diff --git a/.gitignore b/.gitignore index 4aa5e6e60d..28dd2716d3 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,9 @@ elastic-package # IDEA .idea +# VSCode +.vscode + # Build directory /build diff --git a/README.md b/README.md index e23361b7a1..4003076a66 100644 --- a/README.md +++ b/README.md @@ -366,6 +366,24 @@ Use this command to export ingest pipelines with referenced pipelines from the E Use this command to download selected ingest pipelines and its referenced processor pipelines from Elasticsearch. Select data stream or the package root directories to download the pipelines. Pipelines are downloaded as is and will need adjustment to meet your package needs. +### `elastic-package filter [flags]` + +_Context: package_ + +This command gives you a list of all packages based on the given query + +### `elastic-package foreach [flags] -- ` + +_Context: package_ + +Execute a command for each package matching the given filter criteria. + +This command combines filtering capabilities with command execution, allowing you to run +any elastic-package subcommand across multiple packages in a single operation. + +The command uses the same filter flags as the 'filter' command to select packages, +then executes the specified subcommand for each matched package. + ### `elastic-package format` _Context: package_ diff --git a/cmd/filter.go b/cmd/filter.go new file mode 100644 index 0000000000..aa723afa56 --- /dev/null +++ b/cmd/filter.go @@ -0,0 +1,106 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cmd + +import ( + "fmt" + "io" + "os" + + "github.com/spf13/cobra" + + "github.com/elastic/elastic-package/internal/cobraext" + "github.com/elastic/elastic-package/internal/filter" + "github.com/elastic/elastic-package/internal/packages" +) + +const filterLongDescription = `This command gives you a list of all packages based on the given query` + +func setupFilterCommand() *cobraext.Command { + cmd := &cobra.Command{ + Use: "filter [flags]", + Short: "filter integrations based on given flags", + Long: filterLongDescription, + Args: cobra.NoArgs, + RunE: filterCommandAction, + } + + // add filter flags to the command (input, code owner, kibana version, categories) + filter.SetFilterFlags(cmd) + + // add the output package name and absolute path flags to the command + cmd.Flags().StringP(cobraext.FilterOutputFlagName, cobraext.FilterOutputFlagShorthand, "", cobraext.FilterOutputFlagDescription) + cmd.Flags().StringP(cobraext.FilterOutputInfoFlagName, "", cobraext.FilterOutputInfoFlagDefault, cobraext.FilterOutputInfoFlagDescription) + + return cobraext.NewCommand(cmd, cobraext.ContextPackage) +} + +func filterCommandAction(cmd *cobra.Command, args []string) error { + filtered, err := filterPackage(cmd) + if err != nil { + return fmt.Errorf("filtering packages failed: %w", err) + } + + outputFormatStr, err := cmd.Flags().GetString(cobraext.FilterOutputFlagName) + if err != nil { + return fmt.Errorf("getting output format flag failed: %w", err) + } + + outputInfoStr, err := cmd.Flags().GetString(cobraext.FilterOutputInfoFlagName) + if err != nil { + return fmt.Errorf("getting output info flag failed: %w", err) + } + + outputOptions, err := filter.NewOutputOptions(outputInfoStr, outputFormatStr) + if err != nil { + return fmt.Errorf("creating output options failed: %w", err) + } + + if err = printPkgList(filtered, outputOptions, os.Stdout); err != nil { + return fmt.Errorf("printing JSON failed: %w", err) + } + + return nil +} + +func filterPackage(cmd *cobra.Command) ([]packages.PackageDirNameAndManifest, error) { + depth, err := cmd.Flags().GetInt(cobraext.FilterDepthFlagName) + if err != nil { + return nil, fmt.Errorf("getting depth flag failed: %w", err) + } + + excludeDirs, err := cmd.Flags().GetString(cobraext.FilterExcludeDirFlagName) + if err != nil { + return nil, fmt.Errorf("getting exclude-dir flag failed: %w", err) + } + + filters := filter.NewFilterRegistry(depth, excludeDirs) + + if err := filters.Parse(cmd); err != nil { + return nil, fmt.Errorf("parsing filter options failed: %w", err) + } + + if err := filters.Validate(); err != nil { + return nil, fmt.Errorf("validating filter options failed: %w", err) + } + + filtered, errors := filters.Execute() + if errors != nil { + return nil, fmt.Errorf("filtering packages failed: %s", errors.Error()) + } + + return filtered, nil +} + +func printPkgList(pkgs []packages.PackageDirNameAndManifest, outputOptions *filter.OutputOptions, w io.Writer) error { + formatted, err := outputOptions.ApplyTo(pkgs) + if err != nil { + return fmt.Errorf("applying output format failed: %w", err) + } + + // write the formatted packages to the writer + _, err = io.WriteString(w, formatted+"\n") + return err +} diff --git a/cmd/foreach.go b/cmd/foreach.go new file mode 100644 index 0000000000..5347c69db9 --- /dev/null +++ b/cmd/foreach.go @@ -0,0 +1,97 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cmd + +import ( + "fmt" + "slices" + "strings" + + "github.com/spf13/cobra" + + "github.com/elastic/elastic-package/internal/cobraext" + "github.com/elastic/elastic-package/internal/filter" + "github.com/elastic/elastic-package/internal/logger" + "github.com/elastic/elastic-package/internal/multierror" +) + +const foreachLongDescription = `Execute a command for each package matching the given filter criteria. + +This command combines filtering capabilities with command execution, allowing you to run +any elastic-package subcommand across multiple packages in a single operation. + +The command uses the same filter flags as the 'filter' command to select packages, +then executes the specified subcommand for each matched package.` + +// getAllowedSubCommands returns the list of allowed subcommands for the foreach command. +func getAllowedSubCommands() []string { + return []string{ + "build", + "check", + "changelog", + "clean", + "format", + "install", + "lint", + "test", + "uninstall", + } +} + +func setupForeachCommand() *cobraext.Command { + cmd := &cobra.Command{ + Use: "foreach [flags] -- ", + Short: "Execute a command for filtered packages", + Long: foreachLongDescription, + Example: ` # Run system tests for packages with specific inputs + elastic-package foreach --input tcp,udp -- test system -g`, + RunE: foreachCommandAction, + Args: cobra.MinimumNArgs(1), + } + + // Add filter flags + filter.SetFilterFlags(cmd) + + return cobraext.NewCommand(cmd, cobraext.ContextPackage) +} + +func foreachCommandAction(cmd *cobra.Command, args []string) error { + if err := validateSubCommand(args[0]); err != nil { + return fmt.Errorf("validating sub command failed: %w", err) + } + + // reuse filterPackage from cmd/filter.go + filtered, err := filterPackage(cmd) + if err != nil { + return fmt.Errorf("filtering packages failed: %w", err) + } + + errors := multierror.Error{} + + for _, pkg := range filtered { + rootCmd := cmd.Root() + rootCmd.SetArgs(append(args, "--change-directory", pkg.Path)) + if err := rootCmd.Execute(); err != nil { + errors = append(errors, err) + } + } + + logger.Infof("Successfully executed command for %d packages", len(filtered)-len(errors)) + + if errors.Error() != "" { + logger.Errorf("Errors occurred for %d packages", len(errors)) + return fmt.Errorf("errors occurred while executing command for packages: \n%s", errors.Error()) + } + + return nil +} + +func validateSubCommand(subCommand string) error { + if !slices.Contains(getAllowedSubCommands(), subCommand) { + return fmt.Errorf("invalid subcommand: %s. Allowed subcommands are: [%s]", subCommand, strings.Join(getAllowedSubCommands(), ", ")) + } + + return nil +} diff --git a/cmd/root.go b/cmd/root.go index e449ff5169..ae5ffe2d99 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -26,7 +26,9 @@ var commands = []*cobraext.Command{ setupDumpCommand(), setupEditCommand(), setupExportCommand(), + setupFilterCommand(), setupFormatCommand(), + setupForeachCommand(), setupInstallCommand(), setupLinksCommand(), setupLintCommand(), diff --git a/go.mod b/go.mod index acf569a3cb..9453bbf772 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,7 @@ require ( github.com/elastic/package-spec/v3 v3.5.0 github.com/fatih/color v1.18.0 github.com/go-viper/mapstructure/v2 v2.4.0 + github.com/gobwas/glob v0.2.3 github.com/google/go-cmp v0.7.0 github.com/google/go-github/v32 v32.1.0 github.com/google/go-querystring v1.1.0 @@ -35,6 +36,7 @@ require ( github.com/shirou/gopsutil/v3 v3.24.5 github.com/spf13/cobra v1.10.1 github.com/stretchr/testify v1.11.1 + go.yaml.in/yaml/v2 v2.4.2 go4.org/netipx v0.0.0-20231129151722-fdeea329fbba golang.org/x/tools v0.38.0 gopkg.in/dnaeon/go-vcr.v3 v3.2.0 @@ -169,7 +171,6 @@ require ( github.com/yuin/goldmark v1.7.13 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect go.mongodb.org/mongo-driver v1.11.1 // indirect - go.yaml.in/yaml/v2 v2.4.2 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect golang.org/x/crypto v0.43.0 // indirect diff --git a/go.sum b/go.sum index b34f5a5d5c..c06a0c3656 100644 --- a/go.sum +++ b/go.sum @@ -179,6 +179,8 @@ github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1v github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= diff --git a/internal/cobraext/flags.go b/internal/cobraext/flags.go index 1838404fd6..6ff3aa03fb 100644 --- a/internal/cobraext/flags.go +++ b/internal/cobraext/flags.go @@ -133,8 +133,49 @@ const ( FailOnMissingFlagName = "fail-on-missing" FailOnMissingFlagDescription = "fail if tests are missing" - FailFastFlagName = "fail-fast" - FailFastFlagDescription = "fail immediately if any file requires updates (do not overwrite)" + FailFastFlagName = "fail-fast" + FailFastFlagDescription = "fail immediately if any file requires updates (do not overwrite)" + + FilterCategoriesFlagName = "categories" + FilterCategoriesFlagDescription = "integration categories to filter by (comma-separated values)" + + FilterCodeOwnerFlagName = "code-owners" + FilterCodeOwnerFlagDescription = "code owners to filter by (comma-separated values)" + + FilterDepthFlagName = "depth" + FilterDepthFlagDescription = "maximum depth to search for packages" + FilterDepthFlagDefault = 2 + FilterDepthFlagShorthand = "d" + + FilterExcludeDirFlagName = "exclude-dirs" + FilterExcludeDirFlagDescription = "comma-separated list of directories to exclude from search" + + FilterInputFlagName = "inputs" + FilterInputFlagDescription = "name of the inputs to filter by (comma-separated values)" + + FilterKibanaVersionFlagName = "kibana-version" + FilterKibanaVersionFlagDescription = "kibana version to filter by (semver)" + + FilterOutputFlagName = "output" + FilterOutputFlagDescription = "format of the output. Available options: json, yaml (leave empty for newline-separated list)" + FilterOutputFlagShorthand = "o" + + FilterOutputInfoFlagName = "output-info" + FilterOutputInfoFlagDescription = "output information about the packages. Available options: pkgname, dirname, absolute" + FilterOutputInfoFlagDefault = "dirname" + + FilterPackageDirNameFlagName = "package-dirs" + FilterPackageDirNameFlagDescription = "package directories to filter by (comma-separated values)" + + FilterPackagesFlagName = "packages" + FilterPackagesFlagDescription = "package names to filter by (comma-separated values)" + + FilterPackageTypeFlagName = "package-types" + FilterPackageTypeFlagDescription = "package types to filter by (comma-separated values)" + + FilterSpecVersionFlagName = "spec-version" + FilterSpecVersionFlagDescription = "Package spec version to filter by (semver)" + GenerateTestResultFlagName = "generate" GenerateTestResultFlagDescription = "generate test result file" diff --git a/internal/filter/category.go b/internal/filter/category.go new file mode 100644 index 0000000000..5c170ca417 --- /dev/null +++ b/internal/filter/category.go @@ -0,0 +1,62 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package filter + +import ( + "github.com/spf13/cobra" + + "github.com/elastic/elastic-package/internal/cobraext" + "github.com/elastic/elastic-package/internal/packages" +) + +type CategoryFlag struct { + FilterFlagBase + + values []string +} + +func (f *CategoryFlag) Parse(cmd *cobra.Command) error { + category, err := cmd.Flags().GetString(cobraext.FilterCategoriesFlagName) + if err != nil { + return cobraext.FlagParsingError(err, cobraext.FilterCategoriesFlagName) + } + if category == "" { + return nil + } + + categories := splitAndTrim(category, ",") + f.values = categories + f.isApplied = true + return nil +} + +func (f *CategoryFlag) Validate() error { + return nil +} + +func (f *CategoryFlag) Matches(dirName string, manifest *packages.PackageManifest) bool { + return hasAnyMatch(f.values, manifest.Categories) +} + +func (f *CategoryFlag) ApplyTo(pkgs []packages.PackageDirNameAndManifest) ([]packages.PackageDirNameAndManifest, error) { + filtered := make([]packages.PackageDirNameAndManifest, 0, len(pkgs)) + for _, pkg := range pkgs { + if f.Matches(pkg.DirName, pkg.Manifest) { + filtered = append(filtered, pkg) + } + } + return filtered, nil +} + +func initCategoryFlag() *CategoryFlag { + return &CategoryFlag{ + FilterFlagBase: FilterFlagBase{ + name: cobraext.FilterCategoriesFlagName, + description: cobraext.FilterCategoriesFlagDescription, + shorthand: "", + defaultValue: "", + }, + } +} diff --git a/internal/filter/codeowner.go b/internal/filter/codeowner.go new file mode 100644 index 0000000000..d80b982979 --- /dev/null +++ b/internal/filter/codeowner.go @@ -0,0 +1,73 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package filter + +import ( + "fmt" + + "github.com/spf13/cobra" + + "github.com/elastic/elastic-package/internal/cobraext" + "github.com/elastic/elastic-package/internal/packages" + "github.com/elastic/elastic-package/internal/tui" +) + +type CodeOwnerFlag struct { + FilterFlagBase + values []string +} + +func (f *CodeOwnerFlag) Parse(cmd *cobra.Command) error { + codeOwners, err := cmd.Flags().GetString(cobraext.FilterCodeOwnerFlagName) + if err != nil { + return cobraext.FlagParsingError(err, cobraext.FilterCodeOwnerFlagName) + } + if codeOwners == "" { + return nil + } + + f.values = splitAndTrim(codeOwners, ",") + f.isApplied = true + return nil +} + +func (f *CodeOwnerFlag) Validate() error { + validator := tui.Validator{Cwd: "."} + + if f.values != nil { + for _, value := range f.values { + if err := validator.GithubOwner(value); err != nil { + return fmt.Errorf("invalid code owner: %s: %w", value, err) + } + } + } + + return nil +} + +func (f *CodeOwnerFlag) Matches(dirName string, manifest *packages.PackageManifest) bool { + return hasAnyMatch(f.values, []string{manifest.Owner.Github}) +} + +func (f *CodeOwnerFlag) ApplyTo(pkgs []packages.PackageDirNameAndManifest) ([]packages.PackageDirNameAndManifest, error) { + filtered := make([]packages.PackageDirNameAndManifest, 0, len(pkgs)) + for _, pkg := range pkgs { + if f.Matches(pkg.DirName, pkg.Manifest) { + filtered = append(filtered, pkg) + } + } + return filtered, nil +} + +func initCodeOwnerFlag() *CodeOwnerFlag { + return &CodeOwnerFlag{ + FilterFlagBase: FilterFlagBase{ + name: cobraext.FilterCodeOwnerFlagName, + description: cobraext.FilterCodeOwnerFlagDescription, + shorthand: "", + defaultValue: "", + }, + } +} diff --git a/internal/filter/input.go b/internal/filter/input.go new file mode 100644 index 0000000000..d73f7c3d94 --- /dev/null +++ b/internal/filter/input.go @@ -0,0 +1,69 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package filter + +import ( + "github.com/spf13/cobra" + + "github.com/elastic/elastic-package/internal/cobraext" + "github.com/elastic/elastic-package/internal/packages" +) + +type InputFlag struct { + FilterFlagBase + + // flag specific fields + values []string +} + +func (f *InputFlag) Parse(cmd *cobra.Command) error { + input, err := cmd.Flags().GetString(cobraext.FilterInputFlagName) + if err != nil { + return cobraext.FlagParsingError(err, cobraext.FilterInputFlagName) + } + if input == "" { + return nil + } + + f.values = splitAndTrim(input, ",") + f.isApplied = true + return nil +} + +func (f *InputFlag) Validate() error { + return nil +} + +func (f *InputFlag) Matches(dirName string, manifest *packages.PackageManifest) bool { + if f.values != nil { + inputs := extractInputs(manifest) + if !hasAnyMatch(f.values, inputs) { + return false + } + } + return true +} + +func (f *InputFlag) ApplyTo(pkgs []packages.PackageDirNameAndManifest) ([]packages.PackageDirNameAndManifest, error) { + filtered := make([]packages.PackageDirNameAndManifest, 0, len(pkgs)) + + for _, pkg := range pkgs { + if f.Matches(pkg.DirName, pkg.Manifest) { + filtered = append(filtered, pkg) + } + } + return filtered, nil +} + +func initInputFlag() *InputFlag { + return &InputFlag{ + FilterFlagBase: FilterFlagBase{ + name: cobraext.FilterInputFlagName, + description: cobraext.FilterInputFlagDescription, + shorthand: "", + defaultValue: "", + }, + } +} diff --git a/internal/filter/packagedirname.go b/internal/filter/packagedirname.go new file mode 100644 index 0000000000..32f4a640ed --- /dev/null +++ b/internal/filter/packagedirname.go @@ -0,0 +1,77 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package filter + +import ( + "fmt" + + "github.com/gobwas/glob" + "github.com/spf13/cobra" + + "github.com/elastic/elastic-package/internal/cobraext" + "github.com/elastic/elastic-package/internal/packages" +) + +type PackageDirNameFlag struct { + FilterFlagBase + + patterns []glob.Glob +} + +func (f *PackageDirNameFlag) Parse(cmd *cobra.Command) error { + packageDirNamePatterns, err := cmd.Flags().GetString(cobraext.FilterPackageDirNameFlagName) + if err != nil { + return cobraext.FlagParsingError(err, cobraext.FilterPackageDirNameFlagName) + } + + patterns := splitAndTrim(packageDirNamePatterns, ",") + for _, patternString := range patterns { + pattern, err := glob.Compile(patternString) + if err != nil { + return fmt.Errorf("invalid package dir name pattern: %s: %w", patternString, err) + } + f.patterns = append(f.patterns, pattern) + } + + if len(f.patterns) > 0 { + f.isApplied = true + } + + return nil +} + +func (f *PackageDirNameFlag) Validate() error { + return nil +} + +func (f *PackageDirNameFlag) Matches(dirName string, manifest *packages.PackageManifest) bool { + for _, pattern := range f.patterns { + if pattern.Match(dirName) { + return true + } + } + return false +} + +func (f *PackageDirNameFlag) ApplyTo(pkgs []packages.PackageDirNameAndManifest) ([]packages.PackageDirNameAndManifest, error) { + filtered := make([]packages.PackageDirNameAndManifest, 0, len(pkgs)) + for _, pkg := range pkgs { + if f.Matches(pkg.DirName, pkg.Manifest) { + filtered = append(filtered, pkg) + } + } + return filtered, nil +} + +func initPackageDirNameFlag() *PackageDirNameFlag { + return &PackageDirNameFlag{ + FilterFlagBase: FilterFlagBase{ + name: cobraext.FilterPackageDirNameFlagName, + description: cobraext.FilterPackageDirNameFlagDescription, + shorthand: "", + defaultValue: "", + }, + } +} diff --git a/internal/filter/packagename.go b/internal/filter/packagename.go new file mode 100644 index 0000000000..63a415f298 --- /dev/null +++ b/internal/filter/packagename.go @@ -0,0 +1,77 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package filter + +import ( + "fmt" + + "github.com/gobwas/glob" + "github.com/spf13/cobra" + + "github.com/elastic/elastic-package/internal/cobraext" + "github.com/elastic/elastic-package/internal/packages" +) + +type PackageNameFlag struct { + FilterFlagBase + + patterns []glob.Glob +} + +func (f *PackageNameFlag) Parse(cmd *cobra.Command) error { + packageNamePatterns, err := cmd.Flags().GetString(cobraext.FilterPackagesFlagName) + if err != nil { + return cobraext.FlagParsingError(err, cobraext.FilterPackagesFlagName) + } + + patterns := splitAndTrim(packageNamePatterns, ",") + for _, patternString := range patterns { + pattern, err := glob.Compile(patternString) + if err != nil { + return fmt.Errorf("invalid package name pattern: %s: %w", patternString, err) + } + f.patterns = append(f.patterns, pattern) + } + + if len(f.patterns) > 0 { + f.isApplied = true + } + + return nil +} + +func (f *PackageNameFlag) Validate() error { + return nil +} + +func (f *PackageNameFlag) Matches(dirName string, manifest *packages.PackageManifest) bool { + for _, pattern := range f.patterns { + if pattern.Match(manifest.Name) { + return true + } + } + return false +} + +func (f *PackageNameFlag) ApplyTo(pkgs []packages.PackageDirNameAndManifest) ([]packages.PackageDirNameAndManifest, error) { + filtered := make([]packages.PackageDirNameAndManifest, 0, len(pkgs)) + for _, pkg := range pkgs { + if f.Matches(pkg.DirName, pkg.Manifest) { + filtered = append(filtered, pkg) + } + } + return filtered, nil +} + +func initPackageNameFlag() *PackageNameFlag { + return &PackageNameFlag{ + FilterFlagBase: FilterFlagBase{ + name: cobraext.FilterPackagesFlagName, + description: cobraext.FilterPackagesFlagDescription, + shorthand: "", + defaultValue: "", + }, + } +} diff --git a/internal/filter/packagetype.go b/internal/filter/packagetype.go new file mode 100644 index 0000000000..d1945a4b2f --- /dev/null +++ b/internal/filter/packagetype.go @@ -0,0 +1,61 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package filter + +import ( + "github.com/spf13/cobra" + + "github.com/elastic/elastic-package/internal/cobraext" + "github.com/elastic/elastic-package/internal/packages" +) + +type PackageTypeFlag struct { + FilterFlagBase + + // flag specific fields + values []string +} + +func (f *PackageTypeFlag) Parse(cmd *cobra.Command) error { + packageTypes, err := cmd.Flags().GetString(cobraext.FilterPackageTypeFlagName) + if err != nil { + return cobraext.FlagParsingError(err, cobraext.FilterPackageTypeFlagName) + } + if packageTypes == "" { + return nil + } + f.values = splitAndTrim(packageTypes, ",") + f.isApplied = true + return nil +} + +func (f *PackageTypeFlag) Validate() error { + return nil +} + +func (f *PackageTypeFlag) Matches(dirName string, manifest *packages.PackageManifest) bool { + return hasAnyMatch(f.values, []string{manifest.Type}) +} + +func (f *PackageTypeFlag) ApplyTo(pkgs []packages.PackageDirNameAndManifest) ([]packages.PackageDirNameAndManifest, error) { + filtered := make([]packages.PackageDirNameAndManifest, 0, len(pkgs)) + for _, pkg := range pkgs { + if f.Matches(pkg.DirName, pkg.Manifest) { + filtered = append(filtered, pkg) + } + } + return filtered, nil +} + +func initPackageTypeFlag() *PackageTypeFlag { + return &PackageTypeFlag{ + FilterFlagBase: FilterFlagBase{ + name: cobraext.FilterPackageTypeFlagName, + description: cobraext.FilterPackageTypeFlagDescription, + shorthand: "", + defaultValue: "", + }, + } +} diff --git a/internal/filter/registry.go b/internal/filter/registry.go new file mode 100644 index 0000000000..2faf0c1600 --- /dev/null +++ b/internal/filter/registry.go @@ -0,0 +1,108 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package filter + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" + + "github.com/elastic/elastic-package/internal/cobraext" + "github.com/elastic/elastic-package/internal/logger" + "github.com/elastic/elastic-package/internal/multierror" + "github.com/elastic/elastic-package/internal/packages" +) + +var registry = []Filter{ + initCategoryFlag(), + initCodeOwnerFlag(), + initInputFlag(), + initPackageDirNameFlag(), + initPackageNameFlag(), + initPackageTypeFlag(), + initSpecVersionFlag(), +} + +// SetFilterFlags registers all filter flags with the given command. +func SetFilterFlags(cmd *cobra.Command) { + cmd.Flags().IntP(cobraext.FilterDepthFlagName, cobraext.FilterDepthFlagShorthand, cobraext.FilterDepthFlagDefault, cobraext.FilterDepthFlagDescription) + cmd.Flags().StringP(cobraext.FilterExcludeDirFlagName, "", "", cobraext.FilterExcludeDirFlagDescription) + + for _, filterFlag := range registry { + filterFlag.Register(cmd) + } +} + +// FilterRegistry manages a collection of filters for package filtering. +type FilterRegistry struct { + filters []Filter + depth int + excludeDirs string +} + +// NewFilterRegistry creates a new FilterRegistry instance. +func NewFilterRegistry(depth int, excludeDirs string) *FilterRegistry { + return &FilterRegistry{ + filters: []Filter{}, + depth: depth, + excludeDirs: excludeDirs, + } +} + +func (r *FilterRegistry) Parse(cmd *cobra.Command) error { + errs := multierror.Error{} + for _, filter := range registry { + if err := filter.Parse(cmd); err != nil { + errs = append(errs, err) + } + + if filter.IsApplied() { + r.filters = append(r.filters, filter) + } + } + + if errs.Error() != "" { + return fmt.Errorf("error parsing filter options: %s", errs.Error()) + } + + return nil +} + +func (r *FilterRegistry) Validate() error { + for _, filter := range r.filters { + if err := filter.Validate(); err != nil { + return err + } + } + return nil +} + +func (r *FilterRegistry) Execute() (filtered []packages.PackageDirNameAndManifest, errors multierror.Error) { + currentDir, err := os.Getwd() + if err != nil { + return nil, multierror.Error{fmt.Errorf("getting current directory failed: %w", err)} + } + + pkgs, err := packages.ReadAllPackageManifestsFromRepo(currentDir, r.depth, r.excludeDirs) + if err != nil { + return nil, multierror.Error{err} + } + + filtered = pkgs + for _, filter := range r.filters { + filtered, err = filter.ApplyTo(filtered) + if err != nil { + errors = append(errors, err) + } + + if len(filtered) == 0 { + break + } + } + + logger.Infof("Found %d matching package(s)\n", len(filtered)) + return filtered, errors +} diff --git a/internal/filter/specversion.go b/internal/filter/specversion.go new file mode 100644 index 0000000000..c88cc70e6f --- /dev/null +++ b/internal/filter/specversion.go @@ -0,0 +1,75 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package filter + +import ( + "fmt" + + "github.com/Masterminds/semver/v3" + "github.com/spf13/cobra" + + "github.com/elastic/elastic-package/internal/cobraext" + "github.com/elastic/elastic-package/internal/packages" +) + +type SpecVersionFlag struct { + FilterFlagBase + + // package spec version constraint + constraints *semver.Constraints +} + +func (f *SpecVersionFlag) Parse(cmd *cobra.Command) error { + specVersion, err := cmd.Flags().GetString(cobraext.FilterSpecVersionFlagName) + if err != nil { + return cobraext.FlagParsingError(err, cobraext.FilterSpecVersionFlagName) + } + if specVersion == "" { + return nil + } + + f.constraints, err = semver.NewConstraint(specVersion) + if err != nil { + return fmt.Errorf("invalid spec version: %s: %w", specVersion, err) + } + + f.isApplied = true + return nil +} + +func (f *SpecVersionFlag) Validate() error { + // no validation needed for this flag + // checks are done in Parse method + return nil +} + +func (f *SpecVersionFlag) Matches(dirName string, manifest *packages.PackageManifest) bool { + pkgVersion, err := semver.NewVersion(manifest.SpecVersion) + if err != nil { + return false + } + return f.constraints.Check(pkgVersion) +} + +func (f *SpecVersionFlag) ApplyTo(pkgs []packages.PackageDirNameAndManifest) ([]packages.PackageDirNameAndManifest, error) { + filtered := make([]packages.PackageDirNameAndManifest, 0, len(pkgs)) + for _, pkg := range pkgs { + if f.Matches(pkg.DirName, pkg.Manifest) { + filtered = append(filtered, pkg) + } + } + return filtered, nil +} + +func initSpecVersionFlag() *SpecVersionFlag { + return &SpecVersionFlag{ + FilterFlagBase: FilterFlagBase{ + name: cobraext.FilterSpecVersionFlagName, + description: cobraext.FilterSpecVersionFlagDescription, + shorthand: "", + defaultValue: "", + }, + } +} diff --git a/internal/filter/type.go b/internal/filter/type.go new file mode 100644 index 0000000000..646f72d2cc --- /dev/null +++ b/internal/filter/type.go @@ -0,0 +1,148 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package filter + +import ( + "encoding/json" + "fmt" + "slices" + "strings" + + "github.com/spf13/cobra" + "go.yaml.in/yaml/v2" + + "github.com/elastic/elastic-package/internal/packages" +) + +// OutputOptions handles both what information to display and how to format it. +type OutputOptions struct { + infoType string // "pkgname", "dirname", "absolute" + format string // "json", "yaml", "" +} + +// NewOutputOptions creates a new OutputOptions from string parameters. +func NewOutputOptions(infoType, format string) (*OutputOptions, error) { + cfg := &OutputOptions{ + infoType: infoType, + format: format, + } + if err := cfg.validate(); err != nil { + return nil, err + } + return cfg, nil +} + +func (o *OutputOptions) validate() error { + validInfo := []string{"pkgname", "dirname", "absolute"} + validFormats := []string{"json", "yaml", ""} + + if !slices.Contains(validInfo, o.infoType) { + return fmt.Errorf("invalid output info type: %s (valid: pkgname, dirname, absolute)", o.infoType) + } + if !slices.Contains(validFormats, o.format) { + return fmt.Errorf("invalid output format: %s (valid: json, yaml, or empty)", o.format) + } + + return nil +} + +// ApplyTo applies the output configuration to packages and returns formatted output. +func (o *OutputOptions) ApplyTo(pkgs []packages.PackageDirNameAndManifest) (string, error) { + if len(pkgs) == 0 { + return "", nil + } + + values, err := o.extractInfo(pkgs) + if err != nil { + return "", fmt.Errorf("extracting info failed: %w", err) + } + + // Format output + return o.formatOutput(values) +} + +func (o *OutputOptions) extractInfo(pkgs []packages.PackageDirNameAndManifest) ([]string, error) { + + // Extract information + values := make([]string, 0, len(pkgs)) + for _, pkg := range pkgs { + var val string + switch o.infoType { + case "pkgname": + val = pkg.Manifest.Name + case "dirname": + val = pkg.DirName + case "absolute": + val = pkg.Path + } + values = append(values, val) + } + + // Sort for consistent output + slices.Sort(values) + + return values, nil +} + +func (o *OutputOptions) formatOutput(values []string) (string, error) { + switch o.format { + case "": + return strings.Join(values, "\n"), nil + case "json": + data, err := json.Marshal(values) + if err != nil { + return "", fmt.Errorf("failed to marshal to JSON: %w", err) + } + return string(data), nil + case "yaml": + data, err := yaml.Marshal(values) + if err != nil { + return "", fmt.Errorf("failed to marshal to YAML: %w", err) + } + return string(data), nil + default: + return "", fmt.Errorf("unsupported format: %s", o.format) + } +} + +// FilterFlag defines the basic interface for filter flags. +type FilterFlag interface { + String() string + Register(cmd *cobra.Command) + IsApplied() bool +} + +// Filter extends FilterFlag with filtering capabilities. +// It defines the interface for filtering packages based on specific criteria. +type Filter interface { + FilterFlag + Parse(cmd *cobra.Command) error + Validate() error + ApplyTo(pkgs []packages.PackageDirNameAndManifest) ([]packages.PackageDirNameAndManifest, error) + // Matches checks if a package matches the filter criteria. + // dirName is the directory name of the package in package root. + Matches(dirName string, manifest *packages.PackageManifest) bool +} + +// FilterFlagBase provides common functionality for filter flags. +type FilterFlagBase struct { + name string + description string + shorthand string + defaultValue string + isApplied bool +} + +func (f *FilterFlagBase) String() string { + return fmt.Sprintf("name=%s defaultValue=%s applied=%v", f.name, f.defaultValue, f.isApplied) +} + +func (f *FilterFlagBase) Register(cmd *cobra.Command) { + cmd.Flags().StringP(f.name, f.shorthand, f.defaultValue, f.description) +} + +func (f *FilterFlagBase) IsApplied() bool { + return f.isApplied +} diff --git a/internal/filter/utils.go b/internal/filter/utils.go new file mode 100644 index 0000000000..d1b416a019 --- /dev/null +++ b/internal/filter/utils.go @@ -0,0 +1,64 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package filter + +import ( + "slices" + "strings" + + "github.com/elastic/elastic-package/internal/packages" +) + +// splitAndTrim splits a string by delimiter and trims whitespace from each element +func splitAndTrim(s, delimiter string) []string { + if s == "" { + return nil + } + parts := strings.Split(s, delimiter) + result := make([]string, 0, len(parts)) + for _, part := range parts { + trimmed := strings.TrimSpace(part) + if trimmed != "" { + result = append(result, trimmed) + } + } + return result +} + +// hasAnyMatch checks if any item in the items slice exists in the filters slice +func hasAnyMatch(filters []string, items []string) bool { + if len(filters) == 0 { + return true + } + + for _, item := range items { + if slices.Contains(filters, item) { + return true + } + } + + return false +} + +// extractInputs extracts all input types from package policy templates +func extractInputs(manifest *packages.PackageManifest) []string { + uniqueInputs := make(map[string]struct{}) + for _, policyTemplate := range manifest.PolicyTemplates { + if policyTemplate.Input != "" { + uniqueInputs[policyTemplate.Input] = struct{}{} + } + + for _, input := range policyTemplate.Inputs { + uniqueInputs[input.Type] = struct{}{} + } + } + + inputs := make([]string, 0, len(uniqueInputs)) + for input := range uniqueInputs { + inputs = append(inputs, input) + } + + return inputs +} diff --git a/internal/packages/packages.go b/internal/packages/packages.go index 4e55f9dca7..7bcadade60 100644 --- a/internal/packages/packages.go +++ b/internal/packages/packages.go @@ -21,6 +21,8 @@ import ( "github.com/elastic/go-ucfg" "github.com/elastic/go-ucfg/yaml" + + "github.com/elastic/elastic-package/internal/logger" ) const ( @@ -201,6 +203,12 @@ type PackageManifest struct { Elasticsearch *Elasticsearch `config:"elasticsearch" json:"elasticsearch" yaml:"elasticsearch"` } +type PackageDirNameAndManifest struct { + DirName string + Path string + Manifest *PackageManifest +} + type ManifestIndexTemplate struct { IngestPipeline *ManifestIngestPipeline `config:"ingest_pipeline" json:"ingest_pipeline" yaml:"ingest_pipeline"` Mappings *ManifestMappings `config:"mappings" json:"mappings" yaml:"mappings"` @@ -419,6 +427,89 @@ func ReadPackageManifest(path string) (*PackageManifest, error) { return &m, nil } +// ReadAllPackageManifestsFromRepo reads all the package manifests in the given directory. +// It recursively searches for manifest.yml files up to the specified depth. +// - depth: maximum depth to search (1 = current dir + immediate sub dirs) +// - excludeDirs: comma-separated list of directory names to exclude (always excludes .git) +func ReadAllPackageManifestsFromRepo(searchRoot string, depth int, excludeDirs string) ([]PackageDirNameAndManifest, error) { + // Parse exclude directories + excludeMap := map[string]bool{ + ".git": true, // Always exclude .git + "build": true, // Always exclude build + } + + if excludeDirs != "" { + for dir := range strings.SplitSeq(excludeDirs, ",") { + excludeMap[strings.TrimSpace(dir)] = true + } + } + + var packages []PackageDirNameAndManifest + searchRootDepth := strings.Count(searchRoot, string(filepath.Separator)) + + err := filepath.WalkDir(searchRoot, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + + // Calculate current depth relative to search root + currentDepth := strings.Count(path, string(filepath.Separator)) - searchRootDepth + + // If it's a directory, check if we should skip it + if d.IsDir() { + dirName := d.Name() + + // Skip excluded directories (but not the search root) + if excludeMap[dirName] && searchRoot != path { + return filepath.SkipDir + } + + // Skip if we've exceeded the depth limit (but allow processing the current level) + if currentDepth > depth { + return filepath.SkipDir + } + + return nil + } + + // Check if this is a manifest file + if d.Name() != PackageManifestFile { + return nil + } + + // Validate it's a package manifest + ok, err := isPackageManifest(path) + if err != nil { + logger.Debugf("failed to validate package manifest (path: %s): %v", path, err) + return nil + } + if !ok { + return nil + } + + // Extract directory name (just the package directory name, not the full path) + dirName := filepath.Base(filepath.Dir(path)) + manifest, err := ReadPackageManifest(path) + if err != nil { + return fmt.Errorf("failed to read package manifest (path: %s): %w", path, err) + } + + packages = append(packages, PackageDirNameAndManifest{ + DirName: dirName, + Manifest: manifest, + Path: filepath.Dir(path), + }) + + return nil + }) + + if err != nil { + return nil, fmt.Errorf("failed walking directory tree: %w", err) + } + + return packages, nil +} + // ReadTransformDefinitionFile reads and parses the transform definition (elasticsearch/transform//transform.yml) // file for the given transform. It also applies templating to the file, allowing to set the final ingest pipeline name // by adding the package version defined in the package manifest.