From 6e093e9e2f54a8f576ad3c7428c5c9c7dedbf2d4 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Fri, 31 Jan 2020 16:09:09 +0100 Subject: [PATCH] Fix packaging of OSS light modules (#15986) (cherry picked from commit 2cde4ce86b0d67dfcb41f844e82ca66847667a5e) --- metricbeat/magefile.go | 8 ++- metricbeat/scripts/mage/package.go | 86 ++++++++++++++++++++++++++++++ x-pack/metricbeat/magefile.go | 82 +--------------------------- 3 files changed, 95 insertions(+), 81 deletions(-) diff --git a/metricbeat/magefile.go b/metricbeat/magefile.go index 2a96df12bfb5..1b84d3cc3dc4 100644 --- a/metricbeat/magefile.go +++ b/metricbeat/magefile.go @@ -79,7 +79,13 @@ func Package() { // TestPackages tests the generated packages (i.e. file modes, owners, groups). func TestPackages() error { - return devtools.TestPackages(devtools.WithModulesD()) + return devtools.TestPackages( + devtools.WithModulesD(), + devtools.WithModules(), + + // To be increased or removed when more light modules are added + devtools.MinModules(1), + ) } // Dashboards collects all the dashboards and generates index patterns. diff --git a/metricbeat/scripts/mage/package.go b/metricbeat/scripts/mage/package.go index 9fd2529ff7b9..1f527114f02d 100644 --- a/metricbeat/scripts/mage/package.go +++ b/metricbeat/scripts/mage/package.go @@ -18,6 +18,7 @@ package mage import ( + "fmt" "io" "os" "path/filepath" @@ -31,6 +32,7 @@ import ( ) const ( + dirModulesGenerated = "build/package/module" dirModulesDGenerated = "build/package/modules.d" ) @@ -39,6 +41,8 @@ const ( // not supported. You must declare a dependency on either // PrepareModulePackagingOSS or PrepareModulePackagingXPack. func CustomizePackaging() { + mg.Deps(customizeLightModulesPackaging) + var ( modulesDTarget = "modules.d" modulesD = devtools.PackageFile{ @@ -101,6 +105,10 @@ func CustomizePackaging() { // PrepareModulePackagingOSS generates build/package/modules and // build/package/modules.d directories for use in packaging. func PrepareModulePackagingOSS() error { + err := prepareLightModulesPackaging("module") + if err != nil { + return err + } return prepareModulePackaging([]struct{ Src, Dst string }{ {devtools.OSSBeatDir("modules.d"), dirModulesDGenerated}, }...) @@ -109,6 +117,10 @@ func PrepareModulePackagingOSS() error { // PrepareModulePackagingXPack generates build/package/modules and // build/package/modules.d directories for use in packaging. func PrepareModulePackagingXPack() error { + err := prepareLightModulesPackaging("module", devtools.OSSBeatDir("module")) + if err != nil { + return err + } return prepareModulePackaging([]struct{ Src, Dst string }{ {devtools.OSSBeatDir("modules.d"), dirModulesDGenerated}, {"modules.d", dirModulesDGenerated}, @@ -190,6 +202,80 @@ func GenerateDirModulesD() error { return nil } +// customizeLightModulesPackaging customizes packaging to add light modules +func customizeLightModulesPackaging() error { + var ( + moduleTarget = "module" + module = devtools.PackageFile{ + Mode: 0644, + Source: dirModulesGenerated, + } + ) + + for _, args := range devtools.Packages { + pkgType := args.Types[0] + switch pkgType { + case devtools.TarGz, devtools.Zip, devtools.Docker: + args.Spec.Files[moduleTarget] = module + case devtools.Deb, devtools.RPM: + args.Spec.Files["/usr/share/{{.BeatName}}/"+moduleTarget] = module + case devtools.DMG: + args.Spec.Files["/Library/Application Support/{{.BeatVendor}}/{{.BeatName}}/"+moduleTarget] = module + default: + return fmt.Errorf("unhandled package type: %v", pkgType) + } + } + return nil +} + +// prepareLightModulesPackaging generates light modules +func prepareLightModulesPackaging(paths ...string) error { + err := devtools.Clean([]string{dirModulesGenerated}) + if err != nil { + return err + } + if err := os.MkdirAll(dirModulesGenerated, 0755); err != nil { + return err + } + + filePatterns := []string{ + "*/module.yml", + "*/*/manifest.yml", + } + + var tasks []devtools.CopyTask + for _, path := range paths { + for _, pattern := range filePatterns { + matches, err := filepath.Glob(filepath.Join(path, pattern)) + if err != nil { + return err + } + + for _, file := range matches { + rel, _ := filepath.Rel(path, file) + dest := filepath.Join(dirModulesGenerated, rel) + tasks = append(tasks, devtools.CopyTask{ + Source: file, + Dest: dest, + Mode: 0644, + DirMode: 0755, + }) + } + } + } + + if len(tasks) == 0 { + return fmt.Errorf("no light modules found") + } + + for _, task := range tasks { + if err := task.Execute(); err != nil { + return err + } + } + return nil +} + // moduleConfigParts obtain the moduleName and the configName from a config path. // The configName includes the flavor func moduleConfigParts(f string) (moduleName string, configName string, ok bool) { diff --git a/x-pack/metricbeat/magefile.go b/x-pack/metricbeat/magefile.go index 25441b598a7e..8b65953fc38b 100644 --- a/x-pack/metricbeat/magefile.go +++ b/x-pack/metricbeat/magefile.go @@ -9,8 +9,6 @@ package main import ( "context" "fmt" - "os" - "path/filepath" "time" "github.com/magefile/mage/mg" @@ -24,10 +22,6 @@ import ( _ "github.com/elastic/beats/dev-tools/mage/target/compose" ) -const ( - dirModulesGenerated = "build/package/module" -) - func init() { common.RegisterCheckDeps(Update) @@ -78,7 +72,6 @@ func Package() { devtools.UseElasticBeatXPackPackaging() metricbeat.CustomizePackaging() devtools.PackageKibanaDashboardsFromBuildDir() - packageLightModules() mg.Deps(Update, metricbeat.PrepareModulePackagingXPack) mg.Deps(CrossBuild, CrossBuildGoDaemon) @@ -92,7 +85,8 @@ func TestPackages() error { devtools.WithModules(), // To be increased or removed when more light modules are added - devtools.MinModules(3)) + devtools.MinModules(5), + ) } // Fields generates a fields.yml and fields.go for each module. @@ -148,78 +142,6 @@ func PythonUnitTest() error { return devtools.PythonNoseTest(devtools.DefaultPythonTestUnitArgs()) } -// prepareLightModules generates light modules -func prepareLightModules(path string) error { - err := devtools.Clean([]string{dirModulesGenerated}) - if err != nil { - return err - } - if err := os.MkdirAll(dirModulesGenerated, 0755); err != nil { - return err - } - - filePatterns := []string{ - "*/module.yml", - "*/*/manifest.yml", - } - - var files []string - for _, pattern := range filePatterns { - matches, err := filepath.Glob(filepath.Join(path, pattern)) - if err != nil { - return err - } - files = append(files, matches...) - } - - if len(files) == 0 { - return fmt.Errorf("no light modules found") - } - - for _, file := range files { - rel, _ := filepath.Rel(path, file) - dest := filepath.Join(dirModulesGenerated, rel) - err := (&devtools.CopyTask{ - Source: file, - Dest: dest, - Mode: 0644, - DirMode: 0755, - }).Execute() - if err != nil { - return err - } - } - return nil -} - -// packageLightModules customizes packaging to add light modules -func packageLightModules() error { - prepareLightModules("module") - - var ( - moduleTarget = "module" - module = devtools.PackageFile{ - Mode: 0644, - Source: dirModulesGenerated, - } - ) - - for _, args := range devtools.Packages { - pkgType := args.Types[0] - switch pkgType { - case devtools.TarGz, devtools.Zip, devtools.Docker: - args.Spec.Files[moduleTarget] = module - case devtools.Deb, devtools.RPM: - args.Spec.Files["/usr/share/{{.BeatName}}/"+moduleTarget] = module - case devtools.DMG: - args.Spec.Files["/Library/Application Support/{{.BeatVendor}}/{{.BeatName}}/"+moduleTarget] = module - default: - return fmt.Errorf("unhandled package type: %v", pkgType) - } - } - return nil -} - // IntegTest executes integration tests (it uses Docker to run the tests). func IntegTest() { devtools.AddIntegTestUsage()