From 1d21cd397dc9f5dc3e5f34798812ff57c2c679ab Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 22 Oct 2019 11:28:49 +0200 Subject: [PATCH 1/2] Stop using mage:import in custom beats `mage:import` doesn't take into account vendorized dependencies, this is a problen for custom beats, because by default they have libbeat as a vendorized dependency. The solution would be to use go modules, but in the meantime lets remove the use of `mage:import` from the magefiles intended to be used by custom beats. Revert part of #13239 related to custom beats. --- CHANGELOG-developer.next.asciidoc | 2 + .../{beat}/{magefile.go.tmpl => magefile.go} | 20 ++++- generator/metricbeat/{beat}/magefile.go | 82 +++++++++++++++---- 3 files changed, 85 insertions(+), 19 deletions(-) rename generator/beat/{beat}/{magefile.go.tmpl => magefile.go} (89%) diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index 59affd7f7e4e..043ac9b6943a 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -24,6 +24,8 @@ The list below covers the major changes between 7.0.0-rc2 and master only. ==== Bugfixes +- Stop using `mage:import` in community beats. This was ignoring the vendorized beats directory for some mage targets, using the code available in GOPATH, this causes inconsistencies and compilation problems if the version of the code in the GOPATH is different to the vendored one. Use of `mage:import` will continue to be unsupported in custom beats till beats is migrated to go modules, or mage supports vendored dependencies. {issue}13998[13998] {pull}[] + ==== Added - Metricset generator generates beta modules by default now. {pull}10657[10657] diff --git a/generator/beat/{beat}/magefile.go.tmpl b/generator/beat/{beat}/magefile.go similarity index 89% rename from generator/beat/{beat}/magefile.go.tmpl rename to generator/beat/{beat}/magefile.go index 45b45e4aaccc..e40ce218aee5 100644 --- a/generator/beat/{beat}/magefile.go.tmpl +++ b/generator/beat/{beat}/magefile.go @@ -28,9 +28,7 @@ import ( "github.com/magefile/mage/sh" devtools "github.com/elastic/beats/dev-tools/mage" - - // mage:import - _ "github.com/elastic/beats/dev-tools/mage/target/common" + "github.com/elastic/beats/dev-tools/mage/target/common" ) func init() { @@ -66,6 +64,11 @@ func CrossBuildGoDaemon() error { return devtools.CrossBuildGoDaemon() } +// Clean cleans all generated files and build artifacts. +func Clean() error { + return devtools.Clean() +} + // Package packages the Beat for distribution. // Use SNAPSHOT=true to build snapshots. // Use PLATFORMS to control the target platforms. @@ -113,3 +116,14 @@ func GoTestIntegration(ctx context.Context) error { func Config() error { return devtools.Config(devtools.AllConfigTypes, devtools.ConfigFileParams{}, ".") } + +// Check formats code, updates generated content, check for common errors, and +// checks for any modified files. +func Check() { + common.Check() +} + +// Fmt formats source code (.go and .py) and adds license headers. +func Fmt() { + common.Fmt() +} diff --git a/generator/metricbeat/{beat}/magefile.go b/generator/metricbeat/{beat}/magefile.go index 2461371fd8de..79b073fbab9d 100644 --- a/generator/metricbeat/{beat}/magefile.go +++ b/generator/metricbeat/{beat}/magefile.go @@ -20,6 +20,7 @@ package main import ( + "context" "fmt" "time" @@ -27,20 +28,8 @@ import ( "github.com/magefile/mage/sh" devtools "github.com/elastic/beats/dev-tools/mage" + "github.com/elastic/beats/dev-tools/mage/target/common" metricbeat "github.com/elastic/beats/metricbeat/scripts/mage" - - // mage:import - _ "github.com/elastic/beats/dev-tools/mage/target/common" - // mage:import - "github.com/elastic/beats/dev-tools/mage/target/build" - // mage:import - "github.com/elastic/beats/dev-tools/mage/target/pkg" - // mage:import - _ "github.com/elastic/beats/dev-tools/mage/target/test" - // mage:import - _ "github.com/elastic/beats/dev-tools/mage/target/unittest" - // mage:import - _ "github.com/elastic/beats/dev-tools/mage/target/integtest" ) func init() { @@ -49,11 +38,42 @@ func init() { devtools.BeatDescription = "One sentence description of the Beat." } -// CollectAll generates the docs and the fields. +//CollectAll generates the docs and the fields. func CollectAll() { mg.Deps(CollectDocs, FieldsDocs) } +// Build builds the Beat binary. +func Build() error { + return devtools.Build(devtools.DefaultBuildArgs()) +} + +// GolangCrossBuild build the Beat binary inside of the golang-builder. +// Do not use directly, use crossBuild instead. +func GolangCrossBuild() error { + return devtools.GolangCrossBuild(devtools.DefaultGolangCrossBuildArgs()) +} + +// BuildGoDaemon builds the go-daemon binary (use crossBuildGoDaemon). +func BuildGoDaemon() error { + return devtools.BuildGoDaemon() +} + +// CrossBuild cross-builds the beat for all target platforms. +func CrossBuild() error { + return devtools.CrossBuild() +} + +// CrossBuildGoDaemon cross-builds the go-daemon binary using Docker. +func CrossBuildGoDaemon() error { + return devtools.CrossBuildGoDaemon() +} + +// Clean cleans all generated files and build artifacts. +func Clean() error { + return devtools.Clean() +} + // Package packages the Beat for distribution. // Use SNAPSHOT=true to build snapshots. // Use PLATFORMS to control the target platforms. @@ -64,8 +84,13 @@ func Package() { devtools.UseCommunityBeatPackaging() mg.Deps(Update) - mg.Deps(build.CrossBuild, build.CrossBuildGoDaemon) - mg.SerialDeps(devtools.Package, pkg.PackageTest) + mg.Deps(CrossBuild, CrossBuildGoDaemon) + mg.SerialDeps(devtools.Package, TestPackages) +} + +// TestPackages tests the generated packages (i.e. file modes, owners, groups). +func TestPackages() error { + return devtools.TestPackages() } // Update updates the generated files (aka make update). @@ -96,6 +121,20 @@ func CollectDocs() error { return metricbeat.CollectDocs() } +// GoTestUnit executes the Go unit tests. +// Use TEST_COVERAGE=true to enable code coverage profiling. +// Use RACE_DETECTOR=true to enable the race detector. +func GoTestUnit(ctx context.Context) error { + return devtools.GoTest(ctx, devtools.DefaultGoTestUnitArgs()) +} + +// GoTestIntegration executes the Go integration tests. +// Use TEST_COVERAGE=true to enable code coverage profiling. +// Use RACE_DETECTOR=true to enable the race detector. +func GoTestIntegration(ctx context.Context) error { + return devtools.GoTest(ctx, devtools.DefaultGoTestIntegrationArgs()) +} + // Config generates both the short/reference/docker configs. func Config() error { customDeps := devtools.ConfigFileParams{ @@ -106,3 +145,14 @@ func Config() error { } return devtools.Config(devtools.AllConfigTypes, customDeps, ".") } + +// Check formats code, updates generated content, check for common errors, and +// checks for any modified files. +func Check() { + common.Check() +} + +// Fmt formats source code (.go and .py) and adds license headers. +func Fmt() { + common.Fmt() +} From 193c1335c92dcc2c72030451a567c178571a0692 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Thu, 24 Oct 2019 12:16:20 +0200 Subject: [PATCH 2/2] Add space in godoc --- generator/metricbeat/{beat}/magefile.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generator/metricbeat/{beat}/magefile.go b/generator/metricbeat/{beat}/magefile.go index 79b073fbab9d..82c96241f7da 100644 --- a/generator/metricbeat/{beat}/magefile.go +++ b/generator/metricbeat/{beat}/magefile.go @@ -38,7 +38,7 @@ func init() { devtools.BeatDescription = "One sentence description of the Beat." } -//CollectAll generates the docs and the fields. +// CollectAll generates the docs and the fields. func CollectAll() { mg.Deps(CollectDocs, FieldsDocs) }