magefile: use mage to build fields assets (#3372)

* magefile: use mage to build fields assets

Also, stop using the common "update" target and
move more logic into "mage update". This is in
line with how other beats do it now.

* magefile: don't package _meta/kibana.generated

We don't package any dashboards, and index patterns are generated.

* scripts/jenkins: set up Python 3 in windows-build

* Setup Python 3 earlier in the windows test script

Makefile

@@ -25,6 +25,7 @@ NOW=$(shell date -u '+%Y-%m-%dT%H:%M:%S')
 GOBUILD_FLAGS=-ldflags "-s -X $(BEAT_PATH)/vendor/github.com/elastic/beats/libbeat/version.buildTime=$(NOW) -X $(BEAT_PATH)/vendor/github.com/elastic/beats/libbeat/version.commit=$(COMMIT_ID)"
 MAGE_IMPORT_PATH=${BEAT_PATH}/vendor/github.com/magefile/mage
 STATICCHECK_REPO=${BEAT_PATH}/vendor/honnef.co/go/tools/cmd/staticcheck
+EXCLUDE_COMMON_UPDATE_TARGET=true
 
 ES_USER?=apm_server_user
 ES_PASS?=changeme
@@ -85,9 +86,9 @@ endif
 is-beats-updated: python-env
 	@$(PYTHON_ENV)/bin/python ./script/is_beats_updated.py ${BEATS_VERSION}
 
-# Collects all dependencies and then calls update
-.PHONY: collect
-collect: fields go-generate add-headers create-docs notice
+.PHONY: update
+update: go-generate add-headers create-docs notice mage
+	@mage update
 
 .PHONY: go-generate
 go-generate:

magefile.go

@@ -210,13 +210,20 @@ func TestPackagesInstall() error {
 	return nil
 }
 
-// Update updates the generated files (aka make update).
+// Update updates the generated files.
 func Update() error {
-	return sh.Run("make", "update")
+	mg.Deps(Fields, Config)
+	return nil
 }
 
 func Fields() error {
-	return mage.GenerateFieldsYAML("model")
+	if err := mage.GenerateFieldsYAML("model"); err != nil {
+		return err
+	}
+	if err := mage.GenerateAllInOneFieldsGo(); err != nil {
+		return err
+	}
+	return mage.Docs.FieldDocs("fields.yml")
 }
 
 // Use RACE_DETECTOR=true to enable the race detector.
@@ -264,9 +271,8 @@ func customizePackaging() {
 	)
 	for idx := len(mage.Packages) - 1; idx >= 0; idx-- {
 		args := &mage.Packages[idx]
-		pkgType := args.Types[0]
-		switch pkgType {
 
+		switch pkgType := args.Types[0]; pkgType {
 		case mage.Zip, mage.TarGz:
 			// Remove the reference config file from packages.
 			delete(args.Spec.Files, "{{.BeatName}}.reference.yml")
@@ -300,11 +306,19 @@ func customizePackaging() {
 			}
 
 		case mage.DMG:
+			// We do not build macOS packages.
 			mage.Packages = append(mage.Packages[:idx], mage.Packages[idx+1:]...)
+			continue
 
 		default:
 			panic(errors.Errorf("unhandled package type: %v", pkgType))
+		}
 
+		// Remove Kibana dashboard files.
+		for filename, filespec := range args.Spec.Files {
+			if strings.HasPrefix(filespec.Source, "_meta/kibana") {
+				delete(args.Spec.Files, filename)
+			}
 		}
 	}
 }

script/jenkins/windows-build.ps1

@@ -43,6 +43,12 @@ New-Item -ItemType directory -Path build\coverage | Out-Null
 New-Item -ItemType directory -Path build\system-tests | Out-Null
 New-Item -ItemType directory -Path build\system-tests\run | Out-Null
 
+choco install python -y -r --no-progress --version 3.8.1.20200110
+refreshenv
+$env:PATH = "C:\Python38;C:\Python38\Scripts;$env:PATH"
+$env:PYTHON_ENV = "$env:TEMP\python-env"
+python --version
+
 echo "Building fields.yml"
 exec { mage fields }
 

script/jenkins/windows-test.ps1

@@ -27,6 +27,13 @@ $env:PATH = "$env:GOPATH\bin;C:\tools\mingw64\bin;$env:PATH"
 # each run starts from a clean slate.
 $env:MAGEFILE_CACHE = "$env:WORKSPACE\.magefile"
 
+# Setup Python.
+choco install python -y -r --no-progress --version 3.8.1.20200110
+refreshenv
+$env:PATH = "C:\Python38;C:\Python38\Scripts;$env:PATH"
+$env:PYTHON_ENV = "$env:TEMP\python-env"
+python --version
+
 # Configure testing parameters.
 $env:TEST_COVERAGE = "true"
 $env:RACE_DETECTOR = "true"
@@ -59,9 +66,4 @@ $packages = ($packages|group|Select -ExpandProperty Name) -join ","
 exec { go test -race -c -cover -covermode=atomic -coverpkg $packages } "go test FAILURE"
 
 echo "Running python tests"
-choco install python -y -r --no-progress --version 3.8.1.20200110
-refreshenv
-$env:PATH = "C:\Python38;C:\Python38\Scripts;$env:PATH"
-$env:PYTHON_ENV = "$env:TEMP\python-env"
-python --version
 exec { mage pythonUnitTest } "System test FAILURE"