[WIP] Add very basic support for x-pack modules in Metricbeat

x-pack/metricbeat/magefile.go

@@ -0,0 +1,272 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+// +build mage
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"github.com/magefile/mage/mg"
+	"github.com/pkg/errors"
+
+	"github.com/elastic/beats/dev-tools/mage"
+)
+
+func init() {
+	mage.BeatDescription = "Metricbeat is a lightweight shipper for metrics."
+	mage.BeatLicense = "Elastic"
+}
+
+// Build builds the Beat binary.
+func Build() error {
+	return mage.Build(mage.DefaultBuildArgs())
+}
+
+// GolangCrossBuild builds the Beat binary inside of the golang-builder.
+// Do not use directly, use crossBuild instead.
+func GolangCrossBuild() error {
+	return mage.GolangCrossBuild(mage.DefaultGolangCrossBuildArgs())
+}
+
+// CrossBuild cross-builds the beat for all target platforms.
+func CrossBuild() error {
+	return mage.CrossBuild()
+}
+
+// Clean cleans all generated files and build artifacts.
+func Clean() error {
+	return mage.Clean()
+}
+
+// Fields generates a fields.yml and fields.go for each module.
+func Fields() {
+	mg.Deps(fieldsYML, mage.GenerateModuleFieldsGo)
+}
+
+// fieldsYML generates a fields.yml based on metricbeat + x-pack/metricbeat/modules.
+func fieldsYML() error {
+	return mage.GenerateFieldsYAML(mage.OSSBeatDir("module"), "module")
+}
+
+// Dashboards collects all the dashboards and generates index patterns.
+func Dashboards() error {
+	return mage.KibanaDashboards(mage.OSSBeatDir("module"), "module")
+}
+
+// Config generates both the short and reference configs.
+func Config() {
+	mg.Deps(shortConfig, referenceConfig, createDirModulesD)
+}
+
+// Update is an alias for executing fields, dashboards, config.
+func Update() {
+	mg.SerialDeps(Fields, Dashboards, mage.GenerateModuleIncludeListGo)
+}
+
+// Fmt formats source code and adds file headers.
+func Fmt() {
+	mg.Deps(mage.Format)
+}
+
+// Check runs fmt and update then returns an error if any modifications are found.
+func Check() {
+	mg.SerialDeps(mage.Format, Update, mage.Check)
+}
+
+// IntegTest executes integration tests (it uses Docker to run the tests).
+func IntegTest() {
+	mage.AddIntegTestUsage()
+	defer mage.StopIntegTestEnv()
+	mg.SerialDeps(GoIntegTest, PythonIntegTest)
+}
+
+// UnitTest executes the unit tests.
+func UnitTest() {
+	mg.SerialDeps(GoUnitTest, PythonUnitTest)
+}
+
+// GoUnitTest executes the Go unit tests.
+// Use TEST_COVERAGE=true to enable code coverage profiling.
+// Use RACE_DETECTOR=true to enable the race detector.
+func GoUnitTest(ctx context.Context) error {
+	return mage.GoTest(ctx, mage.DefaultGoTestUnitArgs())
+}
+
+// GoIntegTest executes the Go integration tests.
+// Use TEST_COVERAGE=true to enable code coverage profiling.
+// Use RACE_DETECTOR=true to enable the race detector.
+func GoIntegTest(ctx context.Context) error {
+	return mage.RunIntegTest("goIntegTest", func() error {
+		return mage.GoTest(ctx, mage.DefaultGoTestIntegrationArgs())
+	})
+}
+
+// PythonUnitTest executes the python system tests.
+func PythonUnitTest() error {
+	mg.Deps(mage.BuildSystemTestBinary)
+	return mage.PythonNoseTest(mage.DefaultPythonTestUnitArgs())
+}
+
+// PythonIntegTest executes the python system tests in the integration environment (Docker).
+func PythonIntegTest(ctx context.Context) error {
+	if !mage.IsInIntegTestEnv() {
+		mg.Deps(Fields)
+	}
+	return mage.RunIntegTest("pythonIntegTest", func() error {
+		mg.Deps(mage.BuildSystemTestBinary)
+		args := mage.DefaultPythonTestIntegrationArgs()
+		args.Env["MODULES_PATH"] = mage.CWD("module")
+		return mage.PythonNoseTest(args)
+	})
+}
+
+// -----------------------------------------------------------------------------
+// Customizations specific to Metricbeat.
+// - Include modules.d directory in packages.
+// - Disable system/load metricset for Windows.
+
+// customizePackaging modifies the package specs to add the modules.d directory.
+// And for Windows it comments out the system/load metricset because it's
+// not supported.
+func customizePackaging() {
+	var (
+		archiveModulesDir = "modules.d"
+		unixModulesDir    = "/etc/{{.BeatName}}/modules.d"
+
+		modulesDir = mage.PackageFile{
+			Mode:    0644,
+			Source:  "modules.d",
+			Config:  true,
+			Modules: true,
+		}
+		windowsModulesDir = mage.PackageFile{
+			Mode:    0644,
+			Source:  "{{.PackageDir}}/modules.d",
+			Config:  true,
+			Modules: true,
+			Dep: func(spec mage.PackageSpec) error {
+				if err := mage.Copy("modules.d", spec.MustExpand("{{.PackageDir}}/modules.d")); err != nil {
+					return errors.Wrap(err, "failed to copy modules.d dir")
+				}
+
+				return mage.FindReplace(
+					spec.MustExpand("{{.PackageDir}}/modules.d/system.yml"),
+					regexp.MustCompile(`- load`), `#- load`)
+			},
+		}
+		windowsReferenceConfig = mage.PackageFile{
+			Mode:   0644,
+			Source: "{{.PackageDir}}/metricbeat.reference.yml",
+			Dep: func(spec mage.PackageSpec) error {
+				err := mage.Copy("metricbeat.reference.yml",
+					spec.MustExpand("{{.PackageDir}}/metricbeat.reference.yml"))
+				if err != nil {
+					return errors.Wrap(err, "failed to copy reference config")
+				}
+
+				return mage.FindReplace(
+					spec.MustExpand("{{.PackageDir}}/metricbeat.reference.yml"),
+					regexp.MustCompile(`- load`), `#- load`)
+			},
+		}
+	)
+
+	for _, args := range mage.Packages {
+		switch args.OS {
+		case "windows":
+			args.Spec.Files[archiveModulesDir] = windowsModulesDir
+			args.Spec.ReplaceFile("{{.BeatName}}.reference.yml", windowsReferenceConfig)
+		default:
+			pkgType := args.Types[0]
+			switch pkgType {
+			case mage.TarGz, mage.Zip, mage.Docker:
+				args.Spec.Files[archiveModulesDir] = modulesDir
+			case mage.Deb, mage.RPM, mage.DMG:
+				args.Spec.Files[unixModulesDir] = modulesDir
+			default:
+				panic(errors.Errorf("unhandled package type: %v", pkgType))
+			}
+		}
+	}
+}
+
+func shortConfig() error {
+	var configParts = []string{
+		mage.OSSBeatDir("_meta/common.yml"),
+		"{{ elastic_beats_dir }}/libbeat/_meta/config.yml",
+	}
+
+	fmt.Printf("%+v", configParts)
+
+	for i, f := range configParts {
+		configParts[i] = mage.MustExpand(f)
+	}
+
+	configFile := mage.BeatName + ".yml"
+	mage.MustFileConcat(configFile, 0640, configParts...)
+	mage.MustFindReplace(configFile, regexp.MustCompile("beatname"), mage.BeatName)
+	mage.MustFindReplace(configFile, regexp.MustCompile("beat-index-prefix"), mage.BeatIndexPrefix)
+	return nil
+}
+
+func referenceConfig() error {
+	const modulesConfigYml = "build/config.modules.yml"
+	err := mage.GenerateModuleReferenceConfig(modulesConfigYml, mage.OSSBeatDir("module"), "module")
+	if err != nil {
+		return err
+	}
+	defer os.Remove(modulesConfigYml)
+
+	var configParts = []string{
+		mage.OSSBeatDir("_meta/common.reference.yml"),
+		modulesConfigYml,
+		"{{ elastic_beats_dir }}/libbeat/_meta/config.reference.yml",
+	}
+
+	for i, f := range configParts {
+		configParts[i] = mage.MustExpand(f)
+	}
+
+	configFile := mage.BeatName + ".reference.yml"
+	mage.MustFileConcat(configFile, 0640, configParts...)
+	mage.MustFindReplace(configFile, regexp.MustCompile("beatname"), mage.BeatName)
+	mage.MustFindReplace(configFile, regexp.MustCompile("beat-index-prefix"), mage.BeatIndexPrefix)
+	return nil
+}
+
+func createDirModulesD() error {
+	if err := os.RemoveAll("modules.d"); err != nil {
+		return err
+	}
+
+	shortConfigs, err := filepath.Glob("module/*/_meta/config.yml")
+	if err != nil {
+		return err
+	}
+
+	for _, f := range shortConfigs {
+		parts := strings.Split(filepath.ToSlash(f), "/")
+		if len(parts) < 2 {
+			continue
+		}
+		moduleName := parts[1]
+
+		cp := mage.CopyTask{
+			Source: f,
+			Dest:   filepath.Join("modules.d", moduleName+".yml.disabled"),
+			Mode:   0644,
+		}
+		if err = cp.Execute(); err != nil {
+			return err
+		}
+	}
+	return nil
+}

x-pack/metricbeat/make.bat

@@ -0,0 +1,11 @@
+@echo off
+
+REM Windows wrapper for Mage (https://magefile.org/) that installs it
+REM to %GOPATH%\bin from the Beats vendor directory.
+REM
+REM After running this once you may invoke mage.exe directly.
+
+WHERE mage
+IF %ERRORLEVEL% NEQ 0 go install github.com/elastic/beats/vendor/github.com/magefile/mage
+
+mage %*

x-pack/metricbeat/module/foo/_meta/config.yml

@@ -0,0 +1,6 @@
+- module: foo
+  metricsets: ["bar"]
+  enabled: false
+  period: 10s
+  hosts: ["localhost"]
+

x-pack/metricbeat/module/foo/_meta/docs.asciidoc

@@ -0,0 +1,2 @@
+This is the foo module.
+

x-pack/metricbeat/module/foo/_meta/fields.yml

@@ -0,0 +1,11 @@
+- key: foo
+  title: "foo"
+  description: >
+    experimental[]
+
+    foo module
+  fields:
+    - name: foo
+      type: group
+      description: >
+      fields:

x-pack/metricbeat/module/foo/bar/_meta/data.json

@@ -0,0 +1,19 @@
+{
+    "@timestamp":"2016-05-23T08:05:34.853Z",
+    "beat":{
+        "hostname":"beathost",
+        "name":"beathost"
+    },
+    "metricset":{
+        "host":"localhost",
+        "module":"foo",
+        "name":"bar",
+        "rtt":44269
+    },
+    "foo":{
+        "bar":{
+            "example": "bar"
+        }
+    },
+    "type":"metricsets"
+}

x-pack/metricbeat/module/foo/bar/_meta/docs.asciidoc

@@ -0,0 +1 @@
+This is the bar metricset of the module foo.

x-pack/metricbeat/module/foo/bar/_meta/fields.yml

@@ -0,0 +1,9 @@
+- name: bar
+  type: group
+  description: >
+    bar
+  fields:
+    - name: counter
+      type: long
+      description: >
+        Example field