diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index 2aeaf33358ad..2d740169c100 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -24,6 +24,7 @@ The list below covers the major changes between 7.0.0-rc2 and master only. - For "metricbeat style" generated custom beats, the mage target `GoTestIntegration` has changed to `GoIntegTest` and `GoTestUnit` has changed to `GoUnitTest`. {pull}13341[13341] - Build docker and kubernetes features only on supported platforms. {pull}13509[13509] - Need to register new processors to be used in the JS processor in their `init` functions. {pull}13509[13509] +- The custom beat generator now uses mage instead of python, `mage GenerateCustomBeat` can be used to create a new beat, and `mage vendorUpdate` to update the vendored libbeat in a custom beat. {pull}13610[13610] ==== Bugfixes diff --git a/dev-tools/mage/settings.go b/dev-tools/mage/settings.go index 07db69cca2c7..dfe674326071 100644 --- a/dev-tools/mage/settings.go +++ b/dev-tools/mage/settings.go @@ -235,6 +235,14 @@ var ( elasticBeatsDirLock sync.Mutex ) +// SetElasticBeatsDir sets the internal elastic beats dir to a preassigned value +func SetElasticBeatsDir(path string) { + elasticBeatsDirLock.Lock() + defer elasticBeatsDirLock.Unlock() + + elasticBeatsDirValue = path +} + // ElasticBeatsDir returns the path to Elastic beats dir. func ElasticBeatsDir() (string, error) { elasticBeatsDirLock.Lock() diff --git a/docs/devguide/creating-beat-from-metricbeat.asciidoc b/docs/devguide/creating-beat-from-metricbeat.asciidoc index 77b55c48707d..0e4c46736f93 100644 --- a/docs/devguide/creating-beat-from-metricbeat.asciidoc +++ b/docs/devguide/creating-beat-from-metricbeat.asciidoc @@ -12,6 +12,7 @@ must be set up correctly. In addition, the following tools are required: * https://www.python.org/downloads/[python] * https://virtualenv.pypa.io/en/stable/[virtualenv] +* https://github.com/magefile/mage[mage] Virtualenv is easiest installed with your package manager or https://pip.pypa.io/en/stable/[pip]. @@ -43,34 +44,20 @@ This directory is normally located under `$GOPATH/src/github.com/{your-github-na [float] ==== Step 2 - Create the Beat -Run the command: +Run the command from the root `beats` directory: [source,bash] ---- -python ${GOPATH}/src/github.com/elastic/beats/script/generate.py --type=metricbeat +NEWBEAT_TYPE=metricbeat mage GenerateCustomBeat ---- -When prompted, enter the Beat name and path. +When prompted, enter the Beat name and path, along with an initial module and metricset name for your beat. For more details about creating a metricset, see the docs <>. - -[float] -==== Step 3 - Init and create the metricset - -After creating the Beat, change the directory to `$GOPATH/src/github.com/{your-github-name}/{beat}` and run: - -[source,bash] ----- -make setup ----- - -This will do the initial setup for your Beat and also run `make create-metricset`, which will ask you for the -module name and metricset name of your Beat. - -For more details about creating a metricset, see the docs <>. +After creating the Beat, change the directory to `$GOPATH/src/github.com/{your-github-name}/{beat}` [float] -==== Step 4 - Build & Run +==== Step 3 - Build & Run To create a binary run the `make` command. This will create the binary in your beats directory. @@ -84,7 +71,7 @@ To run it, execute the binary. This will automatically load the default configur This will run the beat with debug output enabled to the console to directly see what is happening. Stop the beat with `CTRL-C`. [float] -==== Step 5 - Package +==== Step 4 - Package To create packages and binaries for different platforms, https://www.docker.com/[docker] is required. The first step is to get the most recent packaging tools into your beat: diff --git a/docs/devguide/newbeat.asciidoc b/docs/devguide/newbeat.asciidoc index bc775a467d9f..d70a565e4611 100644 --- a/docs/devguide/newbeat.asciidoc +++ b/docs/devguide/newbeat.asciidoc @@ -122,24 +122,23 @@ mkdir ${GOPATH}/src/github.com/{user} cd ${GOPATH}/src/github.com/{user} -------------------- -Run python and specify the path to the Beat generator: +Run the mage script to generate the custom beat: -NOTE: Python 2 is required (Python 3 will not work). [source,shell] -------------------- -python $GOPATH/src/github.com/elastic/beats/script/generate.py +mage GenerateCustomBeat -------------------- -Python will prompt you to enter information about your Beat. For the `project_name`, enter `Countbeat`. +The mage script will prompt you to enter information about your Beat. For the `project_name`, enter `Countbeat`. For the `github_name`, enter your github id. The `beat` and `beat_path` are set to the correct values automatically (just press Enter to accept each default). For the `full_name`, enter your firstname and lastname. [source,shell] --------- -Beat Name [Examplebeat]: Countbeat -Your Github Name [your-github-name]: {username} -Beat Path [github.com/{github id}/{beat name}]: -Firstname Lastname: {Full Name} +Enter a project name [examplebeat]: Countbeat +Enter a github name [your-github-name]: {username} +Enter a beat path [github.com/{username}/countbeat]: +Enter a full name [Firstname Lastname]: {Full Name} --------- The Beat generator creates a directory called `countbeat` inside of your project folder (e.g. {project folder}/github.com/{github id}/countbeat). diff --git a/generator/beat/{beat}/Makefile b/generator/beat/{beat}/Makefile index 3a96d91280d3..0246f73dc98e 100644 --- a/generator/beat/{beat}/Makefile +++ b/generator/beat/{beat}/Makefile @@ -14,32 +14,6 @@ CHECK_HEADERS_DISABLED=true # Path to the libbeat Makefile -include $(LIBBEAT_MAKEFILE) -# Initial beat setup -.PHONY: setup -setup: pre-setup git-add - -pre-setup: copy-vendor git-init - $(MAKE) -f $(LIBBEAT_MAKEFILE) mage ES_BEATS=$(ES_BEATS) - $(MAKE) -f $(LIBBEAT_MAKEFILE) update BEAT_NAME=$(BEAT_NAME) ES_BEATS=$(ES_BEATS) NO_COLLECT=$(NO_COLLECT) - -# Copy beats into vendor directory .PHONY: copy-vendor -copy-vendor: vendor-check - mkdir -p vendor/github.com/elastic/beats - git archive --remote ${BEAT_GOPATH}/src/github.com/elastic/beats HEAD | tar -x --exclude=x-pack -C vendor/github.com/elastic/beats - mkdir -p vendor/github.com/magefile - cp -R vendor/github.com/elastic/beats/vendor/github.com/magefile/mage vendor/github.com/magefile - -.PHONY: git-init -git-init: - git init - -.PHONY: git-add -git-add: - git add -A - git commit -q -m "Add generated {beat} files" - - -.PHONY: vendor-check -vendor-check: - @if output=$$(git -C ${BEAT_GOPATH}/src/github.com/elastic/beats status --porcelain) && [ ! -z "$${output}" ]; then printf "\033[31mWARNING: elastic/beats has uncommitted changes, these will not be in the vendor directory!\033[0m\n"; fi \ No newline at end of file +copy-vendor: + mage vendorUpdate \ No newline at end of file diff --git a/generator/beat/{beat}/beater/{beat}.go b/generator/beat/{beat}/beater/{beat}.go index 7c0a1027df25..8031896402b7 100644 --- a/generator/beat/{beat}/beater/{beat}.go +++ b/generator/beat/{beat}/beater/{beat}.go @@ -11,8 +11,8 @@ import ( "{beat_path}/config" ) -// {Beat} configuration. -type {Beat} struct { +// {beat} configuration. +type {beat} struct { done chan struct{} config config.Config client beat.Client @@ -25,7 +25,7 @@ func New(b *beat.Beat, cfg *common.Config) (beat.Beater, error) { return nil, fmt.Errorf("Error reading config file: %v", err) } - bt := &{Beat}{ + bt := &{beat}{ done: make(chan struct{}), config: c, } @@ -33,7 +33,7 @@ func New(b *beat.Beat, cfg *common.Config) (beat.Beater, error) { } // Run starts {beat}. -func (bt *{Beat}) Run(b *beat.Beat) error { +func (bt *{beat}) Run(b *beat.Beat) error { logp.Info("{beat} is running! Hit CTRL-C to stop it.") var err error @@ -65,7 +65,7 @@ func (bt *{Beat}) Run(b *beat.Beat) error { } // Stop stops {beat}. -func (bt *{Beat}) Stop() { +func (bt *{beat}) Stop() { bt.client.Close() close(bt.done) } diff --git a/generator/beat/{beat}/magefile.go b/generator/beat/{beat}/magefile.go index bd67f87baade..1c587749c4d3 100644 --- a/generator/beat/{beat}/magefile.go +++ b/generator/beat/{beat}/magefile.go @@ -7,21 +7,23 @@ import ( "time" "github.com/magefile/mage/mg" + "github.com/magefile/mage/sh" devtools "github.com/elastic/beats/dev-tools/mage" + "github.com/elastic/beats/generator/common/beatgen" // mage:import - _ "github.com/elastic/beats/dev-tools/mage/target/common" + "github.com/elastic/beats/dev-tools/mage/target/pkg" // mage:import "github.com/elastic/beats/dev-tools/mage/target/build" // mage:import - "github.com/elastic/beats/dev-tools/mage/target/update" + _ "github.com/elastic/beats/dev-tools/mage/target/common" // mage:import _ "github.com/elastic/beats/dev-tools/mage/target/test" // mage:import _ "github.com/elastic/beats/dev-tools/mage/target/unittest" // mage:import - "github.com/elastic/beats/dev-tools/mage/target/pkg" + _ "github.com/elastic/beats/dev-tools/mage/target/integtest" ) func init() { @@ -31,6 +33,11 @@ func init() { devtools.BeatVendor = "{full_name}" } +// VendorUpdate updates elastic/beats in the vendor dir +func VendorUpdate() error { + return beatgen.VendorUpdate() +} + // Package packages the Beat for distribution. // Use SNAPSHOT=true to build snapshots. // Use PLATFORMS to control the target platforms. @@ -40,17 +47,22 @@ func Package() { devtools.UseCommunityBeatPackaging() - mg.Deps(update.Update) + mg.Deps(Update) mg.Deps(build.CrossBuild, build.CrossBuildGoDaemon) mg.SerialDeps(devtools.Package, pkg.PackageTest) } -// Config generates both the short/reference/docker configs. -func Config() error { - return devtools.Config(devtools.AllConfigTypes, devtools.ConfigFileParams{}, ".") +// Update updates the generated files (aka make update). +func Update() error { + return sh.Run("make", "update") } -//Fields generates a fields.yml for the Beat. +// Fields generates a fields.yml for the Beat. func Fields() error { return devtools.GenerateFieldsYAML() } + +// Config generates both the short/reference/docker configs. +func Config() error { + return devtools.Config(devtools.AllConfigTypes, devtools.ConfigFileParams{}, ".") +} diff --git a/generator/common/Makefile b/generator/common/Makefile index e880b412e32b..a0740683606b 100644 --- a/generator/common/Makefile +++ b/generator/common/Makefile @@ -1,45 +1,36 @@ -BUILD_DIR?=build PWD=$(shell pwd) -PYTHON_ENV?=${BUILD_DIR}/python-env BEAT_TYPE?=beat -BEAT_PATH=${BUILD_DIR}/src/beatpath/testbeat +BEAT_NAME?=beatpath/test${BEAT_TYPE} +BEAT_PATH=${GOPATH}/src/${BEAT_NAME} ES_BEATS=${GOPATH}/src/github.com/elastic/beats PREPARE_COMMAND?= # Runs test build for mock beat .PHONY: test test: prepare-test - . ${PYTHON_ENV}/bin/activate; \ - export GOPATH=${PWD}/build ; \ - export PATH=$${GOPATH}/bin:${PATH}; \ cd ${BEAT_PATH} ; \ - $(MAKE) copy-vendor || exit 1 ; \ - ${PREPARE_COMMAND} \ - $(MAKE) git-init || exit 1 ; \ - $(MAKE) update fmt || exit 1 ; \ + export PATH=$${GOPATH}/bin:$${PATH}; \ git config user.email "beats-jenkins@test.com" || exit 1 ; \ git config user.name "beats-jenkins" || exit 1 ; \ - $(MAKE) git-add || exit 1 ; \ $(MAKE) check CHECK_HEADERS_DISABLED=y || exit 1 ; \ $(MAKE) || exit 1 ; \ $(MAKE) unit .PHONY: prepare-test -prepare-test:: python-env - # Makes sure to use current version of beats for testing - rm -fr ${BUILD_DIR}/src/github.com/elastic/beats/ - git clone -s ${PWD}/../../ ${BUILD_DIR}/src/github.com/elastic/beats/ - +prepare-test:: ${GOPATH}/bin/mage rm -fr ${BEAT_PATH} mkdir -p ${BEAT_PATH} - export GOPATH=${PWD}/build ; \ - . ${PYTHON_ENV}/bin/activate && \ - python ${PWD}/build/src/github.com/elastic/beats/script/generate.py \ - --type=${BEAT_TYPE} \ - --project_name=Testbeat \ - --github_name=ruflin \ - --beat_path=beatpath/testbeat \ - --full_name="Nicolas Ruflin" + cd ${GOPATH}/src/github.com/elastic/beats/ ; \ + export MODULE=elastic ; \ + export METRICSET=test ; \ + export GO111MODULE=off; \ + export NEWBEAT_PROJECT_NAME=Testbeat ; \ + export NEWBEAT_GITHUB_NAME=ruflin ; \ + export NEWBEAT_BEAT_PATH=${BEAT_NAME} ; \ + export NEWBEAT_FULL_NAME="Nicolas Ruflin" ; \ + export NEWBEAT_TYPE=${BEAT_TYPE} ; \ + export NEWBEAT_DEV=1 ; \ + mage GenerateCustomBeat # Runs test build for the created beat .PHONY: test-build @@ -51,15 +42,12 @@ test-build: test $(MAKE) deps ; \ $(MAKE) images -# Sets up the virtual python environment -.PHONY: python-env -python-env: - @test -d ${PYTHON_ENV} || virtualenv ${PYTHON_ENV} - @${PYTHON_ENV}/bin/pip install --upgrade pip PyYAML - @# Work around pip bug. See: https://github.com/pypa/pip/issues/4464 - @find $(PYTHON_ENV) -type d -name dist-packages -exec sh -c "echo dist-packages > {}.pth" ';' +${GOPATH}/bin/mage: + go get -u -d github.com/magefile/mage + cd $${GOPATH}/src/github.com/magefile/mage; \ + go run bootstrap.go # Cleans up environment .PHONY: clean clean: - @rm -rf build + rm -rf ${BEAT_PATH} diff --git a/generator/common/beatgen/beatgen.go b/generator/common/beatgen/beatgen.go new file mode 100644 index 000000000000..cdb3bd1e5c9c --- /dev/null +++ b/generator/common/beatgen/beatgen.go @@ -0,0 +1,193 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package beatgen + +import ( + "bufio" + "fmt" + "go/build" + "os" + "path/filepath" + "strings" + + devtools "github.com/elastic/beats/dev-tools/mage" + "github.com/elastic/beats/generator/common/beatgen/setup" + "github.com/magefile/mage/mg" + "github.com/magefile/mage/sh" + "github.com/pkg/errors" +) + +// ConfigItem represents a value that must be configured for the custom beat +type ConfigItem struct { + Key string + Default func(map[string]string) string + Help string +} + +// required user config for a custom beat +// These are specified in env variables with newbeat_* +var configList = []ConfigItem{ + { + Key: "project_name", + Help: "Enter the beat name", + Default: func(cfg map[string]string) string { + return "examplebeat" + }, + }, + { + Key: "github_name", + Help: "Enter your github name", + Default: func(cfg map[string]string) string { + return "your-github-name" + }, + }, + { + Key: "beat_path", + Help: "Enter the beat path", + Default: func(cfg map[string]string) string { + ghName, _ := cfg["github_name"] + beatName, _ := cfg["project_name"] + return "github.com/" + ghName + "/" + strings.ToLower(beatName) + }, + }, + { + Key: "full_name", + Help: "Enter your full name", + Default: func(cfg map[string]string) string { + return "Firstname Lastname" + }, + }, + { + Key: "type", + Help: "Enter the beat type", + Default: func(cfg map[string]string) string { + return "beat" + }, + }, +} + +// Generate generates a new custom beat +func Generate() error { + cfg, err := getConfig() + if err != nil { + return errors.Wrap(err, "Error getting config") + } + err = setup.GenNewBeat(cfg) + if err != nil { + return errors.Wrap(err, "Error generating new beat") + } + + absBeatPath := filepath.Join(build.Default.GOPATH, "src", cfg["beat_path"]) + + err = os.Chdir(absBeatPath) + if err != nil { + return errors.Wrap(err, "error changing directory") + } + + mg.Deps(setup.CopyVendor) + mg.Deps(setup.RunSetup) + mg.Deps(setup.GitInit) + + if cfg["type"] == "metricbeat" { + //This is runV because it'll ask for user input, so we need stdout. + err = sh.RunV("make", "create-metricset") + if err != nil { + return errors.Wrap(err, "error running create-metricset") + } + } + + mg.Deps(setup.Update) + mg.Deps(setup.GitAdd) + + fmt.Printf("=======================\n") + fmt.Printf("Your custom beat is now available as %s\n", absBeatPath) + fmt.Printf("=======================\n") + + return nil +} + +// VendorUpdate updates the beat vendor directory +func VendorUpdate() error { + err := sh.Rm("./vendor/github.com/elastic/beats") + if err != nil { + return errors.Wrap(err, "error removing vendor dir") + } + + devtools.SetElasticBeatsDir(getAbsoluteBeatsPath()) + mg.SerialDeps(setup.CopyVendor, setup.LinkImportsHelper) + return nil +} + +// returns a "compleated" config object with everything we need +func getConfig() (map[string]string, error) { + userCfg := make(map[string]string) + for _, cfgVal := range configList { + var cfgKey string + var err error + cfgKey, isSet := getEnvConfig(cfgVal.Key) + if !isSet { + cfgKey, err = getValFromUser(cfgVal.Help, cfgVal.Default(userCfg)) + if err != nil { + return userCfg, err + } + } + userCfg[cfgVal.Key] = cfgKey + } + + return userCfg, nil + +} + +func getEnvConfig(cfgKey string) (string, bool) { + EnvKey := fmt.Sprintf("%s_%s", setup.CfgPrefix, strings.ToUpper(cfgKey)) + + envKey := os.Getenv(EnvKey) + + if envKey == "" { + return envKey, false + } + return envKey, true +} + +// getValFromUser returns a config object from the user. If they don't enter one, fallback to the default +func getValFromUser(help, def string) (string, error) { + reader := bufio.NewReader(os.Stdin) + + // Human-readable prompt + fmt.Printf("%s [%s]: ", help, def) + str, err := reader.ReadString('\n') + if err != nil { + return "", err + } + if str == "\n" { + return def, nil + } + return strings.TrimSpace(str), nil + +} + +// getAbsoluteBeatsPath tries to infer the "real" non-vendor beats path +func getAbsoluteBeatsPath() string { + beatsImportPath := "github.com/elastic/beats" + gopath := os.Getenv("GOPATH") + if gopath != "" { + return filepath.Join(gopath, "src", beatsImportPath) + } + return filepath.Join(build.Default.GOPATH, "src", beatsImportPath) + +} diff --git a/generator/common/beatgen/setup/creator.go b/generator/common/beatgen/setup/creator.go new file mode 100644 index 000000000000..26859f101bb3 --- /dev/null +++ b/generator/common/beatgen/setup/creator.go @@ -0,0 +1,87 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package setup + +import ( + "fmt" + "go/build" + "io/ioutil" + "os" + "path/filepath" + "strings" + + devtools "github.com/elastic/beats/dev-tools/mage" + "github.com/pkg/errors" +) + +// CfgPrefix specifies the env variable prefix used to configure the beat +const CfgPrefix = "NEWBEAT" + +// GenNewBeat generates a new custom beat +// We assume our config object is populated and valid here +func GenNewBeat(config map[string]string) error { + if config["type"] != "beat" && config["type"] != "metricbeat" { + return fmt.Errorf("%s is not a valid custom beat type. Valid types are 'beat' and 'metricbeat'", config["type"]) + } + + genPath := devtools.OSSBeatDir("generator", config["type"], "{beat}") + err := filepath.Walk(genPath, func(path string, info os.FileInfo, err error) error { + newBase := filepath.Join(build.Default.GOPATH, "src", config["beat_path"]) + replacePath := strings.Replace(path, genPath, newBase, -1) + + writePath := strings.Replace(replacePath, "{beat}", config["project_name"], -1) + writePath = strings.Replace(writePath, ".go.tmpl", ".go", -1) + if info.IsDir() { + err := os.MkdirAll(writePath, 0755) + if err != nil { + return errors.Wrapf(err, "error creating directory %s", writePath) + } + } else { + + //dump original source file + tmplFile, err := ioutil.ReadFile(path) + if err != nil { + return errors.Wrap(err, "error reading source templatse file") + } + newFile := replaceVars(config, string(tmplFile)) + + err = ioutil.WriteFile(writePath, []byte(newFile), 0644) + if err != nil { + return errors.Wrap(err, "error writing beat file") + } + } + + return nil + }) + + return err +} + +// replaceVars replaces any template vars in a target file +// We're not using the golang template engine as it seems a tad heavy-handed for this use case +// We have a dozen or so files across various languages (go, make, etc) and most just need one or two vars replaced. +func replaceVars(config map[string]string, fileBody string) string { + var newBody = fileBody + config["beat"] = strings.ToLower(config["project_name"]) + for tmplName, tmplValue := range config { + tmplStr := fmt.Sprintf("{%s}", tmplName) + newBody = strings.ReplaceAll(newBody, tmplStr, tmplValue) + } + + return newBody +} diff --git a/generator/common/beatgen/setup/setup.go b/generator/common/beatgen/setup/setup.go new file mode 100644 index 000000000000..7bebb0a735a4 --- /dev/null +++ b/generator/common/beatgen/setup/setup.go @@ -0,0 +1,154 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package setup + +import ( + "fmt" + "os" + "path/filepath" + + devtools "github.com/elastic/beats/dev-tools/mage" + "github.com/magefile/mage/sh" + "github.com/pkg/errors" +) + +// RunSetup runs any remaining setup commands after the vendor directory has been setup +func RunSetup() error { + vendorPath := "./vendor/github.com/" + + //Copy mage stuff + toMkdir := filepath.Join(vendorPath, "magefile") + err := os.MkdirAll(toMkdir, 0755) + if err != nil { + return errors.Wrapf(err, "error making mage directory at %s", toMkdir) + } + + err = sh.Run("cp", "-R", filepath.Join(vendorPath, "elastic/beats/vendor/github.com/magefile/mage"), filepath.Join(vendorPath, "magefile")) + if err != nil { + return errors.Wrapf(err, "error copying vendored magefile to %s", filepath.Join(vendorPath, "magefile")) + } + + //Copy the pkg helper + err = sh.Run("cp", "-R", filepath.Join(vendorPath, "elastic/beats/vendor/github.com/pkg"), vendorPath) + if err != nil { + return errors.Wrapf(err, "error copying pkg to %s", vendorPath) + } + return LinkImportsHelper() +} + +// LinkImportsHelper links generate_imports_helper.py +func LinkImportsHelper() error { + vendorPath := "./vendor/github.com/" + pwd, err := os.Getwd() + if err != nil { + return errors.Wrap(err, "error gettting current directory") + } + return sh.Run("ln", "-sf", filepath.Join(pwd, vendorPath, "elastic/beats/metricbeat/scripts/generate_imports_helper.py"), filepath.Join(pwd, vendorPath, "elastic/beats/script/generate_imports_helper.py")) +} + +// CopyVendor copies a new version of beats into the vendor directory of PWD +// By default this uses git archive, meaning any uncommitted changes will not be copied. +// Set the NEWBEAT_DEV env variable to use a slow `cp` copy that will catch uncommited changes +func CopyVendor() error { + vendorPath := "./vendor/github.com/elastic/" + beatPath, err := devtools.ElasticBeatsDir() + if err != nil { + return errors.Wrap(err, "Could not find ElasticBeatsDir") + } + err = os.MkdirAll(vendorPath, 0755) + if err != nil { + return errors.Wrap(err, "error creating vendor dir") + } + + isClean, err := checkBeatsDirClean() + if err != nil { + return errors.Wrap(err, "error in checkIfBeatsDirClean") + } + + if !isClean { + //Dev mode. Use CP. + fmt.Printf("You have uncommited changes in elastic/beats. Running CopyVendor running in dev mode, elastic/beats will be copied into the vendor directory with cp\n") + vendorPath = filepath.Join(vendorPath, "beats") + + err = sh.Run("cp", "-R", beatPath, vendorPath) + if err != nil { + return errors.Wrap(err, "error copying vendor dir") + } + err = sh.Rm(filepath.Join(vendorPath, ".git")) + if err != nil { + return errors.Wrap(err, "error removing vendor git directory") + } + err = sh.Rm(filepath.Join(vendorPath, "x-pack")) + if err != nil { + return errors.Wrap(err, "error removing x-pack directory") + } + } else { + //not dev mode. Use git archive + vendorPath = filepath.Join(vendorPath, "beats") + err = os.MkdirAll(vendorPath, 0755) + if err != nil { + return errors.Wrap(err, "error creating vendor dir") + } + err = sh.Run("sh", + "-c", + "git archive --remote "+beatPath+" HEAD | tar -x --exclude=x-pack -C "+vendorPath) + if err != nil { + return errors.Wrap(err, "error running git archive") + } + } + + return nil + +} + +// checkIfBeatsDirClean checks to see if the working elastic/beats dir is modified. +// If it is, we'll use a different method to copy beats to vendor/ +func checkBeatsDirClean() (bool, error) { + beatPath, err := devtools.ElasticBeatsDir() + if err != nil { + return false, errors.Wrap(err, "Could not find ElasticBeatsDir") + } + out, err := sh.Output("git", "-C", beatPath, "status", "--porcelain") + if err != nil { + return false, errors.Wrap(err, "Error checking status of elastic/beats repo") + } + + if len(out) == 0 { + return true, nil + } + return false, nil +} + +// GitInit initializes a new git repo in the current directory +func GitInit() error { + return sh.Run("git", "init") +} + +// GitAdd adds the current working directory to git and does an initial commit +func GitAdd() error { + err := sh.Run("git", "add", "-A") + if err != nil { + return errors.Wrap(err, "error running git add") + } + return sh.Run("git", "commit", "-q", "-m", "Initial commit, Add generated files") +} + +// Update updates the generated files (aka make update). +func Update() error { + return sh.Run("make", "update") +} diff --git a/generator/metricbeat/Makefile b/generator/metricbeat/Makefile index 2441db7c351e..c773d75926c1 100644 --- a/generator/metricbeat/Makefile +++ b/generator/metricbeat/Makefile @@ -1,13 +1,7 @@ BEAT_TYPE = metricbeat -PREPARE_COMMAND = MODULE=elastic METRICSET=test make create-metricset; include ../common/Makefile -.PHONY: prepare-test -prepare-test:: python-env - mkdir -p ${BEAT_PATH}/scripts - rsync -a --exclude=build ${PWD}/../../metricbeat/scripts/generate_imports_helper.py ${BEAT_PATH}/scripts - # Collects all dependencies and then calls update .PHONY: collect collect: diff --git a/generator/metricbeat/{beat}/Makefile b/generator/metricbeat/{beat}/Makefile index 996964eab9ec..932b05083789 100644 --- a/generator/metricbeat/{beat}/Makefile +++ b/generator/metricbeat/{beat}/Makefile @@ -12,32 +12,6 @@ CHECK_HEADERS_DISABLED=true # Path to the libbeat Makefile -include $(ES_BEATS)/metricbeat/Makefile -# Initial beat setup -.PHONY: setup -setup: copy-vendor git-init - #call make recursively so we can reload the above include. - #Only needed during the first setup phase, before /vendor exists - $(MAKE) create-metricset update git-add - -# Copy beats into vendor directory .PHONY: copy-vendor -copy-vendor: vendor-check - mkdir -p vendor/github.com/elastic/beats - git archive --remote ${BEAT_GOPATH}/src/github.com/elastic/beats HEAD | tar -x --exclude=x-pack -C vendor/github.com/elastic/beats - ln -sf ${PWD}/vendor/github.com/elastic/beats/metricbeat/scripts/generate_imports_helper.py ${PWD}/vendor/github.com/elastic/beats/script/generate_imports_helper.py - mkdir -p vendor/github.com/magefile - cp -R ${BEAT_GOPATH}/src/github.com/elastic/beats/vendor/github.com/magefile/mage vendor/github.com/magefile - cp -R ${BEAT_GOPATH}/src/github.com/elastic/beats/vendor/github.com/pkg vendor/github.com/ - -.PHONY: git-init -git-init: - git init - -.PHONY: git-add -git-add: - git add -A - git commit -q -m "Add generated {beat} files" - -.PHONY: vendor-check -vendor-check: - @if output=$$(git -C ${BEAT_GOPATH}/src/github.com/elastic/beats status --porcelain) && [ ! -z "$${output}" ]; then printf "\033[31mWARNING: elastic/beats has uncommitted changes, these will not be in the vendor directory!\033[0m\n"; fi \ No newline at end of file +copy-vendor: + mage vendorUpdate \ No newline at end of file diff --git a/generator/metricbeat/{beat}/magefile.go b/generator/metricbeat/{beat}/magefile.go index dc575b5da9d5..db9ac8970c0c 100644 --- a/generator/metricbeat/{beat}/magefile.go +++ b/generator/metricbeat/{beat}/magefile.go @@ -9,6 +9,7 @@ import ( "github.com/magefile/mage/mg" devtools "github.com/elastic/beats/dev-tools/mage" + "github.com/elastic/beats/generator/common/beatgen" metricbeat "github.com/elastic/beats/metricbeat/scripts/mage" // mage:import @@ -36,6 +37,11 @@ func init() { devtools.BeatVendor = "{full_name}" } +// VendorUpdate updates elastic/beats in the vendor dir +func VendorUpdate() error { + return beatgen.VendorUpdate() +} + // CollectAll generates the docs and the fields. func CollectAll() { mg.Deps(collectors.CollectDocs, FieldsDocs) diff --git a/generator/metricbeat/{beat}/main.go b/generator/metricbeat/{beat}/main.go index 793c64f756b1..223a026ff88c 100644 --- a/generator/metricbeat/{beat}/main.go +++ b/generator/metricbeat/{beat}/main.go @@ -9,7 +9,6 @@ import ( _ "{beat_path}/include" ) - func main() { if err := cmd.RootCmd.Execute(); err != nil { os.Exit(1) diff --git a/heartbeat/magefile.go b/heartbeat/magefile.go index 3119eacf6560..7aafc45cf23b 100644 --- a/heartbeat/magefile.go +++ b/heartbeat/magefile.go @@ -28,6 +28,7 @@ import ( "github.com/magefile/mage/sh" devtools "github.com/elastic/beats/dev-tools/mage" + "github.com/elastic/beats/generator/common/beatgen" heartbeat "github.com/elastic/beats/heartbeat/scripts/mage" // mage:import @@ -42,6 +43,11 @@ func init() { devtools.BeatServiceName = "heartbeat-elastic" } +// VendorUpdate updates elastic/beats in the vendor dir +func VendorUpdate() error { + return beatgen.VendorUpdate() +} + // Build builds the Beat binary. func Build() error { return devtools.Build(devtools.DefaultBuildArgs()) diff --git a/magefile.go b/magefile.go index 25eddd0acef6..e932e1412495 100644 --- a/magefile.go +++ b/magefile.go @@ -24,6 +24,7 @@ import ( "os" "path/filepath" + "github.com/elastic/beats/generator/common/beatgen" "github.com/magefile/mage/mg" "github.com/pkg/errors" "go.uber.org/multierr" @@ -45,6 +46,11 @@ var ( } ) +// GenerateCustomBeat generates a new custom beat +func GenerateCustomBeat() error { + return beatgen.Generate() +} + // PackageBeatDashboards packages the dashboards from all Beats into a zip // file. The dashboards must be generated first. func PackageBeatDashboards() error { @@ -129,6 +135,7 @@ func CheckLicenseHeaders() error { licenser.Exclude("x-pack"), licenser.Exclude("generator/beat/{beat}"), licenser.Exclude("generator/metricbeat/{beat}"), + licenser.Exclude("generator/beat/{beat}"), ), licenser( licenser.Check(), diff --git a/script/generate.py b/script/generate.py index 12800cab551f..21ea9f9f781d 100644 --- a/script/generate.py +++ b/script/generate.py @@ -1,123 +1,3 @@ -import argparse -import datetime -import os - -# Creates a new beat or metricbeat based on the given parameters - -project_name = "" -github_name = "" -beat = "" -beat_path = "" -full_name = "" - - -def generate_beat(args): - - global project_name, github_name, beat, beat_path, full_name - - if args.project_name is not None: - project_name = args.project_name - - if args.github_name is not None: - github_name = args.github_name - - if args.beat_path is not None: - beat_path = args.beat_path - - if args.full_name is not None: - full_name = args.full_name - - read_input() - process_file(args.type) - - -def read_input(): - """Requests input form the command line for empty variables if needed. - """ - global project_name, github_name, beat, beat_path, full_name - - if project_name == "": - project_name = raw_input("Beat Name [Examplebeat]: ") or "examplebeat" - - if github_name == "": - github_name = raw_input("Your Github Name [your-github-name]: ") or "your-github-name" - beat = project_name.lower() - - if beat_path == "": - beat_path = raw_input("Beat Path [github.com/" + github_name + "/" + beat + - "]: ") or "github.com/" + github_name + "/" + beat - - if full_name == "": - full_name = raw_input("Firstname Lastname: ") or "Firstname Lastname" - - -def process_file(beat_type): - - # Load path information - template_path = os.path.dirname(os.path.realpath(__file__)) + '/../generator' - go_path = os.environ['GOPATH'] - - for root, dirs, files in os.walk(template_path + '/' + beat_type + '/{beat}'): - - for file in files: - - full_path = root + "/" + file - - # load file - content = "" - with open(full_path) as f: - content = f.read() - - # process content - content = replace_variables(content) - - # Write new path - new_path = replace_variables(full_path).replace(".go.tmpl", ".go") - - # remove generator info and beat name from path - file_path = new_path.replace(template_path + "/" + beat_type + "/" + beat, "") - - # New file path to write file content to - write_file = go_path + "/src/" + beat_path + "/" + file_path - - # Create parent directory if it does not exist yet - dir = os.path.dirname(write_file) - if not os.path.exists(dir): - os.makedirs(dir) - - # Write file to new location - with open(write_file, 'w') as f: - f.write(content) - - -def replace_variables(content): - """Replace all template variables with the actual values - """ - return content.replace("{project_name}", project_name) \ - .replace("{github_name}", github_name) \ - .replace("{beat}", beat) \ - .replace("{Beat}", beat.capitalize()) \ - .replace("{beat_path}", beat_path) \ - .replace("{full_name}", full_name) \ - .replace("{year}", str(datetime.datetime.now().year)) - - -def get_parser(): - """Creates parser to parse script params - """ - parser = argparse.ArgumentParser(description="Creates a beat") - parser.add_argument("--project_name", help="Project name") - parser.add_argument("--github_name", help="Github name") - parser.add_argument("--beat_path", help="Beat path") - parser.add_argument("--full_name", help="Full name") - parser.add_argument("--type", help="Beat type", default="beat", choices=["beat", "metricbeat"]) - - return parser - - if __name__ == "__main__": - - parser = get_parser() - args = parser.parse_args() - - generate_beat(args) + print "This script is deprecated. Please use `mage GenerateCustomBeat`" + exit(1)