Skip to content

Commit

Permalink
Generate fields, dashboards, config from mage
Browse files Browse the repository at this point in the history
Generate the fields, dashboards, and config files from mage to make it reusable across Beats (include x-pack). This allows x-pack/filebeat to generate its own unique fields, dashboards, and config.
  • Loading branch information
andrewkroh committed Oct 14, 2018
1 parent 3ab125f commit cfe009b
Show file tree
Hide file tree
Showing 30 changed files with 3,655 additions and 235 deletions.
15 changes: 13 additions & 2 deletions auditbeat/magefile.go
Original file line number Diff line number Diff line change
Expand Up @@ -98,11 +98,22 @@ func Update() error {
return sh.Run("make", "update")
}

// Fields generates a fields.yml for the Beat.
func Fields() error {
// Fields generates a fields.yml and include/fields.go for the Beat.
func Fields() {
mg.SerialDeps(fieldsYML, mage.GenerateAllInOneFieldsGo)
}

// fieldsYML generates a fields.yml.
func fieldsYML() error {
return mage.GenerateFieldsYAML("module")
}

// Dashboards collects all the dashboards and generates index patterns.
func Dashboards() error {
mg.Deps(Fields)
return mage.KibanaDashboards("module")
}

// GoTestUnit executes the Go unit tests.
// Use TEST_COVERAGE=true to enable code coverage profiling.
// Use RACE_DETECTOR=true to enable the race detector.
Expand Down
68 changes: 45 additions & 23 deletions dev-tools/cmd/kibana_index_pattern/kibana_index_pattern.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,51 +19,73 @@ package main

import (
"flag"
"fmt"
"os"
"log"
"path/filepath"

"github.com/elastic/beats/libbeat/common"
"github.com/elastic/beats/libbeat/kibana"
"github.com/elastic/beats/libbeat/version"
)

var usageText = `
Usage: kibana_index_pattern [flags]
kibana_index_pattern generates Kibana index patterns from the Beat's
fields.yml file. It will create a index pattern file that is usable with both
Kibana 5.x and 6.x.
Options:
`[1:]

var (
beatName string
beatVersion string
indexPattern string
fieldsYAMLFile string
outputDir string
)

func init() {
flag.StringVar(&beatName, "beat", "", "Name of the beat. (Required)")
flag.StringVar(&beatVersion, "version", version.GetDefaultVersion(), "Beat version. (Required)")
flag.StringVar(&indexPattern, "index", "", "Kibana index pattern. (Required)")
flag.StringVar(&fieldsYAMLFile, "fields", "fields.yml", "fields.yml file containing all fields used by the Beat.")
flag.StringVar(&outputDir, "out", "build/kibana", "Output dir.")
}

func main() {
index := flag.String("index", "", "The name of the index pattern. (required)")
beatName := flag.String("beat-name", "", "The name of the beat. (required)")
beatDir := flag.String("beat-dir", "", "The local beat directory. (required)")
beatVersion := flag.String("version", version.GetDefaultVersion(), "The beat version.")
log.SetFlags(0)
flag.Parse()

if *index == "" {
fmt.Fprint(os.Stderr, "The name of the index pattern must be set.")
os.Exit(1)
if beatName == "" {
log.Fatal("Name of the Beat must be set (-beat).")
}

if *beatName == "" {
fmt.Fprint(os.Stderr, "The name of the beat must be set.")
os.Exit(1)
if beatVersion == "" {
log.Fatal("Beat version must be set (-version).")
}

if *beatDir == "" {
fmt.Fprint(os.Stderr, "The beat directory must be set.")
os.Exit(1)
if indexPattern == "" {
log.Fatal("Index pattern must be set (-index).")
}

version5, _ := common.NewVersion("5.0.0")
version6, _ := common.NewVersion("6.0.0")
versions := []*common.Version{version5, version6}
versions := []common.Version{*version5, *version6}
for _, version := range versions {
indexPattern, err := kibana.NewGenerator(indexPattern, beatName, fieldsYAMLFile, outputDir, beatVersion, version)
if err != nil {
log.Fatal(err)
}

indexPatternGenerator, err := kibana.NewGenerator(*index, *beatName, *beatDir, *beatVersion, *version)
file, err := indexPattern.Generate()
if err != nil {
fmt.Fprintf(os.Stderr, err.Error())
os.Exit(1)
log.Fatal(err)
}
pattern, err := indexPatternGenerator.Generate()

// Log output file location.
absFile, err := filepath.Abs(file)
if err != nil {
fmt.Fprintf(os.Stderr, err.Error())
os.Exit(1)
absFile = file
}
fmt.Fprintf(os.Stdout, "-- The index pattern was created under %v\n", pattern)
log.Printf(">> The index pattern was created under %v", absFile)
}
}
118 changes: 118 additions & 0 deletions dev-tools/cmd/module_fields/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
// Licensed to Elasticsearch B.V. under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. Elasticsearch B.V. licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

package main

import (
"bytes"
"flag"
"fmt"
"go/format"
"io/ioutil"
"log"
"os"
"path"

"github.com/elastic/beats/libbeat/asset"
"github.com/elastic/beats/libbeat/generator/fields"
"github.com/elastic/beats/licenses"
)

var usageText = `
Usage: module_fields [flags] [module-dir]
module_fields generates a fields.go file containing a copy of the module's
field.yml data in a format that can be embedded in Beat's binary. module-dir
should be the directory containing modules (e.g. filebeat/module).
Options:
`[1:]

var (
beatName string
license string
)

func init() {
flag.StringVar(&beatName, "beat", "", "Name of the beat. (Required)")
flag.StringVar(&license, "license", "ASL2", "License header for generated file.")
flag.Usage = usageFlag
}

func main() {
log.SetFlags(0)
flag.Parse()

if beatName == "" {
log.Fatal("You must use -beat to specify the beat name.")
}

license, err := licenses.Find(license)
if err != nil {
log.Fatalf("Invalid license specifier: %v", err)
}

args := flag.Args()
if len(args) != 1 {
log.Fatal("module-dir must be passed as an argument.")
}
dir := args[0]

modules, err := fields.GetModules(dir)
if err != nil {
log.Fatalf("Error fetching modules: %v", err)
}

for _, module := range modules {
files, err := fields.CollectFiles(module, dir)
if err != nil {
log.Fatalf("Error fetching files for module %v: %v", module, err)
}

data, err := fields.GenerateFieldsYml(files)
if err != nil {
log.Fatalf("Error fetching files for module %v: %v", module, err)
}

encData, err := asset.EncodeData(string(data))
if err != nil {
log.Fatalf("Error encoding the data: %v", err)
}

var buf bytes.Buffer
asset.Template.Execute(&buf, asset.Data{
License: license,
Beat: beatName,
Name: module,
Data: encData,
Package: module,
})

bs, err := format.Source(buf.Bytes())
if err != nil {
log.Fatalf("Error creating golang file from template: %v", err)
}

err = ioutil.WriteFile(path.Join(dir, module, "fields.go"), bs, 0644)
if err != nil {
log.Fatalf("Error writing fields.go: %v", err)
}
}
}

func usageFlag() {
fmt.Fprintf(os.Stderr, usageText)
flag.PrintDefaults()
}
4 changes: 0 additions & 4 deletions dev-tools/mage/clean.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,6 @@ var DefaultCleanPaths = []string{
"_meta/kibana.generated",
"_meta/kibana/5/index-pattern/{{.BeatName}}.json",
"_meta/kibana/6/index-pattern/{{.BeatName}}.json",

"../x-pack/{{.BeatName}}/build",
"../x-pack/{{.BeatName}}/{{.BeatName}}",
"../x-pack/{{.BeatName}}/{{.BeatName}}.exe",
}

// Clean clean generated build artifacts.
Expand Down
Loading

0 comments on commit cfe009b

Please sign in to comment.