-
Notifications
You must be signed in to change notification settings - Fork 4.9k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Port fields.yml collector to Golang (#6911)
Existing Python and shell code is ported to Golang. The previous Beat specific field collection is generalized and moved to the `Makefile` of `libbeat`. A new variable is added to makefiles: `FIELDS_FILE_PATH`. This specifies where `fields.yml` are. Packetbeat's `fields_base.yml` is renamed to `fields.common.yml` to be the same as in other Beats
- Loading branch information
Showing
21 changed files
with
1,109 additions
and
58 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1 +1,4 @@ | ||
| override FIELDS_FILE_PATH= | ||
| export FIELDS_FILE_PATH | ||
|
|
||
| include ../common/Makefile |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -46,4 +46,4 @@ before-build: | |
|
|
||
| # Collects all dependencies and then calls update | ||
| .PHONY: collect | ||
| collect: | ||
| collect: fields | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,9 +1,16 @@ | ||
| BEAT_TYPE=metricbeat | ||
| PREPARE_COMMAND=MODULE=elastic METRICSET=test make create-metricset ; | ||
| PREPARE_COMMAND=MODULE=elastic METRICSET=test make create-metricset ; FIELDS_FILE_PATH=module | ||
| override FIELDS_FILE_PATH= | ||
| export FIELDS_FILE_PATH | ||
|
|
||
|
|
||
| include ../common/Makefile | ||
|
|
||
| prepare-test:: python-env | ||
|
|
||
| mkdir -p ${BEAT_PATH}/scripts | ||
| rsync -a --exclude=build ${PWD}/../../metricbeat/scripts/generate_imports_helper.py ${BEAT_PATH}/scripts | ||
|
|
||
| # Collects all dependencies and then calls update | ||
| .PHONY: collect | ||
| collect: fields |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,15 +1,10 @@ | ||
| BEAT_NAME=libbeat | ||
| TEST_ENVIRONMENT?=true | ||
| SYSTEM_TESTS=true | ||
| FIELDS_FILE_PATH=processors | ||
|
|
||
| include scripts/Makefile | ||
|
|
||
| # Collects all fields from processors | ||
| .PHONY: fields | ||
| fields: | ||
| @cp _meta/fields.common.yml _meta/fields.generated.yml | ||
| @cat processors/*/_meta/fields.yml >> _meta/fields.generated.yml | ||
|
|
||
| # Collects all dependencies and then calls update | ||
| .PHONY: collect | ||
| collect: fields | ||
| collect: libbeat_fields |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,149 @@ | ||
| package fields | ||
|
|
||
| import ( | ||
| "bufio" | ||
| "fmt" | ||
| "io/ioutil" | ||
| "os" | ||
| "path" | ||
| "path/filepath" | ||
| "strings" | ||
| ) | ||
|
|
||
| var ( | ||
| generatedFieldsYml = filepath.Join("_meta", "fields.generated.yml") | ||
| ) | ||
|
|
||
| // YmlFile holds the info on files and how to write them into the global fields.yml | ||
| type YmlFile struct { | ||
| Path string | ||
| Indent int | ||
| } | ||
|
|
||
| func collectBeatFiles(beatPath string, fieldFiles []*YmlFile) ([]*YmlFile, error) { | ||
| commonFields := filepath.Join(beatPath, "_meta", "fields.common.yml") | ||
| _, err := os.Stat(commonFields) | ||
| if os.IsNotExist(err) { | ||
| return fieldFiles, nil | ||
| } else if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| files := []*YmlFile{ | ||
| &YmlFile{ | ||
| Path: commonFields, | ||
| Indent: 0, | ||
| }, | ||
| } | ||
|
|
||
| return append(files, fieldFiles...), nil | ||
| } | ||
|
|
||
| func writeGeneratedFieldsYml(beatsPath string, fieldFiles []*YmlFile) error { | ||
| outPath := path.Join(beatsPath, generatedFieldsYml) | ||
| f, err := os.Create(outPath) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| defer f.Close() | ||
|
|
||
| fw := bufio.NewWriter(f) | ||
| for _, p := range fieldFiles { | ||
| ff, err := os.Open(p.Path) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| defer ff.Close() | ||
|
|
||
| fs := bufio.NewScanner(ff) | ||
| for fs.Scan() { | ||
| err = writeIndentedLine(fw, fs.Text()+"\n", p.Indent) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| } | ||
| if err := fs.Err(); err != nil { | ||
| return err | ||
| } | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| func writeIndentedLine(fw *bufio.Writer, l string, indent int) error { | ||
| ll := strings.Repeat(" ", indent) + l | ||
| fmt.Fprint(fw, ll) | ||
| return fw.Flush() | ||
| } | ||
|
|
||
| // Generate collects fields.yml files and concatenates them into one global file. | ||
| func Generate(esBeatsPath, beatPath string, files []*YmlFile) error { | ||
| files, err := collectBeatFiles(beatPath, files) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| err = writeGeneratedFieldsYml(beatPath, files) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| return AppendFromLibbeat(esBeatsPath, beatPath) | ||
| } | ||
|
|
||
| // AppendFromLibbeat appends fields.yml of libbeat to the fields.yml | ||
| func AppendFromLibbeat(esBeatsPath, beatPath string) error { | ||
| fieldsMetaPath := path.Join(beatPath, "_meta", "fields.yml") | ||
| generatedPath := path.Join(beatPath, generatedFieldsYml) | ||
|
|
||
| err := createIfNotExists(fieldsMetaPath, generatedPath) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| if isLibbeat(beatPath) { | ||
| out := filepath.Join(esBeatsPath, "libbeat", "fields.yml") | ||
| return copyFileWithFlag(generatedPath, out, os.O_RDWR|os.O_CREATE|os.O_TRUNC) | ||
| } | ||
|
|
||
| libbeatPath := filepath.Join(esBeatsPath, "libbeat", generatedFieldsYml) | ||
| out := filepath.Join(beatPath, "fields.yml") | ||
| err = copyFileWithFlag(libbeatPath, out, os.O_RDWR|os.O_CREATE|os.O_TRUNC) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| return copyFileWithFlag(generatedPath, out, os.O_WRONLY|os.O_APPEND) | ||
| } | ||
|
|
||
| func isLibbeat(beatPath string) bool { | ||
| return filepath.Base(beatPath) == "libbeat" | ||
| } | ||
|
|
||
| func createIfNotExists(inPath, outPath string) error { | ||
| _, err := os.Stat(outPath) | ||
| if os.IsNotExist(err) { | ||
| err := copyFileWithFlag(inPath, outPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC) | ||
| if err != nil { | ||
| fmt.Println("Cannot find _meta/fields.yml") | ||
| } | ||
| return nil | ||
| } | ||
| return err | ||
| } | ||
|
|
||
| func copyFileWithFlag(in, out string, flag int) error { | ||
| input, err := ioutil.ReadFile(in) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| output, err := os.OpenFile(out, flag, 0644) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| defer output.Close() | ||
|
|
||
| _, err = output.Write(input) | ||
| return err | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,74 @@ | ||
| package fields | ||
|
|
||
| import ( | ||
| "io/ioutil" | ||
| "os" | ||
| "path/filepath" | ||
| ) | ||
|
|
||
| var indentByModule = map[string]int{ | ||
| "processors": 0, | ||
| "module": 8, | ||
| "active": 8, | ||
| "protos": 8, | ||
| } | ||
|
|
||
| // CollectModuleFiles looks for fields.yml files under the | ||
| // specified root directory | ||
| func CollectModuleFiles(root string) ([]*YmlFile, error) { | ||
| modules, err := ioutil.ReadDir(root) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| var files []*YmlFile | ||
| for _, m := range modules { | ||
| f, err := collectFiles(m, root) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| files = append(files, f...) | ||
| } | ||
|
|
||
| return files, nil | ||
| } | ||
|
|
||
| func collectFiles(module os.FileInfo, modulesPath string) ([]*YmlFile, error) { | ||
| if !module.IsDir() { | ||
| return nil, nil | ||
| } | ||
|
|
||
| var files []*YmlFile | ||
| fieldsYmlPath := filepath.Join(modulesPath, module.Name(), "_meta", "fields.yml") | ||
| if _, err := os.Stat(fieldsYmlPath); !os.IsNotExist(err) { | ||
| files = append(files, &YmlFile{ | ||
| Path: fieldsYmlPath, | ||
| Indent: 0, | ||
| }) | ||
| } else if !os.IsNotExist(err) && err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| modulesRoot := filepath.Base(modulesPath) | ||
| sets, err := ioutil.ReadDir(filepath.Join(modulesPath, module.Name())) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| for _, s := range sets { | ||
| if !s.IsDir() { | ||
| continue | ||
| } | ||
|
|
||
| fieldsYmlPath = filepath.Join(modulesPath, module.Name(), s.Name(), "_meta", "fields.yml") | ||
| if _, err = os.Stat(fieldsYmlPath); !os.IsNotExist(err) { | ||
| files = append(files, &YmlFile{ | ||
| Path: fieldsYmlPath, | ||
| Indent: indentByModule[modulesRoot], | ||
| }) | ||
| } else if !os.IsNotExist(err) && err != nil { | ||
| return nil, err | ||
| } | ||
| } | ||
| return files, nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.