Update beats framework to 245b3e1

Makefile

@@ -16,6 +16,7 @@ LICENSE_FILE=licenses/APACHE-LICENSE-2.0.txt
 ELASTIC_LICENSE_FILE=licenses/ELASTIC-LICENSE.txt
 NOW=$(shell date -u '+%Y-%m-%dT%H:%M:%S')
 GOBUILD_FLAGS=-i -ldflags "-s -X $(BEAT_PATH)/vendor/github.com/elastic/beats/libbeat/version.buildTime=$(NOW) -X $(BEAT_PATH)/vendor/github.com/elastic/beats/libbeat/version.commit=$(COMMIT_ID)"
+FIELDS_FILE_PATH=processor
 
 # Path to the libbeat Makefile
 -include $(ES_BEATS)/libbeat/scripts/Makefile
@@ -24,7 +25,7 @@ GOBUILD_FLAGS=-i -ldflags "-s -X $(BEAT_PATH)/vendor/github.com/elastic/beats/li
 update-beats:
 	rm -rf vendor/github.com/elastic/beats
 	@govendor fetch github.com/elastic/beats/...@$(BEATS_VERSION)
-	@govendor fetch github.com/elastic/beats/libbeat/kibana@$(BEATS_VERSION)
+	@govendor fetch github.com/elastic/beats/libbeat/generator/fields@$(BEATS_VERSION) github.com/elastic/beats/libbeat/kibana@$(BEATS_VERSION)
 	@BEATS_VERSION=$(BEATS_VERSION) script/update_beats.sh
 	@$(MAKE) update
 	@echo --- Use this commit message: Update beats framework to `cat vendor/vendor.json | python -c 'import sys, json; print([p["revision"] for p in json.load(sys.stdin)["package"] if p["path"] == "github.com/elastic/beats/libbeat/beat"][0][:7])'`
@@ -39,18 +40,7 @@ before-build:
 
 # Collects all dependencies and then calls update
 .PHONY: collect
-collect: imports fields go-generate create-docs notice
-
-# Generates imports for all modules and metricsets
-.PHONY: imports
-imports:
-	@mkdir -p include
-	@mkdir -p processor
-
-.PHONY: fields
-fields:
-	@cat _meta/fields.common.yml > _meta/fields.generated.yml
-	@cat processor/*/_meta/fields.yml >> _meta/fields.generated.yml
+collect: fields go-generate create-docs notice
 
 .PHONY: go-generate
 go-generate:

NOTICE.txt

@@ -211,7 +211,7 @@ SOFTWARE.
 --------------------------------------------------------------------
 Dependency: github.com/elastic/beats
 Version: master
-Revision: c4593443ab5943ac5dd3465d88236b0cf417fab6
+Revision: 245b3e1b2ff12449007d6aa01dceee050d40ffa5
 License type (autodetected): Apache-2.0
 ./vendor/github.com/elastic/beats/LICENSE.txt:
 --------------------------------------------------------------------

_beats/dev-tools/cmd/dashboards/export_dashboards.go

@@ -13,6 +13,7 @@ import (
 	"strings"
 
 	"github.com/elastic/beats/libbeat/common"
+	"github.com/elastic/beats/libbeat/kibana"
 )
 
 var exportAPI = "/api/kibana/dashboards/export"
@@ -29,39 +30,6 @@ func makeURL(url, path string, params url.Values) string {
 	return strings.Join([]string{url, path, "?", params.Encode()}, "")
 }
 
-func ExtractIndexPattern(body []byte) ([]byte, error) {
-	var contents common.MapStr
-
-	err := json.Unmarshal(body, &contents)
-	if err != nil {
-		return nil, err
-	}
-
-	objects, ok := contents["objects"].([]interface{})
-	if !ok {
-		return nil, fmt.Errorf("Key objects not found or wrong type")
-	}
-
-	var result []interface{}
-	for _, obj := range objects {
-		_type, ok := obj.(map[string]interface{})["type"].(string)
-		if !ok {
-			return nil, fmt.Errorf("type key not found or not string")
-		}
-		if _type != "index-pattern" || indexPattern {
-			result = append(result, obj)
-		}
-	}
-	contents["objects"] = result
-
-	newBody, err := json.MarshalIndent(contents, "", "  ")
-	if err != nil {
-		return nil, fmt.Errorf("Error mashaling: %v", err)
-	}
-
-	return newBody, nil
-}
-
 func Export(client *http.Client, conn string, dashboard string, out string) error {
 	params := url.Values{}
 
@@ -90,19 +58,42 @@ func Export(client *http.Client, conn string, dashboard string, out string) erro
 		return fmt.Errorf("HTTP GET %s fails with %s, %s", fullURL, resp.Status, body)
 	}
 
-	body, err = ExtractIndexPattern(body)
+	data, err := kibana.RemoveIndexPattern(body)
 	if err != nil {
 		return fmt.Errorf("fail to extract the index pattern: %v", err)
 	}
 
-	err = ioutil.WriteFile(out, body, 0666)
+	objects := data["objects"].([]interface{})
+	for _, obj := range objects {
+		o := obj.(common.MapStr)
+
+		decodeValue(o, "attributes.uiStateJSON")
+		decodeValue(o, "attributes.visState")
+		decodeValue(o, "attributes.optionsJSON")
+		decodeValue(o, "attributes.panelsJSON")
+		decodeValue(o, "attributes.kibanaSavedObjectMeta.searchSourceJSON")
+	}
 
+	data["objects"] = objects
+	err = ioutil.WriteFile(out, []byte(data.StringToPrint()), 0666)
 	if !quiet {
 		fmt.Printf("The dashboard %s was exported under the %s file\n", dashboard, out)
 	}
 	return err
 }
 
+func decodeValue(data common.MapStr, key string) {
+	v, err := data.GetValue(key)
+	if err != nil {
+		return
+	}
+	s := v.(string)
+	var d common.MapStr
+	json.Unmarshal([]byte(s), &d)
+
+	data.Put(key, d)
+}
+
 func ReadManifest(file string) ([]map[string]string, error) {
 	cfg, err := common.LoadFile(file)
 	if err != nil {

_beats/libbeat/Makefile

@@ -1,15 +1,10 @@
 BEAT_NAME=libbeat
 TEST_ENVIRONMENT?=true
 SYSTEM_TESTS=true
+FIELDS_FILE_PATH=processors
 
 include scripts/Makefile
 
-# Collects all fields from processors
-.PHONY: fields
-fields:
-	@cp _meta/fields.common.yml _meta/fields.generated.yml
-	@cat processors/*/_meta/fields.yml >> _meta/fields.generated.yml
-
 # Collects all dependencies and then calls update
 .PHONY: collect
-collect: fields
+collect: libbeat_fields

_beats/libbeat/scripts/Makefile

@@ -304,9 +304,19 @@ coverage-report:
 	test ! -s ${COVERAGE_DIR}/unit.cov   || go tool cover -html=${COVERAGE_DIR}/unit.cov   -o ${COVERAGE_DIR}/unit.html
 
 
+.PHONY: fields
+fields:
+	echo $(PWD)
+	@go run ${ES_BEATS}/libbeat/scripts/cmd/global_fields/main.go --es_beats_path $(ES_BEATS) --beat_path $(PWD) $(FIELDS_FILE_PATH)
+
+
+.PHONY: libbeat_fields
+libbeat_fields:
+	@$(MAKE) -C ${ES_BEATS}/libbeat fields
+
 .PHONY: update
 update: ## @build Update expects the most recent version of libbeat in the GOPATH
-update: python-env collect
+update: python-env libbeat_fields collect
 	@echo "Updating generated files for ${BEAT_NAME}"
 
 	@# Update config files.
@@ -324,15 +334,7 @@ ifeq ($(BEAT_REF_YAML),true)
 	@chmod 0640 ${BEAT_NAME}.reference.yml
 endif
 
-	@# Update libbeat fields.generated.yml
-	@$(MAKE) -C ${ES_BEATS}/libbeat fields
-
-	@# Update fields.yml
-	@test -s  _meta/fields.generated.yml || cp _meta/fields.yml _meta/fields.generated.yml
-ifeq ($(BEAT_NAME), libbeat)
-	@cp ${ES_BEATS}/libbeat/_meta/fields.generated.yml fields.yml
-else
-	@cat ${ES_BEATS}/libbeat/_meta/fields.generated.yml _meta/fields.generated.yml > fields.yml
+ifneq ($(BEAT_NAME), libbeat)
 	mkdir -p include
 	go run  ${ES_BEATS}/dev-tools/cmd/asset/asset.go -pkg include fields.yml $(BEAT_NAME) > include/fields.go
 endif
@@ -355,10 +357,6 @@ docs:  ## @build Builds the documents for the beat
 docs-preview:  ## @build Preview the documents for the beat in the browser
 	PREVIEW=1 $(MAKE) docs
 
-.PHONY: index-template
-index-template: ## @build Generate index templates for the given $VERSION. This is for manual testing.
-	go run ${ES_BEATS}/dev-tools/cmd/index_template/index_template.go -index ${BEAT_INDEX_PREFIX} -output ${BEAT_GOPATH}/src/${BEAT_PATH}/${BEAT_NAME}.template-es${VERSION}.json -file ${BEAT_GOPATH}/src/${BEAT_PATH}/fields.yml -es.version=${VERSION}
-
 ### KIBANA FILES HANDLING ###
 ES_URL?=http://localhost:9200
 KIBANA_URL?=http://localhost:5601

_beats/libbeat/scripts/cmd/global_fields/main.go

@@ -0,0 +1,56 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/elastic/beats/libbeat/generator/fields"
+)
+
+func main() {
+	esBeatsPath := flag.String("es_beats_path", "..", "Path to elastic/beats")
+	beatPath := flag.String("beat_path", ".", "Path to your Beat")
+	flag.Parse()
+
+	beatFieldsPath := flag.Args()
+	name := filepath.Base(*beatPath)
+
+	err := os.MkdirAll(filepath.Join(*beatPath, "_meta"), 0744)
+	if err != nil {
+		fmt.Printf("Cannot creata _meta dir for %s: %v\n", name, err)
+		os.Exit(1)
+	}
+
+	if len(beatFieldsPath) == 0 {
+		fmt.Println("No field files to collect")
+		err = fields.AppendFromLibbeat(*esBeatsPath, *beatPath)
+		if err != nil {
+			fmt.Printf("Cannot generate global fields.yml for %s: %v\n", name, err)
+			os.Exit(2)
+		}
+		return
+	}
+
+	if *beatPath == "" {
+		fmt.Println("beat_path cannot be empty")
+		os.Exit(1)
+	}
+
+	pathToModules := filepath.Join(*beatPath, beatFieldsPath[0])
+	fieldFiles, err := fields.CollectModuleFiles(pathToModules)
+	if err != nil {
+		fmt.Printf("Cannot collect fields.yml files: %v\n", err)
+		os.Exit(2)
+
+	}
+
+	err = fields.Generate(*esBeatsPath, *beatPath, fieldFiles)
+	if err != nil {
+		fmt.Printf("Cannot generate global fields.yml file for %s: %v\n", name, err)
+		os.Exit(3)
+	}
+
+	fmt.Printf("Generated fields.yml for %s\n", name)
+}