From f3cb33570b7ba75fef9369c668695dbafe79f203 Mon Sep 17 00:00:00 2001 From: ruflin Date: Wed, 25 Jul 2018 11:35:21 +0200 Subject: [PATCH 1/2] Simplify exporting dashboards for Devs A new `mage` command is introduced to simplify exporting dashboards and contributing them to Beats. Currently a contributor is required to know about folder hierarchy under which dashboards are stored and are versioned. To simply this a simple `mage` command is provided: ``` MODULE=mysql ID=the-dashboard-id mage ExportDashboard ``` The Dev must provide the module he wants to export the dashboard for and the dashboard id. The script will take care of extracting it from Kibana, creating directories and files. The current script only supports to export dashboards from localhost:5061 as no params are provided to change the host at the moment. Also this change is only for Metricbeat for now. Filebeat should also be added. Additional changes: * Create directories * Pass module and dashboard id * Add exit code 1 in case of export failure * Improve error handling --- CHANGELOG-developer.asciidoc | 1 + auditbeat/magefile.go | 9 ++++ dev-tools/cmd/dashboards/export_dashboards.go | 13 +++-- dev-tools/mage/dashboard.go | 53 +++++++++++++++++++ docs/devguide/newdashboards.asciidoc | 34 +++--------- filebeat/magefile.go | 9 ++++ metricbeat/magefile.go | 9 ++++ 7 files changed, 97 insertions(+), 31 deletions(-) create mode 100644 dev-tools/mage/dashboard.go diff --git a/CHANGELOG-developer.asciidoc b/CHANGELOG-developer.asciidoc index 1d6db0d7180c..1f4c4878a923 100644 --- a/CHANGELOG-developer.asciidoc +++ b/CHANGELOG-developer.asciidoc @@ -65,3 +65,4 @@ The list below covers the major changes between 6.3.0 and master only. - Add `mage.AddPlatforms` to allow to specify dependent platforms when building a beat. {pull}8889[8889] - Add `cfgwarn.CheckRemoved6xSetting(s)` to display a warning for options removed in 7.0. {pull}8909[8909] - Add docker image building to `mage.Package`. {pull}8898[8898] +- Simplified exporting of dashboards. {pull}7730[7730] diff --git a/auditbeat/magefile.go b/auditbeat/magefile.go index 5100db035331..c9e5f92c2681 100644 --- a/auditbeat/magefile.go +++ b/auditbeat/magefile.go @@ -103,6 +103,15 @@ func Fields() error { return mage.GenerateFieldsYAML("module") } +// ExportDashboard exports a dashboard and writes it into the correct directory +// +// Required ENV variables: +// * MODULE: Name of the module +// * ID: Dashboard id +func ExportDashboard() error { + return mage.ExportDashboard() +} + // GoTestUnit executes the Go unit tests. // Use TEST_COVERAGE=true to enable code coverage profiling. // Use RACE_DETECTOR=true to enable the race detector. diff --git a/dev-tools/cmd/dashboards/export_dashboards.go b/dev-tools/cmd/dashboards/export_dashboards.go index a5338099f5ff..f15c7d289e38 100644 --- a/dev-tools/cmd/dashboards/export_dashboards.go +++ b/dev-tools/cmd/dashboards/export_dashboards.go @@ -89,6 +89,7 @@ func Export(client *http.Client, conn string, spaceID string, dashboard string, for _, obj := range objects { o := obj.(common.MapStr) + // All fields are optional, so errors are not catched decodeValue(o, "attributes.uiStateJSON") decodeValue(o, "attributes.visState") decodeValue(o, "attributes.optionsJSON") @@ -104,23 +105,27 @@ func Export(client *http.Client, conn string, spaceID string, dashboard string, return err } - err = ioutil.WriteFile(out, []byte(data.StringToPrint()), 0666) + err = ioutil.WriteFile(out, []byte(data.StringToPrint()), 0644) if !quiet { log.Printf("The dashboard %s was exported under the %s file\n", dashboard, out) } return err } -func decodeValue(data common.MapStr, key string) { +func decodeValue(data common.MapStr, key string) error { v, err := data.GetValue(key) if err != nil { - return + return err } s := v.(string) var d interface{} - json.Unmarshal([]byte(s), &d) + err = json.Unmarshal([]byte(s), &d) + if err != nil { + return fmt.Errorf("error decoding %s: %v", key, err) + } data.Put(key, d) + return nil } func ReadManifest(file string) ([]map[string]string, error) { diff --git a/dev-tools/mage/dashboard.go b/dev-tools/mage/dashboard.go new file mode 100644 index 000000000000..8690cf370e5e --- /dev/null +++ b/dev-tools/mage/dashboard.go @@ -0,0 +1,53 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mage + +import ( + "fmt" + "path/filepath" + + "github.com/magefile/mage/sh" +) + +// ExportDashboard exports a dashboard from Kibana and writes it into the given module. +func ExportDashboard() error { + module := EnvOr("MODULE", "") + if module == "" { + return fmt.Errorf("MODULE must be specified") + } + + id := EnvOr("ID", "") + if id == "" { + return fmt.Errorf("Dashboad ID must be specified") + } + + beatsDir, err := ElasticBeatsDir() + if err != nil { + return err + } + + // TODO: This is currently hardcoded for KB 6, we need to figure out what we do for KB 7 + file := filepath.Join(beatsDir, BeatName, "module", module, "_meta/kibana/6/dashboard", id+".json") + + dashboardCmd := sh.RunCmd("go", "run", + filepath.Join(beatsDir, "dev-tools/cmd/dashboards/export_dashboards.go"), + "-output", file, "-dashboard", id, + ) + + return dashboardCmd() +} diff --git a/docs/devguide/newdashboards.asciidoc b/docs/devguide/newdashboards.asciidoc index b532655fc209..6d260416f3f3 100644 --- a/docs/devguide/newdashboards.asciidoc +++ b/docs/devguide/newdashboards.asciidoc @@ -213,44 +213,24 @@ https://github.com/elastic/beats/tree/master/dev-tools/cmd/dashboards[dev-tools] for exporting Kibana 5.x dashboards. See the dev-tools https://github.com/elastic/beats/tree/master/dev-tools/README.md[readme] for more info. -///////////////////// -NOT YET IMPLEMENTED -NOTE: You can make use of the Makefile from the Beat GitHub repository to export all the Kibana dashboards for a Beat -from your Elasticsearch. If Elasticsearch is running on localhost, then you just need to run the following command from the Beat repository: - -[source,shell] ------------------------------ -make export-dashboards ------------------------------ - -If Elasticsearch is running on a different host, then you can use the `ES_URL` variable: - -[source,shell] ----------------------------- -ES_URL="http://192.168.3.206:9200" make export-dashboards ----------------------------- - -///////////////////// - ==== Exporting Kibana 6.0 dashboards and newer The `dev-tools/cmd/export_dashboards.go` script helps you export your customized Kibana 6.0 dashboards and newer. You might need to export a single dashboard or all the dashboards available for a module or Beat. - ===== Export a single Kibana dashboard -You can export a single dashboard by passing the dashboard ID in the `-dashboard` flag. - -NOTE: The dashboard ID is available in the dashboard URL. For example, in case the dashboard URL is -`app/kibana#/dashboard/7fea2930-478e-11e7-b1f0-cb29bac6bf8b?_g=()&_a=(description:'Overview%2...`, the dashboard ID is `7fea2930-478e-11e7-b1f0-cb29bac6bf8b`. +To export a single dashboard for a module you can use the following command inside a Beat with modules: [source,shell] --------------- -cd filebeat/module/redis/_meta/kibana/default/dashboard -go run ../../../../../../../dev-tools/cmd/dashboards/export_dashboards.go -id 7fea2930-478e-11e7-b1f0-cb29bac6bf8b -output Filebeat-redis.json +MODULE=redis ID=AV4REOpp5NkDleZmzKkE mage exportDashboard --------------- -This generates the `Filebeat-redis.json` file that contains the dashboard for the Redis module of Filebeat, including the dependencies (visualizations and searches). +This generates a `AV4REOpp5NkDleZmzKkE.json` file inside dashboard directory in the redis module. +It contains all dependencies like visualizations and searches. + +NOTE: The dashboard ID is available in the dashboard URL. For example, in case the dashboard URL is +`app/kibana#/dashboard/AV4REOpp5NkDleZmzKkE?_g=()&_a=(description:'Overview%2...`, the dashboard ID is `AV4REOpp5NkDleZmzKkE`. ===== Export all module/Beat dashboards diff --git a/filebeat/magefile.go b/filebeat/magefile.go index bc31b29c174b..df75f1430504 100644 --- a/filebeat/magefile.go +++ b/filebeat/magefile.go @@ -125,6 +125,15 @@ func GoTestIntegration(ctx context.Context) error { return mage.GoTest(ctx, mage.DefaultGoTestIntegrationArgs()) } +// ExportDashboard exports a dashboard and writes it into the correct directory +// +// Required ENV variables: +// * MODULE: Name of the module +// * ID: Dashboard id +func ExportDashboard() error { + return mage.ExportDashboard() +} + // ----------------------------------------------------------------------------- // Customizations specific to Filebeat. // - Include modules directory in packages (minus _meta and test files). diff --git a/metricbeat/magefile.go b/metricbeat/magefile.go index b9a8f79644b4..efbe279c3b84 100644 --- a/metricbeat/magefile.go +++ b/metricbeat/magefile.go @@ -117,6 +117,15 @@ func GoTestIntegration(ctx context.Context) error { return mage.GoTest(ctx, mage.DefaultGoTestIntegrationArgs()) } +// ExportDashboard exports a dashboard and writes it into the correct directory +// +// Required ENV variables: +// * MODULE: Name of the module +// * ID: Dashboard id +func ExportDashboard() error { + return mage.ExportDashboard() +} + // ----------------------------------------------------------------------------- // Customizations specific to Metricbeat. // - Include modules.d directory in packages. From e9e3e09888e1493fa174c784cb57a06384a66d4f Mon Sep 17 00:00:00 2001 From: beats-jenkins Date: Mon, 19 Nov 2018 07:57:34 +0100 Subject: [PATCH 2/2] introduce relative CWD --- dev-tools/mage/dashboard.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-tools/mage/dashboard.go b/dev-tools/mage/dashboard.go index 8690cf370e5e..4e4279738e2e 100644 --- a/dev-tools/mage/dashboard.go +++ b/dev-tools/mage/dashboard.go @@ -42,7 +42,7 @@ func ExportDashboard() error { } // TODO: This is currently hardcoded for KB 6, we need to figure out what we do for KB 7 - file := filepath.Join(beatsDir, BeatName, "module", module, "_meta/kibana/6/dashboard", id+".json") + file := CWD("module", module, "_meta/kibana/6/dashboard", id+".json") dashboardCmd := sh.RunCmd("go", "run", filepath.Join(beatsDir, "dev-tools/cmd/dashboards/export_dashboards.go"),