Cherry-pick #8045 to 6.x: Support Kibana Spaces (#8545)

Cherry-pick of PR #8045 to 6.x branch. Original message: 

Resolves #7942.

Kibana is implementing a new feature called Spaces, in which Kibana saved objects (such as dashboards) and advanced settings can be restricted to a user-defined namespace. Spaces are identified by a unique ID, e.g. `my-space`. There is also a notion of a Default space which corresponds to how Kibana worked up until the Spaces feature was introduced.

Beats have the ability to import dashboards into Kibana as well as export dashboards out of Kibana. With Spaces, dashboards may belong to a specific space. So Beats must learn to accept an optional Space ID and operate against it when importing or exporting dashboards. This PR teaches Beats to do just that.

Concretely, if a user wishes to import or export dashboards from a specific space, say with ID = `my-space`, they must either:

* Edit `<beat>.yml` and set `setup.kibana.space.id: my-space`, _or_
* Run `<beat> setup` or `<beat> export dashboard` along with the `-E setup.kibana.space.id=my-space` option.

Similarly, if a Beat _developer_ wishes to export dashboards from a specific space, say with ID = `my-space`, they must run:

```sh
go run dev-tools/cmd/dashboards/export_dashboards.go -space-id my-space [other options]
```

CHANGELOG-developer.asciidoc

@@ -50,3 +50,4 @@ The list below covers the major changes between 6.3.0 and master only.
   `cmd.GenRootCmd`, `cmd.GenRootCmdWithRunFlags`, and `cmd.GenRootCmdWithIndexPrefixWithRunFlags`. {pull}7850[7850]
 - Set current year in generator templates. {pull}8396[8396]
 - You can now override default settings of libbeat by using instance.Settings. {pull}8449[8449]
+- Add `-space-id` option to `export_dashboards.go` script to support Kibana Spaces {pull}7942[7942]

CHANGELOG.asciidoc

@@ -95,6 +95,7 @@ https://github.com/elastic/beats/compare/v6.4.0...6.x[Check the HEAD diff]
 - Implement CheckConfig in RunnerFactory to make autodiscover check configs {pull}7961[7961]
 - Add DNS processor with support for performing reverse lookups on IP addresses. {issue}7770[7770]
 - Support for Kafka 2.0.0 in kafka output {pull}8399[8399]
+- Add setting `setup.kibana.space.id` to support Kibana Spaces {pull}7942[7942]
 
 *Auditbeat*
 

auditbeat/auditbeat.yml

@@ -95,6 +95,11 @@ setup.kibana:
   # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601
   #host: "localhost:5601"
 
+  # Kibana Space ID
+  # ID of the Kibana Space into which the dashboards should be loaded. By default,
+  # the Default Space will be used.
+  #space.id:
+
 #============================= Elastic Cloud ==================================
 
 # These settings simplify using auditbeat with the Elastic Cloud (https://cloud.elastic.co/).

dev-tools/cmd/dashboards/export_dashboards.go

@@ -27,6 +27,7 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"path"
 	"path/filepath"
 	"strings"
 
@@ -48,11 +49,14 @@ func makeURL(url, path string, params url.Values) string {
 	return strings.Join([]string{url, path, "?", params.Encode()}, "")
 }
 
-func Export(client *http.Client, conn string, dashboard string, out string) error {
+func Export(client *http.Client, conn string, spaceID string, dashboard string, out string) error {
 	params := url.Values{}
 
 	params.Add("dashboard", dashboard)
 
+	if spaceID != "" {
+		exportAPI = path.Join("/s", spaceID, exportAPI)
+	}
 	fullURL := makeURL(conn, exportAPI, params)
 	if !quiet {
 		log.Printf("Calling HTTP GET %v\n", fullURL)
@@ -138,6 +142,7 @@ var quiet = false
 
 func main() {
 	kibanaURL := flag.String("kibana", "http://localhost:5601", "Kibana URL")
+	spaceID := flag.String("space-id", "", "Space ID")
 	dashboard := flag.String("dashboard", "", "Dashboard ID")
 	fileOutput := flag.String("output", "output.json", "Output file")
 	ymlFile := flag.String("yml", "", "Path to the module.yml file containing the dashboards")
@@ -171,7 +176,7 @@ func main() {
 			if err != nil {
 				log.Fatalf("fail to create directory %s: %v", directory, err)
 			}
-			err = Export(client, *kibanaURL, dashboard["id"], filepath.Join(directory, dashboard["file"]))
+			err = Export(client, *kibanaURL, *spaceID, dashboard["id"], filepath.Join(directory, dashboard["file"]))
 			if err != nil {
 				log.Fatalf("fail to export the dashboards: %s", err)
 			}
@@ -180,7 +185,7 @@ func main() {
 	}
 
 	if len(*dashboard) > 0 {
-		err := Export(client, *kibanaURL, *dashboard, *fileOutput)
+		err := Export(client, *kibanaURL, *spaceID, *dashboard, *fileOutput)
 		if err != nil {
 			log.Fatalf("fail to export the dashboards: %s", err)
 		}

docs/devguide/newdashboards.asciidoc

@@ -283,6 +283,15 @@ go run dev-tools/cmd/dashboards/export_dashboards.go -yml filebeat/module/system
 -------------------
 
 
+===== Export dashboards from a Kibana Space
+
+If you are using the Kibana Spaces feature and want to export dashboards from a specific Space, pass the Space ID to the `export_dashboards.go` script:
+
+[source,shell]
+-------------------
+go run dev-tools/cmd/dashboards/export_dashboards.go -space-id my-space [other-options]
+-------------------
+
 
 ==== Exporting Kibana 5.x dashboards
 

filebeat/filebeat.yml

@@ -122,6 +122,11 @@ setup.kibana:
   # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601
   #host: "localhost:5601"
 
+  # Kibana Space ID
+  # ID of the Kibana Space into which the dashboards should be loaded. By default,
+  # the Default Space will be used.
+  #space.id:
+
 #============================= Elastic Cloud ==================================
 
 # These settings simplify using filebeat with the Elastic Cloud (https://cloud.elastic.co/).

heartbeat/heartbeat.yml

@@ -69,6 +69,11 @@ setup.kibana:
   # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601
   #host: "localhost:5601"
 
+  # Kibana Space ID
+  # ID of the Kibana Space into which the dashboards should be loaded. By default,
+  # the Default Space will be used.
+  #space.id:
+
 #============================= Elastic Cloud ==================================
 
 # These settings simplify using heartbeat with the Elastic Cloud (https://cloud.elastic.co/).

libbeat/_meta/config.yml

@@ -39,6 +39,11 @@ setup.kibana:
   # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601
   #host: "localhost:5601"
 
+  # Kibana Space ID
+  # ID of the Kibana Space into which the dashboards should be loaded. By default,
+  # the Default Space will be used.
+  #space.id:
+
 #============================= Elastic Cloud ==================================
 
 # These settings simplify using beatname with the Elastic Cloud (https://cloud.elastic.co/).

libbeat/kibana/client.go

@@ -25,6 +25,7 @@ import (
 	"io/ioutil"
 	"net/http"
 	"net/url"
+	"path"
 	"strings"
 
 	"github.com/pkg/errors"
@@ -88,7 +89,11 @@ func NewKibanaClient(cfg *common.Config) (*Client, error) {
 
 // NewClientWithConfig creates and returns a kibana client using the given config
 func NewClientWithConfig(config *ClientConfig) (*Client, error) {
-	kibanaURL, err := common.MakeURL(config.Protocol, config.Path, config.Host, 5601)
+	p := config.Path
+	if config.SpaceID != "" {
+		p = path.Join(p, "s", config.SpaceID)
+	}
+	kibanaURL, err := common.MakeURL(config.Protocol, p, config.Host, 5601)
 	if err != nil {
 		return nil, fmt.Errorf("invalid Kibana host: %v", err)
 	}

libbeat/kibana/client_config.go

@@ -28,6 +28,7 @@ type ClientConfig struct {
 	Protocol string            `config:"protocol"`
 	Host     string            `config:"host"`
 	Path     string            `config:"path"`
+	SpaceID  string            `config:"space.id"`
 	Username string            `config:"username"`
 	Password string            `config:"password"`
 	TLS      *tlscommon.Config `config:"ssl"`
@@ -39,6 +40,7 @@ var (
 		Protocol: "http",
 		Host:     "localhost:5601",
 		Path:     "",
+		SpaceID:  "",
 		Username: "",
 		Password: "",
 		Timeout:  90 * time.Second,

libbeat/tests/system/requirements.txt

@@ -29,3 +29,4 @@ urllib3==1.22
 websocket-client==0.47.0
 parameterized==0.6.1
 jsondiff==1.1.2
+semver==2.8.1

libbeat/tests/system/test_dashboard.py

@@ -4,7 +4,8 @@
 import subprocess
 from nose.plugins.attrib import attr
 import unittest
-
+import requests
+import semver
 
 INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False)
 
@@ -36,6 +37,40 @@ def test_load_dashboard(self):
 
         assert self.log_contains("Kibana dashboards successfully loaded") is True
 
+    @unittest.skipUnless(INTEGRATION_TESTS, "integration test")
+    @attr('integration')
+    def test_load_dashboard_into_space(self, create_space=True):
+        """
+        Test loading dashboards into Kibana space
+        """
+        version = self.get_version()
+        if semver.compare(version, "6.5.0") == -1:
+            # Skip for Kibana versions < 6.5.0 as Kibana Spaces not available
+            raise SkipTest
+
+        self.render_config_template()
+        if create_space:
+            self.create_kibana_space()
+
+        beat = self.start_beat(
+            logging_args=["-e", "-d", "*"],
+            extra_args=["setup",
+                        "--dashboards",
+                        "-E", "setup.dashboards.file=" +
+                        os.path.join(self.beat_path, "tests", "files", "testbeat-dashboards.zip"),
+                        "-E", "setup.dashboards.beat=testbeat",
+                        "-E", "setup.kibana.protocol=http",
+                        "-E", "setup.kibana.host=" + self.get_kibana_host(),
+                        "-E", "setup.kibana.port=" + self.get_kibana_port(),
+                        "-E", "setup.kibana.space.id=foo-bar",
+                        "-E", "output.elasticsearch.hosts=['" + self.get_host() + "']",
+                        "-E", "output.file.enabled=false"]
+        )
+
+        beat.check_wait(exit_code=0)
+
+        assert self.log_contains("Kibana dashboards successfully loaded") is True
+
     @unittest.skipUnless(INTEGRATION_TESTS, "integration test")
     @attr('integration')
     def test_load_only_index_patterns(self):
@@ -88,6 +123,36 @@ def test_export_dashboard(self):
 
         os.remove("output.json")
 
+    @unittest.skipUnless(INTEGRATION_TESTS, "integration test")
+    @attr('integration')
+    def test_export_dashboard_from_space(self):
+        """
+        Test export dashboards from Kibana space and remove unsupported characters
+        """
+        version = self.get_version()
+        if semver.compare(version, "6.5.0") == -1:
+            # Skip for Kibana versions < 6.5.0 as Kibana Spaces not available
+            raise SkipTest
+
+        self.test_load_dashboard_into_space(False)
+
+        path = os.path.normpath(self.beat_path + "/../dev-tools/cmd/dashboards/export_dashboards.go")
+        command = path + " -kibana http://" + self.get_kibana_host() + ":" + self.get_kibana_port()
+        command = "go run " + command + " -dashboard Metricbeat-system-overview -space-id foo-bar"
+
+        p = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        content, err = p.communicate()
+
+        assert p.returncode == 0
+
+        assert os.path.isfile("output.json") is True
+
+        with open('output.json') as f:
+            content = f.read()
+            assert "Metricbeat-system-overview" in content
+
+        os.remove("output.json")
+
     def get_host(self):
         return os.getenv('ES_HOST', 'localhost') + ':' + os.getenv('ES_PORT', '9200')
 
@@ -96,3 +161,28 @@ def get_kibana_host(self):
 
     def get_kibana_port(self):
         return os.getenv('KIBANA_PORT', '5601')
+
+    def create_kibana_space(self):
+        url = "http://" + self.get_kibana_host() + ":" + self.get_kibana_port() + \
+            "/api/spaces/space"
+        data = {
+            "id": "foo-bar",
+            "name": "Foo bar space"
+        }
+
+        headers = {
+            "kbn-xsrf": "1"
+        }
+
+        r = requests.post(url, json=data, headers=headers)
+        assert r.status_code == 200
+
+    def get_version(self):
+        url = "http://" + self.get_kibana_host() + ":" + self.get_kibana_port() + \
+            "/api/status"
+
+        r = requests.get(url)
+        body = r.json()
+        version = body["version"]["number"]
+
+        return version

metricbeat/metricbeat.yml

@@ -66,6 +66,11 @@ setup.kibana:
   # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601
   #host: "localhost:5601"
 
+  # Kibana Space ID
+  # ID of the Kibana Space into which the dashboards should be loaded. By default,
+  # the Default Space will be used.
+  #space.id:
+
 #============================= Elastic Cloud ==================================
 
 # These settings simplify using metricbeat with the Elastic Cloud (https://cloud.elastic.co/).

packetbeat/packetbeat.yml

@@ -149,6 +149,11 @@ setup.kibana:
   # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601
   #host: "localhost:5601"
 
+  # Kibana Space ID
+  # ID of the Kibana Space into which the dashboards should be loaded. By default,
+  # the Default Space will be used.
+  #space.id:
+
 #============================= Elastic Cloud ==================================
 
 # These settings simplify using packetbeat with the Elastic Cloud (https://cloud.elastic.co/).

winlogbeat/winlogbeat.yml

@@ -70,6 +70,11 @@ setup.kibana:
   # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601
   #host: "localhost:5601"
 
+  # Kibana Space ID
+  # ID of the Kibana Space into which the dashboards should be loaded. By default,
+  # the Default Space will be used.
+  #space.id:
+
 #============================= Elastic Cloud ==================================
 
 # These settings simplify using winlogbeat with the Elastic Cloud (https://cloud.elastic.co/).