Make -setup load the Beat dashboards

CHANGELOG.asciidoc

@@ -66,6 +66,7 @@ https://github.com/elastic/beats/compare/v5.1.1...master[Check the HEAD diff]
 - RPM/deb packages will now install the config file with 0600 permissions. {pull}3382[3382]
 - Add the option to pass custom HTTP headers to the Elasticsearch output. {pull}3400[3400]
 - Unify `regexp` and `contains` conditionals, for both to support array of strings and convert numbers to strings if required. {pull}3469[3469]
+- Add the option to load the sample dashboards during the Beat startup phase. {pull}3506[3506]
 
 *Metricbeat*
 

filebeat/beater/filebeat.go

@@ -20,8 +20,7 @@ import (
 )
 
 var (
-	once  = flag.Bool("once", false, "Run filebeat only once until all harvesters reach EOF")
-	setup = flag.Bool("setup", false, "Run the setup phase for the modules")
+	once = flag.Bool("once", false, "Run filebeat only once until all harvesters reach EOF")
 )
 
 // Filebeat is a beater object. Contains all objects needed to run the beat
@@ -71,28 +70,34 @@ func New(b *beat.Beat, rawConfig *common.Config) (beat.Beater, error) {
 	return fb, nil
 }
 
-// Setup is called on user request (the -setup flag) to do the initial Beat setup.
-func (fb *Filebeat) Setup(b *beat.Beat) error {
+// modulesSetup is called when modules are configured to do the initial
+// setup.
+func (fb *Filebeat) modulesSetup(b *beat.Beat) error {
 	esConfig := b.Config.Output["elasticsearch"]
 	if esConfig == nil || !esConfig.Enabled() {
-		return fmt.Errorf("Setup requested but the Elasticsearch output is not configured/enabled")
+		return fmt.Errorf("Filebeat modules configured but the Elasticsearch output is not configured/enabled")
 	}
 	esClient, err := elasticsearch.NewConnectedClient(esConfig)
 	if err != nil {
 		return fmt.Errorf("Error creating ES client: %v", err)
 	}
 	defer esClient.Close()
 
-	return fb.moduleRegistry.Setup(esClient)
+	err = fb.moduleRegistry.LoadPipelines(esClient)
+	if err != nil {
+		return err
+	}
+
+	return nil
 }
 
 // Run allows the beater to be run as a beat.
 func (fb *Filebeat) Run(b *beat.Beat) error {
 	var err error
 	config := fb.config
 
-	if *setup {
-		err = fb.Setup(b)
+	if !fb.moduleRegistry.Empty() {
+		err = fb.modulesSetup(b)
 		if err != nil {
 			return err
 		}

filebeat/docs/reference/configuration.asciidoc

@@ -21,6 +21,7 @@ configuration settings, you need to restart {beatname_uc} to pick up the changes
 * <<console-output>>
 * <<configuration-output-ssl>>
 * <<configuration-path>>
+* <<configuration-dashboards>>
 * <<configuration-logging>>
 * <<configuration-processors>>
 

filebeat/docs/reference/configuration/filebeat-options.asciidoc

@@ -588,6 +588,9 @@ include::../../../../libbeat/docs/outputconfig.asciidoc[]
 pass::[<?edit_url https://github.com/elastic/beats/edit/master/libbeat/docs/shared-path-config.asciidoc ?>]
 include::../../../../libbeat/docs/shared-path-config.asciidoc[]
 
+pass::[<?edit_url https://github.com/elastic/beats/edit/master/libbeat/docs/dashboardsconfig.asciidoc ?>]
+include::../../../../libbeat/docs/dashboardsconfig.asciidoc[]
+
 pass::[<?edit_url https://github.com/elastic/beats/edit/master/libbeat/docs/loggingconfig.asciidoc ?>]
 include::../../../../libbeat/docs/loggingconfig.asciidoc[]
 

filebeat/filebeat.full.yml

@@ -897,6 +897,45 @@ output.elasticsearch:
 # the default for the logs path is a logs subdirectory inside the home path.
 #path.logs: ${path.home}/logs
 
+#============================== Dashboards =====================================
+# These settings control loading the sample dashboards to the Kibana index. Loading
+# the dashboards is disabled by default and can be enabled either by setting the
+# options here, or by using the `-setup` CLI flag.
+#dashboards.enabled: false
+
+# The URL from where to download the dashboards archive. By default this URL
+# has a value which is computed based on the Beat name and version. For released
+# versions, this URL points to the dashboard archive on the artifacts.elastic.co
+# website.
+#dashboards.url:
+
+# The directory from where to read the dashboards. It is used instead of the URL
+# when it has a value.
+#dashboards.directory:
+
+# The file archive (zip file) from where to read the dashboards. It is used instead
+# of the URL when it has a value.
+#dashboards.file:
+
+# If this option is enabled, the snapshot URL is used instead of the default URL.
+#dashboard.snapshot: false
+
+# The URL from where to download the snapshot version of the dashboards. By default
+# this has a value which is computed based on the Beat name and version.
+#dashboard.snapshot_url
+
+# In case the archive contains the dashboards from multiple Beats, this lets you
+# select which one to load. You can load all the dashboards in the archive by
+# setting this to the empty string.
+#dashboard.beat: filebeat
+
+# The name of the Kibana index to use for setting the configuration. Default is ".kibana"
+#dashboards.kibana_index: .kibana
+
+# The Elasticsearch index name. This overwrites the index name defined in the
+# dashboards and index pattern. Example: testbeat-*
+#dashboards.index:
+
 #================================ Logging ======================================
 # There are three options for the log output: syslog, file, stderr.
 # Under Windows systems, the log files are per default sent to the file output,

filebeat/fileset/modules.go

@@ -247,8 +247,8 @@ type PipelineLoader interface {
 	LoadJSON(path string, json map[string]interface{}) error
 }
 
-// Setup is called on -setup and loads the pipelines for each configured fileset.
-func (reg *ModuleRegistry) Setup(esClient PipelineLoader) error {
+// LoadPipelines loads the pipelines for each configured fileset.
+func (reg *ModuleRegistry) LoadPipelines(esClient PipelineLoader) error {
 	for module, filesets := range reg.registry {
 		for name, fileset := range filesets {
 			pipelineID, content, err := fileset.GetPipeline()
@@ -273,3 +273,11 @@ func loadPipeline(esClient PipelineLoader, pipelineID string, content map[string
 	logp.Info("Elasticsearch pipeline with ID '%s' loaded", pipelineID)
 	return nil
 }
+
+func (reg *ModuleRegistry) Empty() bool {
+	count := 0
+	for _, filesets := range reg.registry {
+		count += len(filesets)
+	}
+	return count == 0
+}

filebeat/fileset/modules_integration_test.go

@@ -48,7 +48,7 @@ func TestSetupNginx(t *testing.T) {
 	reg, err := newModuleRegistry(modulesPath, configs, nil)
 	assert.NoError(t, err)
 
-	err = reg.Setup(client)
+	err = reg.LoadPipelines(client)
 	assert.NoError(t, err)
 
 	status, _, _ := client.Request("GET", "/_ingest/pipeline/nginx-access-with_plugins", "", nil, nil)

filebeat/tests/system/test_modules.py

@@ -71,7 +71,7 @@ def run_on_file(self, module, fileset, test_file, cfgfile):
 
         cmd = [
             self.filebeat, "-systemTest",
-            "-e", "-d", "*", "-once", "-setup",
+            "-e", "-d", "*", "-once",
             "-c", cfgfile,
             "-modules={}".format(module),
             "-M", "{module}.{fileset}.var.paths=[{test_file}]".format(

heartbeat/docs/reference/configuration.asciidoc

@@ -19,6 +19,7 @@ configuration settings, you need to restart Heartbeat to pick up the changes.
 * <<console-output>>
 * <<configuration-output-ssl>>
 * <<configuration-path>>
+* <<configuration-dashboards>>
 * <<configuration-logging>>
 * <<configuration-processors>>
 

heartbeat/docs/reference/configuration/heartbeat-options.asciidoc

@@ -455,6 +455,9 @@ include::../../../../libbeat/docs/outputconfig.asciidoc[]
 pass::[<?edit_url https://github.com/elastic/beats/edit/master/libbeat/docs/shared-path-config.asciidoc ?>] 
 include::../../../../libbeat/docs/shared-path-config.asciidoc[]
 
+pass::[<?edit_url https://github.com/elastic/beats/edit/master/libbeat/docs/dashboardsconfig.asciidoc ?>]
+include::../../../../libbeat/docs/dashboardsconfig.asciidoc[]
+
 pass::[<?edit_url https://github.com/elastic/beats/edit/master/libbeat/docs/loggingconfig.asciidoc ?>] 
 include::../../../../libbeat/docs/loggingconfig.asciidoc[]
 

heartbeat/heartbeat.full.yml

@@ -745,6 +745,45 @@ output.elasticsearch:
 # the default for the logs path is a logs subdirectory inside the home path.
 #path.logs: ${path.home}/logs
 
+#============================== Dashboards =====================================
+# These settings control loading the sample dashboards to the Kibana index. Loading
+# the dashboards is disabled by default and can be enabled either by setting the
+# options here, or by using the `-setup` CLI flag.
+#dashboards.enabled: false
+
+# The URL from where to download the dashboards archive. By default this URL
+# has a value which is computed based on the Beat name and version. For released
+# versions, this URL points to the dashboard archive on the artifacts.elastic.co
+# website.
+#dashboards.url:
+
+# The directory from where to read the dashboards. It is used instead of the URL
+# when it has a value.
+#dashboards.directory:
+
+# The file archive (zip file) from where to read the dashboards. It is used instead
+# of the URL when it has a value.
+#dashboards.file:
+
+# If this option is enabled, the snapshot URL is used instead of the default URL.
+#dashboard.snapshot: false
+
+# The URL from where to download the snapshot version of the dashboards. By default
+# this has a value which is computed based on the Beat name and version.
+#dashboard.snapshot_url
+
+# In case the archive contains the dashboards from multiple Beats, this lets you
+# select which one to load. You can load all the dashboards in the archive by
+# setting this to the empty string.
+#dashboard.beat: heartbeat
+
+# The name of the Kibana index to use for setting the configuration. Default is ".kibana"
+#dashboards.kibana_index: .kibana
+
+# The Elasticsearch index name. This overwrites the index name defined in the
+# dashboards and index pattern. Example: testbeat-*
+#dashboards.index:
+
 #================================ Logging ======================================
 # There are three options for the log output: syslog, file, stderr.
 # Under Windows systems, the log files are per default sent to the file output,

libbeat/_meta/config.full.yml

@@ -547,6 +547,45 @@ output.elasticsearch:
 # the default for the logs path is a logs subdirectory inside the home path.
 #path.logs: ${path.home}/logs
 
+#============================== Dashboards =====================================
+# These settings control loading the sample dashboards to the Kibana index. Loading
+# the dashboards is disabled by default and can be enabled either by setting the
+# options here, or by using the `-setup` CLI flag.
+#dashboards.enabled: false
+
+# The URL from where to download the dashboards archive. By default this URL
+# has a value which is computed based on the Beat name and version. For released
+# versions, this URL points to the dashboard archive on the artifacts.elastic.co
+# website.
+#dashboards.url:
+
+# The directory from where to read the dashboards. It is used instead of the URL
+# when it has a value.
+#dashboards.directory:
+
+# The file archive (zip file) from where to read the dashboards. It is used instead
+# of the URL when it has a value.
+#dashboards.file:
+
+# If this option is enabled, the snapshot URL is used instead of the default URL.
+#dashboard.snapshot: false
+
+# The URL from where to download the snapshot version of the dashboards. By default
+# this has a value which is computed based on the Beat name and version.
+#dashboard.snapshot_url
+
+# In case the archive contains the dashboards from multiple Beats, this lets you
+# select which one to load. You can load all the dashboards in the archive by
+# setting this to the empty string.
+#dashboard.beat: beatname
+
+# The name of the Kibana index to use for setting the configuration. Default is ".kibana"
+#dashboards.kibana_index: .kibana
+
+# The Elasticsearch index name. This overwrites the index name defined in the
+# dashboards and index pattern. Example: testbeat-*
+#dashboards.index:
+
 #================================ Logging ======================================
 # There are three options for the log output: syslog, file, stderr.
 # Under Windows systems, the log files are per default sent to the file output,

libbeat/beat/beat.go

@@ -44,7 +44,9 @@ import (
 
 	"github.com/elastic/beats/libbeat/cfgfile"
 	"github.com/elastic/beats/libbeat/common"
+	"github.com/elastic/beats/libbeat/dashboards/dashboards"
 	"github.com/elastic/beats/libbeat/logp"
+	"github.com/elastic/beats/libbeat/outputs/elasticsearch"
 	"github.com/elastic/beats/libbeat/paths"
 	"github.com/elastic/beats/libbeat/plugin"
 	"github.com/elastic/beats/libbeat/processors"
@@ -100,10 +102,12 @@ type BeatConfig struct {
 	Logging    logp.Logging              `config:"logging"`
 	Processors processors.PluginConfig   `config:"processors"`
 	Path       paths.Path                `config:"path"`
+	Dashboards *common.Config            `config:"dashboards"`
 }
 
 var (
 	printVersion = flag.Bool("version", false, "Print the version and exit")
+	setup        = flag.Bool("setup", false, "Load the sample Kibana dashboards")
 )
 
 var debugf = logp.MakeDebug("beat")
@@ -209,6 +213,11 @@ func (b *Beat) launch(bt Creator) error {
 
 	svc.HandleSignals(beater.Stop)
 
+	err = b.loadDashboards()
+	if err != nil {
+		return err
+	}
+
 	logp.Info("%s start running.", b.Name)
 	defer logp.Info("%s stopped.", b.Name)
 	defer logp.LogTotalExpvars(&b.Config.Logging)
@@ -285,6 +294,39 @@ func (b *Beat) configure() error {
 	return nil
 }
 
+func (b *Beat) loadDashboards() error {
+	if *setup {
+		// -setup implies dashboards.enabled=true
+		if b.Config.Dashboards == nil {
+			b.Config.Dashboards = common.NewConfig()
+		}
+		err := b.Config.Dashboards.SetBool("enabled", -1, true)
+		if err != nil {
+			return fmt.Errorf("Error setting dashboard.enabled=true: %v", err)
+		}
+	}
+
+	if b.Config.Dashboards != nil && b.Config.Dashboards.Enabled() {
+		esConfig := b.Config.Output["elasticsearch"]
+		if esConfig == nil || !esConfig.Enabled() {
+			return fmt.Errorf("Dashboard loading requested but the Elasticsearch output is not configured/enabled")
+		}
+		esClient, err := elasticsearch.NewConnectedClient(esConfig)
+		if err != nil {
+			return fmt.Errorf("Error creating ES client: %v", err)
+		}
+		defer esClient.Close()
+
+		err = dashboards.ImportDashboards(b.Name, b.Version, esClient, b.Config.Dashboards)
+		if err != nil {
+			return fmt.Errorf("Error importing Kibana dashboards: %v", err)
+		}
+		logp.Info("Kibana dashboards successfully loaded.")
+	}
+
+	return nil
+}
+
 // handleError handles the given error by logging it and then returning the
 // error. If the err is nil or is a GracefulExit error then the method will
 // return nil without logging anything.

libbeat/dashboards/dashboards/config.go

@@ -0,0 +1,23 @@
+package dashboards
+
+type DashboardsConfig struct {
+	Enabled        bool   `config:"enabled"`
+	KibanaIndex    string `config:"kibana_index"`
+	Index          string `config:"index"`
+	Dir            string `config:"directory"`
+	File           string `config:"file"`
+	Beat           string `config:"beat"`
+	URL            string `config:"url"`
+	OnlyDashboards bool   `config:"only_dashboards"`
+	OnlyIndex      bool   `config:"only_index"`
+	Snapshot       bool   `config:"snapshot"`
+	SnapshotURL    string `config:"snapshot_url"`
+}
+
+var defaultDashboardsConfig = DashboardsConfig{
+	KibanaIndex: ".kibana",
+}
+var (
+	defaultURLPattern  = "https://artifacts.elastic.co/downloads/beats/beats-dashboards/beats-dashboards-%s.zip"
+	snapshotURLPattern = "https://beats-nightlies.s3.amazonaws.com/dashboards/beats-dashboards-%s-SNAPSHOT.zip"
+)