Index metricset for elasticsearch Metricbeat module

CHANGELOG.asciidoc

@@ -128,7 +128,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di
 - Add MongoDB module. {pull}6283[6238]
 - Add Ingest pipeline loading to setup. {pull}6814[6814]
 - Add support of log_format combined to NGINX access logs. {pull}6858[6858]
-- Release config reloading feature as GA.
+- Release config reloading feature as GA. {pull}6891[6891]
 - Add support human friendly size for the UDP input. {pull}6886[6886]
 - Add Syslog input to ingest RFC3164 Events via TCP and UDP {pull}6842[6842]
 
@@ -199,7 +199,8 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di
 - Added support for haproxy 1.7 and 1.8. {pull}6793[6793]
 - Add accumulated I/O stats to diskio in the line of `docker stats`. {pull}6701[6701]
 - Ignore virtual filesystem types by default in system module. {pull}6819[6819]
-- Release config reloading feature as GA.
+- Release config reloading feature as GA. {pull}6891[6891]
+- Add experimental Elasticsearch index metricset. {pull}6881[6881]
 
 *Packetbeat*
 

metricbeat/_meta/fields.common.yml

@@ -34,3 +34,7 @@
       description: >
         The document type. Always set to "doc".
 
+    - name: service.name
+      example: elasticsearch
+      description: >
+        Name of the service metricbeat fetches the data from.

metricbeat/docs/fields.asciidoc

@@ -1862,6 +1862,16 @@ required: True
 The document type. Always set to "doc".
 
 
+--
+
+*`service.name`*::
++
+--
+example: elasticsearch
+
+Name of the service metricbeat fetches the data from.
+
+
 --
 
 [[exported-fields-couchbase]]
@@ -3317,6 +3327,88 @@ type: keyword
 Elasticsearch cluster name.
 
 
+--
+
+*`elasticsearch.cluster.id`*::
++
+--
+type: keyword
+
+Elasticsearch cluster id.
+
+
+--
+
+[float]
+== index fields
+
+index
+
+
+
+*`elasticsearch.index.name`*::
++
+--
+type: keyword
+
+Index name.
+
+
+--
+
+
+*`elasticsearch.index.total.docs.count`*::
++
+--
+type: long
+
+Total number of documents in the index.
+
+
+--
+
+*`elasticsearch.index.total.docs.deleted`*::
++
+--
+type: long
+
+Total number of deleted documents in the index.
+
+
+--
+
+*`elasticsearch.index.total.store.size.bytes`*::
++
+--
+type: long
+
+format: bytes
+
+Total size of the index in bytes.
+
+
+--
+
+*`elasticsearch.index.total.segments.count`*::
++
+--
+type: long
+
+Total number of index segments.
+
+
+--
+
+*`elasticsearch.index.total.segments.memory.bytes`*::
++
+--
+type: long
+
+format: bytes
+
+Total number of memory used by the segments in bytes.
+
+
 --
 
 [float]

metricbeat/docs/modules/elasticsearch.asciidoc

@@ -22,7 +22,10 @@ in <<configuration-metricbeat>>. Here is an example configuration:
 ----
 metricbeat.modules:
 - module: elasticsearch
-  metricsets: ["node", "node_stats"]
+  metricsets:
+   #- index
+   - node
+   - node_stats
   period: 10s
   hosts: ["localhost:9200"]
 ----
@@ -34,10 +37,14 @@ This module supports TLS connection when using `ssl` config field, as described
 
 The following metricsets are available:
 
+* <<metricbeat-metricset-elasticsearch-index,index>>
+
 * <<metricbeat-metricset-elasticsearch-node,node>>
 
 * <<metricbeat-metricset-elasticsearch-node_stats,node_stats>>
 
+include::elasticsearch/index.asciidoc[]
+
 include::elasticsearch/node.asciidoc[]
 
 include::elasticsearch/node_stats.asciidoc[]

metricbeat/docs/modules/elasticsearch/index.asciidoc

@@ -0,0 +1,23 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+[[metricbeat-metricset-elasticsearch-index]]
+=== Elasticsearch index metricset
+
+experimental[]
+
+include::../../../module/elasticsearch/index/_meta/docs.asciidoc[]
+
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-elasticsearch,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../module/elasticsearch/index/_meta/data.json[]
+----

metricbeat/docs/modules_list.asciidoc

@@ -33,7 +33,8 @@ This file is generated! See scripts/docs_collector.py
 |<<metricbeat-module-dropwizard,Dropwizard>>  beta[]   |image:./images/icon-no.png[No prebuilt dashboards]    |  
 .1+| .1+|  |<<metricbeat-metricset-dropwizard-collector,collector>> beta[]  
 |<<metricbeat-module-elasticsearch,Elasticsearch>>  beta[]   |image:./images/icon-no.png[No prebuilt dashboards]    |  
-.2+| .2+|  |<<metricbeat-metricset-elasticsearch-node,node>> beta[]  
+.3+| .3+|  |<<metricbeat-metricset-elasticsearch-index,index>> experimental[]  
+|<<metricbeat-metricset-elasticsearch-node,node>> beta[]  
 |<<metricbeat-metricset-elasticsearch-node_stats,node_stats>> beta[]  
 |<<metricbeat-module-etcd,Etcd>>  beta[]   |image:./images/icon-no.png[No prebuilt dashboards]    |  
 .3+| .3+|  |<<metricbeat-metricset-etcd-leader,leader>> beta[]  

metricbeat/include/list.go

@@ -36,6 +36,7 @@ import (
 	_ "github.com/elastic/beats/metricbeat/module/dropwizard"
 	_ "github.com/elastic/beats/metricbeat/module/dropwizard/collector"
 	_ "github.com/elastic/beats/metricbeat/module/elasticsearch"
+	_ "github.com/elastic/beats/metricbeat/module/elasticsearch/index"
 	_ "github.com/elastic/beats/metricbeat/module/elasticsearch/node"
 	_ "github.com/elastic/beats/metricbeat/module/elasticsearch/node_stats"
 	_ "github.com/elastic/beats/metricbeat/module/etcd"

metricbeat/mb/testing/data_generator.go

@@ -57,6 +57,28 @@ func WriteEvents(f mb.EventsFetcher, t testing.TB) error {
 	return nil
 }
 
+// WriteEventsReporterV2 fetches events and writes the first event to a ./_meta/data.json
+// file.
+func WriteEventsReporterV2(f mb.ReportingMetricSetV2, t testing.TB) error {
+	if !*dataFlag {
+		t.Skip("skip data generation tests")
+	}
+
+	events, errs := ReportingFetchV2(f)
+	if len(errs) > 0 {
+		return errs[0]
+	}
+
+	if len(events) == 0 {
+		return fmt.Errorf("no events were generated")
+	}
+
+	e := StandardizeEvent(f, events[0])
+
+	WriteEventToDataJSON(t, e)
+	return nil
+}
+
 // CreateFullEvent builds a full event given the data generated by a MetricSet.
 // This simulates the output of Metricbeat as if it were
 // 2016-05-23T08:05:34.853Z and the hostname is host.example.com.

metricbeat/mb/testing/modules.go

@@ -151,25 +151,38 @@ func NewReportingMetricSetV2(t testing.TB, config interface{}) mb.ReportingMetri
 	return reportingMetricSetV2
 }
 
-type capturingReporterV2 struct {
+// CapturingReporterV2 is a reporter used for testing which stores all events and errors
+type CapturingReporterV2 struct {
 	events []mb.Event
 	errs   []error
 }
 
-func (r *capturingReporterV2) Event(event mb.Event) bool {
+// Event is used to report an event
+func (r *CapturingReporterV2) Event(event mb.Event) bool {
 	r.events = append(r.events, event)
 	return true
 }
 
-func (r *capturingReporterV2) Error(err error) bool {
+// Error is used to report an error
+func (r *CapturingReporterV2) Error(err error) bool {
 	r.errs = append(r.errs, err)
 	return true
 }
 
+// GetEvents returns all reported events
+func (r *CapturingReporterV2) GetEvents() []mb.Event {
+	return r.events
+}
+
+// GetErrors returns all reported errors
+func (r *CapturingReporterV2) GetErrors() []error {
+	return r.errs
+}
+
 // ReportingFetchV2 runs the given reporting metricset and returns all of the
 // events and errors that occur during that period.
 func ReportingFetchV2(metricSet mb.ReportingMetricSetV2) ([]mb.Event, []error) {
-	r := &capturingReporterV2{}
+	r := &CapturingReporterV2{}
 	metricSet.Fetch(r)
 	return r.events, r.errs
 }

metricbeat/metricbeat.reference.yml

@@ -195,7 +195,10 @@ metricbeat.modules:
 
 #---------------------------- Elasticsearch Module ---------------------------
 - module: elasticsearch
-  metricsets: ["node", "node_stats"]
+  metricsets:
+   #- index
+   - node
+   - node_stats
   period: 10s
   hosts: ["localhost:9200"]
 

metricbeat/module/elasticsearch/_meta/config.reference.yml

@@ -1,4 +1,7 @@
 - module: elasticsearch
-  metricsets: ["node", "node_stats"]
+  metricsets:
+   #- index
+   - node
+   - node_stats
   period: 10s
   hosts: ["localhost:9200"]

metricbeat/module/elasticsearch/_meta/fields.yml

@@ -14,3 +14,8 @@
           type: keyword
           description: >
             Elasticsearch cluster name.
+
+        - name: cluster.id
+          type: keyword
+          description: >
+            Elasticsearch cluster id.