Add elasticsearch module with stats metricset (#3903)

This adds the stats metricset for the elasticsearch module. The stats metricset gives and overview over the current state of the cluster.

metricbeat/docker-compose.yml

@@ -19,6 +19,7 @@ services:
       - ${PWD}/module/apache/_meta/env
       - ${PWD}/module/ceph/_meta/env
       - ${PWD}/module/couchbase/_meta/env
+      - ${PWD}/module/elasticsearch/_meta/env
       - ${PWD}/module/haproxy/_meta/env
       - ${PWD}/module/jolokia/_meta/env
       - ${PWD}/module/kafka/_meta/env
@@ -38,22 +39,23 @@ services:
   proxy_dep:
     image: busybox
     depends_on:
-      apache:     { condition: service_healthy }
-      ceph:       { condition: service_healthy }
-      couchbase:  { condition: service_healthy }
-      haproxy:    { condition: service_healthy }
-      jolokia:    { condition: service_healthy }
-      kafka:      { condition: service_healthy }
-      kibana:      { condition: service_healthy }
-      memcached:  { condition: service_healthy }
-      mongodb:    { condition: service_healthy }
-      mysql:      { condition: service_healthy }
-      nginx:      { condition: service_healthy }
-      phpfpm:     { condition: service_healthy }
-      postgresql: { condition: service_healthy }
-      prometheus: { condition: service_healthy }
-      redis:      { condition: service_healthy }
-      zookeeper:  { condition: service_healthy }
+      apache:         { condition: service_healthy }
+      ceph:           { condition: service_healthy }
+      couchbase:      { condition: service_healthy }
+      elasticsearch:  { condition: service_healthy }
+      haproxy:        { condition: service_healthy }
+      jolokia:        { condition: service_healthy }
+      kafka:          { condition: service_healthy }
+      kibana:         { condition: service_healthy }
+      memcached:      { condition: service_healthy }
+      mongodb:        { condition: service_healthy }
+      mysql:          { condition: service_healthy }
+      nginx:          { condition: service_healthy }
+      phpfpm:         { condition: service_healthy }
+      postgresql:     { condition: service_healthy }
+      prometheus:     { condition: service_healthy }
+      redis:          { condition: service_healthy }
+      zookeeper:      { condition: service_healthy }
 
   # Modules
   apache:
@@ -65,6 +67,11 @@ services:
   couchbase:
     build: ${PWD}/module/couchbase/_meta
 
+  elasticsearch:
+    extends:
+      file: ${ES_BEATS}/testing/environments/${TESTING_ENVIRONMENT}.yml
+      service: elasticsearch
+
   haproxy:
     build: ${PWD}/module/haproxy/_meta
 

metricbeat/docs/fields.asciidoc

@@ -19,6 +19,7 @@ grouped in the following categories:
 * <<exported-fields-common>>
 * <<exported-fields-couchbase>>
 * <<exported-fields-docker>>
+* <<exported-fields-elasticsearch>>
 * <<exported-fields-golang>>
 * <<exported-fields-haproxy>>
 * <<exported-fields-jolokia>>
@@ -1843,6 +1844,93 @@ type: long
 Total number of outgoing packets.
 
 
+[[exported-fields-elasticsearch]]
+== elasticsearch Fields
+
+[]experimental
+elasticsearch Module
+
+
+
+[float]
+== elasticsearch Fields
+
+
+
+
+[float]
+== stats Fields
+
+stats
+
+
+
+[float]
+=== elasticsearch.stats.docs.count
+
+type: long
+
+Total number of existing documents.
+
+
+[float]
+=== elasticsearch.stats.docs.deleted
+
+type: long
+
+Total number of deleted documents.
+
+
+[float]
+=== elasticsearch.stats.segments.count
+
+type: long
+
+Total number of segments.
+
+
+[float]
+=== elasticsearch.stats.segments.memory.bytes
+
+type: long
+
+format: bytes
+
+Total size of segments in bytes.
+
+
+[float]
+=== elasticsearch.stats.shards.failed
+
+type: long
+
+Number of failed shards.
+
+
+[float]
+=== elasticsearch.stats.shards.successful
+
+type: long
+
+Number of successful shards.
+
+
+[float]
+=== elasticsearch.stats.shards.total
+
+type: long
+
+Total number of shards.
+
+
+[float]
+=== elasticsearch.stats.store.size.bytes
+
+type: long
+
+Total size of the store in bytes.
+
+
 [[exported-fields-golang]]
 == golang Fields
 

metricbeat/docs/modules/elasticsearch.asciidoc

@@ -0,0 +1,37 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+[[metricbeat-module-elasticsearch]]
+== elasticsearch Module
+
+This is the elasticsearch Module. The elasticsearch module contains a minimal set of metrics to enable monitoring of elasticsearch across multiple versions.
+
+
+
+[float]
+=== Example Configuration
+
+The elasticsearch module supports the standard configuration options that are described
+in <<configuration-metricbeat>>. Here is an example configuration:
+
+[source,yaml]
+----
+metricbeat.modules:
+#- module: elasticsearch
+#  metricsets: ["stats"]
+#  enabled: true
+#  period: 10s
+#  hosts: ["localhost:9200"]
+
+----
+
+[float]
+=== Metricsets
+
+The following metricsets are available:
+
+* <<metricbeat-metricset-elasticsearch-stats,stats>>
+
+include::elasticsearch/stats.asciidoc[]
+

metricbeat/docs/modules/elasticsearch/stats.asciidoc

@@ -0,0 +1,19 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+[[metricbeat-metricset-elasticsearch-stats]]
+include::../../../module/elasticsearch/stats/_meta/docs.asciidoc[]
+
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-elasticsearch,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../module/elasticsearch/stats/_meta/data.json[]
+----

metricbeat/docs/modules_list.asciidoc

@@ -6,6 +6,7 @@ This file is generated! See scripts/docs_collector.py
   * <<metricbeat-module-ceph,ceph>>
   * <<metricbeat-module-couchbase,Couchbase>>
   * <<metricbeat-module-docker,Docker>>
+  * <<metricbeat-module-elasticsearch,elasticsearch>>
   * <<metricbeat-module-golang,golang>>
   * <<metricbeat-module-haproxy,HAProxy>>
   * <<metricbeat-module-jolokia,Jolokia>>
@@ -31,6 +32,7 @@ include::modules/apache.asciidoc[]
 include::modules/ceph.asciidoc[]
 include::modules/couchbase.asciidoc[]
 include::modules/docker.asciidoc[]
+include::modules/elasticsearch.asciidoc[]
 include::modules/golang.asciidoc[]
 include::modules/haproxy.asciidoc[]
 include::modules/jolokia.asciidoc[]

metricbeat/include/list.go

@@ -28,6 +28,8 @@ import (
 	_ "github.com/elastic/beats/metricbeat/module/docker/info"
 	_ "github.com/elastic/beats/metricbeat/module/docker/memory"
 	_ "github.com/elastic/beats/metricbeat/module/docker/network"
+	_ "github.com/elastic/beats/metricbeat/module/elasticsearch"
+	_ "github.com/elastic/beats/metricbeat/module/elasticsearch/stats"
 	_ "github.com/elastic/beats/metricbeat/module/golang"
 	_ "github.com/elastic/beats/metricbeat/module/golang/expvar"
 	_ "github.com/elastic/beats/metricbeat/module/golang/heap"

metricbeat/metricbeat.full.yml

@@ -124,6 +124,14 @@ metricbeat.modules:
     #certificate:           "/etc/pki/client/cert.pem"
     #key:                   "/etc/pki/client/cert.key"
 
+#---------------------------- elasticsearch Module ---------------------------
+#- module: elasticsearch
+#  metricsets: ["stats"]
+#  enabled: true
+#  period: 10s
+#  hosts: ["localhost:9200"]
+
+
 #------------------------------- golang Module -------------------------------
 #- module: golang
 #  metricsets: ["expvar","heap"]

metricbeat/module/elasticsearch/_meta/config.yml

@@ -0,0 +1,6 @@
+#- module: elasticsearch
+#  metricsets: ["stats"]
+#  enabled: true
+#  period: 10s
+#  hosts: ["localhost:9200"]
+

metricbeat/module/elasticsearch/_meta/docs.asciidoc

@@ -0,0 +1,4 @@
+== elasticsearch Module
+
+This is the elasticsearch Module. The elasticsearch module contains a minimal set of metrics to enable monitoring of elasticsearch across multiple versions.
+

metricbeat/module/elasticsearch/_meta/env

@@ -0,0 +1,2 @@
+ELASTICSEARCH_HOST=elasticsearch
+ELASTICSEARCH_PORT=9200

metricbeat/module/elasticsearch/_meta/fields.yml

@@ -0,0 +1,12 @@
+- key: elasticsearch
+  title: "elasticsearch"
+  description: >
+    []experimental
+
+    elasticsearch Module
+  short_config: false
+  fields:
+    - name: elasticsearch
+      type: group
+      description: >
+      fields:

metricbeat/module/elasticsearch/doc.go

@@ -0,0 +1,4 @@
+/*
+Package elasticsearch is a Metricbeat module that contains MetricSets.
+*/
+package elasticsearch

metricbeat/module/elasticsearch/stats/_meta/data.json

@@ -0,0 +1,38 @@
+{
+    "@timestamp": "2016-05-23T08:05:34.853Z",
+    "beat": {
+        "hostname": "host.example.com",
+        "name": "host.example.com"
+    },
+    "elasticsearch": {
+        "stats": {
+            "docs": {
+                "count": 4,
+                "deleted": 0
+            },
+            "segments": {
+                "count": 4,
+                "memory": {
+                    "bytes": 6647
+                }
+            },
+            "shards": {
+                "failed": 0,
+                "successful": 11,
+                "total": 22
+            },
+            "store": {
+                "size": {
+                    "bytes": 16148
+                }
+            }
+        }
+    },
+    "metricset": {
+        "host": "127.0.0.1:9200",
+        "module": "elasticsearch",
+        "name": "stats",
+        "rtt": 115
+    },
+    "type": "metricsets"
+}

metricbeat/module/elasticsearch/stats/_meta/docs.asciidoc

@@ -0,0 +1,3 @@
+=== elasticsearch stats MetricSet
+
+This is the stats metricset of the module elasticsearch. This provides statistics across all indices.

metricbeat/module/elasticsearch/stats/_meta/fields.yml

@@ -0,0 +1,38 @@
+- name: stats
+  type: group
+  description: >
+    stats
+  fields:
+    - name: docs.count
+      type: long
+      description: >
+        Total number of existing documents.
+    - name: docs.deleted
+      type: long
+      description: >
+        Total number of deleted documents.
+    - name: segments.count
+      type: long
+      description: >
+        Total number of segments.
+    - name: segments.memory.bytes
+      type: long
+      format: bytes
+      description: >
+        Total size of segments in bytes.
+    - name: shards.failed
+      type: long
+      description: >
+        Number of failed shards.
+    - name: shards.successful
+      type: long
+      description: >
+        Number of successful shards.
+    - name: shards.total
+      type: long
+      description: >
+        Total number of shards.
+    - name: store.size.bytes
+      type: long
+      description: >
+        Total size of the store in bytes.