Add Slowlog fileset for the Elasticsearch module (#7473)

This is the initial PR for slowlog indexing.

CHANGELOG.asciidoc

@@ -205,6 +205,7 @@ https://github.com/elastic/beats/compare/v6.2.3...master[Check the HEAD diff]
 - Converted part of pipeline from treafik/access metricSet to dissect to improve efficeny. {pull}7209[7209]
 - Add GC fileset to the Elasticsearch module. {pull}7305[7305]
 - Add Audit log fileset to the Elasticsearch module. {pull}7365[7365]
+- Add Slow log fileset to the Elasticsearch module. {pull}7473[7473]
 
 *Heartbeat*
 

filebeat/docs/fields.asciidoc

@@ -976,6 +976,166 @@ type: keyword
 Elasticsearch component.
 
 
+--
+
+[float]
+== slowlog fields
+
+Slowlog events from Elasticsearch
+
+
+*`elasticsearch.slowlog.loglevel`*::
++
+--
+type: keyword
+
+example: INFO
+
+Log level
+
+--
+
+*`elasticsearch.slowlog.logger`*::
++
+--
+type: keyword
+
+example: index.search.slowlog.fetch
+
+Logger name
+
+--
+
+*`elasticsearch.slowlog.node_name`*::
++
+--
+type: keyword
+
+example: v_VJhjV
+
+Name of the node
+
+--
+
+*`elasticsearch.slowlog.index_name`*::
++
+--
+type: keyword
+
+example: metricbeat-6.3.0-2018.06.26
+
+Name of the index
+
+--
+
+*`elasticsearch.slowlog.shard_id`*::
++
+--
+type: keyword
+
+example: 0
+
+Id of the shard
+
+--
+
+*`elasticsearch.slowlog.took`*::
++
+--
+type: text
+
+example: 300ms
+
+Time it took to execute the query
+
+--
+
+*`elasticsearch.slowlog.types`*::
++
+--
+type: keyword
+
+example: 
+
+Types
+
+--
+
+*`elasticsearch.slowlog.stats`*::
++
+--
+type: text
+
+example: 
+
+Statistics
+
+--
+
+*`elasticsearch.slowlog.search_type`*::
++
+--
+type: keyword
+
+example: QUERY_THEN_FETCH
+
+Please add description
+
+--
+
+*`elasticsearch.slowlog.source_query`*::
++
+--
+type: text
+
+example: {"query":{"match_all":{"boost":1.0}}}
+
+Slow query
+
+--
+
+*`elasticsearch.slowlog.extra_source`*::
++
+--
+type: text
+
+example: 
+
+Extra source information
+
+--
+
+*`elasticsearch.slowlog.took_millis`*::
++
+--
+type: keyword
+
+example: 42
+
+Time took in milliseconds
+
+--
+
+*`elasticsearch.slowlog.total_hits`*::
++
+--
+type: keyword
+
+example: 42
+
+Total hits
+
+--
+
+*`elasticsearch.slowlog.total_shards`*::
++
+--
+type: keyword
+
+example: 22
+
+Total queried shards
+
 --
 
 [[exported-fields-host-processor]]

filebeat/filebeat.reference.yml

@@ -104,6 +104,12 @@ filebeat.modules:
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
+  slowlog:
+    enabled: true
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
+
 #------------------------------- Icinga Module -------------------------------
 #- module: icinga
   # Main logs

filebeat/module/elasticsearch/_meta/config.yml

@@ -18,3 +18,9 @@
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
+
+  slowlog:
+    enabled: true
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:

filebeat/module/elasticsearch/slowlog/_meta/fields.yml

@@ -0,0 +1,61 @@
+- name: slowlog
+  description: "Slowlog events from Elasticsearch"
+  example: "[2018-06-29T10:06:14,933][INFO ][index.search.slowlog.query] [v_VJhjV] [metricbeat-6.3.0-2018.06.26][0] took[4.5ms], took_millis[4], total_hits[19435], types[], stats[], search_type[QUERY_THEN_FETCH], total_shards[1], source[{\"query\":{\"match_all\":{\"boost\":1.0}}}],"
+  type: group
+  fields:
+  - name: loglevel
+    description: "Log level"
+    example: "INFO"
+    type: keyword
+  - name: logger
+    description: "Logger name"
+    example: "index.search.slowlog.fetch"
+    type: keyword
+  - name: node_name
+    description: "Name of the node"
+    example: "v_VJhjV"
+    type: keyword
+  - name: index_name
+    description: "Name of the index"
+    example: "metricbeat-6.3.0-2018.06.26"
+    type: keyword
+  - name: shard_id
+    description: "Id of the shard"
+    example: "0"
+    type: keyword
+  - name: took
+    description: "Time it took to execute the query"
+    example: "300ms"
+    type: text
+  - name: types
+    description: "Types"
+    example: ""
+    type: keyword
+  - name: stats
+    description: "Statistics"
+    example: ""
+    type: text
+  - name: search_type
+    description: Please add description
+    example: "QUERY_THEN_FETCH"
+    type: keyword
+  - name: source_query
+    description: "Slow query"
+    example: "{\"query\":{\"match_all\":{\"boost\":1.0}}}"
+    type: text
+  - name: extra_source
+    description: "Extra source information"
+    example: ""
+    type: text
+  - name: took_millis
+    description: "Time took in milliseconds"
+    example: 42
+    type: keyword
+  - name: total_hits
+    description: "Total hits"
+    example: 42
+    type: keyword
+  - name: total_shards
+    description: "Total queried shards"
+    example: 22
+    type: keyword

filebeat/module/elasticsearch/slowlog/config/slowlog.yml

@@ -0,0 +1,10 @@
+type: log
+paths:
+{{ range $i, $path := .paths }}
+ - {{$path}}
+{{ end }}
+exclude_files: [".gz$"]
+
+fields:
+  service.name: "elasticsearch"
+fields_under_root: true

filebeat/module/elasticsearch/slowlog/ingest/pipeline.json

@@ -0,0 +1,31 @@
+{
+  "description": "Pipeline for parsing elasticsearch slowlog logs",
+  "processors": [
+      {
+          "rename": {
+              "field": "@timestamp",
+              "target_field": "event.created"
+          }
+      },
+      {
+          "grok": {
+              "field": "message",
+              "patterns": [
+                  "\\[%{TIMESTAMP_ISO8601:elasticsearch.slowlog.timestamp}\\]\\[%{WORD:elasticsearch.slowlog.loglevel}(\\s*)?\\](\\s*)?\\[%{DATA:elasticsearch.slowlog.logger}\\]\\s*\\[%{WORD:elasticsearch.slowlog.node_name}\\]\\s*\\[%{DATA:elasticsearch.slowlog.index_name}\\]\\[%{DATA:elasticsearch.slowlog.shard_id}\\]\\s*took\\[%{DATA:elasticsearch.slowlog.took}\\],\\s*took_millis\\[%{NUMBER:elasticsearch.slowlog.took_millis:int}\\],\\s*total_hits\\[%{NUMBER:elasticsearch.slowlog.total_hits:int}\\],\\s*types\\[%{DATA:elasticsearch.slowlog.types}\\],\\s*stats\\[%{DATA:elasticsearch.slowlog.stats}\\],\\s*search_type\\[%{DATA:elasticsearch.slowlog.search_type}\\],\\s*total_shards\\[%{NUMBER:elasticsearch.slowlog.total_shards:int}\\],\\s*source\\[%{GREEDYDATA:elasticsearch.slowlog.source_query}\\],(\\s*)?(extra_source\\[%{DATA:elasticsearch.slowlog.extra_source}\\])?,?"
+              ]
+          }
+      },
+      {
+          "rename": {
+              "field": "elasticsearch.slowlog.timestamp",
+              "target_field": "@timestamp"
+          }
+      }
+    ],
+  "on_failure" : [{
+    "set" : {
+      "field" : "error.message",
+      "value" : "{{ _ingest.on_failure_message }}"
+    }
+  }]
+}

filebeat/module/elasticsearch/slowlog/manifest.yml

@@ -0,0 +1,13 @@
+module_version: 1.0
+
+var:
+  - name: paths
+    default:
+      - /var/log/elasticsearch/*_index_search_slowlog.log
+    os.darwin:
+      - /usr/local/elasticsearch/*_index_search_slowlog.log
+    os.windows:
+      - c:/ProgramData/Elastic/Elasticsearch/logs/*_index_search_slowlog.log
+
+ingest_pipeline: ingest/pipeline.json
+input: config/slowlog.yml

filebeat/module/elasticsearch/slowlog/test/test.log

@@ -0,0 +1,4 @@
+[2018-06-29T10:06:14,933][INFO ][index.search.slowlog.query] [v_VJhjV] [metricbeat-6.3.0-2018.06.26][0] took[4.5ms], took_millis[4], total_hits[19435], types[], stats[], search_type[QUERY_THEN_FETCH], total_shards[1], source[{"query":{"match_all":{"boost":1.0}}}],
+[2018-06-29T10:06:14,943][INFO ][index.search.slowlog.fetch] [v_VJhjV] [metricbeat-6.3.0-2018.06.26][0] took[10.8ms], took_millis[10], total_hits[19435], types[], stats[], search_type[QUERY_THEN_FETCH], total_shards[1], source[{"query":{"match_all":{"boost":1.0}}}],
+[2018-06-29T09:01:01,821][INFO ][index.search.slowlog.query] [v_VJhjV] [metricbeat-6.3.0-2018.06.26][0] took[124.3ms], took_millis[124], total_hits[0], types[], stats[], search_type[QUERY_THEN_FETCH], total_shards[1], source[{"size":500,"query":{"match_none":{"boost":1.0}},"version":true,"_source":{"includes":[],"excludes":[]},"stored_fields":"*","docvalue_fields":["@timestamp","ceph.monitor_health.last_updated","docker.container.created","docker.healthcheck.event.end_date","docker.healthcheck.event.start_date","docker.image.created","kubernetes.container.start_time","kubernetes.event.metadata.timestamp.created","kubernetes.node.start_time","kubernetes.pod.start_time","kubernetes.system.start_time","mongodb.status.background_flushing.last_finished","mongodb.status.local_time","php_fpm.pool.start_time","postgresql.activity.backend_start","postgresql.activity.query_start","postgresql.activity.state_change","postgresql.activity.transaction_start","postgresql.bgwriter.stats_reset","postgresql.database.stats_reset","system.process.cpu.start_time"],"script_fields":{},"sort":[{"@timestamp":{"order":"desc","unmapped_type":"boolean"}}],"aggregations":{"2":{"date_histogram":{"field":"@timestamp","time_zone":"Europe/Berlin","interval":"30s","offset":0,"order":{"_key":"asc"},"keyed":false,"min_doc_count":1}}},"highlight":{"pre_tags":["@kibana-highlighted-field@"],"post_tags":["@/kibana-highlighted-field@"],"fragment_size":2147483647,"fields":{"*":{}}}}],
+[2018-06-29T09:01:01,827][INFO ][index.search.slowlog.fetch] [v_VJhjV] [metricbeat-6.3.0-2018.06.26][0] took[7.2ms], took_millis[7], total_hits[0], types[], stats[], search_type[QUERY_THEN_FETCH], total_shards[1], source[{"size":500,"query":{"match_none":{"boost":1.0}},"version":true,"_source":{"includes":[],"excludes":[]},"stored_fields":"*","docvalue_fields":["@timestamp","ceph.monitor_health.last_updated","docker.container.created","docker.healthcheck.event.end_date","docker.healthcheck.event.start_date","docker.image.created","kubernetes.container.start_time","kubernetes.event.metadata.timestamp.created","kubernetes.node.start_time","kubernetes.pod.start_time","kubernetes.system.start_time","mongodb.status.background_flushing.last_finished","mongodb.status.local_time","php_fpm.pool.start_time","postgresql.activity.backend_start","postgresql.activity.query_start","postgresql.activity.state_change","postgresql.activity.transaction_start","postgresql.bgwriter.stats_reset","postgresql.database.stats_reset","system.process.cpu.start_time"],"script_fields":{},"sort":[{"@timestamp":{"order":"desc","unmapped_type":"boolean"}}],"aggregations":{"2":{"date_histogram":{"field":"@timestamp","time_zone":"Europe/Berlin","interval":"30s","offset":0,"order":{"_key":"asc"},"keyed":false,"min_doc_count":1}}},"highlight":{"pre_tags":["@kibana-highlighted-field@"],"post_tags":["@/kibana-highlighted-field@"],"fragment_size":2147483647,"fields":{"*":{}}}}],