Introduce service.type field for all Filebeat modules (#10042)

Based on ECS the `service.type` field is introduce for all Filebeat modules. By default `service.type` is set to the module name. If the service.type is not the module name the `servic.type` config can be used in a module to overwrite the field. The `service.name` fields in the Elasticsearch module were removed as `service.name` could be used for example for the cluster name in this context. `service.type` is the field that always stays the same.
elastic · Jan 15, 2019 · 8a04a80 · 8a04a80
1 parent cda2347
commit 8a04a80
Show file tree

Hide file tree

Showing 56 changed files with 435 additions and 209 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -34,6 +34,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Rename a few `mysql.*` fields to map to ECS. {pull}10008[10008]
 - Rename a few `nginx.error.*` fields to map to ECS. {pull}10007[10007]
 - Filesets with multiple ingest pipelines added in {pull}8914[8914] only work with Elasticsearch >= 6.5.0 {pull}10001[10001]
+- Remove service.name from Elastcsearch module. Replace by service.type. {pull}10042[10042]
 - Add grok pattern to support redis 5.0.3 log timestamp. {issue}9819[9819] {pull}10033[10033]
 
 *Heartbeat*
@@ -118,6 +119,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Support mysql 5.7.22 slowlog starting with time information. {issue}7892[7892] {pull}9647[9647]
 - Add support for ssl_request_log in apache2 module. {issue}8088[8088] {pull}9833[9833]
 - Add support for iis 7.5 log format. {issue}9753[9753] {pull}9967[9967]
+- Add service.type field to all Modules. By default the field is set with the module name. It can be overwritten with `service.type` config. {pull}10042[10042]
 
 *Heartbeat*
 

diff --git a/filebeat/channel/factory.go b/filebeat/channel/factory.go
@@ -48,7 +48,8 @@ type inputOutletConfig struct {
 	Processors           processors.PluginConfig `config:"processors"`
 
 	// implicit event fields
-	Type string `config:"type"` // input.type
+	Type        string `config:"type"`         // input.type
+	ServiceType string `config:"service.type"` // service.type
 
 	// hidden filebeat modules settings
 	Module  string `config:"_module_name"`  // hidden setting
@@ -112,10 +113,13 @@ func (f *OutletFactory) Create(p beat.Pipeline, cfg *common.Config, dynFields *c
 	if config.Fileset != "" {
 		fields.Put("fileset.name", config.Fileset)
 	}
+	if config.ServiceType != "" {
+		fields.Put("service.type", config.ServiceType)
+	} else {
+		fields.Put("service.type", config.Module)
+	}
 	if config.Type != "" {
-		fields["input"] = common.MapStr{
-			"type": config.Type,
-		}
+		fields.Put("input.type", config.Type)
 	}
 
 	client, err := p.ConnectWith(beat.ClientConfig{

diff --git a/filebeat/module/apache/access/test/ssl-request.log-expected.json b/filebeat/module/apache/access/test/ssl-request.log-expected.json
@@ -13,6 +13,7 @@
         "http.version": "1.1",
         "input.type": "log",
         "log.offset": 0,
+        "service.type": "apache",
         "source.ip": "172.30.0.119",
         "url.original": "/nagiosxi/ajaxhelper.php?cmd=getxicoreajax&amp;opts=%7B%22func%22%3A%22get_admin_tasks_html%22%2C%22args%22%3A%22%22%7D&amp;nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21"
     }

diff --git a/filebeat/module/apache/access/test/test.log-expected.json b/filebeat/module/apache/access/test/test.log-expected.json
@@ -12,6 +12,7 @@
         "http.version": "1.1",
         "input.type": "log",
         "log.offset": 0,
+        "service.type": "apache",
         "source.ip": "::1",
         "url.original": "/favicon.ico",
         "user.name": "-"
@@ -30,6 +31,7 @@
         "http.version": "1.1",
         "input.type": "log",
         "log.offset": 73,
+        "service.type": "apache",
         "source.ip": "192.168.33.1",
         "url.original": "/hello",
         "user.name": "-",
@@ -53,6 +55,7 @@
         "http.response.status_code": 408,
         "input.type": "log",
         "log.offset": 238,
+        "service.type": "apache",
         "source.ip": "::1",
         "user.name": "-"
     },
@@ -70,6 +73,7 @@
         "http.version": "1.1",
         "input.type": "log",
         "log.offset": 285,
+        "service.type": "apache",
         "source.ip": "172.17.0.1",
         "url.original": "/stringpatch",
         "user.name": "-",
@@ -96,6 +100,7 @@
         "http.version": "1.1",
         "input.type": "log",
         "log.offset": 443,
+        "service.type": "apache",
         "source.domain": "monitoring-server",
         "url.original": "/status",
         "user.name": "-",

diff --git a/filebeat/module/apache/error/test/test.log-expected.json b/filebeat/module/apache/error/test/test.log-expected.json
@@ -9,6 +9,7 @@
         "log.level": "error",
         "log.offset": 0,
         "message": "File does not exist: /var/www/favicon.ico",
+        "service.type": "apache",
         "source.address": "192.168.33.1"
     },
     {
@@ -22,7 +23,8 @@
         "log.level": "notice",
         "log.offset": 99,
         "message": "AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'",
-        "process.pid": 11379
+        "process.pid": 11379,
+        "service.type": "apache"
     },
     {
         "@timestamp": "2011-09-09T10:42:29.902Z",
@@ -37,6 +39,7 @@
         "message": "File does not exist: /usr/local/apache2/htdocs/favicon.ico",
         "process.pid": 35708,
         "process.thread.id": 4328636416,
+        "service.type": "apache",
         "source.address": "72.15.99.187"
     }
 ]
diff --git a/filebeat/module/auditd/log/test/test.log-expected.json b/filebeat/module/auditd/log/test/test.log-expected.json
@@ -16,7 +16,8 @@
         "event.module": "auditd",
         "fileset.name": "log",
         "input.type": "log",
-        "log.offset": 0
+        "log.offset": 0,
+        "service.type": "auditd"
     },
     {
         "@timestamp": "2017-01-31T20:17:14.891Z",
@@ -51,6 +52,7 @@
         "event.module": "auditd",
         "fileset.name": "log",
         "input.type": "log",
-        "log.offset": 174
+        "log.offset": 174,
+        "service.type": "auditd"
     }
 ]
diff --git a/filebeat/module/elasticsearch/audit/config/audit.yml b/filebeat/module/elasticsearch/audit/config/audit.yml
@@ -9,7 +9,3 @@ exclude_files: [".gz$"]
 processors:
 - add_locale: ~
 {{ end }}
-
-fields:
-  service.name: "elasticsearch"
-fields_under_root: true
diff --git a/filebeat/module/elasticsearch/audit/test/test.log-expected.json b/filebeat/module/elasticsearch/audit/test/test.log-expected.json
@@ -13,7 +13,7 @@
         "input.type": "log",
         "log.offset": 0,
         "message": "[2018-06-19T05:16:15,549] [rest] [authentication_failed]        origin_address=[147.107.128.77], principal=[i030648], uri=[/_xpack/security/_authenticate]",
-        "service.name": "elasticsearch"
+        "service.type": "elasticsearch"
     },
     {
         "@timestamp": "2018-06-19T05:07:52.304Z",
@@ -30,7 +30,7 @@
         "input.type": "log",
         "log.offset": 155,
         "message": "[2018-06-19T05:07:52,304] [v_VJhjV] [rest] [authentication_failed]\torigin_address=[172.22.0.3], principal=[rado], uri=[/_xpack/security/_authenticate]",
-        "service.name": "elasticsearch"
+        "service.type": "elasticsearch"
     },
     {
         "@timestamp": "2018-06-19T05:00:15.778Z",
@@ -48,7 +48,7 @@
         "input.type": "log",
         "log.offset": 306,
         "message": "[2018-06-19T05:00:15,778] [transport] [access_granted]  origin_type=[local_node], origin_address=[192.168.1.165], principal=[_xpack_security], action=[indices:data/read/scroll/clear], request=[ClearScrollRequest]",
-        "service.name": "elasticsearch"
+        "service.type": "elasticsearch"
     },
     {
         "@timestamp": "2018-06-19T05:07:45.544Z",
@@ -64,7 +64,7 @@
         "input.type": "log",
         "log.offset": 519,
         "message": "[2018-06-19T05:07:45,544] [v_VJhjV] [rest] [anonymous_access_denied]\torigin_address=[172.22.0.3], uri=[/_xpack/security/_authenticate]",
-        "service.name": "elasticsearch"
+        "service.type": "elasticsearch"
     },
     {
         "@timestamp": "2018-06-19T05:26:27.268Z",
@@ -80,7 +80,7 @@
         "input.type": "log",
         "log.offset": 654,
         "message": "[2018-06-19T05:26:27,268] [rest] [authentication_failed]\torigin_address=[147.107.128.77], principal=[N078801], uri=[/_xpack/security/_authenticate]",
-        "service.name": "elasticsearch"
+        "service.type": "elasticsearch"
     },
     {
         "@timestamp": "2018-06-19T05:55:26.898Z",
@@ -98,7 +98,7 @@
         "input.type": "log",
         "log.offset": 802,
         "message": "[2018-06-19T05:55:26,898] [transport] [access_denied]\torigin_type=[rest], origin_address=[147.107.128.77], principal=[_anonymous], action=[cluster:monitor/main], request=[MainRequest]",
-        "service.name": "elasticsearch"
+        "service.type": "elasticsearch"
     },
     {
         "@timestamp": "2018-06-19T05:24:15.190Z",
@@ -116,6 +116,6 @@
         "input.type": "log",
         "log.offset": 986,
         "message": "[2018-06-19T05:24:15,190] [v_VJhjV] [rest] [authentication_failed]\torigin_address=[172.18.0.3], principal=[elastic], uri=[/_nodes?filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip], request_body=[body]",
-        "service.name": "elasticsearch"
+        "service.type": "elasticsearch"
     }
 ]
diff --git a/filebeat/module/elasticsearch/deprecation/config/log.yml b/filebeat/module/elasticsearch/deprecation/config/log.yml
@@ -14,7 +14,3 @@ processors:
 - add_locale: ~
 {{ end }}
 
-fields:
-  service.name: "elasticsearch"
-
-fields_under_root: true
diff --git a/filebeat/module/elasticsearch/deprecation/test/elasticsearch_deprecation.log-expected.json b/filebeat/module/elasticsearch/deprecation/test/elasticsearch_deprecation.log-expected.json
@@ -10,7 +10,7 @@
         "log.level": "WARN",
         "log.offset": 0,
         "message": "Deprecated field [template] used, replaced by [index_patterns]",
-        "service.name": "elasticsearch"
+        "service.type": "elasticsearch"
     },
     {
         "@timestamp": "2018-04-23T16:40:13.862Z",
@@ -23,7 +23,7 @@
         "log.level": "WARN",
         "log.offset": 137,
         "message": "Deprecated field [template] used, replaced by [index_patterns]",
-        "service.name": "elasticsearch"
+        "service.type": "elasticsearch"
     },
     {
         "@timestamp": "2018-04-23T16:40:14.792Z",
@@ -36,7 +36,7 @@
         "log.level": "WARN",
         "log.offset": 274,
         "message": "Deprecated field [template] used, replaced by [index_patterns]",
-        "service.name": "elasticsearch"
+        "service.type": "elasticsearch"
     },
     {
         "@timestamp": "2018-04-23T16:40:15.127Z",
@@ -49,6 +49,6 @@
         "log.level": "WARN",
         "log.offset": 411,
         "message": "Deprecated field [template] used, replaced by [index_patterns]",
-        "service.name": "elasticsearch"
+        "service.type": "elasticsearch"
     }
 ]
diff --git a/...t/module/elasticsearch/deprecation/test/other_elasticsearch_deprecation.log-expected.json b/...t/module/elasticsearch/deprecation/test/other_elasticsearch_deprecation.log-expected.json
@@ -10,7 +10,7 @@
         "log.level": "WARN",
         "log.offset": 0,
         "message": "Deprecated field [inline] used, expected [source] instead",
-        "service.name": "elasticsearch"
+        "service.type": "elasticsearch"
     },
     {
         "@timestamp": "2017-11-30T13:38:16.941Z",
@@ -23,7 +23,7 @@
         "log.level": "WARN",
         "log.offset": 118,
         "message": "Deprecated field [inline] used, expected [source] instead",
-        "service.name": "elasticsearch"
+        "service.type": "elasticsearch"
     },
     {
         "@timestamp": "2017-11-30T13:39:28.986Z",
@@ -36,7 +36,7 @@
         "log.level": "WARN",
         "log.offset": 236,
         "message": "Fielddata access on the _uid field is deprecated, use _id instead",
-        "service.name": "elasticsearch"
+        "service.type": "elasticsearch"
     },
     {
         "@timestamp": "2017-11-30T13:39:36.339Z",
@@ -49,7 +49,7 @@
         "log.level": "WARN",
         "log.offset": 362,
         "message": "Fielddata access on the _uid field is deprecated, use _id instead",
-        "service.name": "elasticsearch"
+        "service.type": "elasticsearch"
     },
     {
         "@timestamp": "2017-11-30T13:40:49.540Z",
@@ -62,7 +62,7 @@
         "log.level": "WARN",
         "log.offset": 488,
         "message": "Fielddata access on the _uid field is deprecated, use _id instead",
-        "service.name": "elasticsearch"
+        "service.type": "elasticsearch"
     },
     {
         "@timestamp": "2017-11-30T14:08:37.413Z",
@@ -75,7 +75,7 @@
         "log.level": "WARN",
         "log.offset": 614,
         "message": "Fielddata access on the _uid field is deprecated, use _id instead",
-        "service.name": "elasticsearch"
+        "service.type": "elasticsearch"
     },
     {
         "@timestamp": "2017-11-30T14:08:37.413Z",
@@ -88,7 +88,7 @@
         "log.level": "WARN",
         "log.offset": 740,
         "message": "Fielddata access on the _uid field is deprecated, use _id instead",
-        "service.name": "elasticsearch"
+        "service.type": "elasticsearch"
     },
     {
         "@timestamp": "2017-11-30T14:08:46.006Z",
@@ -101,7 +101,7 @@
         "log.level": "WARN",
         "log.offset": 866,
         "message": "Fielddata access on the _uid field is deprecated, use _id instead",
-        "service.name": "elasticsearch"
+        "service.type": "elasticsearch"
     },
     {
         "@timestamp": "2017-11-30T14:08:46.006Z",
@@ -114,7 +114,7 @@
         "log.level": "WARN",
         "log.offset": 992,
         "message": "Fielddata access on the _uid field is deprecated, use _id instead",
-        "service.name": "elasticsearch"
+        "service.type": "elasticsearch"
     },
     {
         "@timestamp": "2017-12-01T14:05:54.017Z",
@@ -127,7 +127,7 @@
         "log.level": "WARN",
         "log.offset": 1118,
         "message": "[_all] is deprecated in 6.0+ and will be removed in 7.0. As a replacement, you can use [copy_to] on mapping fields to create your own catch all field.",
-        "service.name": "elasticsearch"
+        "service.type": "elasticsearch"
     },
     {
         "@timestamp": "2017-12-01T14:05:54.019Z",
@@ -140,7 +140,7 @@
         "log.level": "WARN",
         "log.offset": 1329,
         "message": "[_all] is deprecated in 6.0+ and will be removed in 7.0. As a replacement, you can use [copy_to] on mapping fields to create your own catch all field.",
-        "service.name": "elasticsearch"
+        "service.type": "elasticsearch"
     },
     {
         "@timestamp": "2017-12-01T14:06:52.059Z",
@@ -153,7 +153,7 @@
         "log.level": "WARN",
         "log.offset": 1540,
         "message": "[_all] is deprecated in 6.0+ and will be removed in 7.0. As a replacement, you can use [copy_to] on mapping fields to create your own catch all field.",
-        "service.name": "elasticsearch"
+        "service.type": "elasticsearch"
     },
     {
         "@timestamp": "2017-12-01T14:46:10.428Z",
@@ -166,7 +166,7 @@
         "log.level": "WARN",
         "log.offset": 1751,
         "message": "Deprecated aggregation order key [_term] used, replaced by [_key]",
-        "service.name": "elasticsearch"
+        "service.type": "elasticsearch"
     },
     {
         "@timestamp": "2017-12-04T16:17:18.271Z",
@@ -179,7 +179,7 @@
         "log.level": "WARN",
         "log.offset": 1882,
         "message": "Deprecated field [template] used, replaced by [index_patterns]",
-        "service.name": "elasticsearch"
+        "service.type": "elasticsearch"
     },
     {
         "@timestamp": "2017-12-04T16:17:18.282Z",
@@ -192,7 +192,7 @@
         "log.level": "WARN",
         "log.offset": 2019,
         "message": "[_default_] mapping is deprecated since it is not useful anymore now that indexes cannot have more than one type",
-        "service.name": "elasticsearch"
+        "service.type": "elasticsearch"
     },
     {
         "@timestamp": "2017-12-04T16:20:43.248Z",
@@ -205,6 +205,6 @@
         "log.level": "WARN",
         "log.offset": 2192,
         "message": "[_default_] mapping is deprecated since it is not useful anymore now that indexes cannot have more than one type",
-        "service.name": "elasticsearch"
+        "service.type": "elasticsearch"
     }
 ]