Remove agent.hostname field from libbeat (elastic#18328)

agent.hostname is not part of ECS so this removes the field. Instead the
agent.name field can be used for this purpose and it it part of ECS.
This adds an alias pointing from agent.hostname to agent.name.

Closes elastic#16377

CHANGELOG.next.asciidoc

@@ -15,6 +15,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Change aws_elb autodiscover provider field name from elb_listener.* to aws.elb.*. {issue}16219[16219] {pull}16402{16402}
 - Remove `AddDockerMetadata` and `AddKubernetesMetadata` processors from the `script` processor. They can still be used as normal processors in the configuration. {issue}16349[16349] {pull}16514[16514]
 - Introduce APM libbeat instrumentation, active when running the beat with ELASTIC_APM_ACTIVE=true. {pull}17938[17938]
+- Remove the non-ECS `agent.hostname` field. Use the `agent.name` or `agent.id` fields for an identifier. {issue}16377[16377] {pull}18328[18328]
 
 *Auditbeat*
 

auditbeat/docs/fields.asciidoc

@@ -2458,10 +2458,12 @@ Contains common beat fields available in all event types.
 *`agent.hostname`*::
 +
 --
-Deprecated - use agent.name or agent.id to identify an agent. Hostname of the agent.
+Deprecated - use agent.name or agent.id to identify an agent.
 
 
-type: keyword
+type: alias
+
+alias to: agent.name
 
 --
 
@@ -2498,7 +2500,7 @@ alias to: host.name
 --
 type: alias
 
-alias to: agent.hostname
+alias to: agent.name
 
 --
 

auditbeat/module/auditd/_meta/kibana/7/dashboard/auditbeat-kernel-executions.json

@@ -164,7 +164,7 @@
         {
             "attributes": {
                 "columns": [
-                    "agent.hostname", 
+                    "agent.name", 
                     "process.args", 
                     "auditd.summary.actor.primary", 
                     "auditd.summary.actor.secondary", 

auditbeat/module/auditd/_meta/kibana/7/dashboard/auditbeat-kernel-overview.json

@@ -146,7 +146,7 @@
         {
             "attributes": {
                 "columns": [
-                    "agent.hostname", 
+                    "agent.name", 
                     "auditd.summary.actor.primary", 
                     "auditd.summary.actor.secondary", 
                     "event.action", 

auditbeat/module/auditd/_meta/kibana/7/dashboard/auditbeat-kernel-sockets.json

@@ -451,7 +451,7 @@
         {
             "attributes": {
                 "columns": [
-                    "agent.hostname", 
+                    "agent.name", 
                     "auditd.summary.how", 
                     "auditd.summary.object.primary", 
                     "auditd.summary.object.secondary", 
@@ -566,7 +566,7 @@
         {
             "attributes": {
                 "columns": [
-                    "agent.hostname", 
+                    "agent.name", 
                     "auditd.summary.how", 
                     "auditd.summary.object.primary", 
                     "auditd.summary.object.secondary", 
@@ -673,7 +673,7 @@
         {
             "attributes": {
                 "columns": [
-                    "agent.hostname", 
+                    "agent.name", 
                     "auditd.summary.how", 
                     "auditd.summary.object.primary", 
                     "auditd.summary.object.secondary", 

filebeat/docs/fields.asciidoc

@@ -3113,10 +3113,12 @@ Contains common beat fields available in all event types.
 *`agent.hostname`*::
 +
 --
-Deprecated - use agent.name or agent.id to identify an agent. Hostname of the agent.
+Deprecated - use agent.name or agent.id to identify an agent.
 
 
-type: keyword
+type: alias
+
+alias to: agent.name
 
 --
 
@@ -3153,7 +3155,7 @@ alias to: host.name
 --
 type: alias
 
-alias to: agent.hostname
+alias to: agent.name
 
 --
 

filebeat/module/osquery/_meta/kibana/7/dashboard/osquery-rootkit.json

@@ -113,7 +113,7 @@
                             "id": "1", 
                             "params": {
                                 "customLabel": "Hosts", 
-                                "field": "agent.hostname"
+                                "field": "agent.name"
                             }, 
                             "schema": "metric", 
                             "type": "cardinality"
@@ -185,7 +185,7 @@
                 "columns": [
                     "osquery.result.name", 
                     "osquery.result.columns.path", 
-                    "agent.hostname"
+                    "agent.name"
                 ], 
                 "description": "", 
                 "hits": 0, 

filebeat/module/santa/_meta/kibana/7/dashboard/filebeat-santa-log-overview.json

@@ -269,7 +269,7 @@
                             "id": "1",
                             "params": {
                                 "customLabel": "Hosts Reporting",
-                                "field": "agent.hostname"
+                                "field": "agent.name"
                             },
                             "schema": "metric",
                             "type": "cardinality"
@@ -373,7 +373,7 @@
         {
             "attributes": {
                 "columns": [
-                    "agent.hostname",
+                    "agent.name",
                     "process.executable",
                     "user.name",
                     "certificate.common_name"

filebeat/tests/system/test_fields.py

@@ -60,10 +60,9 @@ def test_custom_fields_under_root(self):
         assert doc["timestamp"] == 2
         assert "fields" not in doc
 
-    def test_beat_fields(self):
+    def test_agent_name_custom(self):
         """
-        Checks that it's possible to set a custom shipper name. Also
-        tests that agent.hostname has values.
+        Checks that it's possible to set a custom agent name.
         """
         self.render_config_template(
             path=os.path.abspath(self.working_dir) + "/test.log",
@@ -81,5 +80,23 @@ def test_beat_fields(self):
         doc = output[0]
         assert doc["host.name"] == "testShipperName"
         assert doc["agent.name"] == "testShipperName"
-        assert doc["agent.hostname"] == socket.gethostname()
         assert "fields" not in doc
+
+    def test_agent_name_default(self):
+        """
+        Checks that agent.name defaults to the hostname.
+        """
+        self.render_config_template(
+            path=os.path.abspath(self.working_dir) + "/test.log",
+        )
+
+        with open(self.working_dir + "/test.log", "w") as f:
+            f.write("test message\n")
+
+        filebeat = self.start_beat()
+        self.wait_until(lambda: self.output_has(lines=1))
+        filebeat.check_kill_and_wait()
+
+        output = self.read_output()
+        doc = output[0]
+        assert doc["agent.name"] == socket.gethostname()

filebeat/tests/system/test_modules.py

@@ -214,7 +214,7 @@ def _test_expected_events(self, test_file, objects):
 
 def clean_keys(obj):
     # These keys are host dependent
-    host_keys = ["host.name", "agent.name", "agent.hostname", "agent.type", "agent.ephemeral_id", "agent.id"]
+    host_keys = ["host.name", "agent.name", "agent.type", "agent.ephemeral_id", "agent.id"]
     # The create timestamps area always new
     time_keys = ["event.created"]
     # source path and agent.version can be different for each run

heartbeat/docs/fields.asciidoc

@@ -40,10 +40,12 @@ Contains common beat fields available in all event types.
 *`agent.hostname`*::
 +
 --
-Deprecated - use agent.name or agent.id to identify an agent. Hostname of the agent.
+Deprecated - use agent.name or agent.id to identify an agent.
 
 
-type: keyword
+type: alias
+
+alias to: agent.name
 
 --
 
@@ -80,7 +82,7 @@ alias to: host.name
 --
 type: alias
 
-alias to: agent.hostname
+alias to: agent.name
 
 --
 

journalbeat/docs/fields.asciidoc

@@ -33,10 +33,12 @@ Contains common beat fields available in all event types.
 *`agent.hostname`*::
 +
 --
-Deprecated - use agent.name or agent.id to identify an agent. Hostname of the agent.
+Deprecated - use agent.name or agent.id to identify an agent.
 
 
-type: keyword
+type: alias
+
+alias to: agent.name
 
 --
 
@@ -73,7 +75,7 @@ alias to: host.name
 --
 type: alias
 
-alias to: agent.hostname
+alias to: agent.name
 
 --
 

libbeat/_meta/fields.common.yml

@@ -5,10 +5,10 @@
     Contains common beat fields available in all event types.
   fields:
     - name: agent.hostname
-      type: keyword
+      type: alias
+      path: agent.name
       description: >
         Deprecated - use agent.name or agent.id to identify an agent.
-        Hostname of the agent.
 
     - name: beat.timezone
       type: alias
@@ -28,7 +28,7 @@
 
     - name: beat.hostname
       type: alias
-      path: agent.hostname
+      path: agent.name
       migration: true
 
     - name: timeseries.instance

libbeat/publisher/processing/default.go

@@ -149,9 +149,6 @@ func WithAgentMeta() modifier {
 			"name":         info.Hostname,
 			"type":         info.Beat,
 			"version":      info.Version,
-			// hostname is deprecated. To be removed for 8.0. It's not in ECS.
-			// See https://github.com/elastic/beats/issues/16377.
-			"hostname": info.Hostname,
 		}
 		if info.Name != "" {
 			metadata["name"] = info.Name

libbeat/publisher/processing/default_test.go

@@ -183,7 +183,6 @@ func TestProcessorsConfigs(t *testing.T) {
 				},
 				"agent": common.MapStr{
 					"ephemeral_id": "123e4567-e89b-12d3-a456-426655440000",
-					"hostname":     "test.host.name",
 					"name":         "test.host.name",
 					"id":           "123e4567-e89b-12d3-a456-426655440001",
 					"type":         "test",
@@ -210,7 +209,6 @@ func TestProcessorsConfigs(t *testing.T) {
 				},
 				"agent": common.MapStr{
 					"ephemeral_id": "123e4567-e89b-12d3-a456-426655440000",
-					"hostname":     "test.host.name",
 					"name":         "other.test.host.name",
 					"id":           "123e4567-e89b-12d3-a456-426655440001",
 					"type":         "test",

libbeat/tests/system/beat/beat.py

@@ -17,7 +17,7 @@
 
 
 BEAT_REQUIRED_FIELDS = ["@timestamp",
-                        "agent.type", "agent.hostname", "agent.version"]
+                        "agent.type", "agent.name", "agent.version"]
 
 INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False)
 

metricbeat/docs/fields.asciidoc

@@ -4622,10 +4622,12 @@ Contains common beat fields available in all event types.
 *`agent.hostname`*::
 +
 --
-Deprecated - use agent.name or agent.id to identify an agent. Hostname of the agent.
+Deprecated - use agent.name or agent.id to identify an agent.
 
 
-type: keyword
+type: alias
+
+alias to: agent.name
 
 --
 
@@ -4662,7 +4664,7 @@ alias to: host.name
 --
 type: alias
 
-alias to: agent.hostname
+alias to: agent.name
 
 --
 

metricbeat/module/docker/_meta/kibana/7/dashboard/Metricbeat-docker-overview.json

@@ -249,7 +249,7 @@
                             "id": "2", 
                             "params": {
                                 "customLabel": "Hosts", 
-                                "field": "agent.hostname", 
+                                "field": "agent.name", 
                                 "order": "desc", 
                                 "orderBy": "1", 
                                 "size": 5

packetbeat/docs/fields.asciidoc

@@ -452,10 +452,12 @@ Contains common beat fields available in all event types.
 *`agent.hostname`*::
 +
 --
-Deprecated - use agent.name or agent.id to identify an agent. Hostname of the agent.
+Deprecated - use agent.name or agent.id to identify an agent.
 
 
-type: keyword
+type: alias
+
+alias to: agent.name
 
 --
 
@@ -492,7 +494,7 @@ alias to: host.name
 --
 type: alias
 
-alias to: agent.hostname
+alias to: agent.name
 
 --
 

packetbeat/tests/system/packetbeat.py

@@ -9,15 +9,15 @@
 from beat.beat import Proc
 
 TRANS_REQUIRED_FIELDS = ["@timestamp", "type", "status",
-                         "agent.type", "agent.hostname", "agent.version",
+                         "agent.type", "agent.name", "agent.version",
                          "event.kind", "event.category", "event.dataset", "event.start",
                          "source.ip", "destination.ip",
                          "client.ip", "server.ip",
                          "network.type", "network.transport", "network.community_id",
                          ]
 
 FLOWS_REQUIRED_FIELDS = ["@timestamp", "type",
-                         "agent.type", "agent.hostname", "agent.version",
+                         "agent.type", "agent.name", "agent.version",
                          "event.kind", "event.category", "event.dataset", "event.action", "event.start", "event.end", "event.duration",
                          "source.ip", "destination.ip",
                          "flow.id",

packetbeat/tests/system/test_0099_golden_files.py

@@ -64,7 +64,6 @@ def clean_keys(obj):
     keys = [
         "@metadata.version",
         "agent.ephemeral_id",
-        "agent.hostname",
         "agent.id",
         "agent.name",
         "agent.type",

script/ecs-migration-8x.yml

@@ -0,0 +1,17 @@
+# The ECS migration file contains the information about all the Beats fields
+# which were changed between 7.x and 8.0.
+#
+# This file will be used to generate breaking change documentation and migrate
+# Kibana dashboards. See script/renamed_fields.py and script/kibana-migration.py.
+#
+# The format of the file is as following:
+#
+# - from: <original-name-in-7x> # Field name in 7.x.
+#   to: <new-name-in-8x>        # Field name in 8.0.
+#   alias: <true|false>         # True if a migration alias exists in fields.yml.
+#   beat: <beatname>            # Omit for libbeat fields.
+
+## Libbeat
+- from: agent.hostname
+  to: agent.name
+  alias: true