Move cloudfoundry tags with metadata to common metadata fields

CHANGELOG.next.asciidoc

@@ -26,6 +26,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Update to Golang 1.12.1. {pull}11330[11330]
 - Disable Alibaba Cloud and Tencent Cloud metadata providers by default. {pull}13812[12812]
 - API address is a required setting in `add_cloudfoundry_metadata`. {pull}21759[21759]
+- Cloud Foundry tags including metadata are moved to common metadata fields. {pull}22150[22150]
 
 *Auditbeat*
 

x-pack/libbeat/common/cloudfoundry/events.go

@@ -490,21 +490,45 @@ func envelopMap(evt Event) common.MapStr {
 }
 
 func baseMap(evt Event) common.MapStr {
-	return common.MapStr{
-		"cloudfoundry": common.MapStr{
-			"type":     evt.String(),
-			"envelope": envelopMap(evt),
-			"tags":     dedotedTags(evt.Tags()),
-		},
+	tags, meta := tagsToMeta(evt.Tags())
+	cf := common.MapStr{
+		"type":     evt.String(),
+		"envelope": envelopMap(evt),
+	}
+	if len(tags) > 0 {
+		cf["tags"] = tags
+	}
+	result := common.MapStr{
+		"cloudfoundry": cf,
 	}
+	if len(meta) > 0 {
+		result.DeepUpdate(meta)
+	}
+	return result
 }
 
-func dedotedTags(tags map[string]string) common.MapStr {
-	result := common.MapStr{}
-	for name, value := range tags {
-		result[common.DeDot(name)] = value
+func tagsToMeta(eventTags map[string]string) (tags common.MapStr, meta common.MapStr) {
+	tags = common.MapStr{}
+	meta = common.MapStr{}
+	for name, value := range eventTags {
+		switch name {
+		case "app_id":
+			meta.Put("cloudfoundry.app.id", value)
+		case "app_name":
+			meta.Put("cloudfoundry.app.name", value)
+		case "space_id":
+			meta.Put("cloudfoundry.space.id", value)
+		case "space_name":
+			meta.Put("cloudfoundry.space.name", value)
+		case "organization_id":
+			meta.Put("cloudfoundry.org.id", value)
+		case "organization_name":
+			meta.Put("cloudfoundry.org.name", value)
+		default:
+			tags[common.DeDot(name)] = value
+		}
 	}
-	return result
+	return tags, meta
 }
 
 func baseMapWithApp(evt EventWithAppID) common.MapStr {

x-pack/libbeat/common/cloudfoundry/events_test.go

@@ -382,6 +382,87 @@ func TestEventError(t *testing.T) {
 	}, evt.ToFields())
 }
 
+func TestEventTagsWithMetadata(t *testing.T) {
+	eventType := events.Envelope_LogMessage
+	message := "log message"
+	messageType := events.LogMessage_OUT
+	timestamp := int64(1587469726082)
+	appID := "f47ac10b-58cc-4372-a567-0e02b2c3d479"
+	sourceType := "source_type"
+	sourceInstance := "source_instance"
+	cfEvt := makeEnvelope(&eventType)
+	tags := map[string]string{
+		"app_id":            appID,
+		"app_name":          "some-app",
+		"space_id":          "e1114e92-155c-11eb-ada9-27b81025a657",
+		"space_name":        "some-space",
+		"organization_id":   "baeef1ba-155c-11eb-a1af-8f14964c35d2",
+		"organization_name": "some-org",
+		"custom_tag":        "foo",
+	}
+	cfEvt.Tags = tags
+	cfEvt.LogMessage = &events.LogMessage{
+		Message:        []byte(message),
+		MessageType:    &messageType,
+		Timestamp:      &timestamp,
+		AppId:          &appID,
+		SourceType:     &sourceType,
+		SourceInstance: &sourceInstance,
+	}
+	evt := newEventLog(cfEvt)
+
+	assert.Equal(t, EventTypeLog, evt.EventType())
+	assert.Equal(t, "log", evt.String())
+	assert.Equal(t, "origin", evt.Origin())
+	assert.Equal(t, time.Unix(0, 1587469726082), evt.Timestamp())
+	assert.Equal(t, "deployment", evt.Deployment())
+	assert.Equal(t, "job", evt.Job())
+	assert.Equal(t, "index", evt.Index())
+	assert.Equal(t, "ip", evt.IP())
+	assert.Equal(t, tags, evt.Tags())
+	assert.Equal(t, "f47ac10b-58cc-4372-a567-0e02b2c3d479", evt.AppGuid())
+	assert.Equal(t, "log message", evt.Message())
+	assert.Equal(t, EventLogMessageTypeStdout, evt.MessageType())
+	assert.Equal(t, "source_type", evt.SourceType())
+	assert.Equal(t, "source_instance", evt.SourceID())
+
+	assert.Equal(t, common.MapStr{
+		"cloudfoundry": common.MapStr{
+			"type": "log",
+			"log": common.MapStr{
+				"source": common.MapStr{
+					"instance": evt.SourceID(),
+					"type":     evt.SourceType(),
+				},
+			},
+			"envelope": common.MapStr{
+				"origin":     "origin",
+				"deployment": "deployment",
+				"ip":         "ip",
+				"job":        "job",
+				"index":      "index",
+			},
+			"app": common.MapStr{
+				"id":   "f47ac10b-58cc-4372-a567-0e02b2c3d479",
+				"name": "some-app",
+			},
+			"space": common.MapStr{
+				"id":   "e1114e92-155c-11eb-ada9-27b81025a657",
+				"name": "some-space",
+			},
+			"org": common.MapStr{
+				"id":   "baeef1ba-155c-11eb-a1af-8f14964c35d2",
+				"name": "some-org",
+			},
+			"tags": common.MapStr{
+				"custom_tag": "foo",
+			},
+		},
+		"message": "log message",
+		"stream":  "stdout",
+	}, evt.ToFields())
+}
+
 func makeEnvelope(eventType *events.Envelope_EventType) *events.Envelope {
 	timestamp := int64(1587469726082)
 	origin := "origin"

x-pack/libbeat/processors/add_cloudfoundry_metadata/add_cloudfoundry_metadata.go

@@ -69,6 +69,10 @@ func (d *addCloudFoundryMetadata) Run(event *beat.Event) (*beat.Event, error) {
 		// wrong type or not set
 		return event, nil
 	}
+	if hasMetadataFields(event) {
+		// nothing to do, fields already present
+		return event, nil
+	}
 	app, err := d.client.GetAppByGuid(val)
 	if err != nil {
 		d.log.Debugf("failed to get application info for GUID(%s): %v", val, err)
@@ -108,3 +112,21 @@ func (d *addCloudFoundryMetadata) Close() error {
 	}
 	return nil
 }
+
+var metadataFields = []string{
+	"cloudfoundry.app.id",
+	"cloudfoundry.app.name",
+	"cloudfoundry.space.id",
+	"cloudfoundry.space.name",
+	"cloudfoundry.org.id",
+	"cloudfoundry.org.name",
+}
+
+func hasMetadataFields(event *beat.Event) bool {
+	for _, name := range metadataFields {
+		if value, err := event.GetValue(name); value == "" || err != nil {
+			return false
+		}
+	}
+	return true
+}

x-pack/libbeat/processors/add_cloudfoundry_metadata/add_cloudfoundry_metadata_test.go

@@ -82,6 +82,62 @@ func TestCFAppNotFound(t *testing.T) {
 	assert.Equal(t, evt, *observed)
 }
 
+func TestCFAppMetadataAlreadyPresent(t *testing.T) {
+	guid := mustCreateFakeGuid()
+	app := cloudfoundry.AppMeta{
+		Guid:      guid,
+		Name:      "My Fake App",
+		SpaceGuid: mustCreateFakeGuid(),
+		SpaceName: "My Fake Space",
+		OrgGuid:   mustCreateFakeGuid(),
+		OrgName:   "My Fake Org",
+	}
+	p := addCloudFoundryMetadata{
+		log:    logp.NewLogger("add_cloudfoundry_metadata"),
+		client: &fakeClient{app},
+	}
+
+	evt := beat.Event{
+		Fields: common.MapStr{
+			"cloudfoundry": common.MapStr{
+				"app": common.MapStr{
+					"id":   guid,
+					"name": "Other App Name",
+				},
+				"space": common.MapStr{
+					"id":   app.SpaceGuid,
+					"name": app.SpaceName,
+				},
+				"org": common.MapStr{
+					"id":   app.OrgGuid,
+					"name": app.OrgName,
+				},
+			},
+		},
+	}
+	expected := beat.Event{
+		Fields: common.MapStr{
+			"cloudfoundry": common.MapStr{
+				"app": common.MapStr{
+					"id":   guid,
+					"name": "Other App Name",
+				},
+				"space": common.MapStr{
+					"id":   app.SpaceGuid,
+					"name": app.SpaceName,
+				},
+				"org": common.MapStr{
+					"id":   app.OrgGuid,
+					"name": app.OrgName,
+				},
+			},
+		},
+	}
+	observed, err := p.Run(&evt)
+	assert.NoError(t, err)
+	assert.Equal(t, expected, *observed)
+}
+
 func TestCFAppUpdated(t *testing.T) {
 	guid := mustCreateFakeGuid()
 	app := cloudfoundry.AppMeta{

x-pack/libbeat/processors/add_cloudfoundry_metadata/docs/add_cloudfoundry_metadata.asciidoc

@@ -20,6 +20,11 @@ Each event is annotated with:
 * Organization ID
 * Organization Name
 
+NOTE: Pivotal Application Service and Tanzu Application Service include this
+metadata in all events from the firehose since version 2.8. In these cases the
+metadata in the events is used, and `add_cloudfoundry_metadata` processor
+doesn't modify these fields.
+
 
 [source,yaml]
 -------------------------------------------------------------------------------

x-pack/metricbeat/module/cloudfoundry/container/_meta/data.json

@@ -2,26 +2,39 @@
     "@timestamp": "2017-10-12T08:05:34.853Z",
     "cloudfoundry": {
         "app": {
-            "id": "3ce55e14-de73-49af-836d-adc93f3fee39"
+            "id": "8d165a12-fbd8-40cb-b71a-5bc6086df04c",
+            "name": "log-gen"
         },
         "container": {
-            "cpu.pct": 0.19431789913648675,
-            "disk.bytes": 16678912,
-            "disk.quota.bytes": 33554432,
-            "instance_index": 0,
-            "memory.bytes": 8529920,
-            "memory.quota.bytes": 33554432
+            "cpu.pct": 4.231873716293137,
+            "disk.bytes": 122691584,
+            "disk.quota.bytes": 1073741824,
+            "instance_index": 3,
+            "memory.bytes": 52250065,
+            "memory.quota.bytes": 1073741824
         },
         "envelope": {
-            "deployment": "cf-6b7aee31c8d07637ad78",
-            "index": "c2bcf5d6-7ff9-4876-890f-6f8fc6c58668",
-            "ip": "192.168.16.51",
+            "deployment": "cf-9c11cd01425665e2ed6d",
+            "index": "9e1a45be-f8a4-44ef-9c34-22f6e51ce4c7",
+            "ip": "192.168.16.37",
             "job": "diego_cell",
             "origin": "rep"
         },
+        "org": {
+            "id": "4af89198-dd33-4542-9915-f489542bc058",
+            "name": "elastic-logging-org"
+        },
+        "space": {
+            "id": "10ed1559-e399-4034-babf-6424ef888dc1",
+            "name": "logging-space"
+        },
         "tags": {
-            "product": "Pivotal Application Service",
-            "source_id": "3ce55e14-de73-49af-836d-adc93f3fee39"
+            "instance_id": "3",
+            "process_id": "8d165a12-fbd8-40cb-b71a-5bc6086df04c",
+            "process_instance_id": "75568c86-b9e0-4330-4784-928b",
+            "process_type": "web",
+            "product": "VMware Tanzu Application Service",
+            "source_id": "8d165a12-fbd8-40cb-b71a-5bc6086df04c"
         },
         "type": "container"
     },