Skip to content

Commit

Permalink
Move cloudfoundry tags with metadata to common metadata fields (#22150)
Browse files Browse the repository at this point in the history 
In some Cloud Foundry deployments, metadata is included in tags,
move these tags to the common fields so add_cloudfoundry_metadata
is not needed for these cases.
  • Loading branch information
@jsoriano
jsoriano authored Oct 27, 2020
1 parent cf7bdba commit 7c461f8
Show file tree
Hide file tree
Showing 6 changed files with 224 additions and 23 deletions.
46 changes: 35 additions & 11 deletions x-pack/libbeat/common/cloudfoundry/events.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -490,21 +490,45 @@ func envelopMap(evt Event) common.MapStr {
}

func baseMap(evt Event) common.MapStr {
return common.MapStr{
"cloudfoundry": common.MapStr{
"type": evt.String(),
"envelope": envelopMap(evt),
"tags": dedotedTags(evt.Tags()),
},
tags, meta := tagsToMeta(evt.Tags())
cf := common.MapStr{
"type": evt.String(),
"envelope": envelopMap(evt),
}
if len(tags) > 0 {
cf["tags"] = tags
}
result := common.MapStr{
"cloudfoundry": cf,
}
if len(meta) > 0 {
result.DeepUpdate(meta)
}
return result
}

func dedotedTags(tags map[string]string) common.MapStr {
result := common.MapStr{}
for name, value := range tags {
result[common.DeDot(name)] = value
func tagsToMeta(eventTags map[string]string) (tags common.MapStr, meta common.MapStr) {
tags = common.MapStr{}
meta = common.MapStr{}
for name, value := range eventTags {
switch name {
case "app_id":
meta.Put("cloudfoundry.app.id", value)
case "app_name":
meta.Put("cloudfoundry.app.name", value)
case "space_id":
meta.Put("cloudfoundry.space.id", value)
case "space_name":
meta.Put("cloudfoundry.space.name", value)
case "organization_id":
meta.Put("cloudfoundry.org.id", value)
case "organization_name":
meta.Put("cloudfoundry.org.name", value)
default:
tags[common.DeDot(name)] = value
}
}
return result
return tags, meta
}

func baseMapWithApp(evt EventWithAppID) common.MapStr {
Expand Down
81 changes: 81 additions & 0 deletions x-pack/libbeat/common/cloudfoundry/events_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -382,6 +382,87 @@ func TestEventError(t *testing.T) {
}, evt.ToFields())
}

func TestEventTagsWithMetadata(t *testing.T) {
eventType := events.Envelope_LogMessage
message := "log message"
messageType := events.LogMessage_OUT
timestamp := int64(1587469726082)
appID := "f47ac10b-58cc-4372-a567-0e02b2c3d479"
sourceType := "source_type"
sourceInstance := "source_instance"
cfEvt := makeEnvelope(&eventType)
tags := map[string]string{
"app_id": appID,
"app_name": "some-app",
"space_id": "e1114e92-155c-11eb-ada9-27b81025a657",
"space_name": "some-space",
"organization_id": "baeef1ba-155c-11eb-a1af-8f14964c35d2",
"organization_name": "some-org",
"custom_tag": "foo",
}
cfEvt.Tags = tags
cfEvt.LogMessage = &events.LogMessage{
Message: []byte(message),
MessageType: &messageType,
Timestamp: &timestamp,
AppId: &appID,
SourceType: &sourceType,
SourceInstance: &sourceInstance,
}
evt := newEventLog(cfEvt)

assert.Equal(t, EventTypeLog, evt.EventType())
assert.Equal(t, "log", evt.String())
assert.Equal(t, "origin", evt.Origin())
assert.Equal(t, time.Unix(0, 1587469726082), evt.Timestamp())
assert.Equal(t, "deployment", evt.Deployment())
assert.Equal(t, "job", evt.Job())
assert.Equal(t, "index", evt.Index())
assert.Equal(t, "ip", evt.IP())
assert.Equal(t, tags, evt.Tags())
assert.Equal(t, "f47ac10b-58cc-4372-a567-0e02b2c3d479", evt.AppGuid())
assert.Equal(t, "log message", evt.Message())
assert.Equal(t, EventLogMessageTypeStdout, evt.MessageType())
assert.Equal(t, "source_type", evt.SourceType())
assert.Equal(t, "source_instance", evt.SourceID())

assert.Equal(t, common.MapStr{
"cloudfoundry": common.MapStr{
"type": "log",
"log": common.MapStr{
"source": common.MapStr{
"instance": evt.SourceID(),
"type": evt.SourceType(),
},
},
"envelope": common.MapStr{
"origin": "origin",
"deployment": "deployment",
"ip": "ip",
"job": "job",
"index": "index",
},
"app": common.MapStr{
"id": "f47ac10b-58cc-4372-a567-0e02b2c3d479",
"name": "some-app",
},
"space": common.MapStr{
"id": "e1114e92-155c-11eb-ada9-27b81025a657",
"name": "some-space",
},
"org": common.MapStr{
"id": "baeef1ba-155c-11eb-a1af-8f14964c35d2",
"name": "some-org",
},
"tags": common.MapStr{
"custom_tag": "foo",
},
},
"message": "log message",
"stream": "stdout",
}, evt.ToFields())
}

func makeEnvelope(eventType *events.Envelope_EventType) *events.Envelope {
timestamp := int64(1587469726082)
origin := "origin"
Expand Down
22 changes: 22 additions & 0 deletions x-pack/libbeat/processors/add_cloudfoundry_metadata/add_cloudfoundry_metadata.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,10 @@ func (d *addCloudFoundryMetadata) Run(event *beat.Event) (*beat.Event, error) {
// wrong type or not set
return event, nil
}
if hasMetadataFields(event) {
// nothing to do, fields already present
return event, nil
}
app, err := d.client.GetAppByGuid(val)
if err != nil {
d.log.Debugf("failed to get application info for GUID(%s): %v", val, err)
Expand Down Expand Up @@ -108,3 +112,21 @@ func (d *addCloudFoundryMetadata) Close() error {
}
return nil
}

var metadataFields = []string{
"cloudfoundry.app.id",
"cloudfoundry.app.name",
"cloudfoundry.space.id",
"cloudfoundry.space.name",
"cloudfoundry.org.id",
"cloudfoundry.org.name",
}

func hasMetadataFields(event *beat.Event) bool {
for _, name := range metadataFields {
if value, err := event.GetValue(name); value == "" || err != nil {
return false
}
}
return true
}
56 changes: 56 additions & 0 deletions x-pack/libbeat/processors/add_cloudfoundry_metadata/add_cloudfoundry_metadata_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,62 @@ func TestCFAppNotFound(t *testing.T) {
assert.Equal(t, evt, *observed)
}

func TestCFAppMetadataAlreadyPresent(t *testing.T) {
guid := mustCreateFakeGuid()
app := cloudfoundry.AppMeta{
Guid: guid,
Name: "My Fake App",
SpaceGuid: mustCreateFakeGuid(),
SpaceName: "My Fake Space",
OrgGuid: mustCreateFakeGuid(),
OrgName: "My Fake Org",
}
p := addCloudFoundryMetadata{
log: logp.NewLogger("add_cloudfoundry_metadata"),
client: &fakeClient{app},
}

evt := beat.Event{
Fields: common.MapStr{
"cloudfoundry": common.MapStr{
"app": common.MapStr{
"id": guid,
"name": "Other App Name",
},
"space": common.MapStr{
"id": app.SpaceGuid,
"name": app.SpaceName,
},
"org": common.MapStr{
"id": app.OrgGuid,
"name": app.OrgName,
},
},
},
}
expected := beat.Event{
Fields: common.MapStr{
"cloudfoundry": common.MapStr{
"app": common.MapStr{
"id": guid,
"name": "Other App Name",
},
"space": common.MapStr{
"id": app.SpaceGuid,
"name": app.SpaceName,
},
"org": common.MapStr{
"id": app.OrgGuid,
"name": app.OrgName,
},
},
},
}
observed, err := p.Run(&evt)
assert.NoError(t, err)
assert.Equal(t, expected, *observed)
}

func TestCFAppUpdated(t *testing.T) {
guid := mustCreateFakeGuid()
app := cloudfoundry.AppMeta{
Expand Down
5 changes: 5 additions & 0 deletions ...at/processors/add_cloudfoundry_metadata/docs/add_cloudfoundry_metadata.asciidoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,11 @@ Each event is annotated with:
* Organization ID
* Organization Name

NOTE: Pivotal Application Service and Tanzu Application Service include this
metadata in all events from the firehose since version 2.8. In these cases the
metadata in the events is used, and `add_cloudfoundry_metadata` processor
doesn't modify these fields.


[source,yaml]
-------------------------------------------------------------------------------
Expand Down
37 changes: 25 additions & 12 deletions x-pack/metricbeat/module/cloudfoundry/container/_meta/data.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,26 +2,39 @@
"@timestamp": "2017-10-12T08:05:34.853Z",
"cloudfoundry": {
"app": {
"id": "3ce55e14-de73-49af-836d-adc93f3fee39"
"id": "8d165a12-fbd8-40cb-b71a-5bc6086df04c",
"name": "log-gen"
},
"container": {
"cpu.pct": 0.19431789913648675,
"disk.bytes": 16678912,
"disk.quota.bytes": 33554432,
"instance_index": 0,
"memory.bytes": 8529920,
"memory.quota.bytes": 33554432
"cpu.pct": 4.231873716293137,
"disk.bytes": 122691584,
"disk.quota.bytes": 1073741824,
"instance_index": 3,
"memory.bytes": 52250065,
"memory.quota.bytes": 1073741824
},
"envelope": {
"deployment": "cf-6b7aee31c8d07637ad78",
"index": "c2bcf5d6-7ff9-4876-890f-6f8fc6c58668",
"ip": "192.168.16.51",
"deployment": "cf-9c11cd01425665e2ed6d",
"index": "9e1a45be-f8a4-44ef-9c34-22f6e51ce4c7",
"ip": "192.168.16.37",
"job": "diego_cell",
"origin": "rep"
},
"org": {
"id": "4af89198-dd33-4542-9915-f489542bc058",
"name": "elastic-logging-org"
},
"space": {
"id": "10ed1559-e399-4034-babf-6424ef888dc1",
"name": "logging-space"
},
"tags": {
"product": "Pivotal Application Service",
"source_id": "3ce55e14-de73-49af-836d-adc93f3fee39"
"instance_id": "3",
"process_id": "8d165a12-fbd8-40cb-b71a-5bc6086df04c",
"process_instance_id": "75568c86-b9e0-4330-4784-928b",
"process_type": "web",
"product": "VMware Tanzu Application Service",
"source_id": "8d165a12-fbd8-40cb-b71a-5bc6086df04c"
},
"type": "container"
},
Expand Down

0 comments on commit 7c461f8

Please sign in to comment.