diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index f61cd10d1c70..4edde21e9dd7 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -47,6 +47,7 @@ https://github.com/elastic/beats/compare/v6.0.0-alpha1...master[Check the HEAD d *Affecting all Beats* - New cli subcommands interface. {pull}4420[4420] +- Allow source path matching in `add_docker_metadata` processor {pull}4495[4495] *Filebeat* diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 2bf1cc715e5d..a49a1b45bb5d 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -16,6 +16,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -600,6 +601,47 @@ Name of the project in Google Cloud. Region in which this host is running. +[[exported-fields-docker-processor]] +== docker Fields + +beta[] +Docker stats collected from Docker. + + + + +[float] +=== docker.container.id + +type: keyword + +Unique container id. + + +[float] +=== docker.container.image + +type: keyword + +Name of the image the container was built on. + + +[float] +=== docker.container.name + +type: keyword + +Container name. + + +[float] +=== docker.container.labels + +type: object + +Image labels. + + [[exported-fields-icinga]] == Icinga Fields diff --git a/filebeat/filebeat.full.yml b/filebeat/filebeat.full.yml index 2f85dcfd1243..72df1688eafc 100644 --- a/filebeat/filebeat.full.yml +++ b/filebeat/filebeat.full.yml @@ -516,8 +516,10 @@ filebeat.prospectors: # #processors: #- add_docker_metadata: +# host: "unix:///var/run/docker.sock" +# match_source: true +# match_source_index: 4 # match_fields: ["system.process.cgroup.id"] -# host: "unix:///var/run/docker.sock" # # To connect to Docker over TLS you must specify a client and CA certificate. # #ssl: # # certificate_authority: "/etc/pki/root/ca.pem" diff --git a/heartbeat/docs/fields.asciidoc b/heartbeat/docs/fields.asciidoc index ec2b374cd689..e948ad3745dc 100644 --- a/heartbeat/docs/fields.asciidoc +++ b/heartbeat/docs/fields.asciidoc @@ -15,6 +15,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -255,6 +256,47 @@ required: True Indicator if monitor could validate the service to be available. +[[exported-fields-docker-processor]] +== docker Fields + +beta[] +Docker stats collected from Docker. + + + + +[float] +=== docker.container.id + +type: keyword + +Unique container id. + + +[float] +=== docker.container.image + +type: keyword + +Name of the image the container was built on. + + +[float] +=== docker.container.name + +type: keyword + +Container name. + + +[float] +=== docker.container.labels + +type: object + +Image labels. + + [[exported-fields-http]] == HTTP Monitor Fields diff --git a/heartbeat/heartbeat.full.yml b/heartbeat/heartbeat.full.yml index 5b3b5fbd7a9f..aad68d28c082 100644 --- a/heartbeat/heartbeat.full.yml +++ b/heartbeat/heartbeat.full.yml @@ -280,8 +280,10 @@ heartbeat.scheduler: # #processors: #- add_docker_metadata: +# host: "unix:///var/run/docker.sock" +# match_source: true +# match_source_index: 4 # match_fields: ["system.process.cgroup.id"] -# host: "unix:///var/run/docker.sock" # # To connect to Docker over TLS you must specify a client and CA certificate. # #ssl: # # certificate_authority: "/etc/pki/root/ca.pem" diff --git a/libbeat/_meta/config.full.yml b/libbeat/_meta/config.full.yml index 812797c73a98..c369c3e4faba 100644 --- a/libbeat/_meta/config.full.yml +++ b/libbeat/_meta/config.full.yml @@ -82,8 +82,10 @@ # #processors: #- add_docker_metadata: +# host: "unix:///var/run/docker.sock" +# match_source: true +# match_source_index: 4 # match_fields: ["system.process.cgroup.id"] -# host: "unix:///var/run/docker.sock" # # To connect to Docker over TLS you must specify a client and CA certificate. # #ssl: # # certificate_authority: "/etc/pki/root/ca.pem" diff --git a/libbeat/docs/processors-using.asciidoc b/libbeat/docs/processors-using.asciidoc index a3447e2c7dc7..7caddebde696 100644 --- a/libbeat/docs/processors-using.asciidoc +++ b/libbeat/docs/processors-using.asciidoc @@ -572,20 +572,30 @@ from Docker containers: * Image * Labels -Currently it supports enriching events by matching existing fields with the -container ID in them, for example, cgroup IDs retrieved by metricbeat system -module: - [source,yaml] ------------------------------------------------------------------------------- processors: - add_docker_metadata: + host: "unix:///var/run/docker.sock" match_fields: ["system.process.cgroup.id"] - host: "unix:///var/run/docker.sock" - - # To connect to Docker over TLS you must specify a client and CA certificate. - #ssl: - # certificate_authority: "/etc/pki/root/ca.pem" - # certificate: "/etc/pki/client/cert.pem" - # key: "/etc/pki/client/cert.key" + match_source: true + match_source_index: 4 + + # To connect to Docker over TLS you must specify a client and CA certificate. + #ssl: + # certificate_authority: "/etc/pki/root/ca.pem" + # certificate: "/etc/pki/client/cert.pem" + # key: "/etc/pki/client/cert.key" ------------------------------------------------------------------------------- + +It has the following settings: + +`host`:: (Optional) Docker socket (UNIX or TCP socket). It uses + `unix:///var/run/docker.sock` by default. +`match_fields`:: (Optional) A list of fields to match a container id, at least + one of them should hold a container id to get the event enriched. +`match_source`:: (Optional) Match container id from a log path present in + `source` field. Enabled by default. +`match_source_index`:: (Optional) Index in the source path split by / to look + for container id. It defaults to 4 to match + `/var/lib/docker/containers//*.log` diff --git a/libbeat/processors/actions/extract_field.go b/libbeat/processors/actions/extract_field.go new file mode 100644 index 000000000000..bf7c079bf0d9 --- /dev/null +++ b/libbeat/processors/actions/extract_field.go @@ -0,0 +1,91 @@ +package actions + +import ( + "fmt" + "strings" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/processors" +) + +type extract_field struct { + Field string + Separator string + Index int + Target string +} + +/* +This one won't be registered (yet) + +func init() { + processors.RegisterPlugin("extract_field", + configChecked(NewExtractField, + requireFields("field", "separator", "index", "target"), + allowedFields("field", "separator", "index", "target", "when"))) +} +*/ + +func NewExtractField(c common.Config) (processors.Processor, error) { + config := struct { + Field string `config:"field"` + Separator string `config:"separator"` + Index int `config:"index"` + Target string `config:"target"` + }{} + err := c.Unpack(&config) + if err != nil { + return nil, fmt.Errorf("fail to unpack the extract_field configuration: %s", err) + } + + /* remove read only fields */ + for _, readOnly := range processors.MandatoryExportedFields { + if config.Field == readOnly { + return nil, fmt.Errorf("%s is a read only field, cannot override", readOnly) + } + } + + f := extract_field{ + Field: config.Field, + Separator: config.Separator, + Index: config.Index, + Target: config.Target, + } + return f, nil +} + +func (f extract_field) Run(event common.MapStr) (common.MapStr, error) { + fieldValue, err := event.GetValue(f.Field) + if err != nil { + return nil, fmt.Errorf("error getting field '%s' from event", f.Field) + } + + value, ok := fieldValue.(string) + if !ok { + return nil, fmt.Errorf("could not get a string from field '%s'", f.Field) + } + + parts := strings.Split(value, f.Separator) + parts = deleteEmpty(parts) + if len(parts) < f.Index+1 { + return nil, fmt.Errorf("index is out of range for field '%s'", f.Field) + } + + event.Put(f.Target, parts[f.Index]) + + return event, nil +} + +func (f extract_field) String() string { + return "extract_field=" + f.Target +} + +func deleteEmpty(s []string) []string { + var r []string + for _, str := range s { + if str != "" { + r = append(r, str) + } + } + return r +} diff --git a/libbeat/processors/actions/extract_field_test.go b/libbeat/processors/actions/extract_field_test.go new file mode 100644 index 000000000000..119cbb00935b --- /dev/null +++ b/libbeat/processors/actions/extract_field_test.go @@ -0,0 +1,90 @@ +package actions + +import ( + "testing" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" + "github.com/stretchr/testify/assert" +) + +func TestCommonPaths(t *testing.T) { + var tests = []struct { + Value, Field, Separator, Target, Result string + Index int + }{ + // Common docker case + { + Value: "/var/lib/docker/containers/f1510836197d7c34da22cf796dba5640f87c04de5c95cf0adc11b85f1e1c1528/f1510836197d7c34da22cf796dba5640f87c04de5c95cf0adc11b85f1e1c1528-json.log", + Field: "source", + Separator: "/", + Target: "docker.container.id", + Index: 4, + Result: "f1510836197d7c34da22cf796dba5640f87c04de5c95cf0adc11b85f1e1c1528", + }, + { + Value: "/var/lib/foo/bar", + Field: "other_field", + Separator: "/", + Target: "destination", + Index: 3, + Result: "bar", + }, + { + Value: "-var-lib-foo-bar", + Field: "source", + Separator: "-", + Target: "destination", + Index: 2, + Result: "foo", + }, + { + Value: "*var*lib*foo*bar", + Field: "source", + Separator: "*", + Target: "destination", + Index: 0, + Result: "var", + }, + } + + for _, test := range tests { + var testConfig, _ = common.NewConfigFrom(map[string]interface{}{ + "field": test.Field, + "separator": test.Separator, + "index": test.Index, + "target": test.Target, + }) + + // Configure input to + input := common.MapStr{ + test.Field: test.Value, + } + + actual := runExtractField(t, testConfig, input) + + result, err := actual.GetValue(test.Target) + if err != nil { + t.Fatalf("could not get target field: %s", err) + } + assert.Equal(t, result.(string), test.Result) + } +} + +func runExtractField(t *testing.T, config *common.Config, input common.MapStr) common.MapStr { + if testing.Verbose() { + logp.LogInit(logp.LOG_DEBUG, "", false, true, []string{"*"}) + } + + p, err := NewExtractField(*config) + if err != nil { + t.Fatalf("error initializing extract_field: %s", err) + } + + actual, err := p.Run(input) + if err != nil { + t.Fatalf("error running extract_field: %s", err) + } + + return actual +} diff --git a/libbeat/processors/add_docker_metadata/_meta/fields.yml b/libbeat/processors/add_docker_metadata/_meta/fields.yml new file mode 100644 index 000000000000..518041a34df4 --- /dev/null +++ b/libbeat/processors/add_docker_metadata/_meta/fields.yml @@ -0,0 +1,29 @@ +- key: docker + title: docker + description: > + beta[] + + Docker stats collected from Docker. + short_config: false + anchor: docker-processor + fields: + - name: docker + type: group + fields: + - name: container.id + type: keyword + description: > + Unique container id. + - name: container.image + type: keyword + description: > + Name of the image the container was built on. + - name: container.name + type: keyword + description: > + Container name. + - name: container.labels + type: object + object_type: keyword + description: > + Image labels. diff --git a/libbeat/processors/add_docker_metadata/add_docker_metadata.go b/libbeat/processors/add_docker_metadata/add_docker_metadata.go index 682a3c0485cc..8bd2e29cd5cf 100644 --- a/libbeat/processors/add_docker_metadata/add_docker_metadata.go +++ b/libbeat/processors/add_docker_metadata/add_docker_metadata.go @@ -7,6 +7,7 @@ import ( "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/libbeat/processors" + "github.com/elastic/beats/libbeat/processors/actions" ) func init() { @@ -14,8 +15,9 @@ func init() { } type addDockerMetadata struct { - watcher Watcher - fields []string + watcher Watcher + fields []string + sourceProcessor processors.Processor } func newDockerMetadataProcessor(cfg common.Config) (processors.Processor, error) { @@ -41,14 +43,45 @@ func buildDockerMetadataProcessor(cfg common.Config, watcherConstructor WatcherC return nil, err } + // Use extract_field processor to get container id from source file path + var sourceProcessor processors.Processor + if config.MatchSource { + var procConf, _ = common.NewConfigFrom(map[string]interface{}{ + "field": "source", + "separator": "/", + "index": config.SourceIndex, + "target": "docker.container.id", + }) + sourceProcessor, err = actions.NewExtractField(*procConf) + if err != nil { + return nil, err + } + + // Ensure `docker.container.id` is matched: + config.Fields = append(config.Fields, "docker.container.id") + } + return &addDockerMetadata{ - watcher: watcher, - fields: config.Fields, + watcher: watcher, + fields: config.Fields, + sourceProcessor: sourceProcessor, }, nil } func (d *addDockerMetadata) Run(event common.MapStr) (common.MapStr, error) { var cid string + var err error + + // Process source field + if d.sourceProcessor != nil { + if event["source"] != nil { + event, err = d.sourceProcessor.Run(event) + if err != nil { + return nil, err + } + } + } + for _, field := range d.fields { value, err := event.GetValue(field) if err != nil { diff --git a/libbeat/processors/add_docker_metadata/add_docker_metadata_test.go b/libbeat/processors/add_docker_metadata/add_docker_metadata_test.go index 2b3445c10f8c..1814a08935be 100644 --- a/libbeat/processors/add_docker_metadata/add_docker_metadata_test.go +++ b/libbeat/processors/add_docker_metadata/add_docker_metadata_test.go @@ -98,6 +98,78 @@ func TestMatchContainer(t *testing.T) { }, result) } +func TestMatchSource(t *testing.T) { + // Use defaults + testConfig, err := common.NewConfigFrom(map[string]interface{}{}) + assert.NoError(t, err) + + p, err := buildDockerMetadataProcessor(*testConfig, MockWatcherFactory( + map[string]*Container{ + "FABADA": &Container{ + ID: "FABADA", + Image: "image", + Name: "name", + Labels: map[string]string{ + "a": "1", + "b": "2", + }, + }, + })) + assert.NoError(t, err, "initializing add_docker_metadata processor") + + input := common.MapStr{ + "source": "/var/lib/docker/containers/FABADA/foo.log", + } + result, err := p.Run(input) + assert.NoError(t, err, "processing an event") + + assert.EqualValues(t, common.MapStr{ + "docker": common.MapStr{ + "container": common.MapStr{ + "id": "FABADA", + "image": "image", + "labels": common.MapStr{ + "a": "1", + "b": "2", + }, + "name": "name", + }, + }, + "source": "/var/lib/docker/containers/FABADA/foo.log", + }, result) +} + +func TestDisableSource(t *testing.T) { + // Use defaults + testConfig, err := common.NewConfigFrom(map[string]interface{}{ + "match_source": false, + }) + assert.NoError(t, err) + + p, err := buildDockerMetadataProcessor(*testConfig, MockWatcherFactory( + map[string]*Container{ + "FABADA": &Container{ + ID: "FABADA", + Image: "image", + Name: "name", + Labels: map[string]string{ + "a": "1", + "b": "2", + }, + }, + })) + assert.NoError(t, err, "initializing add_docker_metadata processor") + + input := common.MapStr{ + "source": "/var/lib/docker/containers/FABADA/foo.log", + } + result, err := p.Run(input) + assert.NoError(t, err, "processing an event") + + // remains unchanged + assert.EqualValues(t, input, result) +} + // Mock container watcher func MockWatcherFactory(containers map[string]*Container) WatcherConstructor { diff --git a/libbeat/processors/add_docker_metadata/config.go b/libbeat/processors/add_docker_metadata/config.go index 7f7b477c996d..0efcef7e6ec7 100644 --- a/libbeat/processors/add_docker_metadata/config.go +++ b/libbeat/processors/add_docker_metadata/config.go @@ -2,9 +2,11 @@ package add_docker_metadata // Config for docker processor type Config struct { - Host string `config:"host"` - TLS *TLSConfig `config:"ssl"` - Fields []string `config:"match_fields"` + Host string `config:"host"` + TLS *TLSConfig `config:"ssl"` + Fields []string `config:"match_fields"` + MatchSource bool `config:"match_source"` + SourceIndex int `config:"match_source_index"` } // TLSConfig for docker socket connection @@ -16,6 +18,8 @@ type TLSConfig struct { func defaultConfig() Config { return Config{ - Host: "unix:///var/run/docker.sock", + Host: "unix:///var/run/docker.sock", + MatchSource: true, + SourceIndex: 4, } } diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 93b933f390a3..c17d004a0f61 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -19,6 +19,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -3213,66 +3214,83 @@ type: long Number of items/documents that are replicas. -[[exported-fields-docker]] -== Docker Fields +[[exported-fields-docker-processor]] +== docker Fields beta[] Docker stats collected from Docker. + [float] -== docker Fields +=== docker.container.id -Information and statistics about docker's running containers. +type: keyword +Unique container id. [float] -== container Fields +=== docker.container.image -Docker container metrics. +type: keyword +Name of the image the container was built on. [float] -=== docker.container.command +=== docker.container.name type: keyword -Command that was executed in the Docker container. +Container name. [float] -=== docker.container.created +=== docker.container.labels -type: date +type: object + +Image labels. + + +[[exported-fields-docker]] +== Docker Fields + +beta[] +Docker stats collected from Docker. -Date when the container was created. [float] -=== docker.container.id +== docker Fields -type: keyword +Information and statistics about docker's running containers. -Unique container id. [float] -=== docker.container.image +== container Fields -type: keyword +Docker container metrics. -Name of the image the container was built on. [float] -=== docker.container.name +=== docker.container.command type: keyword -Container name. +Command that was executed in the Docker container. + + +[float] +=== docker.container.created + +type: date + +Date when the container was created. [float] @@ -3306,14 +3324,6 @@ type: long Size of the files that have been created or changed since creation. -[float] -=== docker.container.labels - -type: object - -Image labels. - - [float] === docker.container.tags diff --git a/metricbeat/metricbeat.full.yml b/metricbeat/metricbeat.full.yml index b763ea5309ae..5391f95bfc71 100644 --- a/metricbeat/metricbeat.full.yml +++ b/metricbeat/metricbeat.full.yml @@ -539,8 +539,10 @@ metricbeat.modules: # #processors: #- add_docker_metadata: +# host: "unix:///var/run/docker.sock" +# match_source: true +# match_source_index: 4 # match_fields: ["system.process.cgroup.id"] -# host: "unix:///var/run/docker.sock" # # To connect to Docker over TLS you must specify a client and CA certificate. # #ssl: # # certificate_authority: "/etc/pki/root/ca.pem" diff --git a/metricbeat/module/docker/container/_meta/fields.yml b/metricbeat/module/docker/container/_meta/fields.yml index adeabd0e19ab..b22f2d6f30c3 100644 --- a/metricbeat/module/docker/container/_meta/fields.yml +++ b/metricbeat/module/docker/container/_meta/fields.yml @@ -11,18 +11,6 @@ type: date description: > Date when the container was created. - - name: id - type: keyword - description: > - Unique container id. - - name: image - type: keyword - description: > - Name of the image the container was built on. - - name: name - type: keyword - description: > - Container name. - name: status type: keyword description: > @@ -40,11 +28,6 @@ type: long description: > Size of the files that have been created or changed since creation. - - name: labels - type: object - object_type: keyword - description: > - Image labels. - name: tags type: array description: > diff --git a/packetbeat/docs/fields.asciidoc b/packetbeat/docs/fields.asciidoc index 1e65b85aef87..ca4431030fc9 100644 --- a/packetbeat/docs/fields.asciidoc +++ b/packetbeat/docs/fields.asciidoc @@ -18,6 +18,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -1498,6 +1499,47 @@ type: long Requestor's UDP payload size (in bytes). +[[exported-fields-docker-processor]] +== docker Fields + +beta[] +Docker stats collected from Docker. + + + + +[float] +=== docker.container.id + +type: keyword + +Unique container id. + + +[float] +=== docker.container.image + +type: keyword + +Name of the image the container was built on. + + +[float] +=== docker.container.name + +type: keyword + +Container name. + + +[float] +=== docker.container.labels + +type: object + +Image labels. + + [[exported-fields-flows_event]] == Flow Event Fields diff --git a/packetbeat/packetbeat.full.yml b/packetbeat/packetbeat.full.yml index 63a2d659198a..2f34b9b54602 100644 --- a/packetbeat/packetbeat.full.yml +++ b/packetbeat/packetbeat.full.yml @@ -537,8 +537,10 @@ packetbeat.protocols: # #processors: #- add_docker_metadata: +# host: "unix:///var/run/docker.sock" +# match_source: true +# match_source_index: 4 # match_fields: ["system.process.cgroup.id"] -# host: "unix:///var/run/docker.sock" # # To connect to Docker over TLS you must specify a client and CA certificate. # #ssl: # # certificate_authority: "/etc/pki/root/ca.pem" diff --git a/winlogbeat/docs/fields.asciidoc b/winlogbeat/docs/fields.asciidoc index bb826805f3f3..f0918dffd33a 100644 --- a/winlogbeat/docs/fields.asciidoc +++ b/winlogbeat/docs/fields.asciidoc @@ -15,6 +15,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> @@ -182,6 +183,47 @@ The event log API type used to read the record. The possible values are "wineven The Event Logging API was designed for Windows Server 2003, Windows XP, or Windows 2000 operating systems. In Windows Vista, the event logging infrastructure was redesigned. On Windows Vista or later operating systems, the Windows Event Log API is used. Winlogbeat automatically detects which API to use for reading event logs. +[[exported-fields-docker-processor]] +== docker Fields + +beta[] +Docker stats collected from Docker. + + + + +[float] +=== docker.container.id + +type: keyword + +Unique container id. + + +[float] +=== docker.container.image + +type: keyword + +Name of the image the container was built on. + + +[float] +=== docker.container.name + +type: keyword + +Container name. + + +[float] +=== docker.container.labels + +type: object + +Image labels. + + [[exported-fields-eventlog]] == Event Log Record Fields diff --git a/winlogbeat/winlogbeat.full.yml b/winlogbeat/winlogbeat.full.yml index fbdcc9f9fbc5..baafa65fce15 100644 --- a/winlogbeat/winlogbeat.full.yml +++ b/winlogbeat/winlogbeat.full.yml @@ -111,8 +111,10 @@ winlogbeat.event_logs: # #processors: #- add_docker_metadata: +# host: "unix:///var/run/docker.sock" +# match_source: true +# match_source_index: 4 # match_fields: ["system.process.cgroup.id"] -# host: "unix:///var/run/docker.sock" # # To connect to Docker over TLS you must specify a client and CA certificate. # #ssl: # # certificate_authority: "/etc/pki/root/ca.pem"