diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 33afd9eb33c..7182cb17c9a 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -138,6 +138,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Remove experimental flag from `setup.template.append_fields` {pull}16576[16576] - Add `add_cloudfoundry_metadata` processor to annotate events with Cloud Foundry application data. {pull}16621[16621] - Add Kerberos support to Kafka input and output. {pull}16781[16781] +- Add `add_cloudfoundry_metadata` processor to annotate events with Cloud Foundry application data. {pull}16621[16621 +- Add support for kubernetes provider to recognize namespace level defaults {pull}16321[16321] *Auditbeat* diff --git a/filebeat/docs/autodiscover-hints.asciidoc b/filebeat/docs/autodiscover-hints.asciidoc index a7caddb3048..8201de0e620 100644 --- a/filebeat/docs/autodiscover-hints.asciidoc +++ b/filebeat/docs/autodiscover-hints.asciidoc @@ -155,6 +155,24 @@ annotations: co.elastic.logs.sidecar/exclude_lines: '^DBG' ----- +[float] +===== Configuring Namespace Defaults + +Hints can be configured on the Namespace's annotations as defaults to use when Pod level annotations are missing. +The resultant hints are a combination of Pod annotations and Namespace annotations with the Pod's taking precedence. To +enable Namespace defaults configure the `add_resource_metadata` for Namespace objects as follows: + +["source","yaml",subs="attributes"] +------------------------------------------------------------------------------------- +filebeat.autodiscover: + providers: + - type: kubernetes + hints.enabled: true + add_resource_metadata: + namespace: + enabled: true +------------------------------------------------------------------------------------- + [float] diff --git a/libbeat/autodiscover/providers/kubernetes/kubernetes.go b/libbeat/autodiscover/providers/kubernetes/kubernetes.go index a1e007acbe9..ec3480fa00b 100644 --- a/libbeat/autodiscover/providers/kubernetes/kubernetes.go +++ b/libbeat/autodiscover/providers/kubernetes/kubernetes.go @@ -140,7 +140,8 @@ func (p *Provider) publish(event bus.Event) { event["config"] = config } else { // If there isn't a default template then attempt to use builders - if config := p.builders.GetConfig(p.eventer.GenerateHints(event)); config != nil { + e := p.eventer.GenerateHints(event) + if config := p.builders.GetConfig(e); config != nil { event["config"] = config } } diff --git a/libbeat/autodiscover/providers/kubernetes/pod.go b/libbeat/autodiscover/providers/kubernetes/pod.go index 44f85a2363a..b7239e3004e 100644 --- a/libbeat/autodiscover/providers/kubernetes/pod.go +++ b/libbeat/autodiscover/providers/kubernetes/pod.go @@ -23,6 +23,7 @@ import ( "github.com/gofrs/uuid" k8s "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" "github.com/elastic/beats/v7/libbeat/autodiscover/builder" "github.com/elastic/beats/v7/libbeat/common" @@ -42,6 +43,7 @@ type pod struct { watcher kubernetes.Watcher nodeWatcher kubernetes.Watcher namespaceWatcher kubernetes.Watcher + namespaceStore cache.Store } // NewPodEventer creates an eventer that can discover and process pod objects @@ -159,15 +161,27 @@ func (p *pod) GenerateHints(event bus.Event) bus.Event { // Try to build a config with enabled builders. Send a provider agnostic payload. // Builders are Beat specific. e := bus.Event{} - var annotations common.MapStr var kubeMeta, container common.MapStr + + annotations := make(common.MapStr, 0) rawMeta, ok := event["kubernetes"] if ok { kubeMeta = rawMeta.(common.MapStr) // The builder base config can configure any of the field values of kubernetes if need be. e["kubernetes"] = kubeMeta if rawAnn, ok := kubeMeta["annotations"]; ok { - annotations = rawAnn.(common.MapStr) + anns, _ := rawAnn.(common.MapStr) + if len(anns) != 0 { + annotations = anns.Clone() + } + } + + // Look at all the namespace level default annotations and do a merge with priority going to the pod annotations. + if rawNsAnn, ok := kubeMeta["namespace_annotations"]; ok { + nsAnn, _ := rawNsAnn.(common.MapStr) + if len(nsAnn) != 0 { + annotations.DeepUpdateNoOverwrite(nsAnn) + } } } if host, ok := event["host"]; ok { @@ -186,12 +200,14 @@ func (p *pod) GenerateHints(event bus.Event) bus.Event { } cname := builder.GetContainerName(container) + + // Generate hints based on the cumulative of both namespace and pod annotations. hints := builder.GenerateHints(annotations, cname, p.config.Prefix) p.logger.Debugf("Generated hints %+v", hints) + if len(hints) != 0 { e["hints"] = hints } - p.logger.Debugf("Generated builder event %+v", e) return e @@ -296,6 +312,18 @@ func (p *pod) emitEvents(pod *kubernetes.Pod, flag string, containers []kubernet safemapstr.Put(annotations, k, v) } kubemeta["annotations"] = annotations + if p.namespaceWatcher != nil { + if rawNs, ok, err := p.namespaceWatcher.Store().GetByKey(pod.Namespace); ok && err == nil { + if namespace, ok := rawNs.(*kubernetes.Namespace); ok { + nsAnn := common.MapStr{} + + for k, v := range namespace.GetAnnotations() { + safemapstr.Put(nsAnn, k, v) + } + kubemeta["namespace_annotations"] = nsAnn + } + } + } // Without this check there would be overlapping configurations with and without ports. if len(c.Ports) == 0 { diff --git a/libbeat/autodiscover/providers/kubernetes/pod_test.go b/libbeat/autodiscover/providers/kubernetes/pod_test.go index b105aa0991f..f63dbdf1e73 100644 --- a/libbeat/autodiscover/providers/kubernetes/pod_test.go +++ b/libbeat/autodiscover/providers/kubernetes/pod_test.go @@ -150,6 +150,174 @@ func TestGenerateHints(t *testing.T) { }, }, }, + // Scenarios tested: + // Have one set of hints come from the pod and the other come from namespaces + // The resultant hints should have a combination of both + { + event: bus.Event{ + "kubernetes": common.MapStr{ + "annotations": getNestedAnnotations(common.MapStr{ + "co.elastic.logs/multiline.pattern": "^test", + "co.elastic.logs/json.keys_under_root": "true", + "not.to.include": "true", + }), + "namespace_annotations": getNestedAnnotations(common.MapStr{ + "co.elastic.metrics/module": "prometheus", + "co.elastic.metrics/period": "10s", + "co.elastic.metrics.foobar/period": "15s", + }), + "container": common.MapStr{ + "name": "foobar", + "id": "abc", + "runtime": "docker", + }, + "namespace": "ns", + }, + }, + result: bus.Event{ + "kubernetes": common.MapStr{ + "annotations": getNestedAnnotations(common.MapStr{ + "co.elastic.logs/multiline.pattern": "^test", + "co.elastic.logs/json.keys_under_root": "true", + "not.to.include": "true", + }), + "namespace_annotations": getNestedAnnotations(common.MapStr{ + "co.elastic.metrics/period": "10s", + "co.elastic.metrics.foobar/period": "15s", + "co.elastic.metrics/module": "prometheus", + }), + "container": common.MapStr{ + "name": "foobar", + "id": "abc", + "runtime": "docker", + }, + "namespace": "ns", + }, + "hints": common.MapStr{ + "logs": common.MapStr{ + "multiline": common.MapStr{ + "pattern": "^test", + }, + "json": common.MapStr{ + "keys_under_root": "true", + }, + }, + "metrics": common.MapStr{ + "module": "prometheus", + "period": "15s", + }, + }, + "container": common.MapStr{ + "name": "foobar", + "id": "abc", + "runtime": "docker", + }, + }, + }, + // Scenarios tested: + // Have one set of hints come from the pod and the same keys come from namespaces + // The resultant hints should honor only pods and not namespace. + { + event: bus.Event{ + "kubernetes": common.MapStr{ + "annotations": getNestedAnnotations(common.MapStr{ + "co.elastic.metrics/module": "prometheus", + "co.elastic.metrics/period": "10s", + "co.elastic.metrics.foobar/period": "15s", + "not.to.include": "true", + }), + "namespace_annotations": getNestedAnnotations(common.MapStr{ + "co.elastic.metrics/module": "dropwizard", + "co.elastic.metrics/period": "60s", + "co.elastic.metrics.foobar/period": "25s", + }), + "namespace": "ns", + "container": common.MapStr{ + "name": "foobar", + "id": "abc", + "runtime": "docker", + }, + }, + }, + result: bus.Event{ + "kubernetes": common.MapStr{ + "annotations": getNestedAnnotations(common.MapStr{ + "co.elastic.metrics/module": "prometheus", + "co.elastic.metrics/period": "10s", + "co.elastic.metrics.foobar/period": "15s", + "not.to.include": "true", + }), + "namespace_annotations": getNestedAnnotations(common.MapStr{ + "co.elastic.metrics/module": "dropwizard", + "co.elastic.metrics/period": "60s", + "co.elastic.metrics.foobar/period": "25s", + }), + "container": common.MapStr{ + "name": "foobar", + "id": "abc", + "runtime": "docker", + }, + "namespace": "ns", + }, + "hints": common.MapStr{ + "metrics": common.MapStr{ + "module": "prometheus", + "period": "15s", + }, + }, + "container": common.MapStr{ + "name": "foobar", + "id": "abc", + "runtime": "docker", + }, + }, + }, + // Scenarios tested: + // Have no hints on the pod and have namespace level defaults. + // The resultant hints should honor only namespace defaults. + { + event: bus.Event{ + "kubernetes": common.MapStr{ + "namespace_annotations": getNestedAnnotations(common.MapStr{ + "co.elastic.metrics/module": "prometheus", + "co.elastic.metrics/period": "10s", + "co.elastic.metrics.foobar/period": "15s", + }), + "container": common.MapStr{ + "name": "foobar", + "id": "abc", + "runtime": "docker", + }, + "namespace": "ns", + }, + }, + result: bus.Event{ + "kubernetes": common.MapStr{ + "namespace_annotations": getNestedAnnotations(common.MapStr{ + "co.elastic.metrics/module": "prometheus", + "co.elastic.metrics/period": "10s", + "co.elastic.metrics.foobar/period": "15s", + }), + "container": common.MapStr{ + "name": "foobar", + "id": "abc", + "runtime": "docker", + }, + "namespace": "ns", + }, + "hints": common.MapStr{ + "metrics": common.MapStr{ + "module": "prometheus", + "period": "15s", + }, + }, + "container": common.MapStr{ + "name": "foobar", + "id": "abc", + "runtime": "docker", + }, + }, + }, } cfg := defaultConfig() diff --git a/libbeat/autodiscover/providers/kubernetes/service.go b/libbeat/autodiscover/providers/kubernetes/service.go index 8708833cd8d..c952143d3c8 100644 --- a/libbeat/autodiscover/providers/kubernetes/service.go +++ b/libbeat/autodiscover/providers/kubernetes/service.go @@ -124,15 +124,27 @@ func (s *service) GenerateHints(event bus.Event) bus.Event { // Try to build a config with enabled builders. Send a provider agnostic payload. // Builders are Beat specific. e := bus.Event{} - var annotations common.MapStr var kubeMeta common.MapStr + + annotations := make(common.MapStr, 0) rawMeta, ok := event["kubernetes"] if ok { kubeMeta = rawMeta.(common.MapStr) // The builder base config can configure any of the field values of kubernetes if need be. e["kubernetes"] = kubeMeta if rawAnn, ok := kubeMeta["annotations"]; ok { - annotations = rawAnn.(common.MapStr) + anns, _ := rawAnn.(common.MapStr) + if len(anns) != 0 { + annotations = anns.Clone() + } + } + + // Look at all the namespace level default annotations and do a merge with priority going to the pod annotations. + if rawNsAnn, ok := kubeMeta["namespace_annotations"]; ok { + nsAnn, _ := rawNsAnn.(common.MapStr) + if len(nsAnn) != 0 { + annotations.DeepUpdateNoOverwrite(nsAnn) + } } } if host, ok := event["host"]; ok { @@ -144,6 +156,7 @@ func (s *service) GenerateHints(event bus.Event) bus.Event { hints := builder.GenerateHints(annotations, "", s.config.Prefix) s.logger.Debugf("Generated hints %+v", hints) + if len(hints) != 0 { e["hints"] = hints } @@ -191,6 +204,19 @@ func (s *service) emit(svc *kubernetes.Service, flag string) { } kubemeta["annotations"] = annotations + if s.namespaceWatcher != nil { + if rawNs, ok, err := s.namespaceWatcher.Store().GetByKey(svc.Namespace); ok && err == nil { + if namespace, ok := rawNs.(*kubernetes.Namespace); ok { + nsAnns := common.MapStr{} + + for k, v := range namespace.GetAnnotations() { + safemapstr.Put(nsAnns, k, v) + } + kubemeta["namespace_annotations"] = nsAnns + } + } + } + for _, port := range svc.Spec.Ports { event := bus.Event{ "provider": s.uuid, diff --git a/libbeat/autodiscover/providers/kubernetes/service_test.go b/libbeat/autodiscover/providers/kubernetes/service_test.go index a17ef8a569d..6d6582b3ff2 100644 --- a/libbeat/autodiscover/providers/kubernetes/service_test.go +++ b/libbeat/autodiscover/providers/kubernetes/service_test.go @@ -98,6 +98,127 @@ func TestGenerateHints_Service(t *testing.T) { }, }, }, + // Scenarios tested: + // Have one set of annotations come from service and the other from namespace defaults + // The resultant should have both + { + event: bus.Event{ + "kubernetes": common.MapStr{ + "annotations": getNestedAnnotations(common.MapStr{ + "co.elastic.metrics/module": "prometheus", + "not.to.include": "true", + }), + "namespace_annotations": getNestedAnnotations(common.MapStr{ + "co.elastic.metrics/period": "10s", + }), + "service": common.MapStr{ + "name": "foobar", + }, + "namespace": "ns", + }, + }, + result: bus.Event{ + "kubernetes": common.MapStr{ + "annotations": getNestedAnnotations(common.MapStr{ + "co.elastic.metrics/module": "prometheus", + "not.to.include": "true", + }), + "service": common.MapStr{ + "name": "foobar", + }, + "namespace_annotations": getNestedAnnotations(common.MapStr{ + "co.elastic.metrics/period": "10s", + }), + "namespace": "ns", + }, + "hints": common.MapStr{ + "metrics": common.MapStr{ + "module": "prometheus", + "period": "10s", + }, + }, + }, + }, + // Scenarios tested: + // Have the same set of annotations come from both namespace and service. + // The resultant should have the ones from service alone + { + event: bus.Event{ + "kubernetes": common.MapStr{ + "annotations": getNestedAnnotations(common.MapStr{ + "co.elastic.metrics/module": "prometheus", + "co.elastic.metrics/period": "10s", + "not.to.include": "true", + }), + "namespace_annotations": getNestedAnnotations(common.MapStr{ + "co.elastic.metrics/module": "dropwizard", + "co.elastic.metrics/period": "60s", + }), + "namespace": "ns", + "service": common.MapStr{ + "name": "foobar", + }, + }, + }, + result: bus.Event{ + "kubernetes": common.MapStr{ + "annotations": getNestedAnnotations(common.MapStr{ + "co.elastic.metrics/module": "prometheus", + "co.elastic.metrics/period": "10s", + "not.to.include": "true", + }), + "namespace_annotations": getNestedAnnotations(common.MapStr{ + "co.elastic.metrics/module": "dropwizard", + "co.elastic.metrics/period": "60s", + }), + "namespace": "ns", + "service": common.MapStr{ + "name": "foobar", + }, + }, + "hints": common.MapStr{ + "metrics": common.MapStr{ + "module": "prometheus", + "period": "10s", + }, + }, + }, + }, + // Scenarios tested: + // Have no annotations on the service and only have namespace level defaults + // The resultant should have honored the namespace defaults + { + event: bus.Event{ + "kubernetes": common.MapStr{ + "namespace_annotations": getNestedAnnotations(common.MapStr{ + "co.elastic.metrics/module": "prometheus", + "co.elastic.metrics/period": "10s", + }), + "service": common.MapStr{ + "name": "foobar", + }, + "namespace": "ns", + }, + }, + result: bus.Event{ + "kubernetes": common.MapStr{ + "namespace_annotations": getNestedAnnotations(common.MapStr{ + "co.elastic.metrics/module": "prometheus", + "co.elastic.metrics/period": "10s", + }), + "service": common.MapStr{ + "name": "foobar", + }, + "namespace": "ns", + }, + "hints": common.MapStr{ + "metrics": common.MapStr{ + "module": "prometheus", + "period": "10s", + }, + }, + }, + }, } cfg := defaultConfig() diff --git a/metricbeat/docs/autodiscover-hints.asciidoc b/metricbeat/docs/autodiscover-hints.asciidoc index e12754d3313..1bc56d8f54d 100644 --- a/metricbeat/docs/autodiscover-hints.asciidoc +++ b/metricbeat/docs/autodiscover-hints.asciidoc @@ -126,6 +126,25 @@ annotations: co.elastic.metrics.sidecar/hosts: '${data.host}:8080' ------------------------------------------------------------------------------------- +[float] +===== Configuring Namespace Defaults + +Hints can be configured on the Namespace's annotations as defaults to use when Pod level annotations are missing. +The resultant hints are a combination of Pod annotations and Namespace annotations with the Pod's taking precedence. To +enable Namespace defaults configure the `add_resource_metadata` for Namespace objects as follows: + +["source","yaml",subs="attributes"] +------------------------------------------------------------------------------------- +metricbeat.autodiscover: + providers: + - type: kubernetes + hints.enabled: true + add_resource_metadata: + namespace: + enabled: true +------------------------------------------------------------------------------------- + + [float] === Docker