From 0bba2ae2d97b8df8a3e399736c8a3a2801d49ff8 Mon Sep 17 00:00:00 2001 From: Vijay Samuel Date: Thu, 13 Feb 2020 23:13:01 -0800 Subject: [PATCH 1/3] Add support for kubernetes provider to recognize namespace level default hints --- CHANGELOG.next.asciidoc | 2 ++ .../providers/kubernetes/kubernetes.go | 5 +++- .../autodiscover/providers/kubernetes/pod.go | 24 ++++++++++++++++++- .../providers/kubernetes/service.go | 22 +++++++++++++++++ 4 files changed, 51 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 33afd9eb33c2..7182cb17c9ae 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -138,6 +138,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Remove experimental flag from `setup.template.append_fields` {pull}16576[16576] - Add `add_cloudfoundry_metadata` processor to annotate events with Cloud Foundry application data. {pull}16621[16621] - Add Kerberos support to Kafka input and output. {pull}16781[16781] +- Add `add_cloudfoundry_metadata` processor to annotate events with Cloud Foundry application data. {pull}16621[16621 +- Add support for kubernetes provider to recognize namespace level defaults {pull}16321[16321] *Auditbeat* diff --git a/libbeat/autodiscover/providers/kubernetes/kubernetes.go b/libbeat/autodiscover/providers/kubernetes/kubernetes.go index a1e007acbe9a..2865e2d5576d 100644 --- a/libbeat/autodiscover/providers/kubernetes/kubernetes.go +++ b/libbeat/autodiscover/providers/kubernetes/kubernetes.go @@ -140,8 +140,11 @@ func (p *Provider) publish(event bus.Event) { event["config"] = config } else { // If there isn't a default template then attempt to use builders - if config := p.builders.GetConfig(p.eventer.GenerateHints(event)); config != nil { + e := p.eventer.GenerateHints(event) + if config := p.builders.GetConfig(e); config != nil { event["config"] = config + } else { + } } diff --git a/libbeat/autodiscover/providers/kubernetes/pod.go b/libbeat/autodiscover/providers/kubernetes/pod.go index 44f85a2363a3..38d23ae97fcc 100644 --- a/libbeat/autodiscover/providers/kubernetes/pod.go +++ b/libbeat/autodiscover/providers/kubernetes/pod.go @@ -23,6 +23,7 @@ import ( "github.com/gofrs/uuid" k8s "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" "github.com/elastic/beats/v7/libbeat/autodiscover/builder" "github.com/elastic/beats/v7/libbeat/common" @@ -42,6 +43,7 @@ type pod struct { watcher kubernetes.Watcher nodeWatcher kubernetes.Watcher namespaceWatcher kubernetes.Watcher + namespaceStore cache.Store } // NewPodEventer creates an eventer that can discover and process pod objects @@ -188,10 +190,18 @@ func (p *pod) GenerateHints(event bus.Event) bus.Event { cname := builder.GetContainerName(container) hints := builder.GenerateHints(annotations, cname, p.config.Prefix) p.logger.Debugf("Generated hints %+v", hints) + + // Fall back to defaults on the namespace if there were no hints on the pods + if len(hints) == 0 { + if rawAnn, ok := kubeMeta["defaults"]; ok { + annotations = rawAnn.(common.MapStr) + hints = builder.GenerateHints(annotations, cname, p.config.Prefix) + } + } + if len(hints) != 0 { e["hints"] = hints } - p.logger.Debugf("Generated builder event %+v", e) return e @@ -296,6 +306,18 @@ func (p *pod) emitEvents(pod *kubernetes.Pod, flag string, containers []kubernet safemapstr.Put(annotations, k, v) } kubemeta["annotations"] = annotations + if p.namespaceWatcher != nil { + if rawNs, ok, err := p.namespaceWatcher.Store().GetByKey(pod.Namespace); ok && err == nil { + if namespace, ok := rawNs.(*kubernetes.Namespace); ok { + defaults := common.MapStr{} + + for k, v := range namespace.GetAnnotations() { + safemapstr.Put(defaults, k, v) + } + kubemeta["defaults"] = defaults + } + } + } // Without this check there would be overlapping configurations with and without ports. if len(c.Ports) == 0 { diff --git a/libbeat/autodiscover/providers/kubernetes/service.go b/libbeat/autodiscover/providers/kubernetes/service.go index 8708833cd8dd..a33145d3014b 100644 --- a/libbeat/autodiscover/providers/kubernetes/service.go +++ b/libbeat/autodiscover/providers/kubernetes/service.go @@ -144,6 +144,15 @@ func (s *service) GenerateHints(event bus.Event) bus.Event { hints := builder.GenerateHints(annotations, "", s.config.Prefix) s.logger.Debugf("Generated hints %+v", hints) + + // Fall back to defaults on the namespace if there were no hints on the pods + if len(hints) == 0 { + if rawAnn, ok := kubeMeta["defaults"]; ok { + annotations = rawAnn.(common.MapStr) + hints = builder.GenerateHints(annotations, "", s.config.Prefix) + } + } + if len(hints) != 0 { e["hints"] = hints } @@ -191,6 +200,19 @@ func (s *service) emit(svc *kubernetes.Service, flag string) { } kubemeta["annotations"] = annotations + if s.namespaceWatcher != nil { + if rawNs, ok, err := s.namespaceWatcher.Store().GetByKey(svc.Namespace); ok && err == nil { + if namespace, ok := rawNs.(*kubernetes.Namespace); ok { + defaults := common.MapStr{} + + for k, v := range namespace.GetAnnotations() { + safemapstr.Put(defaults, k, v) + } + kubemeta["defaults"] = defaults + } + } + } + for _, port := range svc.Spec.Ports { event := bus.Event{ "provider": s.uuid, From c377c019afcab12fa05fa5f2c7abd146eb0024ae Mon Sep 17 00:00:00 2001 From: Vijay Samuel Date: Mon, 24 Feb 2020 10:44:35 -0800 Subject: [PATCH 2/3] Incorporate review comments --- .../providers/kubernetes/kubernetes.go | 2 - .../autodiscover/providers/kubernetes/pod.go | 32 ++-- .../providers/kubernetes/pod_test.go | 168 ++++++++++++++++++ .../providers/kubernetes/service.go | 30 ++-- .../providers/kubernetes/service_test.go | 121 +++++++++++++ 5 files changed, 325 insertions(+), 28 deletions(-) diff --git a/libbeat/autodiscover/providers/kubernetes/kubernetes.go b/libbeat/autodiscover/providers/kubernetes/kubernetes.go index 2865e2d5576d..ec3480fa00b7 100644 --- a/libbeat/autodiscover/providers/kubernetes/kubernetes.go +++ b/libbeat/autodiscover/providers/kubernetes/kubernetes.go @@ -143,8 +143,6 @@ func (p *Provider) publish(event bus.Event) { e := p.eventer.GenerateHints(event) if config := p.builders.GetConfig(e); config != nil { event["config"] = config - } else { - } } diff --git a/libbeat/autodiscover/providers/kubernetes/pod.go b/libbeat/autodiscover/providers/kubernetes/pod.go index 38d23ae97fcc..b7239e3004e8 100644 --- a/libbeat/autodiscover/providers/kubernetes/pod.go +++ b/libbeat/autodiscover/providers/kubernetes/pod.go @@ -161,15 +161,27 @@ func (p *pod) GenerateHints(event bus.Event) bus.Event { // Try to build a config with enabled builders. Send a provider agnostic payload. // Builders are Beat specific. e := bus.Event{} - var annotations common.MapStr var kubeMeta, container common.MapStr + + annotations := make(common.MapStr, 0) rawMeta, ok := event["kubernetes"] if ok { kubeMeta = rawMeta.(common.MapStr) // The builder base config can configure any of the field values of kubernetes if need be. e["kubernetes"] = kubeMeta if rawAnn, ok := kubeMeta["annotations"]; ok { - annotations = rawAnn.(common.MapStr) + anns, _ := rawAnn.(common.MapStr) + if len(anns) != 0 { + annotations = anns.Clone() + } + } + + // Look at all the namespace level default annotations and do a merge with priority going to the pod annotations. + if rawNsAnn, ok := kubeMeta["namespace_annotations"]; ok { + nsAnn, _ := rawNsAnn.(common.MapStr) + if len(nsAnn) != 0 { + annotations.DeepUpdateNoOverwrite(nsAnn) + } } } if host, ok := event["host"]; ok { @@ -188,17 +200,11 @@ func (p *pod) GenerateHints(event bus.Event) bus.Event { } cname := builder.GetContainerName(container) + + // Generate hints based on the cumulative of both namespace and pod annotations. hints := builder.GenerateHints(annotations, cname, p.config.Prefix) p.logger.Debugf("Generated hints %+v", hints) - // Fall back to defaults on the namespace if there were no hints on the pods - if len(hints) == 0 { - if rawAnn, ok := kubeMeta["defaults"]; ok { - annotations = rawAnn.(common.MapStr) - hints = builder.GenerateHints(annotations, cname, p.config.Prefix) - } - } - if len(hints) != 0 { e["hints"] = hints } @@ -309,12 +315,12 @@ func (p *pod) emitEvents(pod *kubernetes.Pod, flag string, containers []kubernet if p.namespaceWatcher != nil { if rawNs, ok, err := p.namespaceWatcher.Store().GetByKey(pod.Namespace); ok && err == nil { if namespace, ok := rawNs.(*kubernetes.Namespace); ok { - defaults := common.MapStr{} + nsAnn := common.MapStr{} for k, v := range namespace.GetAnnotations() { - safemapstr.Put(defaults, k, v) + safemapstr.Put(nsAnn, k, v) } - kubemeta["defaults"] = defaults + kubemeta["namespace_annotations"] = nsAnn } } } diff --git a/libbeat/autodiscover/providers/kubernetes/pod_test.go b/libbeat/autodiscover/providers/kubernetes/pod_test.go index b105aa0991fe..f63dbdf1e734 100644 --- a/libbeat/autodiscover/providers/kubernetes/pod_test.go +++ b/libbeat/autodiscover/providers/kubernetes/pod_test.go @@ -150,6 +150,174 @@ func TestGenerateHints(t *testing.T) { }, }, }, + // Scenarios tested: + // Have one set of hints come from the pod and the other come from namespaces + // The resultant hints should have a combination of both + { + event: bus.Event{ + "kubernetes": common.MapStr{ + "annotations": getNestedAnnotations(common.MapStr{ + "co.elastic.logs/multiline.pattern": "^test", + "co.elastic.logs/json.keys_under_root": "true", + "not.to.include": "true", + }), + "namespace_annotations": getNestedAnnotations(common.MapStr{ + "co.elastic.metrics/module": "prometheus", + "co.elastic.metrics/period": "10s", + "co.elastic.metrics.foobar/period": "15s", + }), + "container": common.MapStr{ + "name": "foobar", + "id": "abc", + "runtime": "docker", + }, + "namespace": "ns", + }, + }, + result: bus.Event{ + "kubernetes": common.MapStr{ + "annotations": getNestedAnnotations(common.MapStr{ + "co.elastic.logs/multiline.pattern": "^test", + "co.elastic.logs/json.keys_under_root": "true", + "not.to.include": "true", + }), + "namespace_annotations": getNestedAnnotations(common.MapStr{ + "co.elastic.metrics/period": "10s", + "co.elastic.metrics.foobar/period": "15s", + "co.elastic.metrics/module": "prometheus", + }), + "container": common.MapStr{ + "name": "foobar", + "id": "abc", + "runtime": "docker", + }, + "namespace": "ns", + }, + "hints": common.MapStr{ + "logs": common.MapStr{ + "multiline": common.MapStr{ + "pattern": "^test", + }, + "json": common.MapStr{ + "keys_under_root": "true", + }, + }, + "metrics": common.MapStr{ + "module": "prometheus", + "period": "15s", + }, + }, + "container": common.MapStr{ + "name": "foobar", + "id": "abc", + "runtime": "docker", + }, + }, + }, + // Scenarios tested: + // Have one set of hints come from the pod and the same keys come from namespaces + // The resultant hints should honor only pods and not namespace. + { + event: bus.Event{ + "kubernetes": common.MapStr{ + "annotations": getNestedAnnotations(common.MapStr{ + "co.elastic.metrics/module": "prometheus", + "co.elastic.metrics/period": "10s", + "co.elastic.metrics.foobar/period": "15s", + "not.to.include": "true", + }), + "namespace_annotations": getNestedAnnotations(common.MapStr{ + "co.elastic.metrics/module": "dropwizard", + "co.elastic.metrics/period": "60s", + "co.elastic.metrics.foobar/period": "25s", + }), + "namespace": "ns", + "container": common.MapStr{ + "name": "foobar", + "id": "abc", + "runtime": "docker", + }, + }, + }, + result: bus.Event{ + "kubernetes": common.MapStr{ + "annotations": getNestedAnnotations(common.MapStr{ + "co.elastic.metrics/module": "prometheus", + "co.elastic.metrics/period": "10s", + "co.elastic.metrics.foobar/period": "15s", + "not.to.include": "true", + }), + "namespace_annotations": getNestedAnnotations(common.MapStr{ + "co.elastic.metrics/module": "dropwizard", + "co.elastic.metrics/period": "60s", + "co.elastic.metrics.foobar/period": "25s", + }), + "container": common.MapStr{ + "name": "foobar", + "id": "abc", + "runtime": "docker", + }, + "namespace": "ns", + }, + "hints": common.MapStr{ + "metrics": common.MapStr{ + "module": "prometheus", + "period": "15s", + }, + }, + "container": common.MapStr{ + "name": "foobar", + "id": "abc", + "runtime": "docker", + }, + }, + }, + // Scenarios tested: + // Have no hints on the pod and have namespace level defaults. + // The resultant hints should honor only namespace defaults. + { + event: bus.Event{ + "kubernetes": common.MapStr{ + "namespace_annotations": getNestedAnnotations(common.MapStr{ + "co.elastic.metrics/module": "prometheus", + "co.elastic.metrics/period": "10s", + "co.elastic.metrics.foobar/period": "15s", + }), + "container": common.MapStr{ + "name": "foobar", + "id": "abc", + "runtime": "docker", + }, + "namespace": "ns", + }, + }, + result: bus.Event{ + "kubernetes": common.MapStr{ + "namespace_annotations": getNestedAnnotations(common.MapStr{ + "co.elastic.metrics/module": "prometheus", + "co.elastic.metrics/period": "10s", + "co.elastic.metrics.foobar/period": "15s", + }), + "container": common.MapStr{ + "name": "foobar", + "id": "abc", + "runtime": "docker", + }, + "namespace": "ns", + }, + "hints": common.MapStr{ + "metrics": common.MapStr{ + "module": "prometheus", + "period": "15s", + }, + }, + "container": common.MapStr{ + "name": "foobar", + "id": "abc", + "runtime": "docker", + }, + }, + }, } cfg := defaultConfig() diff --git a/libbeat/autodiscover/providers/kubernetes/service.go b/libbeat/autodiscover/providers/kubernetes/service.go index a33145d3014b..c952143d3c84 100644 --- a/libbeat/autodiscover/providers/kubernetes/service.go +++ b/libbeat/autodiscover/providers/kubernetes/service.go @@ -124,15 +124,27 @@ func (s *service) GenerateHints(event bus.Event) bus.Event { // Try to build a config with enabled builders. Send a provider agnostic payload. // Builders are Beat specific. e := bus.Event{} - var annotations common.MapStr var kubeMeta common.MapStr + + annotations := make(common.MapStr, 0) rawMeta, ok := event["kubernetes"] if ok { kubeMeta = rawMeta.(common.MapStr) // The builder base config can configure any of the field values of kubernetes if need be. e["kubernetes"] = kubeMeta if rawAnn, ok := kubeMeta["annotations"]; ok { - annotations = rawAnn.(common.MapStr) + anns, _ := rawAnn.(common.MapStr) + if len(anns) != 0 { + annotations = anns.Clone() + } + } + + // Look at all the namespace level default annotations and do a merge with priority going to the pod annotations. + if rawNsAnn, ok := kubeMeta["namespace_annotations"]; ok { + nsAnn, _ := rawNsAnn.(common.MapStr) + if len(nsAnn) != 0 { + annotations.DeepUpdateNoOverwrite(nsAnn) + } } } if host, ok := event["host"]; ok { @@ -145,14 +157,6 @@ func (s *service) GenerateHints(event bus.Event) bus.Event { hints := builder.GenerateHints(annotations, "", s.config.Prefix) s.logger.Debugf("Generated hints %+v", hints) - // Fall back to defaults on the namespace if there were no hints on the pods - if len(hints) == 0 { - if rawAnn, ok := kubeMeta["defaults"]; ok { - annotations = rawAnn.(common.MapStr) - hints = builder.GenerateHints(annotations, "", s.config.Prefix) - } - } - if len(hints) != 0 { e["hints"] = hints } @@ -203,12 +207,12 @@ func (s *service) emit(svc *kubernetes.Service, flag string) { if s.namespaceWatcher != nil { if rawNs, ok, err := s.namespaceWatcher.Store().GetByKey(svc.Namespace); ok && err == nil { if namespace, ok := rawNs.(*kubernetes.Namespace); ok { - defaults := common.MapStr{} + nsAnns := common.MapStr{} for k, v := range namespace.GetAnnotations() { - safemapstr.Put(defaults, k, v) + safemapstr.Put(nsAnns, k, v) } - kubemeta["defaults"] = defaults + kubemeta["namespace_annotations"] = nsAnns } } } diff --git a/libbeat/autodiscover/providers/kubernetes/service_test.go b/libbeat/autodiscover/providers/kubernetes/service_test.go index a17ef8a569d7..6d6582b3ff2d 100644 --- a/libbeat/autodiscover/providers/kubernetes/service_test.go +++ b/libbeat/autodiscover/providers/kubernetes/service_test.go @@ -98,6 +98,127 @@ func TestGenerateHints_Service(t *testing.T) { }, }, }, + // Scenarios tested: + // Have one set of annotations come from service and the other from namespace defaults + // The resultant should have both + { + event: bus.Event{ + "kubernetes": common.MapStr{ + "annotations": getNestedAnnotations(common.MapStr{ + "co.elastic.metrics/module": "prometheus", + "not.to.include": "true", + }), + "namespace_annotations": getNestedAnnotations(common.MapStr{ + "co.elastic.metrics/period": "10s", + }), + "service": common.MapStr{ + "name": "foobar", + }, + "namespace": "ns", + }, + }, + result: bus.Event{ + "kubernetes": common.MapStr{ + "annotations": getNestedAnnotations(common.MapStr{ + "co.elastic.metrics/module": "prometheus", + "not.to.include": "true", + }), + "service": common.MapStr{ + "name": "foobar", + }, + "namespace_annotations": getNestedAnnotations(common.MapStr{ + "co.elastic.metrics/period": "10s", + }), + "namespace": "ns", + }, + "hints": common.MapStr{ + "metrics": common.MapStr{ + "module": "prometheus", + "period": "10s", + }, + }, + }, + }, + // Scenarios tested: + // Have the same set of annotations come from both namespace and service. + // The resultant should have the ones from service alone + { + event: bus.Event{ + "kubernetes": common.MapStr{ + "annotations": getNestedAnnotations(common.MapStr{ + "co.elastic.metrics/module": "prometheus", + "co.elastic.metrics/period": "10s", + "not.to.include": "true", + }), + "namespace_annotations": getNestedAnnotations(common.MapStr{ + "co.elastic.metrics/module": "dropwizard", + "co.elastic.metrics/period": "60s", + }), + "namespace": "ns", + "service": common.MapStr{ + "name": "foobar", + }, + }, + }, + result: bus.Event{ + "kubernetes": common.MapStr{ + "annotations": getNestedAnnotations(common.MapStr{ + "co.elastic.metrics/module": "prometheus", + "co.elastic.metrics/period": "10s", + "not.to.include": "true", + }), + "namespace_annotations": getNestedAnnotations(common.MapStr{ + "co.elastic.metrics/module": "dropwizard", + "co.elastic.metrics/period": "60s", + }), + "namespace": "ns", + "service": common.MapStr{ + "name": "foobar", + }, + }, + "hints": common.MapStr{ + "metrics": common.MapStr{ + "module": "prometheus", + "period": "10s", + }, + }, + }, + }, + // Scenarios tested: + // Have no annotations on the service and only have namespace level defaults + // The resultant should have honored the namespace defaults + { + event: bus.Event{ + "kubernetes": common.MapStr{ + "namespace_annotations": getNestedAnnotations(common.MapStr{ + "co.elastic.metrics/module": "prometheus", + "co.elastic.metrics/period": "10s", + }), + "service": common.MapStr{ + "name": "foobar", + }, + "namespace": "ns", + }, + }, + result: bus.Event{ + "kubernetes": common.MapStr{ + "namespace_annotations": getNestedAnnotations(common.MapStr{ + "co.elastic.metrics/module": "prometheus", + "co.elastic.metrics/period": "10s", + }), + "service": common.MapStr{ + "name": "foobar", + }, + "namespace": "ns", + }, + "hints": common.MapStr{ + "metrics": common.MapStr{ + "module": "prometheus", + "period": "10s", + }, + }, + }, + }, } cfg := defaultConfig() From 54d0b5c4726bcc01d48f6f281fc7b020258a9ba5 Mon Sep 17 00:00:00 2001 From: Vijay Samuel Date: Mon, 2 Mar 2020 22:26:03 -0800 Subject: [PATCH 3/3] Add documentation --- filebeat/docs/autodiscover-hints.asciidoc | 18 ++++++++++++++++++ metricbeat/docs/autodiscover-hints.asciidoc | 19 +++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/filebeat/docs/autodiscover-hints.asciidoc b/filebeat/docs/autodiscover-hints.asciidoc index a7caddb30482..8201de0e6204 100644 --- a/filebeat/docs/autodiscover-hints.asciidoc +++ b/filebeat/docs/autodiscover-hints.asciidoc @@ -155,6 +155,24 @@ annotations: co.elastic.logs.sidecar/exclude_lines: '^DBG' ----- +[float] +===== Configuring Namespace Defaults + +Hints can be configured on the Namespace's annotations as defaults to use when Pod level annotations are missing. +The resultant hints are a combination of Pod annotations and Namespace annotations with the Pod's taking precedence. To +enable Namespace defaults configure the `add_resource_metadata` for Namespace objects as follows: + +["source","yaml",subs="attributes"] +------------------------------------------------------------------------------------- +filebeat.autodiscover: + providers: + - type: kubernetes + hints.enabled: true + add_resource_metadata: + namespace: + enabled: true +------------------------------------------------------------------------------------- + [float] diff --git a/metricbeat/docs/autodiscover-hints.asciidoc b/metricbeat/docs/autodiscover-hints.asciidoc index e12754d3313b..1bc56d8f54dc 100644 --- a/metricbeat/docs/autodiscover-hints.asciidoc +++ b/metricbeat/docs/autodiscover-hints.asciidoc @@ -126,6 +126,25 @@ annotations: co.elastic.metrics.sidecar/hosts: '${data.host}:8080' ------------------------------------------------------------------------------------- +[float] +===== Configuring Namespace Defaults + +Hints can be configured on the Namespace's annotations as defaults to use when Pod level annotations are missing. +The resultant hints are a combination of Pod annotations and Namespace annotations with the Pod's taking precedence. To +enable Namespace defaults configure the `add_resource_metadata` for Namespace objects as follows: + +["source","yaml",subs="attributes"] +------------------------------------------------------------------------------------- +metricbeat.autodiscover: + providers: + - type: kubernetes + hints.enabled: true + add_resource_metadata: + namespace: + enabled: true +------------------------------------------------------------------------------------- + + [float] === Docker