Bring back add_resource_metadata section

Signed-off-by: chrismark <[email protected]>

CHANGELOG.next.asciidoc

@@ -490,7 +490,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add kubernetes.node.hostname metadata of Kubernetes node. {pull}22189[22189]
 - Include node metadata and namespace metadata in pod's metadata by default for autodiscovery. {pull}22189[22189]
 - Include node metadata and namespace metadata in pod's metadata by default for add_kuberentes_metadata. {pull}22189[22189]
-- Deprecate add_resource_metadata of kubernetes autodiscovery. {pull}22189[22189]
+- Enable always add_resource_metadata for Pods and Services of kubernetes autodiscovery. {pull}22189[22189]
+- Add add_resource_metadata option setting (always enabled) for add_kubernetes_metadata setting. {pull}22189[22189]
 - Added Kafka version 2.2 to the list of supported versions. {pull}22328[22328]
 
 *Auditbeat*

filebeat/docs/autodiscover-hints.asciidoc

@@ -175,7 +175,20 @@ tokenizer. The second input handles everything but debug logs.
 =====  Namespace Defaults
 
 Hints can be configured on the Namespace's annotations as defaults to use when Pod level annotations are missing.
-The resultant hints are a combination of Pod annotations and Namespace annotations with the Pod's taking precedence.
+The resultant hints are a combination of Pod annotations and Namespace annotations with the Pod's taking precedence. To
+enable Namespace defaults configure the `add_resource_metadata` for Namespace objects as follows:
+
+["source","yaml",subs="attributes"]
+-------------------------------------------------------------------------------------
+filebeat.autodiscover:
+  providers:
+    - type: kubernetes
+      hints.enabled: true
+      add_resource_metadata:
+        namespace:
+          include_annotations: ["nsannotation1"]
+-------------------------------------------------------------------------------------
+
 
 
 [float]

libbeat/autodiscover/providers/kubernetes/config.go

@@ -23,6 +23,8 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/elastic/beats/v7/libbeat/common/kubernetes/metadata"
+
 	"github.com/elastic/beats/v7/libbeat/autodiscover/template"
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/libbeat/common/cfgwarn"
@@ -51,6 +53,8 @@ type Config struct {
 	Builders  []*common.Config        `config:"builders"`
 	Appenders []*common.Config        `config:"appenders"`
 	Templates template.MapperSettings `config:"templates"`
+
+	AddResourceMetadata *metadata.AddResourceMetadataConfig `config:"add_resource_metadata"`
 }
 
 func defaultConfig() *Config {

libbeat/autodiscover/providers/kubernetes/pod.go

@@ -82,7 +82,10 @@ func NewPodEventer(uuid uuid.UUID, cfg *common.Config, client k8s.Interface, pub
 	if config.Namespace != "" {
 		options.Namespace = config.Namespace
 	}
-
+	metaConf := config.AddResourceMetadata
+	if metaConf == nil {
+		metaConf = metadata.GetDefaultResourceMetadataConfig()
+	}
 	nodeWatcher, err := kubernetes.NewWatcher(client, &kubernetes.Node{}, options, nil)
 	if err != nil {
 		return nil, fmt.Errorf("couldn't create watcher for %T due to error %+v", &kubernetes.Node{}, err)
@@ -93,7 +96,7 @@ func NewPodEventer(uuid uuid.UUID, cfg *common.Config, client k8s.Interface, pub
 	if err != nil {
 		return nil, fmt.Errorf("couldn't create watcher for %T due to error %+v", &kubernetes.Namespace{}, err)
 	}
-	metaGen := metadata.GetPodMetaGen(cfg, watcher, nodeWatcher, namespaceWatcher)
+	metaGen := metadata.GetPodMetaGen(cfg, watcher, nodeWatcher, namespaceWatcher, metaConf)
 
 	p := &pod{
 		config:           config,

libbeat/common/kubernetes/metadata/metadata.go

@@ -57,9 +57,9 @@ func GetPodMetaGen(
 	cfg *common.Config,
 	podWatcher kubernetes.Watcher,
 	nodeWatcher kubernetes.Watcher,
-	namespaceWatcher kubernetes.Watcher) MetaGen {
+	namespaceWatcher kubernetes.Watcher,
+	metaConf *AddResourceMetadataConfig) MetaGen {
 
-	metaConf := GetDefaultResourceMetadataConfig()
 	nodeMetaGen := NewNodeMetadataGenerator(metaConf.Node, nodeWatcher.Store())
 	namespaceMetaGen := NewNamespaceMetadataGenerator(metaConf.Namespace, namespaceWatcher.Store())
 	metaGen := NewPodMetadataGenerator(cfg, podWatcher.Store(), nodeMetaGen, namespaceMetaGen)

libbeat/common/kubernetes/metadata/resource.go

@@ -61,12 +61,7 @@ func (r *Resource) Generate(kind string, obj kubernetes.Resource, options ...Fie
 		labelMap.Delete(label)
 	}
 
-	annotationsMap := common.MapStr{}
-	if len(r.config.IncludeAnnotations) == 0 {
-		annotationsMap = generateMap(accessor.GetAnnotations(), r.config.LabelsDedot)
-	} else {
-		annotationsMap = generateMapSubset(accessor.GetAnnotations(), r.config.IncludeAnnotations, r.config.LabelsDedot)
-	}
+	annotationsMap := generateMapSubset(accessor.GetAnnotations(), r.config.IncludeAnnotations, r.config.LabelsDedot)
 
 	meta := common.MapStr{
 		strings.ToLower(kind): common.MapStr{

libbeat/docs/shared-autodiscover.asciidoc

@@ -229,10 +229,26 @@ running configuration for a container, 60s by default.
   either take `node` or `cluster` as values. `node` scope allows discovery of resources in
   the specified node. `cluster` scope allows cluster wide discovery. Only `pod` and `node` resources
   can be discovered at node scope.
+`add_resource_metadata`:: (Optional) Specify labels and annotations filters for the extra metadata coming from Node and Namespace.
+ `add_resource_metadata` can be done for `node` or `namespace`. By default all labels will be included
+  while annotations are not added by default. This settings are useful when labels' and annotations'
+  storing requires special handling to avoid overloading the storage output.
+  Example:
+
+["source","yaml",subs="attributes"]
+-------------------------------------------------------------------------------------
+      add_resource_metadata:
+        namespace:
+          include_labels: ["namespacelabel1"]
+        node:
+          include_labels: ["nodelabel2"]
+          include_annotations: ["nodeannotation1"]
+-------------------------------------------------------------------------------------
+
 `unique`:: (Optional) Defaults to `false`. Marking an autodiscover provider as unique results into
   making the provider to enable the provided templates only when it will gain the leader lease.
   This setting can only be combined with `cluster` scope. When `unique` is enabled enabled, `resource`
-  setting is not taken into account.
+  and `add_resource_metadata` settings are not taken into account.
 `leader_lease`:: (Optional) Defaults to `{beatname_lc}-cluster-leader`. This will be name of the lock lease.
   One can monitor the status of the lease with `kubectl describe lease beats-cluster-leader`.
   Different Beats that refer to the same leader lease will be competetitors in holding the lease

libbeat/processors/add_kubernetes_metadata/config.go

@@ -21,6 +21,7 @@ import (
 	"time"
 
 	"github.com/elastic/beats/v7/libbeat/common"
+	"github.com/elastic/beats/v7/libbeat/common/kubernetes/metadata"
 )
 
 type kubeAnnotatorConfig struct {
@@ -35,6 +36,8 @@ type kubeAnnotatorConfig struct {
 	Matchers        PluginConfig  `config:"matchers"`
 	DefaultMatchers Enabled       `config:"default_matchers"`
 	DefaultIndexers Enabled       `config:"default_indexers"`
+
+	AddResourceMetadata *metadata.AddResourceMetadataConfig `config:"add_resource_metadata"`
 }
 
 type Enabled struct {

libbeat/processors/add_kubernetes_metadata/docs/add_kubernetes_metadata.asciidoc

@@ -19,6 +19,8 @@ Each event is annotated with:
 * Namespace
 * Labels
 
+In addition Node's metadata and Namespace's metadata are being added in Pod's metadata.
+
 The `add_kubernetes_metadata` processor has two basic building blocks which are:
 
 * Indexers
@@ -116,6 +118,21 @@ mode.
 `namespace`:: (Optional) Select the namespace from which to collect the
 metadata. If it is not set, the processor collects metadata from all namespaces.
 It is unset by default.
+`add_resource_metadata`:: (Optional) Specify labels and annotations filters for the extra metadata coming from Node and Namespace.
+ `add_resource_metadata` can be done for `node` or `namespace`. By default all labels will be included
+while annotations are not added by default. This settings are useful when labels' and annotations'
+storing requires special handling to avoid overloading the storage output.
+Example:
+
+["source","yaml",subs="attributes"]
+-------------------------------------------------------------------------------------
+      add_resource_metadata:
+        namespace:
+          include_labels: ["namespacelabel1"]
+        node:
+          include_labels: ["nodelabel2"]
+          include_annotations: ["nodeannotation1"]
+-------------------------------------------------------------------------------------
 `kube_config`:: (Optional) Use given config file as configuration for Kubernetes
 client. It defaults to `KUBECONFIG` environment variable if present.
 `default_indexers.enabled`:: (Optional) Enable/Disable default pod indexers, in

libbeat/processors/add_kubernetes_metadata/kubernetes.go

@@ -171,14 +171,18 @@ func (k *kubernetesAnnotator) init(config kubeAnnotatorConfig, cfg *common.Confi
 			return
 		}
 
+		metaConf := config.AddResourceMetadata
+		if metaConf == nil {
+			metaConf = metadata.GetDefaultResourceMetadataConfig()
+		}
+
 		options := kubernetes.WatchOptions{
 			SyncTimeout: config.SyncPeriod,
 			Node:        "",
 		}
 		if config.Namespace != "" {
 			options.Namespace = config.Namespace
 		}
-
 		nodeWatcher, err := kubernetes.NewWatcher(client, &kubernetes.Node{}, options, nil)
 		if err != nil {
 			k.log.Errorf("couldn't create watcher for %T due to error %+v", &kubernetes.Node{}, err)
@@ -190,7 +194,7 @@ func (k *kubernetesAnnotator) init(config kubeAnnotatorConfig, cfg *common.Confi
 			k.log.Errorf("couldn't create watcher for %T due to error %+v", &kubernetes.Namespace{}, err)
 		}
 		// TODO: refactor the above section to a common function to be used by NeWPodEventer too
-		metaGen := metadata.GetPodMetaGen(cfg, watcher, nodeWatcher, namespaceWatcher)
+		metaGen := metadata.GetPodMetaGen(cfg, watcher, nodeWatcher, namespaceWatcher, metaConf)
 
 		k.indexers = NewIndexers(config.Indexers, metaGen)
 		k.watcher = watcher

metricbeat/docs/autodiscover-hints.asciidoc

@@ -155,7 +155,20 @@ polled every 60s whereas the "/metrics" endpoint is polled every 60s.
 =====  Namespace Defaults
 
 Hints can be configured on the Namespace's annotations as defaults to use when Pod level annotations are missing.
-The resultant hints are a combination of Pod annotations and Namespace annotations with the Pod's taking precedence.
+The resultant hints are a combination of Pod annotations and Namespace annotations with the Pod's taking precedence. To
+enable Namespace defaults configure the `add_resource_metadata` for Namespace objects as follows:
+
+["source","yaml",subs="attributes"]
+-------------------------------------------------------------------------------------
+metricbeat.autodiscover:
+  providers:
+    - type: kubernetes
+      hints.enabled: true
+      add_resource_metadata:
+        namespace:
+          include_annotations: ["nsannotation1"]
+-------------------------------------------------------------------------------------
+
 
 [float]
 === Docker