Merge branch 'master' of github.com:elastic/beats

elastic · Sep 9, 2020 · 5100e6a · 5100e6a
2 parents b2625ca + cce29d4
commit 5100e6a
Show file tree

Hide file tree

Showing 23 changed files with 191 additions and 81 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -337,6 +337,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fill cloud.account.name with accountID if account alias doesn't exist. {pull}20736[20736]
 - Update fields.yml in the azure module, missing metrics field. {pull}20918[20918]
 - The `elasticsearch/index` metricset only requests wildcard expansion for hidden indices if the monitored Elasticsearch cluster supports it. {pull}20938[20938]
+- Disable Kafka metricsets based on Jolokia by default. They require a different configuration. {pull}20989[20989]
 
 *Packetbeat*
 
@@ -404,6 +405,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add leader election for Kubernetes autodiscover. {pull}20281[20281]
 - Add capability of enriching process metadata with contianer id also for non-privileged containers in `add_process_metadata` processor. {pull}19767[19767]
 - Add replace_fields config option in add_host_metadata for replacing host fields. {pull}20490[20490] {issue}20464[20464]
+- Add container ECS fields in kubernetes metadata. {pull}20984[20984]
 
 *Auditbeat*
 
@@ -672,6 +674,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Request prometheus endpoints to be gzipped by default {pull}20766[20766]
 - Release all kubernetes `state` metricsets as GA {pull}20901[20901]
 - Add billing metricset into googlecloud module. {pull}20812[20812] {issue}20738[20738]
+- Sanitize `event.host`. {pull}21022[21022]
 
 *Packetbeat*
 

diff --git a/NOTICE.txt b/NOTICE.txt
@@ -13971,11 +13971,11 @@ Contents of probable licence file $GOMODCACHE/github.com/xdg/[email protected]
 
 --------------------------------------------------------------------------------
 Dependency : go.elastic.co/apm
-Version: v1.8.1-0.20200902013556-b34fe04da73f
+Version: v1.8.1-0.20200909061013-2aef45b9cf4b
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/go.elastic.co/[email protected].20200902013556-b34fe04da73f/LICENSE:
+Contents of probable licence file $GOMODCACHE/go.elastic.co/[email protected].20200909061013-2aef45b9cf4b/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004

diff --git a/go.mod b/go.mod
@@ -151,7 +151,7 @@ require (
 	github.com/vmware/govmomi v0.0.0-20170802214208-2cad15190b41
 	github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c
 	github.com/yuin/gopher-lua v0.0.0-20170403160031-b402f3114ec7 // indirect
-	go.elastic.co/apm v1.8.1-0.20200902013556-b34fe04da73f
+	go.elastic.co/apm v1.8.1-0.20200909061013-2aef45b9cf4b
 	go.elastic.co/apm/module/apmelasticsearch v1.7.2
 	go.elastic.co/apm/module/apmhttp v1.7.2
 	go.elastic.co/ecszap v0.1.1-0.20200424093508-cdd95a104193
@@ -168,7 +168,7 @@ require (
 	golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae
 	golang.org/x/text v0.3.2
 	golang.org/x/time v0.0.0-20191024005414-555d28b269f0
-	golang.org/x/tools v0.0.0-20200806022845-90696ccdc692
+	golang.org/x/tools v0.0.0-20200904185747-39188db58858
 	google.golang.org/api v0.15.0
 	google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb
 	google.golang.org/grpc v1.29.1

diff --git a/go.sum b/go.sum
@@ -699,8 +699,8 @@ github.com/yuin/gopher-lua v0.0.0-20170403160031-b402f3114ec7 h1:0gYLpmzecnaDCoe
 github.com/yuin/gopher-lua v0.0.0-20170403160031-b402f3114ec7/go.mod h1:aEV29XrmTYFr3CiRxZeGHpkvbwq+prZduBqMaascyCU=
 go.elastic.co/apm v1.7.2 h1:0nwzVIPp4PDBXSYYtN19+1W5V+sj+C25UjqxDVoKcA8=
 go.elastic.co/apm v1.7.2/go.mod h1:tCw6CkOJgkWnzEthFN9HUP1uL3Gjc/Ur6m7gRPLaoH0=
-go.elastic.co/apm v1.8.1-0.20200902013556-b34fe04da73f h1:Z5e1eChG4ZcP0+jFtztaE5X2dOYRypQcOJGOUB4WgvA=
-go.elastic.co/apm v1.8.1-0.20200902013556-b34fe04da73f/go.mod h1:qoOSi09pnzJDh5fKnfY7bPmQgl8yl2tULdOu03xhui0=
+go.elastic.co/apm v1.8.1-0.20200909061013-2aef45b9cf4b h1:Sf+V3eV91ZuXjF3824SABFgXU+z4ZEuIX5ikDvt2lCE=
+go.elastic.co/apm v1.8.1-0.20200909061013-2aef45b9cf4b/go.mod h1:qoOSi09pnzJDh5fKnfY7bPmQgl8yl2tULdOu03xhui0=
 go.elastic.co/apm/module/apmelasticsearch v1.7.2 h1:5STGHLZLSeAzxordMc+dFVKiyVtMmxADOV+TgRaXXJg=
 go.elastic.co/apm/module/apmelasticsearch v1.7.2/go.mod h1:ZyNFuyWdt42GBZkz0SogoLzDBrBGj4orxpiUuxYeYq8=
 go.elastic.co/apm/module/apmhttp v1.7.2 h1:2mRh7SwBuEVLmJlX+hsMdcSg9xaielCLElaPn/+i34w=

diff --git a/libbeat/autodiscover/providers/kubernetes/pod.go b/libbeat/autodiscover/providers/kubernetes/pod.go
@@ -335,19 +335,29 @@ func (p *pod) emitEvents(pod *kubernetes.Pod, flag string, containers []kubernet
 		// so it works also on `stop` if containers have been already deleted.
 		eventID := fmt.Sprintf("%s.%s", pod.GetObjectMeta().GetUID(), c.Name)
 
+		meta := p.metagen.Generate(
+			pod,
+			metadata.WithFields("container.name", c.Name),
+			metadata.WithFields("container.image", c.Image),
+		)
+
 		cmeta := common.MapStr{
-			"id":      cid,
-			"name":    c.Name,
-			"image":   c.Image,
+			"id": cid,
+			"image": common.MapStr{
+				"name": c.Image,
+			},
 			"runtime": runtimes[c.Name],
 		}
-		meta := p.metagen.Generate(pod, metadata.WithFields("container.name", c.Name),
-			metadata.WithFields("container.image", c.Image))
 
 		// Information that can be used in discovering a workload
 		kubemeta := meta.Clone()
-		kubemeta["container"] = cmeta
 		kubemeta["annotations"] = annotations
+		kubemeta["container"] = common.MapStr{
+			"id":      cid,
+			"name":    c.Name,
+			"image":   c.Image,
+			"runtime": runtimes[c.Name],
+		}
 		if len(nsAnn) != 0 {
 			kubemeta["namespace_annotations"] = nsAnn
 		}
@@ -364,6 +374,7 @@ func (p *pod) emitEvents(pod *kubernetes.Pod, flag string, containers []kubernet
 				"kubernetes": kubemeta,
 				"meta": common.MapStr{
 					"kubernetes": meta,
+					"container":  cmeta,
 				},
 			}
 			events = append(events, event)
@@ -380,6 +391,7 @@ func (p *pod) emitEvents(pod *kubernetes.Pod, flag string, containers []kubernet
 				"kubernetes": kubemeta,
 				"meta": common.MapStr{
 					"kubernetes": meta,
+					"container":  cmeta,
 				},
 			}
 			events = append(events, event)

diff --git a/libbeat/autodiscover/providers/kubernetes/pod_test.go b/libbeat/autodiscover/providers/kubernetes/pod_test.go
@@ -446,15 +446,21 @@ func TestEmitEvent(t *testing.T) {
 					"meta": common.MapStr{
 						"kubernetes": common.MapStr{
 							"namespace": "default",
-							"container": common.MapStr{
-								"name":  "filebeat",
-								"image": "elastic/filebeat:6.3.0",
-							}, "pod": common.MapStr{
+							"pod": common.MapStr{
 								"name": "filebeat",
 								"uid":  "005f3b90-4b9d-12f8-acf0-31020a840133",
 							}, "node": common.MapStr{
 								"name": "node",
 							},
+							"container": common.MapStr{
+								"name":  "filebeat",
+								"image": "elastic/filebeat:6.3.0",
+							},
+						},
+						"container": common.MapStr{
+							"image":   common.MapStr{"name": "elastic/filebeat:6.3.0"},
+							"id":      "foobar",
+							"runtime": "docker",
 						},
 					},
 					"config": []*common.Config{},
@@ -565,15 +571,21 @@ func TestEmitEvent(t *testing.T) {
 					"meta": common.MapStr{
 						"kubernetes": common.MapStr{
 							"namespace": "default",
-							"container": common.MapStr{
-								"name":  "filebeat",
-								"image": "elastic/filebeat:6.3.0",
-							}, "pod": common.MapStr{
+							"pod": common.MapStr{
 								"name": "filebeat",
 								"uid":  "005f3b90-4b9d-12f8-acf0-31020a840133",
 							}, "node": common.MapStr{
 								"name": "node",
 							},
+							"container": common.MapStr{
+								"name":  "filebeat",
+								"image": "elastic/filebeat:6.3.0",
+							},
+						},
+						"container": common.MapStr{
+							"image":   common.MapStr{"name": "elastic/filebeat:6.3.0"},
+							"runtime": "docker",
+							"id":      "foobar",
 						},
 					},
 					"config": []*common.Config{},
@@ -604,15 +616,21 @@ func TestEmitEvent(t *testing.T) {
 					"meta": common.MapStr{
 						"kubernetes": common.MapStr{
 							"namespace": "default",
-							"container": common.MapStr{
-								"name":  "filebeat",
-								"image": "elastic/filebeat:6.3.0",
-							}, "pod": common.MapStr{
+							"pod": common.MapStr{
 								"name": "filebeat",
 								"uid":  "005f3b90-4b9d-12f8-acf0-31020a840133",
 							}, "node": common.MapStr{
 								"name": "node",
 							},
+							"container": common.MapStr{
+								"name":  "filebeat",
+								"image": "elastic/filebeat:6.3.0",
+							},
+						},
+						"container": common.MapStr{
+							"image":   common.MapStr{"name": "elastic/filebeat:6.3.0"},
+							"id":      "foobar",
+							"runtime": "docker",
 						},
 					},
 					"config": []*common.Config{},
@@ -769,15 +787,21 @@ func TestEmitEvent(t *testing.T) {
 					"meta": common.MapStr{
 						"kubernetes": common.MapStr{
 							"namespace": "default",
-							"container": common.MapStr{
-								"name":  "filebeat",
-								"image": "elastic/filebeat:6.3.0",
-							}, "pod": common.MapStr{
+							"pod": common.MapStr{
 								"name": "filebeat",
 								"uid":  "005f3b90-4b9d-12f8-acf0-31020a840133",
 							}, "node": common.MapStr{
 								"name": "node",
 							},
+							"container": common.MapStr{
+								"name":  "filebeat",
+								"image": "elastic/filebeat:6.3.0",
+							},
+						},
+						"container": common.MapStr{
+							"image":   common.MapStr{"name": "elastic/filebeat:6.3.0"},
+							"runtime": "",
+							"id":      "",
 						},
 					},
 					"config": []*common.Config{},
@@ -874,13 +898,19 @@ func TestEmitEvent(t *testing.T) {
 							"container": common.MapStr{
 								"name":  "filebeat",
 								"image": "elastic/filebeat:6.3.0",
-							}, "pod": common.MapStr{
+							},
+							"pod": common.MapStr{
 								"name": "filebeat",
 								"uid":  "005f3b90-4b9d-12f8-acf0-31020a840133",
 							}, "node": common.MapStr{
 								"name": "node",
 							},
 						},
+						"container": common.MapStr{
+							"image":   common.MapStr{"name": "elastic/filebeat:6.3.0"},
+							"id":      "",
+							"runtime": "",
+						},
 					},
 					"config": []*common.Config{},
 				},
@@ -976,13 +1006,19 @@ func TestEmitEvent(t *testing.T) {
 							"container": common.MapStr{
 								"name":  "filebeat",
 								"image": "elastic/filebeat:6.3.0",
-							}, "pod": common.MapStr{
+							},
+							"pod": common.MapStr{
 								"name": "filebeat",
 								"uid":  "005f3b90-4b9d-12f8-acf0-31020a840133",
 							}, "node": common.MapStr{
 								"name": "node",
 							},
 						},
+						"container": common.MapStr{
+							"image":   common.MapStr{"name": "elastic/filebeat:6.3.0"},
+							"runtime": "",
+							"id":      "",
+						},
 					},
 					"config": []*common.Config{},
 				},

diff --git a/libbeat/docs/security/api-keys.asciidoc b/libbeat/docs/security/api-keys.asciidoc
@@ -14,6 +14,8 @@ API key. For different clusters, you need to use an API key per cluster.
 NOTE: For security reasons, we recommend using a unique API key per {beatname_uc} instance.
 You can create as many API keys per user as necessary.
 
+IMPORTANT: Review <<feature-roles>> before creating API keys for {beatname_uc}.
+
 [float]
 [[beats-api-key-publish]]
 === Create an API key for publishing
@@ -41,6 +43,8 @@ POST /_security/api_key
 <1> Name of the API key
 <2> Granted privileges, see <<feature-roles>>
 
+NOTE: See <<privileges-to-publish-events>> for the list of privileges required to publish events.
+
 The return value will look something like this:
 
 [source,console-result,subs="attributes,callouts"]
@@ -89,6 +93,8 @@ POST /_security/api_key
 <1> Name of the API key
 <2> Granted privileges, see <<feature-roles>>
 
+NOTE: See <<privileges-to-publish-monitoring>> for the list of privileges required to send monitoring data.
+
 The return value will look something like this:
 
 [source,console-result,subs="attributes,callouts"]

diff --git a/libbeat/processors/add_kubernetes_metadata/indexers.go b/libbeat/processors/add_kubernetes_metadata/indexers.go
@@ -183,14 +183,19 @@ func NewContainerIndexer(_ common.Config, metaGen metadata.MetaGen) (Indexer, er
 func (c *ContainerIndexer) GetMetadata(pod *kubernetes.Pod) []MetadataIndex {
 	var m []MetadataIndex
 	for _, status := range append(pod.Status.ContainerStatuses, pod.Status.InitContainerStatuses...) {
-		cID := kubernetes.ContainerID(status)
+		cID, runtime := kubernetes.ContainerIDWithRuntime(status)
 		if cID == "" {
 			continue
 		}
 		m = append(m, MetadataIndex{
 			Index: cID,
-			Data: c.metaGen.Generate(pod, metadata.WithFields("container.name", status.Name),
-				metadata.WithFields("container.image", status.Image)),
+			Data: c.metaGen.Generate(
+				pod,
+				metadata.WithFields("container.name", status.Name),
+				metadata.WithFields("container.image", status.Image),
+				metadata.WithFields("container.id", cID),
+				metadata.WithFields("container.runtime", runtime),
+			),
 		})
 	}
 
@@ -234,14 +239,30 @@ func (h *IPPortIndexer) GetMetadata(pod *kubernetes.Pod) []MetadataIndex {
 		Data:  h.metaGen.Generate(pod),
 	})
 
+	cIDs := make(map[string]string)
+	runtimes := make(map[string]string)
+	for _, status := range append(pod.Status.ContainerStatuses, pod.Status.InitContainerStatuses...) {
+		cID, runtime := kubernetes.ContainerIDWithRuntime(status)
+		if cID == "" {
+			continue
+		}
+		cIDs[status.Name] = cID
+		runtimes[status.Name] = runtime
+	}
+
 	for _, container := range pod.Spec.Containers {
 		for _, port := range container.Ports {
 			if port.ContainerPort != 0 {
 
 				m = append(m, MetadataIndex{
 					Index: fmt.Sprintf("%s:%d", pod.Status.PodIP, port.ContainerPort),
-					Data: h.metaGen.Generate(pod, metadata.WithFields("container.name", container.Name),
-						metadata.WithFields("container.image", container.Image)),
+					Data: h.metaGen.Generate(
+						pod,
+						metadata.WithFields("container.name", container.Name),
+						metadata.WithFields("container.image", container.Image),
+						metadata.WithFields("container.id", cIDs[container.Name]),
+						metadata.WithFields("container.runtime", runtimes[container.Name]),
+					),
 				})
 			}
 		}