From 658185456413c918423f1195c4447011622e56db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Fri, 12 Jul 2019 13:31:38 +0200 Subject: [PATCH] Cherry-pick #12632 to 7.2: Use CRI paths in kubernetes manifests (#12742) * Use CRI paths in kubernetes manifests (#12632) We added a new `container` input in #12162, this change makes use of it to read logs from their CRI paths. Making Filebeat work with deployments that are not using Docker runtime. (cherry picked from commit 32c3b216deb4a6d34decdfb7b949b8c6b977f085) * Update CHANGELOG.next.asciidoc --- CHANGELOG.next.asciidoc | 2 + deploy/kubernetes/filebeat-kubernetes.yaml | 61 ++++++++----------- .../filebeat/filebeat-configmap.yaml | 44 ++++++------- .../filebeat/filebeat-daemonset.yaml | 17 +++--- 4 files changed, 56 insertions(+), 68 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index a8cb90118ec5..601c2af9838d 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -83,6 +83,8 @@ https://github.com/elastic/beats/compare/v7.2.0...7.2[Check the HEAD diff] *Filebeat* +- Update Kubernetes deployment manifest to use `container` input. {pull}12632[12632] + *Heartbeat* *Journalbeat* diff --git a/deploy/kubernetes/filebeat-kubernetes.yaml b/deploy/kubernetes/filebeat-kubernetes.yaml index c8981babf20b..76271592c7b3 100644 --- a/deploy/kubernetes/filebeat-kubernetes.yaml +++ b/deploy/kubernetes/filebeat-kubernetes.yaml @@ -8,22 +8,28 @@ metadata: k8s-app: filebeat data: filebeat.yml: |- - filebeat.config: - inputs: - # Mounted `filebeat-inputs` configmap: - path: ${path.config}/inputs.d/*.yml - # Reload inputs configs as they change: - reload.enabled: false - modules: - path: ${path.config}/modules.d/*.yml - # Reload module configs as they change: - reload.enabled: false + filebeat.inputs: + - type: container + paths: + - /var/log/containers/*.log + processors: + - add_kubernetes_metadata: + in_cluster: true + host: ${NODE_NAME} + matchers: + - logs_path: + logs_path: "/var/log/containers/" - # To enable hints based autodiscover, remove `filebeat.config.inputs` configuration and uncomment this: + # To enable hints based autodiscover, remove `filebeat.inputs` configuration and uncomment this: #filebeat.autodiscover: # providers: # - type: kubernetes + # host: ${NODE_NAME} # hints.enabled: true + # hints.default_config: + # type: container + # paths: + # - /var/log/containers/*${data.kubernetes.container.id}.log processors: - add_cloud_metadata: @@ -36,22 +42,6 @@ data: username: ${ELASTICSEARCH_USERNAME} password: ${ELASTICSEARCH_PASSWORD} --- -apiVersion: v1 -kind: ConfigMap -metadata: - name: filebeat-inputs - namespace: kube-system - labels: - k8s-app: filebeat -data: - kubernetes.yml: |- - - type: docker - containers.ids: - - "*" - processors: - - add_kubernetes_metadata: - in_cluster: true ---- apiVersion: extensions/v1beta1 kind: DaemonSet metadata: @@ -87,6 +77,10 @@ spec: value: - name: ELASTIC_CLOUD_AUTH value: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName securityContext: runAsUser: 0 # If using Red Hat OpenShift uncomment this: @@ -102,14 +96,14 @@ spec: mountPath: /etc/filebeat.yml readOnly: true subPath: filebeat.yml - - name: inputs - mountPath: /usr/share/filebeat/inputs.d - readOnly: true - name: data mountPath: /usr/share/filebeat/data - name: varlibdockercontainers mountPath: /var/lib/docker/containers readOnly: true + - name: varlog + mountPath: /var/log + readOnly: true volumes: - name: config configMap: @@ -118,10 +112,9 @@ spec: - name: varlibdockercontainers hostPath: path: /var/lib/docker/containers - - name: inputs - configMap: - defaultMode: 0600 - name: filebeat-inputs + - name: varlog + hostPath: + path: /var/log # data folder stores a registry of read status for all files, so we don't send everything again on a Filebeat pod restart - name: data hostPath: diff --git a/deploy/kubernetes/filebeat/filebeat-configmap.yaml b/deploy/kubernetes/filebeat/filebeat-configmap.yaml index 0b7f3dd5024f..e37a759e60eb 100644 --- a/deploy/kubernetes/filebeat/filebeat-configmap.yaml +++ b/deploy/kubernetes/filebeat/filebeat-configmap.yaml @@ -8,22 +8,28 @@ metadata: k8s-app: filebeat data: filebeat.yml: |- - filebeat.config: - inputs: - # Mounted `filebeat-inputs` configmap: - path: ${path.config}/inputs.d/*.yml - # Reload inputs configs as they change: - reload.enabled: false - modules: - path: ${path.config}/modules.d/*.yml - # Reload module configs as they change: - reload.enabled: false + filebeat.inputs: + - type: container + paths: + - /var/log/containers/*.log + processors: + - add_kubernetes_metadata: + in_cluster: true + host: ${NODE_NAME} + matchers: + - logs_path: + logs_path: "/var/log/containers/" - # To enable hints based autodiscover, remove `filebeat.config.inputs` configuration and uncomment this: + # To enable hints based autodiscover, remove `filebeat.inputs` configuration and uncomment this: #filebeat.autodiscover: # providers: # - type: kubernetes + # host: ${NODE_NAME} # hints.enabled: true + # hints.default_config: + # type: container + # paths: + # - /var/log/containers/*${data.kubernetes.container.id}.log processors: - add_cloud_metadata: @@ -35,19 +41,3 @@ data: hosts: ['${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9200}'] username: ${ELASTICSEARCH_USERNAME} password: ${ELASTICSEARCH_PASSWORD} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: filebeat-inputs - namespace: kube-system - labels: - k8s-app: filebeat -data: - kubernetes.yml: |- - - type: docker - containers.ids: - - "*" - processors: - - add_kubernetes_metadata: - in_cluster: true diff --git a/deploy/kubernetes/filebeat/filebeat-daemonset.yaml b/deploy/kubernetes/filebeat/filebeat-daemonset.yaml index f554dee185b0..98fd31b73d47 100644 --- a/deploy/kubernetes/filebeat/filebeat-daemonset.yaml +++ b/deploy/kubernetes/filebeat/filebeat-daemonset.yaml @@ -33,6 +33,10 @@ spec: value: - name: ELASTIC_CLOUD_AUTH value: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName securityContext: runAsUser: 0 # If using Red Hat OpenShift uncomment this: @@ -48,14 +52,14 @@ spec: mountPath: /etc/filebeat.yml readOnly: true subPath: filebeat.yml - - name: inputs - mountPath: /usr/share/filebeat/inputs.d - readOnly: true - name: data mountPath: /usr/share/filebeat/data - name: varlibdockercontainers mountPath: /var/lib/docker/containers readOnly: true + - name: varlog + mountPath: /var/log + readOnly: true volumes: - name: config configMap: @@ -64,10 +68,9 @@ spec: - name: varlibdockercontainers hostPath: path: /var/lib/docker/containers - - name: inputs - configMap: - defaultMode: 0600 - name: filebeat-inputs + - name: varlog + hostPath: + path: /var/log # data folder stores a registry of read status for all files, so we don't send everything again on a Filebeat pod restart - name: data hostPath: