From 18d20b15f12283237920fcaeeea9ae9858e664d5 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Tue, 6 Jul 2021 10:34:57 +0300 Subject: [PATCH] Add instructions on testing metricbeat kubernetes module (#26643) (#26722) (cherry picked from commit 1862f78f80edbe5835fc794494b62c4c1e9c0fed) Co-authored-by: Michael Katsoulis --- .../test/docs/01_playground/ek_stack.yaml | 79 ++++ .../test/docs/01_playground/metricbeat.yaml | 336 ++++++++++++++++++ .../docs/01_playground/playground-ubuntu.yaml | 82 ----- .../kubernetes/_meta/test/docs/README.md | 156 +++----- .../kubernetes/_meta/test/docs/darwin.md | 27 -- 5 files changed, 467 insertions(+), 213 deletions(-) create mode 100644 metricbeat/module/kubernetes/_meta/test/docs/01_playground/ek_stack.yaml create mode 100644 metricbeat/module/kubernetes/_meta/test/docs/01_playground/metricbeat.yaml delete mode 100644 metricbeat/module/kubernetes/_meta/test/docs/01_playground/playground-ubuntu.yaml delete mode 100644 metricbeat/module/kubernetes/_meta/test/docs/darwin.md diff --git a/metricbeat/module/kubernetes/_meta/test/docs/01_playground/ek_stack.yaml b/metricbeat/module/kubernetes/_meta/test/docs/01_playground/ek_stack.yaml new file mode 100644 index 00000000000..7365ce2e6ef --- /dev/null +++ b/metricbeat/module/kubernetes/_meta/test/docs/01_playground/ek_stack.yaml @@ -0,0 +1,79 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + deployment.kubernetes.io/revision: "1" + labels: + app: elasticsearch + name: elasticsearch + namespace: default +spec: + replicas: 1 + selector: + matchLabels: + app: elasticsearch + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + labels: + app: elasticsearch + spec: + containers: + - image: docker.elastic.co/elasticsearch/elasticsearch:8.0.0-SNAPSHOT + imagePullPolicy: IfNotPresent + name: elasticsearch + env: + - name: "discovery.type" + value: "single-node" + +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: elasticsearch + name: elasticsearch + namespace: default +spec: + ports: + - port: 9200 + protocol: TCP + targetPort: 9200 + selector: + app: elasticsearch + sessionAffinity: None + type: ClusterIP + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + deployment.kubernetes.io/revision: "1" + labels: + app: kibana + name: kibana + namespace: default +spec: + replicas: 1 + selector: + matchLabels: + app: kibana + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + labels: + app: kibana + spec: + containers: + - image: docker.elastic.co/kibana/kibana:8.0.0-SNAPSHOT + imagePullPolicy: IfNotPresent + name: kibana diff --git a/metricbeat/module/kubernetes/_meta/test/docs/01_playground/metricbeat.yaml b/metricbeat/module/kubernetes/_meta/test/docs/01_playground/metricbeat.yaml new file mode 100644 index 00000000000..ba304382637 --- /dev/null +++ b/metricbeat/module/kubernetes/_meta/test/docs/01_playground/metricbeat.yaml @@ -0,0 +1,336 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: metricbeat-daemonset-config + namespace: kube-system + labels: + k8s-app: metricbeat +data: + metricbeat.yml: |- + metricbeat.config.modules: + # Mounted `metricbeat-daemonset-modules` configmap: + path: ${path.config}/modules.d/*.yml + # Reload module configs as they change: + reload.enabled: false + + metricbeat.autodiscover: + providers: + - type: kubernetes + scope: cluster + node: ${NODE_NAME} + unique: true + templates: + - config: + - module: kubernetes + hosts: ["kube-state-metrics:8080"] + period: 10s + add_metadata: true + metricsets: + - state_node + - state_deployment + - state_daemonset + - state_replicaset + - state_pod + - state_container + - state_cronjob + - state_resourcequota + - state_statefulset + - state_service + - module: kubernetes + metricsets: + - apiserver + hosts: ["https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}"] + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + ssl.certificate_authorities: + - /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + period: 30s + # Uncomment this to get k8s events: + #- module: kubernetes + # metricsets: + # - event + # To enable hints based autodiscover uncomment this: + #- type: kubernetes + # node: ${NODE_NAME} + # hints.enabled: true + + processors: + - add_cloud_metadata: + + cloud.id: ${ELASTIC_CLOUD_ID} + cloud.auth: ${ELASTIC_CLOUD_AUTH} + + output.elasticsearch: + hosts: ['${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9200}'] + username: ${ELASTICSEARCH_USERNAME} + password: ${ELASTICSEARCH_PASSWORD} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: metricbeat-daemonset-modules + namespace: kube-system + labels: + k8s-app: metricbeat +data: + system.yml: |- + - module: system + period: 10s + metricsets: + - cpu + - load + - memory + - network + - process + - process_summary + #- core + #- diskio + #- socket + processes: ['.*'] + process.include_top_n: + by_cpu: 5 # include top 5 processes by CPU + by_memory: 5 # include top 5 processes by memory + + - module: system + period: 1m + metricsets: + - filesystem + - fsstat + processors: + - drop_event.when.regexp: + system.filesystem.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib|snap)($|/)' + kubernetes.yml: |- + - module: kubernetes + metricsets: + - node + - system + - pod + - container + - volume + period: 10s + host: ${NODE_NAME} + hosts: ["https://${NODE_NAME}:10250"] + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + ssl.verification_mode: "none" + # If there is a CA bundle that contains the issuer of the certificate used in the Kubelet API, + # remove ssl.verification_mode entry and use the CA, for instance: + #ssl.certificate_authorities: + #- /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt + # Currently `proxy` metricset is not supported on Openshift, comment out section + - module: kubernetes + metricsets: + - proxy + period: 10s + host: ${NODE_NAME} + hosts: ["localhost:10249"] +--- +# Deploy a Metricbeat instance per node for node metrics retrieval +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: metricbeat + namespace: kube-system + labels: + k8s-app: metricbeat +spec: + selector: + matchLabels: + k8s-app: metricbeat + template: + metadata: + labels: + k8s-app: metricbeat + spec: + serviceAccountName: metricbeat + terminationGracePeriodSeconds: 30 + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: metricbeat + image: docker.elastic.co/beats/metricbeat:8.0.0-SNAPSHOT + command: [ "sleep" ] + args: [ "infinity" ] + env: + - name: ELASTICSEARCH_HOST + value: elasticsearch.default.svc.cluster.local + - name: ELASTICSEARCH_PORT + value: "9200" + - name: ELASTICSEARCH_USERNAME + value: elastic + - name: ELASTICSEARCH_PASSWORD + value: "changeme" + - name: ELASTIC_CLOUD_ID + value: + - name: ELASTIC_CLOUD_AUTH + value: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + securityContext: + runAsUser: 0 + # If using Red Hat OpenShift uncomment this: + #privileged: true + resources: + limits: + memory: 200Mi + requests: + cpu: 100m + memory: 100Mi + volumeMounts: + - name: config + mountPath: /etc/metricbeat.yml + readOnly: true + subPath: metricbeat.yml + - name: data + mountPath: /usr/share/metricbeat/data + - name: modules + mountPath: /usr/share/metricbeat/modules.d + readOnly: true + - name: proc + mountPath: /hostfs/proc + readOnly: true + - name: cgroup + mountPath: /hostfs/sys/fs/cgroup + readOnly: true + volumes: + - name: proc + hostPath: + path: /proc + - name: cgroup + hostPath: + path: /sys/fs/cgroup + - name: config + configMap: + defaultMode: 0640 + name: metricbeat-daemonset-config + - name: modules + configMap: + defaultMode: 0640 + name: metricbeat-daemonset-modules + - name: data + hostPath: + # When metricbeat runs as non-root user, this directory needs to be writable by group (g+w) + path: /var/lib/metricbeat-data + type: DirectoryOrCreate +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: metricbeat +subjects: + - kind: ServiceAccount + name: metricbeat + namespace: kube-system +roleRef: + kind: ClusterRole + name: metricbeat + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: metricbeat + namespace: kube-system +subjects: + - kind: ServiceAccount + name: metricbeat + namespace: kube-system +roleRef: + kind: Role + name: metricbeat + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: metricbeat-kubeadm-config + namespace: kube-system +subjects: + - kind: ServiceAccount + name: metricbeat + namespace: kube-system +roleRef: + kind: Role + name: metricbeat-kubeadm-config + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metricbeat + labels: + k8s-app: metricbeat +rules: + - apiGroups: [""] + resources: + - nodes + - namespaces + - events + - pods + - services + verbs: ["get", "list", "watch"] + # Enable this rule only if planing to use Kubernetes keystore + #- apiGroups: [""] + # resources: + # - secrets + # verbs: ["get"] + - apiGroups: ["extensions"] + resources: + - replicasets + verbs: ["get", "list", "watch"] + - apiGroups: ["apps"] + resources: + - statefulsets + - deployments + - replicasets + verbs: ["get", "list", "watch"] + - apiGroups: + - "" + resources: + - nodes/stats + verbs: + - get + - nonResourceURLs: + - "/metrics" + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: metricbeat + # should be the namespace where metricbeat is running + namespace: kube-system + labels: + k8s-app: metricbeat +rules: + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: ["get", "create", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: metricbeat-kubeadm-config + namespace: kube-system + labels: + k8s-app: metricbeat +rules: + - apiGroups: [""] + resources: + - configmaps + resourceNames: + - kubeadm-config + verbs: ["get"] +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: metricbeat + namespace: kube-system + labels: + k8s-app: metricbeat +--- diff --git a/metricbeat/module/kubernetes/_meta/test/docs/01_playground/playground-ubuntu.yaml b/metricbeat/module/kubernetes/_meta/test/docs/01_playground/playground-ubuntu.yaml deleted file mode 100644 index 38c7538f169..00000000000 --- a/metricbeat/module/kubernetes/_meta/test/docs/01_playground/playground-ubuntu.yaml +++ /dev/null @@ -1,82 +0,0 @@ - -apiVersion: v1 -kind: ServiceAccount -metadata: - name: metricbeat-kube - ---- - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: metricbeat -rules: -- apiGroups: - - "" - resources: - - nodes/stats - - nodes/metrics - - nodes/log - - nodes/spec - - nodes/proxy - verbs: - - get -- apiGroups: [""] - resources: ["pods", "nodes"] - verbs: ["get", "watch", "list"] -- apiGroups: ["extensions", "apps"] - resources: ["deployments", "replicasets", "statefulsets"] - verbs: ["get", "list", "watch"] -- apiGroups: ["batch"] - resources: ["jobs"] - verbs: ["get", "list", "watch"] - ---- - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: metricbeat-kube -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: metricbeat -subjects: -- kind: ServiceAccount - name: metricbeat-kube - namespace: default ---- - -apiVersion: v1 -kind: Pod -metadata: - name: playground -spec: - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet - serviceAccountName: metricbeat-kube - containers: - - name: ubuntu - image: ubuntu:latest - command: [ "sleep" ] - args: [ "infinity" ] - volumeMounts: - - name: data - mountPath: /usr/share/filebeat/data - - name: varlibdockercontainers - mountPath: /var/lib/docker/containers - readOnly: true - - name: varlog - mountPath: /var/log - volumes: - - name: varlibdockercontainers - hostPath: - path: /var/lib/docker/containers - - name: varlog - hostPath: - path: /var/log - - name: data - hostPath: - path: /var/lib/filebeat-data - type: DirectoryOrCreate - diff --git a/metricbeat/module/kubernetes/_meta/test/docs/README.md b/metricbeat/module/kubernetes/_meta/test/docs/README.md index f2a13a07fee..c3d7bc6c750 100644 --- a/metricbeat/module/kubernetes/_meta/test/docs/README.md +++ b/metricbeat/module/kubernetes/_meta/test/docs/README.md @@ -1,139 +1,87 @@ -# Testing on OSX +# Testing Metricbeat -A previous document regarding testing metricbeat at OSX existed, and have been moved to [./darwin.md](darwin.md) -# Testing on Linux +## Create kubernetes cluster using kind -## Create Elasticsearch + Kibana instances +Follow instructions at https://kind.sigs.k8s.io/docs/user/quick-start/#installation and install kind. -You can rely on your EK tuple of choice as long as it is addresable from the kubernetes cluster. - -To boot a docker based EK this should suffice, be sure to replace image tags according to version: - -```bash -# Run Elasticsearch -docker run --name es -d -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" docker.elastic.co/elasticsearch/elasticsearch:7.3.0 - -# Run Kibana -docker run --name kibana -d --link es:elasticsearch -p 5601:5601 \ - docker.elastic.co/kibana/kibana:7.3.0 +Create a kind kubernetes cluster. ``` - -## Prepare assets - -Generate binary and other assets for the beats branch you want to test, then copy them to a folder layed out to run: - -- create folder `/home/myuser/playground/metricbeat` -- copy to that folder `metricbeat` binary and `metricbeat.yml` -- recursive copy `modules.d` from source repo to destination folder -- recursive copy `_meta/kibana.generated/{version}/dashboard/` to `kibana/{version}/dashboard/` - -Configure `metricbeat.yml` and modules, do not use `localhost` to point to elasticsearch and kibana but the public ip of the host (one that will be routable from minikube) - - -## Create minikube cluster - -Follow instructions https://kubernetes.io/docs/tasks/tools/install-minikube/ and start the minikube cluster. - -Usually we should be ok with the kubernetes version that minikube creates, but you can force it by using `--kubernetes-version` flag. - -``` -minikube start --kubernetes-version v1.15.0 +kind create cluster --image 'kindest/node:v1.21.1' ``` -## Playground Pod - -A playground Pod hosts the ubuntu container metricbeat will be running. A working playground is provided under [./01_playground](./01_playground) subfolder. - -This file contains: +## Deploy Kube-state-metrics -- a service account. -- a cluster role, if you are consuming kubernetes API resources, make sure that the APIGroup/Version, Resource and verb are listed here. -- a cluster role binding that links the service account to the service role -- an Ubuntu Pod: - - uses `hostNetwork`, so it can reach ports at the host instance (for instance, the kubelet) - - executes `sleep infinity`, so that it never exists, but does nothing - - in order to be useful for filebeat, it mounts `/var/log/`, `/var/lib/docker/containers` and `/var/lib/filebeat-data` - -At the time of writing this the Pod has been only used for 2 tests from the same person (hello), there is a lot of room for improvement. - -To deploy the pod _as is_ you need to: +Prerequisite for collecting kubernetes meaningful metrics is kube-state-metrics. +Deploy it to your cluster manually by +```bash +git clone git@github.com:kubernetes/kube-state-metrics.git +cd kube-state-metrics/ +kubectl apply -k . ``` -kubectl apply -f https://raw.githubusercontent.com/elastic/beats/master/metricbeat/module/kubernetes/_meta/test/docs/01_playground/playground-ubuntu.yaml -``` - -## Test - -Binary and assets needed for the test that we prepared above need to be copied to the playground pod. Use `kubectl` to copy the directory, further iterations might only need to copy the changing assets. +## Create ELK stack -Replace source folder and Pod namespace/name +You can spin up an ELK stack in two ways +1. [Proposed] Using elastic cloud https://cloud.elastic.co +2. Locally on your kind cluster (EK tuple will suffice). +```bash +# Deploy Elasticsearch and Kibana +kubectl apply -f ../01_playground/ek_stack.yaml -``` -kubectl cp --no-preserve /home/myuser/playground/metricbeat playground:/metricbeat +# Expose Kibana with port forwarding. In your browser visit localhost:5601 +kubectl port-forward deployment/kibana 5601:5601 ``` -Now you can exec into the container and launch metricbeat -``` - kubectl exec -ti playground /bin/bash +## Playground Metricbeat Pod - cd /metricbeat - ./metricbeat -c metricbeat.yml -e +A slightly modified (as of beats/deploy/kubernetes/metricbeat-kubernetes.yaml) all-in-one metricbeat manifest resides under 01_playground directory. +The daemonset executes an infinite sleep command instead of starting metricbeat. - ``` +ELASTICSEARCH_HOST, ELASTICSEARCH_PORT, ELASTICSEARCH_USERNAME, ELASTICSEARCH_PASSWORD variables are set according to local kind EK stack. -### Test Iterations +In case of Elastic Cloud deployment configure the variables ELASTIC_CLOUD_ID and ELASTIC_CLOUD_AUTH properly. -When copying new assets to an already used playground Pod, you will most probably run into an issue: +Deploy metricbeat ``` -tar: metricbeat/kibana/7/dashboard/Metricbeat-aerospike-overview.json: Cannot open: Not a directory -tar: metricbeat/kibana/7/dashboard/Metricbeat-apache-overview.json: Cannot open: Not a directory -tar: metricbeat/kibana/7/dashboard/Metricbeat-ceph-overview.json: Cannot open: Not a directory -tar: metricbeat/kibana/7/dashboard/Metricbeat-consul-overview.json: Cannot open: Not a directory +kubectl apply -f ../01_playground/metricbeat.yaml ``` -I haven't looked much into this, there seems to be something going on when kubernetes untars the bundled directory. As a workaround, delete the metricbeat directory at the Pod before copying a new set of assets. +## Build and launch metricbeat process -# Testing kubernetes loads +Next step is to build metricbeat binary and copy it in the running metricbeat pod. -## Kube-state-metrics +Under beats/metricbeat execute -Kube-state-metrics needs to be deployed for all the `state_` prefix metricsets at kubernetes. Yamls are to be found at the [upstream project](https://github.com/kubernetes/kube-state-metrics/tree/master/kubernetes) +```bash +# Build metricbeat +GOOS=linux GOARCH=amd64 go build -Installing kube-state-metrics can be done either installing the yamls one by one from their remote location or cloning and installing the folder contents. Be sure to checkout the target release version before installing. +# Copy binary in pod +kubectl cp ./metricbeat `kubectl get pod -n kube-system -l k8s-app=metricbeat -o jsonpath='{.items[].metadata.name}'`:/usr/share/metricbeat/ -n kube-system +```` +The above command only copies metricbeat binary. +In case of configuration files updates it can be modified to copy also those files in the right container paths. +```bash +# Exec in the container and launch metricbeat +kubectl exec `kubectl get pod -n kube-system -l k8s-app=metricbeat -o jsonpath='{.items[].metadata.name}'` -n kube-system -- bash -c "metricbeat -e -c /etc/metricbeat.yml" ``` -git clone git@github.com:kubernetes/kube-state-metrics.git -cd kube-state-metrics/ - -git checkout -b release-1.7 origin/release-1.7 -kubectl apply -f kubernetes/ -``` - +Metricbeat will launch and the process logs will appear in the terminal. -## Core components test +You can as well exec in metricbeat pod with bash command and then run metricbeat. +This gives the flexibility to easily start and stop the process. -Testing core components (kubelet, apiserver, controller manager, scheduler) requires a diverse range of objects to be created. Using [Sonobuoy](https://github.com/heptio/sonobuoy) is the fastest path for testing,getting metrics and filling dashboards. -Refer to the documentation at Sonobuoy, at the time of this writing installing and running can be achieved with a couple commands +### Test Iterations +In case a new update is needed in the binary or configurations files +1. delete the running metricbeat pod. +```bash +# Delete metricbeat +kubectl delete pod `kubectl get pod -n kube-system -l k8s-app=metricbeat -o jsonpath='{.items[].metadata.name}'` ``` -go get -u -v github.com/heptio/sonobuoy -sonobuoy run --wait -``` - -## Regular kubernetes components - -You can find at [./02_objects](./02_objects) example kubernetes objects used during development and testing. -For now only a CronJob example is added, add your kubernetes object of choice under that folder if you consider it will be useful for other people when developing, testing and troubleshooting. - -# Going further - -- All improvements are welcome. -- Different ways to test are welcome and can live here side by side. -- Using kind seems to be a lot more lightweight. -- Probably some steps above can be tackled using [telepresence](https://www.telepresence.io/). -- Probably Sonobuoy can be replaced with kubernetes e2e tests. +2. Execute previous step (Build and launch metricbeat process) diff --git a/metricbeat/module/kubernetes/_meta/test/docs/darwin.md b/metricbeat/module/kubernetes/_meta/test/docs/darwin.md deleted file mode 100644 index 8c9bbb010ed..00000000000 --- a/metricbeat/module/kubernetes/_meta/test/docs/darwin.md +++ /dev/null @@ -1,27 +0,0 @@ -# Testing on OS X - -To test the kubernetes module on Mac OS X you can use the following setup. [Minikube](https://github.com/kubernetes/minikube) is used for the testing and it is assumed that you have [brew](https://brew.sh/) installed. - -First install minikube: - -``` -brew install Caskroom/cask/minikube -``` - -Start minikube exposing the metrics endpoint externally: - -``` -minikube start --extra-config apiserver.InsecureBindAddress=0.0.0.0 -``` - -Now setup your metricbeat config to connect to the minikube kubernetes: - -``` -- module: kubernetes - metricsets: ["node","container","volume","pod","system"] - enabled: true - period: 10s - hosts: ["192.168.99.100:10255"] -``` - -Replace the IP address with the IP of your virtual box inside which minikube is running.