Add support for graceful termination of agent pods when deploying via Helm chart

operations/helm/charts/grafana-agent/CHANGELOG.md

@@ -10,6 +10,10 @@ internal API changes are not present.
 Unreleased
 ----------
 
+### Enhancements
+
+- Add support for graceful termination of agent pods using `terminationGracePeriodSeconds` and `lifecycle` hooks on agent container. (@onematchfox)
+
 0.35.0 (2024-02-27)
 -------------------
 
@@ -339,7 +343,6 @@ Unreleased
 - Add PodMonitors, ServiceMonitors, and Probes to the agent ClusterRole. (@captncraig)
 - Add podLabels values. (@therealmanny)
 
-
 0.8.1 (2023-03-06)
 ------------------
 
@@ -372,7 +375,6 @@ Unreleased
 - Helm chart: Add support for templates inside of configMap.content (@ts-mini)
 - Add the necessary rbac to support eventhandler integration (@nvanheuverzwijn)
 
-
 0.6.0 (2023-02-13)
 ------------------
 

operations/helm/charts/grafana-agent/README.md

@@ -51,6 +51,7 @@ use the older mode (called "static mode"), set the `agent.mode` value to
 | agent.extraArgs | list | `[]` | Extra args to pass to `agent run`: https://grafana.com/docs/agent/latest/flow/reference/cli/run/ |
 | agent.extraEnv | list | `[]` | Extra environment variables to pass to the agent container. |
 | agent.extraPorts | list | `[]` | Extra ports to expose on the Agent |
+| agent.lifecycle | object | `{}` | Lifecycle for the agent container |
 | agent.listenAddr | string | `"0.0.0.0"` | Address to listen for traffic on. 0.0.0.0 exposes the UI to other containers. |
 | agent.listenPort | int | `80` | Port to listen for traffic on. |
 | agent.listenScheme | string | `"HTTP"` | Scheme is needed for readiness probes. If enabling tls in your configs, set to "HTTPS" |
@@ -95,6 +96,7 @@ use the older mode (called "static mode"), set the `agent.mode` value to
 | controller.podLabels | object | `{}` | Extra pod labels to add. |
 | controller.priorityClassName | string | `""` | priorityClassName to apply to Grafana Agent pods. |
 | controller.replicas | int | `1` | Number of pods to deploy. Ignored when controller.type is 'daemonset'. |
+| controller.terminationGracePeriodSeconds | int | `30` | Grace period to allow the Grafana Agent pods to shutdown before being killed. |
 | controller.tolerations | list | `[]` | Tolerations to apply to Grafana Agent pods. |
 | controller.topologySpreadConstraints | list | `[]` | Topology Spread Constraints to apply to Grafana Agent pods. |
 | controller.type | string | `"daemonset"` | Type of controller to use for deploying Grafana Agent in the cluster. Must be one of 'daemonset', 'deployment', or 'statefulset'. |

operations/helm/charts/grafana-agent/ci/graceful-shutdown-values.yaml

@@ -0,0 +1,11 @@
+agent:
+  lifecycle:
+    preStop:
+      exec:
+        command:
+          - sleep
+          - "60"
+
+controller:
+  type: deployment
+  terminationGracePeriodSeconds: 300

operations/helm/charts/grafana-agent/templates/containers/_agent.yaml

@@ -40,6 +40,10 @@
   envFrom:
     {{- toYaml .Values.agent.envFrom | nindent 4 }}
   {{- end }}
+  {{- with .Values.agent.lifecycle }}
+  lifecycle:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
   ports:
     - containerPort: {{ .Values.agent.listenPort }}
       name: http-metrics

operations/helm/charts/grafana-agent/templates/controllers/_pod.yaml

@@ -54,6 +54,7 @@ spec:
   nodeSelector:
     {{- toYaml . | nindent 4 }}
   {{- end }}
+  terminationGracePeriodSeconds: {{ .Values.controller.terminationGracePeriodSeconds }}
   {{- with .Values.controller.tolerations }}
   tolerations:
     {{- toYaml . | nindent 4 }}

operations/helm/charts/grafana-agent/values.yaml

@@ -96,6 +96,9 @@ agent:
   # -- Resource requests and limits to apply to the Grafana Agent container.
   resources: {}
 
+  # -- Lifecycle for the agent container
+  lifecycle: {}
+
 image:
   # -- Grafana Agent image registry (defaults to docker.io)
   registry: "docker.io"
@@ -179,6 +182,11 @@ controller:
   # -- nodeSelector to apply to Grafana Agent pods.
   nodeSelector: {}
 
+  # -- Grace period to allow the Grafana Agent pods to shutdown before being
+  # killed. Useful for ensuring gracefully shutdown flushing/transferring all
+  # data prior to shutdown.
+  terminationGracePeriodSeconds: 30
+
   # -- Tolerations to apply to Grafana Agent pods.
   tolerations: []
 

operations/helm/tests/additional-serviceaccount-label/grafana-agent/templates/controllers/daemonset.yaml

@@ -70,6 +70,7 @@ spec:
               cpu: 1m
               memory: 5Mi
       dnsPolicy: ClusterFirst
+      terminationGracePeriodSeconds: 30
       volumes:
         - name: config
           configMap:

operations/helm/tests/clustering/grafana-agent/templates/controllers/statefulset.yaml

@@ -75,6 +75,7 @@ spec:
               cpu: 1m
               memory: 5Mi
       dnsPolicy: ClusterFirst
+      terminationGracePeriodSeconds: 30
       volumes:
         - name: config
           configMap:

operations/helm/tests/controller-volumes-extra/grafana-agent/templates/controllers/daemonset.yaml

@@ -73,6 +73,7 @@ spec:
               cpu: 1m
               memory: 5Mi
       dnsPolicy: ClusterFirst
+      terminationGracePeriodSeconds: 30
       volumes:
         - name: config
           configMap:

operations/helm/tests/create-daemonset-hostnetwork/grafana-agent/templates/controllers/daemonset.yaml

@@ -71,6 +71,7 @@ spec:
               memory: 5Mi
       hostNetwork: true
       dnsPolicy: ClusterFirstWithHostNet
+      terminationGracePeriodSeconds: 30
       volumes:
         - name: config
           configMap:

operations/helm/tests/create-daemonset/grafana-agent/templates/controllers/daemonset.yaml

@@ -70,6 +70,7 @@ spec:
               cpu: 1m
               memory: 5Mi
       dnsPolicy: ClusterFirst
+      terminationGracePeriodSeconds: 30
       volumes:
         - name: config
           configMap:

operations/helm/tests/create-deployment-autoscaling/grafana-agent/templates/controllers/deployment.yaml

@@ -73,6 +73,7 @@ spec:
               cpu: 1m
               memory: 5Mi
       dnsPolicy: ClusterFirst
+      terminationGracePeriodSeconds: 30
       volumes:
         - name: config
           configMap:

operations/helm/tests/create-deployment/grafana-agent/templates/controllers/deployment.yaml

@@ -71,6 +71,7 @@ spec:
               cpu: 1m
               memory: 5Mi
       dnsPolicy: ClusterFirst
+      terminationGracePeriodSeconds: 30
       volumes:
         - name: config
           configMap:

operations/helm/tests/create-statefulset-autoscaling/grafana-agent/templates/controllers/statefulset.yaml

@@ -75,6 +75,7 @@ spec:
               cpu: 1m
               memory: 5Mi
       dnsPolicy: ClusterFirst
+      terminationGracePeriodSeconds: 30
       volumes:
         - name: config
           configMap:

operations/helm/tests/create-statefulset/grafana-agent/templates/controllers/statefulset.yaml

@@ -73,6 +73,7 @@ spec:
               cpu: 1m
               memory: 5Mi
       dnsPolicy: ClusterFirst
+      terminationGracePeriodSeconds: 30
       volumes:
         - name: config
           configMap:

operations/helm/tests/custom-config/grafana-agent/templates/controllers/daemonset.yaml

@@ -70,6 +70,7 @@ spec:
               cpu: 1m
               memory: 5Mi
       dnsPolicy: ClusterFirst
+      terminationGracePeriodSeconds: 30
       volumes:
         - name: config
           configMap:

operations/helm/tests/default-values/grafana-agent/templates/controllers/daemonset.yaml

@@ -70,6 +70,7 @@ spec:
               cpu: 1m
               memory: 5Mi
       dnsPolicy: ClusterFirst
+      terminationGracePeriodSeconds: 30
       volumes:
         - name: config
           configMap:

operations/helm/tests/enable-servicemonitor-tls/grafana-agent/templates/controllers/daemonset.yaml

@@ -70,6 +70,7 @@ spec:
               cpu: 1m
               memory: 5Mi
       dnsPolicy: ClusterFirst
+      terminationGracePeriodSeconds: 30
       volumes:
         - name: config
           configMap:

operations/helm/tests/enable-servicemonitor/grafana-agent/templates/controllers/daemonset.yaml

@@ -70,6 +70,7 @@ spec:
               cpu: 1m
               memory: 5Mi
       dnsPolicy: ClusterFirst
+      terminationGracePeriodSeconds: 30
       volumes:
         - name: config
           configMap:

operations/helm/tests/envFrom/grafana-agent/templates/controllers/daemonset.yaml

@@ -73,6 +73,7 @@ spec:
               cpu: 1m
               memory: 5Mi
       dnsPolicy: ClusterFirst
+      terminationGracePeriodSeconds: 30
       volumes:
         - name: config
           configMap:

operations/helm/tests/existing-config/grafana-agent/templates/controllers/daemonset.yaml

@@ -70,6 +70,7 @@ spec:
               cpu: 1m
               memory: 5Mi
       dnsPolicy: ClusterFirst
+      terminationGracePeriodSeconds: 30
       volumes:
         - name: config
           configMap:

operations/helm/tests/extra-env/grafana-agent/templates/controllers/daemonset.yaml

@@ -79,6 +79,7 @@ spec:
               cpu: 1m
               memory: 5Mi
       dnsPolicy: ClusterFirst
+      terminationGracePeriodSeconds: 30
       volumes:
         - name: config
           configMap:

operations/helm/tests/extra-ports/grafana-agent/templates/controllers/daemonset.yaml

@@ -73,6 +73,7 @@ spec:
               cpu: 1m
               memory: 5Mi
       dnsPolicy: ClusterFirst
+      terminationGracePeriodSeconds: 30
       volumes:
         - name: config
           configMap:

operations/helm/tests/faro-ingress/grafana-agent/templates/controllers/daemonset.yaml

@@ -73,6 +73,7 @@ spec:
               cpu: 1m
               memory: 5Mi
       dnsPolicy: ClusterFirst
+      terminationGracePeriodSeconds: 30
       volumes:
         - name: config
           configMap:

operations/helm/tests/global-image-pullsecrets/grafana-agent/templates/controllers/daemonset.yaml

@@ -75,6 +75,7 @@ spec:
               cpu: 1m
               memory: 5Mi
       dnsPolicy: ClusterFirst
+      terminationGracePeriodSeconds: 30
       volumes:
         - name: config
           configMap:

operations/helm/tests/global-image-registry/grafana-agent/templates/controllers/daemonset.yaml

@@ -70,6 +70,7 @@ spec:
               cpu: 1m
               memory: 5Mi
       dnsPolicy: ClusterFirst
+      terminationGracePeriodSeconds: 30
       volumes:
         - name: config
           configMap:

operations/helm/tests/graceful-shutdown/grafana-agent/templates/configmap.yaml

@@ -0,0 +1,42 @@
+---
+# Source: grafana-agent/templates/configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: grafana-agent
+  labels:
+    helm.sh/chart: grafana-agent
+    app.kubernetes.io/name: grafana-agent
+    app.kubernetes.io/instance: grafana-agent
+    app.kubernetes.io/version: "vX.Y.Z"
+    app.kubernetes.io/managed-by: Helm
+data:
+  config.river: |-
+    logging {
+    	level  = "info"
+    	format = "logfmt"
+    }
+
+    discovery.kubernetes "pods" {
+    	role = "pod"
+    }
+
+    discovery.kubernetes "nodes" {
+    	role = "node"
+    }
+
+    discovery.kubernetes "services" {
+    	role = "service"
+    }
+
+    discovery.kubernetes "endpoints" {
+    	role = "endpoints"
+    }
+
+    discovery.kubernetes "endpointslices" {
+    	role = "endpointslice"
+    }
+
+    discovery.kubernetes "ingresses" {
+    	role = "ingress"
+    }

operations/helm/tests/graceful-shutdown/grafana-agent/templates/controllers/deployment.yaml

@@ -0,0 +1,84 @@
+---
+# Source: grafana-agent/templates/controllers/deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: grafana-agent
+  labels:
+    helm.sh/chart: grafana-agent
+    app.kubernetes.io/name: grafana-agent
+    app.kubernetes.io/instance: grafana-agent
+    app.kubernetes.io/version: "vX.Y.Z"
+    app.kubernetes.io/managed-by: Helm
+spec:
+  replicas: 1
+  minReadySeconds: 10
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: grafana-agent
+      app.kubernetes.io/instance: grafana-agent
+  template:
+    metadata:
+      annotations:
+        kubectl.kubernetes.io/default-container: grafana-agent
+      labels:
+        app.kubernetes.io/name: grafana-agent
+        app.kubernetes.io/instance: grafana-agent
+    spec:
+      serviceAccountName: grafana-agent
+      containers:
+        - name: grafana-agent
+          image: docker.io/grafana/agent:v0.40.1
+          imagePullPolicy: IfNotPresent
+          args:
+            - run
+            - /etc/agent/config.river
+            - --storage.path=/tmp/agent
+            - --server.http.listen-addr=0.0.0.0:80
+            - --server.http.ui-path-prefix=/
+          env:
+            - name: AGENT_MODE
+              value: flow
+            - name: AGENT_DEPLOY_MODE
+              value: "helm"
+            - name: HOSTNAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          lifecycle:
+            preStop:
+              exec:
+                command:
+                - sleep
+                - "60"
+          ports:
+            - containerPort: 80
+              name: http-metrics
+          readinessProbe:
+            httpGet:
+              path: /-/ready
+              port: 80
+              scheme: HTTP
+            initialDelaySeconds: 10
+            timeoutSeconds: 1
+          volumeMounts:
+            - name: config
+              mountPath: /etc/agent
+        - name: config-reloader
+          image: ghcr.io/jimmidyson/configmap-reload:v0.12.0
+          args:
+            - --volume-dir=/etc/agent
+            - --webhook-url=http://localhost:80/-/reload
+          volumeMounts:
+            - name: config
+              mountPath: /etc/agent
+          resources:
+            requests:
+              cpu: 1m
+              memory: 5Mi
+      dnsPolicy: ClusterFirst
+      terminationGracePeriodSeconds: 300
+      volumes:
+        - name: config
+          configMap:
+            name: grafana-agent