Ensure logging pods only run on Linux nodes

Ensure logging pods only run on Linux nodes by adding tolerations and node selectors for fluentbit and fluentd. The tolerations have "NoSchedule" for Windows nodes, and the node selectors look for nodes with a Linux-based OS.
rancher · Oct 12, 2020 · 055db82 · 055db82
1 parent 5b2da0a
commit 055db82
Show file tree

Hide file tree

Showing 10 changed files with 162 additions and 29 deletions.
diff --git a/packages/rancher-logging/overlay/templates/loggings/eks/logging.yaml b/packages/rancher-logging/overlay/templates/loggings/eks/logging.yaml
@@ -16,10 +16,14 @@ spec:
       Tag: "eks"
       Path: "/var/log/messages"
       Parser: "syslog"
-    {{ with .Values.fluentbit_tolerations }}
+    {{ with .Values.images.fluentbit.tolerations }}
     tolerations:
       {{ toYaml . | nindent 6 }}
     {{ end }}
+    {{ with .Values.images.fluentbit.nodeSelector }}
+    nodeSelector:
+      {{ toYaml . | nindent 6 }}
+    {{ end }}
   fluentd:
     image:
       repository: {{ template "system_default_registry" . }}{{ .Values.images.fluentd.repository }}
@@ -28,4 +32,12 @@ spec:
       repository: {{ template "system_default_registry" . }}{{ .Values.images.config_reloader.repository }}
       tag: {{ .Values.images.config_reloader.tag }}
     disablePvc: {{ .Values.disablePvc }}
+    {{ with .Values.images.fluentd.tolerations }}
+    tolerations:
+      {{ toYaml . | nindent 6 }}
+    {{ end }}
+    {{ with .Values.images.fluentd.nodeSelector }}
+    nodeSelector:
+      {{ toYaml . | nindent 6 }}
+    {{ end }}
 {{- end }}
diff --git a/packages/rancher-logging/overlay/templates/loggings/k3s/logging-k3s-openrc.yaml b/packages/rancher-logging/overlay/templates/loggings/k3s/logging-k3s-openrc.yaml
@@ -19,10 +19,14 @@ spec:
       - source: "/var/log/"
         destination: "/var/log"
         readOnly: true
-    {{ with .Values.fluentbit_tolerations }}
+    {{ with .Values.images.fluentbit.tolerations }}
     tolerations:
       {{ toYaml . | nindent 6 }}
     {{ end }}
+    {{ with .Values.images.fluentbit.nodeSelector }}
+    nodeSelector:
+      {{ toYaml . | nindent 6 }}
+    {{ end }}
   fluentd:
     image:
       repository: {{ template "system_default_registry" . }}{{ .Values.images.fluentd.repository }}
@@ -31,4 +35,12 @@ spec:
       repository: {{ template "system_default_registry" . }}{{ .Values.images.config_reloader.repository }}
       tag: {{ .Values.images.config_reloader.tag }}
     disablePvc: {{ .Values.disablePvc }}
+    {{ with .Values.images.fluentd.tolerations }}
+    tolerations:
+      {{ toYaml . | nindent 6 }}
+    {{ end }}
+    {{ with .Values.images.fluentd.nodeSelector }}
+    nodeSelector:
+      {{ toYaml . | nindent 6 }}
+    {{ end }}
 {{- end }}
diff --git a/packages/rancher-logging/overlay/templates/loggings/k3s/logging-k3s-systemd.yaml b/packages/rancher-logging/overlay/templates/loggings/k3s/logging-k3s-systemd.yaml
@@ -19,10 +19,14 @@ spec:
       - source: "/var/log/"
         destination: "/var/log"
         readOnly: true
-    {{ with .Values.fluentbit_tolerations }}
+    {{ with .Values.images.fluentbit.tolerations }}
     tolerations:
       {{ toYaml . | nindent 6 }}
     {{ end }}
+    {{ with .Values.images.fluentbit.nodeSelector }}
+    nodeSelector:
+      {{ toYaml . | nindent 6 }}
+    {{ end }}
   fluentd:
     image:
       repository: {{ template "system_default_registry" . }}{{ .Values.images.fluentd.repository }}
@@ -31,4 +35,12 @@ spec:
       repository: {{ template "system_default_registry" . }}{{ .Values.images.config_reloader.repository }}
       tag: {{ .Values.images.config_reloader.tag }}
     disablePvc: {{ .Values.disablePvc }}
+    {{ with .Values.images.fluentd.tolerations }}
+    tolerations:
+      {{ toYaml . | nindent 6 }}
+    {{ end }}
+    {{ with .Values.images.fluentd.nodeSelector }}
+    nodeSelector:
+      {{ toYaml . | nindent 6 }}
+    {{ end }}
 {{- end }}
diff --git a/packages/rancher-logging/overlay/templates/loggings/rke/logging-containers-rke.yaml b/packages/rancher-logging/overlay/templates/loggings/rke/logging-containers-rke.yaml
@@ -17,10 +17,14 @@ spec:
       - source: "/var/log/containers/"
         destination: "/var/log/containers/"
         readOnly: true
-    {{ with .Values.fluentbit_tolerations }}
+    {{ with .Values.images.fluentbit.tolerations }}
     tolerations:
       {{ toYaml . | nindent 6 }}
     {{ end }}
+    {{ with .Values.images.fluentbit.nodeSelector }}
+    nodeSelector:
+      {{ toYaml . | nindent 6 }}
+    {{ end }}
   fluentd:
     image:
       repository: {{ template "system_default_registry" . }}{{ .Values.images.fluentd.repository }}
@@ -29,5 +33,13 @@ spec:
       repository: {{ template "system_default_registry" . }}{{ .Values.images.config_reloader.repository }}
       tag: {{ .Values.images.config_reloader.tag }}
     disablePvc: {{ .Values.disablePvc }}
+    {{ with .Values.images.fluentd.tolerations }}
+    tolerations:
+      {{ toYaml . | nindent 6 }}
+    {{ end }}
+    {{ with .Values.images.fluentd.nodeSelector }}
+    nodeSelector:
+      {{ toYaml . | nindent 6 }}
+    {{ end }}
   {{- end }}
 
diff --git a/packages/rancher-logging/overlay/templates/loggings/rke/logging-rke.yaml b/packages/rancher-logging/overlay/templates/loggings/rke/logging-rke.yaml
@@ -19,10 +19,14 @@ spec:
       - source: "/var/lib/rancher/rke/log"
         destination: "/rke"
         readOnly: true
-    {{ with .Values.fluentbit_tolerations }}
+    {{ with .Values.images.fluentbit.tolerations }}
     tolerations:
       {{ toYaml . | nindent 6 }}
     {{ end }}
+    {{ with .Values.images.fluentbit.nodeSelector }}
+    nodeSelector:
+      {{ toYaml . | nindent 6 }}
+    {{ end }}
   fluentd:
     image:
       repository: {{ template "system_default_registry" . }}{{ .Values.images.fluentd.repository }}
@@ -31,4 +35,12 @@ spec:
       repository: {{ template "system_default_registry" . }}{{ .Values.images.config_reloader.repository }}
       tag: {{ .Values.images.config_reloader.tag }}
     disablePvc: {{ .Values.disablePvc }}
+    {{ with .Values.images.fluentd.tolerations }}
+    tolerations:
+      {{ toYaml . | nindent 6 }}
+    {{ end }}
+    {{ with .Values.images.fluentd.nodeSelector }}
+    nodeSelector:
+      {{ toYaml . | nindent 6 }}
+    {{ end }}
 {{- end }}
diff --git a/packages/rancher-logging/overlay/templates/loggings/rke2/daemonset.yaml b/packages/rancher-logging/overlay/templates/loggings/rke2/daemonset.yaml
@@ -23,6 +23,14 @@ spec:
               name: logdir
             - mountPath: /fluent-bit/etc/
               name: config
+          {{ with .Values.images.fluentd.tolerations }}
+          tolerations:
+            {{ toYaml . | nindent 6 }}
+          {{ end }}
+          {{ with .Values.images.fluentd.nodeSelector }}
+          nodeSelector:
+            {{ toYaml . | nindent 6 }}
+          {{ end }}
       volumes:
         - name: logdir
           hostPath:

diff --git a/packages/rancher-logging/overlay/templates/loggings/rke2/logging-rke2-containers.yaml b/packages/rancher-logging/overlay/templates/loggings/rke2/logging-rke2-containers.yaml
@@ -17,10 +17,14 @@ spec:
       - source: "/var/log/containers/"
         destination: "/var/log/containers/"
         readOnly: true
-    {{ with .Values.fluentbit_tolerations }}
+    {{ with .Values.images.fluentbit.tolerations }}
     tolerations:
       {{ toYaml . | nindent 6 }}
     {{ end }}
+    {{ with .Values.images.fluentbit.nodeSelector }}
+    nodeSelector:
+      {{ toYaml . | nindent 6 }}
+    {{ end }}
   fluentd:
     image:
       repository: {{ template "system_default_registry" . }}{{ .Values.images.fluentd.repository }}
@@ -29,4 +33,12 @@ spec:
       repository: {{ template "system_default_registry" . }}{{ .Values.images.config_reloader.repository }}
       tag: {{ .Values.images.config_reloader.tag }}
     disablePvc: {{ .Values.disablePvc }}
+    {{ with .Values.images.fluentd.tolerations }}
+    tolerations:
+      {{ toYaml . | nindent 6 }}
+    {{ end }}
+    {{ with .Values.images.fluentd.nodeSelector }}
+    nodeSelector:
+      {{ toYaml . | nindent 6 }}
+    {{ end }}
 {{- end }}
diff --git a/packages/rancher-logging/overlay/templates/loggings/rke2/logging-rke2-journald.yaml b/packages/rancher-logging/overlay/templates/loggings/rke2/logging-rke2-journald.yaml
@@ -17,10 +17,14 @@ spec:
       - source: "/etc/rancher/logging/logs/"
         destination: "/etc/rancher/logging/logs/"
         readOnly: true
-    {{ with .Values.fluentbit_tolerations }}
+    {{ with .Values.images.fluentbit.tolerations }}
     tolerations:
       {{ toYaml . | nindent 6 }}
     {{ end }}
+    {{ with .Values.images.fluentbit.nodeSelector }}
+    nodeSelector:
+      {{ toYaml . | nindent 6 }}
+    {{ end }}
   fluentd:
     image:
       repository: {{ template "system_default_registry" . }}{{ .Values.images.fluentd.repository }}
@@ -29,4 +33,12 @@ spec:
       repository: {{ template "system_default_registry" . }}{{ .Values.images.config_reloader.repository }}
       tag: {{ .Values.images.config_reloader.tag }}
     disablePvc: {{ .Values.disablePvc }}
+    {{ with .Values.images.fluentd.tolerations }}
+    tolerations:
+      {{ toYaml . | nindent 6 }}
+    {{ end }}
+    {{ with .Values.images.fluentd.nodeSelector }}
+    nodeSelector:
+      {{ toYaml . | nindent 6 }}
+    {{ end }}
 {{- end }}
diff --git a/packages/rancher-logging/overlay/templates/loggings/root/logging.yaml b/packages/rancher-logging/overlay/templates/loggings/root/logging.yaml
@@ -11,10 +11,14 @@ spec:
     image:
       repository: {{ template "system_default_registry" . }}{{ .Values.images.fluentbit.repository }}
       tag: {{ .Values.images.fluentbit.tag }}
-    {{ with .Values.fluentbit_tolerations }}
+    {{ with .Values.images.fluentbit.tolerations }}
     tolerations:
       {{ toYaml . | nindent 6 }}
     {{ end }}
+    {{ with .Values.images.fluentbit.nodeSelector }}
+    nodeSelector:
+      {{ toYaml . | nindent 6 }}
+    {{ end }}
   fluentd:
     image:
       repository: {{ template "system_default_registry" . }}{{ .Values.images.fluentd.repository }}
@@ -23,3 +27,11 @@ spec:
       repository: {{ template "system_default_registry" . }}{{ .Values.images.config_reloader.repository }}
       tag: {{ .Values.images.config_reloader.tag }}
     disablePvc: {{ .Values.disablePvc }}
+    {{ with .Values.images.fluentd.tolerations }}
+    tolerations:
+      {{ toYaml . | nindent 6 }}
+    {{ end }}
+    {{ with .Values.images.fluentd.nodeSelector }}
+    nodeSelector:
+      {{ toYaml . | nindent 6 }}
+    {{ end }}
diff --git a/packages/rancher-logging/rancher-logging.patch b/packages/rancher-logging/rancher-logging.patch
@@ -20,18 +20,6 @@ diff -x '*.tgz' -x '*.lock' -uNr packages/rancher-logging/charts-original/Chart.
 +  catalog.cattle.io/release-name: rancher-logging
 +  catalog.cattle.io/ui-component: logging
 +  catalog.cattle.io/provides-gvr: logging.banzaicloud.io.clusterflow/v1beta1
-diff -x '*.tgz' -x '*.lock' -uNr packages/rancher-logging/charts-original/templates/deployment.yaml packages/rancher-logging/charts/templates/deployment.yaml
---- packages/rancher-logging/charts-original/templates/deployment.yaml
-+++ packages/rancher-logging/charts/templates/deployment.yaml
-@@ -30,7 +30,7 @@
-     {{- end }}
-       containers:
-         - name: {{ .Chart.Name }}
--          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
-+          image: "{{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }}"
-           imagePullPolicy: {{ .Values.image.pullPolicy }}
-           resources:
-             {{- toYaml .Values.resources | nindent 12 }}
 diff -x '*.tgz' -x '*.lock' -uNr packages/rancher-logging/charts-original/templates/_helpers.tpl packages/rancher-logging/charts/templates/_helpers.tpl
 --- packages/rancher-logging/charts-original/templates/_helpers.tpl
 +++ packages/rancher-logging/charts/templates/_helpers.tpl
@@ -47,6 +35,18 @@ diff -x '*.tgz' -x '*.lock' -uNr packages/rancher-logging/charts-original/templa
 +{{- "" -}}
 +{{- end -}}
 +{{- end -}}
+diff -x '*.tgz' -x '*.lock' -uNr packages/rancher-logging/charts-original/templates/deployment.yaml packages/rancher-logging/charts/templates/deployment.yaml
+--- packages/rancher-logging/charts-original/templates/deployment.yaml
++++ packages/rancher-logging/charts/templates/deployment.yaml
+@@ -30,7 +30,7 @@
+     {{- end }}
+       containers:
+         - name: {{ .Chart.Name }}
+-          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
++          image: "{{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+           imagePullPolicy: {{ .Values.image.pullPolicy }}
+           resources:
+             {{- toYaml .Values.resources | nindent 12 }}
 diff -x '*.tgz' -x '*.lock' -uNr packages/rancher-logging/charts-original/values.yaml packages/rancher-logging/charts/values.yaml
 --- packages/rancher-logging/charts-original/values.yaml
 +++ packages/rancher-logging/charts/values.yaml
@@ -68,7 +68,24 @@ diff -x '*.tgz' -x '*.lock' -uNr packages/rancher-logging/charts-original/values
 
  resources: {}
    # We usually recommend not to specify default resources and to leave this as a conscious
-@@ -76,4 +76,43 @@
+@@ -32,9 +32,14 @@
+   #   cpu: 100m
+   #   memory: 128Mi
+
+-nodeSelector: {}
++nodeSelector:
++  kubernetes.io/os: linux
+
+-tolerations: []
++tolerations:
++  - key: cattle.io/os
++    operator: "Equal"
++    value: "linux"
++    effect: NoSchedule
+
+ affinity: {}
+
+@@ -76,4 +81,55 @@
  monitoring:
    # Create a Prometheus Operator ServiceMonitor object
    serviceMonitor:
@@ -96,21 +113,33 @@ diff -x '*.tgz' -x '*.lock' -uNr packages/rancher-logging/charts-original/values
 +  fluentbit:
 +    repository: rancher/fluent-fluent-bit
 +    tag: 1.5.4
++    nodeSelector:
++      kubernetes.io/os: linux
++    tolerations:
++      - key: node-role.kubernetes.io/controlplane
++        value: "true"
++        effect: NoSchedule
++      - key: node-role.kubernetes.io/etcd
++        value: "true"
++        effect: NoExecute
++      - key: cattle.io/os
++        operator: "Equal"
++        value: "linux"
++        effect: NoSchedule
 +  fluentd:
 +    repository: rancher/banzaicloud-fluentd
 +    tag: v1.11.2-alpine-2
++    nodeSelector:
++      kubernetes.io/os: linux
++    tolerations:
++      - key: cattle.io/os
++        operator: "Equal"
++        value: "linux"
++        effect: NoSchedule
 +  syslog_forwarder:
 +    repository: rancher/fluent-bit-out-syslog
 +    tag: 0.1.0
 +
 +global:
 +  cattle:
 +    systemDefaultRegistry: ""
-+
-+fluentbit_tolerations:
-+  - key: node-role.kubernetes.io/controlplane
-+    value: "true"
-+    effect: NoSchedule
-+  - key: node-role.kubernetes.io/etcd
-+    value: "true"
-+    effect: NoExecute