run BMO deployment as non-root (v2)

BMO ironic has no reason to run as root. Make it run as "ironic" user. dnsmasq requires elevated capabiities. k8s is missing the feature of ambient capabilities, so it requires us to setcap the binaries with expected capabilities and container must be running with "allowPrivilegeEscalation: true" in the manifest to allow elevation. Read the ambient capabilities KEP for more details: https://github.com/kubernetes/enhancements/blob/master/keps/sig-security/2763-ambient-capabilities/README.md Add securityContext to BMO deployment manifest and keepalived component, with correct UIDs and GIDs. This is important to be able to share files via /shared. Modify keepalived image to run as ironic user, which we use the same UID and GID as the ironic-image. This commit requires ironic-image with PR metal3-io/ironic-image#410 to be merged to work. This v2 of the PR fixes issues identified after merging 1st PR: - mariadb was missing securityContext and failed to run - keepalived changes were not backwards compatible, and due using only single tag for all versions, new image broke all release branches
NeerajNagure · Apr 6, 2023 · ec66ee2 · ec66ee2
1 parent 58b4139
commit ec66ee2
Show file tree

Hide file tree

Showing 6 changed files with 295 additions and 195 deletions.
diff --git a/ironic-deployment/base/ironic.yaml b/ironic-deployment/base/ironic.yaml
@@ -19,134 +19,190 @@ spec:
     spec:
       hostNetwork: true
       containers:
-        - name: ironic-dnsmasq
-          image: quay.io/metal3-io/ironic
-          imagePullPolicy: Always
-          securityContext:
-             capabilities:
-               add: ["NET_ADMIN", "NET_RAW"]
-          command:
-            - /bin/rundnsmasq
-          livenessProbe:
-           exec:
-             command: ["sh", "-c", "ss -lun | grep :67 && ss -lun | grep :69"]
-           initialDelaySeconds: 30
-           periodSeconds: 30
-           timeoutSeconds: 10
-           successThreshold: 1
-           failureThreshold: 10
-          readinessProbe:
-           exec:
-             command: ["sh", "-c", "ss -lun | grep :67 && ss -lun | grep :69"]
-           initialDelaySeconds: 30
-           periodSeconds: 30
-           timeoutSeconds: 10
-           successThreshold: 1
-           failureThreshold: 10
-          volumeMounts:
-            - mountPath: /shared
-              name: ironic-data-volume
-          envFrom:
-            - configMapRef:
-                name: ironic-bmo-configmap
-        - name: ironic
-          image: quay.io/metal3-io/ironic
-          imagePullPolicy: Always
-          command:
-            - /bin/runironic
-          livenessProbe:
-           exec:
-             command: ["sh", "-c", "curl -sSf http://127.0.0.1:6385 || curl -sSfk https://127.0.0.1:6385"]
-           initialDelaySeconds: 30
-           periodSeconds: 30
-           timeoutSeconds: 10
-           successThreshold: 1
-           failureThreshold: 10
-          readinessProbe:
-           exec:
-             command: ["sh", "-c", "curl -sSf http://127.0.0.1:6385 || curl -sSfk https://127.0.0.1:6385"]
-           initialDelaySeconds: 30
-           periodSeconds: 30
-           timeoutSeconds: 10
-           successThreshold: 1
-           failureThreshold: 10
-          volumeMounts:
-            - mountPath: /shared
-              name: ironic-data-volume
-          envFrom:
-            - configMapRef:
-                name: ironic-bmo-configmap
-        - name: ironic-log-watch
-          image: quay.io/metal3-io/ironic
-          imagePullPolicy: Always
-          command:
-            - /bin/runlogwatch.sh
-          volumeMounts:
-            - mountPath: /shared
-              name: ironic-data-volume
-        - name: ironic-inspector
-          image: quay.io/metal3-io/ironic
-          imagePullPolicy: Always
-          readinessProbe:
-            exec:
-              command: ["sh", "-c", "curl -sSf http://127.0.0.1:5050 || curl -sSf -k https://127.0.0.1:5050"]
-            initialDelaySeconds: 30
-            periodSeconds: 30
-            timeoutSeconds: 10
-            successThreshold: 1
-            failureThreshold: 10
-          livenessProbe:
-            exec:
-              command: ["sh", "-c", "curl -sSf http://127.0.0.1:5050 || curl -sSf -k https://127.0.0.1:5050"]
-            initialDelaySeconds: 30
-            periodSeconds: 30
-            timeoutSeconds: 10
-            successThreshold: 1
-            failureThreshold: 10
-          command:
-            - /bin/runironic-inspector
-          envFrom:
-            - configMapRef:
-                name: ironic-bmo-configmap
-        - name: ironic-httpd
-          image: quay.io/metal3-io/ironic
-          imagePullPolicy: Always
-          command:
-            - /bin/runhttpd
-          livenessProbe:
-           exec:
-             command: ["sh", "-c", "curl -sSfk http://127.0.0.1:6180/images"]
-           initialDelaySeconds: 30
-           periodSeconds: 30
-           timeoutSeconds: 10
-           successThreshold: 1
-           failureThreshold: 10
-          readinessProbe:
-           exec:
-             command: ["sh", "-c", "curl -sSfk http://127.0.0.1:6180/images"]
-           initialDelaySeconds: 30
-           periodSeconds: 30
-           timeoutSeconds: 10
-           successThreshold: 1
-           failureThreshold: 10
-          volumeMounts:
-            - mountPath: /shared
-              name: ironic-data-volume
-          envFrom:
-            - configMapRef:
-                name: ironic-bmo-configmap
+      - name: ironic-dnsmasq
+        image: quay.io/metal3-io/ironic
+        imagePullPolicy: Always
+        securityContext:
+          # Must be true so dnsmasq may get the capabilities via file caps
+          # KEP: https://github.com/kubernetes/enhancements/blob/master/keps/sig-security/2763-ambient-capabilities/README.md
+          allowPrivilegeEscalation: true
+          capabilities:
+            drop:
+            - ALL
+            add:
+            - NET_ADMIN
+            - NET_BIND_SERVICE
+            - NET_RAW
+          privileged: false
+          runAsUser: 997 # ironic
+          runAsGroup: 994 # ironic
+        command:
+        - /bin/rundnsmasq
+        livenessProbe:
+          exec:
+            command: ["sh", "-c", "ss -lun | grep :67 && ss -lun | grep :69"]
+          initialDelaySeconds: 30
+          periodSeconds: 30
+          timeoutSeconds: 10
+          successThreshold: 1
+          failureThreshold: 10
+        readinessProbe:
+          exec:
+            command: ["sh", "-c", "ss -lun | grep :67 && ss -lun | grep :69"]
+          initialDelaySeconds: 30
+          periodSeconds: 30
+          timeoutSeconds: 10
+          successThreshold: 1
+          failureThreshold: 10
+        volumeMounts:
+        - mountPath: /shared
+          name: ironic-data-volume
+        envFrom:
+        - configMapRef:
+            name: ironic-bmo-configmap
+      - name: ironic
+        image: quay.io/metal3-io/ironic
+        imagePullPolicy: Always
+        command:
+        - /bin/runironic
+        livenessProbe:
+          exec:
+            command: ["sh", "-c", "curl -sSf http://127.0.0.1:6385 || curl -sSfk https://127.0.0.1:6385"]
+          initialDelaySeconds: 30
+          periodSeconds: 30
+          timeoutSeconds: 10
+          successThreshold: 1
+          failureThreshold: 10
+        readinessProbe:
+          exec:
+            command: ["sh", "-c", "curl -sSf http://127.0.0.1:6385 || curl -sSfk https://127.0.0.1:6385"]
+          initialDelaySeconds: 30
+          periodSeconds: 30
+          timeoutSeconds: 10
+          successThreshold: 1
+          failureThreshold: 10
+        volumeMounts:
+        - mountPath: /shared
+          name: ironic-data-volume
+        envFrom:
+        - configMapRef:
+            name: ironic-bmo-configmap
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - ALL
+          privileged: false
+          runAsUser: 997 # ironic
+          runAsGroup: 994 # ironic
+      - name: ironic-log-watch
+        image: quay.io/metal3-io/ironic
+        imagePullPolicy: Always
+        command:
+        - /bin/runlogwatch.sh
+        volumeMounts:
+        - mountPath: /shared
+          name: ironic-data-volume
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - ALL
+          privileged: false
+          runAsUser: 997 # ironic
+          runAsGroup: 994 # ironic
+      - name: ironic-inspector
+        image: quay.io/metal3-io/ironic
+        imagePullPolicy: Always
+        readinessProbe:
+          exec:
+            command: ["sh", "-c", "curl -sSf http://127.0.0.1:5050 || curl -sSf -k https://127.0.0.1:5050"]
+          initialDelaySeconds: 30
+          periodSeconds: 30
+          timeoutSeconds: 10
+          successThreshold: 1
+          failureThreshold: 10
+        livenessProbe:
+          exec:
+            command: ["sh", "-c", "curl -sSf http://127.0.0.1:5050 || curl -sSf -k https://127.0.0.1:5050"]
+          initialDelaySeconds: 30
+          periodSeconds: 30
+          timeoutSeconds: 10
+          successThreshold: 1
+          failureThreshold: 10
+        command:
+        - /bin/runironic-inspector
+        envFrom:
+        - configMapRef:
+            name: ironic-bmo-configmap
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - ALL
+          privileged: false
+          runAsUser: 996 # ironic-inspector
+          runAsGroup: 993 # ironicinspector
+      - name: ironic-httpd
+        image: quay.io/metal3-io/ironic
+        imagePullPolicy: Always
+        command:
+        - /bin/runhttpd
+        livenessProbe:
+          exec:
+            command: ["sh", "-c", "curl -sSfk http://127.0.0.1:6180/images"]
+          initialDelaySeconds: 30
+          periodSeconds: 30
+          timeoutSeconds: 10
+          successThreshold: 1
+          failureThreshold: 10
+        readinessProbe:
+          exec:
+            command: ["sh", "-c", "curl -sSfk http://127.0.0.1:6180/images"]
+          initialDelaySeconds: 30
+          periodSeconds: 30
+          timeoutSeconds: 10
+          successThreshold: 1
+          failureThreshold: 10
+        volumeMounts:
+        - mountPath: /shared
+          name: ironic-data-volume
+        envFrom:
+        - configMapRef:
+            name: ironic-bmo-configmap
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - ALL
+          privileged: false
+          runAsUser: 997 # ironic
+          runAsGroup: 994 # ironic
       initContainers:
-        - name: ironic-ipa-downloader
-          image: quay.io/metal3-io/ironic-ipa-downloader
-          imagePullPolicy: Always
-          command:
-            - /usr/local/bin/get-resource.sh
-          envFrom:
-            - configMapRef:
-                name: ironic-bmo-configmap
-          volumeMounts:
-            - mountPath: /shared
-              name: ironic-data-volume
+      - name: ironic-ipa-downloader
+        image: quay.io/metal3-io/ironic-ipa-downloader
+        imagePullPolicy: Always
+        command:
+        - /usr/local/bin/get-resource.sh
+        envFrom:
+        - configMapRef:
+            name: ironic-bmo-configmap
+        volumeMounts:
+        - mountPath: /shared
+          name: ironic-data-volume
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - ALL
+          privileged: false
+          runAsUser: 997 # ironic
+          runAsGroup: 994 # ironic
       volumes:
-        - name: ironic-data-volume
-          emptyDir: {}
+      - name: ironic-data-volume
+        emptyDir: {}
+      securityContext:
+        runAsNonRoot: true
+        seccompProfile:
+          type: RuntimeDefault
+        fsGroup: 994
diff --git a/ironic-deployment/components/keepalived/keepalived_patch.yaml b/ironic-deployment/components/keepalived/keepalived_patch.yaml
@@ -10,8 +10,19 @@ spec:
       - image: quay.io/metal3-io/keepalived
         name: ironic-endpoint-keepalived
         securityContext:
-              capabilities:
-                add: ["NET_ADMIN", "NET_RAW"]
+          # Must be true so dnsmasq may get the capabilities via file caps
+          # KEP: https://github.com/kubernetes/enhancements/blob/master/keps/sig-security/2763-ambient-capabilities/README.md
+          allowPrivilegeEscalation: true
+          capabilities:
+            drop:
+            - ALL
+            add:
+            - NET_ADMIN
+            - NET_BROADCAST
+            - NET_RAW
+          privileged: false
+          runAsUser: 65532
+          runAsGroup: 65532
         envFrom:
-          - configMapRef:
-              name: ironic-bmo-configmap
+        - configMapRef:
+            name: ironic-bmo-configmap