From ec66ee29f332572e66131447e5ecf49cba71480d Mon Sep 17 00:00:00 2001 From: Tuomo Tanskanen Date: Fri, 31 Mar 2023 13:39:27 +0300 Subject: [PATCH] run BMO deployment as non-root (v2) BMO ironic has no reason to run as root. Make it run as "ironic" user. dnsmasq requires elevated capabiities. k8s is missing the feature of ambient capabilities, so it requires us to setcap the binaries with expected capabilities and container must be running with "allowPrivilegeEscalation: true" in the manifest to allow elevation. Read the ambient capabilities KEP for more details: https://github.com/kubernetes/enhancements/blob/master/keps/sig-security/2763-ambient-capabilities/README.md Add securityContext to BMO deployment manifest and keepalived component, with correct UIDs and GIDs. This is important to be able to share files via /shared. Modify keepalived image to run as ironic user, which we use the same UID and GID as the ironic-image. This commit requires ironic-image with PR https://github.com/metal3-io/ironic-image/pull/410 to be merged to work. This v2 of the PR fixes issues identified after merging 1st PR: - mariadb was missing securityContext and failed to run - keepalived changes were not backwards compatible, and due using only single tag for all versions, new image broke all release branches --- ironic-deployment/base/ironic.yaml | 314 +++++++++++------- .../keepalived/keepalived_patch.yaml | 19 +- .../components/mariadb/mariadb_patch.yaml | 112 ++++--- resources/keepalived-docker/Dockerfile | 10 +- .../keepalived-docker/configure-nonroot.sh | 20 ++ .../keepalived-docker/manage-keepalived.sh | 15 +- 6 files changed, 295 insertions(+), 195 deletions(-) create mode 100755 resources/keepalived-docker/configure-nonroot.sh diff --git a/ironic-deployment/base/ironic.yaml b/ironic-deployment/base/ironic.yaml index ea5bdf35be..31609189ac 100644 --- a/ironic-deployment/base/ironic.yaml +++ b/ironic-deployment/base/ironic.yaml @@ -19,134 +19,190 @@ spec: spec: hostNetwork: true containers: - - name: ironic-dnsmasq - image: quay.io/metal3-io/ironic - imagePullPolicy: Always - securityContext: - capabilities: - add: ["NET_ADMIN", "NET_RAW"] - command: - - /bin/rundnsmasq - livenessProbe: - exec: - command: ["sh", "-c", "ss -lun | grep :67 && ss -lun | grep :69"] - initialDelaySeconds: 30 - periodSeconds: 30 - timeoutSeconds: 10 - successThreshold: 1 - failureThreshold: 10 - readinessProbe: - exec: - command: ["sh", "-c", "ss -lun | grep :67 && ss -lun | grep :69"] - initialDelaySeconds: 30 - periodSeconds: 30 - timeoutSeconds: 10 - successThreshold: 1 - failureThreshold: 10 - volumeMounts: - - mountPath: /shared - name: ironic-data-volume - envFrom: - - configMapRef: - name: ironic-bmo-configmap - - name: ironic - image: quay.io/metal3-io/ironic - imagePullPolicy: Always - command: - - /bin/runironic - livenessProbe: - exec: - command: ["sh", "-c", "curl -sSf http://127.0.0.1:6385 || curl -sSfk https://127.0.0.1:6385"] - initialDelaySeconds: 30 - periodSeconds: 30 - timeoutSeconds: 10 - successThreshold: 1 - failureThreshold: 10 - readinessProbe: - exec: - command: ["sh", "-c", "curl -sSf http://127.0.0.1:6385 || curl -sSfk https://127.0.0.1:6385"] - initialDelaySeconds: 30 - periodSeconds: 30 - timeoutSeconds: 10 - successThreshold: 1 - failureThreshold: 10 - volumeMounts: - - mountPath: /shared - name: ironic-data-volume - envFrom: - - configMapRef: - name: ironic-bmo-configmap - - name: ironic-log-watch - image: quay.io/metal3-io/ironic - imagePullPolicy: Always - command: - - /bin/runlogwatch.sh - volumeMounts: - - mountPath: /shared - name: ironic-data-volume - - name: ironic-inspector - image: quay.io/metal3-io/ironic - imagePullPolicy: Always - readinessProbe: - exec: - command: ["sh", "-c", "curl -sSf http://127.0.0.1:5050 || curl -sSf -k https://127.0.0.1:5050"] - initialDelaySeconds: 30 - periodSeconds: 30 - timeoutSeconds: 10 - successThreshold: 1 - failureThreshold: 10 - livenessProbe: - exec: - command: ["sh", "-c", "curl -sSf http://127.0.0.1:5050 || curl -sSf -k https://127.0.0.1:5050"] - initialDelaySeconds: 30 - periodSeconds: 30 - timeoutSeconds: 10 - successThreshold: 1 - failureThreshold: 10 - command: - - /bin/runironic-inspector - envFrom: - - configMapRef: - name: ironic-bmo-configmap - - name: ironic-httpd - image: quay.io/metal3-io/ironic - imagePullPolicy: Always - command: - - /bin/runhttpd - livenessProbe: - exec: - command: ["sh", "-c", "curl -sSfk http://127.0.0.1:6180/images"] - initialDelaySeconds: 30 - periodSeconds: 30 - timeoutSeconds: 10 - successThreshold: 1 - failureThreshold: 10 - readinessProbe: - exec: - command: ["sh", "-c", "curl -sSfk http://127.0.0.1:6180/images"] - initialDelaySeconds: 30 - periodSeconds: 30 - timeoutSeconds: 10 - successThreshold: 1 - failureThreshold: 10 - volumeMounts: - - mountPath: /shared - name: ironic-data-volume - envFrom: - - configMapRef: - name: ironic-bmo-configmap + - name: ironic-dnsmasq + image: quay.io/metal3-io/ironic + imagePullPolicy: Always + securityContext: + # Must be true so dnsmasq may get the capabilities via file caps + # KEP: https://github.com/kubernetes/enhancements/blob/master/keps/sig-security/2763-ambient-capabilities/README.md + allowPrivilegeEscalation: true + capabilities: + drop: + - ALL + add: + - NET_ADMIN + - NET_BIND_SERVICE + - NET_RAW + privileged: false + runAsUser: 997 # ironic + runAsGroup: 994 # ironic + command: + - /bin/rundnsmasq + livenessProbe: + exec: + command: ["sh", "-c", "ss -lun | grep :67 && ss -lun | grep :69"] + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + successThreshold: 1 + failureThreshold: 10 + readinessProbe: + exec: + command: ["sh", "-c", "ss -lun | grep :67 && ss -lun | grep :69"] + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + successThreshold: 1 + failureThreshold: 10 + volumeMounts: + - mountPath: /shared + name: ironic-data-volume + envFrom: + - configMapRef: + name: ironic-bmo-configmap + - name: ironic + image: quay.io/metal3-io/ironic + imagePullPolicy: Always + command: + - /bin/runironic + livenessProbe: + exec: + command: ["sh", "-c", "curl -sSf http://127.0.0.1:6385 || curl -sSfk https://127.0.0.1:6385"] + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + successThreshold: 1 + failureThreshold: 10 + readinessProbe: + exec: + command: ["sh", "-c", "curl -sSf http://127.0.0.1:6385 || curl -sSfk https://127.0.0.1:6385"] + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + successThreshold: 1 + failureThreshold: 10 + volumeMounts: + - mountPath: /shared + name: ironic-data-volume + envFrom: + - configMapRef: + name: ironic-bmo-configmap + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + runAsUser: 997 # ironic + runAsGroup: 994 # ironic + - name: ironic-log-watch + image: quay.io/metal3-io/ironic + imagePullPolicy: Always + command: + - /bin/runlogwatch.sh + volumeMounts: + - mountPath: /shared + name: ironic-data-volume + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + runAsUser: 997 # ironic + runAsGroup: 994 # ironic + - name: ironic-inspector + image: quay.io/metal3-io/ironic + imagePullPolicy: Always + readinessProbe: + exec: + command: ["sh", "-c", "curl -sSf http://127.0.0.1:5050 || curl -sSf -k https://127.0.0.1:5050"] + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + successThreshold: 1 + failureThreshold: 10 + livenessProbe: + exec: + command: ["sh", "-c", "curl -sSf http://127.0.0.1:5050 || curl -sSf -k https://127.0.0.1:5050"] + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + successThreshold: 1 + failureThreshold: 10 + command: + - /bin/runironic-inspector + envFrom: + - configMapRef: + name: ironic-bmo-configmap + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + runAsUser: 996 # ironic-inspector + runAsGroup: 993 # ironicinspector + - name: ironic-httpd + image: quay.io/metal3-io/ironic + imagePullPolicy: Always + command: + - /bin/runhttpd + livenessProbe: + exec: + command: ["sh", "-c", "curl -sSfk http://127.0.0.1:6180/images"] + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + successThreshold: 1 + failureThreshold: 10 + readinessProbe: + exec: + command: ["sh", "-c", "curl -sSfk http://127.0.0.1:6180/images"] + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + successThreshold: 1 + failureThreshold: 10 + volumeMounts: + - mountPath: /shared + name: ironic-data-volume + envFrom: + - configMapRef: + name: ironic-bmo-configmap + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + runAsUser: 997 # ironic + runAsGroup: 994 # ironic initContainers: - - name: ironic-ipa-downloader - image: quay.io/metal3-io/ironic-ipa-downloader - imagePullPolicy: Always - command: - - /usr/local/bin/get-resource.sh - envFrom: - - configMapRef: - name: ironic-bmo-configmap - volumeMounts: - - mountPath: /shared - name: ironic-data-volume + - name: ironic-ipa-downloader + image: quay.io/metal3-io/ironic-ipa-downloader + imagePullPolicy: Always + command: + - /usr/local/bin/get-resource.sh + envFrom: + - configMapRef: + name: ironic-bmo-configmap + volumeMounts: + - mountPath: /shared + name: ironic-data-volume + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + runAsUser: 997 # ironic + runAsGroup: 994 # ironic volumes: - - name: ironic-data-volume - emptyDir: {} + - name: ironic-data-volume + emptyDir: {} + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + fsGroup: 994 diff --git a/ironic-deployment/components/keepalived/keepalived_patch.yaml b/ironic-deployment/components/keepalived/keepalived_patch.yaml index 5d24688234..a6a03cdfbd 100644 --- a/ironic-deployment/components/keepalived/keepalived_patch.yaml +++ b/ironic-deployment/components/keepalived/keepalived_patch.yaml @@ -10,8 +10,19 @@ spec: - image: quay.io/metal3-io/keepalived name: ironic-endpoint-keepalived securityContext: - capabilities: - add: ["NET_ADMIN", "NET_RAW"] + # Must be true so dnsmasq may get the capabilities via file caps + # KEP: https://github.com/kubernetes/enhancements/blob/master/keps/sig-security/2763-ambient-capabilities/README.md + allowPrivilegeEscalation: true + capabilities: + drop: + - ALL + add: + - NET_ADMIN + - NET_BROADCAST + - NET_RAW + privileged: false + runAsUser: 65532 + runAsGroup: 65532 envFrom: - - configMapRef: - name: ironic-bmo-configmap + - configMapRef: + name: ironic-bmo-configmap diff --git a/ironic-deployment/components/mariadb/mariadb_patch.yaml b/ironic-deployment/components/mariadb/mariadb_patch.yaml index ca75d16de8..acffad2448 100644 --- a/ironic-deployment/components/mariadb/mariadb_patch.yaml +++ b/ironic-deployment/components/mariadb/mariadb_patch.yaml @@ -6,58 +6,66 @@ spec: template: spec: containers: - - name: mariadb - image: quay.io/metal3-io/mariadb - imagePullPolicy: Always - livenessProbe: - exec: - command: ["sh", "-c", "mysqladmin status -uironic -p$(printenv MARIADB_PASSWORD)"] - initialDelaySeconds: 30 - periodSeconds: 30 - timeoutSeconds: 10 - successThreshold: 1 - failureThreshold: 10 - readinessProbe: - exec: - command: ["sh", "-c", "mysqladmin status -uironic -p$(printenv MARIADB_PASSWORD)"] - initialDelaySeconds: 30 - periodSeconds: 30 - timeoutSeconds: 10 - successThreshold: 1 - failureThreshold: 10 - volumeMounts: - - mountPath: /shared - name: ironic-data-volume - - name: cert-mariadb - mountPath: "/certs/mariadb" - readOnly: true - - name: cert-mariadb-ca - mountPath: "/certs/ca/mariadb" - readOnly: true - env: - - name: MARIADB_PASSWORD - valueFrom: - secretKeyRef: - name: mariadb-password - key: password - - name: RESTART_CONTAINER_CERTIFICATE_UPDATED - valueFrom: - configMapKeyRef: - name: ironic-bmo-configmap - key: RESTART_CONTAINER_CERTIFICATE_UPDATED - - name: ironic - env: - - name: MARIADB_PASSWORD - valueFrom: - secretKeyRef: - name: mariadb-password - key: password - - name: IRONIC_USE_MARIADB - value: "true" - volumeMounts: - - name: cert-mariadb-ca - mountPath: "/certs/ca/mariadb" - readOnly: true + - name: mariadb + image: quay.io/metal3-io/mariadb + imagePullPolicy: Always + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + runAsUser: 27 + runAsGroup: 27 + livenessProbe: + exec: + command: ["sh", "-c", "mysqladmin status -uironic -p$(printenv MARIADB_PASSWORD)"] + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + successThreshold: 1 + failureThreshold: 10 + readinessProbe: + exec: + command: ["sh", "-c", "mysqladmin status -uironic -p$(printenv MARIADB_PASSWORD)"] + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + successThreshold: 1 + failureThreshold: 10 + volumeMounts: + - mountPath: /shared + name: ironic-data-volume + - name: cert-mariadb + mountPath: "/certs/mariadb" + readOnly: true + - name: cert-mariadb-ca + mountPath: "/certs/ca/mariadb" + readOnly: true + env: + - name: MARIADB_PASSWORD + valueFrom: + secretKeyRef: + name: mariadb-password + key: password + - name: RESTART_CONTAINER_CERTIFICATE_UPDATED + valueFrom: + configMapKeyRef: + name: ironic-bmo-configmap + key: RESTART_CONTAINER_CERTIFICATE_UPDATED + - name: ironic + env: + - name: MARIADB_PASSWORD + valueFrom: + secretKeyRef: + name: mariadb-password + key: password + - name: IRONIC_USE_MARIADB + value: "true" + volumeMounts: + - name: cert-mariadb-ca + mountPath: "/certs/ca/mariadb" + readOnly: true volumes: - name: cert-mariadb secret: diff --git a/resources/keepalived-docker/Dockerfile b/resources/keepalived-docker/Dockerfile index a8a484564b..f7c752cd18 100644 --- a/resources/keepalived-docker/Dockerfile +++ b/resources/keepalived-docker/Dockerfile @@ -4,11 +4,13 @@ ARG BASE_IMAGE=ubuntu:22.04 FROM $BASE_IMAGE ARG DEBIAN_FRONTEND=noninteractive -COPY sample.keepalived.conf /etc/keepalived/keepalived.conf -COPY manage-keepalived.sh manage-keepalived.sh - RUN apt-get -y update && \ apt-get -y install keepalived && \ apt-get -y clean -ENTRYPOINT ["/bin/bash", "manage-keepalived.sh"] +COPY sample.keepalived.conf /etc/keepalived/keepalived.conf +COPY manage-keepalived.sh configure-nonroot.sh /bin/ + +RUN /bin/configure-nonroot.sh && rm /bin/configure-nonroot.sh + +CMD ["/bin/bash", "/bin/manage-keepalived.sh"] diff --git a/resources/keepalived-docker/configure-nonroot.sh b/resources/keepalived-docker/configure-nonroot.sh new file mode 100755 index 0000000000..6bebd789af --- /dev/null +++ b/resources/keepalived-docker/configure-nonroot.sh @@ -0,0 +1,20 @@ +#!/usr/bin/bash + +set -eux + +# create nonroot image matching the keepalived manifest +NONROOT_USER="nonroot" +NONROOT_GROUP="nonroot" +NONROOT_UID=65532 +NONROOT_GID=65532 + +# run as non-root, allow editing the keepalived.conf during startup +groupadd -g "${NONROOT_GID}" "${NONROOT_GROUP}" +useradd -u "${NONROOT_UID}" -g "${NONROOT_GID}" -m "${NONROOT_USER}" + +mkdir -p /run/keepalived +chown -R root:"${NONROOT_GROUP}" /etc/keepalived /run/keepalived +chmod 2775 /etc/keepalived /run/keepalived +chmod 664 /etc/keepalived/keepalived.conf + +setcap "cap_net_raw,cap_net_broadcast,cap_net_admin=+eip" /usr/sbin/keepalived diff --git a/resources/keepalived-docker/manage-keepalived.sh b/resources/keepalived-docker/manage-keepalived.sh index bf4841c4e1..392077ba74 100644 --- a/resources/keepalived-docker/manage-keepalived.sh +++ b/resources/keepalived-docker/manage-keepalived.sh @@ -1,8 +1,11 @@ -#!/bin/bash -export assignedIP="$PROVISIONING_IP/32" -export interface=$PROVISIONING_INTERFACE +#!/usr/bin/bash -sed -i "s~INTERFACE~$interface~g" /etc/keepalived/keepalived.conf -sed -i "s~CHANGEIP~$assignedIP~g" /etc/keepalived/keepalived.conf +set -eux -exec /usr/sbin/keepalived -n -l \ No newline at end of file +export assignedIP="${PROVISIONING_IP}/32" +export interface="${PROVISIONING_INTERFACE}" + +sed -i "s~INTERFACE~${interface}~g" /etc/keepalived/keepalived.conf +sed -i "s~CHANGEIP~${assignedIP}~g" /etc/keepalived/keepalived.conf + +exec /usr/sbin/keepalived -n -l -p /run/keepalived/keepalived.pid -r /run/keepalived/vrrp.pid