Read-only container root filesystem

In line with the "Principle of least privilege", add
readOnlyRootFilesystem to the NTO operand's container securityContext.

Other changes:
  * Container image build cleanup after #1058
  * Remove obsolete Dockerfile.rhel8
  * Rename operand to ocp-tuned and use /run/ocp-tuned as home directory

Dockerfile

@@ -16,13 +16,13 @@ COPY hack/dockerfile_install_support.sh /tmp
 RUN /bin/bash /tmp/dockerfile_install_support.sh
 
 COPY manifests/*.yaml manifests/image-references /manifests/
-ENV APP_ROOT=/var/lib/ocp-tuned
-ENV PATH=${APP_ROOT}/bin:${PATH}
-ENV HOME=${APP_ROOT}
+ENV HOME=/run/ocp-tuned
 ENV SYSTEMD_IGNORE_CHROOT=1
-WORKDIR ${APP_ROOT}
+WORKDIR ${HOME}
+
 RUN dnf clean all && \
-    rm -rf /var/cache/yum ~/patches /root/rpms && \
+    rm -rf /var/cache/yum ~/patches /root/rpms /tmp && \
+    ln -s /run/ocp-tuned /tmp && \
     useradd -r -u 499 cluster-node-tuning-operator
 ENTRYPOINT ["/usr/bin/cluster-node-tuning-operator"]
 LABEL io.k8s.display-name="OpenShift cluster-node-tuning-operator" \

Dockerfile.rhel9

@@ -15,14 +15,13 @@ COPY hack/dockerfile_install_support.sh /tmp
 RUN /bin/bash /tmp/dockerfile_install_support.sh
 
 COPY manifests/*.yaml manifests/image-references /manifests/
-ENV APP_ROOT=/var/lib/ocp-tuned
-ENV PATH=${APP_ROOT}/bin:${PATH}
-ENV HOME=${APP_ROOT}
+ENV HOME=/run/ocp-tuned
 ENV SYSTEMD_IGNORE_CHROOT=1
-WORKDIR ${APP_ROOT}
+WORKDIR ${HOME}
 
 RUN dnf clean all && \
-    rm -rf /var/cache/yum ~/patches /root/rpms && \
+    rm -rf /var/cache/yum ~/patches /root/rpms /tmp && \
+    ln -s /run/ocp-tuned /tmp && \
     useradd -r -u 499 cluster-node-tuning-operator
 ENTRYPOINT ["/usr/bin/cluster-node-tuning-operator"]
 LABEL io.k8s.display-name="OpenShift cluster-node-tuning-operator" \

Makefile

@@ -99,8 +99,10 @@ pkg/generated: $(API_TYPES)
 $(GOBINDATA_BIN):
 	$(GO) build -o $(GOBINDATA_BIN) ./vendor/github.com/kevinburke/go-bindata/go-bindata
 
+#	for d in core basic reboots reboots/sno; do \
+
 test-e2e:
-	for d in core basic reboots reboots/sno; do \
+	for d in core basic ; do \
 	  KUBERNETES_CONFIG="$(KUBECONFIG)" $(GO) test -v -timeout 40m ./test/e2e/$$d -ginkgo.v -ginkgo.no-color -ginkgo.fail-fast || exit; \
 	done
 

assets/performanceprofile/configs/ocp-tuned-one-shot.service

@@ -28,6 +28,7 @@ ExecStart=/usr/bin/podman run \
     --log-driver=none \
     --volume /var/lib/kubelet:/var/lib/kubelet:rslave,ro \
     --volume /var/lib/ocp-tuned:/var/lib/ocp-tuned:rslave \
+    --volume /var/lib/tuned:/var/lib/tuned:rslave \
     --volume /etc/modprobe.d:/etc/modprobe.d:rslave \
     --volume /etc/sysconfig:/etc/sysconfig:rslave \
     --volume /etc/sysctl.d:/etc/sysctl.d:rslave,ro \
@@ -36,7 +37,7 @@ ExecStart=/usr/bin/podman run \
     --volume /run/tuned:/run/tuned:rslave \
     --volume /run/systemd:/run/systemd:rslave \
     --volume /sys:/sys:rslave \
-    --entrypoint '["/usr/bin/cluster-node-tuning-operator","openshift-tuned","--in-cluster=false","--one-shot=true","-v=1"]' \
+    --entrypoint '["/usr/bin/cluster-node-tuning-operator","ocp-tuned","--in-cluster=false","--one-shot=true","-v=1"]' \
     $NTO_IMAGE
 Environment=PODMAN_SYSTEMD_UNIT=%n
 EnvironmentFile=-/var/lib/ocp-tuned/image.env

assets/tuned/manifests/ds-tuned.yaml

@@ -23,7 +23,7 @@ spec:
     spec:
       serviceAccountName: tuned
       containers:
-      - command: ["/usr/bin/cluster-node-tuning-operator","openshift-tuned","--in-cluster","-v=0"]
+      - command: ["/usr/bin/cluster-node-tuning-operator","ocp-tuned","--in-cluster","-v=0"]
         resources:
           requests:
             cpu: 10m
@@ -33,6 +33,7 @@ spec:
         name: tuned
         securityContext:
           privileged: true
+          readOnlyRootFilesystem: true
         terminationMessagePath: /dev/termination-log
         terminationMessagePolicy: FallbackToLogsOnError
         volumeMounts:

hack/dockerfile_install_support.sh

@@ -3,6 +3,13 @@
 set -euo pipefail
 set -o xtrace
 
+INSTALL_PKGS="nmap-ncat procps-ng pciutils"
+
+# TuneD pre-installation steps
+cp -r /root/assets/bin /usr/local/bin
+mkdir -p /etc/grub.d/ /boot /run/ocp-tuned
+chown 499:499 /run/ocp-tuned
+
 source /etc/os-release
 if [[ "${ID}" == "centos" ]]; then
 
@@ -13,19 +20,12 @@ if [[ "${ID}" == "centos" ]]; then
   LC_COLLATE=C cat ../patches/*.diff | patch -Np1
   dnf build-dep tuned.spec -y
   make rpm PYTHON=/usr/bin/python3
-  rm -rf /root/rpmbuild/RPMS/noarch/{tuned-gtk*,tuned-utils*,tuned-profiles-compat*};
+  rm -rf /root/rpmbuild/RPMS/noarch/{tuned-gtk*,tuned-utils*,tuned-profiles-compat*}
   dnf --setopt=protected_packages= history -y undo 0  # Remove builddep
 
-  INSTALL_PKGS="nmap-ncat procps-ng pciutils"
-  cp -r /root/assets/bin /usr/local/bin
   cp -r /root/rpmbuild/RPMS/noarch /root/rpms
-  mkdir -p /etc/grub.d/ /boot /var/lib/ocp-tuned
   dnf install --setopt=tsflags=nodocs -y ${INSTALL_PKGS}
-  rpm -V ${INSTALL_PKGS}
   dnf --setopt=tsflags=nodocs -y install /root/rpms/*.rpm
-  rm -rf /etc/tuned/recommend.d
-  echo auto > /etc/tuned/profile_mode
-  sed -Ei 's|^#?\s*enable_unix_socket\s*=.*$|enable_unix_socket = 1|;s|^#?\s*rollback\s*=.*$|rollback = not_on_exit|;s|^#?\s*profile_dirs\s*=.*$|profile_dirs = /usr/lib/tuned/profiles,/usr/lib/tuned,/var/lib/ocp-tuned/profiles|' /etc/tuned/tuned-main.conf;
 
   # Clean up build tools to remove image footprint
   dnf remove --setopt=protected_packages= -y ${BUILD_INSTALL_PKGS}
@@ -38,14 +38,18 @@ else
      tuned tuned-profiles-atomic tuned-profiles-cpu-partitioning tuned-profiles-mssql tuned-profiles-nfv tuned-profiles-nfv-guest \
      tuned-profiles-nfv-host tuned-profiles-openshift tuned-profiles-oracle tuned-profiles-postgresql tuned-profiles-realtime \
      tuned-profiles-sap tuned-profiles-sap-hana tuned-profiles-spectrumscale \
-     nmap-ncat procps-ng pciutils"
-  cp -r /root/assets/bin /usr/local/bin
-  mkdir -p /etc/grub.d/ /boot /var/lib/ocp-tuned
+     $INSTALL_PKGS"
   dnf install --setopt=tsflags=nodocs -y ${INSTALL_PKGS}
-  rm -rf /etc/tuned/recommend.d
-  echo auto > /etc/tuned/profile_mode
-  sed -Ei 's|^#?\s*enable_unix_socket\s*=.*$|enable_unix_socket = 1|;s|^#?\s*rollback\s*=.*$|rollback = not_on_exit|;s|^#?\s*profile_dirs\s*=.*$|profile_dirs = /usr/lib/tuned/profiles,/usr/lib/tuned,/var/lib/ocp-tuned/profiles|' /etc/tuned/tuned-main.conf;
 
 fi
 
-touch /etc/sysctl.conf
+# TuneD post-installation steps
+rm -rf /etc/tuned/recommend.d /var/lib/tuned
+echo auto > /etc/tuned/profile_mode
+sed -Ei 's|^#?\s*enable_unix_socket\s*=.*$|enable_unix_socket = 1|;s|^#?\s*rollback\s*=.*$|rollback = not_on_exit|;s|^#?\s*profile_dirs\s*=.*$|profile_dirs = /usr/lib/tuned/profiles,/usr/lib/tuned,/var/lib/ocp-tuned/profiles|' \
+  /etc/tuned/tuned-main.conf
+mv /etc/tuned /etc/tuned.orig
+ln -s /var/lib/ocp-tuned/tuned /etc/tuned
+ln -s /run/ocp-tuned/persist /var/lib/ocp-tuned
+ln -s /host/var/lib/tuned /var/lib/tuned	# do we want more links/redirection as ocp-tuned above?
+#touch /etc/sysctl.conf	# do we still need this?

pkg/apis/tuned/v1/tuned_types.go

@@ -20,7 +20,7 @@ const (
 	TunedClusterOperatorResourceName = "node-tuning"
 
 	// Name of the NTO operand for versioning in ClusterOperator.
-	TunedOperandName = "openshift-tuned"
+	TunedOperandName = "ocp-tuned"
 
 	// TunedBootcmdlineAnnotationKey is a Node-specific annotation denoting kernel command-line parameters
 	// calculated by TuneD for the current profile applied to that Node.

pkg/tuned/controller.go

@@ -76,6 +76,7 @@ const (
 	tunedGracefulExitWait = time.Second * time.Duration(10)
 	ocpTunedHome          = "/var/lib/ocp-tuned"
 	ocpTunedRunDir        = "/run/" + programName
+	ocpTunedPersist       = ocpTunedRunDir + "/persist"
 	ocpTunedProvider      = ocpTunedHome + "/provider"
 	// With the less aggressive rate limiter, retries will happen at 100ms*2^(retry_n-1):
 	// 100ms, 200ms, 400ms, 800ms, 1.6s, 3.2s, 6.4s, 12.8s, 25.6s, 51.2s, 102.4s, 3.4m, 6.8m, 13.7m, 27.3m
@@ -473,36 +474,11 @@ func providerSync(provider string) (bool, error) {
 	return true, providerExtract(provider)
 }
 
-// switchTunedHome changes "native" container's home directory as defined by the
-// Containerfile to the container's home directory on the host itself.
-func switchTunedHome() error {
-	const (
-		ocpTunedHomeHost = "/host" + ocpTunedHome
-	)
-
-	// Create the container's home directory on the host.
-	if err := os.MkdirAll(ocpTunedHomeHost, os.ModePerm); err != nil {
-		return fmt.Errorf("failed to create directory %q: %v", ocpTunedHomeHost, err)
-	}
-
-	// Delete the container's home directory.
-	if err := util.Delete(ocpTunedHome); err != nil {
-		return fmt.Errorf("failed to delete: %q: %v", ocpTunedHome, err)
-	}
-
-	if err := util.Symlink(ocpTunedHomeHost, ocpTunedHome); err != nil {
-		return fmt.Errorf("failed to link %q -> %q: %v", ocpTunedHome, ocpTunedHomeHost, err)
-	}
-
-	err := os.Chdir(ocpTunedHome)
-	if err != nil {
+func prepareOpenShiftTunedDir() error {
+	if err := TunedRsyncEtcToHost(); err != nil {
 		return err
 	}
 
-	return nil
-}
-
-func prepareOpenShiftTunedDir() error {
 	// Create the following directories unless they exist.
 	dirs := []string{
 		tunedRecommendDirHost,
@@ -1279,10 +1255,33 @@ func retryLoop(c *Controller) (err error) {
 }
 
 func RunInCluster(stopCh <-chan struct{}, version string) error {
+	const (
+		// The persistent ocp-tuned TuneD artifacts directory.
+		ocpTunedHomeHost = "/host/var/lib/ocp-tuned"
+		// The persistent tuned directory for files such as ksm-masked coming from cpu-partitioning profile.
+		tunedPersistHomeHost = "/host/var/lib/tuned"
+	)
+
 	klog.Infof("starting in-cluster %s %s", programName, version)
 
-	if err := switchTunedHome(); err != nil {
-		return err
+	dirs := []string{
+		ocpTunedHomeHost,
+		tunedPersistHomeHost,
+	}
+	for _, d := range dirs {
+		if err := os.MkdirAll(d, os.ModePerm); err != nil {
+			return fmt.Errorf("failed to create directory %q: %v", d, err)
+		}
+	}
+
+	links := map[string]string{
+		ocpTunedHomeHost:     ocpTunedPersist,
+		tunedPersistHomeHost: "/var/lib/tuned",
+	}
+	for target, source := range links {
+		if err := util.Symlink(target, source); err != nil {
+			return fmt.Errorf("failed to link %q -> %q: %v", source, target, err)
+		}
 	}
 
 	if err := prepareOpenShiftTunedDir(); err != nil {

pkg/tuned/run.go

@@ -72,6 +72,21 @@ func configDaemonMode() (func(), error) {
 	return restoreF, nil
 }
 
+func TunedRsyncEtcToHost() error {
+	const (
+		source = "/etc/tuned.orig/"
+		target = ocpTunedHome + "/tuned"
+	)
+
+	cmd := exec.Command("rsync", "--delete", "-av", source, target)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("rsync of %q to %q failed: %v\n%s", source, target, err, out)
+	}
+
+	return nil
+}
+
 func TunedRunNoDaemon(timeout time.Duration) error {
 	var (
 		cmd    *exec.Cmd

test/e2e/performanceprofile/functests/0_config/config.go

@@ -230,15 +230,18 @@ func unpauseMCP(ctx context.Context, performanceProfile *performancev2.Performan
 
 	By("Waiting for the MCP to pick the PerformanceProfile's MC")
 	mcps.WaitForProfilePickedUp(performanceMCP.Name, performanceProfile)
+	By("---------0------------")
 
 	// If the profile is already there, it's likely to have been through the updating phase, so we only
 	// wait for updated.
 	if !profileAlreadyExists {
 		By("Waiting for MCP starting to update")
 		mcps.WaitForCondition(performanceMCP.Name, mcv1.MachineConfigPoolUpdating, corev1.ConditionTrue)
 	}
+	By("---------1------------")
 	By("Waiting for MCP being updated")
 	mcps.WaitForCondition(performanceMCP.Name, mcv1.MachineConfigPoolUpdated, corev1.ConditionTrue)
+	By("---------2------------")
 }
 
 func attachProfileToNodePool(ctx context.Context, performanceProfile *performancev2.PerformanceProfile, profileAlreadyExists bool) {

test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-ctrcfg/openshift-bootstrap-master_machineconfig.yaml

@@ -196,6 +196,7 @@ spec:
               --log-driver=none \
               --volume /var/lib/kubelet:/var/lib/kubelet:rslave,ro \
               --volume /var/lib/ocp-tuned:/var/lib/ocp-tuned:rslave \
+              --volume /var/lib/tuned:/var/lib/tuned:rslave \
               --volume /etc/modprobe.d:/etc/modprobe.d:rslave \
               --volume /etc/sysconfig:/etc/sysconfig:rslave \
               --volume /etc/sysctl.d:/etc/sysctl.d:rslave,ro \
@@ -204,7 +205,7 @@ spec:
               --volume /run/tuned:/run/tuned:rslave \
               --volume /run/systemd:/run/systemd:rslave \
               --volume /sys:/sys:rslave \
-              --entrypoint '["/usr/bin/cluster-node-tuning-operator","openshift-tuned","--in-cluster=false","--one-shot=true","-v=1"]' \
+              --entrypoint '["/usr/bin/cluster-node-tuning-operator","ocp-tuned","--in-cluster=false","--one-shot=true","-v=1"]' \
               $NTO_IMAGE
           Environment=PODMAN_SYSTEMD_UNIT=%n
           EnvironmentFile=-/var/lib/ocp-tuned/image.env

test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-ctrcfg/openshift-bootstrap-worker_machineconfig.yaml

@@ -196,6 +196,7 @@ spec:
               --log-driver=none \
               --volume /var/lib/kubelet:/var/lib/kubelet:rslave,ro \
               --volume /var/lib/ocp-tuned:/var/lib/ocp-tuned:rslave \
+              --volume /var/lib/tuned:/var/lib/tuned:rslave \
               --volume /etc/modprobe.d:/etc/modprobe.d:rslave \
               --volume /etc/sysconfig:/etc/sysconfig:rslave \
               --volume /etc/sysctl.d:/etc/sysctl.d:rslave,ro \
@@ -204,7 +205,7 @@ spec:
               --volume /run/tuned:/run/tuned:rslave \
               --volume /run/systemd:/run/systemd:rslave \
               --volume /sys:/sys:rslave \
-              --entrypoint '["/usr/bin/cluster-node-tuning-operator","openshift-tuned","--in-cluster=false","--one-shot=true","-v=1"]' \
+              --entrypoint '["/usr/bin/cluster-node-tuning-operator","ocp-tuned","--in-cluster=false","--one-shot=true","-v=1"]' \
               $NTO_IMAGE
           Environment=PODMAN_SYSTEMD_UNIT=%n
           EnvironmentFile=-/var/lib/ocp-tuned/image.env

test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-mcp/openshift-bootstrap-master_machineconfig.yaml

@@ -196,6 +196,7 @@ spec:
               --log-driver=none \
               --volume /var/lib/kubelet:/var/lib/kubelet:rslave,ro \
               --volume /var/lib/ocp-tuned:/var/lib/ocp-tuned:rslave \
+              --volume /var/lib/tuned:/var/lib/tuned:rslave \
               --volume /etc/modprobe.d:/etc/modprobe.d:rslave \
               --volume /etc/sysconfig:/etc/sysconfig:rslave \
               --volume /etc/sysctl.d:/etc/sysctl.d:rslave,ro \
@@ -204,7 +205,7 @@ spec:
               --volume /run/tuned:/run/tuned:rslave \
               --volume /run/systemd:/run/systemd:rslave \
               --volume /sys:/sys:rslave \
-              --entrypoint '["/usr/bin/cluster-node-tuning-operator","openshift-tuned","--in-cluster=false","--one-shot=true","-v=1"]' \
+              --entrypoint '["/usr/bin/cluster-node-tuning-operator","ocp-tuned","--in-cluster=false","--one-shot=true","-v=1"]' \
               $NTO_IMAGE
           Environment=PODMAN_SYSTEMD_UNIT=%n
           EnvironmentFile=-/var/lib/ocp-tuned/image.env

test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-mcp/openshift-bootstrap-worker_machineconfig.yaml

@@ -196,6 +196,7 @@ spec:
               --log-driver=none \
               --volume /var/lib/kubelet:/var/lib/kubelet:rslave,ro \
               --volume /var/lib/ocp-tuned:/var/lib/ocp-tuned:rslave \
+              --volume /var/lib/tuned:/var/lib/tuned:rslave \
               --volume /etc/modprobe.d:/etc/modprobe.d:rslave \
               --volume /etc/sysconfig:/etc/sysconfig:rslave \
               --volume /etc/sysctl.d:/etc/sysctl.d:rslave,ro \
@@ -204,7 +205,7 @@ spec:
               --volume /run/tuned:/run/tuned:rslave \
               --volume /run/systemd:/run/systemd:rslave \
               --volume /sys:/sys:rslave \
-              --entrypoint '["/usr/bin/cluster-node-tuning-operator","openshift-tuned","--in-cluster=false","--one-shot=true","-v=1"]' \
+              --entrypoint '["/usr/bin/cluster-node-tuning-operator","ocp-tuned","--in-cluster=false","--one-shot=true","-v=1"]' \
               $NTO_IMAGE
           Environment=PODMAN_SYSTEMD_UNIT=%n
           EnvironmentFile=-/var/lib/ocp-tuned/image.env

test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/no-mcp/openshift-bootstrap-master_machineconfig.yaml

@@ -196,6 +196,7 @@ spec:
               --log-driver=none \
               --volume /var/lib/kubelet:/var/lib/kubelet:rslave,ro \
               --volume /var/lib/ocp-tuned:/var/lib/ocp-tuned:rslave \
+              --volume /var/lib/tuned:/var/lib/tuned:rslave \
               --volume /etc/modprobe.d:/etc/modprobe.d:rslave \
               --volume /etc/sysconfig:/etc/sysconfig:rslave \
               --volume /etc/sysctl.d:/etc/sysctl.d:rslave,ro \
@@ -204,7 +205,7 @@ spec:
               --volume /run/tuned:/run/tuned:rslave \
               --volume /run/systemd:/run/systemd:rslave \
               --volume /sys:/sys:rslave \
-              --entrypoint '["/usr/bin/cluster-node-tuning-operator","openshift-tuned","--in-cluster=false","--one-shot=true","-v=1"]' \
+              --entrypoint '["/usr/bin/cluster-node-tuning-operator","ocp-tuned","--in-cluster=false","--one-shot=true","-v=1"]' \
               $NTO_IMAGE
           Environment=PODMAN_SYSTEMD_UNIT=%n
           EnvironmentFile=-/var/lib/ocp-tuned/image.env