From def59d9d466fb80f6880137757a681bcbe419da0 Mon Sep 17 00:00:00 2001 From: Oren Cohen Date: Sun, 26 Mar 2023 15:27:23 +0300 Subject: [PATCH] Switch to HPP-CSI for CI tests Up until now, we were using the legacy way of deploying hostpath-provisioner. We can start consuming HPP-CSI instead, which does not require setting up MachineConfigs and can take advantage of the underlying storage backend provided by the cloud provider. Signed-off-by: Oren Cohen --- hack/hpp/00_hpp_mc.yaml | 84 ------------------- hack/hpp/10_hpp_cr.yaml | 10 --- hack/hpp/10_hpp_pool_cr.yaml | 12 +-- hack/hpp/20_hpp_pool_sc.yaml | 20 +++++ hack/hpp/20_hpp_sc.yaml | 8 -- hack/hpp/30_hpp_csi_sc.yaml | 8 -- hack/hpp/30_hpp_pool_sc.yaml | 10 --- hack/hpp/configure_hpp_ha.sh | 53 ++++++++++++ hack/hpp/configure_hpp_legacy.sh | 61 -------------- ...igure_hpp_pool.sh => configure_hpp_sno.sh} | 27 +++--- hack/hpp/deploy_hpp.sh | 11 +-- 11 files changed, 99 insertions(+), 205 deletions(-) delete mode 100644 hack/hpp/00_hpp_mc.yaml delete mode 100644 hack/hpp/10_hpp_cr.yaml create mode 100644 hack/hpp/20_hpp_pool_sc.yaml delete mode 100644 hack/hpp/20_hpp_sc.yaml delete mode 100644 hack/hpp/30_hpp_csi_sc.yaml delete mode 100644 hack/hpp/30_hpp_pool_sc.yaml create mode 100755 hack/hpp/configure_hpp_ha.sh delete mode 100755 hack/hpp/configure_hpp_legacy.sh rename hack/hpp/{configure_hpp_pool.sh => configure_hpp_sno.sh} (50%) diff --git a/hack/hpp/00_hpp_mc.yaml b/hack/hpp/00_hpp_mc.yaml deleted file mode 100644 index ac62cf6deb..0000000000 --- a/hack/hpp/00_hpp_mc.yaml +++ /dev/null @@ -1,84 +0,0 @@ ---- -# -# Create and mount a dedicated partition for PersistentVolumes -# provisioned by kubevirt-hostpath-provisioner. -# -# See documentation: -# - https://github.com/kubevirt/hostpath-provisioner-operator/blob/master/contrib/machineconfig-selinux-hpp.yaml -# - -kind: MachineConfig -apiVersion: machineconfiguration.openshift.io/v1 -metadata: - name: 100-worker-hostpath-provisioner - labels: - machineconfiguration.openshift.io/role: worker -spec: - config: - ignition: - version: 3.2.0 - systemd: - units: - - contents: | - [Unit] - Description=Create directory for hostpath provisioner - - [Service] - Type=oneshot - RemainAfterExit=yes - ExecStart=mkdir -p /var/hpvolumes - - [Install] - WantedBy=local-fs.target - enabled: true - name: hostpath-provisioner-mkdir.service - - contents: | - # VirtualMachines have a dedicated disk for HPP while BM servers - # use their rootdisk for HPP - [Unit] - Description=Create filesystem for hostpath provisioner - ConditionPathExists=/dev/vdb - ConditionVirtualization=vm - - [Service] - Type=oneshot - RemainAfterExit=yes - ExecStart=/bin/bash -c 'lsblk --noheadings --output=FSTYPE /dev/vdb | grep -Fw ext4 || /usr/sbin/mkfs.ext4 -q -L hpvolumes /dev/vdb' - - [Install] - WantedBy=local-fs.target - enabled: true - name: hostpath-provisioner-mkfs.service - - contents: | - [Unit] - Description=Mount filesystem for hostpath provisioner - ConditionPathExists=/dev/disk/by-label/hpvolumes - ConditionVirtualization=vm - Requires=hostpath-provisioner-mkdir.service hostpath-provisioner-mkfs.service - After=hostpath-provisioner-mkdir.service hostpath-provisioner-mkfs.service - Before=hostpath-provisioner-selinux.service - - [Mount] - What=/dev/disk/by-label/hpvolumes - Where=/var/hpvolumes - - [Install] - WantedBy=local-fs.target - enabled: true - name: var-hpvolumes.mount - - contents: | - [Unit] - Description=Set SELinux chcon for hostpath provisioner - Requires=local-fs.target hostpath-provisioner-mkdir.service - After=local-fs.target hostpath-provisioner-mkdir.service var-hpvolumes.mount - Before=kubelet.service - - [Service] - Type=oneshot - RemainAfterExit=yes - ExecStart=/usr/bin/chcon -Rt container_file_t /var/hpvolumes - - [Install] - WantedBy=local-fs.target - enabled: true - name: hostpath-provisioner-selinux.service diff --git a/hack/hpp/10_hpp_cr.yaml b/hack/hpp/10_hpp_cr.yaml deleted file mode 100644 index eb66cd3cc0..0000000000 --- a/hack/hpp/10_hpp_cr.yaml +++ /dev/null @@ -1,10 +0,0 @@ ---- -kind: HostPathProvisioner -apiVersion: hostpathprovisioner.kubevirt.io/v1beta1 -metadata: - name: hostpath-provisioner -spec: - imagePullPolicy: IfNotPresent - pathConfig: - path: "/var/hpvolumes" - useNamingPrefix: false diff --git a/hack/hpp/10_hpp_pool_cr.yaml b/hack/hpp/10_hpp_pool_cr.yaml index 58623e9f58..f0a2928644 100644 --- a/hack/hpp/10_hpp_pool_cr.yaml +++ b/hack/hpp/10_hpp_pool_cr.yaml @@ -1,21 +1,23 @@ --- -kind: HostPathProvisioner apiVersion: hostpathprovisioner.kubevirt.io/v1beta1 +kind: HostPathProvisioner metadata: name: hostpath-provisioner spec: imagePullPolicy: IfNotPresent storagePools: - - name: local + - name: hpp-csi-local-basic + path: "/var/hpp-csi-local-basic" + - name: hpp-csi-pvc-block pvcTemplate: - storageClassName: local-block-hpp volumeMode: Block + storageClassName: local-block-hpp # Replaced by configure_hpp_*.sh accessModes: - ReadWriteOnce resources: requests: - storage: 5Gi - path: "/var/hpvolumes" + storage: 70Gi + path: "/var/hpp-csi-pvc-block" workload: nodeSelector: kubernetes.io/os: linux diff --git a/hack/hpp/20_hpp_pool_sc.yaml b/hack/hpp/20_hpp_pool_sc.yaml new file mode 100644 index 0000000000..b200cdfb2c --- /dev/null +++ b/hack/hpp/20_hpp_pool_sc.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: hostpath-provisioner +provisioner: kubevirt.io.hostpath-provisioner +reclaimPolicy: Delete +volumeBindingMode: WaitForFirstConsumer +parameters: + storagePool: hpp-csi-local-basic +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: hostpath-csi-pvc-block +provisioner: kubevirt.io.hostpath-provisioner +reclaimPolicy: Delete +volumeBindingMode: WaitForFirstConsumer +parameters: + storagePool: hpp-csi-pvc-block diff --git a/hack/hpp/20_hpp_sc.yaml b/hack/hpp/20_hpp_sc.yaml deleted file mode 100644 index df0f0247bd..0000000000 --- a/hack/hpp/20_hpp_sc.yaml +++ /dev/null @@ -1,8 +0,0 @@ ---- -kind: StorageClass -apiVersion: storage.k8s.io/v1 -metadata: - name: hostpath-provisioner -provisioner: kubevirt.io/hostpath-provisioner -reclaimPolicy: Delete -volumeBindingMode: WaitForFirstConsumer diff --git a/hack/hpp/30_hpp_csi_sc.yaml b/hack/hpp/30_hpp_csi_sc.yaml deleted file mode 100644 index 1e0c3aaa58..0000000000 --- a/hack/hpp/30_hpp_csi_sc.yaml +++ /dev/null @@ -1,8 +0,0 @@ ---- -kind: StorageClass -apiVersion: storage.k8s.io/v1 -metadata: - name: hostpath-csi -provisioner: kubevirt.io.hostpath-provisioner -reclaimPolicy: Delete -volumeBindingMode: WaitForFirstConsumer diff --git a/hack/hpp/30_hpp_pool_sc.yaml b/hack/hpp/30_hpp_pool_sc.yaml deleted file mode 100644 index 0f76f2d5c0..0000000000 --- a/hack/hpp/30_hpp_pool_sc.yaml +++ /dev/null @@ -1,10 +0,0 @@ ---- -kind: StorageClass -apiVersion: storage.k8s.io/v1 -metadata: - name: hostpath-csi -provisioner: kubevirt.io.hostpath-provisioner -reclaimPolicy: Delete -volumeBindingMode: Immediate # must be set to WFFC for non SNO clusters -parameters: - storagePool: local diff --git a/hack/hpp/configure_hpp_ha.sh b/hack/hpp/configure_hpp_ha.sh new file mode 100755 index 0000000000..6294b7124d --- /dev/null +++ b/hack/hpp/configure_hpp_ha.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +# +# Configures HPP-CSI on an OCP cluster using the StoragePool feature. +# +# Deploys two storage classes +# * hostpath-csi-basic - uses root filesystem of the worker nodes +# * hostpath-csi-pvc-block - utilize another storage class as a backend +# +# Requires HPP operator to be deployed on the cluster. + +set -ex + +readonly SCRIPT_DIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")") +HPP_VOLUME_SIZE=${HPP_VOLUME_SIZE:-${VOLUME_SIZE:-70}}Gi + + +CLUSTER_PLATFORM=$( + oc get infrastructure cluster \ + --output=jsonpath='{$.status.platform}' +) + +case "${CLUSTER_PLATFORM}" in + Azure) + HPP_BACKEND_STORAGE_CLASS=managed-csi + ;; + AWS) + HPP_BACKEND_STORAGE_CLASS=gp3-csi + ;; + GCP) + HPP_BACKEND_STORAGE_CLASS=standard-csi + ;; + BareMetal) + HPP_BACKEND_STORAGE_CLASS=ocs-storagecluster-ceph-rbd + ;; + None) + # UPI Installation + HPP_BACKEND_STORAGE_CLASS=${HPP_BACKEND_STORAGE_CLASS:-ocs-storagecluster-ceph-rbd} + ;; + *) + echo "[ERROR] Unsupported cluster platform: [${CLUSTER_PLATFORM}]" >&2 + exit 1 + ;; +esac + + +# Create HPP CustomResource using the StoragePool feature +sed "${SCRIPT_DIR}/10_hpp_pool_cr.yaml" \ + -e "s|^\( \+storageClassName\): .*|\1: ${HPP_BACKEND_STORAGE_CLASS}|g" \ +| oc create --filename=- + +# Create HPP StorageClass using the StoragePool feature +oc create --filename="${SCRIPT_DIR}/20_hpp_pool_sc.yaml" diff --git a/hack/hpp/configure_hpp_legacy.sh b/hack/hpp/configure_hpp_legacy.sh deleted file mode 100755 index b2f03df554..0000000000 --- a/hack/hpp/configure_hpp_legacy.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash - -# -# Configures HPP on an OCP cluster using the legacy. -# -# Requires HPP operator to be deployed on the cluster. It is usually deployed -# as part of CNV by the HCO operator. -# -# See documentation: -# - https://github.com/kubevirt/hostpath-provisioner-operator/blob/master/README.md -# - -set -ex - -readonly SCRIPT_DIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")") -readonly HCO_NAMESPACE="kubevirt-hyperconverged" - - -echo_debug() -{ - echo "$@" >&2 -} - -# Wait until master and worker MCP are Updated -# or timeout after 90min. -wait_mcp_for_updated() -{ - local mcp_updated="false" - - sleep 30 - - for i in {1..60} - do - echo_debug "Attempt ${i}/60" - sleep 30 - if oc wait mcp --all --for condition=updated --timeout=1m; then - echo_debug "MCP is Updated" - mcp_updated="true" - break - fi - done - - if [[ "$mcp_updated" == "false" ]]; then - echo_debug "Error: MCP didn't get Updated!!" - exit 1 - fi -} - -# Create and mount a dedicated partition for PersistentVolumes provisioned by HPP -# => https://github.com/kubevirt/hostpath-provisioner-operator/blob/master/contrib/machineconfig-selinux-hpp.yaml -oc create --filename="${SCRIPT_DIR}/00_hpp_mc.yaml" -n ${HCO_NAMESPACE} || true # Don't fail if resource already exists -wait_mcp_for_updated - -# Create HPP CustomResource and StorageClass -oc create --filename="${SCRIPT_DIR}/10_hpp_cr.yaml" -n ${HCO_NAMESPACE} -oc create --filename="${SCRIPT_DIR}/20_hpp_sc.yaml" -oc create --filename="${SCRIPT_DIR}/30_hpp_csi_sc.yaml" - -# Set HPP as default StorageClass for the cluster -oc annotate storageclasses --all storageclass.kubernetes.io/is-default-class- -oc annotate storageclass hostpath-provisioner storageclass.kubernetes.io/is-default-class='true' diff --git a/hack/hpp/configure_hpp_pool.sh b/hack/hpp/configure_hpp_sno.sh similarity index 50% rename from hack/hpp/configure_hpp_pool.sh rename to hack/hpp/configure_hpp_sno.sh index cfdf5fb3f2..ef865a6a9e 100755 --- a/hack/hpp/configure_hpp_pool.sh +++ b/hack/hpp/configure_hpp_sno.sh @@ -1,18 +1,18 @@ #!/bin/bash # -# Configures HPP on an OCP cluster using the StoragePool feature. +# Configures HPP-CSI on a SNO cluster using the StoragePool feature. # -# Requires HPP operator to be deployed on the cluster. It is usually deployed -# as part of CNV by the HCO operator. -# -# See documentation: -# - https://docs.google.com/document/d/1v_kPxJKhy3WYVOIlTRviEpJbigqraE8Hte7BCKJNVBM +# Deploys two storage classes +# * hostpath-csi-basic - uses root filesystem of the worker nodes +# * hostpath-csi-pvc-block - utilize another storage class as a backend # +# Requires HPP operator to be deployed on the cluster. set -ex readonly SCRIPT_DIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")") +VOLUME_BINDING_MODE="WaitForFirstConsumer" CLUSTER_PLATFORM=$( oc get infrastructure cluster \ @@ -28,6 +28,10 @@ case "${CLUSTER_PLATFORM}" in HPP_BACKEND_STORAGE_CLASS=gp3-csi HPP_VOLUME_SIZE=128Gi ;; + GCP) + HPP_BACKEND_STORAGE_CLASS=standard-csi + HPP_VOLUME_SIZE=128Gi + ;; BareMetal|None) HPP_BACKEND_STORAGE_CLASS=local-block-hpp HPP_VOLUME_SIZE=5Gi @@ -38,16 +42,11 @@ case "${CLUSTER_PLATFORM}" in ;; esac -# Create HPP CustomResource and StorageClass using the StoragePool feature +# Create HPP CustomResource using the StoragePool feature sed "${SCRIPT_DIR}/10_hpp_pool_cr.yaml" \ -e "s|^\( \+storage\): .*|\1: ${HPP_VOLUME_SIZE}|g" \ -e "s|^\( \+storageClassName\): .*|\1: ${HPP_BACKEND_STORAGE_CLASS}|g" \ | oc create --filename=- -oc create --filename="${SCRIPT_DIR}/30_hpp_pool_sc.yaml" - -# Set HPP-CSI as default StorageClass for the cluster -oc annotate storageclasses --all storageclass.kubernetes.io/is-default-class- -oc annotate storageclass hostpath-csi storageclass.kubernetes.io/is-default-class='true' -# Wait for HPP to be ready -oc wait hostpathprovisioner hostpath-provisioner --for=condition='Available' --timeout='10m' +# Create HPP StorageClass using the StoragePool feature +oc create --filename="${SCRIPT_DIR}/20_hpp_pool_sc.yaml" diff --git a/hack/hpp/deploy_hpp.sh b/hack/hpp/deploy_hpp.sh index 0b0aca6986..dda66e527d 100755 --- a/hack/hpp/deploy_hpp.sh +++ b/hack/hpp/deploy_hpp.sh @@ -1,9 +1,7 @@ #!/bin/bash # -# Configures HPP on an OCP cluster: -# - on regular clusters, HPP is deployed the legacy way -# - on SNO clusters, HPP is deployed using the StoragePool feature +# Configures HPP on an OCP cluster using the StoragePool feature # set -ex @@ -22,8 +20,11 @@ CLUSTER_VERSION=$( if [[ "$CLUSTER_VERSION" != *"okd"* ]]; then # skipping configuring HPP in case of an OKD cluster if [[ "${CLUSTER_TOPOLOGY}" != 'SingleReplica' ]]; then - "${SCRIPT_DIR}"/configure_hpp_legacy.sh + "${SCRIPT_DIR}"/configure_hpp_ha.sh else - "${SCRIPT_DIR}"/configure_hpp_pool.sh + "${SCRIPT_DIR}"/configure_hpp_sno.sh fi + +# Wait for HPP to be ready +oc wait hostpathprovisioner hostpath-provisioner --for=condition='Available' --timeout='10m' fi