Update golang to 1.19.4 and kubevirtci to latest (#156)

Signed-off-by: Alexander Wels <[email protected]>

Signed-off-by: Alexander Wels <[email protected]>

Makefile

@@ -14,13 +14,13 @@
 
 .PHONY: cluster-up cluster-down cluster-sync cluster-clean
 
-KUBEVIRT_PROVIDER?=k8s-1.23
+KUBEVIRT_PROVIDER?=k8s-1.25
 HPP_IMAGE?=hostpath-provisioner
 HPP_CSI_IMAGE?=hostpath-csi-driver
 TAG?=latest
 DOCKER_REPO?=quay.io/kubevirt
 ARTIFACTS_PATH?=_out
-GOLANG_VER?=1.18.6
+GOLANG_VER?=1.19.4
 GOOS?=linux
 GOARCH?=amd64
 BUILDAH_PLATFORM_FLAG?=--platform $(GOOS)/$(GOARCH)

cluster-sync/clean.sh

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-KUBEVIRT_PROVIDER=${KUBEVIRT_PROVIDER:-"k8s-1.23"}
+KUBEVIRT_PROVIDER=${KUBEVIRT_PROVIDER:-"k8s-1.25"}
 
 source ./cluster-up/hack/common.sh
 source ./cluster-up/cluster/${KUBEVIRT_PROVIDER}/provider.sh

cluster-sync/sync.sh

@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-KUBEVIRT_PROVIDER=${KUBEVIRT_PROVIDER:-"k8s-1.23"}
+KUBEVIRT_PROVIDER=${KUBEVIRT_PROVIDER:-"k8s-1.25"}
 KUBEVIRT_NUM_NODES=${KUBEVIRT_NUM_NODES:-1}
 HPP_NAMESPACE=${HPP_NAMESPACE:-"hostpath-provisioner"}
 

cluster-up/cluster/K8S.md

@@ -13,7 +13,7 @@ There's a docker registry available which is exposed at `localhost:5000`.
 The env variable `KUBEVIRT_PROVIDER` tells kubevirtci what cluster version to spin up.
 
 ```bash
-export KUBEVIRT_PROVIDER=k8s-1.21   # choose kubevirtci provider version by subdirectory name
+export KUBEVIRT_PROVIDER=k8s-1.22   # choose kubevirtci provider version by subdirectory name
 ```
 
 ## Bringing the cluster up
@@ -28,8 +28,8 @@ The cluster can be accessed as usual:
 ```bash
 $ cluster/kubectl.sh get nodes
 NAME      STATUS     ROLES          AGE       VERSION
-node01    NotReady   control-plane  31s       v1.21.1
-node02    NotReady   <none>         5s        v1.21.1
+node01    NotReady   control-plane  31s       v1.22.1
+node02    NotReady   <none>         5s        v1.22.1
 ```
 
 Note: for further configuration environment variables please see [cluster-up/hack/common.sh](../hack/common.sh)
@@ -51,6 +51,13 @@ export KUBEVIRT_CGROUPV2=true
 make cluster-up
 ```
 
+## Use slim provider (without pre-pulled images of the optional components such as CDI, CNAO etc)
+
+```bash
+export KUBEVIRT_SLIM=true
+make cluster-up
+```
+
 ## Enabling IPv6 connectivity
 
 In order to be able to reach from the cluster to the host's IPv6 network, IPv6

cluster-up/cluster/K8S_DEV_GUIDE.md

@@ -39,18 +39,18 @@ Upon finishing deployment of a K8s deploy, we will have 3 containers:
 
 The containers are running and look like this:
 ```
-[root@modi01 1.21.0]# docker ps
+[root@modi01 1.22.0]# docker ps
 CONTAINER ID        IMAGE                   COMMAND                  CREATED             STATUS              PORTS                                                                                                                          NAMES
-3589e85efc7d        kubevirtci/k8s-1.21.0   "/bin/bash -c '/vm.s…"   About an hour ago   Up About an hour                                                                                                                                   k8s-1.21.0-node01
-4742dc02add2        registry:2.7.1          "/entrypoint.sh /etc…"   About an hour ago   Up About an hour                                                                                                                                   k8s-1.21.0-registry
-13787e7d4ac9        kubevirtci/k8s-1.21.0   "/bin/bash -c /dnsma…"   About an hour ago   Up About an hour    127.0.0.1:8443->8443/tcp, 0.0.0.0:32794->2201/tcp, 0.0.0.0:32793->5000/tcp, 0.0.0.0:32792->5901/tcp, 0.0.0.0:32791->6443/tcp   k8s-1.21.0-dnsmasq
+3589e85efc7d        kubevirtci/k8s-1.22.0   "/bin/bash -c '/vm.s…"   About an hour ago   Up About an hour                                                                                                                                   k8s-1.22.0-node01
+4742dc02add2        registry:2.7.1          "/entrypoint.sh /etc…"   About an hour ago   Up About an hour                                                                                                                                   k8s-1.22.0-registry
+13787e7d4ac9        kubevirtci/k8s-1.22.0   "/bin/bash -c /dnsma…"   About an hour ago   Up About an hour    127.0.0.1:8443->8443/tcp, 0.0.0.0:32794->2201/tcp, 0.0.0.0:32793->5000/tcp, 0.0.0.0:32792->5901/tcp, 0.0.0.0:32791->6443/tcp   k8s-1.22.0-dnsmasq
 ```
 
 Nodes:
 ```
 [root@modi01 kubevirtci]# oc get nodes
 NAME     STATUS   ROLES             AGE   VERSION
-node01   Ready    control-plane     83m   v1.21.0
+node01   Ready    control-plane     83m   v1.22.0
 ```
 
 # Inner look of a deployed cluster

cluster-up/cluster/ephemeral-provider-common.sh

@@ -4,6 +4,9 @@ set -e
 
 KUBEVIRT_WITH_ETC_IN_MEMORY=${KUBEVIRT_WITH_ETC_IN_MEMORY:-false}
 KUBEVIRT_WITH_ETC_CAPACITY=${KUBEVIRT_WITH_ETC_CAPACITY:-none}
+KUBEVIRT_DNS_HOST_PORT=${KUBEVIRT_DNS_HOST_PORT:-31111}
+
+export KUBEVIRTCI_PODMAN_SOCKET=${KUBEVIRTCI_PODMAN_SOCKET:-"/run/podman/podman.sock"}
 
 if [ -z "${KUBEVIRTCI_TAG}" ] && [ -z "${KUBEVIRTCI_GOCLI_CONTAINER}" ]; then
     >&2 echo "FATAL: either KUBEVIRTCI_TAG or KUBEVIRTCI_GOCLI_CONTAINER must be set"
@@ -14,20 +17,26 @@ if [ -n "${KUBEVIRTCI_TAG}" ] && [ -n "${KUBEVIRTCI_GOCLI_CONTAINER}" ]; then
     >&2 echo "WARNING: KUBEVIRTCI_GOCLI_CONTAINER is set and will take precedence over the also set KUBEVIRTCI_TAG"
 fi
 
+detect_podman_socket() {
+    if curl --unix-socket "${KUBEVIRTCI_PODMAN_SOCKET}" http://d/v3.0.0/libpod/info >/dev/null 2>&1; then
+        echo "${KUBEVIRTCI_PODMAN_SOCKET}"
+    fi
+}
+
 if [ "${KUBEVIRTCI_RUNTIME}" = "podman" ]; then
-    _cri_bin="podman --remote --url=unix://${XDG_RUNTIME_DIR}/podman/podman.sock"
-    _docker_socket="${XDG_RUNTIME_DIR}/podman/podman.sock"
+    _cri_socket=$(detect_podman_socket)
+    _cri_bin="podman --remote --url=unix://$_cri_socket"
 elif [ "${KUBEVIRTCI_RUNTIME}" = "docker" ]; then
     _cri_bin=docker
-    _docker_socket="/var/run/docker.sock"
+    _cri_socket="/var/run/docker.sock"
 else
-    if curl --unix-socket "${XDG_RUNTIME_DIR}/podman/podman.sock" http://d/v3.0.0/libpod/info >/dev/null 2>&1; then
-        _cri_bin="podman --remote --url=unix://${XDG_RUNTIME_DIR}/podman/podman.sock"
-        _docker_socket="${XDG_RUNTIME_DIR}/podman/podman.sock"
+    _cri_socket=$(detect_podman_socket)
+    if [ -n "$_cri_socket" ]; then
+        _cri_bin="podman --remote --url=unix://$_cri_socket"
         >&2 echo "selecting podman as container runtime"
     elif docker ps >/dev/null 2>&1; then
         _cri_bin=docker
-        _docker_socket="/var/run/docker.sock"
+        _cri_socket="/var/run/docker.sock"
         >&2 echo "selecting docker as container runtime"
     else
         >&2 echo "no working container runtime found. Neither docker nor podman seems to work."
@@ -36,7 +45,7 @@ else
 fi
 
 _cli_container="${KUBEVIRTCI_GOCLI_CONTAINER:-quay.io/kubevirtci/gocli:${KUBEVIRTCI_TAG}}"
-_cli="${_cri_bin} run --privileged --net=host --rm ${USE_TTY} -v ${_docker_socket}:/var/run/docker.sock"
+_cli="${_cri_bin} run --privileged --net=host --rm ${USE_TTY} -v ${_cri_socket}:/var/run/docker.sock"
 # gocli will try to mount /lib/modules to make it accessible to dnsmasq in
 # in case it exists
 if [ -d /lib/modules ]; then
@@ -79,6 +88,9 @@ function _registry_volume() {
 function _add_common_params() {
     # shellcheck disable=SC2155
     local params="--nodes ${KUBEVIRT_NUM_NODES} --memory ${KUBEVIRT_MEMORY_SIZE} --cpu 6 --secondary-nics ${KUBEVIRT_NUM_SECONDARY_NICS} --random-ports --background --prefix $provider_prefix ${KUBEVIRT_PROVIDER} ${KUBEVIRT_PROVIDER_EXTRA_ARGS}"
+
+    params=" --dns-port $KUBEVIRT_DNS_HOST_PORT $params"
+
     if [[ $TARGET =~ windows_sysprep.* ]] && [ -n "$WINDOWS_SYSPREP_NFS_DIR" ]; then
         params=" --nfs-data $WINDOWS_SYSPREP_NFS_DIR $params"
     elif [[ $TARGET =~ windows.* ]] && [ -n "$WINDOWS_NFS_DIR" ]; then
@@ -89,6 +101,8 @@ function _add_common_params() {
 
     if [ -n "${KUBEVIRTCI_PROVISION_CHECK}" ]; then
         params=" --container-registry=quay.io --container-suffix=:latest $params"
+    elif [[ ${KUBEVIRT_SLIM} == "true" ]]; then
+        params=" --slim $params"
     fi
 
     if [ $KUBEVIRT_WITH_ETC_IN_MEMORY == "true" ]; then
@@ -102,6 +116,10 @@ function _add_common_params() {
         params=" --enable-istio $params"
     fi
 
+    if [ $KUBEVIRT_PSA == "true" ]; then
+        params=" --enable-psa $params"
+    fi
+
     if [ $KUBEVIRT_DEPLOY_NFS_CSI == "true" ]; then
         params=" --enable-nfs-csi $params"
     fi
@@ -113,14 +131,6 @@ function _add_common_params() {
 
     if [[ $KUBEVIRT_DEPLOY_PROMETHEUS == "true" ]] &&
         [[ $KUBEVIRT_PROVIDER_EXTRA_ARGS != *"--enable-prometheus"* ]]; then
-
-        if [[ ($KUBEVIRT_PROVIDER =~ k8s-1\.1.*) || ($KUBEVIRT_PROVIDER =~ k8s-1.20) ]]; then
-            echo "ERROR: cluster up failed because prometheus is only supported for providers >= k8s-1.21\n"
-            echo "the current provider is $KUBEVIRT_PROVIDER, consider updating to a newer version, or\n"
-            echo "disabling Prometheus using export KUBEVIRT_DEPLOY_PROMETHEUS=false"
-            exit 1
-        fi
-
         params=" --enable-prometheus $params"
 
         if [[ $KUBEVIRT_DEPLOY_PROMETHEUS_ALERTMANAGER == "true" ]] &&
@@ -140,6 +150,15 @@ function _add_common_params() {
     if [ -n "$KUBEVIRT_REALTIME_SCHEDULER" ]; then
         params=" --enable-realtime-scheduler $params"
     fi
+
+    if [ -n "$KUBEVIRT_FIPS" ]; then
+        params=" --enable-fips $params"
+    fi
+
+    if [ -n "$KUBEVIRTCI_PROXY" ]; then
+        params=" --docker-proxy=$KUBEVIRTCI_PROXY $params"
+    fi
+
     echo $params
 }
 

cluster-up/cluster/k8s-1.25/provider.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -e
+
+if [ "${KUBEVIRT_CGROUPV2}" == "true" ]; then
+    export KUBEVIRT_PROVIDER_EXTRA_ARGS="${KUBEVIRT_PROVIDER_EXTRA_ARGS} --kernel-args='systemd.unified_cgroup_hierarchy=1'"
+fi
+
+# shellcheck disable=SC1090
+source "${KUBEVIRTCI_PATH}/cluster/k8s-provider-common.sh"

cluster-up/cluster/k8s-1.26-centos9/provider.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -e
+
+if [ "${KUBEVIRT_CGROUPV2}" == "true" ]; then
+    export KUBEVIRT_PROVIDER_EXTRA_ARGS="${KUBEVIRT_PROVIDER_EXTRA_ARGS} --kernel-args='systemd.unified_cgroup_hierarchy=1'"
+fi
+
+# shellcheck disable=SC1090
+source "${KUBEVIRTCI_PATH}/cluster/k8s-provider-common.sh"

cluster-up/cluster/k8s-1.26/provider.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -e
+
+if [ "${KUBEVIRT_CGROUPV2}" == "true" ]; then
+    export KUBEVIRT_PROVIDER_EXTRA_ARGS="${KUBEVIRT_PROVIDER_EXTRA_ARGS} --kernel-args='systemd.unified_cgroup_hierarchy=1'"
+fi
+
+# shellcheck disable=SC1090
+source "${KUBEVIRTCI_PATH}/cluster/k8s-provider-common.sh"

cluster-up/cluster/k8s-provider-common.sh

@@ -9,32 +9,45 @@ source "${KUBEVIRTCI_PATH}/cluster/ephemeral-provider-common.sh"
 #if UNLIMITEDSWAP is set to true - Kubernetes workloads can use as much swap memory as they request, up to the system limit.
 #otherwise Kubernetes workloads can use as much swap memory as they request, up to the system limit by default
 function configure_swap_memory () {
-  if [ "$KUBEVIRT_SWAP_ON" == "true" ] && [[  ($KUBEVIRT_PROVIDER =~ k8s-1\.1.*) ||  ($KUBEVIRT_PROVIDER =~ k8s-1.20) ||  ($KUBEVIRT_PROVIDER =~ k8s-1.21) ]]; then
-      echo "ERROR: swap is not supported on kubevirtci version < 1.22"
-      exit 1
-
-  elif [ "$KUBEVIRT_SWAP_ON" == "true" ] ;then
-
-    for nodeNum in $(seq -f "%02g" 1 $KUBEVIRT_NUM_NODES); do
-        if [ ! -z $KUBEVIRT_SWAP_SIZE_IN_GB  ]; then
-          $ssh node${nodeNum} -- sudo dd if=/dev/zero of=/swapfile count=$KUBEVIRT_SWAP_SIZE_IN_GB bs=1G
-          $ssh node${nodeNum} -- sudo mkswap /swapfile
-        fi
-
-        $ssh node${nodeNum} -- sudo swapon -a
-
-        if [ ! -z $KUBEVIRT_SWAPPINESS ]; then
-          $ssh node${nodeNum} -- "sudo /bin/su -c \"echo vm.swappiness = $KUBEVIRT_SWAPPINESS >> /etc/sysctl.conf\""
-          $ssh node${nodeNum} -- sudo sysctl vm.swappiness=$KUBEVIRT_SWAPPINESS
-        fi
+    if [ "$KUBEVIRT_SWAP_ON" == "true" ] ;then
+      for nodeNum in $(seq -f "%02g" 1 $KUBEVIRT_NUM_NODES); do
+          if [ ! -z $KUBEVIRT_SWAP_SIZE_IN_GB  ]; then
+            $ssh node${nodeNum} -- sudo dd if=/dev/zero of=/swapfile count=$KUBEVIRT_SWAP_SIZE_IN_GB bs=1G
+            $ssh node${nodeNum} -- sudo mkswap /swapfile
+          fi
+
+          $ssh node${nodeNum} -- sudo swapon -a
+
+          if [ ! -z $KUBEVIRT_SWAPPINESS ]; then
+            $ssh node${nodeNum} -- "sudo /bin/su -c \"echo vm.swappiness = $KUBEVIRT_SWAPPINESS >> /etc/sysctl.conf\""
+            $ssh node${nodeNum} -- sudo sysctl vm.swappiness=$KUBEVIRT_SWAPPINESS
+          fi
+
+          if [ $KUBEVIRT_UNLIMITEDSWAP == "true" ]; then
+            $ssh node${nodeNum} -- "sudo sed -i ':a;N;\$!ba;s/memorySwap: {}/memorySwap:\n  swapBehavior: UnlimitedSwap/g'  /var/lib/kubelet/config.yaml"
+            $ssh node${nodeNum} -- sudo systemctl restart kubelet
+          fi
+      done
+    fi
+}
 
-        if [ $KUBEVIRT_UNLIMITEDSWAP == "true" ]; then
-          $ssh node${nodeNum} -- "sudo sed -i ':a;N;\$!ba;s/memorySwap: {}/memorySwap:\n  swapBehavior: UnlimitedSwap/g'  /var/lib/kubelet/config.yaml"
-          $ssh node${nodeNum} -- sudo systemctl restart kubelet
-        fi
-  done
-fi
+function configure_ksm_module () {
+    if [ "$KUBEVIRT_KSM_ON" == "true" ] ;then
+      for nodeNum in $(seq -f "%02g" 1 $KUBEVIRT_NUM_NODES); do
+        $ssh node${nodeNum} -- "echo 1 | sudo tee /sys/kernel/mm/ksm/run >/dev/null"
+          if [ ! -z $KUBEVIRT_KSM_SLEEP_BETWEEN_SCANS_MS ]; then
+            $ssh node${nodeNum} -- "echo ${KUBEVIRT_KSM_SLEEP_BETWEEN_SCANS_MS} | sudo tee /sys/kernel/mm/ksm/sleep_millisecs >/dev/null "
+          fi
+          if [ ! -z $KUBEVIRT_KSM_PAGES_TO_SCAN ]; then
+            $ssh node${nodeNum} -- "echo ${KUBEVIRT_KSM_PAGES_TO_SCAN} | sudo tee /sys/kernel/mm/ksm/pages_to_scan >/dev/null "
+          fi
+      done
+    fi
+}
 
+function configure_memory_overcommitment_behavior () {
+  configure_swap_memory
+  configure_ksm_module
 }
 
 function deploy_cnao() {
@@ -62,11 +75,7 @@ function wait_for_cnao_ready() {
 }
 
 function deploy_istio() {
-    if [ "$KUBEVIRT_DEPLOY_ISTIO" == "true" ] && [[ $KUBEVIRT_PROVIDER =~ k8s-1\.1.* ]]; then
-        echo "ERROR: Istio is not supported on kubevirtci version < 1.20"
-        exit 1
-
-    elif [ "$KUBEVIRT_DEPLOY_ISTIO" == "true" ]; then
+    if [ "$KUBEVIRT_DEPLOY_ISTIO" == "true" ]; then
         if [ "$KUBEVIRT_WITH_CNAO" == "true" ]; then
             $kubectl create -f /opt/istio/istio-operator-with-cnao.cr.yaml
         else
@@ -96,6 +105,16 @@ function wait_for_istio_ready() {
     fi
 }
 
+# copy_istio_cni_conf_files copy the generated Istio CNI net conf file
+# (at '/etc/cni/multus/net.d/') to where Multus expect CNI net conf files ('/etc/cni/net.d/')
+function copy_istio_cni_conf_files() {
+    if [ "$KUBEVIRT_DEPLOY_ISTIO" == "true" ] && [ "$KUBEVIRT_WITH_CNAO" == "true" ]; then
+        for nodeNum in $(seq -f "%02g" 1 $KUBEVIRT_NUM_NODES); do
+            $ssh node${nodeNum} -- sudo cp -uv /etc/cni/multus/net.d/*istio*.conf /etc/cni/net.d/
+        done
+    fi
+}
+
 function deploy_cdi() {
     if [ "$KUBEVIRT_DEPLOY_CDI" == "true" ]; then
         $kubectl create -f /opt/cdi-*-operator.yaml
@@ -121,23 +140,23 @@ function up() {
     fi
     eval ${_cli:?} run $params
 
-    # Copy k8s config and kubectl
+    ${_cli} scp --prefix $provider_prefix /etc/kubernetes/admin.conf - >${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/.kubeconfig
+
+    # Set server and disable tls check
+    export KUBECONFIG=${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/.kubeconfig
+    kubectl config set-cluster kubernetes --server="https://$(_main_ip):$(_port k8s)"
+    kubectl config set-cluster kubernetes --insecure-skip-tls-verify=true
+
     # Workaround https://github.com/containers/conmon/issues/315 by not dumping the file to stdout for the time being
     if [[ ${_cri_bin} = podman* ]]; then
-        ${_cli} scp --prefix ${provider_prefix:?} /usr/bin/kubectl /kubevirtci_config/.kubectl
-        ${_cli} scp --prefix $provider_prefix /etc/kubernetes/admin.conf /kubevirtci_config/.kubeconfig
+        k8s_version=$(kubectl get node node01 --no-headers -o=custom-columns=VERSION:.status.nodeInfo.kubeletVersion)
+        curl -Ls "https://dl.k8s.io/release/${k8s_version}/bin/linux/amd64/kubectl" -o ${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/.kubectl
     else
         ${_cli} scp --prefix ${provider_prefix:?} /usr/bin/kubectl - >${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/.kubectl
-        ${_cli} scp --prefix $provider_prefix /etc/kubernetes/admin.conf - >${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/.kubeconfig
     fi
 
     chmod u+x ${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/.kubectl
 
-    # Set server and disable tls check
-    export KUBECONFIG=${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/.kubeconfig
-    ${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/.kubectl config set-cluster kubernetes --server="https://$(_main_ip):$(_port k8s)"
-    ${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/.kubectl config set-cluster kubernetes --insecure-skip-tls-verify=true
-
     # Make sure that local config is correct
     prepare_config
     ssh="${_cli} --prefix $provider_prefix ssh"
@@ -152,7 +171,7 @@ function up() {
     fi
     $kubectl label node -l $label node-role.kubernetes.io/worker=''
 
-    configure_swap_memory
+    configure_memory_overcommitment_behavior
 
     deploy_cnao
     deploy_istio
@@ -163,4 +182,8 @@ function up() {
         sleep 5
     done
 
+    # FIXME: remove 'copy_istio_cni_conf_files()' as soon as [1] and [2] are resolved
+    # [1] https://github.com/kubevirt/kubevirtci/issues/906
+    # [2] https://github.com/k8snetworkplumbingwg/multus-cni/issues/982
+    copy_istio_cni_conf_files
 }