From 5cc9764e151139c8cbab1491704504a9baa70091 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Mon, 4 Nov 2019 17:51:32 +0100 Subject: [PATCH 1/4] deploy: fix number of expected pods https://github.com/kubernetes-csi/csi-driver-host-path/pull/90 added external-resizer without increasing the number of running pods that the script waits for, so it may have proceeded too early. --- deploy/util/deploy-hostpath.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deploy/util/deploy-hostpath.sh b/deploy/util/deploy-hostpath.sh index df20799e8..478d42083 100755 --- a/deploy/util/deploy-hostpath.sh +++ b/deploy/util/deploy-hostpath.sh @@ -150,8 +150,9 @@ done # about the deployment here, otherwise we wouldn't know what to wait # for: the expectation is that we run attacher, provisioner, # snapshotter, resizer, socat and hostpath plugin in the default namespace. +expected_running_pods=6 cnt=0 -while [ $(kubectl get pods 2>/dev/null | grep '^csi-hostpath.* Running ' | wc -l) -lt 5 ] || ! kubectl describe volumesnapshotclasses.snapshot.storage.k8s.io 2>/dev/null >/dev/null; do +while [ $(kubectl get pods 2>/dev/null | grep '^csi-hostpath.* Running ' | wc -l) -lt $expected_running_pods ] || ! kubectl describe volumesnapshotclasses.snapshot.storage.k8s.io 2>/dev/null >/dev/null; do if [ $cnt -gt 30 ]; then echo "Running pods:" kubectl describe pods From dbea8ad5fc070869b519ce4a300ee8b3f027e427 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Mon, 4 Nov 2019 18:04:54 +0100 Subject: [PATCH 2/4] deploy: additional debug output about snapshotter The canary external-snapshotter was running on 1.14/15/16, but didn't create the volumesnapshotclasses.snapshot.storage.k8s.io CRD, so the `kubectl describe volumesnapshotclasses.snapshot.storage.k8s.io` check failed without that being visible in the debug output. --- deploy/util/deploy-hostpath.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deploy/util/deploy-hostpath.sh b/deploy/util/deploy-hostpath.sh index 478d42083..4e0a25dc6 100755 --- a/deploy/util/deploy-hostpath.sh +++ b/deploy/util/deploy-hostpath.sh @@ -154,8 +154,9 @@ expected_running_pods=6 cnt=0 while [ $(kubectl get pods 2>/dev/null | grep '^csi-hostpath.* Running ' | wc -l) -lt $expected_running_pods ] || ! kubectl describe volumesnapshotclasses.snapshot.storage.k8s.io 2>/dev/null >/dev/null; do if [ $cnt -gt 30 ]; then - echo "Running pods:" + echo "$(kubectl get pods 2>/dev/null | grep '^csi-hostpath.* Running ' | wc -l) running pods:" kubectl describe pods + (set -x; kubectl describe volumesnapshotclasses.snapshot.storage.k8s.io) || true echo >&2 "ERROR: hostpath deployment not ready after over 5min" exit 1 From 33b0b2f3a2e06f91916a7807fe38928ec0e85e51 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Mon, 4 Nov 2019 18:08:05 +0100 Subject: [PATCH 3/4] deploy: fix canary jobs on 1.14/15/16 The Prow canary jobs correctly showed that the upcoming external-snapshotter 2.0.0 is not going to be compatible with the existing deployments because of API changes. But now that this has been discovered, it's not useful anymore to keep those jobs in the failing state. We need them to run again until some other sidecar breaks its API. This can be achieved by documenting that external-snapshotter cannot be updated to its canary version and then just keeping it at the version specified for the deployment. The same can be done for other sidecars if the need arises. --- deploy/kubernetes-1.14/README.md | 4 ++++ deploy/kubernetes-1.15/README.md | 4 ++++ deploy/kubernetes-1.16/README.md | 4 ++++ deploy/util/deploy-hostpath.sh | 19 ++++++++++++++++++- 4 files changed, 30 insertions(+), 1 deletion(-) diff --git a/deploy/kubernetes-1.14/README.md b/deploy/kubernetes-1.14/README.md index 40face28b..08052770d 100644 --- a/deploy/kubernetes-1.14/README.md +++ b/deploy/kubernetes-1.14/README.md @@ -6,3 +6,7 @@ introduced in Kubernetes 1.14, so features depending on those (like topology) will not work on Kubernetes 1.13. But because this example deployment does not enable those features, it can run on Kubernetes 1.13. + +The following canary images are known to be incompatible with this +deployment: +- csi-snapshotter (canary uses VolumeSnapshot v1beta) diff --git a/deploy/kubernetes-1.15/README.md b/deploy/kubernetes-1.15/README.md index c5b4c0f20..a370d55c6 100644 --- a/deploy/kubernetes-1.15/README.md +++ b/deploy/kubernetes-1.15/README.md @@ -5,3 +5,7 @@ The sidecars depend on 1.15 API changes for migration and resizing, and 1.14 API changes for CSIDriver and CSINode. However the hostpath driver doesn't use those features, so this deployment can work on older Kubernetes versions. + +The following canary images are known to be incompatible with this +deployment: +- csi-snapshotter (canary uses VolumeSnapshot v1beta) diff --git a/deploy/kubernetes-1.16/README.md b/deploy/kubernetes-1.16/README.md index c1d3c1462..4ac48fb8b 100644 --- a/deploy/kubernetes-1.16/README.md +++ b/deploy/kubernetes-1.16/README.md @@ -1,3 +1,7 @@ The deployment for Kubernetes 1.16 enables ephemeral inline volumes via its CSIDriverInfo and thus is incompatible with Kubernetes < 1.16 because the `VolumeLifecycleModes` field is rejected by those release. + +The following canary images are known to be incompatible with this +deployment: +- csi-snapshotter (canary uses VolumeSnapshot v1beta) diff --git a/deploy/util/deploy-hostpath.sh b/deploy/util/deploy-hostpath.sh index 4e0a25dc6..3c48e90ea 100755 --- a/deploy/util/deploy-hostpath.sh +++ b/deploy/util/deploy-hostpath.sh @@ -32,6 +32,15 @@ BASE_DIR=$(dirname "$0") # - IMAGE_TAG # These are used as fallback when the more specific variables are unset or empty. # +# IMAGE_TAG=canary is ignored for images that are blacklisted in the deployment's +# top-level README.md. This is meant for images which have known API breakages and +# thus cannot work in those deployments anymore. For the purpose of this script, +# any line matching " *-* < optional additional text>" is fine, for example: +# +# The following canary images are known to be incompatible with this +# deployment: +# - csi-snapshotter (canary uses VolumeSnapshot v1beta) +# # Beware that the .yaml files do not have "imagePullPolicy: Always". That means that # also the "canary" images will only be pulled once. This is good for testing # (starting a pod multiple times will always run with the same canary image), but @@ -132,7 +141,15 @@ for i in $(ls ${BASE_DIR}/hostpath/*.yaml | sort); do # Only do this for the images which are meant to be configurable. if update_image "$name"; then prefix=$(eval echo \${${varname}_REGISTRY:-${IMAGE_REGISTRY:-${registry}}}/ | sed -e 's;none/;;') - suffix=$(eval echo :\${${varname}_TAG:-${IMAGE_TAG:-${tag}}}) + if [ "$IMAGE_TAG" = "canary" ] && + [ -f ${BASE_DIR}/README.md ] && + grep -q "^ *-* *$name *" ${BASE_DIR}/README.md; then + # Ignore IMAGE_TAG=canary for this particular image because its + # canary image is blacklisted in the deployment's README.md. + suffix=$(eval echo :\${${varname}_TAG:-${tag}}) + else + suffix=$(eval echo :\${${varname}_TAG:-${IMAGE_TAG:-${tag}}}) + fi line="$(echo "$nocomments" | sed -e "s;$image;${prefix}${name}${suffix};")" fi echo " using $line" >&2 From a5064ab6df7c4254281ef4301da173f61ac76b82 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Mon, 4 Nov 2019 19:17:23 +0100 Subject: [PATCH 4/4] deploy: blacklist canary images via canary-blacklist.txt This is a bit less brittle than embedding this information in the README.md. --- deploy/kubernetes-1.14/README.md | 4 ---- deploy/kubernetes-1.14/canary-blacklist.txt | 3 +++ deploy/kubernetes-1.15/README.md | 4 ---- deploy/kubernetes-1.15/canary-blacklist.txt | 3 +++ deploy/kubernetes-1.16/README.md | 4 ---- deploy/kubernetes-1.16/canary-blacklist.txt | 3 +++ deploy/util/deploy-hostpath.sh | 21 +++++++++++---------- 7 files changed, 20 insertions(+), 22 deletions(-) create mode 100644 deploy/kubernetes-1.14/canary-blacklist.txt create mode 100644 deploy/kubernetes-1.15/canary-blacklist.txt create mode 100644 deploy/kubernetes-1.16/canary-blacklist.txt diff --git a/deploy/kubernetes-1.14/README.md b/deploy/kubernetes-1.14/README.md index 08052770d..40face28b 100644 --- a/deploy/kubernetes-1.14/README.md +++ b/deploy/kubernetes-1.14/README.md @@ -6,7 +6,3 @@ introduced in Kubernetes 1.14, so features depending on those (like topology) will not work on Kubernetes 1.13. But because this example deployment does not enable those features, it can run on Kubernetes 1.13. - -The following canary images are known to be incompatible with this -deployment: -- csi-snapshotter (canary uses VolumeSnapshot v1beta) diff --git a/deploy/kubernetes-1.14/canary-blacklist.txt b/deploy/kubernetes-1.14/canary-blacklist.txt new file mode 100644 index 000000000..bf6d57004 --- /dev/null +++ b/deploy/kubernetes-1.14/canary-blacklist.txt @@ -0,0 +1,3 @@ +# The following canary images are known to be incompatible with this +# deployment: +csi-snapshotter diff --git a/deploy/kubernetes-1.15/README.md b/deploy/kubernetes-1.15/README.md index a370d55c6..c5b4c0f20 100644 --- a/deploy/kubernetes-1.15/README.md +++ b/deploy/kubernetes-1.15/README.md @@ -5,7 +5,3 @@ The sidecars depend on 1.15 API changes for migration and resizing, and 1.14 API changes for CSIDriver and CSINode. However the hostpath driver doesn't use those features, so this deployment can work on older Kubernetes versions. - -The following canary images are known to be incompatible with this -deployment: -- csi-snapshotter (canary uses VolumeSnapshot v1beta) diff --git a/deploy/kubernetes-1.15/canary-blacklist.txt b/deploy/kubernetes-1.15/canary-blacklist.txt new file mode 100644 index 000000000..bf6d57004 --- /dev/null +++ b/deploy/kubernetes-1.15/canary-blacklist.txt @@ -0,0 +1,3 @@ +# The following canary images are known to be incompatible with this +# deployment: +csi-snapshotter diff --git a/deploy/kubernetes-1.16/README.md b/deploy/kubernetes-1.16/README.md index 4ac48fb8b..c1d3c1462 100644 --- a/deploy/kubernetes-1.16/README.md +++ b/deploy/kubernetes-1.16/README.md @@ -1,7 +1,3 @@ The deployment for Kubernetes 1.16 enables ephemeral inline volumes via its CSIDriverInfo and thus is incompatible with Kubernetes < 1.16 because the `VolumeLifecycleModes` field is rejected by those release. - -The following canary images are known to be incompatible with this -deployment: -- csi-snapshotter (canary uses VolumeSnapshot v1beta) diff --git a/deploy/kubernetes-1.16/canary-blacklist.txt b/deploy/kubernetes-1.16/canary-blacklist.txt new file mode 100644 index 000000000..bf6d57004 --- /dev/null +++ b/deploy/kubernetes-1.16/canary-blacklist.txt @@ -0,0 +1,3 @@ +# The following canary images are known to be incompatible with this +# deployment: +csi-snapshotter diff --git a/deploy/util/deploy-hostpath.sh b/deploy/util/deploy-hostpath.sh index 3c48e90ea..81e92bd9c 100755 --- a/deploy/util/deploy-hostpath.sh +++ b/deploy/util/deploy-hostpath.sh @@ -32,14 +32,15 @@ BASE_DIR=$(dirname "$0") # - IMAGE_TAG # These are used as fallback when the more specific variables are unset or empty. # -# IMAGE_TAG=canary is ignored for images that are blacklisted in the deployment's -# top-level README.md. This is meant for images which have known API breakages and -# thus cannot work in those deployments anymore. For the purpose of this script, -# any line matching " *-* < optional additional text>" is fine, for example: +# IMAGE_TAG=canary is ignored for images that are blacklisted in the +# deployment's optional canary-blacklist.txt file. This is meant for +# images which have known API breakages and thus cannot work in those +# deployments anymore. That text file must have the name of the blacklisted +# image on a line by itself, other lines are ignored. Example: # -# The following canary images are known to be incompatible with this -# deployment: -# - csi-snapshotter (canary uses VolumeSnapshot v1beta) +# # The following canary images are known to be incompatible with this +# # deployment: +# csi-snapshotter # # Beware that the .yaml files do not have "imagePullPolicy: Always". That means that # also the "canary" images will only be pulled once. This is good for testing @@ -142,10 +143,10 @@ for i in $(ls ${BASE_DIR}/hostpath/*.yaml | sort); do if update_image "$name"; then prefix=$(eval echo \${${varname}_REGISTRY:-${IMAGE_REGISTRY:-${registry}}}/ | sed -e 's;none/;;') if [ "$IMAGE_TAG" = "canary" ] && - [ -f ${BASE_DIR}/README.md ] && - grep -q "^ *-* *$name *" ${BASE_DIR}/README.md; then + [ -f ${BASE_DIR}/canary-blacklist.txt ] && + grep -q "^$name\$" ${BASE_DIR}/canary-blacklist.txt; then # Ignore IMAGE_TAG=canary for this particular image because its - # canary image is blacklisted in the deployment's README.md. + # canary image is blacklisted in the deployment's blacklist. suffix=$(eval echo :\${${varname}_TAG:-${tag}}) else suffix=$(eval echo :\${${varname}_TAG:-${IMAGE_TAG:-${tag}}})