From f1697d2cac2e184d60b96d45720ed7a37a4ef9ed Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Tue, 8 Oct 2019 14:44:56 -0700 Subject: [PATCH 01/16] Do full git clones in travis. Shallow clones are causing test-subtree errors when the depth is exactly 50. --- travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/travis.yml b/travis.yml index 5b0425226..a713bed3b 100644 --- a/travis.yml +++ b/travis.yml @@ -2,6 +2,8 @@ language: go sudo: required services: - docker +git: + depth: false matrix: include: - go: 1.12.4 From e0fde8c4f5684f5c259c8c7a1d116def587cabf0 Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Wed, 30 Oct 2019 18:08:41 -0700 Subject: [PATCH 02/16] Add new variables for 1.16 and remove 1.13 --- prow.sh | 38 ++++++++++---------------------------- 1 file changed, 10 insertions(+), 28 deletions(-) diff --git a/prow.sh b/prow.sh index d93b6fa7d..e3308e77b 100755 --- a/prow.sh +++ b/prow.sh @@ -101,7 +101,8 @@ configvar CSI_PROW_GO_VERSION_GINKGO "${CSI_PROW_GO_VERSION_BUILD}" "Go version # kind version to use. If the pre-installed version is different, # the desired version is downloaded from https://github.com/kubernetes-sigs/kind/releases/download/ # (if available), otherwise it is built from source. -configvar CSI_PROW_KIND_VERSION "v0.5.0" "kind" +# TODO: https://github.com/kubernetes-csi/csi-release-tools/issues/39 +configvar CSI_PROW_KIND_VERSION "86bc23d84ac12dcb56a0528890736e2c347c2dc3" "kind" # ginkgo test runner version to use. If the pre-installed version is # different, the desired version is built from source. @@ -136,7 +137,6 @@ configvar CSI_PROW_KUBERNETES_VERSION 1.15.3 "Kubernetes" # # If the version is prefixed with "release-", then nothing # is overridden. -override_k8s_version "1.13.10" override_k8s_version "1.14.6" override_k8s_version "1.15.3" @@ -183,7 +183,7 @@ configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csip # # When no deploy script is found (nothing in `deploy` directory, # CSI_PROW_HOSTPATH_REPO=none), nothing gets deployed. -configvar CSI_PROW_HOSTPATH_VERSION "v1.2.0-rc8" "hostpath driver" +configvar CSI_PROW_HOSTPATH_VERSION "v1.2.0" "hostpath driver" configvar CSI_PROW_HOSTPATH_REPO https://github.com/kubernetes-csi/csi-driver-host-path "hostpath repo" configvar CSI_PROW_DEPLOYMENT "" "deployment" configvar CSI_PROW_HOSTPATH_DRIVER_NAME "hostpath.csi.k8s.io" "the hostpath driver name" @@ -200,9 +200,10 @@ configvar CSI_PROW_HOSTPATH_CANARY "" "hostpath image" # all generated files are present. # # CSI_PROW_E2E_REPO=none disables E2E testing. -configvar CSI_PROW_E2E_VERSION_1_13 v1.14.0 "E2E version for Kubernetes 1.13.x" # we can't use the one from 1.13.x because it didn't have --storage.testdriver +# TOOO: remove versioned variables and make e2e version match k8s version configvar CSI_PROW_E2E_VERSION_1_14 v1.14.0 "E2E version for Kubernetes 1.14.x" configvar CSI_PROW_E2E_VERSION_1_15 v1.15.0 "E2E version for Kubernetes 1.15.x" +configvar CSI_PROW_E2E_VERSION_1_16 v1.16.0 "E2E version for Kubernetes 1.16.x" # TODO: add new CSI_PROW_E2E_VERSION entry for future Kubernetes releases configvar CSI_PROW_E2E_VERSION_LATEST master "E2E version for Kubernetes master" # testing against Kubernetes master is already tracking a moving target, so we might as well use a moving E2E version configvar CSI_PROW_E2E_REPO_LATEST https://github.com/kubernetes/kubernetes "E2E repo for Kubernetes >= 1.13.x" # currently the same for all versions @@ -292,11 +293,6 @@ regex_join () { # alpha in previous Kubernetes releases. This was considered too # error prone. Therefore we use E2E tests that match the Kubernetes # version that is getting tested. -# -# However, for 1.13.x testing we have to use the E2E tests from 1.14 -# because 1.13 didn't have --storage.testdriver yet, so for that (and only -# that version) we have to define alpha tests differently. -configvar CSI_PROW_E2E_ALPHA_1_13 '\[Feature: \[Testpattern:.Dynamic.PV..block.volmode.\] should.create.and.delete.block.persistent.volumes' "alpha tests for Kubernetes 1.13" # Raw block was an alpha feature in 1.13. configvar CSI_PROW_E2E_ALPHA_LATEST '\[Feature:' "alpha tests for Kubernetes >= 1.14" # there's no need to update this, adding a new case for CSI_PROW_E2E for a new Kubernetes is enough configvar CSI_PROW_E2E_ALPHA "$(get_versioned_variable CSI_PROW_E2E_ALPHA "${csi_prow_kubernetes_version_suffix}")" "alpha tests" @@ -312,12 +308,12 @@ configvar CSI_PROW_E2E_ALPHA "$(get_versioned_variable CSI_PROW_E2E_ALPHA "${csi # kubernetes-csi components must be updated, either by disabling # the failing test for "latest" or by updating the test and not running # it anymore for older releases. -configvar CSI_PROW_E2E_ALPHA_GATES_1_13 'VolumeSnapshotDataSource=true,BlockVolume=true,CSIBlockVolume=true' "alpha feature gates for Kubernetes 1.13" configvar CSI_PROW_E2E_ALPHA_GATES_1_14 'VolumeSnapshotDataSource=true,ExpandCSIVolumes=true' "alpha feature gates for Kubernetes 1.14" configvar CSI_PROW_E2E_ALPHA_GATES_1_15 'VolumeSnapshotDataSource=true,ExpandCSIVolumes=true' "alpha feature gates for Kubernetes 1.15" +configvar CSI_PROW_E2E_ALPHA_GATES_1_16 'VolumeSnapshotDataSource=true' "alpha feature gates for Kubernetes 1.16" # TODO: add new CSI_PROW_ALPHA_GATES_xxx entry for future Kubernetes releases and # add new gates to CSI_PROW_E2E_ALPHA_GATES_LATEST. -configvar CSI_PROW_E2E_ALPHA_GATES_LATEST 'VolumeSnapshotDataSource=true,ExpandCSIVolumes=true' "alpha feature gates for latest Kubernetes" +configvar CSI_PROW_E2E_ALPHA_GATES_LATEST 'VolumeSnapshotDataSource=true' "alpha feature gates for latest Kubernetes" configvar CSI_PROW_E2E_ALPHA_GATES "$(get_versioned_variable CSI_PROW_E2E_ALPHA_GATES "${csi_prow_kubernetes_version_suffix}")" "alpha E2E feature gates" # Some tests are known to be unusable in a KinD cluster. For example, @@ -723,22 +719,6 @@ install_sanity () ( run_with_go "${CSI_PROW_GO_VERSION_SANITY}" go test -c -o "${CSI_PROW_WORK}/csi-sanity" "${CSI_PROW_SANITY_IMPORT_PATH}/cmd/csi-sanity" || die "building csi-sanity failed" ) -# Whether the hostpath driver supports raw block devices depends on which version -# we are testing. It would be much nicer if we could determine that by querying the -# installed driver's capabilities instead of having to do a version check. -hostpath_supports_block () { - local result - result="$(docker exec csi-prow-control-plane docker image ls --format='{{.Repository}} {{.Tag}} {{.ID}}' | grep hostpath | while read -r repo tag id; do - if [ "$tag" == "v1.0.1" ]; then - # Old version because the revision label is missing: didn't have support yet. - echo "false" - return - fi - done)" - # If not set, then it must be a newer driver with support. - echo "${result:-true}" -} - # The default implementation of this function generates a external # driver test configuration for the hostpath driver. # @@ -755,12 +735,14 @@ SnapshotClass: DriverInfo: Name: ${CSI_PROW_HOSTPATH_DRIVER_NAME} Capabilities: - block: $(hostpath_supports_block) + block: true persistence: true dataSource: true multipods: true nodeExpansion: true controllerExpansion: true + snapshotDataSource: true + singleNodeVolume: true EOF } From 5e773d2db08fa624d3b9352dd577f885efb7a2b2 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Thu, 31 Oct 2019 08:43:31 +0100 Subject: [PATCH 03/16] update CI to use Go 1.13.3 This is the latest release. Updating is useful to ensure that we have all of the latest fixes and enhancements. --- travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/travis.yml b/travis.yml index a713bed3b..494c11444 100644 --- a/travis.yml +++ b/travis.yml @@ -6,7 +6,7 @@ git: depth: false matrix: include: - - go: 1.12.4 + - go: 1.13.3 before_script: - mkdir -p bin - wget https://github.com/golang/dep/releases/download/v0.5.1/dep-linux-amd64 -O bin/dep From c8a1c4af933311a7e63765cd2b64ca45a0fb7dba Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Thu, 31 Oct 2019 11:49:23 +0100 Subject: [PATCH 04/16] better handling of Go version Some operations are sensitive to the version of Go that is used. In the past, formatting of source differed depending on the version. Right now it is the content of the vendor directory which changes when switch back and forth between 1.12 and 1.13. We don't want to impose a certain workflow on developers, like forcing all invocations of Go to run inside a container. If developers want that, they can set up their development environment accordingly. But we should warn about this aspect to raise awareness. "make" invocations which involve Go now compare against the projects Go version (specified in travis.yml) once at the beginning. This is only a warning because we don't know which future version will be compatible with the project. Vendor directory handling gets updated, too: verification is now a separate script (became too complex for make) and there is a corresponding "update-vendor.sh". In contrast to verification, updating vendor is not integrated into make and thus itself invokes the go version check. --- build.make | 50 ++++++++---------------------------- update-vendor.sh | 23 +++++++++++++++++ verify-go-version.sh | 51 +++++++++++++++++++++++++++++++++++++ verify-vendor.sh | 60 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 145 insertions(+), 39 deletions(-) create mode 100755 update-vendor.sh create mode 100755 verify-go-version.sh create mode 100755 verify-vendor.sh diff --git a/build.make b/build.make index cbf6d4553..6b89f5b20 100644 --- a/build.make +++ b/build.make @@ -62,7 +62,7 @@ ARCH := $(if $(GOARCH),$(GOARCH),$(shell go env GOARCH)) # Specific packages can be excluded from each of the tests below by setting the *_FILTER_CMD variables # to something like "| grep -v 'github.com/kubernetes-csi/project/pkg/foobar'". See usage below. -build-%: +build-%: check-go-version-go mkdir -p bin CGO_ENABLED=0 GOOS=linux go build -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$* ./cmd/$* if [ "$$ARCH" = "amd64" ]; then \ @@ -97,7 +97,7 @@ push: $(CMDS:%=push-%) clean: -rm -rf bin -test: +test: check-go-version-go .PHONY: test-go test: test-go @@ -154,43 +154,7 @@ test-fmt: test: test-vendor test-vendor: @ echo; echo "### $@:" - @ if [ -f Gopkg.toml ]; then \ - echo "Repo uses 'dep' for vendoring."; \ - case "$$(dep version 2>/dev/null | grep 'version *:')" in \ - *v0.[56789]*) dep check && echo "vendor up-to-date" || false;; \ - *) echo "skipping check, dep >= 0.5 required";; \ - esac; \ - elif [ -f go.mod ]; then \ - echo "Repo uses 'go mod'."; \ - if [ "$${JOB_NAME}" ] && \ - ( [ "$${JOB_TYPE}" != "presubmit" ] || \ - [ $$( (git diff "${PULL_BASE_SHA}..HEAD" -- go.mod go.sum vendor release-tools; \ - git diff "${PULL_BASE_SHA}..HEAD" | grep -e '^@@.*@@ import (' -e '^[+-]import') | \ - wc -l) -eq 0 ] ); then \ - echo "Skipping vendor check because the Prow pre-submit job does not affect dependencies."; \ - elif ! GO111MODULE=on go mod tidy; then \ - echo "ERROR: vendor check failed."; \ - false; \ - elif [ $$(git status --porcelain -- go.mod go.sum | wc -l) -gt 0 ]; then \ - echo "ERROR: go module files *not* up-to-date, they did get modified by 'GO111MODULE=on go mod tidy':"; \ - git diff -- go.mod go.sum; \ - false; \ - elif [ -d vendor ]; then \ - if ! GO111MODULE=on go mod vendor; then \ - echo "ERROR: vendor check failed."; \ - false; \ - elif [ $$(git status --porcelain -- vendor | wc -l) -gt 0 ]; then \ - echo "ERROR: vendor directory *not* up-to-date, it did get modified by 'GO111MODULE=on go mod vendor':"; \ - git status -- vendor; \ - git diff -- vendor; \ - false; \ - else \ - echo "Go dependencies and vendor directory up-to-date."; \ - fi; \ - else \ - echo "Go dependencies up-to-date."; \ - fi; \ - fi + @ ./release-tools/verify-vendor.sh .PHONY: test-subtree test: test-subtree @@ -216,3 +180,11 @@ test-shellcheck: ./release-tools/verify-shellcheck.sh "$$dir" || ret=1; \ done; \ exit $$ret + +# Targets in the makefile can depend on check-go-version- +# to trigger a warning if the x.y version of that binary does not match +# what the project uses. Make ensures that this is only checked once per +# invocation. +.PHONY: check-go-version-% +check-go-version-%: + ./release-tools/verify-go-version.sh "$*" diff --git a/update-vendor.sh b/update-vendor.sh new file mode 100755 index 000000000..6f4c27aeb --- /dev/null +++ b/update-vendor.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ -f Gopkg.toml ]; then + echo "Repo uses 'dep' for vendoring." + (set -x; dep ensure) +elif [ -f go.mod ]; then + release-tools/verify-go-version.sh "go" + (set -x; env GO111MODULE=on go mod tidy && env GO111MODULE=on go mod vendor) +fi diff --git a/verify-go-version.sh b/verify-go-version.sh new file mode 100755 index 000000000..f242e769d --- /dev/null +++ b/verify-go-version.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +GO="$1" + +if [ ! "$GO" ]; then + echo >&2 "usage: $0 " + exit 1 +fi + +die () { + echo "ERROR: $*" + exit 1 +} + +version=$("$GO" version) || die "determining version of $GO failed" +# shellcheck disable=SC2001 +majorminor=$(echo "$version" | sed -e 's/.*go\([0-9]*\)\.\([0-9]*\).*/\1.\2/') +# shellcheck disable=SC2001 +expected=$(grep "^ *- go:" "release-tools/travis.yml" | sed -e 's/.*go: *\([0-9]*\)\.\([0-9]*\).*/\1.\2/') + +if [ "$majorminor" != "$expected" ]; then + cat >&2 </dev/null | grep 'version *:')" in + *v0.[56789]*) + if dep check; then + echo "vendor up-to-date" + else + exit 1 + fi + ;; + *) echo "skipping check, dep >= 0.5 required";; + esac +elif [ -f go.mod ]; then + echo "Repo uses 'go mod'." + # shellcheck disable=SC2235 + if [ "${JOB_NAME}" ] && + ( [ "${JOB_TYPE}" != "presubmit" ] || + [ "$( (git diff "${PULL_BASE_SHA}..HEAD" -- go.mod go.sum vendor release-tools; + git diff "${PULL_BASE_SHA}..HEAD" | grep -e '^@@.*@@ import (' -e '^[+-]import') | + wc -l)" -eq 0 ] ); then + echo "Skipping vendor check because the Prow pre-submit job does not affect dependencies." + elif ! (set -x; env GO111MODULE=on go mod tidy); then + echo "ERROR: vendor check failed." + exit 1 + elif [ "$(git status --porcelain -- go.mod go.sum | wc -l)" -gt 0 ]; then + echo "ERROR: go module files *not* up-to-date, they did get modified by 'GO111MODULE=on go mod tidy':"; + git diff -- go.mod go.sum + exit 1 + elif [ -d vendor ]; then + if ! (set -x; env GO111MODULE=on go mod vendor); then + echo "ERROR: vendor check failed." + exit 1 + elif [ "$(git status --porcelain -- vendor | wc -l)" -gt 0 ]; then + echo "ERROR: vendor directory *not* up-to-date, it did get modified by 'GO111MODULE=on go mod vendor':" + git status -- vendor + git diff -- vendor + exit 1 + else + echo "Go dependencies and vendor directory up-to-date." + fi + else + echo "Go dependencies up-to-date." + fi +fi From 23df4aef51ee82896bc8326a693c3fdbd0baa07f Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Mon, 4 Nov 2019 11:16:38 +0100 Subject: [PATCH 05/16] prow.sh: use vendor directory if available This avoids dependencies on the Go module cache or the upstream code hosting. --- build.make | 11 +++++++---- prow.sh | 12 +++++++++--- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/build.make b/build.make index 6b89f5b20..1b6f35fe1 100644 --- a/build.make +++ b/build.make @@ -22,6 +22,9 @@ # including build.make. REGISTRY_NAME=quay.io/k8scsi +# Can be set to -mod=vendor to ensure that the "vendor" directory is used. +GOFLAGS_VENDOR= + # Revision that gets built into each binary via the main.version # string. Uses the `git describe` output based on the most recent # version tag with a short revision suffix or, if nothing has been @@ -64,9 +67,9 @@ ARCH := $(if $(GOARCH),$(GOARCH),$(shell go env GOARCH)) build-%: check-go-version-go mkdir -p bin - CGO_ENABLED=0 GOOS=linux go build -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$* ./cmd/$* + CGO_ENABLED=0 GOOS=linux go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$* ./cmd/$* if [ "$$ARCH" = "amd64" ]; then \ - CGO_ENABLED=0 GOOS=windows go build -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$*.exe ./cmd/$* ; \ + CGO_ENABLED=0 GOOS=windows go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$*.exe ./cmd/$* ; \ fi container-%: build-% @@ -103,13 +106,13 @@ test: check-go-version-go test: test-go test-go: @ echo; echo "### $@:" - go test `go list ./... | grep -v -e 'vendor' -e '/test/e2e$$' $(TEST_GO_FILTER_CMD)` $(TESTARGS) + go test $(GOFLAGS_VENDOR) `go list $(GOFLAGS_VENDOR) ./... | grep -v -e 'vendor' -e '/test/e2e$$' $(TEST_GO_FILTER_CMD)` $(TESTARGS) .PHONY: test-vet test: test-vet test-vet: @ echo; echo "### $@:" - go vet `go list ./... | grep -v vendor $(TEST_VET_FILTER_CMD)` + go test $(GOFLAGS_VENDOR) `go list $(GOFLAGS_VENDOR) ./... | grep -v vendor $(TEST_VET_FILTER_CMD)` .PHONY: test-fmt test: test-fmt diff --git a/prow.sh b/prow.sh index e3308e77b..fe68bb7d9 100755 --- a/prow.sh +++ b/prow.sh @@ -85,6 +85,12 @@ get_versioned_variable () { echo "$value" } +# If we have a vendor directory, then use it. We must be careful to only +# use this for "make" invocations inside the project's repo itself because +# setting it globally can break other go usages (like "go get " +# which is disabled with GOFLAGS=-mod=vendor). +configvar GOFLAGS_VENDOR "$( [ -d vendor ] && echo '-mod=vendor' )" "Go flags for using the vendor directory" + # Go versions can be specified seperately for different tasks # If the pre-installed Go is missing or a different # version, the required version here will get installed @@ -928,7 +934,7 @@ main () { images= if ${CSI_PROW_BUILD_JOB}; then # A successful build is required for testing. - run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make all || die "'make all' failed" + run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make all "GOFLAGS_VENDOR=${GOFLAGS_VENDOR}" || die "'make all' failed" # We don't want test failures to prevent E2E testing below, because the failure # might have been minor or unavoidable, for example when experimenting with # changes in "release-tools" in a PR (that fails the "is release-tools unmodified" @@ -938,13 +944,13 @@ main () { warn "installing 'dep' failed, cannot test vendoring" ret=1 fi - if ! run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make -k test 2>&1 | make_test_to_junit; then + if ! run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make -k test "GOFLAGS_VENDOR=${GOFLAGS_VENDOR}" 2>&1 | make_test_to_junit; then warn "'make test' failed, proceeding anyway" ret=1 fi fi # Required for E2E testing. - run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make container || die "'make container' failed" + run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make container "GOFLAGS_VENDOR=${GOFLAGS_VENDOR}" || die "'make container' failed" fi if tests_need_kind; then From 806784565969caa302d40f98f6fb6dd3dafb39ba Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Mon, 4 Nov 2019 15:40:31 +0100 Subject: [PATCH 06/16] travis.yml: also use vendor directory --- travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/travis.yml b/travis.yml index 494c11444..bfd7647ba 100644 --- a/travis.yml +++ b/travis.yml @@ -13,9 +13,9 @@ before_script: - chmod u+x bin/dep - export PATH=$PWD/bin:$PATH script: -- make -k all test +- make -k all test GOFLAGS_VENDOR=$( [ -d vendor ] && echo '-mod=vendor' ) after_success: - if [ "${TRAVIS_PULL_REQUEST}" == "false" ]; then docker login -u "${DOCKER_USERNAME}" -p "${DOCKER_PASSWORD}" quay.io; - make push; + make push GOFLAGS_VENDOR=$( [ -d vendor ] && echo '-mod=vendor' ); fi From f41c1351a38ad936cbdec298fff3f49b8e2e8613 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Tue, 5 Nov 2019 16:40:29 +0100 Subject: [PATCH 07/16] prow.sh: also log output of system containers Depending on the error, those logs are needed to debug failures. --- prow.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/prow.sh b/prow.sh index e3308e77b..0777ae377 100755 --- a/prow.sh +++ b/prow.sh @@ -677,16 +677,16 @@ EOF } -# Gets logs of all containers in the default namespace. When passed -f, kubectl will +# Gets logs of all containers in all namespaces. When passed -f, kubectl will # keep running and capture new output. Prints the pid of all background processes. # The caller must kill (when using -f) and/or wait for them. # # May be called multiple times and thus appends. start_loggers () { - kubectl get pods -o go-template --template='{{range .items}}{{.metadata.name}} {{range .spec.containers}}{{.name}} {{end}}{{"\n"}}{{end}}' | while read -r pod containers; do + kubectl get pods --all-namespaces -o go-template --template='{{range .items}}{{.metadata.namespace}} {{.metadata.name}} {{range .spec.containers}}{{.name}} {{end}}{{"\n"}}{{end}}' | while read -r namespace pod containers; do for container in $containers; do - mkdir -p "${ARTIFACTS}/$pod" - kubectl logs "$@" "$pod" "$container" >>"${ARTIFACTS}/$pod/$container.log" & + mkdir -p "${ARTIFACTS}/$namespace/$pod" + kubectl logs -n "$namespace" "$@" "$pod" "$container" >>"${ARTIFACTS}/$namespace/$pod/$container.log" & echo "$!" done done From 83a4ef15dfda6403862437f8acf3606f255e888a Mon Sep 17 00:00:00 2001 From: Peeyush Gupta Date: Tue, 19 Nov 2019 08:58:58 -0500 Subject: [PATCH 08/16] Adding build for ppc64le --- build.make | 1 + 1 file changed, 1 insertion(+) diff --git a/build.make b/build.make index 1b6f35fe1..7075a37ee 100644 --- a/build.make +++ b/build.make @@ -70,6 +70,7 @@ build-%: check-go-version-go CGO_ENABLED=0 GOOS=linux go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$* ./cmd/$* if [ "$$ARCH" = "amd64" ]; then \ CGO_ENABLED=0 GOOS=windows go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$*.exe ./cmd/$* ; \ + CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$*-ppc64le ./cmd/$* fi container-%: build-% From 1eaaaa1cb49713e8e4af87d009c24d310cf66260 Mon Sep 17 00:00:00 2001 From: Mucahit Kurt Date: Sat, 16 Nov 2019 06:23:08 +0300 Subject: [PATCH 09/16] Delete kind cluster after tests run. Inside a real Prow job it is better to clean up at runtime instead of leaving that to the Prow job cleanup code because the later sometimes times out. Signed-off-by: Mucahit Kurt --- prow.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/prow.sh b/prow.sh index 3b9621a21..8b431c564 100755 --- a/prow.sh +++ b/prow.sh @@ -580,6 +580,19 @@ EOF export KUBECONFIG } +# Deletes kind cluster inside a prow job +delete_cluster_inside_prow_job() { + # Inside a real Prow job it is better to clean up at runtime + # instead of leaving that to the Prow job cleanup code + # because the later sometimes times out (https://github.com/kubernetes-csi/csi-release-tools/issues/24#issuecomment-554765872). + if [ "$JOB_NAME" ]; then + if kind get clusters | grep -q csi-prow; then + run kind delete cluster --name=csi-prow || die "kind delete failed" + fi + unset KUBECONFIG + fi +} + # Looks for the deployment as specified by CSI_PROW_DEPLOYMENT and CSI_PROW_KUBERNETES_VERSION # in the given directory. find_deployment () { @@ -1017,6 +1030,7 @@ main () { fi fi fi + delete_cluster_inside_prow_job fi if tests_need_alpha_cluster && [ "${CSI_PROW_E2E_ALPHA_GATES}" ]; then @@ -1047,6 +1061,7 @@ main () { fi fi fi + delete_cluster_inside_prow_job fi fi From 003c14b2d4ae3b1463db5e5b3ff91f39b03f5ba8 Mon Sep 17 00:00:00 2001 From: Grant Griffiths Date: Mon, 11 Nov 2019 23:49:42 -0800 Subject: [PATCH 10/16] Add snapshotter CRDs after cluster setup Signed-off-by: Grant Griffiths --- prow.sh | 80 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/prow.sh b/prow.sh index 3b9621a21..bc9c9f03b 100755 --- a/prow.sh +++ b/prow.sh @@ -322,6 +322,9 @@ configvar CSI_PROW_E2E_ALPHA_GATES_1_16 'VolumeSnapshotDataSource=true' "alpha f configvar CSI_PROW_E2E_ALPHA_GATES_LATEST 'VolumeSnapshotDataSource=true' "alpha feature gates for latest Kubernetes" configvar CSI_PROW_E2E_ALPHA_GATES "$(get_versioned_variable CSI_PROW_E2E_ALPHA_GATES "${csi_prow_kubernetes_version_suffix}")" "alpha E2E feature gates" +# Which external-snapshotter tag to use for the snapshotter CRD and snapshot-controller deployment +configvar CSI_SNAPSHOTTER_VERSION 'v2.0.0-rc4' "external-snapshotter version tag" + # Some tests are known to be unusable in a KinD cluster. For example, # stopping kubelet with "ssh systemctl stop kubelet" simply # doesn't work. Such tests should be written in a way that they verify @@ -657,6 +660,59 @@ install_hostpath () { fi } +# Installs all nessesary snapshotter CRDs +install_snapshot_crds() { + # Wait until volumesnapshot CRDs are in place. + CRD_BASE_DIR="https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}/config/crd" + kubectl apply -f "${CRD_BASE_DIR}/snapshot.storage.k8s.io_volumesnapshotclasses.yaml" --validate=false + kubectl apply -f "${CRD_BASE_DIR}/snapshot.storage.k8s.io_volumesnapshots.yaml" --validate=false + kubectl apply -f "${CRD_BASE_DIR}/snapshot.storage.k8s.io_volumesnapshotcontents.yaml" --validate=false + cnt=0 + until kubectl get volumesnapshotclasses.snapshot.storage.k8s.io \ + && kubectl get volumesnapshots.snapshot.storage.k8s.io \ + && kubectl get volumesnapshotcontents.snapshot.storage.k8s.io; do + if [ $cnt -gt 30 ]; then + echo >&2 "ERROR: snapshot CRDs not ready after over 1 min" + exit 1 + fi + echo "$(date +%H:%M:%S)" "waiting for snapshot CRDs, attempt #$cnt" + cnt=$((cnt + 1)) + sleep 2 + done +} + +# Install snapshot controller and associated RBAC, retrying until the pod is running. +install_snapshot_controller() { + kubectl apply -f "https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}/deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml" + cnt=0 + until kubectl get clusterrolebinding snapshot-controller-role; do + if [ $cnt -gt 30 ]; then + echo "Cluster role bindings:" + kubectl describe clusterrolebinding + echo >&2 "ERROR: snapshot controller RBAC not ready after over 5 min" + exit 1 + fi + echo "$(date +%H:%M:%S)" "waiting for snapshot RBAC setup complete, attempt #$cnt" + cnt=$((cnt + 1)) + sleep 10 + done + + + kubectl apply -f "https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml" + cnt=0 + until kubectl get statefulset snapshot-controller | grep snapshot-controller | grep "1/1"; do + if [ $cnt -gt 30 ]; then + echo "Running statefulsets:" + kubectl describe statefulsets + echo >&2 "ERROR: snapshot controller not ready after over 5 min" + exit 1 + fi + echo "$(date +%H:%M:%S)" "waiting for snapshot controller deployment to complete, attempt #$cnt" + cnt=$((cnt + 1)) + sleep 10 + done +} + # collect logs and cluster status (like the version of all components, Kubernetes version, test version) collect_cluster_info () { cat < Date: Sat, 30 Nov 2019 00:29:00 +0530 Subject: [PATCH 11/16] Use kind v0.6.0 kind v0.6.0 appends the kubeconfig with the default config at ~/.kube/config. --- prow.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/prow.sh b/prow.sh index d719e9808..b88230629 100755 --- a/prow.sh +++ b/prow.sh @@ -107,8 +107,7 @@ configvar CSI_PROW_GO_VERSION_GINKGO "${CSI_PROW_GO_VERSION_BUILD}" "Go version # kind version to use. If the pre-installed version is different, # the desired version is downloaded from https://github.com/kubernetes-sigs/kind/releases/download/ # (if available), otherwise it is built from source. -# TODO: https://github.com/kubernetes-csi/csi-release-tools/issues/39 -configvar CSI_PROW_KIND_VERSION "86bc23d84ac12dcb56a0528890736e2c347c2dc3" "kind" +configvar CSI_PROW_KIND_VERSION "v0.6.0" "kind" # ginkgo test runner version to use. If the pre-installed version is # different, the desired version is built from source. @@ -579,8 +578,7 @@ EOF die "Cluster creation failed again, giving up. See the 'kind-cluster' artifact directory for additional logs." fi fi - KUBECONFIG="$(kind get kubeconfig-path --name=csi-prow)" - export KUBECONFIG + export KUBECONFIG="${HOME}/.kube/config" } # Deletes kind cluster inside a prow job From 9a7a685ee169d669a0182532315aeecd8a8715dc Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Mon, 2 Dec 2019 17:45:57 -0800 Subject: [PATCH 12/16] Create a kind cluster with two worker nodes so that the topology feature can be tested. Test cases that test accessing volumes from multiple nodes need to be skipped --- prow.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/prow.sh b/prow.sh index b88230629..815bc9d8a 100755 --- a/prow.sh +++ b/prow.sh @@ -330,7 +330,11 @@ configvar CSI_SNAPSHOTTER_VERSION 'v2.0.0-rc4' "external-snapshotter version tag # whether they can run with the current cluster provider, but until # they are, we filter them out by name. Like the other test selection # variables, this is again a space separated list of regular expressions. -configvar CSI_PROW_E2E_SKIP 'Disruptive' "tests that need to be skipped" +# +# "different node" test skips can be removed once +# https://github.com/kubernetes/kubernetes/pull/82678 has been backported +# to all the K8s versions we test against +configvar CSI_PROW_E2E_SKIP 'Disruptive|different\s+node' "tests that need to be skipped" # This is the directory for additional result files. Usually set by Prow, but # if not (for example, when invoking manually) it defaults to the work directory. @@ -526,6 +530,7 @@ apiVersion: kind.sigs.k8s.io/v1alpha3 nodes: - role: control-plane - role: worker +- role: worker EOF # kubeadm has API dependencies between apiVersion and Kubernetes version @@ -840,10 +845,6 @@ run_e2e () ( install_e2e || die "building e2e.test failed" install_ginkgo || die "installing ginkgo failed" - # TODO (?): multi-node cluster (depends on https://github.com/kubernetes-csi/csi-driver-host-path/pull/14). - # When running on a multi-node cluster, we need to figure out where the - # hostpath driver was deployed and set ClientNodeName accordingly. - generate_test_driver >"${CSI_PROW_WORK}/test-driver.yaml" || die "generating test-driver.yaml failed" # Rename, merge and filter JUnit files. Necessary in case that we run the E2E suite again From 53888ae7d59b862bfded46b883570c98f5fc5fff Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Tue, 3 Dec 2019 18:18:38 -0800 Subject: [PATCH 13/16] Improve README by adding an explicit Kubernetes dependency section --- README.md | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index cc40f161c..08c828846 100644 --- a/README.md +++ b/README.md @@ -141,17 +141,6 @@ The `vendor` directory is optional. It is still present in projects because it avoids downloading sources during CI builds. If this is no longer deemed necessary, then a project can also remove the directory. -When using packages that are part of the Kubernetes source code, the -commands above are not enough because the [lack of semantic -versioning](https://github.com/kubernetes/kubernetes/issues/72638) -prevents `go mod` from finding newer releases. Importing directly from -`kubernetes/kubernetes` also needs `replace` statements to override -the fake `v0.0.0` versions -(https://github.com/kubernetes/kubernetes/issues/79384). The -`go-get-kubernetes.sh` script can be used to update all packages in -lockstep to a different Kubernetes version. It takes a single version -number like "1.16.0". - Conversion of a repository that uses `dep` to `go mod` can be done with: GO111MODULE=on go mod init @@ -160,3 +149,18 @@ Conversion of a repository that uses `dep` to `go mod` can be done with: GO111MODULE=on go mod vendor git rm -f Gopkg.toml Gopkg.lock git add go.mod go.sum vendor + +### Updating Kubernetes dependencies + +When using packages that are part of the Kubernetes source code, the +commands above are not enough because the [lack of semantic +versioning](https://github.com/kubernetes/kubernetes/issues/72638) +prevents `go mod` from finding newer releases. Importing directly from +`kubernetes/kubernetes` also needs `replace` statements to override +the fake `v0.0.0` versions +(https://github.com/kubernetes/kubernetes/issues/79384). The +`go-get-kubernetes.sh` script can be used to update all packages in +lockstep to a different Kubernetes version. Example usage: +``` +$ ./release-tools/go-get-kubernetes.sh 1.16.4 +``` From 4ad69492c97834b52d74d0b67b6f47b0590e4be1 Mon Sep 17 00:00:00 2001 From: Grant Griffiths Date: Tue, 3 Dec 2019 23:48:29 -0800 Subject: [PATCH 14/16] Improve snapshot pod running checks and improve version_gt Signed-off-by: Grant Griffiths --- prow.sh | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/prow.sh b/prow.sh index d719e9808..27c705129 100755 --- a/prow.sh +++ b/prow.sh @@ -713,10 +713,11 @@ install_snapshot_controller() { kubectl apply -f "https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml" cnt=0 - until kubectl get statefulset snapshot-controller | grep snapshot-controller | grep "1/1"; do + expected_running_pods=$(curl https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/"${CSI_SNAPSHOTTER_VERSION}"/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml | grep replicas | cut -d ':' -f 2-) + while [ "$(kubectl get pods -l app=snapshot-controller | grep 'Running' -c)" -lt "$expected_running_pods" ]; do if [ $cnt -gt 30 ]; then - echo "Running statefulsets:" - kubectl describe statefulsets + echo "snapshot-controller pod status:" + kubectl describe pods -l app=snapshot-controller echo >&2 "ERROR: snapshot controller not ready after over 5 min" exit 1 fi @@ -996,8 +997,30 @@ make_test_to_junit () { fi } +# version_gt returns true if arg1 is greater than arg2. +# +# This function expects versions to be one of the following formats: +# X.Y.Z, release-X.Y.Z, vX.Y.Z +# +# where X,Y, and Z are any number. +# +# Partial versions (1.2, release-1.2) work as well. +# The follow substrings are stripped before version comparison: +# - "v" +# - "release-" +# +# Usage: +# version_gt release-1.3 v1.2.0 (returns true) +# version_gt v1.1.1 v1.2.0 (returns false) +# version_gt 1.1.1 v1.2.0 (returns false) +# version_gt 1.3.1 v1.2.0 (returns true) +# version_gt 1.1.1 release-1.2.0 (returns false) +# version_gt 1.2.0 1.2.2 (returns false) function version_gt() { - test "$(printf '%s\n' "$@" | sort -V | head -n 1)" != "$1"; + versions=$(for ver in "$@"; do ver=${ver#release-}; echo "${ver#v}"; done) + greaterVersion=${1#"release-"}; + greaterVersion=${greaterVersion#"v"}; + test "$(printf '%s' "$versions" | sort -V | head -n 1)" != "$greaterVersion" } main () { From a4e629966848d0097461f49f8aab0dfeb68d7a68 Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Wed, 4 Dec 2019 14:12:50 -0800 Subject: [PATCH 15/16] fix syntax for ppc64le build --- build.make | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.make b/build.make index 7075a37ee..a9b9d25dc 100644 --- a/build.make +++ b/build.make @@ -70,7 +70,7 @@ build-%: check-go-version-go CGO_ENABLED=0 GOOS=linux go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$* ./cmd/$* if [ "$$ARCH" = "amd64" ]; then \ CGO_ENABLED=0 GOOS=windows go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$*.exe ./cmd/$* ; \ - CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$*-ppc64le ./cmd/$* + CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$*-ppc64le ./cmd/$* ; \ fi container-%: build-% From b98b2aed087f4e768db673a9d43dff4c7abd1482 Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Mon, 16 Dec 2019 19:13:38 -0800 Subject: [PATCH 16/16] Enable snapshot tests in 1.17 to be run in non-alpha jobs. This requires adding one more parallel e2e test run with a special focus flag because snapshot tests are still guarded with a "[Feature:VolumeSnapshotDataSource]" tag. The setting that skips all tests with "[Feature:.*]" has to be removed because it overrides the focus. We don't have serial snapshot tests yet. This needs to be modified again if we add any in the future. --- prow.sh | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/prow.sh b/prow.sh index 9bfdc2e28..c160f886c 100755 --- a/prow.sh +++ b/prow.sh @@ -132,7 +132,7 @@ configvar CSI_PROW_BUILD_JOB true "building code in repo enabled" # use the same settings as for "latest" Kubernetes. This works # as long as there are no breaking changes in Kubernetes, like # deprecating or changing the implementation of an alpha feature. -configvar CSI_PROW_KUBERNETES_VERSION 1.15.3 "Kubernetes" +configvar CSI_PROW_KUBERNETES_VERSION 1.17.0 "Kubernetes" # This is a hack to workaround the issue that each version # of kind currently only supports specific patch versions of @@ -142,7 +142,6 @@ configvar CSI_PROW_KUBERNETES_VERSION 1.15.3 "Kubernetes" # # If the version is prefixed with "release-", then nothing # is overridden. -override_k8s_version "1.14.6" override_k8s_version "1.15.3" # CSI_PROW_KUBERNETES_VERSION reduced to first two version numbers and @@ -206,9 +205,9 @@ configvar CSI_PROW_HOSTPATH_CANARY "" "hostpath image" # # CSI_PROW_E2E_REPO=none disables E2E testing. # TOOO: remove versioned variables and make e2e version match k8s version -configvar CSI_PROW_E2E_VERSION_1_14 v1.14.0 "E2E version for Kubernetes 1.14.x" configvar CSI_PROW_E2E_VERSION_1_15 v1.15.0 "E2E version for Kubernetes 1.15.x" configvar CSI_PROW_E2E_VERSION_1_16 v1.16.0 "E2E version for Kubernetes 1.16.x" +configvar CSI_PROW_E2E_VERSION_1_17 v1.17.0 "E2E version for Kubernetes 1.17.x" # TODO: add new CSI_PROW_E2E_VERSION entry for future Kubernetes releases configvar CSI_PROW_E2E_VERSION_LATEST master "E2E version for Kubernetes master" # testing against Kubernetes master is already tracking a moving target, so we might as well use a moving E2E version configvar CSI_PROW_E2E_REPO_LATEST https://github.com/kubernetes/kubernetes "E2E repo for Kubernetes >= 1.13.x" # currently the same for all versions @@ -278,6 +277,14 @@ tests_need_alpha_cluster () { tests_enabled "parallel-alpha" "serial-alpha" } +# Regex for non-alpha, feature-tagged tests that should be run. +# +# Starting with 1.17, snapshots is beta, but the E2E tests still have the +# [Feature:] tag. They need to be explicitly enabled. +configvar CSI_PROW_E2E_FOCUS_1_15 '^' "non-alpha, feature-tagged tests for Kubernetes = 1.15" # no tests to run, match nothing +configvar CSI_PROW_E2E_FOCUS_1_16 '^' "non-alpha, feature-tagged tests for Kubernetes = 1.16" # no tests to run, match nothing +configvar CSI_PROW_E2E_FOCUS_LATEST '\[Feature:VolumeSnapshotDataSource\]' "non-alpha, feature-tagged tests for Kubernetes >= 1.17" +configvar CSI_PROW_E2E_FOCUS "$(get_versioned_variable CSI_PROW_E2E_FOCUS "${csi_prow_kubernetes_version_suffix}")" "non-alpha, feature-tagged tests" # Serial vs. parallel is always determined by these regular expressions. # Individual regular expressions are seperated by spaces for readability @@ -313,12 +320,11 @@ configvar CSI_PROW_E2E_ALPHA "$(get_versioned_variable CSI_PROW_E2E_ALPHA "${csi # kubernetes-csi components must be updated, either by disabling # the failing test for "latest" or by updating the test and not running # it anymore for older releases. -configvar CSI_PROW_E2E_ALPHA_GATES_1_14 'VolumeSnapshotDataSource=true,ExpandCSIVolumes=true' "alpha feature gates for Kubernetes 1.14" configvar CSI_PROW_E2E_ALPHA_GATES_1_15 'VolumeSnapshotDataSource=true,ExpandCSIVolumes=true' "alpha feature gates for Kubernetes 1.15" configvar CSI_PROW_E2E_ALPHA_GATES_1_16 'VolumeSnapshotDataSource=true' "alpha feature gates for Kubernetes 1.16" # TODO: add new CSI_PROW_ALPHA_GATES_xxx entry for future Kubernetes releases and # add new gates to CSI_PROW_E2E_ALPHA_GATES_LATEST. -configvar CSI_PROW_E2E_ALPHA_GATES_LATEST 'VolumeSnapshotDataSource=true' "alpha feature gates for latest Kubernetes" +configvar CSI_PROW_E2E_ALPHA_GATES_LATEST '' "alpha feature gates for latest Kubernetes" configvar CSI_PROW_E2E_ALPHA_GATES "$(get_versioned_variable CSI_PROW_E2E_ALPHA_GATES "${csi_prow_kubernetes_version_suffix}")" "alpha E2E feature gates" # Which external-snapshotter tag to use for the snapshotter CRD and snapshot-controller deployment @@ -1111,6 +1117,16 @@ main () { warn "E2E parallel failed" ret=1 fi + + # Run tests that are feature tagged, but non-alpha + # Ignore: Double quote to prevent globbing and word splitting. + # shellcheck disable=SC2086 + if ! run_e2e parallel ${CSI_PROW_GINKO_PARALLEL} \ + -focus="External.Storage.*($(regex_join "${CSI_PROW_E2E_FOCUS}"))" \ + -skip="$(regex_join "${CSI_PROW_E2E_SERIAL}")"; then + warn "E2E parallel features failed" + ret=1 + fi fi if tests_enabled "serial"; then