Parallelize end-to-end tests

.circleci/config.yml

@@ -61,6 +61,7 @@ jobs:
     working_directory: /home/circleci/go/src/github.com/fluxcd/flux
     machine:
       image: ubuntu-1604:201903-01
+    resource_class: large
     environment:
       GO_VERSION: 1.13.1
       # We don't need a GOPATH but CircleCI defines it, so we override it
@@ -90,7 +91,10 @@ jobs:
             sudo killall apt-get || true
 
             sudo apt-get update
-            sudo apt-get install -y git rng-tools memcached
+            sudo apt-get install -y git rng-tools memcached parallel
+            # avoid needing to invoke parallel with --will-cite,
+            # see e.g see https://bugs.launchpad.net/ubuntu/+source/parallel/+bug/1779764
+            mkdir -p "$HOME/.parallel" && touch "$HOME/.parallel/will-cite"
             git version
             docker version
       - restore_cache:

Makefile

@@ -5,7 +5,7 @@ SUDO := $(shell docker info > /dev/null 2> /dev/null || echo "sudo")
 
 TEST_FLAGS?=
 
-BATS_VERSION := 1.1.0
+BATS_COMMIT := 3a1c2f28be260f8687ff83183cef4963faabedd6
 SHELLCHECK_VERSION := 0.7.0
 SHFMT_VERSION := 2.6.4
 
@@ -135,12 +135,13 @@ cache/%/shfmt-$(SHFMT_VERSION):
 	mkdir -p cache/$*
 	curl --fail -L -o $@ "https://github.com/mvdan/sh/releases/download/v$(SHFMT_VERSION)/shfmt_v$(SHFMT_VERSION)_`echo $* | tr - _`"
 
-test/e2e/bats: cache/bats-v$(BATS_VERSION).tar.gz
+test/e2e/bats: cache/bats-core-$(BATS_COMMIT).tar.gz
 	mkdir -p $@
 	tar -C $@ --strip-components 1 -xzf $< 
 
-cache/bats-v$(BATS_VERSION).tar.gz:
-	curl --fail -L -o $@ https://github.com/bats-core/bats-core/archive/v$(BATS_VERSION).tar.gz
+cache/bats-core-$(BATS_COMMIT).tar.gz:
+	# Use 2opremio's fork until https://github.com/bats-core/bats-core/pull/255 is merged
+	curl --fail -L -o $@ https://github.com/2opremio/bats-core/archive/$(BATS_COMMIT).tar.gz
 
 $(GOBIN)/fluxctl: $(FLUXCTL_DEPS) $(GENERATED_TEMPLATES_FILE)
 	go install ./cmd/fluxctl

test/e2e/10_helm_chart.bats

@@ -1,10 +1,10 @@
 #!/usr/bin/env bats
 
-load lib/env
-load lib/install
-load lib/poll
-
 function setup() {
+  load lib/env
+  load lib/install
+  load lib/poll
+
   kubectl create namespace "$FLUX_NAMESPACE"
   install_git_srv
   install_tiller

test/e2e/11_fluxctl_install.bats

@@ -1,10 +1,10 @@
 #!/usr/bin/env bats
 
-load lib/env
-load lib/install
-load lib/poll
-
 function setup() {
+  load lib/env
+  load lib/install
+  load lib/poll
+
   kubectl create namespace "$FLUX_NAMESPACE"
   install_git_srv
   install_flux_with_fluxctl

test/e2e/12_sync.bats

@@ -1,13 +1,13 @@
 #!/usr/bin/env bats
 
-load lib/env
-load lib/install
-load lib/poll
-load lib/defer
-
 clone_dir=""
 
 function setup() {
+  load lib/env
+  load lib/install
+  load lib/poll
+  load lib/defer
+
   kubectl create namespace "$FLUX_NAMESPACE"
   # Install flux and the git server, allowing external access
   install_git_srv git_srv_result

test/e2e/13_sync_gc.bats

@@ -1,11 +1,11 @@
 #!/usr/bin/env bats
 
-load lib/env
-load lib/install
-load lib/poll
-load lib/defer
-
 function setup() {
+  load lib/env
+  load lib/install
+  load lib/poll
+  load lib/defer
+
   kubectl create namespace "$FLUX_NAMESPACE"
   # Install flux and the git server, allowing external access
   install_git_srv git_srv_result

test/e2e/20_commit_signing.bats

@@ -1,12 +1,12 @@
 #!/usr/bin/env bats
 
-load lib/defer
-load lib/env
-load lib/gpg
-load lib/install
-load lib/poll
-
 function setup() {
+  load lib/defer
+  load lib/env
+  load lib/gpg
+  load lib/install
+  load lib/poll
+
   kubectl create namespace "${FLUX_NAMESPACE}" &> /dev/null
 
   # Install the git server, allowing external access

test/e2e/20_commit_verification.bats

@@ -1,12 +1,12 @@
 #!/usr/bin/env bats
 
-load lib/env
-load lib/gpg
-load lib/install
-load lib/poll
-load lib/defer
-
 function setup() {
+  load lib/env
+  load lib/gpg
+  load lib/install
+  load lib/poll
+  load lib/defer
+
   kubectl create namespace "${FLUX_NAMESPACE}"
 
   # Create a temporary GNUPGHOME

test/e2e/22_manifest_generation.bats

@@ -1,11 +1,11 @@
 #!/usr/bin/env bats
 
-load lib/env
-load lib/install
-load lib/poll
-load lib/defer
-
 function setup() {
+  load lib/env
+  load lib/install
+  load lib/poll
+  load lib/defer
+
   kubectl create namespace "$FLUX_NAMESPACE"
   # Install flux and the git server, allowing external access
   install_git_srv git_srv_result "22_manifest_generation/gitsrv"

test/e2e/lib/env.bash

@@ -10,3 +10,8 @@ KNOWN_HOSTS=$(cat "${FIXTURES_DIR}/known_hosts")
 export KNOWN_HOSTS
 GITCONFIG=$(cat "${FIXTURES_DIR}/gitconfig")
 export GITCONFIG
+
+# Wire the test to the right cluster when tests are run in parallel
+if eval [ -n '$KUBECONFIG_SLOT_'"${BATS_JOB_SLOT}" ]; then
+  eval export KUBECONFIG='$KUBECONFIG_SLOT_'"${BATS_JOB_SLOT}"
+fi

test/e2e/run.bash

@@ -12,34 +12,48 @@ CACHE_DIR="${FLUX_ROOT_DIR}/cache/$CURRENT_OS_ARCH"
 
 KIND_VERSION="v0.5.1"
 KIND_CACHE_PATH="${CACHE_DIR}/kind-$KIND_VERSION"
-KIND_CLUSTER=flux-e2e
-USING_KIND=false
+KIND_CLUSTER_PREFIX=flux-e2e
+BATS_EXTRA_ARGS=""
 
 # shellcheck disable=SC1090
 source "${E2E_DIR}/lib/defer.bash"
 trap run_deferred EXIT
 
-# Check if there is a kubernetes cluster running, otherwise use Kind
-if ! kubectl version > /dev/null 2>&1; then
+function install_kind() {
   if [ ! -f "${KIND_CACHE_PATH}" ]; then
     echo '>>> Downloading Kind'
     mkdir -p "${CACHE_DIR}"
     curl -sL "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-${CURRENT_OS_ARCH}" -o "${KIND_CACHE_PATH}"
   fi
-  echo '>>> Creating Kind Kubernetes cluster'
   cp "${KIND_CACHE_PATH}" "${FLUX_ROOT_DIR}/test/bin/kind"
   chmod +x "${FLUX_ROOT_DIR}/test/bin/kind"
-  kind create cluster --name "${KIND_CLUSTER}" --wait 5m
-  defer kind --name "${KIND_CLUSTER}" delete cluster > /dev/null 2>&1 || true
-  KUBECONFIG="$(kind --name="${KIND_CLUSTER}" get kubeconfig-path)"
-  export KUBECONFIG
-  USING_KIND=true
-  kubectl get pods --all-namespaces
+}
+
+# Create multiple Kind clusters and run jobs in parallel?
+# Let users specify how many, e.g. with E2E_KIND_CLUSTER_NUM=3 make e2e
+if [ -n "$CI" ]; then
+  # Use four Kind clusters when running the tests in CircleCI
+  E2E_KIND_CLUSTER_NUM=4
 fi
+E2E_KIND_CLUSTER_NUM=${E2E_KIND_CLUSTER_NUM:-1}
+
+# Check if there is a kubernetes cluster running, otherwise use Kind
+if ! kubectl version > /dev/null 2>&1; then
+  install_kind
 
-if [ "${USING_KIND}" = 'true' ]; then
-  echo '>>> Loading images into the Kind cluster'
-  kind --name "${KIND_CLUSTER}" load docker-image 'docker.io/fluxcd/flux:latest'
+  echo '>>> Creating Kind Kubernetes cluster(s)'
+  seq 1 "${E2E_KIND_CLUSTER_NUM}" | time parallel -- kind create cluster --name "${KIND_CLUSTER_PREFIX}-{}" --wait 5m
+  for I in $(seq 1 "${E2E_KIND_CLUSTER_NUM}"); do
+    defer kind --name "${KIND_CLUSTER_PREFIX}-${I}" delete cluster > /dev/null 2>&1 || true
+    # Wire tests with the right cluster based on their BATS_JOB_SLOT env variable
+    eval export KUBECONFIG_SLOT_"${I}"="$(kind --name="${KIND_CLUSTER_PREFIX}-${I}" get kubeconfig-path)"
+  done
+
+  echo '>>> Loading images into the Kind cluster(s)'
+  seq 1 "${E2E_KIND_CLUSTER_NUM}" | time parallel -- kind --name "${KIND_CLUSTER_PREFIX}-{}" load docker-image 'docker.io/fluxcd/flux:latest'
+  if [ "${E2E_KIND_CLUSTER_NUM}" -gt 1 ]; then
+    BATS_EXTRA_ARGS="--jobs ${E2E_KIND_CLUSTER_NUM}"
+  fi
 fi
 
 echo '>>> Running the tests'
@@ -48,5 +62,5 @@ E2E_TESTS=${E2E_TESTS:-.}
 (
   cd "${E2E_DIR}"
   # shellcheck disable=SC2086
-  "${E2E_DIR}/bats/bin/bats" -t ${E2E_TESTS}
+  "${E2E_DIR}/bats/bin/bats" -t ${BATS_EXTRA_ARGS} ${E2E_TESTS}
 )