Merge pull request #2577 from fluxcd/e2e/sync-gc

End-to-end test for garbage collection

test/e2e/12_sync.bats

@@ -17,10 +17,15 @@ function setup() {
   export GIT_SSH_COMMAND="$git_ssh_cmd"
   # shellcheck disable=SC2154
   git_port_forward_pid="${git_srv_result[1]}"
+  # Teardown the created port-forward to gitsrv and restore Git settings.
+  defer kill "$git_port_forward_pid"
+
   install_flux_with_fluxctl
+
   # Clone the repo and
   clone_dir="$(mktemp -d)"
   git clone -b master ssh://git@localhost/git-server/repos/cluster.git "$clone_dir"
+  defer rm -rf "$clone_dir"
   # shellcheck disable=SC2164
   cd "$clone_dir"
 }
@@ -41,7 +46,7 @@ function setup() {
   sed -i'.bak' 's%stefanprodan/podinfo:.*%stefanprodan/podinfo:3.1.5%' "${clone_dir}/workloads/podinfo-dep.yaml"
   git -c '[email protected]' -c 'user.name=Foo' commit -am "Bump podinfo"
   head_hash=$(git rev-list -n 1 HEAD)
-  git push
+  git push >&3
   poll_until_equals "podinfo image" "stefanprodan/podinfo:3.1.5" "kubectl get pod -n demo -l app=podinfo -o\"jsonpath={['items'][0]['spec']['containers'][0]['image']}\""
   git pull -f --tags
   sync_tag_hash=$(git rev-list -n 1 flux)
@@ -81,9 +86,7 @@ function setup() {
 }
 
 function teardown() {
-  rm -rf "$clone_dir"
-  # Teardown the created port-forward to gitsrv and restore Git settings.
-  kill "$git_port_forward_pid"
+  run_deferred
   # Uninstall Flux and the global resources it installs.
   uninstall_flux_with_fluxctl
   # Removing the namespace also takes care of removing gitsrv.

test/e2e/13_sync_gc.bats

@@ -0,0 +1,61 @@
+#!/usr/bin/env bats
+
+load lib/env
+load lib/install
+load lib/poll
+load lib/defer
+
+git_port_forward_pid=""
+
+function setup() {
+  kubectl create namespace "$FLUX_NAMESPACE"
+  # Install flux and the git server, allowing external access
+  install_git_srv flux-git-deploy git_srv_result
+  # shellcheck disable=SC2154
+  git_ssh_cmd="${git_srv_result[0]}"
+  export GIT_SSH_COMMAND="$git_ssh_cmd"
+  # shellcheck disable=SC2154
+  git_port_forward_pid="${git_srv_result[1]}"
+  defer kill "$git_port_forward_pid"
+  install_flux_with_fluxctl "13_sync_gc"
+}
+
+@test "Sync with garbage collection test" {
+  # Wait until flux deploys the workloads, which indicates it has at least started a sync
+  poll_until_true 'workload podinfo' 'kubectl -n demo describe deployment/podinfo'
+
+  # make sure we have _finished_ a sync run
+  fluxctl --k8s-fwd-ns "${FLUX_NAMESPACE}" sync
+
+  # Clone the repo and check the sync tag
+  local clone_dir
+  clone_dir="$(mktemp -d)"
+  defer rm -rf "$clone_dir"
+  git clone -b master ssh://git@localhost/git-server/repos/cluster.git "$clone_dir"
+  cd "$clone_dir"
+  local sync_tag_hash
+  sync_tag_hash=$(git rev-list -n 1 flux)
+  head_hash=$(git rev-list -n 1 HEAD)
+  [ "$sync_tag_hash" = "$head_hash" ]
+
+  # Remove a manifest and commit that
+  git rm workloads/podinfo-dep.yaml
+  git -c '[email protected]' -c 'user.name=Foo' commit -m "Remove podinfo deployment"
+  head_hash=$(git rev-list -n 1 HEAD)
+  git push >&3
+
+  fluxctl --k8s-fwd-ns "${FLUX_NAMESPACE}" sync
+
+  poll_until_equals "podinfo deployment removed" "[]" "kubectl get deploy -n demo -o\"jsonpath={['items']}\""
+  git pull -f --tags >&3
+  sync_tag_hash=$(git rev-list -n 1 flux)
+  [ "$sync_tag_hash" = "$head_hash" ]
+}
+
+function teardown() {
+  run_deferred
+  # Removing the namespace also takes care of removing Flux and gitsrv.
+  kubectl delete namespace "$FLUX_NAMESPACE"
+  # Only remove the demo workloads after Flux, so that they cannot be recreated.
+  kubectl delete namespace "$DEMO_NAMESPACE"
+}

test/e2e/20_commit_signing.bats

@@ -19,10 +19,13 @@ function setup() {
   export GIT_SSH_COMMAND="$git_ssh_cmd"
   # shellcheck disable=SC2154
   git_port_forward_pid="${git_srv_result[1]}"
+  # Teardown the created port-forward to gitsrv.
+  defer kill "$git_port_forward_pid"
 
   # Create a temporary GNUPGHOME
   tmp_gnupghome=$(mktemp -d)
   export GNUPGHOME="$tmp_gnupghome"
+  defer rm -rf "$tmp_gnupghome"
 
   # Install Flux, with a new GPG key and signing enabled
   gpg_key=$(create_gpg_key)
@@ -66,11 +69,9 @@ function setup() {
 }
 
 function teardown() {
-  # Teardown the created port-forward to gitsrv.
-  kill "$git_port_forward_pid"
+  run_deferred
   # Kill the agent and remove temporary GNUPGHOME
   gpgconf --kill gpg-agent
-  rm -rf "$tmp_gnupghome"
   # Uninstall Flux and the global resources it installs.
   uninstall_flux_gpg
   # Removing the namespace also takes care of removing Flux and gitsrv.

test/e2e/fixtures/kustom/13_sync_gc/gc_patch.json

@@ -0,0 +1,3 @@
+[
+  { "op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--sync-garbage-collection" }
+]

test/e2e/fixtures/kustom/13_sync_gc/kustomization.yaml

@@ -0,0 +1,12 @@
+bases:
+- "../base"
+patchesJson6902:
+## this patch is for test-specific patches; supply a filename to
+## install_flux_with_fluxctl and it will use that rather than the
+## (empty) default.
+- target:
+    group: apps
+    version: v1
+    kind: Deployment
+    name: flux
+  path: gc_patch.json

test/e2e/fixtures/kustom/base/e2e_patch.json

@@ -0,0 +1,9 @@
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: --git-poll-interval=10s
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: --sync-interval=10s
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: --registry-exclude-image=*

test/e2e/fixtures/kustom/base/kustomization.yaml

@@ -0,0 +1,15 @@
+resources:
+- flux-account.yaml
+- flux-deployment.yaml
+- flux-secret.yaml
+- memcache-dep.yaml
+- memcache-svc.yaml
+patchesJson6902:
+# use a poll interval of 10s (to make tests quicker) and disable
+# registry polling (to avoid overloading kind)
+- target:
+    group: apps
+    version: v1
+    kind: Deployment
+    name: flux
+  path: e2e_patch.json

test/e2e/fixtures/test_patch.yaml

@@ -0,0 +1 @@
+[]

test/e2e/lib/defer.bash

@@ -1,8 +1,16 @@
 #!/usr/bin/env bash
 
+# This lets you call `defer` to record an action to do later;
+# `run_deferred` should be called in an EXIT trap, either explicitly:
+#
+#    trap run_deferred EXIT
+#
+# or when using with tests, by calling it in the teardown function
+# (which bats will arrange to run).
+
 declare -a on_exit_items
 
-function on_exit() {
+function run_deferred() {
   if [ "${#on_exit_items[@]}" -gt 0 ]; then
     echo -e '\nRunning deferred items, please do not interrupt until they are done:'
   fi
@@ -12,8 +20,6 @@ function on_exit() {
   done
 }
 
-trap on_exit EXIT
-
 function defer() {
   on_exit_items=("$*" "${on_exit_items[@]}")
 }

test/e2e/lib/install.bash

@@ -51,16 +51,25 @@ function uninstall_flux_with_helm() {
 fluxctl_install_cmd="fluxctl install --git-url=ssh://git@gitsrv/git-server/repos/cluster.git --git-email=foo"
 
 function install_flux_with_fluxctl() {
-  local eol=$'\n'
-  # Use the local Flux image instead of the latest release, use a poll interval of 10s
-  # (to make tests quicker) and disable registry polling (to avoid overloading kind)
-  $fluxctl_install_cmd --namespace "${FLUX_NAMESPACE}" |
-    sed 's%docker\.io/fluxcd/flux:.*%fluxcd/flux:latest%' |
-    sed "s%--git-email=foo%--git-email=foo\\$eol        - --git-poll-interval=10s%" |
-    sed "s%--git-email=foo%--git-email=foo\\$eol        - --sync-interval=10s%" |
-    sed "s%--git-email=foo%--git-email=foo\\$eol        - --registry-exclude-image=\*%" |
-    kubectl apply -f -
-  kubectl -n "${FLUX_NAMESPACE}" rollout status deployment/flux
+  local kustomtmp
+  kustomtmp="$(mktemp -d)"
+  defer "if [ -d \"${kustomtmp}\" ]; then rm -r \"${kustomtmp}\"; fi"
+  mkdir "${kustomtmp}/base"
+  # This generates the base manifests, which we'll then patch with a kustomization
+  echo ">>> writing base configuration to ${kustomtmp}/base" >&3
+  $fluxctl_install_cmd --namespace "${FLUX_NAMESPACE}" -o "${kustomtmp}/base/"
+  # Everything goes into one directory, but not everything is
+  # necessarily used by the kustomization
+  cp -R "${E2E_DIR}"/fixtures/kustom/* "${kustomtmp}/"
+  local kustomization
+  kustomization="base"
+  if [ -n "$1" ]; then
+    # use the kustomization given instead; ../base will still be
+    # there to be used as a base
+    kustomization="$1"
+  fi
+  kubectl apply -k "${kustomtmp}/${kustomization}" >&3
+  kubectl -n "${FLUX_NAMESPACE}" rollout status -w --timeout=30s deployment/flux
   # Add the known hosts file manually (it's much easier than editing the manifests to add a volume)
   local flux_podname
   flux_podname=$(kubectl get pod -n "${FLUX_NAMESPACE}" -l name=flux -o jsonpath="{['items'][0].metadata.name}")

test/e2e/run.bash

@@ -17,6 +17,7 @@ USING_KIND=false
 
 # shellcheck disable=SC1090
 source "${E2E_DIR}/lib/defer.bash"
+trap run_deferred EXIT
 
 # Check if there is a kubernetes cluster running, otherwise use Kind
 if ! kubectl version > /dev/null 2>&1; then