e2e: extend fluxctl tests

test/e2e/14_release_image.bats

@@ -22,7 +22,6 @@ function setup() {
   REGISTRY_PORT="${registry_result[0]}"
   # Teardown the created port-forward to the registry.
   defer kill "${registry_result[1]}"
-  echo "REGISTRY_PORT=$REGISTRY_PORT" >&3
   # create empty images for the test
   push_empty_image "localhost:$REGISTRY_PORT" 'bitnami/ghost:3.0.2-debian-9-r3' '2020-01-20T13:53:05.47178071Z'
   push_empty_image "localhost:$REGISTRY_PORT" 'bitnami/ghost:3.1.1-debian-9-r0' '2020-02-20T13:53:05.47178071Z'
@@ -53,9 +52,9 @@ function setup() {
   head_hash=$(git rev-list -n 1 HEAD)
   poll_until_equals "sync tag" "$head_hash" 'git pull -f --tags > /dev/null 2>&1; git rev-list -n 1 flux'
 
-  # Wait for the registry scanner to fo its magic on stefanprodan/podinfo and bitnami/ghost
-  poll_until_true "stefanprodan/podinfo to be scanned" "kubectl logs -n $FLUX_NAMESPACE deploy/flux | grep -q \"component=warmer updated=stefanprodan/podinfo\"" 5 50
-  poll_until_true "bitnami/ghost to be scanned" "kubectl logs -n $FLUX_NAMESPACE deploy/flux | grep -q \"component=warmer updated=bitnami/ghost\"" 5 50
+  # Wait for the registry scanner to do its magic on stefanprodan/podinfo and bitnami/ghost
+  poll_until_true "stefanprodan/podinfo to be scanned" "kubectl logs -n $FLUX_NAMESPACE deploy/flux | grep -q \"component=warmer updated=stefanprodan/podinfo\"" 50
+  poll_until_true "bitnami/ghost to be scanned" "kubectl logs -n $FLUX_NAMESPACE deploy/flux | grep -q \"component=warmer updated=bitnami/ghost\"" 50
 
   # Manually release podinfo to version 3.0.5
   fluxctl --k8s-fwd-ns "${FLUX_NAMESPACE}" deautomate --workload=demo:deployment/podinfo
@@ -81,12 +80,12 @@ function setup() {
   poll_until_true "helmrelease/ghost glob:3.1.1-debian-9-* to be released" 'git pull > /dev/null 2>&1; grep -q 3.1.1-debian-9-r0 releases/ghost.yaml'
 
   # Test `fluxctl release --update-all-images` by deautomating the podinfo deployment, pushing a newer podinfo image to the
-  # registry (matching its automation pattern) and making sure Flux the container to that image.
+  # registry (matching its automation pattern) and making sure Flux updates the podinfo container to that image.
   local time_before_new_image
   time_before_new_image="$(date -u +%Y-%m-%dT%T.0Z)"
   fluxctl --k8s-fwd-ns "${FLUX_NAMESPACE}" deautomate --workload=demo:deployment/podinfo
   push_empty_image "localhost:$REGISTRY_PORT" 'stefanprodan/podinfo:3.1.5' '2020-12-20T13:53:05.47178071Z'
-  poll_until_true "stefanprodan/podinfo to be re-scanned" "kubectl logs --since-time=${time_before_new_image} -n $FLUX_NAMESPACE deploy/flux | grep -q \"component=warmer updated=stefanprodan/podinfo\"" 5 50
+  poll_until_true "stefanprodan/podinfo to be re-scanned" "kubectl logs --since-time=${time_before_new_image} -n $FLUX_NAMESPACE deploy/flux | grep -q \"component=warmer updated=stefanprodan/podinfo\"" 50
   fluxctl --k8s-fwd-ns "${FLUX_NAMESPACE}" release --force --workload=demo:deployment/podinfo --update-all-images
   poll_until_true "deployment/podinfo version 3.1.5 to be released" 'git pull > /dev/null 2>&1; grep -q stefanprodan/podinfo:3.1.5 workloads/podinfo-dep.yaml'
 }

test/e2e/15_fluxctl_list.bats

@@ -0,0 +1,91 @@
+#!/usr/bin/env bats
+
+function setup() {
+  load lib/env
+  load lib/install
+  load lib/poll
+  load lib/defer
+  load lib/registry
+
+  kubectl create namespace "$FLUX_NAMESPACE"
+
+  # Install the git server, allowing external access
+  install_git_srv
+
+  # Install a local registry, with some empty images to be used later in the test
+  install_registry registry_result
+  # shellcheck disable=SC2154
+  REGISTRY_PORT="${registry_result[0]}"
+  # Teardown the created port-forward to the registry.
+  defer kill "${registry_result[1]}"
+  # create empty images for the test
+  push_empty_image "localhost:$REGISTRY_PORT" 'bitnami/ghost:3.0.2-debian-9-r3' '2020-01-20T13:53:05.47178071Z'
+  push_empty_image "localhost:$REGISTRY_PORT" 'bitnami/ghost:3.1.1-debian-9-r0' '2020-02-20T13:53:05.47178071Z'
+  push_empty_image "localhost:$REGISTRY_PORT" 'stefanprodan/podinfo:3.1.0' '2020-03-20T13:53:05.47178071Z'
+  push_empty_image "localhost:$REGISTRY_PORT" 'stefanprodan/podinfo:3.0.5' '2020-04-20T13:53:05.47178071Z'
+  REGISTRY_SERVICE_IP=$(kubectl -n "$FLUX_NAMESPACE" get service registry -o 'jsonpath={.spec.clusterIP}')
+
+  # Finally, install Flux
+  local -A template_values
+  # shellcheck disable=SC2034
+  template_values['REGISTRY_SERVICE_IP']="$REGISTRY_SERVICE_IP"
+  # Intentionally reuse the setup from the release_image test
+  install_flux_with_fluxctl '14_release_image' 'template_values'
+}
+
+@test "Fluxctl list-workloads and list-images" {
+  # avoid races with the image of podinfo being automatically updated
+  poll_until_true 'lock podinfo' "fluxctl --k8s-fwd-ns ${FLUX_NAMESPACE} lock --workload=demo:deployment/podinfo"
+
+  # Wait for the registry scanner to do its magic on stefanprodan/podinfo and bitnami/ghost
+  poll_until_true "stefanprodan/podinfo to be scanned" "kubectl logs -n $FLUX_NAMESPACE deploy/flux | grep -q \"component=warmer updated=stefanprodan/podinfo\"" 50
+  poll_until_true "bitnami/ghost to be scanned" "kubectl logs -n $FLUX_NAMESPACE deploy/flux | grep -q \"component=warmer updated=bitnami/ghost\"" 50
+
+  # Test fluxctl list-services
+  run fluxctl --k8s-fwd-ns "${FLUX_NAMESPACE}" list-workloads --namespace "${DEMO_NAMESPACE}"
+
+  [ "$status" -eq 0 ]
+  special_chars_output="$(echo "$output" | cat -vet)"
+  [ "$special_chars_output" = "WORKLOAD                  CONTAINER    IMAGE                            RELEASE  POLICY$
+demo:deployment/podinfo   podinfod     stefanprodan/podinfo:3.1.0       ready    automated,locked$
+                          init         alpine:3.10.1                             $
+demo:helmrelease/ghost    chart-image  bitnami/ghost:3.1.1-debian-9-r0           $
+demo:helmrelease/mongodb  chart-image  bitnami/mongodb:4.0.13                    $
+demo:helmrelease/redis    chart-image  bitnami/redis:5.0.7                       automated,locked$" ] || (
+    echo "unexpected output: $special_chars_output" >&3
+    exit 1
+  )
+
+  # Test fluxctl list-images
+  run fluxctl --k8s-fwd-ns "${FLUX_NAMESPACE}" list-images --namespace "${DEMO_NAMESPACE}"
+  [ "$status" -eq 0 ]
+  special_chars_output="$(echo "$output" | cat -vet)"
+  [ "$special_chars_output" = "WORKLOAD                  CONTAINER    IMAGE                  CREATED$
+demo:deployment/podinfo   podinfod     stefanprodan/podinfo   $
+                                       '-> 3.1.0              20 Mar 20 13:53 UTC$
+                                           3.0.5              20 Apr 20 13:53 UTC$
+                          init                                image data not available$
+                                       '-> (untagged)         ?$
+demo:helmrelease/ghost    chart-image  bitnami/ghost          $
+                                       '-> 3.1.1-debian-9-r0  20 Feb 20 13:53 UTC$
+                                           3.0.2-debian-9-r3  20 Jan 20 13:53 UTC$
+demo:helmrelease/mongodb  chart-image                         image data not available$
+                                       '-> (untagged)         ?$
+demo:helmrelease/redis    chart-image                         image data not available$
+                                       '-> (untagged)         ?$" ] || (
+    echo "unexpected output: $special_chars_output" >&3
+    exit 1
+  )
+
+}
+
+function teardown() {
+  run_deferred
+  # Although the namespace delete below takes care of removing most Flux
+  # elements, the global resources will not be removed without this.
+  uninstall_flux_with_fluxctl
+  # Removing the namespace also takes care of removing Flux and gitsrv.
+  kubectl delete namespace "$FLUX_NAMESPACE"
+  # Only remove the demo workloads after Flux, so that they cannot be recreated.
+  kubectl delete namespace "$DEMO_NAMESPACE"
+}

test/e2e/16_fluxctl_sync.bats

@@ -0,0 +1,53 @@
+#!/usr/bin/env bats
+
+clone_dir=""
+
+function setup() {
+  load lib/env
+  load lib/install
+  load lib/poll
+  load lib/defer
+
+  kubectl create namespace "$FLUX_NAMESPACE"
+  # Install flux and the git server, allowing external access
+  install_git_srv git_srv_result
+  # shellcheck disable=SC2154
+  export GIT_SSH_COMMAND="${git_srv_result[0]}"
+  # Teardown the created port-forward to gitsrv and restore Git settings.
+  defer kill "${git_srv_result[1]}"
+
+  install_flux_with_fluxctl '15_fluxctl_sync'
+
+  # Clone the repo
+  clone_dir="$(mktemp -d)"
+  defer rm -rf "'$clone_dir'"
+  git clone -b master ssh://git@localhost/git-server/repos/cluster.git "$clone_dir"
+  # shellcheck disable=SC2164
+  cd "$clone_dir"
+}
+
+@test "fluxctl sync" {
+
+  # Sync
+  poll_until_true 'fluxctl sync succeeds' "fluxctl --k8s-fwd-ns ${FLUX_NAMESPACE} sync"
+
+  # Wait until flux deploys the workloads
+  poll_until_true 'workload podinfo' 'kubectl -n demo describe deployment/podinfo'
+
+  # Check the sync tag
+  local head_hash
+  head_hash=$(git rev-list -n 1 HEAD)
+  poll_until_equals "sync tag" "$head_hash" 'git pull -f --tags > /dev/null 2>&1; git rev-list -n 1 flux'
+
+}
+
+function teardown() {
+  run_deferred
+  # Although the namespace delete below takes care of removing most Flux
+  # elements, the global resources will not be removed without this.
+  uninstall_flux_with_fluxctl
+  # Removing the namespace also takes care of removing gitsrv.
+  kubectl delete namespace "$FLUX_NAMESPACE"
+  # Only remove the demo workloads after Flux, so that they cannot be recreated.
+  kubectl delete namespace "$DEMO_NAMESPACE"
+}

test/e2e/17_fluxctl_policies.bats

@@ -0,0 +1,107 @@
+#!/usr/bin/env bats
+
+clone_dir=""
+
+function setup() {
+  load lib/env
+  load lib/install
+  load lib/poll
+  load lib/defer
+
+  kubectl create namespace "$FLUX_NAMESPACE"
+  # Install flux and the git server, allowing external access
+  install_git_srv git_srv_result
+  # shellcheck disable=SC2154
+  export GIT_SSH_COMMAND="${git_srv_result[0]}"
+  # Teardown the created port-forward to gitsrv and restore Git settings.
+  defer kill "${git_srv_result[1]}"
+
+  install_flux_with_fluxctl
+
+  # Clone the repo
+  clone_dir="$(mktemp -d)"
+  defer rm -rf "'$clone_dir'"
+  git clone -b master ssh://git@localhost/git-server/repos/cluster.git "$clone_dir"
+  # shellcheck disable=SC2164
+  cd "$clone_dir"
+}
+
+# TODO: make annotation checks more precise with a yaml-aware tool as opposed to simply grepping for precense
+#       anywhere in the files
+@test "fluxctl policy/(de)automate/(un)lock" {
+
+  # Check that podinfo is starting up in the state assumed by the test
+  grep -q 'fluxcd.io/automated: "true"' workloads/podinfo-dep.yaml # automated
+  ! grep -q 'fluxcd.io/locked' workloads/podinfo-dep.yaml          # unlocked
+  grep -q 'fluxcd.io/tag.init: regex:^3.10.*' workloads/podinfo-dep.yaml
+  grep -q 'fluxcd.io/tag.podinfod: semver:~3.1' workloads/podinfo-dep.yaml
+
+  ###########
+  ## Automate
+  ###########
+
+  # de-automate (polling since Flux may not be ready yet)
+  poll_until_true 'fluxctl deautomate' "fluxctl --k8s-fwd-ns ${FLUX_NAMESPACE} deautomate --workload=demo:deployment/podinfo"
+  git pull
+  ! grep -q 'fluxcd.io/automated' workloads/podinfo-dep.yaml
+
+  # re-automate
+  fluxctl --k8s-fwd-ns "${FLUX_NAMESPACE}" automate --workload=demo:deployment/podinfo
+  git pull
+  grep -q "fluxcd.io/automated: 'true'" workloads/podinfo-dep.yaml
+
+  # de-automate again, with the policy command
+  fluxctl --k8s-fwd-ns "${FLUX_NAMESPACE}" policy --deautomate --workload=demo:deployment/podinfo
+  git pull
+  ! grep -q 'fluxcd.io/automated' workloads/podinfo-dep.yaml
+
+  # re-automate, with the policy command
+  fluxctl --k8s-fwd-ns "${FLUX_NAMESPACE}" policy --automate --workload=demo:deployment/podinfo
+  git pull
+  grep -q "fluxcd.io/automated: 'true'" workloads/podinfo-dep.yaml
+
+  #######
+  ## Lock
+  #######
+
+  # lock
+  fluxctl --k8s-fwd-ns "${FLUX_NAMESPACE}" lock --workload=demo:deployment/podinfo
+  git pull
+  grep -q "fluxcd.io/locked: 'true'" workloads/podinfo-dep.yaml
+
+  # unlock
+  fluxctl --k8s-fwd-ns "${FLUX_NAMESPACE}" unlock --workload=demo:deployment/podinfo
+  git pull
+  ! grep -q 'fluxcd.io/locked' workloads/podinfo-dep.yaml
+
+  # re-lock, with the policy command
+  fluxctl --k8s-fwd-ns "${FLUX_NAMESPACE}" policy --lock --workload=demo:deployment/podinfo
+  git pull
+  grep -q "fluxcd.io/locked: 'true'" workloads/podinfo-dep.yaml
+
+  # unlock again, with the policy command
+  fluxctl --k8s-fwd-ns "${FLUX_NAMESPACE}" policy --unlock --workload=demo:deployment/podinfo
+  git pull
+  ! grep -q 'fluxcd.io/locked' workloads/podinfo-dep.yaml
+
+  ##############
+  ## Policy tags
+  ##############
+
+  # Update podinfo tag
+  fluxctl --k8s-fwd-ns "${FLUX_NAMESPACE}" policy --workload=demo:deployment/podinfo --tag='podinfod=3.5.*'
+  git pull
+  grep -q "fluxcd.io/tag.podinfod: glob:3.5.*" workloads/podinfo-dep.yaml
+
+}
+
+function teardown() {
+  run_deferred
+  # Although the namespace delete below takes care of removing most Flux
+  # elements, the global resources will not be removed without this.
+  uninstall_flux_with_fluxctl
+  # Removing the namespace also takes care of removing gitsrv.
+  kubectl delete namespace "$FLUX_NAMESPACE"
+  # Only remove the demo workloads after Flux, so that they cannot be recreated.
+  kubectl delete namespace "$DEMO_NAMESPACE"
+}

test/e2e/fixtures/kustom/15_fluxctl_sync/fluxctl_sync.yaml

@@ -0,0 +1,7 @@
+# make sure automatic syncs don't kick in
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: --sync-interval=525600m
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: --git-poll-interval=525600m

test/e2e/fixtures/kustom/15_fluxctl_sync/kustomization.yaml

@@ -0,0 +1,9 @@
+bases:
+- "../base/flux"
+patchesJson6902:
+- target:
+    group: apps
+    version: v1
+    kind: Deployment
+    name: flux
+  path: fluxctl_sync.yaml