Refactor e2e-test (#56)

.github/workflows/e2e-test.yaml

@@ -30,13 +30,15 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - run: echo "${{ secrets.CR_PAT }}" | docker login ghcr.io -u "$GITHUB_ACTOR" --password-stdin
-      - run: make -C e2e_test ci-setup
-      - run: make -C e2e_test -j3 deploy
+      - run: git config user.name ci
+      - run: git config user.email [email protected]
+      - run: make -C e2e_test setup
+      - run: make -C e2e_test -j4 deploy
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - run: make -C e2e_test test
+      - run: make -C e2e_test -j4 test
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           GITHUB_PR_NUMBER: ${{ github.event.pull_request.number }}
-      - run: make -C e2e_test ci-cleanup
+      - run: make -C e2e_test cleanup
         if: always()

e2e_test/Makefile

@@ -3,41 +3,41 @@ KUBECONFIG := output/kubeconfig.yaml
 export KUBECONFIG
 
 GITHUB_RUN_ID ?= 0
-MAIN_BRANCH_NAME := e2e-test-$(GITHUB_RUN_ID)-main
+FIXTURE_BRANCH := e2e-test-$(GITHUB_RUN_ID)
 
 GITHUB_PR_NUMBER ?= 0
 
 all:
 
-test: testcase1 testcase2
-
-.NOTPARALLEL: testcase1 testcase2
-testcase1:
-	# create a pull request
-	git checkout -b $(MAIN_BRANCH_NAME)-fixture1
-	sed -i -e 's/name: echoserver/name: echoserver-fixture1/g' helloworld/deployment/echoserver.yaml
-	git commit -a -m "e2e-test-fixture1"
-	git push origin $(MAIN_BRANCH_NAME)-fixture1
-	gh pr create --base $(MAIN_BRANCH_NAME) --title "e2e-test: sync success" --body "This is created by e2e-test of #$(GITHUB_PR_NUMBER)"
-	gh pr merge --squash
+test:
 	# wait for sync
-	git checkout $(MAIN_BRANCH_NAME)
-	git pull origin --ff-only $(MAIN_BRANCH_NAME)
-	./wait-for-sync-status.sh helloworld Synced
-	kubectl -n helloworld rollout status deployment echoserver-fixture1
-
-testcase2:
-	# create a pull request
-	git checkout -b $(MAIN_BRANCH_NAME)-fixture2
-	sed -i -e 's/app: echoserver/app: echoserver-fixture2/g' helloworld/deployment/echoserver.yaml
-	git commit -a -m "e2e-test-fixture2"
-	git push origin $(MAIN_BRANCH_NAME)-fixture2
-	gh pr create --base $(MAIN_BRANCH_NAME) --title "e2e-test: sync failure" --body "This is created by e2e-test of #$(GITHUB_PR_NUMBER)"
+	./wait-for-sync-status.sh test1-fixture $(FIXTURE_BRANCH)-test1-main Synced Succeeded
+	./wait-for-sync-status.sh test2-fixture $(FIXTURE_BRANCH)-test2-main Synced Succeeded
+	# test1 (sync success)
+	git checkout $(FIXTURE_BRANCH)-test1-main
+	git checkout -b $(FIXTURE_BRANCH)-test1-topic
+	sed -i -e 's/name: echoserver/name: echoserver-test1/g' test1-fixture/deployment/echoserver.yaml
+	git commit -a -m $(FIXTURE_BRANCH)-test1-topic
+	git push origin $(FIXTURE_BRANCH)-test1-topic
+	gh pr create --base $(FIXTURE_BRANCH)-test1-main --title "e2e-test: test1" --body "This is created by e2e-test of #$(GITHUB_PR_NUMBER)"
+	gh pr merge --squash
+	git checkout $(FIXTURE_BRANCH)-test1-main
+	git pull origin --ff-only $(FIXTURE_BRANCH)-test1-main
+	git branch -D $(FIXTURE_BRANCH)-test1-topic
+	# test2 (sync failure)
+	git checkout $(FIXTURE_BRANCH)-test2-main
+	git checkout -b $(FIXTURE_BRANCH)-test2-topic
+	sed -i -e 's/app: echoserver/app: echoserver-test2/g' test2-fixture/deployment/echoserver.yaml
+	git commit -a -m $(FIXTURE_BRANCH)-test2-topic
+	git push origin $(FIXTURE_BRANCH)-test2-topic
+	gh pr create --base $(FIXTURE_BRANCH)-test2-main --title "e2e-test: test2" --body "This is created by e2e-test of #$(GITHUB_PR_NUMBER)"
 	gh pr merge --squash
+	git checkout $(FIXTURE_BRANCH)-test2-main
+	git pull origin --ff-only $(FIXTURE_BRANCH)-test2-main
+	git branch -D $(FIXTURE_BRANCH)-test2-topic
 	# wait for sync
-	git checkout $(MAIN_BRANCH_NAME)
-	git pull origin --ff-only $(MAIN_BRANCH_NAME)
-	./wait-for-sync-status.sh helloworld OutOfSync
+	./wait-for-sync-status.sh test1-fixture $(FIXTURE_BRANCH)-test1-main Synced Succeeded
+	./wait-for-sync-status.sh test2-fixture $(FIXTURE_BRANCH)-test2-main OutOfSync Failed
 
 # environment
 cluster: $(KUBECONFIG)
@@ -52,11 +52,8 @@ deploy: deploy-argocd deploy-controller
 
 deploy-argocd: cluster
 	kustomize build argocd | kubectl apply -f -
-	git branch $(MAIN_BRANCH_NAME)
-	git push origin $(MAIN_BRANCH_NAME)
-	kustomize build applications | sed -e "s/MAIN_BRANCH_NAME/$(MAIN_BRANCH_NAME)/g" | kubectl apply -f -
+	kustomize build applications | sed -e "s/FIXTURE_BRANCH/$(FIXTURE_BRANCH)/g" | kubectl apply -f -
 	kubectl -n argocd rollout status statefulsets argocd-application-controller
-	./wait-for-sync-status.sh helloworld Synced
 
 deploy-controller: cluster build-controller
 	kind load docker-image controller:latest --name $(CLUSTER_NAME)
@@ -70,13 +67,20 @@ build-controller:
 undeploy:
 	kubectl delete namespace/argocd namespace/argocd-commenter-system
 
-# CI specific tasks
-ci-setup:
-	git config user.name ci
-	git config user.email [email protected]
+setup:
+	git branch -f $(FIXTURE_BRANCH)-test1-main
+	git branch -f $(FIXTURE_BRANCH)-test2-main
+	git push origin -f $(FIXTURE_BRANCH)-test1-main $(FIXTURE_BRANCH)-test2-main
 
-ci-cleanup:
-	-git push origin --delete $(MAIN_BRANCH_NAME)
-	-git push origin --delete $(MAIN_BRANCH_NAME)-fixture1
-	-git push origin --delete $(MAIN_BRANCH_NAME)-fixture2
+cleanup:
+	-git branch -D \
+		$(FIXTURE_BRANCH)-test1-main \
+		$(FIXTURE_BRANCH)-test1-topic \
+		$(FIXTURE_BRANCH)-test2-main \
+		$(FIXTURE_BRANCH)-test2-topic
+	-git push origin --delete \
+		$(FIXTURE_BRANCH)-test1-main \
+		$(FIXTURE_BRANCH)-test1-topic \
+		$(FIXTURE_BRANCH)-test2-main \
+		$(FIXTURE_BRANCH)-test2-topic
 	-kubectl -n argocd-commenter-system logs -l control-plane=controller-manager --all-containers

e2e_test/README.md

@@ -0,0 +1,19 @@
+# end-to-end test
+
+To run locally:
+
+```sh
+# set up a cluster and deploy argocd
+make deploy-argocd
+
+# set up main branches
+make setup
+
+# run the controller
+
+# create pull requests
+make test
+
+# clean up branches
+make cleanup
+```

e2e_test/applications/kustomization.yaml

@@ -1,2 +1,3 @@
 resources:
-  - helloworld.yaml
+  - test1-fixture.yaml
+  - test2-fixture.yaml

e2e_test/applications/helloworld.yaml → e2e_test/applications/test1-fixture.yaml

@@ -1,17 +1,17 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
-  name: helloworld
+  name: test1-fixture
   namespace: argocd
 spec:
   project: default
   source:
     repoURL: https://github.com/int128/argocd-commenter
-    targetRevision: MAIN_BRANCH_NAME
-    path: e2e_test/helloworld
+    targetRevision: FIXTURE_BRANCH-test1-main
+    path: e2e_test/test1-fixture
   destination:
     server: https://kubernetes.default.svc
-    namespace: helloworld
+    namespace: test1-fixture
   syncPolicy:
     automated:
       prune: true

e2e_test/applications/test2-fixture.yaml

@@ -0,0 +1,20 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: test2-fixture
+  namespace: argocd
+spec:
+  project: default
+  source:
+    repoURL: https://github.com/int128/argocd-commenter
+    targetRevision: FIXTURE_BRANCH-test2-main
+    path: e2e_test/test2-fixture
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: test2-fixture
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    retry:
+      limit: 1  # reduce test time

e2e_test/test1-fixture/kustomization.yaml

@@ -0,0 +1,4 @@
+namespace: test1-fixture
+resources:
+  - namespace/test1-fixture.yaml
+  - deployment/echoserver.yaml

e2e_test/helloworld/namespace/helloworld.yaml → e2e_test/test1-fixture/namespace/test1-fixture.yaml

@@ -1,4 +1,4 @@
 apiVersion: v1
 kind: Namespace
 metadata:
-  name: helloworld
+  name: test1-fixture

e2e_test/test2-fixture/deployment/echoserver.yaml

@@ -0,0 +1,19 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: echoserver
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: echoserver
+  template:
+    metadata:
+      labels:
+        app: echoserver
+    spec:
+      containers:
+        - image: gcr.io/google_containers/echoserver:1.8
+          name: echoserver
+          ports:
+            - containerPort: 8080

e2e_test/test2-fixture/kustomization.yaml

@@ -0,0 +1,4 @@
+namespace: test2-fixture
+resources:
+  - namespace/test2-fixture.yaml
+  - deployment/echoserver.yaml

e2e_test/test2-fixture/namespace/test2-fixture.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: test2-fixture

e2e_test/wait-for-sync-status.sh

@@ -3,14 +3,17 @@ set -o pipefail
 set -eu
 
 application_name="$1"
-want_revision="$(git rev-parse HEAD)"
-want_status="$2"
+want_branch="$2"
+want_revision="$(git rev-parse "$want_branch")"
+want_sync_status="$3"
+want_phase="$4"
 
-for (( i = 0; i < 10; i++ )); do
-  status="$(kubectl -n argocd get application "$application_name" '-ojsonpath={.status.sync.status}/{.status.sync.revision}')"
-  echo "[wait-for-sync-status] got:  $status"
-  echo "[wait-for-sync-status] want: $want_status/$want_revision"
-  if [[ $status == $want_status/$want_revision ]]; then
+for (( i = 0; i < 20; i++ )); do
+  got="$(kubectl -n argocd get application "$application_name" '-ojsonpath={.status.sync.status}/{.status.operationState.phase}/{.status.sync.revision}')"
+  want="$want_sync_status/$want_phase/$want_revision"
+  echo "[wait-for-sync-status] got:  $got"
+  echo "[wait-for-sync-status] want: $want"
+  if [[ $got == $want ]]; then
     exit 0
   fi
   echo "[wait-for-sync-status] retry after ${i}s"