Adding a job to run extended tests (only on main) (microsoft#1322)

* Adding a job to run extended tests (only on main) * add missing devcontainer tag Co-authored-by: Ross Smith <[email protected]>
vjmanda · Feb 18, 2022 · 2f462c4 · 2f462c4
1 parent c22a6f6
commit 2f462c4
Show file tree

Hide file tree

Showing 3 changed files with 119 additions and 13 deletions.
diff --git a/.github/workflows/deploy_tre_reusable.yml b/.github/workflows/deploy_tre_reusable.yml
@@ -369,19 +369,19 @@ jobs:
           LOCATION: "${{ secrets.LOCATION }}"
           BUNDLE_TYPE: ${{ matrix.BUNDLE_TYPE }}
 
-  e2e_tests:
-    name: "Run E2E Tests"
+  e2e_tests_smoke:
+    name: "Run E2E Tests (Smoke)"
     runs-on: ubuntu-latest
     environment: Dev
     needs: [register_bundles, build_additional_images]
     steps:
       - name: Checkout
         uses: actions/checkout@v2
 
-      - name: Run e2e Tests
+      - name: Run E2E Tests (Smoke)
         uses: ./.github/actions/devcontainer_run_command
         with:
-          COMMAND: "make test-e2e"
+          COMMAND: "make test-e2e-smoke"
           ACTIONS_ACR_NAME: ${{ secrets.ACTIONS_ACR_NAME }}
           ACTIONS_ACR_URI: ${{ secrets.ACTIONS_ACR_URI }}
           ACTIONS_ACR_PASSWORD: ${{ secrets.ACTIONS_ACR_PASSWORD }}
@@ -407,7 +407,7 @@ jobs:
         with:
           github-token: ${{ github.token }}
           ms-teams-webhook-uri: ${{ secrets.MS_TEAMS_WEBHOOK_URI }}
-          notification-summary: "E2E Tests failed"
+          notification-summary: "E2E Tests (Smoke) failed"
           notification-color: dc3545
           timezone: Europe/Zurich
 
@@ -427,13 +427,80 @@ jobs:
         uses: actions/upload-artifact@v2
         with:
           name: E2E Test Results (Python ${{ matrix.python-version }})
-          path: "./e2e_tests/pytest_e2e.xml"
+          path: "./e2e_tests/pytest_e2e_smoke.xml"
 
       - name: Publish Test Results
         if: always()
         uses: EnricoMi/publish-unit-test-result-action@v1
         with:
-          files: "./e2e_tests/pytest_e2e.xml"
+          files: "./e2e_tests/pytest_e2e_smoke.xml"
+
+  e2e_tests_extended:
+    name: "Run E2E Tests (Extended)"
+    if: ${{ github.ref == 'refs/heads/main' }}
+    runs-on: ubuntu-latest
+    environment: Dev
+    needs: [register_bundles, build_additional_images]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Run E2E Tests (Extended)
+        uses: ./.github/actions/devcontainer_run_command
+        with:
+          COMMAND: "make test-e2e-extended"
+          ACTIONS_ACR_NAME: ${{ secrets.ACTIONS_ACR_NAME }}
+          ACTIONS_ACR_URI: ${{ secrets.ACTIONS_ACR_URI }}
+          ACTIONS_ACR_PASSWORD: ${{ secrets.ACTIONS_ACR_PASSWORD }}
+          ACTIONS_DEVCONTAINER_TAG: ${{ secrets.ACTIONS_DEVCONTAINER_TAG }}
+          ARM_TENANT_ID: "${{ secrets.ARM_TENANT_ID }}"
+          ARM_CLIENT_ID: "${{ secrets.ARM_CLIENT_ID }}"
+          ARM_CLIENT_SECRET: "${{ secrets.ARM_CLIENT_SECRET }}"
+          ARM_SUBSCRIPTION_ID: "${{ secrets.ARM_SUBSCRIPTION_ID }}"
+          LOCATION: "${{ secrets.LOCATION }}"
+          RESOURCE: "${{ secrets.API_CLIENT_ID }}"
+          AUTH_TENANT_ID: "${{ secrets.AAD_TENANT_ID }}"
+          TEST_APP_ID: "${{ secrets.TEST_APP_ID }}"
+          TEST_USER_NAME: "${{ secrets.TEST_USER_NAME }}"
+          TEST_USER_PASSWORD: "${{ secrets.TEST_USER_PASSWORD }}"
+          TEST_WORKSPACE_APP_ID: "${{ secrets.TEST_WORKSPACE_APP_ID }}"
+          TRE_ID: "${{ secrets.TRE_ID }}"
+          IS_API_SECURED: false
+
+      - name: Notify dedicated teams channel
+        uses: sachinkundu/[email protected]
+        if: ${{ failure() && github.ref == 'refs/heads/main'}}
+        continue-on-error: true
+        with:
+          github-token: ${{ github.token }}
+          ms-teams-webhook-uri: ${{ secrets.MS_TEAMS_WEBHOOK_URI }}
+          notification-summary: "E2E Tests (Extended) failed"
+          notification-color: dc3545
+          timezone: Europe/Zurich
+
+      - name: Notify dedicated teams channel
+        uses: sachinkundu/[email protected]
+        if: ${{ success() && github.ref == 'refs/heads/main' }}
+        continue-on-error: true
+        with:
+          github-token: ${{ github.token }}
+          ms-teams-webhook-uri: ${{ secrets.MS_TEAMS_WEBHOOK_URI }}
+          notification-summary: "Deployment and tests passed successfully"
+          notification-color: 28a745
+          timezone: Europe/Zurich
+
+      - name: Upload Test Results
+        if: always()
+        uses: actions/upload-artifact@v2
+        with:
+          name: E2E Test (Extended) Results (Python ${{ matrix.python-version }})
+          path: "./e2e_tests/pytest_e2e_extended.xml"
+
+      - name: Publish Test Results
+        if: always()
+        uses: EnricoMi/publish-unit-test-result-action@v1
+        with:
+          files: "./e2e_tests/pytest_e2e_extended.xml"
 
   # Waiting for: https://github.com/microsoft/AzureTRE/issues/1165
   # tre_stop:

diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
@@ -6,21 +6,53 @@ on:
   workflow_dispatch:
 
 jobs:
-  e2e_tests:
-    name: "Run E2E Tests"
+  e2e_tests-smoke:
+    name: "Run E2E Tests (Smoke)"
     runs-on: ubuntu-latest
     environment: Dev
     steps:
       - name: Checkout
         uses: actions/checkout@v2
 
-      - name: Run e2e Tests
+      - name: Run E2E Tests
         uses: ./.github/actions/devcontainer_run_command
         with:
-          DISPLAY_NAME: "Run e2e Tests"
-          COMMAND: "make test-e2e"
+          DISPLAY_NAME: "Run E2E Tests (Smoke)"
+          COMMAND: "make test-e2e-smoke"
           ACTIONS_ACR_NAME: ${{ secrets.ACTIONS_ACR_NAME }}
           ACTIONS_ACR_PASSWORD: ${{ secrets.ACTIONS_ACR_PASSWORD }}
+          ACTIONS_DEVCONTAINER_TAG: ${{ secrets.ACTIONS_DEVCONTAINER_TAG }}
+          ARM_TENANT_ID: "${{ secrets.ARM_TENANT_ID }}"
+          ARM_CLIENT_ID: "${{ secrets.ARM_CLIENT_ID }}"
+          ARM_CLIENT_SECRET: "${{ secrets.ARM_CLIENT_SECRET }}"
+          ARM_SUBSCRIPTION_ID: "${{ secrets.ARM_SUBSCRIPTION_ID }}"
+          LOCATION: "${{ secrets.LOCATION }}"
+          RESOURCE: "${{ secrets.API_CLIENT_ID }}"
+          AUTH_TENANT_ID: "${{ secrets.AAD_TENANT_ID }}"
+          TEST_APP_ID: "${{ secrets.TEST_APP_ID }}"
+          TEST_USER_NAME: "${{ secrets.TEST_USER_NAME }}"
+          TEST_USER_PASSWORD: "${{ secrets.TEST_USER_PASSWORD }}"
+          TEST_WORKSPACE_APP_ID: "${{ secrets.TEST_WORKSPACE_APP_ID }}"
+          TRE_ID: "${{ secrets.TRE_ID }}"
+          IS_API_SECURED: "false"
+
+  e2e_tests-extended:
+    name: "Run E2E Tests (Extended)"
+    if: ${{ github.ref == 'refs/heads/main' }}
+    runs-on: ubuntu-latest
+    environment: Dev
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Run E2E Tests (Extended)
+        uses: ./.github/actions/devcontainer_run_command
+        with:
+          DISPLAY_NAME: "Run E2E Tests (Extended)"
+          COMMAND: "make test-e2e-extended"
+          ACTIONS_ACR_NAME: ${{ secrets.ACTIONS_ACR_NAME }}
+          ACTIONS_ACR_PASSWORD: ${{ secrets.ACTIONS_ACR_PASSWORD }}
+          ACTIONS_DEVCONTAINER_TAG: ${{ secrets.ACTIONS_DEVCONTAINER_TAG }}
           ARM_TENANT_ID: "${{ secrets.ARM_TENANT_ID }}"
           ARM_CLIENT_ID: "${{ secrets.ARM_CLIENT_ID }}"
           ARM_CLIENT_SECRET: "${{ secrets.ARM_CLIENT_SECRET }}"

diff --git a/Makefile b/Makefile
@@ -232,13 +232,20 @@ static-web-upload:
 	&& . ./devops/scripts/load_env.sh ./templates/core/tre.env \
 	&& ./templates/core/terraform/scripts/upload_static_web.sh
 
-test-e2e:
+test-e2e-smoke:
 	$(call target_title, "Running E2E smoke tests") && \
 	export SCOPE="api://${RESOURCE}/user_impersonation" && \
 	export WORKSPACE_SCOPE="api://${TEST_WORKSPACE_APP_ID}/user_impersonation" && \
 	cd e2e_tests && \
 	python -m pytest -m smoke --verify $${IS_API_SECURED:-true} --junit-xml pytest_e2e.xml
 
+test-e2e-extended:
+	$(call target_title, "Running E2E smoke tests") && \
+	export SCOPE="api://${RESOURCE}/user_impersonation" && \
+	export WORKSPACE_SCOPE="api://${TEST_WORKSPACE_APP_ID}/user_impersonation" && \
+	cd e2e_tests && \
+	python -m pytest -m extended --verify $${IS_API_SECURED:-true} --junit-xml pytest_e2e.xml
+
 setup-local-debugging-api:
 	$(call target_title,"Setting up the ability to debug the API") \
 	&& . ./devops/scripts/check_dependencies.sh nodocker \