fix(ci): reduce smoke test run time (#6841)

.github/workflows/docker-unified.yml

@@ -436,6 +436,14 @@ jobs:
   smoke_test:
     name: Run Smoke Tests
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        test_strategy: [
+          "no_cypress", 
+          "cypress_suite1",
+          "cypress_rest"
+        ]
     needs:
       [
         setup,
@@ -458,6 +466,14 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: "3.7"
+      - name: Pull images in background
+        run: |
+          docker pull acryldata/datahub-actions:head &
+          docker pull confluentinc/cp-kafka:7.2.0 &
+          docker pull elasticsearch:7.9.3 &
+          docker pull mariadb:10.5.8 &
+          docker pull confluentinc/cp-schema-registry:7.2.0 &
+          docker pull confluentinc/cp-zookeeper:7.2.0 &
       - name: Install dependencies
         run: ./metadata-ingestion/scripts/install_deps.sh
       - name: Build datahub cli
@@ -493,23 +509,35 @@ jobs:
         if: ${{ needs.setup.outputs.publish != 'true' }}
         with:
           image: ${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
-      - name: Download MCE Consumer image
+      - name: Download MAE Consumer image
         uses: ishworkh/docker-image-artifact-download@v1
         if: ${{ needs.setup.outputs.publish != 'true' }}
         with:
           image: ${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
+      - name: check background image download status
+        run: |
+          jobs
+      - name: run quickstart
+        env:
+          DATAHUB_TELEMETRY_ENABLED: false
+          DATAHUB_VERSION: ${{ needs.setup.outputs.unique_tag }}
+        run: |
+          ./smoke-test/run-quickstart.sh
       - name: Smoke test
         env:
+          RUN_QUICKSTART: false
           DATAHUB_VERSION: ${{ needs.setup.outputs.unique_tag }}
           CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
+          CLEANUP_DATA: "false"
+          TEST_STRATEGY: ${{ matrix.test_strategy }}
         run: |
           echo "$DATAHUB_VERSION"
           ./smoke-test/smoke.sh
       - name: store logs
         if: failure()
         run: |
           docker ps -a
-          docker logs datahub-gms >& gms.log
+          docker logs datahub-gms >& gms-${{ matrix.test_strategy }}.log
       - name: Upload logs
         uses: actions/upload-artifact@v3
         if: failure()
@@ -520,12 +548,12 @@ jobs:
         uses: actions/upload-artifact@v3
         if: failure()
         with:
-          name: cypress-snapshots
+          name: cypress-snapshots-${{ matrix.test_strategy }}
           path: smoke-test/tests/cypress/cypress/screenshots/
       - uses: actions/upload-artifact@v3
         if: always()
         with:
-          name: Test Results (smoke tests)
+          name: Test Results (smoke tests) ${{ matrix.test_strategy }}
           path: |
             **/build/reports/tests/test/**
             **/build/test-results/test/**

smoke-test/run-quickstart.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+set -euxo pipefail
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+cd "$DIR"
+
+python3 -m venv venv
+source venv/bin/activate
+pip install --upgrade pip wheel setuptools
+pip install -r requirements.txt
+
+mkdir -p ~/.datahub/plugins/frontend/auth/
+echo "test_user:test_pass" >> ~/.datahub/plugins/frontend/auth/user.props
+
+echo "DATAHUB_VERSION = $DATAHUB_VERSION"
+DATAHUB_TELEMETRY_ENABLED=false  \
+DOCKER_COMPOSE_BASE="file://$( dirname "$DIR" )" \
+
+datahub docker quickstart --standalone_consumers --dump-logs-on-failure

smoke-test/smoke.sh

@@ -14,22 +14,23 @@ set -euxo pipefail
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 cd "$DIR"
 
-python3 -m venv venv
-source venv/bin/activate
-pip install --upgrade pip wheel setuptools
-pip install -r requirements.txt
-
-mkdir -p ~/.datahub/plugins/frontend/auth/
-echo "test_user:test_pass" >> ~/.datahub/plugins/frontend/auth/user.props
+if [ "${RUN_QUICKSTART:-true}" == "true" ]; then
+    source ./run-quickstart.sh
+fi
 
-echo "DATAHUB_VERSION = $DATAHUB_VERSION"
-DATAHUB_TELEMETRY_ENABLED=false  \
-DOCKER_COMPOSE_BASE="file://$( dirname "$DIR" )" \
-datahub docker quickstart --standalone_consumers --dump-logs-on-failure
+source venv/bin/activate
 
 (cd ..; ./gradlew :smoke-test:yarnInstall)
 
 export CYPRESS_ADMIN_USERNAME=${ADMIN_USERNAME:-datahub}
 export CYPRESS_ADMIN_PASSWORD=${ADMIN_PASSWORD:-datahub}
 
-pytest -rP --durations=20 -vv --continue-on-collection-errors --junit-xml=junit.smoke.xml
+if [[ -z "${TEST_STRATEGY}" ]]; then
+    pytest -rP --durations=20 -vv --continue-on-collection-errors --junit-xml=junit.smoke.xml
+else
+    if [ "$TEST_STRATEGY" == "no_cypress" ]; then
+        pytest -rP --durations=20 -vv --continue-on-collection-errors --junit-xml=junit.smoke_non_cypress.xml -k 'not test_run_cypress'
+    else
+        pytest -rP --durations=20 -vv --continue-on-collection-errors --junit-xml=junit.smoke_cypress_${TEST_STRATEGY}.xml tests/cypress/integration_test.py
+    fi
+fi

smoke-test/test_e2e.py

@@ -575,10 +575,10 @@ def test_ingest_with_system_metadata():
                             {
                                 "com.linkedin.identity.CorpUserInfo": {
                                     "active": True,
-                                    "displayName": "Data Hub",
+                                    "displayName": "DataHub",
                                     "email": "[email protected]",
                                     "title": "CEO",
-                                    "fullName": "Data Hub",
+                                    "fullName": "DataHub",
                                 }
                             }
                         ],
@@ -608,10 +608,10 @@ def test_ingest_with_blank_system_metadata():
                             {
                                 "com.linkedin.identity.CorpUserInfo": {
                                     "active": True,
-                                    "displayName": "Data Hub",
+                                    "displayName": "DataHub",
                                     "email": "[email protected]",
                                     "title": "CEO",
-                                    "fullName": "Data Hub",
+                                    "fullName": "DataHub",
                                 }
                             }
                         ],
@@ -638,10 +638,10 @@ def test_ingest_without_system_metadata():
                             {
                                 "com.linkedin.identity.CorpUserInfo": {
                                     "active": True,
-                                    "displayName": "Data Hub",
+                                    "displayName": "DataHub",
                                     "email": "[email protected]",
                                     "title": "CEO",
-                                    "fullName": "Data Hub",
+                                    "fullName": "DataHub",
                                 }
                             }
                         ],

smoke-test/tests/cypress/cypress.json

@@ -6,7 +6,8 @@
   "projectId": "hkrxk5",
   "defaultCommandTimeout": 10000,
   "retries": {
-    "runMode": 1,
+    "runMode": 2,
     "openMode": 0
-  }
+  },
+  "video": false
 }

smoke-test/tests/cypress/cypress/integration/analytics/analytics.js

@@ -2,8 +2,6 @@ describe('analytics', () => {
   it('can go to a chart and see analytics in Section Views', () => {
     cy.login();
 
-    cy.goToAnalytics();
-
     cy.goToChart("urn:li:chart:(looker,cypress_baz1)");
     cy.waitTextVisible("Baz Chart 1");
     cy.openEntityTab("Dashboards");

smoke-test/tests/cypress/cypress/integration/login/login.js

@@ -4,6 +4,6 @@ describe('login', () => {
     cy.get('input[data-testid=username]').type('datahub');
     cy.get('input[data-testid=password]').type('datahub');
     cy.contains('Sign In').click();
-    cy.contains('Welcome back, Data Hub');
+    cy.contains('Welcome back, DataHub');
   });
 })

smoke-test/tests/cypress/cypress/plugins/index.js

@@ -19,4 +19,5 @@
 module.exports = (on, config) => {
   // `on` is used to hook into various events Cypress emits
   // `config` is the resolved Cypress config
+  require('cypress-timestamps/plugin')(on);
 }

smoke-test/tests/cypress/cypress/support/commands.js

@@ -90,7 +90,7 @@ Cypress.Commands.add("goToDomain", (urn) => {
 
 Cypress.Commands.add("goToAnalytics", () => {
   cy.visit("/analytics");
-  cy.waitTextVisible("Data Landscape Summary");
+  cy.contains("Data Landscape Summary", {timeout: 10000});
 });
 
 Cypress.Commands.add("goToUserList", () => {

smoke-test/tests/cypress/cypress/support/index.js

@@ -17,4 +17,11 @@
 import './commands'
 
 // Alternatively you can use CommonJS syntax:
-// require('./commands')
+// require('./commands')
+
+// https://github.com/bahmutov/cypress-timestamps
+require('cypress-timestamps/support')({
+    terminal: true, // by default the terminal output is disabled
+    error: true,
+    commandLog: true,
+});

smoke-test/tests/cypress/integration_test.py

@@ -1,5 +1,6 @@
-from typing import List
+from typing import Set, List
 
+import datetime
 import pytest
 import subprocess
 import os
@@ -99,48 +100,84 @@
     ONBOARDING_IDS.extend(id_list)
 
 
+def print_now():
+    print(f"current time is {datetime.datetime.now(datetime.timezone.utc)}")
+
+
 def ingest_data():
+    print_now()
     print("creating onboarding data file")
     create_datahub_step_state_aspects(
         get_admin_username(),
         ONBOARDING_IDS,
         f"{CYPRESS_TEST_DATA_DIR}/{TEST_ONBOARDING_DATA_FILENAME}",
     )
 
+    print_now()
     print("ingesting test data")
     ingest_file_via_rest(f"{CYPRESS_TEST_DATA_DIR}/{TEST_DATA_FILENAME}")
     ingest_file_via_rest(f"{CYPRESS_TEST_DATA_DIR}/{TEST_DBT_DATA_FILENAME}")
     ingest_file_via_rest(f"{CYPRESS_TEST_DATA_DIR}/{TEST_SCHEMA_BLAME_DATA_FILENAME}")
     ingest_file_via_rest(f"{CYPRESS_TEST_DATA_DIR}/{TEST_ONBOARDING_DATA_FILENAME}")
+    print_now()
+    print("completed ingesting test data")
 
 
 @pytest.fixture(scope="module", autouse=True)
 def ingest_cleanup_data():
     ingest_data()
     yield
+    print_now()
     print("removing test data")
     delete_urns_from_file(f"{CYPRESS_TEST_DATA_DIR}/{TEST_DATA_FILENAME}")
     delete_urns_from_file(f"{CYPRESS_TEST_DATA_DIR}/{TEST_DBT_DATA_FILENAME}")
     delete_urns_from_file(f"{CYPRESS_TEST_DATA_DIR}/{TEST_SCHEMA_BLAME_DATA_FILENAME}")
     delete_urns_from_file(f"{CYPRESS_TEST_DATA_DIR}/{TEST_ONBOARDING_DATA_FILENAME}")
 
+    print_now()
     print("deleting onboarding data file")
     if os.path.exists(f"{CYPRESS_TEST_DATA_DIR}/{TEST_ONBOARDING_DATA_FILENAME}"):
         os.remove(f"{CYPRESS_TEST_DATA_DIR}/{TEST_ONBOARDING_DATA_FILENAME}")
+    print_now()
+    print("deleted onboarding data")
+
+
+def _get_spec_map(items: Set[str]) -> str:
+    if len(items) == 0:
+        return ""
+    return ",".join([f"**/{item}/*.js" for item in items])
 
 
 def test_run_cypress(frontend_session, wait_for_healthchecks):
     # Run with --record option only if CYPRESS_RECORD_KEY is non-empty
     record_key = os.getenv("CYPRESS_RECORD_KEY")
-    if record_key:
-        print("Running Cypress tests with recording")
-        command = "NO_COLOR=1 npx cypress run --record"
+    tag_arg = ""
+    test_strategy = os.getenv("TEST_STRATEGY", None)
+    if record_key is not None:
+        record_arg = " --record "
+        tag_arg = f" --tag {test_strategy} "
     else:
-        print("Running Cypress tests without recording")
-        # command = "NO_COLOR=1 npx cypress --version"
-        command = "NO_COLOR=1 npx cypress run"
-        # Add --headed --spec '**/mutations/mutations.js' (change spec name)
-        # in case you want to see the browser for debugging
+        record_arg = " "
+
+    rest_specs = set(os.listdir("tests/cypress/cypress/integration"))
+    cypress_suite1_specs = {"mutations", "search", "views"}
+    rest_specs.difference_update(set(cypress_suite1_specs))
+    strategy_spec_map = {
+        "cypress_suite1": cypress_suite1_specs,
+        "cypress_rest": rest_specs,
+    }
+    print(f"test strategy is {test_strategy}")
+    test_spec_arg = ""
+    if test_strategy is not None:
+        specs = _get_spec_map(strategy_spec_map.get(test_strategy))
+        test_spec_arg = f" --spec '{specs}' "
+
+    print("Running Cypress tests with command")
+    command = f"NO_COLOR=1 npx cypress run {record_arg} {test_spec_arg} {tag_arg}"
+    print(command)
+    # Add --headed --spec '**/mutations/mutations.js' (change spec name)
+    # in case you want to see the browser for debugging
+    print_now()
     proc = subprocess.Popen(
         command,
         shell=True,
@@ -155,4 +192,5 @@ def test_run_cypress(frontend_session, wait_for_healthchecks):
     print("stderr output:")
     print(stderr.decode("utf-8"))
     print("return code", return_code)
+    print_now()
     assert return_code == 0

smoke-test/tests/cypress/package.json

@@ -4,6 +4,7 @@
   "main": "index.js",
   "license": "MIT",
   "devDependencies": {
-    "cypress": "^9.7.0"
+    "cypress": "^9.7.0",
+    "cypress-timestamps": "^1.2.0"
   }
 }