fix(ci): reduce smoke test run time

.github/workflows/docker-unified.yml

@@ -415,6 +415,14 @@ jobs:
   smoke_test:
     name: Run Smoke Tests
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        test_strategy: [
+          "no_cypress", 
+          "cypress_suite1",
+          "cypress_rest"
+        ]
     needs:
       [
         setup,
@@ -437,6 +445,14 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: "3.7"
+      - name: Pull images in background
+        run: |
+          docker pull acryldata/datahub-actions:head &
+          docker pull confluentinc/cp-kafka:7.2.0 &
+          docker pull elasticsearch:7.9.3 &
+          docker pull mariadb:10.5.8 &
+          docker pull confluentinc/cp-schema-registry:7.2.0 &
+          docker pull confluentinc/cp-zookeeper:7.2.0 &
       - name: Install dependencies
         run: ./metadata-ingestion/scripts/install_deps.sh
       - name: Build datahub cli
@@ -472,23 +488,35 @@ jobs:
         if: ${{ needs.setup.outputs.publish != 'true' }}
         with:
           image: ${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
-      - name: Download MCE Consumer image
+      - name: Download MAE Consumer image
         uses: ishworkh/docker-image-artifact-download@v1
         if: ${{ needs.setup.outputs.publish != 'true' }}
         with:
           image: ${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
+      - name: check background image download status
+        run: |
+          jobs
+      - name: run quickstart
+        env:
+          DATAHUB_TELEMETRY_ENABLED: false
+          DATAHUB_VERSION: ${{ needs.setup.outputs.unique_tag }}
+        run: |
+          ./smoke-test/run-quickstart.sh
       - name: Smoke test
         env:
+          RUN_QUICKSTART: false
           DATAHUB_VERSION: ${{ needs.setup.outputs.unique_tag }}
           CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
+          CLEANUP_DATA: "false"
+          TEST_STRATEGY: ${{ matrix.test_strategy }}
         run: |
           echo "$DATAHUB_VERSION"
           ./smoke-test/smoke.sh
       - name: store logs
         if: failure()
         run: |
           docker ps -a
-          docker logs datahub-gms >& gms.log
+          docker logs datahub-gms >& gms-${{ matrix.test_strategy }}.log
       - name: Upload logs
         uses: actions/upload-artifact@v3
         if: failure()
@@ -499,12 +527,12 @@ jobs:
         uses: actions/upload-artifact@v3
         if: failure()
         with:
-          name: cypress-snapshots
+          name: cypress-snapshots-${{ matrix.test_strategy }}
           path: smoke-test/tests/cypress/cypress/screenshots/
       - uses: actions/upload-artifact@v3
         if: always()
         with:
-          name: Test Results (smoke tests)
+          name: Test Results (smoke tests) ${{ matrix.test_strategy }}
           path: |
             **/build/reports/tests/test/**
             **/build/test-results/test/**

smoke-test/run-quickstart.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+set -euxo pipefail
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+cd "$DIR"
+
+python3 -m venv venv
+source venv/bin/activate
+pip install --upgrade pip wheel setuptools
+pip install -r requirements.txt
+
+mkdir -p ~/.datahub/plugins/frontend/auth/
+echo "test_user:test_pass" >> ~/.datahub/plugins/frontend/auth/user.props
+
+echo "DATAHUB_VERSION = $DATAHUB_VERSION"
+DATAHUB_TELEMETRY_ENABLED=false  \
+DOCKER_COMPOSE_BASE="file://$( dirname "$DIR" )" \
+
+datahub docker quickstart --standalone_consumers --dump-logs-on-failure

smoke-test/smoke.sh

@@ -14,22 +14,23 @@ set -euxo pipefail
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 cd "$DIR"
 
-python3 -m venv venv
-source venv/bin/activate
-pip install --upgrade pip wheel setuptools
-pip install -r requirements.txt
-
-mkdir -p ~/.datahub/plugins/frontend/auth/
-echo "test_user:test_pass" >> ~/.datahub/plugins/frontend/auth/user.props
+if [ "${RUN_QUICKSTART:-true}" == "true" ]; then
+    source ./run-quickstart.sh
+fi
 
-echo "DATAHUB_VERSION = $DATAHUB_VERSION"
-DATAHUB_TELEMETRY_ENABLED=false  \
-DOCKER_COMPOSE_BASE="file://$( dirname "$DIR" )" \
-datahub docker quickstart --standalone_consumers --dump-logs-on-failure
+source venv/bin/activate
 
 (cd ..; ./gradlew :smoke-test:yarnInstall)
 
 export CYPRESS_ADMIN_USERNAME=${ADMIN_USERNAME:-datahub}
 export CYPRESS_ADMIN_PASSWORD=${ADMIN_PASSWORD:-datahub}
 
-pytest -rP --durations=20 -vv --continue-on-collection-errors --junit-xml=junit.smoke.xml
+if [[ -z "${TEST_STRATEGY}" ]]; then
+    pytest -rP --durations=20 -vv --continue-on-collection-errors --junit-xml=junit.smoke.xml
+else
+    if [ "$TEST_STRATEGY" == "no_cypress" ]; then
+        pytest -rP --durations=20 -vv --continue-on-collection-errors --junit-xml=junit.smoke_non_cypress.xml -k 'not test_run_cypress'
+    else
+        pytest -rP --durations=20 -vv --continue-on-collection-errors --junit-xml=junit.smoke_cypress_${TEST_STRATEGY}.xml tests/cypress/integration_test.py
+    fi
+fi

smoke-test/test_e2e.py

@@ -575,10 +575,10 @@ def test_ingest_with_system_metadata():
                             {
                                 "com.linkedin.identity.CorpUserInfo": {
                                     "active": True,
-                                    "displayName": "Data Hub",
+                                    "displayName": "DataHub",
                                     "email": "[email protected]",
                                     "title": "CEO",
-                                    "fullName": "Data Hub",
+                                    "fullName": "DataHub",
                                 }
                             }
                         ],
@@ -608,10 +608,10 @@ def test_ingest_with_blank_system_metadata():
                             {
                                 "com.linkedin.identity.CorpUserInfo": {
                                     "active": True,
-                                    "displayName": "Data Hub",
+                                    "displayName": "DataHub",
                                     "email": "[email protected]",
                                     "title": "CEO",
-                                    "fullName": "Data Hub",
+                                    "fullName": "DataHub",
                                 }
                             }
                         ],
@@ -638,10 +638,10 @@ def test_ingest_without_system_metadata():
                             {
                                 "com.linkedin.identity.CorpUserInfo": {
                                     "active": True,
-                                    "displayName": "Data Hub",
+                                    "displayName": "DataHub",
                                     "email": "[email protected]",
                                     "title": "CEO",
-                                    "fullName": "Data Hub",
+                                    "fullName": "DataHub",
                                 }
                             }
                         ],

smoke-test/tests/cypress/cypress/integration/login/login.js

@@ -4,6 +4,6 @@ describe('login', () => {
     cy.get('input[data-testid=username]').type('datahub');
     cy.get('input[data-testid=password]').type('datahub');
     cy.contains('Sign In').click();
-    cy.contains('Welcome back, Data Hub');
+    cy.contains('Welcome back, DataHub');
   });
 })

smoke-test/tests/cypress/integration_test.py

@@ -1,5 +1,6 @@
-from typing import List
+from typing import Set, List
 
+import datetime
 import pytest
 import subprocess
 import os
@@ -99,48 +100,83 @@
     ONBOARDING_IDS.extend(id_list)
 
 
+def print_now():
+    print(f"current time is {datetime.datetime.now()}")
+
+
 def ingest_data():
+    print_now()
     print("creating onboarding data file")
     create_datahub_step_state_aspects(
         get_admin_username(),
         ONBOARDING_IDS,
         f"{CYPRESS_TEST_DATA_DIR}/{TEST_ONBOARDING_DATA_FILENAME}",
     )
 
+    print_now()
     print("ingesting test data")
     ingest_file_via_rest(f"{CYPRESS_TEST_DATA_DIR}/{TEST_DATA_FILENAME}")
     ingest_file_via_rest(f"{CYPRESS_TEST_DATA_DIR}/{TEST_DBT_DATA_FILENAME}")
     ingest_file_via_rest(f"{CYPRESS_TEST_DATA_DIR}/{TEST_SCHEMA_BLAME_DATA_FILENAME}")
     ingest_file_via_rest(f"{CYPRESS_TEST_DATA_DIR}/{TEST_ONBOARDING_DATA_FILENAME}")
+    print_now()
+    print("completed ingesting test data")
 
 
 @pytest.fixture(scope="module", autouse=True)
 def ingest_cleanup_data():
     ingest_data()
     yield
+    print_now()
     print("removing test data")
     delete_urns_from_file(f"{CYPRESS_TEST_DATA_DIR}/{TEST_DATA_FILENAME}")
     delete_urns_from_file(f"{CYPRESS_TEST_DATA_DIR}/{TEST_DBT_DATA_FILENAME}")
     delete_urns_from_file(f"{CYPRESS_TEST_DATA_DIR}/{TEST_SCHEMA_BLAME_DATA_FILENAME}")
     delete_urns_from_file(f"{CYPRESS_TEST_DATA_DIR}/{TEST_ONBOARDING_DATA_FILENAME}")
 
+    print_now()
     print("deleting onboarding data file")
     if os.path.exists(f"{CYPRESS_TEST_DATA_DIR}/{TEST_ONBOARDING_DATA_FILENAME}"):
         os.remove(f"{CYPRESS_TEST_DATA_DIR}/{TEST_ONBOARDING_DATA_FILENAME}")
+    print_now()
+    print("deleted onboarding data")
+
+
+def _get_spec_map(items: Set[str]) -> str:
+    if len(items) == 0:
+        return ""
+    return ",".join([f"**/{item}/*.js" for item in items])
 
 
 def test_run_cypress(frontend_session, wait_for_healthchecks):
     # Run with --record option only if CYPRESS_RECORD_KEY is non-empty
     record_key = os.getenv("CYPRESS_RECORD_KEY")
-    if record_key:
-        print("Running Cypress tests with recording")
-        command = "NO_COLOR=1 npx cypress run --record"
+    if record_key is None:
+        record_arg = " --record "
     else:
-        print("Running Cypress tests without recording")
-        # command = "NO_COLOR=1 npx cypress --version"
-        command = "NO_COLOR=1 npx cypress run"
-        # Add --headed --spec '**/mutations/mutations.js' (change spec name)
-        # in case you want to see the browser for debugging
+        record_arg = " "
+
+    rest_specs = set(os.listdir("tests/cypress/cypress/integration"))
+
+    cypress_suite1_specs = {"mutations", "search", "glossary", "views"}
+    rest_specs.difference_update(set(cypress_suite1_specs))
+
+    test_strategy = os.getenv("TEST_STRATEGY", None)
+    print(f"test strategy is {test_strategy}")
+    if test_strategy == "cypress_suite1":
+        specs = _get_spec_map(cypress_suite1_specs)
+        test_spec_arg = f" --spec '{specs}' "
+    elif test_strategy == "cypress_rest":
+        specs = _get_spec_map(rest_specs)
+        test_spec_arg = f" --spec '{specs}' "
+    else:
+        test_spec_arg = " "
+
+    print("Running Cypress tests with command")
+    command = f"NO_COLOR=1 npx cypress run {record_arg} {test_spec_arg}"
+    print(command)
+    # Add --headed --spec '**/mutations/mutations.js' (change spec name)
+    # in case you want to see the browser for debugging
     proc = subprocess.Popen(
         command,
         shell=True,

smoke-test/tests/utils.py

@@ -7,7 +7,6 @@
 import requests_wrapper as requests
 
 from datahub.cli import cli_utils
-from datahub.cli.docker_cli import check_local_docker_containers
 from datahub.ingestion.run.pipeline import Pipeline
 
 TIME: int = 1581407189000
@@ -114,6 +113,9 @@ def ingest_file_via_rest(filename: str) -> Pipeline:
 
 
 def delete_urns_from_file(filename: str) -> None:
+    if not cli_utils.get_boolean_env_variable("CLEANUP_DATA", True):
+        print("Not cleaning data to save time")
+        return
     session = requests.Session()
     session.headers.update(
         {
@@ -196,7 +198,7 @@ def create_datahub_step_state_aspects(
     """
     For a specific user, creates dataHubStepState aspects for each onboarding id in the list
     """
-    aspects_dict: List[Dict[str, any]] = [
+    aspects_dict: List[Dict[str, Any]] = [
         create_datahub_step_state_aspect(username, onboarding_id)
         for onboarding_id in onboarding_ids
     ]