From dfd54398b0dc54021c8603b7a26a665f05173bfe Mon Sep 17 00:00:00 2001 From: Zain Nasir <49543216+zainasir@users.noreply.github.com> Date: Fri, 7 Feb 2025 18:55:22 -0500 Subject: [PATCH] Update circleci localdb tests to use the new setup (#11385) * update localdb tests to use new setup * push both images * push both images * fix paths * use spec file pattern variable --- .circleci/config.yml | 485 +++++++++++++++++++++---------------------- 1 file changed, 232 insertions(+), 253 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 63cc08cac52..39fe800fc1f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,259 +5,60 @@ defaults: &defaults version: 2.1 commands: build_push_image: + description: "Build and push cBioPortal image to Dockerhub. Check parameters below for configurable options." parameters: + docker_repo: + description: Docker repo name + type: string + default: "cbioportal/cbioportal-dev" + app_type: + description: Type of cBioPortal App (web OR web-and-data OR both) + type: enum + enum: ["web", "web-and-data"] push: description: Push image to DockerHub - type: string - default: "false" + type: enum + enum: ["true", "false"] steps: - run: - name: Build cBioPortal docker image - environment: - DOCKER_REPO: cbioportal/cbioportal-dev + name: Build and Push cBioPortal docker image command: | export DOCKER_TAG=$CIRCLE_SHA1 - URL="https://hub.docker.com/v2/repositories/cbioportal/cbioportal-dev/tags/$DOCKER_TAG-web-shenandoah" + export DOCKER_REPO=<> + export APP_TYPE=<> + + if [ $APP_TYPE = "web" ]; then + URL="https://hub.docker.com/v2/repositories/$DOCKER_REPO/tags/$DOCKER_TAG-web-shenandoah" + elif [ $APP_TYPE = "web-and-data" ]; then + URL="https://hub.docker.com/v2/repositories/$DOCKER_REPO/tags/$DOCKER_TAG" + else + echo "Unknown app type provided. Accepted values are ['web', 'web-and-data']" + exit 1 + fi + TAG_FOUND=$(curl -s $URL | jq -r .name) - if [ $TAG_FOUND = "$DOCKER_TAG-web-shenandoah" ]; then + if [ $APP_TYPE = "web" ] && [ $TAG_FOUND = "$DOCKER_TAG-web-shenandoah" ]; then echo "Image already exists. Pulling remote image and skipping build step!" docker pull $DOCKER_REPO:$DOCKER_TAG-web-shenandoah exit 0 fi - cd cbioportal-test - ./scripts/build-push-image.sh --src=/tmp/repos/cbioportal --push=<> --skip_web_and_data=true - if [ "<>" = "false" ]; then - EXISTS=$(docker inspect --type=image $DOCKER_REPO:$DOCKER_TAG-web-shenandoah > /dev/null; echo $?); - else - EXISTS=$(docker manifest inspect $DOCKER_REPO:$DOCKER_TAG-web-shenandoah > /dev/null; echo $?); + if [ $APP_TYPE = "web-and-data" ] && [ $TAG_FOUND = "$DOCKER_TAG" ]; then + echo "Image already exists. Pulling remote image and skipping build step!" + docker pull $DOCKER_REPO:$DOCKER_TAG + exit 0 fi - if [ $EXISTS -eq 0 ]; then - echo "Build succeeded!" + + cd cbioportal-test + if [ $APP_TYPE = "web" ]; then + ./scripts/build-push-image.sh --src=/tmp/repos/cbioportal --push=<> --skip_web_and_data=true + elif [ $APP_TYPE = "web-and-data" ]; then + ./scripts/build-push-image.sh --src=/tmp/repos/cbioportal --push=<> --skip_web=true else - echo "Build failed!" + echo "Unknown app type provided. Accepted values are ['web', 'web-and-data']" exit 1 fi jobs: - build_backend: - docker: - - image: maven:3-eclipse-temurin-21 - working_directory: /tmp/repos/cbioportal - steps: - - checkout - - run: - name: Concatenate poms to use as cache key for mvn deps - command: cat $(git ls-files '*pom.xml*') > poms_combined - - restore_cache: - keys: - - v1-mvn-dependencies-{{ checksum "poms_combined" }} - - v1-mvn-dependencies- - - run: - name: Build and unzip jar - command: | - cp src/main/resources/application.properties.EXAMPLE src/main/resources/application.properties && \ - mvn -DskipTests clean package - - save_cache: - paths: - - ~/.m2 - key: v1-mvn-dependencies-{{ checksum "poms_combined" }} - - persist_to_workspace: - root: /tmp/repos - paths: - - cbioportal - - pull_frontend_codebase: - <<: *defaults - working_directory: /tmp/repos - steps: - - checkout - - run: - name: Pull frontend code - command: | - export FRONTEND_VERSION=$(grep '' pom.xml | sed 's///g' | sed 's|||' | tr -d '[:blank:]') && \ - export FRONTEND_ORG=$(grep 'frontend\.groupId' pom.xml | sed 's///g' | sed 's|||' | tr -d '[:blank:]' | cut -d. -f3) && \ - git clone https://github.com/$FRONTEND_ORG/cbioportal-frontend.git && \ - cd cbioportal-frontend && \ - git fetch --tags - - persist_to_workspace: - root: /tmp/repos - paths: - - cbioportal-frontend - - install_yarn: - <<: *defaults - working_directory: /tmp/repos/cbioportal-frontend - steps: - - attach_workspace: - at: /tmp/repos - #- run: - # name: "Install yarn at specific version" - # command: - # sudo npm install --global yarn@1.22.4 - - run: - name: "Show yarn and node versions" - command: | - node --version - yarn --version - # cache build artifacts. Use concatenation of all source files as cache - # key. If there are no changes to src/ and yarn.lock, no need to rebuild - - run: - name: "Concatenate all source files to use as hash key for caching dist folder" - command: "cat yarn.lock $(find src/ -type f | sort) webpack.config.js vendor-bundles.webpack.config.js > has_source_changed" - - restore_cache: - keys: - - v4-dependencies-plus-dist-{{ checksum "has_source_changed" }} - - v4-dependencies-{{ checksum "yarn.lock" }} - # Download and cache dependencies - - run: yarn -# - run: -# name: "Make sure lock file is still the same" -# command: 'git diff --exit-code yarn.lock > /dev/null || (echo -e "New package lock file at $(cat yarn.lock | curl -F c=@- https://ptpb.pw | grep url) (include this file in your PR to fix this test)"; git diff --exit-code yarn.lock; exit 1)' - - save_cache: - paths: - - node_modules - key: v4-dependencies-{{ checksum "yarn.lock" }} - - run: - name: "Run build if no dist folder" - command: 'ls dist || yarn run build' - environment: - DISABLE_SOURCEMAP: true - NO_PARALLEL: true - no_output_timeout: 25m - - run: cd /tmp/repos/cbioportal-frontend/end-to-end-test && (ls node_modules || yarn install --frozen-lockfile --ignore-engines) - - save_cache: - paths: - - node_modules - - dist - - common-dist - key: v4-dependencies-plus-dist-{{ checksum "has_source_changed" }} - - persist_to_workspace: - root: /tmp/repos - paths: - - cbioportal-frontend - - end_to_end_tests_localdb: - working_directory: /tmp/repos/cbioportal-frontend - machine: - enabled: true - image: ubuntu-2204:2023.02.1 - resource_class: large - steps: - - attach_workspace: - at: /tmp/repos - - run: - name: Setup python libraries - command: | - pip3 install requests pyyaml - - run: - name: Install dependencies - command: | - sudo apt-get update && \ - sudo apt-get install jq - - run: - name: Determine what backend image to run - command: | - if [[ -n "${CIRCLE_PR_USERNAME}" ]]; then \ - sed -i '/BACKEND.*/d' env/custom.sh && \ - echo -e "\nexport BACKEND=$CIRCLE_PR_USERNAME:$CIRCLE_SHA1" >> $PORTAL_SOURCE_DIR/env/custom.sh; \ - else \ - echo -e "\nexport BACKEND=$CIRCLE_PROJECT_USERNAME:$CIRCLE_SHA1" >> $PORTAL_SOURCE_DIR/env/custom.sh; \ - fi - - run: - name: Setup e2e-environment - command: | - source $PORTAL_SOURCE_DIR/env/custom.sh || true && \ - cd $TEST_HOME/runtime-config && \ - ./setup_environment.sh && ./setup_environment.sh >> $BASH_ENV - - run: - name: Build custom backend - command: | - mkdir -p $E2E_WORKSPACE; \ - mv /tmp/repos/cbioportal $E2E_WORKSPACE - - run: - name: Setup docker compose assets - command: | - $TEST_HOME/docker_compose/setup.sh - no_output_timeout: 25m - - run: - name: Create MySQL data directory - command: | - docker volume rm --force cbioportal-docker-compose_cbioportal_mysql_data && mkdir -p $CBIO_DB_DATA_DIR && rm -rf $CBIO_DB_DATA_DIR/* - - restore_cache: - keys: - - v8-cbio-database-files-{{ checksum "/tmp/db_data_md5key" }} - - restore_cache: - keys: - - v8-keycloak-database-files-{{ checksum "e2e-localdb-workspace/keycloak/keycloak-config-generated.json" }} - - run: - name: Init database - command: | - cd $TEST_HOME/docker_compose && echo $CBIO_DB_DATA_DIR && ls -la $CBIO_DB_DATA_DIR && \ - [ "$(ls -A $CBIO_DB_DATA_DIR)" ] && echo "DB initialization is not needed." || ./initdb.sh - - run: - name: Change owner of MySQL database files (needed by cache) - command: | - sudo chmod -R 777 $CBIO_DB_DATA_DIR && \ - sudo chown -R circleci:circleci $CBIO_DB_DATA_DIR - - save_cache: - paths: - - /tmp/repos/cbioportal-frontend/e2e-localdb-workspace/cbio_db_data - key: v9-cbio-database-files-{{ checksum "/tmp/db_data_md5key" }} - - run: - name: Start cbioportal and other services - command: | - $TEST_HOME/docker_compose/start.sh - - run: - name: Change owner of keycloak MySQL database files (needed by cache) - command: | - if (ls "$KC_DB_DATA_DIR"/* 2> /dev/null > /dev/null); then \ - sudo chmod -R 777 $KC_DB_DATA_DIR && \ - sudo chown -R circleci:circleci $KC_DB_DATA_DIR; \ - fi - - save_cache: - paths: - - /tmp/repos/cbioportal-frontend/e2e-localdb-workspace/kc_db_data - key: v9-keycloak-database-files-{{ checksum "e2e-localdb-workspace/keycloak/keycloak-config-generated.json" }} - - run: - name: Run end-2-end tests with studies in local database - command: | - cd $PORTAL_SOURCE_DIR && \ - $TEST_HOME/docker_compose/test.sh - - run: - name: "Make sure all screenshots are tracked (otherwise the test will always be successful)" - command: 'for f in $TEST_HOME/screenshots/reference/*.png; do git ls-files --error-unmatch $f > /dev/null 2> /dev/null || (echo -e "\033[0;31m $f not tracked \033[0m" && touch screenshots_not_tracked); done; ls screenshots_not_tracked > /dev/null 2> /dev/null && exit 1 || exit 0' - - store_artifacts: - path: /tmp/repos/cbioportal-frontend/end-to-end-test/local/screenshots - destination: /screenshots - - store_artifacts: - path: /tmp/repos/cbioportal-frontend/end-to-end-test/shared/image-compare - destination: /image-compare - - store_artifacts: - path: /tmp/repos/cbioportal-frontend/end-to-end-test/local/errorShots - destination: /errorShots - - store_test_results: - path: /tmp/repos/cbioportal-frontend/end-to-end-test/local/junit - - store_artifacts: - path: /tmp/repos/cbioportal-frontend/end-to-end-test/local/junit - - store_artifacts: - path: /tmp/repos/cbioportal-frontend/end-to-end-test/shared/imageCompare.html - destination: /imageCompare.html - - store_artifacts: - path: /tmp/repos/cbioportal-frontend/end-to-end-test/local/junit/customReport.json - destination: /customReport.json - - store_artifacts: - path: /tmp/repos/cbioportal-frontend/end-to-end-test/local/junit/errors/ - destination: /errors - - environment: - PORTAL_SOURCE_DIR: /tmp/repos/cbioportal-frontend/ - TEST_HOME: /tmp/repos/cbioportal-frontend/end-to-end-test/local - E2E_WORKSPACE: /tmp/repos/cbioportal-frontend/e2e-localdb-workspace - CBIO_DB_DATA_DIR: /tmp/repos/cbioportal-frontend/e2e-localdb-workspace/cbio_db_data - KC_DB_DATA_DIR: /tmp/repos/cbioportal-frontend/e2e-localdb-workspace/kc_db_data - DOCKER_IMAGE_SESSION_SERVICE: cbioportal/session-service:0.5.0 - FRONTEND_TEST_DO_NOT_LOAD_EXTERNAL_FRONTEND: true - pull_cbioportal_test_codebase: machine: image: ubuntu-2204:2024.08.1 @@ -306,7 +107,60 @@ jobs: paths: - cbioportal - build_image: + build_frontend: + <<: *defaults + working_directory: /tmp/repos + environment: + BRANCH_ENV: master + steps: + - attach_workspace: + at: /tmp/repos + - run: + name: Generate cache key for dist + command: | + cd cbioportal-frontend + echo "Concatenate all source files to use as hash key." + cat yarn.lock $(find src/ -type f | sort) webpack.config.js vendor-bundles.webpack.config.js > has_source_changed + - restore_cache: + keys: + - v5-dependencies-plus-dist-{{ checksum "cbioportal-frontend/has_source_changed" }} + - v5-dependencies-{{ checksum "cbioportal-frontend/yarn.lock" }} + - run: + name: Download dependencies + command: | + cd cbioportal-frontend + yarn + - save_cache: + name: Save dependency cache + key: v5-dependencies-{{ checksum "cbioportal-frontend/yarn.lock" }} + paths: + - cbioportal-frontend/node_modules + - run: + name: Build frontend application + environment: + DISABLE_SOURCEMAP: true + NO_PARALLEL: true + command: | + cd cbioportal-frontend + ls dist || yarn run build + - run: + name: Build frontend tests + command: | + cd cbioportal-frontend/end-to-end-test + ls node_modules || yarn install --frozen-lockfile --ignore-engines + - save_cache: + name: Save dependency plus dist cache + key: v5-dependencies-plus-dist-{{ checksum "cbioportal-frontend/has_source_changed" }} + paths: + - cbioportal-frontend/node_modules + - cbioportal-frontend/dist + - cbioportal-frontend/common-dist + - persist_to_workspace: + root: /tmp/repos + paths: + - cbioportal-frontend + + build_web_image: machine: image: ubuntu-2204:2024.08.1 resource_class: medium @@ -317,6 +171,7 @@ jobs: - attach_workspace: at: /tmp/repos - build_push_image: + app_type: "web" push: "false" - run: name: Save cbioportal image as tar @@ -328,7 +183,30 @@ jobs: paths: - "*.tar" - push_image: + build_web_and_data_image: + machine: + image: ubuntu-2204:2024.08.1 + resource_class: medium + working_directory: /tmp/repos + environment: + DOCKER_REPO: cbioportal/cbioportal-dev + steps: + - attach_workspace: + at: /tmp/repos + - build_push_image: + app_type: "web-and-data" + push: "false" + - run: + name: Save cbioportal image as tar + command: | + export DOCKER_TAG=$CIRCLE_SHA1 + docker save -o $DOCKER_TAG.tar $DOCKER_REPO:$DOCKER_TAG + - persist_to_workspace: + root: /tmp/repos + paths: + - "*.tar" + + push_web_image: machine: image: ubuntu-2204:2024.08.1 resource_class: medium @@ -337,8 +215,104 @@ jobs: - attach_workspace: at: /tmp/repos - build_push_image: + app_type: "web" push: "true" + push_web_and_data_image: + machine: + image: ubuntu-2204:2024.08.1 + resource_class: medium + working_directory: /tmp/repos + steps: + - attach_workspace: + at: /tmp/repos + - build_push_image: + app_type: "web-and-data" + push: "true" + + run_e2e_localdb_tests: + working_directory: /tmp/repos + machine: + image: ubuntu-2204:2024.08.1 + resource_class: large + environment: + DOCKER_REPO: cbioportal/cbioportal-dev + DOCKER_IMAGE_MYSQL: cbioportal/mysql:8.0-database-test + steps: + - attach_workspace: + at: /tmp/repos + - run: + name: Load cbioportal image + command: | + export DOCKER_TAG=$CIRCLE_SHA1 + docker load -i $DOCKER_TAG.tar + - run: + name: Generate keycloak config + environment: + STUDIES: 'ascn_test_study study_hg38 teststudy_genepanels study_es_0 lgg_ucsf_2014_test_generic_assay' + command: | + git clone https://github.com/cbioportal/cbioportal-docker-compose.git + cd cbioportal-test + ./utils/gen-keycloak-config.sh --studies="$STUDIES" --template='/tmp/repos/cbioportal-docker-compose/dev/keycloak/keycloak-config.json' --out='/tmp/repos/keycloak-config-generated.json' + - run: + name: Start backend server with keycloak + environment: + KEYCLOAK_CONFIG_PATH: '/tmp/repos/keycloak-config-generated.json' + APPLICATION_PROPERTIES_PATH: '/tmp/repos/cbioportal-frontend/end-to-end-test/local/runtime-config/portal.properties' + command: | + cd cbioportal-test + export DOCKER_IMAGE_CBIOPORTAL=$DOCKER_REPO:$CIRCLE_SHA1 + nohup ./scripts/docker-compose.sh --portal_type='keycloak' >> /tmp/repos/docker-compose-logs.txt 2>&1 & + - run: + name: Check keycloak connection at localhost + command: | + cd cbioportal-test + ./utils/check-connection.sh --url=localhost:8081 --max_retries=50 + - run: + name: Check backend connection at localhost + command: | + cd cbioportal-test + ./utils/check-connection.sh --url=localhost:8080 --max_retries=50 + - run: + name: Run end-2-end tests with studies in local database + environment: + CBIOPORTAL_URL: http://localhost:8080 + JUNIT_REPORT_PATH: ./local/junit + SCREENSHOT_DIRECTORY: ./local/screenshots + PORTAL_SOURCE_DIR: /tmp/repos/cbioportal-frontend + TEST_HOME: /tmp/repos/cbioportal-frontend/end-to-end-test/local + FRONTEND_TEST_DO_NOT_LOAD_EXTERNAL_FRONTEND: true + SPEC_FILE_PATTERN: ./local/specs/**/*.spec.js + command: | + cd $PORTAL_SOURCE_DIR && \ + $TEST_HOME/docker_compose/test.sh + - run: + name: Make sure all screenshots are tracked (otherwise the test will always be successful) + command: | + for f in $TEST_HOME/screenshots/reference/*.png; do git ls-files --error-unmatch $f > /dev/null 2> /dev/null || (echo -e "\033[0;31m $f not tracked \033[0m" && touch screenshots_not_tracked); done; ls screenshots_not_tracked > /dev/null 2> /dev/null && exit 1 || exit 0 + - store_artifacts: + path: /tmp/repos/cbioportal-frontend/end-to-end-test/local/screenshots + destination: /screenshots + - store_artifacts: + path: /tmp/repos/cbioportal-frontend/end-to-end-test/shared/image-compare + destination: /image-compare + - store_artifacts: + path: /tmp/repos/cbioportal-frontend/end-to-end-test/local/errorShots + destination: /errorShots + - store_test_results: + path: /tmp/repos/cbioportal-frontend/end-to-end-test/local/junit + - store_artifacts: + path: /tmp/repos/cbioportal-frontend/end-to-end-test/local/junit + - store_artifacts: + path: /tmp/repos/cbioportal-frontend/end-to-end-test/shared/imageCompare.html + destination: /imageCompare.html + - store_artifacts: + path: /tmp/repos/cbioportal-frontend/end-to-end-test/local/junit/customReport.json + destination: /customReport.json + - store_artifacts: + path: /tmp/repos/cbioportal-frontend/end-to-end-test/local/junit/errors/ + destination: /errors + run_api_tests: machine: image: ubuntu-2204:2024.08.1 @@ -518,44 +492,49 @@ jobs: git push workflows: - end_to_end_tests: - jobs: - - build_backend - - pull_frontend_codebase - - install_yarn: - requires: - - pull_frontend_codebase - - end_to_end_tests_localdb: - requires: - - build_backend - - pull_frontend_codebase - - install_yarn tests: jobs: - checkout_pr - pull_cbioportal_test_codebase - pull_cbioportal_frontend_codebase - - build_image: + - build_frontend: + requires: + - pull_cbioportal_frontend_codebase + - build_web_image: requires: - checkout_pr - pull_cbioportal_test_codebase - - push_image: + - build_web_and_data_image: + requires: + - checkout_pr + - pull_cbioportal_test_codebase + - push_web_image: context: - api-tests requires: - checkout_pr - pull_cbioportal_test_codebase + - push_web_and_data_image: + context: + - api-tests + requires: + - checkout_pr + - pull_cbioportal_test_codebase + - run_e2e_localdb_tests: + requires: + - build_frontend + - build_web_and_data_image - run_api_tests: context: - api-tests requires: - - build_image + - build_web_image - pull_cbioportal_frontend_codebase - run_security_tests: context: - docker-scout requires: - - build_image + - build_web_image - update_security_status_badge: requires: - run_security_tests