From 55ee0a0b821b3247b34d89d075fcf785a14d2032 Mon Sep 17 00:00:00 2001 From: David Ventresco Date: Mon, 27 Jan 2025 11:25:54 -0500 Subject: [PATCH 1/3] Add SBT to cve-scan workflow --- .github/workflows/cve-scan-pr.yml | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/.github/workflows/cve-scan-pr.yml b/.github/workflows/cve-scan-pr.yml index 949d1d8734..b94ad11cdc 100644 --- a/.github/workflows/cve-scan-pr.yml +++ b/.github/workflows/cve-scan-pr.yml @@ -15,25 +15,22 @@ jobs: - name: Check out repo uses: actions/checkout@v4 + - name: Setup sbt launcher + uses: sbt/setup-sbt@v1 + - name: Build Docker image of HMDA platform run: | env JAVA_OPTS="-Xss256m -Xmx4096m" sbt "project hmda-platform" dockerPublishLocalSkipTests - continue-on-error: false - name: Tag and name Docker image run: docker tag hmda/hmda-platform:latest pr-cve-scan:latest - - name: Set up Homebrew - uses: Homebrew/actions/setup-homebrew@master - - - name: Install Grype - run: | - brew install grype + - name: Scan image + uses: anchore/scan-action@v6 + with: + image: pr-cve-scan:latest + output-file: grype-report.txt - - name: Run Grype scan - run: | - grype pr-cve-scan:latest > grype-report.txt - - name: Upload Grype report to artifact uses: actions/upload-artifact@v4 with: From f84b686ccd099693f591a52d510043d7ec13c364 Mon Sep 17 00:00:00 2001 From: David Ventresco Date: Mon, 27 Jan 2025 12:22:49 -0500 Subject: [PATCH 2/3] Add helper jobs & change failure of grype scan --- .github/workflows/cve-scan-pr.yml | 1 + .github/workflows/helper-functions.yaml | 42 +++ .github/workflows/unit-test.yaml | 388 ++++-------------------- 3 files changed, 95 insertions(+), 336 deletions(-) create mode 100644 .github/workflows/helper-functions.yaml diff --git a/.github/workflows/cve-scan-pr.yml b/.github/workflows/cve-scan-pr.yml index b94ad11cdc..3ece28caf0 100644 --- a/.github/workflows/cve-scan-pr.yml +++ b/.github/workflows/cve-scan-pr.yml @@ -30,6 +30,7 @@ jobs: with: image: pr-cve-scan:latest output-file: grype-report.txt + fail-build: false - name: Upload Grype report to artifact uses: actions/upload-artifact@v4 diff --git a/.github/workflows/helper-functions.yaml b/.github/workflows/helper-functions.yaml new file mode 100644 index 0000000000..bf0dc6e842 --- /dev/null +++ b/.github/workflows/helper-functions.yaml @@ -0,0 +1,42 @@ +name: "Helper Functions" +on: + workflow_call: + inputs: + service: + required: true + type: string + output-file: + required: true + type: string +jobs: + test-service: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup sbt launcher + uses: sbt/setup-sbt@v1 + + - name: Setup JDK + uses: actions/setup-java@v4 + with: + java-version: '11' + distribution: 'adopt' + + - name: Run hmda-platform Tests + run: | + sbt "project ${{ inputs.service }}" "testOnly -- -l actions-ignore" 2>&1 | tee -a ${{ inputs.output-file }} + continue-on-error: true + + - name: Check Test Results + run: | + if [ $(grep 'All tests passed.' ${{ inputs.output-file }} | wc -l) -ne 1 ]; then + echo "One or more projects had failures. Please review the logs." + cat ${{ inputs.output-file }} + exit 1 + else + echo "All tests passed." + exit 0 + fi \ No newline at end of file diff --git a/.github/workflows/unit-test.yaml b/.github/workflows/unit-test.yaml index fede2a4b1e..592d91c472 100644 --- a/.github/workflows/unit-test.yaml +++ b/.github/workflows/unit-test.yaml @@ -7,364 +7,80 @@ on: jobs: hmda_platform_tests: - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup JDK - uses: actions/setup-java@v4 - with: - java-version: '11' - distribution: 'adopt' - - - name: Run hmda-platform Tests - run: | - sbt "project hmda-platform" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file - continue-on-error: true - - - name: Check Test Results - run: | - if [ $(grep 'All tests passed.' log-file | wc -l) -ne 1 ]; then - echo "One or more projects had failures. Please review the logs." - exit 1 - else - echo "All tests passed." - exit 0 - fi + uses: ./.github/workflows/helper-functions.yaml + with: + service: "hmda-platform" + output-file: "output-log" check_digit_tests: - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup JDK - uses: actions/setup-java@v4 - with: - java-version: '11' - distribution: 'adopt' - - - name: Run check-digit Tests - run: | - sbt "project check-digit" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file - continue-on-error: true - - - name: Check Test Results - run: | - if [ $(grep 'All tests passed.' log-file | wc -l) -ne 1 ]; then - echo "One or more projects had failures. Please review the logs." - exit 1 - else - echo "All tests passed." - exit 0 - fi + uses: ./.github/workflows/helper-functions.yaml + with: + service: "check-digit" + output-file: "output-log" common_tests: - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup JDK - uses: actions/setup-java@v4 - with: - java-version: '11' - distribution: 'adopt' - - - name: Run common Tests - run: | - sbt "project common" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file - continue-on-error: true - - - name: Check Test Results - run: | - if [ $(grep 'All tests passed.' log-file | wc -l) -ne 1 ]; then - echo "One or more projects had failures. Please review the logs." - exit 1 - else - echo "All tests passed." - exit 0 - fi + uses: ./.github/workflows/helper-functions.yaml + with: + service: "common" + output-file: "output-log" data_browser_tests: - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup JDK - uses: actions/setup-java@v4 - with: - java-version: '11' - distribution: 'adopt' - - - name: Run data-browser Tests - run: | - sbt "project data-browser" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file - continue-on-error: true - - - name: Check Test Results - run: | - if [ $(grep 'All tests passed.' log-file | wc -l) -ne 1 ]; then - echo "One or more projects had failures. Please review the logs." - exit 1 - else - echo "All tests passed." - exit 0 - fi + uses: ./.github/workflows/helper-functions.yaml + with: + service: "data-browser" + output-file: "output-log" hmda_analytics_tests: - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup JDK - uses: actions/setup-java@v4 - with: - java-version: '11' - distribution: 'adopt' - - - name: Run hmda-analytics Tests - run: | - sbt "project hmda-analytics" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file - continue-on-error: true + uses: ./.github/workflows/helper-functions.yaml + with: + service: "hmda-analytics" + output-file: "output-log" - - name: Check Test Results - run: | - if [ $(grep 'All tests passed.' log-file | wc -l) -ne 1 ]; then - echo "One or more projects had failures. Please review the logs." - exit 1 - else - echo "All tests passed." - exit 0 - fi hmda_data_publisher_tests: - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup JDK - uses: actions/setup-java@v4 - with: - java-version: '11' - distribution: 'adopt' - - - name: Run hmda-data-publisher Tests - run: | - sbt "project hmda-data-publisher" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file - continue-on-error: true - - - name: Check Test Results - run: | - if [ $(grep 'All tests passed.' log-file | wc -l) -ne 1 ]; then - echo "One or more projects had failures. Please review the logs." - exit 1 - else - echo "All tests passed." - exit 0 - fi + uses: ./.github/workflows/helper-functions.yaml + with: + service: "hmda-data-publisher" + output-file: "output-log" hmda_quarterly_data_service_tests: - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup JDK - uses: actions/setup-java@v4 - with: - java-version: '11' - distribution: 'adopt' - - - name: Run hmda-quarterly-data-service Tests - run: | - sbt "project hmda-quarterly-data-service" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file - continue-on-error: true - - - name: Check Test Results - run: | - if [ $(grep 'All tests passed.' log-file | wc -l) -ne 1 ]; then - echo "One or more projects had failures. Please review the logs." - exit 1 - else - echo "All tests passed." - exit 0 - fi + uses: ./.github/workflows/helper-functions.yaml + with: + service: "hmda-quarterly-data-service" + output-file: "output-log" hmda_reporting_tests: - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup JDK - uses: actions/setup-java@v4 - with: - java-version: '11' - distribution: 'adopt' - - - name: Run hmda-reporting Tests - run: | - sbt "project hmda-reporting" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file - continue-on-error: true - - - name: Check Test Results - run: | - if [ $(grep 'All tests passed.' log-file | wc -l) -ne 1 ]; then - echo "One or more projects had failures. Please review the logs." - exit 1 - else - echo "All tests passed." - exit 0 - fi + uses: ./.github/workflows/helper-functions.yaml + with: + service: "hmda-reporting" + output-file: "output-log" institutions_api_tests: - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup JDK - uses: actions/setup-java@v4 - with: - java-version: '11' - distribution: 'adopt' - - - name: Run institutions-api Tests - run: | - sbt "project institutions-api" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file - continue-on-error: true - - - name: Check Test Results - run: | - if [ $(grep 'All tests passed.' log-file | wc -l) -ne 1 ]; then - echo "One or more projects had failures. Please review the logs." - exit 1 - else - echo "All tests passed." - exit 0 - fi + uses: ./.github/workflows/helper-functions.yaml + with: + service: "institutions-api" + output-file: "output-log" modified_lar_tests: - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup JDK - uses: actions/setup-java@v4 - with: - java-version: '11' - distribution: 'adopt' - - - name: Run modified-lar Tests - run: | - sbt "project modified-lar" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file - continue-on-error: true - - - name: Check Test Results - run: | - if [ $(grep 'All tests passed.' log-file | wc -l) -ne 1 ]; then - echo "One or more projects had failures. Please review the logs." - exit 1 - else - echo "All tests passed." - exit 0 - fi + uses: ./.github/workflows/helper-functions.yaml + with: + service: "modified-lar" + output-file: "output-log" rate_limit_tests: - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup JDK - uses: actions/setup-java@v4 - with: - java-version: '11' - distribution: 'adopt' - - - name: Run rate-limit Tests - run: | - sbt "project rate-limit" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file - continue-on-error: true - - - name: Check Test Results - run: | - if [ $(grep 'All tests passed.' log-file | wc -l) -ne 1 ]; then - echo "One or more projects had failures. Please review the logs." - exit 1 - else - echo "All tests passed." - exit 0 - fi + uses: ./.github/workflows/helper-functions.yaml + with: + service: "rate-limit" + output-file: "output-log" ratespread_calculator_tests: - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v4 + uses: ./.github/workflows/helper-functions.yaml + with: + service: "ratespread-calculator" + output-file: "output-log" - - name: Setup JDK - uses: actions/setup-java@v4 - with: - java-version: '11' - distribution: 'adopt' - - - name: Run ratespread-calculator Tests - run: | - sbt "project ratespread-calculator" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file - continue-on-error: true - - - name: Check Test Results - run: | - if [ $(grep 'All tests passed.' log-file | wc -l) -ne 1 ]; then - echo "One or more projects had failures. Please review the logs." - exit 1 - else - echo "All tests passed." - exit 0 - fi submission_errors_tests: - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup JDK - uses: actions/setup-java@v4 - with: - java-version: '11' - distribution: 'adopt' - - - name: Run submission-errors Tests - run: | - sbt "project submission-errors" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file - continue-on-error: true - - - name: Check Test Results - run: | - if [ $(grep 'All tests passed.' log-file | wc -l) -ne 1 ]; then - echo "One or more projects had failures. Please review the logs." - exit 1 - else - echo "All tests passed." - exit 0 - fi \ No newline at end of file + uses: ./.github/workflows/helper-functions.yaml + with: + service: "submission-errors" + output-file: "output-log" \ No newline at end of file From 969a2b1d1826c2bc25ca22ca99e2061713794325 Mon Sep 17 00:00:00 2001 From: David Ventresco Date: Mon, 27 Jan 2025 12:41:32 -0500 Subject: [PATCH 3/3] Change severity level & output format of Grype --- .github/workflows/cve-scan-pr.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/cve-scan-pr.yml b/.github/workflows/cve-scan-pr.yml index 3ece28caf0..a08cd75665 100644 --- a/.github/workflows/cve-scan-pr.yml +++ b/.github/workflows/cve-scan-pr.yml @@ -31,6 +31,8 @@ jobs: image: pr-cve-scan:latest output-file: grype-report.txt fail-build: false + severity-cutoff: high + output-format: table - name: Upload Grype report to artifact uses: actions/upload-artifact@v4