From 1ea182750c1cd73a27172408446fba09d4df6424 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Wa=C5=9B?= Date: Thu, 24 Nov 2022 14:49:48 +0100 Subject: [PATCH 1/2] Revert "Upload test results and reports as a composite action" This reverts commit 758c02e27dda2859fdb935ee35ce42a803d4f312. It does not correctly handle failures and fixing it would require too much overhead to keep it in a composite action. --- .github/actions/upload/action.yml | 31 ----------- .github/workflows/ci.yml | 86 +++++++++++++++++++++++++++---- 2 files changed, 76 insertions(+), 41 deletions(-) delete mode 100644 .github/actions/upload/action.yml diff --git a/.github/actions/upload/action.yml b/.github/actions/upload/action.yml deleted file mode 100644 index a99bb90b3363..000000000000 --- a/.github/actions/upload/action.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: upload -description: "Upload test results" -inputs: - name-suffix: - default: ${{ github.job }} - test-report-retention-days: - default: 5 - -runs: - using: composite - steps: - - name: Upload test results - uses: actions/upload-artifact@v3 - # Upload all test reports only on failure, because the artifacts are large - if: failure() - with: - name: result ${{ inputs.name-suffix }} - path: | - **/target/surefire-reports - **/target/checkstyle-* - - name: Upload test report - uses: actions/upload-artifact@v3 - # Always upload the test report for the annotate.yml workflow, - # but only the single XML file to keep the artifact small - if: always() - with: - # Name prefix is checked in the `Annotate checks` workflow - name: test report ${{ inputs.name-suffix }} - path: | - **/surefire-reports/TEST-*.xml - retention-days: ${{ inputs.test-report-retention-days }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5f083252f046..518d4834be59 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -173,9 +173,26 @@ jobs: if [ ! -f gib-impacted.log ] || grep -q testing/trino-test-jdbc-compatibility-old-server gib-impacted.log; then $MAVEN test ${MAVEN_TEST} -pl :trino-test-jdbc-compatibility-old-server fi - - uses: ./.github/actions/upload + - name: Upload test results + uses: actions/upload-artifact@v3 + # Upload all test reports only on failure, because the artifacts are large + if: failure() with: - test-report-retention-days: ${{ env.TEST_REPORT_RETENTION_DAYS }} + name: result ${{ github.job }} + path: | + **/target/surefire-reports + **/target/checkstyle-* + - name: Upload test report + uses: actions/upload-artifact@v3 + # Always upload the test report for the annotate.yml workflow, + # but only the single XML file to keep the artifact small + if: always() + with: + # Name prefix is checked in the `Annotate checks` workflow + name: test report ${{ github.job }} + path: | + **/surefire-reports/TEST-*.xml + retention-days: ${{ env.TEST_REPORT_RETENTION_DAYS }} - name: Clean local Maven repo # Avoid creating a cache entry because this job doesn't download all dependencies if: steps.cache.outputs.cache-hit != 'true' @@ -278,10 +295,26 @@ jobs: run: | source plugin/trino-hive-hadoop2/conf/hive-tests-${{ matrix.config }}.sh && plugin/trino-hive-hadoop2/bin/run_hive_alluxio_tests.sh - - uses: ./.github/actions/upload + - name: Upload test results + uses: actions/upload-artifact@v3 + # Upload all test reports only on failure, because the artifacts are large + if: failure() with: - name-suffix: ${{ github.job }} (${{ matrix.config }}) - test-report-retention-days: ${{ env.TEST_REPORT_RETENTION_DAYS }} + name: result ${{ github.job }} + path: | + **/target/surefire-reports + **/target/checkstyle-* + - name: Upload test report + uses: actions/upload-artifact@v3 + # Always upload the test report for the annotate.yml workflow, + # but only the single XML file to keep the artifact small + if: always() + with: + # Name prefix is checked in the `Annotate checks` workflow + name: test report ${{ github.job }} (${{ matrix.config }}) + path: | + **/surefire-reports/TEST-*.xml + retention-days: ${{ env.TEST_REPORT_RETENTION_DAYS }} - name: Clean local Maven repo # Avoid creating a cache entry because this job doesn't download all dependencies if: steps.cache.outputs.cache-hit != 'true' @@ -331,9 +364,26 @@ jobs: !:trino-sqlserver, !:trino-test-jdbc-compatibility-old-server, !:trino-tests' - - uses: ./.github/actions/upload + - name: Upload test results + uses: actions/upload-artifact@v3 + # Upload all test reports only on failure, because the artifacts are large + if: failure() with: - test-report-retention-days: ${{ env.TEST_REPORT_RETENTION_DAYS }} + name: result ${{ github.job }} + path: | + **/target/surefire-reports + **/target/checkstyle-* + - name: Upload test report + uses: actions/upload-artifact@v3 + # Always upload the test report for the annotate.yml workflow, + # but only the single XML file to keep the artifact small + if: always() + with: + # Name prefix is checked in the `Annotate checks` workflow + name: test report ${{ github.job }} + path: | + **/surefire-reports/TEST-*.xml + retention-days: ${{ env.TEST_REPORT_RETENTION_DAYS }} - name: Clean local Maven repo # Avoid creating a cache entry because this job doesn't download all dependencies if: steps.cache.outputs.cache-hit != 'true' @@ -504,10 +554,26 @@ jobs: # ", :, <, >, |, *, ?, \, / are not allowed in artifact names, replace it with an underscore name=$(echo -n "${{ matrix.modules }}" | sed -e 's/[":<>|\*\?\\\/]/_/g') echo "ARTIFACT_NAME=$name" >> $GITHUB_ENV - - uses: ./.github/actions/upload + - name: Upload test results + uses: actions/upload-artifact@v3 + # Upload all test reports only on failure, because the artifacts are large + if: failure() with: - name-suffix: ${{ env.ARTIFACT_NAME }} - test-report-retention-days: ${{ env.TEST_REPORT_RETENTION_DAYS }} + name: result ${{ env.ARTIFACT_NAME }} + path: | + **/target/surefire-reports + **/target/checkstyle-* + - name: Upload test report + uses: actions/upload-artifact@v3 + # Always upload the test report for the annotate.yml workflow, + # but only the single XML file to keep the artifact small + if: always() + with: + # Name prefix is checked in the `Annotate checks` workflow + name: test report ${{ github.job }} (${{ env.ARTIFACT_NAME }}) + path: | + **/surefire-reports/TEST-*.xml + retention-days: ${{ env.TEST_REPORT_RETENTION_DAYS }} - name: Clean local Maven repo # Avoid creating a cache entry because this job doesn't download all dependencies if: steps.cache.outputs.cache-hit != 'true' From df43ba3aba13f1b96a685d6ad44190723a91eb6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Wa=C5=9B?= Date: Thu, 24 Nov 2022 14:56:29 +0100 Subject: [PATCH 2/2] Attempt to upload heap dump if tests failed --- .github/workflows/ci.yml | 45 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 518d4834be59..5c133a6bc634 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,6 +31,7 @@ env: # This value should be greater than the time taken for the longest image pull. TESTCONTAINERS_PULL_PAUSE_TIMEOUT: 600 TEST_REPORT_RETENTION_DAYS: 5 + HEAP_DUMP_RETENTION_DAYS: 14 # used by actions/cache to retry the download after this time: https://github.com/actions/cache/blob/main/workarounds.md#cache-segment-restore-timeout SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5 @@ -153,6 +154,8 @@ jobs: test-jdbc-compatibility: runs-on: ubuntu-latest timeout-minutes: 30 + env: + SECRETS_PRESENT: ${{ secrets.SECRETS_PRESENT }} steps: - uses: actions/checkout@v3 with: @@ -193,6 +196,15 @@ jobs: path: | **/surefire-reports/TEST-*.xml retention-days: ${{ env.TEST_REPORT_RETENTION_DAYS }} + - name: Upload heap dump + uses: actions/upload-artifact@v3 + if: failure() && env.SECRETS_PRESENT == '' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository + with: + name: heap dump ${{ github.job }} + if-no-files-found: 'ignore' + path: | + **/*.hprof + retention-days: ${{ env.HEAP_DUMP_RETENTION_DAYS }} - name: Clean local Maven repo # Avoid creating a cache entry because this job doesn't download all dependencies if: steps.cache.outputs.cache-hit != 'true' @@ -207,6 +219,8 @@ jobs: - config-hdp3 # TODO: config-apache-hive3 timeout-minutes: 60 + env: + SECRETS_PRESENT: ${{ secrets.SECRETS_PRESENT }} steps: - uses: actions/checkout@v3 with: @@ -315,6 +329,15 @@ jobs: path: | **/surefire-reports/TEST-*.xml retention-days: ${{ env.TEST_REPORT_RETENTION_DAYS }} + - name: Upload heap dump + uses: actions/upload-artifact@v3 + if: failure() && env.SECRETS_PRESENT == '' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository + with: + name: heap dump ${{ github.job }} (${{ matrix.config }}) + if-no-files-found: 'ignore' + path: | + **/*.hprof + retention-days: ${{ env.HEAP_DUMP_RETENTION_DAYS }} - name: Clean local Maven repo # Avoid creating a cache entry because this job doesn't download all dependencies if: steps.cache.outputs.cache-hit != 'true' @@ -323,6 +346,8 @@ jobs: test-other-modules: runs-on: ubuntu-latest timeout-minutes: 60 + env: + SECRETS_PRESENT: ${{ secrets.SECRETS_PRESENT }} steps: - uses: actions/checkout@v3 with: @@ -384,6 +409,15 @@ jobs: path: | **/surefire-reports/TEST-*.xml retention-days: ${{ env.TEST_REPORT_RETENTION_DAYS }} + - name: Upload heap dump + uses: actions/upload-artifact@v3 + if: failure() && env.SECRETS_PRESENT == '' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository + with: + name: heap dump ${{ github.job }} + if-no-files-found: 'ignore' + path: | + **/*.hprof + retention-days: ${{ env.HEAP_DUMP_RETENTION_DAYS }} - name: Clean local Maven repo # Avoid creating a cache entry because this job doesn't download all dependencies if: steps.cache.outputs.cache-hit != 'true' @@ -462,6 +496,8 @@ jobs: fail-fast: false matrix: ${{ fromJson(needs.build-test-matrix.outputs.matrix) }} timeout-minutes: 60 + env: + SECRETS_PRESENT: ${{ secrets.SECRETS_PRESENT }} steps: - uses: actions/checkout@v3 with: @@ -574,6 +610,15 @@ jobs: path: | **/surefire-reports/TEST-*.xml retention-days: ${{ env.TEST_REPORT_RETENTION_DAYS }} + - name: Upload heap dump + uses: actions/upload-artifact@v3 + if: failure() && env.SECRETS_PRESENT == '' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository + with: + name: heap dump ${{ github.job }} (${{ env.ARTIFACT_NAME }}) + if-no-files-found: 'ignore' + path: | + **/*.hprof + retention-days: ${{ env.HEAP_DUMP_RETENTION_DAYS }} - name: Clean local Maven repo # Avoid creating a cache entry because this job doesn't download all dependencies if: steps.cache.outputs.cache-hit != 'true'