Merge pull request hyperledger#24 from jflo/gha_acceptTest

splitting reference tests
garyschulte · Dec 11, 2023 · 3b762f3 · 3b762f3
2 parents 7058b11 + 8ede971
commit 3b762f3
Show file tree

Hide file tree

Showing 2 changed files with 41 additions and 120 deletions.
diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml
@@ -2,36 +2,20 @@ name: acceptance-tests
 on:
   workflow_dispatch:
 
-
-
 env:
   GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false"
-  total-runners: 32
+  total-runners: 16
 
 jobs:
-  runner-indexes:
-    runs-on: ubuntu-22.04
-    name: Generate runner indexes
-    outputs:
-      json: ${{ steps.generate-index-list.outputs.json }}
-    steps:
-      - id: generate-index-list
-        run: |
-          MAX_INDEX=$((${{ env.total-runners }}-1))
-          INDEX_LIST=$(seq 0 ${MAX_INDEX})
-          INDEX_JSON=$(jq --null-input --compact-output '. |= [inputs]' <<< ${INDEX_LIST})
-          echo "::set-output name=json::${INDEX_JSON}"
   acceptanceTestEthereum:
     runs-on: ubuntu-22.04
-    name: "Acceptance Runner #${{ matrix.runner-index }}: Run acceptance tests in parallel"
-    needs:
-      - runner-indexes
+    name: "Acceptance Runner #${{ matrix.runner_index }}: Run acceptance tests in parallel"
     # TODO: return check for PR approval here!
     if: ${{ github.actor != 'dependabot[bot]'}}
     strategy:
       fail-fast: false
       matrix:
-        runner-index: ${{ fromjson(needs.runner-indexes.outputs.json) }}
+        runner_index: [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]
     steps:
       - name: Checkout Repo
         uses: actions/checkout@v3
@@ -40,24 +24,31 @@ jobs:
         with:
           distribution: adopt
           java-version: 17
-      - name: Compile
+      - name: get acceptance test report
+        uses: dawidd6/action-download-artifact@v2
+        with:
+          branch: main
+          name: acceptance-test-results
+          path: tmp/junit-xml-reports-downloaded
+          if_no_artifact_found: true
+      - name: setup gradle
         uses: gradle/gradle-build-action@v2
       - name: Split tests
         id: split-tests
-        uses: chaosaffe/split-tests@v1-alpha.1
+        uses: r7kamura/split-tests-by-timings@v0
         with:
+          reports: tmp/junit-xml-reports-downloaded
           glob: 'acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/**/*Test.java'
-          split-total: ${{ env.total-runners }}
-          split-index: ${{ matrix.runner-index }}
-
+          total: ${{env.total-runners}}
+          index: ${{ matrix.runner_index }}
       - name: write out test list
-        run: echo "${{ steps.split-tests.outputs.test-suite }}" >> testList.txt
+        run: echo "${{ steps.split-tests.outputs.paths }}" >> testList.txt
       - name: format gradle args
         #regex means: first truncate file paths to align with package name, then swap path delimiter with package delimiter,
         #then drop file extension, then insert --tests option between each.
-        run: cat testList.txt | sed -e 's@acceptance-tests/tests/src/test/java/@@g' -e 's@/@.@g' -e 's/\.java//g' -e 's/\ /\ --tests\ /g' >> gradleArgs.txt
+        run: cat testList.txt | sed -e 's@acceptance-tests/tests/src/test/java/@--tests\ @g;s@/@.@g;s/\.java//g'  > gradleArgs.txt
       - name: run acceptance tests
-        run: ./gradlew acceptanceTest --tests `cat gradleArgs.txt` -Dorg.gradle.parallel=true -Dorg.gradle.caching=true
+        run: ./gradlew acceptanceTest `cat gradleArgs.txt` -Dorg.gradle.parallel=true -Dorg.gradle.caching=true
       - name: cleanup tempfiles
         run: rm testList.txt gradleArgs.txt
       - name: Upload Acceptance Test Results

diff --git a/.github/workflows/reference-tests.yml b/.github/workflows/reference-tests.yml
@@ -2,31 +2,15 @@ name: reference-tests
 on:
   workflow_dispatch:
 
-
 env:
   GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false"
   total-runners: 6
 
 jobs:
-  runner-indexes:
-    runs-on: ubuntu-22.04
-    name: Generate runner indexes
-    outputs:
-      json: ${{ steps.generate-index-list.outputs.json }}
-    steps:
-      - id: generate-index-list
-        run: |
-          MAX_INDEX=$((${{ env.total-runners }}-1))
-          INDEX_LIST=$(seq 0 ${MAX_INDEX})
-          INDEX_JSON=$(jq --null-input --compact-output '. |= [inputs]' <<< ${INDEX_LIST})
-          echo "::set-output name=json::${INDEX_JSON}"
   prepareReferenceTestEthereum:
     runs-on: ubuntu-22.04
-    timeout-minutes: 30
     if: ${{ github.actor != 'dependabot[bot]' }}
     steps:
-      - name: export runner UID
-        run: echo "runner_uid=$UID" >> $GITHUB_ENV
       - name: Checkout Repo
         uses: actions/checkout@v3
         with:
@@ -37,38 +21,20 @@ jobs:
         with:
           distribution: adopt
           java-version: 17
-          cache: gradle
-      - name: Restore classes
-        uses: actions/cache@v3
-        with:
-          path: "**/*.class"
-          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.sha }}
-      - name: generate ethereum reference tests
+      - name: setup gradle
         uses: gradle/gradle-build-action@v2
-        with:
-          arguments: ethereum:referencetests:blockchainReferenceTests ethereum:referencetests:generalstateReferenceTests ethereum:referencetests:generalstateRegressionReferenceTests
-      - name: Correct Ownership in GITHUB_WORKSPACE directory
-        uses: peter-murray/reset-workspace-ownership-action@v1
-        with:
-          user_id: ${{ env.runner_uid }}
-      - name: Cache reference tests
-        uses: actions/cache@v3
-        with:
-          path: "ethereum/referencetests/build/generated/sources/reference-test/**"
-          key: ${{ runner.os }}-reftests-${{ github.sha }}
+      - name: execute generate reference tests
+        run: ./gradlew ethereum:referencetests:blockchainReferenceTests ethereum:referencetests:generalstateReferenceTests ethereum:referencetests:generalstateRegressionReferenceTests -Dorg.gradle.parallel=true -Dorg.gradle.caching=true
   referenceTestEthereum:
     runs-on: ubuntu-22.04
     needs:
       - prepareReferenceTestEthereum
-      - runner-indexes
-    timeout-minutes: 30
     if: ${{ github.actor != 'dependabot[bot]' }}
     strategy:
+      fail-fast: false
       matrix:
-        runner-index: ${{ fromjson(needs.runner-indexes.outputs.json) }}
+        runner-index: [0,1,2,3,4,5]
     steps:
-      - name: export runner UID
-        run: echo "runner_uid=$UID" >> $GITHUB_ENV
       - name: Checkout Repo
         uses: actions/checkout@v3
         with:
@@ -78,63 +44,27 @@ jobs:
         with:
           distribution: adopt
           java-version: 17
-          cache: gradle
-      - name: Restore classes
-        uses: actions/cache@v3
-        with:
-          path: "**/*.class"
-          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.sha }}
-      - name: Restore reference tests
-        uses: actions/cache@v3
-        with:
-          path: "ethereum/referencetests/build/generated/sources/reference-test/**"
-          key: ${{ runner.os }}-reftests-${{ github.sha }}
+      - name: get acceptance test report
+        uses: dawidd6/action-download-artifact@v2
+        with:
+          branch: main
+          name: acceptance-test-results
+          path: tmp/ref-xml-reports-downloaded
+          if_no_artifact_found: true
+      - name: setup gradle
+        uses: gradle/gradle-build-action@v2
       - name: Split tests
         id: split-tests
-        uses: chaosaffe/split-tests@v1-alpha.1
+        uses: r7kamura/split-tests-by-timings@v0
         with:
+          reports: tmp/ref-xml-reports-downloaded
           glob: 'ethereum/referencetests/build/generated/sources/reference-test/**/*.java'
-          split-total: ${{ env.total-runners }}
-          split-index: ${{ matrix.runner-index }}
+          total: ${{env.total-runners}}
+          index: ${{ matrix.runner-index }}
+      - name: compose gradle args
+        run: echo ${{ steps.split-tests.outputs.paths }} | sed -e 's/^.*java\///' -e 's@/@.@g' -e 's/\.java//' -e 's/^/--tests /' > refTestArgs.txt
       - name: run reference tests
-        run: ./gradlew ethereum:referenceTests:referenceTests $(echo ${{ steps.split-tests.outputs.test-suite }} | sed -e 's/^.*java\///' -e 's@/@.@g' -e 's/\.java//' -e 's/^/--tests /')
-      - name: Correct Ownership in GITHUB_WORKSPACE directory
-        uses: peter-murray/reset-workspace-ownership-action@v1
-        with:
-          user_id: ${{ env.runner_uid }}
-      - name: Upload Test Report
-        uses: actions/upload-artifact@v3
-        if: always() # always run even if the previous step fails
-        with:
-          name: junit-reference-${{ matrix.runner-index }}-test-results
-          path: '**/build/test-results/referenceTests/TEST-*.xml'
-          retention-days: 1
-  referenceTestQbft:
-    runs-on: ubuntu-22.04
-    timeout-minutes: 30
-    if: ${{ github.actor != 'dependabot[bot]' }}
-    steps:
-      - name: export runner UID
-        run: echo "runner_uid=$UID" >> $GITHUB_ENV
-      - name: Checkout Repo
-        uses: actions/checkout@v3
-        with:
-          submodules: recursive
-      - name: Set up Java
-        uses: actions/setup-java@v3
-        with:
-          distribution: adopt
-          java-version: 17
-          cache: gradle
-      - name: Restore classes
-        uses: actions/cache@v3
-        with:
-          path: "**/*.class"
-          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.sha }}
-      - name: qbft reference tests
-        uses: gradle/gradle-build-action@v2
-        with:
-          arguments: consensus:qbft:referencetests
+        run: ./gradlew ethereum:referenceTests:referenceTests `cat refTestArgs.txt` -Dorg.gradle.parallel=true -Dorg.gradle.caching=true
       - name: Correct Ownership in GITHUB_WORKSPACE directory
         uses: peter-murray/reset-workspace-ownership-action@v1
         with:
@@ -143,6 +73,6 @@ jobs:
         uses: actions/upload-artifact@v3
         if: always() # always run even if the previous step fails
         with:
-          name: junit-qbft-reference-test-results
+          name: acceptance-test-results
           path: '**/build/test-results/referenceTests/TEST-*.xml'
-          retention-days: 1
+