-
Notifications
You must be signed in to change notification settings - Fork 1.4k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
ci: Fix workflow to update screenshots (#4993)
The initial commit was untested, but having the workflow in the main branch allowed me to iterate on the workflow and test it in a branch. The following changes were made: - Standardize how workflow inputs are read (using inputs, which works for all triggers, instead of github.event.inputs, which only works for some triggers) - Improve the description of test_filter in selenium-lab-tests. - Add missing fields in the workflow_call trigger of selenium-lab-tests.yaml. (Unfortunately, the two triggers need to have the inputs defined separately, and with slightly different fields.) - Log the chosen ref in selenium-lab-tests, to help debug inputs. - Fix variable expansion in selenium-lab-tests so that test_filter can contain spaces. - Fix ignore_test_status, which did not work at all and broken the nightly tests last night. - Fix dependencies between jobs in update-screenshots. - Do not set status for the subordinate selenium-lab-tests job if started from update-screenshots. - Always upload screenshots from selenium-lab-tests if started from update-screenshots (normally only uploaded on failure). - Set the PR status from upload-screenshots. - Fix unpacking of screenshots from artifacts in upload-screenshots. - Fix updating of changed screenshots. - Fix handling of a lack of screenshot changes. - Split up the jobs for easier debugging in upload-screenshots. Still to do: - Fix setting of final PR status - Use a personal access token so that tests can be triggered from the PR push
- Loading branch information
1 parent
541badc
commit 25fc81e
Showing
3 changed files
with
144 additions
and
43 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -12,52 +12,131 @@ on: | |
| required: true | ||
|
|
||
| jobs: | ||
| set-pending-status: | ||
| name: Set Pending Status | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - uses: actions/checkout@v3 | ||
| with: | ||
| ref: refs/pull/${{ inputs.pr }}/head | ||
|
|
||
| - name: Set commit status to pending | ||
| uses: ./.github/workflows/custom-actions/set-commit-status | ||
| with: | ||
| context: Update All Screenshots | ||
| state: pending | ||
| token: ${{ secrets.GITHUB_TOKEN }} | ||
|
|
||
| run-lab-tests: | ||
| name: Get Selenium Lab Screenshots | ||
| needs: [set-pending-status] | ||
| uses: ./.github/workflows/selenium-lab-tests.yaml | ||
| with: | ||
| pr: ${{ github.event.inputs.pr }} | ||
| test_filter: 'layout' | ||
| pr: ${{ inputs.pr }} | ||
| test_filter: layout | ||
| ignore_test_status: true | ||
|
|
||
| update-pr: | ||
| name: Update PR | ||
| runs-on: ubuntu-latest | ||
| needs: [run-lab-tests] | ||
| steps: | ||
| - uses: actions/checkout@v3 | ||
| with: | ||
| ref: refs/pull/${{ github.event.inputs.pr }}/head | ||
| ref: refs/pull/${{ inputs.pr }}/head | ||
|
|
||
| - name: Set commit status to pending | ||
| uses: ./.github/workflows/custom-actions/set-commit-status | ||
| with: | ||
| context: Update All Screenshots | ||
| state: pending | ||
| token: ${{ secrets.GITHUB_TOKEN }} | ||
|
|
||
| - name: Get artifacts | ||
| uses: actions/download-artifact@v3 | ||
| with: | ||
| path: new-screenshots | ||
|
|
||
| - name: Update screenshots | ||
| - name: Move screenshots | ||
| run: | | ||
| # Unpack screenshots from the lab. | ||
| for i in screenshots-*.zip; do | ||
| unzip -d test/test/assets/screenshots/ "$i" | ||
| # The download-artifact action puts zip file contents into folders by | ||
| # the name of the artifact, so we can't have them downloaded directly | ||
| # to the screenshots folder. Here we move them into the correct | ||
| # locations. | ||
| cd new-screenshots | ||
| # "$i" is a folder for an artifact (eg "screenshots-ChromeAndroid"), | ||
| # and it contains a single folder with the name we use in our | ||
| # screenshots (eg "chrome-Android"). We want the contents of that | ||
| # inner folder to be copied to test/test/assets/screenshots/, where a | ||
| # folder with the same name (eg "chrome-Android") already exists. | ||
| for i in screenshots*; do | ||
| cp -a $i/* ../test/test/assets/screenshots/ | ||
| done | ||
| # Update the official screenshots for any that has visibly changed. | ||
| # This is not a byte-for-byte comparison. | ||
| - name: Update screenshots | ||
| run: | | ||
| # Install prerequisites. | ||
| npm ci | ||
| # Update the official screenshots for any that have visibly changed. | ||
| # This is not a byte-for-byte comparison, but based on pixel diffs. | ||
| ./build/updateScreenshots.py | ||
| # Emulate the actions bot. | ||
| git config user.name "github-actions[bot]" | ||
| git config user.email "41898282+github-actions[bot]@users.noreply.github.com" | ||
| # Update the PR. | ||
| git add test/test/assets/screenshots/*/*.png | ||
| git commit -m ':robot: Update all screenshots' | ||
| # Commit the changes. Ignore failure, in case there are no changes. | ||
| git add test/test/assets/screenshots/*/*.png || true | ||
| git commit -m ':robot: Update all screenshots' || true | ||
| PR_API_URL="/repos/${{ github.repository }}/pulls/${{ github.event.inputs.pr }}" | ||
| REMOTE=$(gh api $GH_API_URL | jq -r .head.repo.html_url) | ||
| BRANCH=$(gh api $GH_API_URL | jq -r .head.ref) | ||
| - name: Update PR | ||
| env: | ||
| GH_TOKEN: ${{ github.token }} | ||
| run: | | ||
| # Update the PR. | ||
| PR_API_URL="/repos/${{ github.repository }}/pulls/${{ inputs.pr }}" | ||
| REMOTE=$(gh api $PR_API_URL | jq -r .head.repo.html_url) | ||
| BRANCH=$(gh api $PR_API_URL | jq -r .head.ref) | ||
| git push "$REMOTE" "$BRANCH" | ||
| # If there were no changes, this will do nothing, but succeed. | ||
| git push "$REMOTE" HEAD:"$BRANCH" | ||
| - name: Debug | ||
| uses: mxschmitt/[email protected] | ||
| with: | ||
| limit-access-to-actor: true | ||
| if: failure() | ||
|
|
||
| set-final-status: | ||
| name: Set Final Status | ||
| runs-on: ubuntu-latest | ||
| needs: [run-lab-tests, update-pr] | ||
| # Will run on success or failure, but not if the workflow is cancelled. | ||
| if: ${{ success() || failure() }} | ||
| steps: | ||
| - uses: actions/checkout@v3 | ||
| with: | ||
| ref: refs/pull/${{ inputs.pr }}/head | ||
|
|
||
| - name: Compute final status | ||
| id: compute | ||
| run: | | ||
| LAB_TEST_STATUS="${{ needs.run-lab-tests.status }}" | ||
| UPDATE_PR_STATUS="${{ needs.update-pr.status }}" | ||
| # If run-lab-tests succeeded, use the status of update-pr, otherwise | ||
| # use run-lab-tests (which is "failed" or "error"). | ||
| if [ "$LAB_TEST_STATUS" == "success" ]; then | ||
| echo "status=$UPDATE_PR_STATUS" >> $GITHUB_OUTPUT | ||
| else | ||
| echo "status=$LAST_TEST_STATUS" >> $GITHUB_OUTPUT | ||
| fi | ||
| - name: Report final status | ||
| uses: ./.github/workflows/custom-actions/set-commit-status | ||
| with: | ||
| context: Update All Screenshots | ||
| state: ${{ steps.compute.outputs.status }} | ||
| token: ${{ secrets.GITHUB_TOKEN }} | ||