-
Notifications
You must be signed in to change notification settings - Fork 1.4k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
ci: Fix workflow to update screenshots #4993
Changes from 1 commit
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -9,7 +9,7 @@ on: | |
description: "A PR number to build and test in the lab. If empty, will build and test from main." | ||
required: false | ||
test_filter: | ||
description: "A filter to run a subset of the tests. If empty, all tests will run." | ||
description: "A regex filter to run a subset of the tests. If empty, all tests will run." | ||
required: false | ||
workflow_call: | ||
# Allows for reuse from other workflows, such as "Update All Screenshots" | ||
|
@@ -18,11 +18,15 @@ on: | |
pr: | ||
description: "A PR number to build and test in the lab. If empty, will build and test from main." | ||
required: false | ||
type: string | ||
test_filter: | ||
description: "A filter to run a subset of the tests. If empty, all tests will run." | ||
description: "A regex filter to run a subset of the tests. If empty, all tests will run." | ||
required: false | ||
type: string | ||
ignore_test_status: | ||
description: "If true, ignore test failures and treat the workflow as a success." | ||
required: false | ||
type: boolean | ||
schedule: | ||
# Runs every night at 2am PST / 10am UTC, testing against the main branch. | ||
- cron: '0 10 * * *' | ||
|
@@ -42,12 +46,12 @@ jobs: | |
- name: Compute ref | ||
id: compute | ||
run: | | ||
if [[ "${{ github.event.inputs.pr }}" != "" ]]; then | ||
LAB_TEST_REF="refs/pull/${{ github.event.inputs.pr }}/head" | ||
if [[ "${{ inputs.pr }}" != "" ]]; then | ||
LAB_TEST_REF="refs/pull/${{ inputs.pr }}/head" | ||
else | ||
LAB_TEST_REF="main" | ||
fi | ||
echo "REF=$LAB_TEST_REF" >> $GITHUB_OUTPUT | ||
echo "REF=$LAB_TEST_REF" | tee -a $GITHUB_OUTPUT | ||
|
||
# Configure the build matrix based on our grid's YAML config. | ||
# The matrix contents will be computed by this first job and deserialized | ||
|
@@ -112,6 +116,7 @@ jobs: | |
ref: ${{ needs.compute-ref.outputs.REF }} | ||
|
||
- name: Set commit status to pending | ||
if: ${{ inputs.ignore_test_status == false }} | ||
uses: ./.github/workflows/custom-actions/set-commit-status | ||
with: | ||
context: Selenium / Build | ||
|
@@ -129,8 +134,9 @@ jobs: | |
retention-days: 1 | ||
|
||
- name: Report final commit status | ||
# Will run on success or failure, but not if the workflow is cancelled. | ||
if: ${{ success() || failure() }} | ||
# Will run on success or failure, but not if the workflow is cancelled | ||
# or if we were asked to ignore the test status. | ||
if: ${{ (success() || failure()) && inputs.ignore_test_status == false }} | ||
uses: ./.github/workflows/custom-actions/set-commit-status | ||
with: | ||
context: Selenium / Build | ||
|
@@ -154,6 +160,7 @@ jobs: | |
ref: ${{ needs.compute-ref.outputs.REF }} | ||
|
||
- name: Set commit status to pending | ||
if: ${{ inputs.ignore_test_status == false }} | ||
uses: ./.github/workflows/custom-actions/set-commit-status | ||
with: | ||
context: Selenium / ${{ matrix.browser }} | ||
|
@@ -197,25 +204,28 @@ jobs: | |
# the container. | ||
- name: Test Player | ||
run: | | ||
# Use of an array keeps elements in tact, and allows an element to | ||
# contain spaces without being expanded into multiple arguments in a | ||
# shell command. | ||
extra_flags=() | ||
|
||
# Generate a coverage report from uncompiled code on ChromeLinux. | ||
# It should be the uncompiled build, or else we won't execute any | ||
# coverage instrumentation on full-stack player integration tests. | ||
if [[ "${{ matrix.browser }}" == "ChromeLinux" ]]; then | ||
extra_flags="--html-coverage-report --uncompiled" | ||
else | ||
extra_flags="" | ||
extra_flags+=(--html-coverage-report --uncompiled) | ||
fi | ||
|
||
filter="${{ github.event.inputs.test_filter }}" | ||
if [[ "$filter" != "" ]]; then | ||
extra_flags="$extra_flags --filter $filter" | ||
fi | ||
|
||
ignore_test_status="${{ github.event.inputs.ignore_test_status }}" | ||
if [[ "$ignore_test_status" == "true" ]]; then | ||
extra_flags="$extra_flags || true" | ||
if [[ "${{ inputs.test_filter }}" != "" ]]; then | ||
echo "Adding filter: ${{ inputs.test_filter }}" | ||
extra_flags+=(--filter "${{ inputs.test_filter }}") | ||
fi | ||
|
||
# Do not automatically fail when a command fails. This allows us to | ||
# implement the ignore_test_status input by capturing the exit code | ||
# and examining it. | ||
set +e | ||
# Run the tests with any extra flags. | ||
python3 build/test.py \ | ||
--no-build \ | ||
--reporters spec --spec-hide-passed \ | ||
|
@@ -225,7 +235,18 @@ jobs: | |
--grid-config build/shaka-lab.yaml \ | ||
--grid-address selenium-grid.lab:4444 \ | ||
--browsers ${{ matrix.browser }} \ | ||
$extra_flags | ||
"${extra_flags[@]}" | ||
# Capture the test exit code immediately after running the tests. | ||
# There cannot be any other command between test.py and here. | ||
exit_code=$? | ||
|
||
# If ignoring test status, treat this as an exit code of 0 (success). | ||
if [[ "${{ inputs.ignore_test_status }}" == "true" ]]; then | ||
exit_code=0 | ||
fi | ||
|
||
# Report the captured (and possibly overridden) exit status. | ||
exit $exit_code | ||
|
||
- name: Find coverage report (ChromeLinux only) | ||
id: coverage | ||
|
@@ -269,7 +290,7 @@ jobs: | |
# Upload new screenshots and diffs on failure; ignore if missing | ||
- name: Upload screenshots | ||
uses: actions/upload-artifact@v3 | ||
if: ${{ failure() }} | ||
if: ${{ failure() || inputs.ignore_test_status }} | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The description of There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This means the job runs on failure, or if ignore_test_status is true. Normally, we would only upload screenshot artifacts if the tests failed. But if we're running the workflow specifically to get the screenshots updated, we upload them no matter what. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Comment updated. ignore_test_status modifies many things that normally depend on test status. |
||
with: | ||
# In this workflow, "browser" is the selenium node name, which can | ||
# contain both browser and OS, such as "ChromeLinux". | ||
|
@@ -281,8 +302,9 @@ jobs: | |
retention-days: 5 | ||
|
||
- name: Report final commit status | ||
# Will run on success or failure, but not if the workflow is cancelled. | ||
if: ${{ success() || failure() }} | ||
# Will run on success or failure, but not if the workflow is cancelled | ||
# or if we were asked to ignore the test status. | ||
if: ${{ (success() || failure()) && inputs.ignore_test_status == false }} | ||
uses: ./.github/workflows/custom-actions/set-commit-status | ||
with: | ||
context: Selenium / ${{ matrix.browser }} | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -12,52 +12,131 @@ on: | |
required: true | ||
|
||
jobs: | ||
set-pending-status: | ||
name: Set Pending Status | ||
runs-on: ubuntu-latest | ||
steps: | ||
- uses: actions/checkout@v3 | ||
with: | ||
ref: refs/pull/${{ inputs.pr }}/head | ||
|
||
- name: Set commit status to pending | ||
uses: ./.github/workflows/custom-actions/set-commit-status | ||
with: | ||
context: Update All Screenshots | ||
state: pending | ||
token: ${{ secrets.GITHUB_TOKEN }} | ||
|
||
run-lab-tests: | ||
name: Get Selenium Lab Screenshots | ||
needs: [set-pending-status] | ||
uses: ./.github/workflows/selenium-lab-tests.yaml | ||
with: | ||
pr: ${{ github.event.inputs.pr }} | ||
test_filter: 'layout' | ||
pr: ${{ inputs.pr }} | ||
test_filter: layout | ||
ignore_test_status: true | ||
|
||
update-pr: | ||
name: Update PR | ||
runs-on: ubuntu-latest | ||
needs: [run-lab-tests] | ||
steps: | ||
- uses: actions/checkout@v3 | ||
with: | ||
ref: refs/pull/${{ github.event.inputs.pr }}/head | ||
ref: refs/pull/${{ inputs.pr }}/head | ||
|
||
- name: Set commit status to pending | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Say, the first two steps of this workflow are the exact same as the steps of There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. No, those are separate jobs. Each job runs with a clean slate. You're thinking of "steps" within jobs. Calling another workflow (run-lab-tests) requires its own job. So the jobs are: 1. Set pending status, 2. run test workflow, 3. update PR, 4. set final status |
||
uses: ./.github/workflows/custom-actions/set-commit-status | ||
with: | ||
context: Update All Screenshots | ||
state: pending | ||
token: ${{ secrets.GITHUB_TOKEN }} | ||
|
||
- name: Get artifacts | ||
uses: actions/download-artifact@v3 | ||
with: | ||
path: new-screenshots | ||
|
||
- name: Update screenshots | ||
- name: Move screenshots | ||
run: | | ||
# Unpack screenshots from the lab. | ||
for i in screenshots-*.zip; do | ||
unzip -d test/test/assets/screenshots/ "$i" | ||
# The download-artifact action puts zip file contents into folders by | ||
# the name of the artifact, so we can't have them downloaded directly | ||
# to the screenshots folder. Here we move them into the correct | ||
# locations. | ||
cd new-screenshots | ||
|
||
# "$i" is a folder for an artifact (eg "screenshots-ChromeAndroid"), | ||
# and it contains a single folder with the name we use in our | ||
# screenshots (eg "chrome-Android"). We want the contents of that | ||
# inner folder to be copied to test/test/assets/screenshots/, where a | ||
# folder with the same name (eg "chrome-Android") already exists. | ||
for i in screenshots*; do | ||
cp -a $i/* ../test/test/assets/screenshots/ | ||
done | ||
|
||
# Update the official screenshots for any that has visibly changed. | ||
# This is not a byte-for-byte comparison. | ||
- name: Update screenshots | ||
run: | | ||
# Install prerequisites. | ||
npm ci | ||
|
||
# Update the official screenshots for any that have visibly changed. | ||
# This is not a byte-for-byte comparison, but based on pixel diffs. | ||
./build/updateScreenshots.py | ||
|
||
# Emulate the actions bot. | ||
git config user.name "github-actions[bot]" | ||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com" | ||
|
||
# Update the PR. | ||
git add test/test/assets/screenshots/*/*.png | ||
git commit -m ':robot: Update all screenshots' | ||
# Commit the changes. Ignore failure, in case there are no changes. | ||
git add test/test/assets/screenshots/*/*.png || true | ||
git commit -m ':robot: Update all screenshots' || true | ||
|
||
PR_API_URL="/repos/${{ github.repository }}/pulls/${{ github.event.inputs.pr }}" | ||
REMOTE=$(gh api $GH_API_URL | jq -r .head.repo.html_url) | ||
BRANCH=$(gh api $GH_API_URL | jq -r .head.ref) | ||
- name: Update PR | ||
env: | ||
GH_TOKEN: ${{ github.token }} | ||
run: | | ||
# Update the PR. | ||
PR_API_URL="/repos/${{ github.repository }}/pulls/${{ inputs.pr }}" | ||
REMOTE=$(gh api $PR_API_URL | jq -r .head.repo.html_url) | ||
BRANCH=$(gh api $PR_API_URL | jq -r .head.ref) | ||
|
||
git push "$REMOTE" "$BRANCH" | ||
# If there were no changes, this will do nothing, but succeed. | ||
git push "$REMOTE" HEAD:"$BRANCH" | ||
|
||
- name: Debug | ||
uses: mxschmitt/[email protected] | ||
with: | ||
limit-access-to-actor: true | ||
if: failure() | ||
|
||
set-final-status: | ||
name: Set Final Status | ||
runs-on: ubuntu-latest | ||
needs: [run-lab-tests, update-pr] | ||
# Will run on success or failure, but not if the workflow is cancelled. | ||
if: ${{ success() || failure() }} | ||
steps: | ||
- uses: actions/checkout@v3 | ||
with: | ||
ref: refs/pull/${{ inputs.pr }}/head | ||
|
||
- name: Compute final status | ||
id: compute | ||
run: | | ||
LAB_TEST_STATUS="${{ needs.run-lab-tests.status }}" | ||
UPDATE_PR_STATUS="${{ needs.update-pr.status }}" | ||
|
||
# If run-lab-tests succeeded, use the status of update-pr, otherwise | ||
# use run-lab-tests (which is "failed" or "error"). | ||
if [ "$LAB_TEST_STATUS" == "success" ]; then | ||
echo "status=$UPDATE_PR_STATUS" >> $GITHUB_OUTPUT | ||
else | ||
echo "status=$LAST_TEST_STATUS" >> $GITHUB_OUTPUT | ||
fi | ||
|
||
- name: Report final status | ||
uses: ./.github/workflows/custom-actions/set-commit-status | ||
with: | ||
context: Update All Screenshots | ||
state: ${{ steps.compute.outputs.status }} | ||
token: ${{ secrets.GITHUB_TOKEN }} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
"in tact" -> "intact"
Unless you're saying that using an array makes the elements remain tactful and sensitive.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Done