-
Notifications
You must be signed in to change notification settings - Fork 364
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Infrastructure: Run regression tests in parallel in TravisCI and impr…
…ove test selection logic (pull #1465) Fixes issues #1449 and 1450 by running regression tests in parallel and improving logic that determines which tests to run. To run in parallel: 1. Updates `.travis.yml` to use the [Ava feature](https://github.com/avajs/ava#parallel-runs-in-ci) that allows tests in parallel across three VMs in the CI 2. Updates Ava version to include [this bug fix](avajs/ava#2461) -- the fix prevents a false error when Ava tries to parallelize less than three test files (it parallelizes by files, so one test file is not parallelized). 3. Updates the script that runs the regression tests in the CI to only the test files for the example or test file you are editing to take advantage of the parallelization option. Before, it filtered tests by using a regex on the name of the test. Now, it explicitly lists all the test FILES that should be run. When running `bash regression-tests.sh` locally from a branch that is not master, the script will now run as if the current branch is a pull request against master. You can now test locally that file changes in your branch will trigger the correct tests. The logic is: 1. If `package.json` has been edited, or if test utility files have been edited, or if example utility files have been edited, then all regression tests will be run in the CI. 2. Otherwise, if an example is edited, then the regression tests for all of the examples contained in the same example directory are run. For example: if you edit one of the two checkbox examples, then both the checkbox-1 and checkbox-2 example regression tests will be run in the CI. 3. All test files that have been edited will be run in the CI.
1 parent
962481a
commit 32f0774
Showing
4 changed files
with
260 additions
and
181 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,40 +1,68 @@ | ||
#!/bin/bash | ||
|
||
if ! git diff --name-only $TRAVIS_COMMIT_RANGE | grep -qP '(test/|examples/|package\.json)' | ||
if [[ "$CI" != "true" ]] | ||
then | ||
echo "Examples files were not updated, not running example regression tests." | ||
exit | ||
# When running this script locally, compare the current branch to master | ||
COMMIT_RANGE="..master" | ||
|
||
elif [[ "$TRAVIS_PULL_REQUEST" != "false" ]] | ||
then | ||
# If we are on a PR build, we can use TRAVIS_COMMIT_RANGE | ||
COMMIT_RANGE=$TRAVIS_COMMIT_RANGE | ||
|
||
else | ||
# If we are on a branch build, and it has been force pushed, then TRAVIS_PULL_REQUEST will | ||
# not contain useful information for the branch build. | ||
COMMIT_RANGE="origin/master...$TRAVIS_BRANCH" | ||
git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" | ||
git fetch origin master | ||
fi | ||
|
||
TEST_DIRS=$(git diff --name-only $TRAVIS_COMMIT_RANGE | grep -oP 'test/tests/\K[\w-]+(?=_)' | uniq) | ||
EXAMPLE_DIRS=$(git diff --name-only $TRAVIS_COMMIT_RANGE | grep -oP 'examples/\K[\w-]+(?=/)' | uniq) | ||
AVACMD="npm run regression -- -t" | ||
ARGS='' | ||
|
||
# Only add match args if the example/js or example/css directories or test/index.hs or the | ||
# test/utils.js directories have not been edited. If they have been edited, run all tests. | ||
TEST_FILES=$(git diff --name-only $COMMIT_RANGE | grep -oP 'test/tests/\K.*' | uniq) | ||
TEST_INFRA=$(git diff --name-only $COMMIT_RANGE | grep -oP 'test/(util|index)') | ||
|
||
TEST_INFRA=$(git diff --name-only $TRAVIS_COMMIT_RANGE | grep -oP 'test/(util|index)') | ||
EXAMPLE_DIRS=$(git diff --name-only $COMMIT_RANGE | grep -oP 'examples/\K[\w-]+(?=/)' | uniq) | ||
EXAMPLE_INFRA=$(echo "$EXAMPLE_DIRS" | grep -P '^(js|css)$') | ||
|
||
ARGS='' | ||
PACKAGE_UPDATE=$(git diff --name-only $COMMIT_RANGE | grep -P '(package\.json)') | ||
|
||
if [ -z $TEST_INFRA ] && [ -z $EXAMPLE_INFRA ] | ||
if [[ $TEST_INFRA || $EXAMPLE_INFRA || $PACKAGE_UPDATE ]] | ||
then | ||
|
||
# If the example/js or example/css directories or the test/index.js or the test/utils.js | ||
# or the package.json files have been edited, run all tests. | ||
|
||
ARGS="test/tests/*.js" | ||
|
||
else | ||
|
||
# Otherwise, run only relevant tests | ||
|
||
for D in $EXAMPLE_DIRS | ||
do | ||
# Remove this if statement if we add regression tests for landmark pages | ||
if [ $D != 'landmarks' ] | ||
# Remove this if statement when we add regression tests for landmark pages | ||
if [[ $D != 'landmarks' ]] | ||
then | ||
ARGS="${ARGS} --match ${D}/*" | ||
ARGS="${ARGS} test/tests/${D}*.js" | ||
fi | ||
done | ||
|
||
for F in $TEST_DIRS | ||
for F in $TEST_FILES | ||
do | ||
ARGS="${ARGS} --match ${F}/*" | ||
ARGS="${ARGS} test/tests/${F}" | ||
done | ||
fi | ||
|
||
AVACMD="npm run regression -- -t -c 1 test/tests/*.js ${ARGS}" | ||
echo "$ $AVACMD" | ||
# If there are no arguments, no regression test relevant files have been edited | ||
if [[ -z $ARGS ]] | ||
then | ||
echo "Examples files were not updated, not running example regression tests." | ||
exit | ||
fi | ||
|
||
echo "$" $AVACMD $ARGS | ||
|
||
$AVACMD | ||
$AVACMD $ARGS |