Run contrib tests with unit tests and integration tests for fewer CI …

…delays (#7709) ### Problem The contrib shards consistently are the blocker for CI runs because they appear near the end of the shards. While discussing options to fix this, we realized that there is no compelling reason for the contrib folder to have its own dedicated shard(s). Instead, it would be valuable to use the split between unit tests vs. integration tests with the contrib code too. ### Solution Merge contrib tests into the unit tests and integration tests shards. ### Result The unit tests and integration tests shards will take slightly longer, but we will no longer be blocked by contrib at the end of the run.
pantsbuild · Jun 5, 2019 · 922cd76 · 922cd76
1 parent 5590016
commit 922cd76
Show file tree

Hide file tree

Showing 3 changed files with 27 additions and 106 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -945,23 +945,26 @@ matrix:
         - *py27_linux_test_config_env
         - CACHE_NAME=linuxunittests.py27
       script:
-        - travis_wait 60 ./build-support/bin/ci.sh -2lp
+        - ./build-support/bin/ci.sh -2l
+        - travis_wait 60 ./build-support/bin/ci.sh -2p
 
     - <<: *py36_linux_test_config
       name: "Unit tests (Py3.6 PEX)"
       env:
         - *py36_linux_test_config_env
         - CACHE_NAME=linuxunittests.py36
       script:
-        - travis_wait 60 ./build-support/bin/ci.sh -lp
+        - ./build-support/bin/ci.sh -l
+        - travis_wait 60 ./build-support/bin/ci.sh -p
 
     - <<: *py37_linux_test_config
       name: "Unit tests (Py3.7 PEX)"
       env:
         - *py37_linux_test_config_env
         - CACHE_NAME=linuxunittests.py37
       script:
-        - travis_wait 60 ./build-support/bin/ci.sh -7lp
+        - ./build-support/bin/ci.sh -7l
+        - travis_wait 60 ./build-support/bin/ci.sh -7p
 
     - <<: *py27_linux_build_wheels_ucs2
     - <<: *py27_linux_build_wheels_ucs4
@@ -1655,41 +1658,6 @@ matrix:
     - <<: *linux_rust_tests
     - <<: *osx_rust_tests
 
-    - <<: *py27_linux_test_config
-      name: "Python contrib tests (Py2.7 PEX)"
-      stage: *test
-      env:
-        - *py27_linux_test_config_env
-        - CACHE_NAME=linuxcontribtests.py27
-      script:
-        - ./build-support/bin/ci.sh -2n
-
-    - <<: *py36_linux_test_config
-      name: "Python contrib tests (Py3.6 PEX)"
-      env:
-        - *py36_linux_test_config_env
-        - CACHE_NAME=linuxcontribtests.py36
-      script:
-        - ./build-support/bin/ci.sh -n
-
-    - <<: *py36_linux_test_config
-      name: "Python contrib tests with Pantsd (Py3.6 PEX)"
-      stage: *test_cron
-      env:
-        - *py36_linux_test_config_env
-        - *run_tests_under_pantsd
-        - CACHE_NAME=linuxcontribtests.py36
-      script:
-        - ./build-support/bin/ci.sh -n
-
-    - <<: *py37_linux_test_config
-      name: "Python contrib tests (Py3.7 PEX)"
-      env:
-        - *py37_linux_test_config_env
-        - CACHE_NAME=linuxcontribtests.py37
-      script:
-        - ./build-support/bin/ci.sh -7n
-
     - <<: *py27_osx_10_12_sanity_check
     - <<: *py36_osx_10_12_sanity_check
     - <<: *py37_osx_10_12_sanity_check

diff --git a/build-support/bin/ci.sh b/build-support/bin/ci.sh
@@ -13,7 +13,7 @@ function usage() {
   cat <<EOF
 Runs commons tests for local or hosted CI.
 
-Usage: $0 (-h|-2fxbkmrjlpuneycitzsw)
+Usage: $0 (-h|-2fxbkmrjlpecitzsw)
  -h           print out this help message
  -2           Run using Python 2.7 (defaults to using Python 3.6).
  -7           Run using Python 3.7 (defaults to using Python 3.6).
@@ -27,18 +27,9 @@ Usage: $0 (-h|-2fxbkmrjlpuneycitzsw)
  -j           run core jvm tests
  -l           run internal backends python tests
  -p           run core python tests
- -u SHARD_NUMBER/TOTAL_SHARDS
-              if running core python tests, divide them into
-              TOTAL_SHARDS shards and just run those in SHARD_NUMBER
-              to run only even tests: '-u 0/2', odd: '-u 1/2'
- -n           run contrib python tests
  -e           run rust tests
  -s           run clippy on rust code
  -a           run cargo audit of rust dependencies
- -y SHARD_NUMBER/TOTAL_SHARDS
-              if running contrib python tests, divide them into
-              TOTAL_SHARDS shards and just run those in SHARD_NUMBER
-              to run only even tests: '-u 0/2', odd: '-u 1/2'
  -c           run pants integration tests (includes examples and testprojects)
  -i SHARD_NUMBER/TOTAL_SHARDS
               if running integration tests, divide them into
@@ -55,10 +46,9 @@ EOF
 }
 
 # No python test sharding (1 shard) by default.
-python_contrib_shard="0/1"
 python_intg_shard="0/1"
 
-while getopts "h27fxbmrjlpeasny:ci:tz" opt; do
+while getopts "h27fxbmrjlpeasci:tz" opt; do
   case ${opt} in
     h) usage ;;
     2) python_two="true" ;;
@@ -74,8 +64,6 @@ while getopts "h27fxbmrjlpeasny:ci:tz" opt; do
     e) run_rust_tests="true" ;;
     a) run_cargo_audit="true" ;;
     s) run_rust_clippy="true" ;;
-    n) run_contrib="true" ;;
-    y) python_contrib_shard=${OPTARG} ;;
     c) run_integration="true" ;;
     i) python_intg_shard=${OPTARG} ;;
     t) run_lint="true" ;;
@@ -210,23 +198,15 @@ if [[ "${run_python:-false}" == "true" ]]; then
     ./pants.pex --tag='-integration' --filter-type='python_tests' filter src/python:: tests/python:: > all_targets.txt
     comm -23 <(sort all_targets.txt) <(sort "${known_v2_failures_file}") > v2_targets.txt
     ./pants.pex --no-v1 --v2 --target-spec-file=v2_targets.txt test.pytest -- "${PYTEST_PASSTHRU_ARGS[@]}"
-  ) || die "Core Python test failure"
+  ) || die "Core Python test failure (V2 test runner)"
   (
-    ./pants.pex --target-spec-file="${known_v2_failures_file}" test.pytest -- "${PYTEST_PASSTHRU_ARGS[@]}"
-  ) || die "Core Python test failure"
-  end_travis_section
-fi
-
-if [[ "${run_contrib:-false}" == "true" ]]; then
-  if [[ "0/1" != "${python_contrib_shard}" ]]; then
-    shard_desc=" [shard ${python_contrib_shard}]"
-  fi
-  start_travis_section "ContribTests" "Running contrib python tests${shard_desc}"
+    ./pants.pex --target-spec-file="${known_v2_failures_file}" \
+      test.pytest --chroot -- "${PYTEST_PASSTHRU_ARGS[@]}"
+  ) || die "Core Python test failure (V1 test runner)"
   (
-    ./pants.pex --exclude-target-regexp='.*/testprojects/.*' test.pytest \
-    "--test-pytest-test-shard=${python_contrib_shard}" \
-    contrib:: -- "${PYTEST_PASSTHRU_ARGS[@]}"
-  ) || die "Contrib python test failure"
+    ./pants.pex --tag='-integration' --exclude-target-regexp='.*/testprojects/.*' \
+      test.pytest contrib:: -- "${PYTEST_PASSTHRU_ARGS[@]}"
+  ) || die "Contrib Python test failure"
   end_travis_section
 fi
 
@@ -298,7 +278,12 @@ if [[ "${run_integration:-false}" == "true" ]]; then
     ./pants.pex --tag='+integration' test.pytest \
       "--test-pytest-test-shard=${python_intg_shard}" \
       src/python:: tests/python:: -- "${PYTEST_PASSTHRU_ARGS[@]}"
-  ) || die "Pants Integration test failure"
+  ) || die "Pants integration test failure"
+  (
+    ./pants.pex --tag='+integration' --exclude-target-regexp='.*/testprojects/.*' \
+      test.pytest --test-pytest-test-shard="${python_intg_shard}" \
+      contrib:: -- "${PYTEST_PASSTHRU_ARGS[@]}"
+  ) || die "Contrib integration test failure"
   end_travis_section
 fi
 

diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache
@@ -888,23 +888,26 @@ matrix:
         - *py27_linux_test_config_env
         - CACHE_NAME=linuxunittests.py27
       script:
-        - travis_wait 60 ./build-support/bin/ci.sh -2lp
+        - ./build-support/bin/ci.sh -2l
+        - travis_wait 60 ./build-support/bin/ci.sh -2p
 
     - <<: *py36_linux_test_config
       name: "Unit tests (Py3.6 PEX)"
       env:
         - *py36_linux_test_config_env
         - CACHE_NAME=linuxunittests.py36
       script:
-        - travis_wait 60 ./build-support/bin/ci.sh -lp
+        - ./build-support/bin/ci.sh -l
+        - travis_wait 60 ./build-support/bin/ci.sh -p
 
     - <<: *py37_linux_test_config
       name: "Unit tests (Py3.7 PEX)"
       env:
         - *py37_linux_test_config_env
         - CACHE_NAME=linuxunittests.py37
       script:
-        - travis_wait 60 ./build-support/bin/ci.sh -7lp
+        - ./build-support/bin/ci.sh -7l
+        - travis_wait 60 ./build-support/bin/ci.sh -7p
 
     - <<: *py27_linux_build_wheels_ucs2
     - <<: *py27_linux_build_wheels_ucs4
@@ -960,41 +963,6 @@ matrix:
     - <<: *linux_rust_tests
     - <<: *osx_rust_tests
 
-    - <<: *py27_linux_test_config
-      name: "Python contrib tests (Py2.7 PEX)"
-      stage: *test
-      env:
-        - *py27_linux_test_config_env
-        - CACHE_NAME=linuxcontribtests.py27
-      script:
-        - ./build-support/bin/ci.sh -2n
-
-    - <<: *py36_linux_test_config
-      name: "Python contrib tests (Py3.6 PEX)"
-      env:
-        - *py36_linux_test_config_env
-        - CACHE_NAME=linuxcontribtests.py36
-      script:
-        - ./build-support/bin/ci.sh -n
-
-    - <<: *py36_linux_test_config
-      name: "Python contrib tests with Pantsd (Py3.6 PEX)"
-      stage: *test_cron
-      env:
-        - *py36_linux_test_config_env
-        - *run_tests_under_pantsd
-        - CACHE_NAME=linuxcontribtests.py36
-      script:
-        - ./build-support/bin/ci.sh -n
-
-    - <<: *py37_linux_test_config
-      name: "Python contrib tests (Py3.7 PEX)"
-      env:
-        - *py37_linux_test_config_env
-        - CACHE_NAME=linuxcontribtests.py37
-      script:
-        - ./build-support/bin/ci.sh -7n
-
     - <<: *py27_osx_10_12_sanity_check
     - <<: *py36_osx_10_12_sanity_check
     - <<: *py37_osx_10_12_sanity_check