From 5890f68b1b1f078af5ebd058e5d5ab17e849976e Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Tue, 18 Dec 2018 09:57:27 -0700 Subject: [PATCH 001/105] Default in CI to Py3, not Py2 --- .travis.yml | 36 ++++++++++++++++++------------------ build-support/bin/ci.sh | 15 +++++++-------- 2 files changed, 25 insertions(+), 26 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1ee7d1f5720..4bbb177bc5f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -203,92 +203,92 @@ matrix: - <<: *default_test_config name: "Py2 - Unit tests for pants and pants-plugins" script: - - ./build-support/bin/travis-ci.sh -lp + - ./build-support/bin/travis-ci.sh -2lp - <<: *default_test_config name: "Py3 - Unit tests for pants and pants-plugins" script: - - ./build-support/bin/travis-ci.sh -3lp + - ./build-support/bin/travis-ci.sh -lp - <<: *default_test_config name: "Py2 - Python contrib tests" script: - - ./build-support/bin/travis-ci.sh -n + - ./build-support/bin/travis-ci.sh -2n - <<: *default_test_config name: "Py3 - Python contrib tests" script: - - ./build-support/bin/travis-ci.sh -3n + - ./build-support/bin/travis-ci.sh -n - <<: *default_test_config name: "Integration tests for pants - shard 1" script: - - ./build-support/bin/travis-ci.sh -c3 -i 0/7 + - ./build-support/bin/travis-ci.sh -c -i 0/7 - <<: *default_test_config name: "Integration tests for pants - shard 2" script: - - ./build-support/bin/travis-ci.sh -c3 -i 1/7 + - ./build-support/bin/travis-ci.sh -c -i 1/7 - <<: *default_test_config name: "Integration tests for pants - shard 3" script: - - ./build-support/bin/travis-ci.sh -c3 -i 2/7 + - ./build-support/bin/travis-ci.sh -c -i 2/7 - <<: *default_test_config name: "Integration tests for pants - shard 4" script: - - ./build-support/bin/travis-ci.sh -c3 -i 3/7 + - ./build-support/bin/travis-ci.sh -c -i 3/7 - <<: *default_test_config name: "Integration tests for pants - shard 5" script: - - ./build-support/bin/travis-ci.sh -c3 -i 4/7 + - ./build-support/bin/travis-ci.sh -c -i 4/7 - <<: *default_test_config name: "Integration tests for pants - shard 6" script: - - ./build-support/bin/travis-ci.sh -c3 -i 5/7 + - ./build-support/bin/travis-ci.sh -c -i 5/7 - <<: *default_test_config name: "Integration tests for pants - shard 7" script: - - ./build-support/bin/travis-ci.sh -c3 -i 6/7 + - ./build-support/bin/travis-ci.sh -c -i 6/7 - <<: *default_cron_test_config name: "Integration tests for pants (Python 2) - shard 1" script: - - ./build-support/bin/travis-ci.sh -c -i 0/7 + - ./build-support/bin/travis-ci.sh -c2 -i 0/7 - <<: *default_cron_test_config name: "Integration tests for pants (Python 2) - shard 2" script: - - ./build-support/bin/travis-ci.sh -c -i 1/7 + - ./build-support/bin/travis-ci.sh -c2 -i 1/7 - <<: *default_cron_test_config name: "Integration tests for pants (Python 2) - shard 3" script: - - ./build-support/bin/travis-ci.sh -c -i 2/7 + - ./build-support/bin/travis-ci.sh -c2 -i 2/7 - <<: *default_cron_test_config name: "Integration tests for pants (Python 2) - shard 4" script: - - ./build-support/bin/travis-ci.sh -c -i 3/7 + - ./build-support/bin/travis-ci.sh -c2 -i 3/7 - <<: *default_cron_test_config name: "Integration tests for pants (Python 2) - shard 5" script: - - ./build-support/bin/travis-ci.sh -c -i 4/7 + - ./build-support/bin/travis-ci.sh -c2 -i 4/7 - <<: *default_cron_test_config name: "Integration tests for pants (Python 2) - shard 6" script: - - ./build-support/bin/travis-ci.sh -c -i 5/7 + - ./build-support/bin/travis-ci.sh -c2 -i 5/7 - <<: *default_cron_test_config name: "Integration tests for pants (Python 2) - shard 7" script: - - ./build-support/bin/travis-ci.sh -c -i 6/7 + - ./build-support/bin/travis-ci.sh -c2 -i 6/7 # Rust on linux - <<: *linux_with_fuse diff --git a/build-support/bin/ci.sh b/build-support/bin/ci.sh index b07a7515217..e9db2a361cf 100755 --- a/build-support/bin/ci.sh +++ b/build-support/bin/ci.sh @@ -13,10 +13,9 @@ function usage() { cat < Date: Sat, 22 Dec 2018 15:59:29 -0700 Subject: [PATCH 002/105] Change CI to either use ./pants or ./pants3 --- build-support/bin/ci.sh | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/build-support/bin/ci.sh b/build-support/bin/ci.sh index e9db2a361cf..1173e107816 100755 --- a/build-support/bin/ci.sh +++ b/build-support/bin/ci.sh @@ -122,7 +122,8 @@ if [[ "${run_bootstrap:-true}" == "true" ]]; then if [[ "${run_bootstrap_clean:-false}" == "true" ]]; then ./build-support/python/clean.sh || die "Failed to clean before bootstrapping pants." fi - ./pants ${bootstrap_compile_args[@]} binary \ + pants_script="./pants"; [[ "${python_two:-false}" == "false" ]] && pants_script="./pants3" + ${pants_script} ${bootstrap_compile_args[@]} binary \ src/python/pants/bin:pants_local_binary && \ mv dist/pants_local_binary.pex pants.pex && \ ./pants.pex -V @@ -130,15 +131,6 @@ if [[ "${run_bootstrap:-true}" == "true" ]]; then end_travis_section fi -# NB: Ordering matters here. We (currently) always bootstrap a Python 2 pex. -if [[ "${python_two:-false}" == "false" ]]; then - banner "Setting interpreter constraints to Python 3!" - export PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS='["CPython>=3.6,<4"]' - # TODO: Clear interpreters, otherwise this constraint does not end up applying due to a cache - # bug between the `./pants binary` and further runs. - ./pants.pex clean-all -fi - if [[ "${run_sanity_checks:-false}" == "true" ]]; then start_travis_section "SanityCheck" "Sanity checking bootstrapped pants and repo BUILD files" sanity_tests=( From 6ed019d3c128ffe8895e9b361a9f0cbe136bfa79 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Sun, 23 Dec 2018 10:40:37 -0700 Subject: [PATCH 003/105] Brew install Python3 on OS X shards --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 4bbb177bc5f..780c64538df 100644 --- a/.travis.yml +++ b/.travis.yml @@ -92,6 +92,8 @@ osx_test_config: &osx_test_config os: osx stage: *test language: generic + before_install: + - brew update && brew install python3 script: - MODE=debug ./build-support/bin/travis-ci.sh -m From 60268121422e84fe56ebbd2c783913e348cabb23 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Sun, 23 Dec 2018 10:41:07 -0700 Subject: [PATCH 004/105] Change language to Py 3.6 to add support for non-travis_ci.sh shards --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 780c64538df..7dafc44585e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -80,7 +80,7 @@ default_test_config: &default_test_config - libssl-dev stage: *test language: python - python: &python_version "2.7" + python: &python_version "3.6" before_install: - PATH="/usr/lib/jvm/java-8-openjdk-amd64/jre/bin":$PATH - JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 From 0ef79c384f3e57b86f0e4574929bce84e8f47879 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Sun, 23 Dec 2018 10:42:05 -0700 Subject: [PATCH 005/105] Squash merge py3 --- .travis.yml | 3 ++- build-support/pants_venv | 6 +++++- build-support/python/clean.sh | 1 + build-support/virtualenv | 7 ++++--- pants.ini | 9 +++++---- pants3 | 8 ++++++++ src/docs/export.md | 2 +- src/docs/intellij.md | 2 +- .../pants/backend/python/subsystems/python_setup.py | 2 +- 9 files changed, 28 insertions(+), 12 deletions(-) create mode 100755 pants3 diff --git a/.travis.yml b/.travis.yml index 7dafc44585e..a12b1336923 100644 --- a/.travis.yml +++ b/.travis.yml @@ -50,7 +50,8 @@ cache: # using its own isolated cache: # https://github.com/pantsbuild/pants/issues/2485 - ${HOME}/.npm - - build-support/pants_dev_deps.venv + - build-support/pants_dev_deps.py2.venv + - build-support/pants_dev_deps.py3.venv - src/rust/engine/target # Stages are documented here: https://docs.travis-ci.com/user/build-stages diff --git a/build-support/pants_venv b/build-support/pants_venv index 4b4c3719c02..587ffc47066 100755 --- a/build-support/pants_venv +++ b/build-support/pants_venv @@ -12,7 +12,8 @@ REQUIREMENTS=( ) function venv_dir() { - echo ${REPO_ROOT}/build-support/pants_dev_deps.venv + py_version="py2"; [ "${PANTS_USE_PYTHON3}" = true ] && py_version="py3" + echo ${REPO_ROOT}/build-support/pants_dev_deps.${py_version}.venv } function activate_venv() { @@ -21,6 +22,9 @@ function activate_venv() { function create_venv() { rm -rf "$(venv_dir)" + # Also delete `pants_dev_deps.venv` to remove legacy folder now that + # we use `pants_dev_deps.py2.venv` and `pants_dev_deps.py3.venv` folders + rm -rf "${REPO_ROOT}/build-support/pants_dev_deps.venv" "${REPO_ROOT}/build-support/virtualenv" "$(venv_dir)" } diff --git a/build-support/python/clean.sh b/build-support/python/clean.sh index 65119ffb0aa..a0a0dab3bdc 100755 --- a/build-support/python/clean.sh +++ b/build-support/python/clean.sh @@ -3,5 +3,6 @@ PANTS_BASE=$(dirname $0)/../.. rm -rf ${HOME}/.pex rm -rf ${PANTS_BASE}/build-support/pants_dev_deps.venv +rm -rf ${PANTS_BASE}/build-support/pants_dev_deps.py{2,3}.venv rm -rf ${PANTS_BASE}/.pants.d find ${PANTS_BASE} -name '*.pyc' | xargs rm -f diff --git a/build-support/virtualenv b/build-support/virtualenv index d34e285fa3d..3478f2a704d 100755 --- a/build-support/virtualenv +++ b/build-support/virtualenv @@ -14,10 +14,11 @@ fi source ${REPO_ROOT}/build-support/common.sh if [[ -z "${PY}" ]]; then - if which python2.7 >/dev/null; then - PY=`which python2.7` + python_bin_name="python2.7"; [ "${PANTS_USE_PYTHON3}" = true ] && python_bin_name="python3" + if which "${python_bin_name}" >/dev/null; then + PY=`which ${python_bin_name}` else - die 'No python2.7 interpreter found on the path. Python will not work!' + die "No ${python_bin_name} interpreter found on the path. Python will not work!" fi fi diff --git a/pants.ini b/pants.ini index ba51a2631ef..50b104e6f53 100644 --- a/pants.ini +++ b/pants.ini @@ -369,10 +369,11 @@ verify_commit: False [python-setup] -# We only support pants running under 2.7 for now with 3.3+ support to be added later. -# Any example or test targets that are meant to test interpreters outside pants own -# acceptable set should specify an explicit compatibility constraint. -interpreter_constraints: ["CPython>=2.7,<3"] +# We will support Python 2 until release 1.16, at which we will drop Python 2 support and +# require Python 3.6+ to run Pants. +# Note this does not mean client code has to use these specified versions, only that +# an appropriate interpreter must be discoverable. +interpreter_constraints: ["CPython>=2.7,<3","CPython>=3.6,<4"] interpreter_cache_dir: %(pants_bootstrapdir)s/python_cache/interpreters resolver_cache_dir: %(pants_bootstrapdir)s/python_cache/requirements diff --git a/pants3 b/pants3 new file mode 100755 index 00000000000..0675a2e6738 --- /dev/null +++ b/pants3 @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +# This bootstrap script invokes Pants using a Python 3 interpreter. + +export PANTS_USE_PYTHON3=true +./pants "$@" diff --git a/src/docs/export.md b/src/docs/export.md index 4902de7b38a..dc09e3128c1 100644 --- a/src/docs/export.md +++ b/src/docs/export.md @@ -101,7 +101,7 @@ The following is an abbreviated export file from a command in the pants repo: "python_setup": { "interpreters": { "CPython-2.7.10": { - "binary": "/Users/user/pants/build-support/pants_dev_deps.venv/bin/python2.7", + "binary": "/Users/user/pants/build-support/pants_dev_deps.py2.venv/bin/python2.7", "chroot": "/Users/user/pants/.pants.d/python-setup/chroots/e8da2c200f36ca0a1b8a60c12590a59209250b1a" } }, diff --git a/src/docs/intellij.md b/src/docs/intellij.md index e8f3fa0ab8f..df442bc80ed 100644 --- a/src/docs/intellij.md +++ b/src/docs/intellij.md @@ -44,7 +44,7 @@ SDK". ![image](images/intellij-new-pythonsdk.png) This will be a "local" interpreter and you'll need to select the virtual -environment bootstrapped above; it's in `build-support/pants_dev_deps.venv`. +environment bootstrapped above; it's in `build-support/pants_dev_deps.py2.venv` if you want to use Python 2 or `build-support/pants_dev_deps.py3.venv` if you want to use Python 3 for the underlying Pants engine. ![image](images/intellij-select-venv.png) diff --git a/src/python/pants/backend/python/subsystems/python_setup.py b/src/python/pants/backend/python/subsystems/python_setup.py index 76bc246196b..4b6009db65e 100644 --- a/src/python/pants/backend/python/subsystems/python_setup.py +++ b/src/python/pants/backend/python/subsystems/python_setup.py @@ -26,7 +26,7 @@ class PythonSetup(Subsystem): @classmethod def register_options(cls, register): super(PythonSetup, cls).register_options(register) - register('--interpreter-constraints', advanced=True, default=['CPython>=2.7,<3'], type=list, + register('--interpreter-constraints', advanced=True, default=['CPython>=2.7,<3', 'CPython>=3.6,<4'], type=list, metavar='', help="Constrain the selected Python interpreter. Specify with requirement syntax, " "e.g. 'CPython>=2.7,<3' (A CPython interpreter with version >=2.7 AND version <3)" From 99785dd07ca591e2cefeab4651d341e2a4a42444 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Sun, 23 Dec 2018 12:54:40 -0700 Subject: [PATCH 006/105] Brew calls Python3 simply python --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a12b1336923..bb8c12c5602 100644 --- a/.travis.yml +++ b/.travis.yml @@ -94,7 +94,8 @@ osx_test_config: &osx_test_config stage: *test language: generic before_install: - - brew update && brew install python3 + # install Python 3 + - brew install python script: - MODE=debug ./build-support/bin/travis-ci.sh -m From f4cc591a225671dd25f62a03ef81831eccbc3231 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Sun, 23 Dec 2018 12:59:24 -0700 Subject: [PATCH 007/105] Also brew install Python 3 for Rust macOS shard --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index bb8c12c5602..43c64a9a0b5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -317,6 +317,8 @@ matrix: language: generic before_install: - brew tap caskroom/cask && brew update && brew cask install osxfuse + # Install Python 3 + - brew install python before_script: - ulimit -c unlimited - ulimit -n 8192 From b6d127a701b20628a8912e4acb449c07883b81b9 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Sun, 23 Dec 2018 18:47:45 -0700 Subject: [PATCH 008/105] Install Python 3 on macOS using `brew upgrade python` Travis recommends doing this for some reason --- .travis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 43c64a9a0b5..1ea94a60a11 100644 --- a/.travis.yml +++ b/.travis.yml @@ -94,8 +94,8 @@ osx_test_config: &osx_test_config stage: *test language: generic before_install: - # install Python 3 - - brew install python + # Upgrade to Python 3 + - brew upgrade python script: - MODE=debug ./build-support/bin/travis-ci.sh -m @@ -317,8 +317,8 @@ matrix: language: generic before_install: - brew tap caskroom/cask && brew update && brew cask install osxfuse - # Install Python 3 - - brew install python + # Upgrade to Python 3 + - brew upgrade python before_script: - ulimit -c unlimited - ulimit -n 8192 From 116acdc34657ad71d2c0307e8be9200d8cd76a43 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Wed, 9 Jan 2019 09:12:55 -0700 Subject: [PATCH 009/105] Squash merge newest py3 changes --- build-support/pants_venv | 12 +++++++++--- pants3 | 5 +++++ src/rust/engine/src/cffi_build.rs | 7 +++++++ 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/build-support/pants_venv b/build-support/pants_venv index 587ffc47066..47a344549e9 100755 --- a/build-support/pants_venv +++ b/build-support/pants_venv @@ -22,9 +22,15 @@ function activate_venv() { function create_venv() { rm -rf "$(venv_dir)" - # Also delete `pants_dev_deps.venv` to remove legacy folder now that - # we use `pants_dev_deps.py2.venv` and `pants_dev_deps.py3.venv` folders - rm -rf "${REPO_ROOT}/build-support/pants_dev_deps.venv" + # If `pants_dev_deps.venv` still exists, delete both its legacy folder and the cached + # Python interpreter folder, which contains problematic symlinks. + # Note we only perform these removals for the first time, to avoid continually deleting + # the cache when switching between using `pants_dev_deps.py2.venv` and `pants_dev_deps.py3.venv`. + legacy_venv_dir="${REPO_ROOT}/build-support/pants_dev_deps.venv" + if [ -d "${legacy_venv_dir}" ]; then + rm -rf "${legacy_venv_dir}" + rm -rf "${HOME}/.cache/pants/python_cache/interpreters" + fi "${REPO_ROOT}/build-support/virtualenv" "$(venv_dir)" } diff --git a/pants3 b/pants3 index 0675a2e6738..cd5d1f5f068 100755 --- a/pants3 +++ b/pants3 @@ -4,5 +4,10 @@ # This bootstrap script invokes Pants using a Python 3 interpreter. +# Use Py3 under-the-hood export PANTS_USE_PYTHON3=true + +# Use Py3 for subprocesses +export PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS='["CPython>=3.6,<4"]' + ./pants "$@" diff --git a/src/rust/engine/src/cffi_build.rs b/src/rust/engine/src/cffi_build.rs index 0c81313bf68..a865ddbe30c 100644 --- a/src/rust/engine/src/cffi_build.rs +++ b/src/rust/engine/src/cffi_build.rs @@ -109,6 +109,13 @@ fn main() -> Result<(), CffiBuildError> { // (which is equivalent to `-ldylib=stdc++`). // * Specifying `rustc-link-lib=stdc++` // (which is equivalent to `rustc-link-lib=dylib=stdc++). + + // NB: When built with Python 3, `native_engine.so` only works with a Python 3 interpreter. + // When built with Python 2, it works with both Python 2 and Python 3. + // So, we check to see if the under-the-hood interpreter has changed and rebuild the native engine + // when needed. + println!("rerun-if-env-changed=PANTS_USE_PYTHON3"); + if cfg!(target_os = "linux") { println!("cargo:rustc-link-lib=static=stdc++"); } From eceb62d37bd578617a64574bd54b77cee60255d2 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Wed, 9 Jan 2019 09:18:27 -0700 Subject: [PATCH 010/105] Move CI changes into mustache file, don't hardcode --- build-support/travis/travis.yml.mustache | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index 40eface4b64..ccbbeba9d90 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -58,7 +58,8 @@ cache: # using its own isolated cache: # https://github.com/pantsbuild/pants/issues/2485 - ${HOME}/.npm - - build-support/pants_dev_deps.venv + - build-support/pants_dev_deps.py2.venv + - build-support/pants_dev_deps.py3.venv - src/rust/engine/target # Stages are documented here: https://docs.travis-ci.com/user/build-stages @@ -89,7 +90,7 @@ default_test_config: &default_test_config - libssl-dev stage: *test language: python - python: &python_version "2.7" + python: &python_version "3.6" before_install: {{>before_install}} before_script: @@ -100,6 +101,9 @@ osx_test_config: &osx_test_config os: osx stage: *test language: generic + before_install: + # Upgrade to Python 3 + - brew upgrade python script: - MODE=debug ./build-support/bin/travis-ci.sh -m @@ -217,22 +221,22 @@ matrix: - <<: *default_test_config name: "Py2 - Unit tests for pants and pants-plugins" script: - - ./build-support/bin/travis-ci.sh -blp + - ./build-support/bin/travis-ci.sh -2blp - <<: *default_test_config name: "Py3 - Unit tests for pants and pants-plugins" script: - - ./build-support/bin/travis-ci.sh -3blp + - ./build-support/bin/travis-ci.sh -blp - <<: *default_test_config name: "Py2 - Python contrib tests" script: - - ./build-support/bin/travis-ci.sh -bn + - ./build-support/bin/travis-ci.sh -2bn - <<: *default_test_config name: "Py3 - Python contrib tests" script: - - ./build-support/bin/travis-ci.sh -3bn + - ./build-support/bin/travis-ci.sh -bn {{#integration_shards}} - <<: *default_test_config @@ -240,7 +244,7 @@ matrix: env: - CACHE_NAME=integrationshard{{.}} # Ensure distinct caches. script: - - ./build-support/bin/travis-ci.sh -bc3 -i {{.}}/{{integration_shards_length}} + - ./build-support/bin/travis-ci.sh -bc -i {{.}}/{{integration_shards_length}} {{/integration_shards}} {{#integration_shards}} @@ -249,7 +253,7 @@ matrix: env: - CACHE_NAME=cronshard{{.}} # Ensure distinct caches. script: - - ./build-support/bin/travis-ci.sh -bc -i {{.}}/{{integration_shards_length}} + - ./build-support/bin/travis-ci.sh -bc2 -i {{.}}/{{integration_shards_length}} {{/integration_shards}} @@ -276,6 +280,8 @@ matrix: language: generic before_install: - brew tap caskroom/cask && brew update && brew cask install osxfuse + # Upgrade to Python 3 + - brew upgrade python before_script: - ulimit -c unlimited - ulimit -n 8192 From fccb7369320a9b73e4d4d0260e991a063365a713 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Wed, 9 Jan 2019 12:44:24 -0700 Subject: [PATCH 011/105] Constrain subprocesses to corresponding interpreter --- build-support/bin/ci.sh | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/build-support/bin/ci.sh b/build-support/bin/ci.sh index 3580a73a9dd..d06e5a67160 100755 --- a/build-support/bin/ci.sh +++ b/build-support/bin/ci.sh @@ -119,12 +119,19 @@ if [[ "${run_pre_commit_checks:-false}" == "true" ]]; then fi if [[ "${run_bootstrap:-true}" == "true" ]]; then - start_travis_section "Bootstrap" "Bootstrapping pants" + # determine interpreter + if [[ "${python_two:-false}" == "false" ]]; then + py_version_number="3" + pants_script="./pants3" + else + py_version_number="2" + pants_script="./pants" + fi + start_travis_section "Bootstrap" "Bootstrapping pants with Python ${py_version_number} under-the-hood." ( if [[ "${run_bootstrap_clean:-false}" == "true" ]]; then ./build-support/python/clean.sh || die "Failed to clean before bootstrapping pants." fi - pants_script="./pants"; [[ "${python_two:-false}" == "false" ]] && pants_script="./pants3" ${pants_script} ${bootstrap_compile_args[@]} binary \ src/python/pants/bin:pants_local_binary && \ mv dist/pants_local_binary.pex pants.pex && \ @@ -139,6 +146,16 @@ fi # integration tests that shell out to `./pants`, so we set this env var for those cases. export RUN_PANTS_FROM_PEX=1 +# NB: In addition to choosing which interpreter we use under-the-hood to bootstrap Pants, +# we must also set interpreter constraints for any subprocesses to use the corresponding interpreter. +if [[ "${python_two:-false}" == "false" ]]; then + export PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS='["CPython>=3.6,<4"]' +fi +banner "Using Python ${py_version_number} to execute subprocesses." +# TODO: Clear interpreters, otherwise this constraint does not end up applying due to a cache +# bug between the `./pants binary` and further runs. +./pants.pex clean-all + if [[ "${run_sanity_checks:-false}" == "true" ]]; then start_travis_section "SanityCheck" "Sanity checking bootstrapped pants and repo BUILD files" sanity_tests=( From bd54d8526482080b2c0bbe41091d4c7d72abe521 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Thu, 10 Jan 2019 09:19:49 -0700 Subject: [PATCH 012/105] Move setting interpreter constraints up in order Locally, I ran into an issue when trying to bootstrap with Py3. The command ./pants -V was running with a Py2 subprocess even though the PEX was Py3 under-the-hood, so I ran into the _Py_Dealloc problem. Now we properly set the constraint before anything else. --- build-support/bin/ci.sh | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/build-support/bin/ci.sh b/build-support/bin/ci.sh index d06e5a67160..39a6b6af4b3 100755 --- a/build-support/bin/ci.sh +++ b/build-support/bin/ci.sh @@ -118,15 +118,20 @@ if [[ "${run_pre_commit_checks:-false}" == "true" ]]; then end_travis_section fi +# Determine interpreter for both under-the-hood and for subprocesses. +# Order matters here. We must constrain subprocesses before running the bootstrap stage, +# or we will encounter the _Py_Dealloc error when running `./pants.pex -V` with Python 3 under-the-hood. +if [[ "${python_two:-false}" == "false" ]]; then + py_version_number="3" + pants_script="./pants3" + export PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS='["CPython>=3.6,<4"]' +else + py_version_number="2" + pants_script="./pants" +fi +banner "Using Python ${py_version_number} to execute subprocesses." + if [[ "${run_bootstrap:-true}" == "true" ]]; then - # determine interpreter - if [[ "${python_two:-false}" == "false" ]]; then - py_version_number="3" - pants_script="./pants3" - else - py_version_number="2" - pants_script="./pants" - fi start_travis_section "Bootstrap" "Bootstrapping pants with Python ${py_version_number} under-the-hood." ( if [[ "${run_bootstrap_clean:-false}" == "true" ]]; then @@ -146,12 +151,6 @@ fi # integration tests that shell out to `./pants`, so we set this env var for those cases. export RUN_PANTS_FROM_PEX=1 -# NB: In addition to choosing which interpreter we use under-the-hood to bootstrap Pants, -# we must also set interpreter constraints for any subprocesses to use the corresponding interpreter. -if [[ "${python_two:-false}" == "false" ]]; then - export PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS='["CPython>=3.6,<4"]' -fi -banner "Using Python ${py_version_number} to execute subprocesses." # TODO: Clear interpreters, otherwise this constraint does not end up applying due to a cache # bug between the `./pants binary` and further runs. ./pants.pex clean-all From be59b779eedc600cbd17baf8947897e2153805aa Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Thu, 10 Jan 2019 09:58:46 -0700 Subject: [PATCH 013/105] Attempt to fix contrib hasher encoding issue --- .../scrooge/tasks/java_thrift_library_fingerprint_strategy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/scrooge/src/python/pants/contrib/scrooge/tasks/java_thrift_library_fingerprint_strategy.py b/contrib/scrooge/src/python/pants/contrib/scrooge/tasks/java_thrift_library_fingerprint_strategy.py index de5c5ff3f61..d78b20435e0 100644 --- a/contrib/scrooge/src/python/pants/contrib/scrooge/tasks/java_thrift_library_fingerprint_strategy.py +++ b/contrib/scrooge/src/python/pants/contrib/scrooge/tasks/java_thrift_library_fingerprint_strategy.py @@ -39,7 +39,7 @@ def compute_fingerprint(self, target): default_java_namespace = self._thrift_defaults.default_java_namespace(target) if default_java_namespace: - hasher.update(default_java_namespace) + hasher.update(default_java_namespace.encode('utf-8')) if target.include_paths: hasher.update(str(target.include_paths).encode('utf-8')) From 421c8d319da8c099ade15292a2872585887821d1 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Thu, 10 Jan 2019 12:15:47 -0700 Subject: [PATCH 014/105] Fix contrib go hasher unicode issue --- .../pants/contrib/go/tasks/go_binary_fingerprint_strategy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/go/src/python/pants/contrib/go/tasks/go_binary_fingerprint_strategy.py b/contrib/go/src/python/pants/contrib/go/tasks/go_binary_fingerprint_strategy.py index 4cb3a0bbb17..56cc75129a3 100644 --- a/contrib/go/src/python/pants/contrib/go/tasks/go_binary_fingerprint_strategy.py +++ b/contrib/go/src/python/pants/contrib/go/tasks/go_binary_fingerprint_strategy.py @@ -32,7 +32,7 @@ def compute_fingerprint(self, target): return fp hasher = hashlib.sha1() - hasher.update(fp) + hasher.update(fp.encode('utf-8')) hasher.update(str(self._get_build_flags_func(target)).encode('utf-8')) return hasher.hexdigest() if PY3 else hasher.hexdigest().decode('utf-8') From ebceebdfffb8f33bf720edce47834fb6d4417478 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Thu, 10 Jan 2019 12:20:02 -0700 Subject: [PATCH 015/105] Remove unused `as` in except clause Was causing the linting shard to fail --- .../src/python/pants/contrib/confluence/util/confluence_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/confluence/src/python/pants/contrib/confluence/util/confluence_util.py b/contrib/confluence/src/python/pants/contrib/confluence/util/confluence_util.py index fa35978048d..1d40c573fef 100644 --- a/contrib/confluence/src/python/pants/contrib/confluence/util/confluence_util.py +++ b/contrib/confluence/src/python/pants/contrib/confluence/util/confluence_util.py @@ -171,7 +171,7 @@ def addattachment(self, page, filename): except (IOError, OSError) as e: log.error('Failed to read data from file %s: %s' % (filename, str(e))) return None - except XMLRPCError as e: + except XMLRPCError: log.error('Failed to add file attachment %s to page: %s' % (filename, page.get('title', '[unknown title]'))) return None From feb4920fd0bcb68b181feead42c7c3261cefe396 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Thu, 10 Jan 2019 12:37:51 -0700 Subject: [PATCH 016/105] Apply homebrew workaround to all osx shards --- .travis.yml | 10 +++++----- build-support/travis/travis.yml.mustache | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 868092d5c1a..4ddd26fd64e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -119,6 +119,11 @@ default_linux_test_config: &default_linux_test_config default_osx_config: &default_osx_config os: osx language: generic + # TODO: workaround for Homebrew bug triggered by running `brew update`. + # Follow at https://github.com/Homebrew/brew/issues/5513 and remove this workaround once its fixed. + env: + - HOMEBREW_LOGS=~/homebrew-logs + - HOMEBREW_TEMP=~/homebrew-temp before_install: - curl -L https://github.com/stedolan/jq/releases/download/jq-1.5/jq-osx-amd64 -o /usr/local/bin/jq - chmod 755 /usr/local/bin/jq @@ -635,11 +640,6 @@ matrix: - CACHE_NAME=macosrusttests # Fuse actually works on this image. It hangs on many others. osx_image: xcode8.3 - # TODO: workaround for Homebrew bug triggered by running `brew update`. - # Follow at https://github.com/Homebrew/brew/issues/5513 and remove this workaround once its fixed. - env: - - HOMEBREW_LOGS=~/homebrew-logs - - HOMEBREW_TEMP=~/homebrew-temp before_install: - curl -L https://github.com/stedolan/jq/releases/download/jq-1.5/jq-osx-amd64 -o /usr/local/bin/jq - chmod 755 /usr/local/bin/jq diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index 8ac481508bd..5d7c18aae58 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -110,6 +110,11 @@ default_linux_test_config: &default_linux_test_config default_osx_config: &default_osx_config os: osx language: generic + # TODO: workaround for Homebrew bug triggered by running `brew update`. + # Follow at https://github.com/Homebrew/brew/issues/5513 and remove this workaround once its fixed. + env: + - HOMEBREW_LOGS=~/homebrew-logs + - HOMEBREW_TEMP=~/homebrew-temp before_install: {{>before_install_osx}} @@ -336,11 +341,6 @@ matrix: - CACHE_NAME=macosrusttests # Fuse actually works on this image. It hangs on many others. osx_image: xcode8.3 - # TODO: workaround for Homebrew bug triggered by running `brew update`. - # Follow at https://github.com/Homebrew/brew/issues/5513 and remove this workaround once its fixed. - env: - - HOMEBREW_LOGS=~/homebrew-logs - - HOMEBREW_TEMP=~/homebrew-temp before_install: {{>before_install_osx}} - brew tap caskroom/cask && brew update && brew cask install osxfuse From 1f386a75d12532e9a7b0dd4675fec31dd69063c4 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Thu, 10 Jan 2019 16:11:27 -0700 Subject: [PATCH 017/105] Bootstrap both Py2 and Py3 pexes --- .travis.yml | 288 +++++++++++------- .../travis/before_install_osx.mustache | 2 - build-support/travis/travis.yml.mustache | 214 ++++++++----- 3 files changed, 323 insertions(+), 181 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4ddd26fd64e..eeb4f67494d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -79,7 +79,7 @@ pants_run_cache_config: &pants_run_cache_config # https://github.com/pantsbuild/pants/issues/2485 - ${HOME}/.npm -default_linux_config: &default_linux_config +base_linux_config: &base_linux_config os: linux dist: xenial sudo: required @@ -96,14 +96,21 @@ default_linux_config: &default_linux_config - jq - unzip language: python - python: &python_version "3.6" before_install: - ./build-support/bin/install_aws_cli_for_ci.sh after_failure: - ./build-support/bin/ci-failure.sh -default_linux_test_config: &default_linux_test_config - <<: *default_linux_config +py2_linux_config: &py2_linux_config + <<: *base_linux_config + python: &python_version "2.7" + +py3_linux_config: &py3_linux_config + <<: *base_linux_config + python: &python_version "3.6" + +base_linux_test_config: &base_linux_test_config + <<: *base_linux_config <<: *pants_run_cache_config stage: *test before_install: @@ -114,9 +121,21 @@ default_linux_test_config: &default_linux_test_config - sudo sysctl fs.inotify.max_user_watches=524288 - ./build-support/bin/install_aws_cli_for_ci.sh before_script: - - ./build-support/bin/get_ci_bootstrapped_pants_pex.sh ${BOOTSTRAPPED_PEX_BUCKET} ${BOOTSTRAPPED_PEX_KEY_PREFIX}.linux + - ./build-support/bin/get_ci_bootstrapped_pants_pex.sh ${BOOTSTRAPPED_PEX_BUCKET} ${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX} + +py2_linux_test_config: &py2_linux_test_config + <<: *py2_linux_config + <<: *base_linux_test_config + env: + - BOOTSTRAPPED_PEX_KEY_SUFFIX="py2.linux" + +py3_linux_test_config: &py3_linux_test_config + <<: *py3_linux_config + <<: *base_linux_test_config + env: + - BOOTSTRAPPED_PEX_KEY_SUFFIX="py3.linux" -default_osx_config: &default_osx_config +base_osx_config: &base_osx_config os: osx language: generic # TODO: workaround for Homebrew bug triggered by running `brew update`. @@ -128,17 +147,114 @@ default_osx_config: &default_osx_config - curl -L https://github.com/stedolan/jq/releases/download/jq-1.5/jq-osx-amd64 -o /usr/local/bin/jq - chmod 755 /usr/local/bin/jq - ./build-support/bin/install_aws_cli_for_ci.sh + +py2_osx_config: &py2_osx_config + <<: *base_osx_config + +py3_osx_config: &py3_osx_config + <<: *base_osx_config + before_install: # Upgrade to Python 3 - brew upgrade python -default_osx_test_config: &default_osx_test_config - <<: *default_osx_config +base_osx_test_config: &base_osx_test_config + <<: *base_osx_config <<: *pants_run_cache_config stage: *test before_script: - ulimit -c unlimited - ulimit -n 8192 - - ./build-support/bin/get_ci_bootstrapped_pants_pex.sh ${BOOTSTRAPPED_PEX_BUCKET} ${BOOTSTRAPPED_PEX_KEY_PREFIX}.osx + - ./build-support/bin/get_ci_bootstrapped_pants_pex.sh ${BOOTSTRAPPED_PEX_BUCKET} ${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX} + +py2_osx_test_config: &py2_osx_test_config + <<: *py2_osx_config + <<: *base_osx_test_config + env: + - BOOTSTRAPPED_PEX_KEY_SUFFIX="py2.osx" + +py3_osx_test_config: &py3_osx_test_config + <<: *py3_osx_config + <<: *base_osx_test_config + env: + - BOOTSTRAPPED_PEX_KEY_SUFFIX="py3.osx" + +base_linux_build_engine: &base_linux_build_engine + <<: *base_linux_config + <<: *native_engine_cache_config + stage: *bootstrap + services: + - docker + env: + - PREPARE_DEPLOY=1 # To deploy fs_util. + before_script: + - ulimit -c unlimited + script: + - docker build --rm -t travis_ci + --build-arg "TRAVIS_USER=$(id -un)" + --build-arg "TRAVIS_UID=$(id -u)" + --build-arg "TRAVIS_GROUP=$(id -gn)" + --build-arg "TRAVIS_GID=$(id -g)" + build-support/docker/travis_ci/ + # Note that: + # * We mount ${HOME} to cache the ${HOME}/.cache/pants/rust-toolchain. + # * With no args ci.sh just bootstraps a pants.pex. + # * We also build fs_util, to take advantage of the rust code built during bootstrapping. + - docker run --rm -t + -v "${HOME}:/travis/home" + -v "${TRAVIS_BUILD_DIR}:/travis/workdir" + travis_ci:latest + sh -c "./build-support/bin/ci.sh && ./build-support/bin/release.sh -f" + - aws --no-sign-request --region us-east-1 s3 cp ${TRAVIS_BUILD_DIR}/pants.pex ${BOOTSTRAPPED_PEX_URL_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX} + +py2_linux_build_engine: &py2_linux_build_engine + <<: *py2_linux_config + <<: *base_linux_build_engine + name: "Build Linux native engine and Python 2 pants.pex" + env: + - CACHE_NAME=linuxpexbuild_py2 + - BOOTSTRAPPED_PEX_KEY_SUFFIX="py2.linux" + +py3_linux_build_engine: &py3_linux_build_engine + <<: *py3_linux_config + <<: *base_linux_build_engine + name: "Build Linux native engine and Python 3 pants.pex" + env: + - CACHE_NAME=linuxpexbuild_py3 + - BOOTSTRAPPED_PEX_KEY_SUFFIX="py3.linux" + +base_osx_build_engine: &base_osx_build_engine + <<: *base_osx_config + <<: *native_engine_cache_config + stage: *bootstrap + # We request the oldest image we can (corresponding to OSX 10.11) for maximum compatibility. + # We use 10.11 as a minimum to avoid https://github.com/rust-lang/regex/issues/489. + # See: https://docs.travis-ci.com/user/reference/osx/#OS-X-Version + osx_image: xcode8 + env: + - PREPARE_DEPLOY=1 # To deploy fs_util. + script: + # With no args ci.sh just bootstraps a pants.pex. We also build fs_util, to take advantage + # of the rust code built during bootstrapping. + - ./build-support/bin/ci.sh && ./build-support/bin/release.sh -f + - aws --no-sign-request --region us-east-1 s3 cp ${TRAVIS_BUILD_DIR}/pants.pex ${BOOTSTRAPPED_PEX_URL_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX} + after_failure: + - ./build-support/bin/ci-failure.sh + +py2_osx_build_engine: &py2_osx_build_engine + <<: *py2_osx_config + <<: *base_osx_build_engine + name: "Build OSX native engine and Python 2 pants.pex" + env: + - CACHE_NAME=osxpexbuild_py2 + - BOOTSTRAPPED_PEX_KEY_SUFFIX="py2.osx" + +py3_osx_build_engine: &py3_osx_build_engine + <<: *py3_osx_config + <<: *base_osx_build_engine + name: "Build OSX native engine and Python 3 pants.pex" + env: + - CACHE_NAME=osxpexbuild_py3 + - BOOTSTRAPPED_PEX_KEY_SUFFIX="py3.osx" linux_with_fuse: &linux_with_fuse before_install: @@ -155,56 +271,12 @@ linux_with_fuse: &linux_with_fuse matrix: include: - - <<: *default_linux_config - <<: *native_engine_cache_config - name: "Build Linux native engine and pants.pex" - stage: *bootstrap - services: - - docker - env: - - CACHE_NAME=linuxpexbuild - - PREPARE_DEPLOY=1 # To deploy fs_util. - before_script: - - ulimit -c unlimited - script: - - docker build --rm -t travis_ci - --build-arg "TRAVIS_USER=$(id -un)" - --build-arg "TRAVIS_UID=$(id -u)" - --build-arg "TRAVIS_GROUP=$(id -gn)" - --build-arg "TRAVIS_GID=$(id -g)" - build-support/docker/travis_ci/ - # Note that: - # * We mount ${HOME} to cache the ${HOME}/.cache/pants/rust-toolchain. - # * With no args ci.sh just bootstraps a pants.pex. - # * We also build fs_util, to take advantage of the rust code built during bootstrapping. - - docker run --rm -t - -v "${HOME}:/travis/home" - -v "${TRAVIS_BUILD_DIR}:/travis/workdir" - travis_ci:latest - sh -c "./build-support/bin/ci.sh && ./build-support/bin/release.sh -f" - - aws --no-sign-request --region us-east-1 s3 cp ${TRAVIS_BUILD_DIR}/pants.pex ${BOOTSTRAPPED_PEX_URL_PREFIX}.linux - - # Build macOS engine - - <<: *default_osx_config - <<: *native_engine_cache_config - name: "Build OSX native engine and pants.pex" - stage: *bootstrap - # We request the oldest image we can (corresponding to OSX 10.11) for maximum compatibility. - # We use 10.11 as a minimum to avoid https://github.com/rust-lang/regex/issues/489. - # See: https://docs.travis-ci.com/user/reference/osx/#OS-X-Version - osx_image: xcode8 - env: - - CACHE_NAME=macospexbuild - - PREPARE_DEPLOY=1 # To deploy fs_util. - script: - # With no args ci.sh just bootstraps a pants.pex. We also build fs_util, to take advantage - # of the rust code built during bootstrapping. - - ./build-support/bin/ci.sh && ./build-support/bin/release.sh -f - - aws --no-sign-request --region us-east-1 s3 cp ${TRAVIS_BUILD_DIR}/pants.pex ${BOOTSTRAPPED_PEX_URL_PREFIX}.osx - after_failure: - - ./build-support/bin/ci-failure.sh + - <<: *py2_linux_build_engine + - <<: *py3_linux_build_engine + - <<: *py2_osx_build_engine + - <<: *py3_osx_build_engine - - <<: *default_linux_test_config + - <<: *py3_linux_test_config name: "Build Linux wheels" stage: *test env: @@ -214,7 +286,7 @@ matrix: script: - ./build-support/bin/release.sh -n - - <<: *default_osx_test_config + - <<: *py3_osx_test_config name: "Build OSX wheels" osx_image: xcode8 env: @@ -225,7 +297,7 @@ matrix: - ./build-support/bin/release.sh -n # TODO: Update this to use 10.14 once it is available - - <<: *default_osx_test_config + - <<: *py3_osx_test_config name: "OSX 10.12 sanity check" osx_image: xcode9.2 env: @@ -233,7 +305,7 @@ matrix: script: - MODE=debug ./build-support/bin/travis-ci.sh -bm - - <<: *default_osx_test_config + - <<: *py3_osx_test_config name: "OSX 10.13 sanity check" osx_image: xcode10.1 env: @@ -278,7 +350,7 @@ matrix: script: - ./build-support/bin/release.sh -p && mkdir -p dist/deploy/pex/ && mv dist/pants*.pex dist/deploy/pex/ - - <<: *default_linux_test_config + - <<: *py3_linux_test_config <<: *linux_with_fuse name: "Self-checks, lint, and JVM tests" env: @@ -286,175 +358,175 @@ matrix: script: - ./build-support/bin/travis-ci.sh -fbmrjt - - <<: *default_linux_test_config + - <<: *py2_linux_test_config name: "Py2 - Unit tests for pants and pants-plugins" env: - CACHE_NAME=linuxunittests.py2 script: - ./build-support/bin/travis-ci.sh -2blp - - <<: *default_linux_test_config + - <<: *py3_linux_test_config name: "Py3 - Unit tests for pants and pants-plugins" env: - CACHE_NAME=linuxunittests.py3 script: - ./build-support/bin/travis-ci.sh -blp - - <<: *default_linux_test_config + - <<: *py2_linux_test_config name: "Py2 - Python contrib tests" env: - CACHE_NAME=linuxcontribtests.py2 script: - ./build-support/bin/travis-ci.sh -2bn - - <<: *default_linux_test_config + - <<: *py3_linux_test_config name: "Py3 - Python contrib tests" env: - CACHE_NAME=linuxcontribtests.py3 script: - ./build-support/bin/travis-ci.sh -bn - - <<: *default_linux_test_config + - <<: *py3_linux_test_config name: "Integration tests for pants - shard 0" env: - CACHE_NAME=integrationshard0 script: - ./build-support/bin/travis-ci.sh -bc -i 0/20 - - <<: *default_linux_test_config + - <<: *py3_linux_test_config name: "Integration tests for pants - shard 1" env: - CACHE_NAME=integrationshard1 script: - ./build-support/bin/travis-ci.sh -bc -i 1/20 - - <<: *default_linux_test_config + - <<: *py3_linux_test_config name: "Integration tests for pants - shard 2" env: - CACHE_NAME=integrationshard2 script: - ./build-support/bin/travis-ci.sh -bc -i 2/20 - - <<: *default_linux_test_config + - <<: *py3_linux_test_config name: "Integration tests for pants - shard 3" env: - CACHE_NAME=integrationshard3 script: - ./build-support/bin/travis-ci.sh -bc -i 3/20 - - <<: *default_linux_test_config + - <<: *py3_linux_test_config name: "Integration tests for pants - shard 4" env: - CACHE_NAME=integrationshard4 script: - ./build-support/bin/travis-ci.sh -bc -i 4/20 - - <<: *default_linux_test_config + - <<: *py3_linux_test_config name: "Integration tests for pants - shard 5" env: - CACHE_NAME=integrationshard5 script: - ./build-support/bin/travis-ci.sh -bc -i 5/20 - - <<: *default_linux_test_config + - <<: *py3_linux_test_config name: "Integration tests for pants - shard 6" env: - CACHE_NAME=integrationshard6 script: - ./build-support/bin/travis-ci.sh -bc -i 6/20 - - <<: *default_linux_test_config + - <<: *py3_linux_test_config name: "Integration tests for pants - shard 7" env: - CACHE_NAME=integrationshard7 script: - ./build-support/bin/travis-ci.sh -bc -i 7/20 - - <<: *default_linux_test_config + - <<: *py3_linux_test_config name: "Integration tests for pants - shard 8" env: - CACHE_NAME=integrationshard8 script: - ./build-support/bin/travis-ci.sh -bc -i 8/20 - - <<: *default_linux_test_config + - <<: *py3_linux_test_config name: "Integration tests for pants - shard 9" env: - CACHE_NAME=integrationshard9 script: - ./build-support/bin/travis-ci.sh -bc -i 9/20 - - <<: *default_linux_test_config + - <<: *py3_linux_test_config name: "Integration tests for pants - shard 10" env: - CACHE_NAME=integrationshard10 script: - ./build-support/bin/travis-ci.sh -bc -i 10/20 - - <<: *default_linux_test_config + - <<: *py3_linux_test_config name: "Integration tests for pants - shard 11" env: - CACHE_NAME=integrationshard11 script: - ./build-support/bin/travis-ci.sh -bc -i 11/20 - - <<: *default_linux_test_config + - <<: *py3_linux_test_config name: "Integration tests for pants - shard 12" env: - CACHE_NAME=integrationshard12 script: - ./build-support/bin/travis-ci.sh -bc -i 12/20 - - <<: *default_linux_test_config + - <<: *py3_linux_test_config name: "Integration tests for pants - shard 13" env: - CACHE_NAME=integrationshard13 script: - ./build-support/bin/travis-ci.sh -bc -i 13/20 - - <<: *default_linux_test_config + - <<: *py3_linux_test_config name: "Integration tests for pants - shard 14" env: - CACHE_NAME=integrationshard14 script: - ./build-support/bin/travis-ci.sh -bc -i 14/20 - - <<: *default_linux_test_config + - <<: *py3_linux_test_config name: "Integration tests for pants - shard 15" env: - CACHE_NAME=integrationshard15 script: - ./build-support/bin/travis-ci.sh -bc -i 15/20 - - <<: *default_linux_test_config + - <<: *py3_linux_test_config name: "Integration tests for pants - shard 16" env: - CACHE_NAME=integrationshard16 script: - ./build-support/bin/travis-ci.sh -bc -i 16/20 - - <<: *default_linux_test_config + - <<: *py3_linux_test_config name: "Integration tests for pants - shard 17" env: - CACHE_NAME=integrationshard17 script: - ./build-support/bin/travis-ci.sh -bc -i 17/20 - - <<: *default_linux_test_config + - <<: *py3_linux_test_config name: "Integration tests for pants - shard 18" env: - CACHE_NAME=integrationshard18 script: - ./build-support/bin/travis-ci.sh -bc -i 18/20 - - <<: *default_linux_test_config + - <<: *py3_linux_test_config name: "Integration tests for pants - shard 19" env: - CACHE_NAME=integrationshard19 script: - ./build-support/bin/travis-ci.sh -bc -i 19/20 - - <<: *default_linux_test_config + - <<: *py2_linux_test_config name: "Integration tests for pants (Python 2) - shard 0" stage: *cron env: @@ -462,7 +534,7 @@ matrix: script: - ./build-support/bin/travis-ci.sh -bc2 -i 0/20 - - <<: *default_linux_test_config + - <<: *py2_linux_test_config name: "Integration tests for pants (Python 2) - shard 1" stage: *cron env: @@ -470,7 +542,7 @@ matrix: script: - ./build-support/bin/travis-ci.sh -bc2 -i 1/20 - - <<: *default_linux_test_config + - <<: *py2_linux_test_config name: "Integration tests for pants (Python 2) - shard 2" stage: *cron env: @@ -478,7 +550,7 @@ matrix: script: - ./build-support/bin/travis-ci.sh -bc2 -i 2/20 - - <<: *default_linux_test_config + - <<: *py2_linux_test_config name: "Integration tests for pants (Python 2) - shard 3" stage: *cron env: @@ -486,7 +558,7 @@ matrix: script: - ./build-support/bin/travis-ci.sh -bc2 -i 3/20 - - <<: *default_linux_test_config + - <<: *py2_linux_test_config name: "Integration tests for pants (Python 2) - shard 4" stage: *cron env: @@ -494,7 +566,7 @@ matrix: script: - ./build-support/bin/travis-ci.sh -bc2 -i 4/20 - - <<: *default_linux_test_config + - <<: *py2_linux_test_config name: "Integration tests for pants (Python 2) - shard 5" stage: *cron env: @@ -502,7 +574,7 @@ matrix: script: - ./build-support/bin/travis-ci.sh -bc2 -i 5/20 - - <<: *default_linux_test_config + - <<: *py2_linux_test_config name: "Integration tests for pants (Python 2) - shard 6" stage: *cron env: @@ -510,7 +582,7 @@ matrix: script: - ./build-support/bin/travis-ci.sh -bc2 -i 6/20 - - <<: *default_linux_test_config + - <<: *py2_linux_test_config name: "Integration tests for pants (Python 2) - shard 7" stage: *cron env: @@ -518,7 +590,7 @@ matrix: script: - ./build-support/bin/travis-ci.sh -bc2 -i 7/20 - - <<: *default_linux_test_config + - <<: *py2_linux_test_config name: "Integration tests for pants (Python 2) - shard 8" stage: *cron env: @@ -526,7 +598,7 @@ matrix: script: - ./build-support/bin/travis-ci.sh -bc2 -i 8/20 - - <<: *default_linux_test_config + - <<: *py2_linux_test_config name: "Integration tests for pants (Python 2) - shard 9" stage: *cron env: @@ -534,7 +606,7 @@ matrix: script: - ./build-support/bin/travis-ci.sh -bc2 -i 9/20 - - <<: *default_linux_test_config + - <<: *py2_linux_test_config name: "Integration tests for pants (Python 2) - shard 10" stage: *cron env: @@ -542,7 +614,7 @@ matrix: script: - ./build-support/bin/travis-ci.sh -bc2 -i 10/20 - - <<: *default_linux_test_config + - <<: *py2_linux_test_config name: "Integration tests for pants (Python 2) - shard 11" stage: *cron env: @@ -550,7 +622,7 @@ matrix: script: - ./build-support/bin/travis-ci.sh -bc2 -i 11/20 - - <<: *default_linux_test_config + - <<: *py2_linux_test_config name: "Integration tests for pants (Python 2) - shard 12" stage: *cron env: @@ -558,7 +630,7 @@ matrix: script: - ./build-support/bin/travis-ci.sh -bc2 -i 12/20 - - <<: *default_linux_test_config + - <<: *py2_linux_test_config name: "Integration tests for pants (Python 2) - shard 13" stage: *cron env: @@ -566,7 +638,7 @@ matrix: script: - ./build-support/bin/travis-ci.sh -bc2 -i 13/20 - - <<: *default_linux_test_config + - <<: *py2_linux_test_config name: "Integration tests for pants (Python 2) - shard 14" stage: *cron env: @@ -574,7 +646,7 @@ matrix: script: - ./build-support/bin/travis-ci.sh -bc2 -i 14/20 - - <<: *default_linux_test_config + - <<: *py2_linux_test_config name: "Integration tests for pants (Python 2) - shard 15" stage: *cron env: @@ -582,7 +654,7 @@ matrix: script: - ./build-support/bin/travis-ci.sh -bc2 -i 15/20 - - <<: *default_linux_test_config + - <<: *py2_linux_test_config name: "Integration tests for pants (Python 2) - shard 16" stage: *cron env: @@ -590,7 +662,7 @@ matrix: script: - ./build-support/bin/travis-ci.sh -bc2 -i 16/20 - - <<: *default_linux_test_config + - <<: *py2_linux_test_config name: "Integration tests for pants (Python 2) - shard 17" stage: *cron env: @@ -598,7 +670,7 @@ matrix: script: - ./build-support/bin/travis-ci.sh -bc2 -i 17/20 - - <<: *default_linux_test_config + - <<: *py2_linux_test_config name: "Integration tests for pants (Python 2) - shard 18" stage: *cron env: @@ -606,7 +678,7 @@ matrix: script: - ./build-support/bin/travis-ci.sh -bc2 -i 18/20 - - <<: *default_linux_test_config + - <<: *py2_linux_test_config name: "Integration tests for pants (Python 2) - shard 19" stage: *cron env: @@ -634,7 +706,7 @@ matrix: - ./build-support/bin/travis-ci.sh -be # Rust on macOS - - <<: *default_osx_test_config + - <<: *py3_osx_test_config name: "OSX Rust + platform-specific tests" env: - CACHE_NAME=macosrusttests @@ -644,8 +716,6 @@ matrix: - curl -L https://github.com/stedolan/jq/releases/download/jq-1.5/jq-osx-amd64 -o /usr/local/bin/jq - chmod 755 /usr/local/bin/jq - ./build-support/bin/install_aws_cli_for_ci.sh - # Upgrade to Python 3 - - brew upgrade python - brew tap caskroom/cask && brew update && brew cask install osxfuse script: - ./build-support/bin/travis-ci.sh -bez diff --git a/build-support/travis/before_install_osx.mustache b/build-support/travis/before_install_osx.mustache index d7fdefeae78..f56bd2147e3 100644 --- a/build-support/travis/before_install_osx.mustache +++ b/build-support/travis/before_install_osx.mustache @@ -1,5 +1,3 @@ - curl -L https://github.com/stedolan/jq/releases/download/jq-1.5/jq-osx-amd64 -o /usr/local/bin/jq - chmod 755 /usr/local/bin/jq - ./build-support/bin/install_aws_cli_for_ci.sh -# Upgrade to Python 3 -- brew upgrade python diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index 5d7c18aae58..13f90cf28ed 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -75,7 +75,7 @@ pants_run_cache_config: &pants_run_cache_config # https://github.com/pantsbuild/pants/issues/2485 - ${HOME}/.npm -default_linux_config: &default_linux_config +base_linux_config: &base_linux_config os: linux dist: xenial sudo: required @@ -92,22 +92,41 @@ default_linux_config: &default_linux_config - jq - unzip language: python - python: &python_version "3.6" before_install: - ./build-support/bin/install_aws_cli_for_ci.sh after_failure: - ./build-support/bin/ci-failure.sh -default_linux_test_config: &default_linux_test_config - <<: *default_linux_config +py2_linux_config: &py2_linux_config + <<: *base_linux_config + python: &python_version "2.7" + +py3_linux_config: &py3_linux_config + <<: *base_linux_config + python: &python_version "3.6" + +base_linux_test_config: &base_linux_test_config + <<: *base_linux_config <<: *pants_run_cache_config stage: *test before_install: {{>before_install_linux}} before_script: - - ./build-support/bin/get_ci_bootstrapped_pants_pex.sh ${BOOTSTRAPPED_PEX_BUCKET} ${BOOTSTRAPPED_PEX_KEY_PREFIX}.linux + - ./build-support/bin/get_ci_bootstrapped_pants_pex.sh ${BOOTSTRAPPED_PEX_BUCKET} ${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX} + +py2_linux_test_config: &py2_linux_test_config + <<: *py2_linux_config + <<: *base_linux_test_config + env: + - BOOTSTRAPPED_PEX_KEY_SUFFIX="py2.linux" -default_osx_config: &default_osx_config +py3_linux_test_config: &py3_linux_test_config + <<: *py3_linux_config + <<: *base_linux_test_config + env: + - BOOTSTRAPPED_PEX_KEY_SUFFIX="py3.linux" + +base_osx_config: &base_osx_config os: osx language: generic # TODO: workaround for Homebrew bug triggered by running `brew update`. @@ -118,14 +137,113 @@ default_osx_config: &default_osx_config before_install: {{>before_install_osx}} -default_osx_test_config: &default_osx_test_config - <<: *default_osx_config +py2_osx_config: &py2_osx_config + <<: *base_osx_config + +py3_osx_config: &py3_osx_config + <<: *base_osx_config + before_install: + # Upgrade to Python 3 + - brew upgrade python + +base_osx_test_config: &base_osx_test_config + <<: *base_osx_config <<: *pants_run_cache_config stage: *test before_script: - ulimit -c unlimited - ulimit -n 8192 - - ./build-support/bin/get_ci_bootstrapped_pants_pex.sh ${BOOTSTRAPPED_PEX_BUCKET} ${BOOTSTRAPPED_PEX_KEY_PREFIX}.osx + - ./build-support/bin/get_ci_bootstrapped_pants_pex.sh ${BOOTSTRAPPED_PEX_BUCKET} ${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX} + +py2_osx_test_config: &py2_osx_test_config + <<: *py2_osx_config + <<: *base_osx_test_config + env: + - BOOTSTRAPPED_PEX_KEY_SUFFIX="py2.osx" + +py3_osx_test_config: &py3_osx_test_config + <<: *py3_osx_config + <<: *base_osx_test_config + env: + - BOOTSTRAPPED_PEX_KEY_SUFFIX="py3.osx" + +base_linux_build_engine: &base_linux_build_engine + <<: *base_linux_config + <<: *native_engine_cache_config + stage: *bootstrap + services: + - docker + env: + - PREPARE_DEPLOY=1 # To deploy fs_util. + before_script: + - ulimit -c unlimited + script: + - docker build --rm -t travis_ci + --build-arg "TRAVIS_USER=$(id -un)" + --build-arg "TRAVIS_UID=$(id -u)" + --build-arg "TRAVIS_GROUP=$(id -gn)" + --build-arg "TRAVIS_GID=$(id -g)" + build-support/docker/travis_ci/ + # Note that: + # * We mount ${HOME} to cache the ${HOME}/.cache/pants/rust-toolchain. + # * With no args ci.sh just bootstraps a pants.pex. + # * We also build fs_util, to take advantage of the rust code built during bootstrapping. + - docker run --rm -t + -v "${HOME}:/travis/home" + -v "${TRAVIS_BUILD_DIR}:/travis/workdir" + travis_ci:latest + sh -c "./build-support/bin/ci.sh && ./build-support/bin/release.sh -f" + - aws --no-sign-request --region us-east-1 s3 cp ${TRAVIS_BUILD_DIR}/pants.pex ${BOOTSTRAPPED_PEX_URL_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX} + +py2_linux_build_engine: &py2_linux_build_engine + <<: *py2_linux_config + <<: *base_linux_build_engine + name: "Build Linux native engine and Python 2 pants.pex" + env: + - CACHE_NAME=linuxpexbuild_py2 + - BOOTSTRAPPED_PEX_KEY_SUFFIX="py2.linux" + +py3_linux_build_engine: &py3_linux_build_engine + <<: *py3_linux_config + <<: *base_linux_build_engine + name: "Build Linux native engine and Python 3 pants.pex" + env: + - CACHE_NAME=linuxpexbuild_py3 + - BOOTSTRAPPED_PEX_KEY_SUFFIX="py3.linux" + +base_osx_build_engine: &base_osx_build_engine + <<: *base_osx_config + <<: *native_engine_cache_config + stage: *bootstrap + # We request the oldest image we can (corresponding to OSX 10.11) for maximum compatibility. + # We use 10.11 as a minimum to avoid https://github.com/rust-lang/regex/issues/489. + # See: https://docs.travis-ci.com/user/reference/osx/#OS-X-Version + osx_image: xcode8 + env: + - PREPARE_DEPLOY=1 # To deploy fs_util. + script: + # With no args ci.sh just bootstraps a pants.pex. We also build fs_util, to take advantage + # of the rust code built during bootstrapping. + - ./build-support/bin/ci.sh && ./build-support/bin/release.sh -f + - aws --no-sign-request --region us-east-1 s3 cp ${TRAVIS_BUILD_DIR}/pants.pex ${BOOTSTRAPPED_PEX_URL_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX} + after_failure: + - ./build-support/bin/ci-failure.sh + +py2_osx_build_engine: &py2_osx_build_engine + <<: *py2_osx_config + <<: *base_osx_build_engine + name: "Build OSX native engine and Python 2 pants.pex" + env: + - CACHE_NAME=osxpexbuild_py2 + - BOOTSTRAPPED_PEX_KEY_SUFFIX="py2.osx" + +py3_osx_build_engine: &py3_osx_build_engine + <<: *py3_osx_config + <<: *base_osx_build_engine + name: "Build OSX native engine and Python 3 pants.pex" + env: + - CACHE_NAME=osxpexbuild_py3 + - BOOTSTRAPPED_PEX_KEY_SUFFIX="py3.osx" linux_with_fuse: &linux_with_fuse before_install: @@ -137,56 +255,12 @@ linux_with_fuse: &linux_with_fuse matrix: include: - - <<: *default_linux_config - <<: *native_engine_cache_config - name: "Build Linux native engine and pants.pex" - stage: *bootstrap - services: - - docker - env: - - CACHE_NAME=linuxpexbuild - - PREPARE_DEPLOY=1 # To deploy fs_util. - before_script: - - ulimit -c unlimited - script: - - docker build --rm -t travis_ci - --build-arg "TRAVIS_USER=$(id -un)" - --build-arg "TRAVIS_UID=$(id -u)" - --build-arg "TRAVIS_GROUP=$(id -gn)" - --build-arg "TRAVIS_GID=$(id -g)" - build-support/docker/travis_ci/ - # Note that: - # * We mount ${HOME} to cache the ${HOME}/.cache/pants/rust-toolchain. - # * With no args ci.sh just bootstraps a pants.pex. - # * We also build fs_util, to take advantage of the rust code built during bootstrapping. - - docker run --rm -t - -v "${HOME}:/travis/home" - -v "${TRAVIS_BUILD_DIR}:/travis/workdir" - travis_ci:latest - sh -c "./build-support/bin/ci.sh && ./build-support/bin/release.sh -f" - - aws --no-sign-request --region us-east-1 s3 cp ${TRAVIS_BUILD_DIR}/pants.pex ${BOOTSTRAPPED_PEX_URL_PREFIX}.linux - - # Build macOS engine - - <<: *default_osx_config - <<: *native_engine_cache_config - name: "Build OSX native engine and pants.pex" - stage: *bootstrap - # We request the oldest image we can (corresponding to OSX 10.11) for maximum compatibility. - # We use 10.11 as a minimum to avoid https://github.com/rust-lang/regex/issues/489. - # See: https://docs.travis-ci.com/user/reference/osx/#OS-X-Version - osx_image: xcode8 - env: - - CACHE_NAME=macospexbuild - - PREPARE_DEPLOY=1 # To deploy fs_util. - script: - # With no args ci.sh just bootstraps a pants.pex. We also build fs_util, to take advantage - # of the rust code built during bootstrapping. - - ./build-support/bin/ci.sh && ./build-support/bin/release.sh -f - - aws --no-sign-request --region us-east-1 s3 cp ${TRAVIS_BUILD_DIR}/pants.pex ${BOOTSTRAPPED_PEX_URL_PREFIX}.osx - after_failure: - - ./build-support/bin/ci-failure.sh - - - <<: *default_linux_test_config + - <<: *py2_linux_build_engine + - <<: *py3_linux_build_engine + - <<: *py2_osx_build_engine + - <<: *py3_osx_build_engine + + - <<: *py3_linux_test_config name: "Build Linux wheels" stage: *test env: @@ -196,7 +270,7 @@ matrix: script: - ./build-support/bin/release.sh -n - - <<: *default_osx_test_config + - <<: *py3_osx_test_config name: "Build OSX wheels" osx_image: xcode8 env: @@ -207,7 +281,7 @@ matrix: - ./build-support/bin/release.sh -n # TODO: Update this to use 10.14 once it is available - - <<: *default_osx_test_config + - <<: *py3_osx_test_config name: "OSX 10.12 sanity check" osx_image: xcode9.2 env: @@ -215,7 +289,7 @@ matrix: script: - MODE=debug ./build-support/bin/travis-ci.sh -bm - - <<: *default_osx_test_config + - <<: *py3_osx_test_config name: "OSX 10.13 sanity check" osx_image: xcode10.1 env: @@ -260,7 +334,7 @@ matrix: script: - ./build-support/bin/release.sh -p && mkdir -p dist/deploy/pex/ && mv dist/pants*.pex dist/deploy/pex/ - - <<: *default_linux_test_config + - <<: *py3_linux_test_config <<: *linux_with_fuse name: "Self-checks, lint, and JVM tests" env: @@ -268,28 +342,28 @@ matrix: script: - ./build-support/bin/travis-ci.sh -fbmrjt - - <<: *default_linux_test_config + - <<: *py2_linux_test_config name: "Py2 - Unit tests for pants and pants-plugins" env: - CACHE_NAME=linuxunittests.py2 script: - ./build-support/bin/travis-ci.sh -2blp - - <<: *default_linux_test_config + - <<: *py3_linux_test_config name: "Py3 - Unit tests for pants and pants-plugins" env: - CACHE_NAME=linuxunittests.py3 script: - ./build-support/bin/travis-ci.sh -blp - - <<: *default_linux_test_config + - <<: *py2_linux_test_config name: "Py2 - Python contrib tests" env: - CACHE_NAME=linuxcontribtests.py2 script: - ./build-support/bin/travis-ci.sh -2bn - - <<: *default_linux_test_config + - <<: *py3_linux_test_config name: "Py3 - Python contrib tests" env: - CACHE_NAME=linuxcontribtests.py3 @@ -297,7 +371,7 @@ matrix: - ./build-support/bin/travis-ci.sh -bn {{#integration_shards}} - - <<: *default_linux_test_config + - <<: *py3_linux_test_config name: "Integration tests for pants - shard {{.}}" env: - CACHE_NAME=integrationshard{{.}} @@ -306,7 +380,7 @@ matrix: {{/integration_shards}} {{#integration_shards}} - - <<: *default_linux_test_config + - <<: *py2_linux_test_config name: "Integration tests for pants (Python 2) - shard {{.}}" stage: *cron env: @@ -335,7 +409,7 @@ matrix: - ./build-support/bin/travis-ci.sh -be # Rust on macOS - - <<: *default_osx_test_config + - <<: *py3_osx_test_config name: "OSX Rust + platform-specific tests" env: - CACHE_NAME=macosrusttests From c8eb069ddf0c155e846806b0fe3fe89aba28c79d Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Thu, 10 Jan 2019 16:27:21 -0700 Subject: [PATCH 018/105] Avoid overriding yaml values for env and before_install --- .travis.yml | 15 +++++++++++---- build-support/travis/travis.yml.mustache | 15 +++++++++++---- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index eeb4f67494d..f1d3319d3b7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -140,10 +140,10 @@ base_osx_config: &base_osx_config language: generic # TODO: workaround for Homebrew bug triggered by running `brew update`. # Follow at https://github.com/Homebrew/brew/issues/5513 and remove this workaround once its fixed. - env: + env: &base_osx_config_env - HOMEBREW_LOGS=~/homebrew-logs - HOMEBREW_TEMP=~/homebrew-temp - before_install: + before_install: &base_osx_config_before_install - curl -L https://github.com/stedolan/jq/releases/download/jq-1.5/jq-osx-amd64 -o /usr/local/bin/jq - chmod 755 /usr/local/bin/jq - ./build-support/bin/install_aws_cli_for_ci.sh @@ -154,6 +154,7 @@ py2_osx_config: &py2_osx_config py3_osx_config: &py3_osx_config <<: *base_osx_config before_install: + - <<: *base_osx_config_before_install # Upgrade to Python 3 - brew upgrade python @@ -170,12 +171,14 @@ py2_osx_test_config: &py2_osx_test_config <<: *py2_osx_config <<: *base_osx_test_config env: + - <<: *base_osx_config_env - BOOTSTRAPPED_PEX_KEY_SUFFIX="py2.osx" py3_osx_test_config: &py3_osx_test_config <<: *py3_osx_config <<: *base_osx_test_config env: + - <<: *base_osx_config_env - BOOTSTRAPPED_PEX_KEY_SUFFIX="py3.osx" base_linux_build_engine: &base_linux_build_engine @@ -184,7 +187,7 @@ base_linux_build_engine: &base_linux_build_engine stage: *bootstrap services: - docker - env: + env: &base_linux_build_engine_env - PREPARE_DEPLOY=1 # To deploy fs_util. before_script: - ulimit -c unlimited @@ -211,6 +214,7 @@ py2_linux_build_engine: &py2_linux_build_engine <<: *base_linux_build_engine name: "Build Linux native engine and Python 2 pants.pex" env: + - <<: *base_linux_build_engine_env - CACHE_NAME=linuxpexbuild_py2 - BOOTSTRAPPED_PEX_KEY_SUFFIX="py2.linux" @@ -219,6 +223,7 @@ py3_linux_build_engine: &py3_linux_build_engine <<: *base_linux_build_engine name: "Build Linux native engine and Python 3 pants.pex" env: + - <<: *base_linux_build_engine_env - CACHE_NAME=linuxpexbuild_py3 - BOOTSTRAPPED_PEX_KEY_SUFFIX="py3.linux" @@ -230,7 +235,7 @@ base_osx_build_engine: &base_osx_build_engine # We use 10.11 as a minimum to avoid https://github.com/rust-lang/regex/issues/489. # See: https://docs.travis-ci.com/user/reference/osx/#OS-X-Version osx_image: xcode8 - env: + env: &base_osx_build_engine_env - PREPARE_DEPLOY=1 # To deploy fs_util. script: # With no args ci.sh just bootstraps a pants.pex. We also build fs_util, to take advantage @@ -245,6 +250,7 @@ py2_osx_build_engine: &py2_osx_build_engine <<: *base_osx_build_engine name: "Build OSX native engine and Python 2 pants.pex" env: + - <<: *base_osx_build_engine_env - CACHE_NAME=osxpexbuild_py2 - BOOTSTRAPPED_PEX_KEY_SUFFIX="py2.osx" @@ -253,6 +259,7 @@ py3_osx_build_engine: &py3_osx_build_engine <<: *base_osx_build_engine name: "Build OSX native engine and Python 3 pants.pex" env: + - <<: *base_osx_build_engine_env - CACHE_NAME=osxpexbuild_py3 - BOOTSTRAPPED_PEX_KEY_SUFFIX="py3.osx" diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index 13f90cf28ed..e50905c3c9b 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -131,10 +131,10 @@ base_osx_config: &base_osx_config language: generic # TODO: workaround for Homebrew bug triggered by running `brew update`. # Follow at https://github.com/Homebrew/brew/issues/5513 and remove this workaround once its fixed. - env: + env: &base_osx_config_env - HOMEBREW_LOGS=~/homebrew-logs - HOMEBREW_TEMP=~/homebrew-temp - before_install: + before_install: &base_osx_config_before_install {{>before_install_osx}} py2_osx_config: &py2_osx_config @@ -143,6 +143,7 @@ py2_osx_config: &py2_osx_config py3_osx_config: &py3_osx_config <<: *base_osx_config before_install: + - <<: *base_osx_config_before_install # Upgrade to Python 3 - brew upgrade python @@ -159,12 +160,14 @@ py2_osx_test_config: &py2_osx_test_config <<: *py2_osx_config <<: *base_osx_test_config env: + - <<: *base_osx_config_env - BOOTSTRAPPED_PEX_KEY_SUFFIX="py2.osx" py3_osx_test_config: &py3_osx_test_config <<: *py3_osx_config <<: *base_osx_test_config env: + - <<: *base_osx_config_env - BOOTSTRAPPED_PEX_KEY_SUFFIX="py3.osx" base_linux_build_engine: &base_linux_build_engine @@ -173,7 +176,7 @@ base_linux_build_engine: &base_linux_build_engine stage: *bootstrap services: - docker - env: + env: &base_linux_build_engine_env - PREPARE_DEPLOY=1 # To deploy fs_util. before_script: - ulimit -c unlimited @@ -200,6 +203,7 @@ py2_linux_build_engine: &py2_linux_build_engine <<: *base_linux_build_engine name: "Build Linux native engine and Python 2 pants.pex" env: + - <<: *base_linux_build_engine_env - CACHE_NAME=linuxpexbuild_py2 - BOOTSTRAPPED_PEX_KEY_SUFFIX="py2.linux" @@ -208,6 +212,7 @@ py3_linux_build_engine: &py3_linux_build_engine <<: *base_linux_build_engine name: "Build Linux native engine and Python 3 pants.pex" env: + - <<: *base_linux_build_engine_env - CACHE_NAME=linuxpexbuild_py3 - BOOTSTRAPPED_PEX_KEY_SUFFIX="py3.linux" @@ -219,7 +224,7 @@ base_osx_build_engine: &base_osx_build_engine # We use 10.11 as a minimum to avoid https://github.com/rust-lang/regex/issues/489. # See: https://docs.travis-ci.com/user/reference/osx/#OS-X-Version osx_image: xcode8 - env: + env: &base_osx_build_engine_env - PREPARE_DEPLOY=1 # To deploy fs_util. script: # With no args ci.sh just bootstraps a pants.pex. We also build fs_util, to take advantage @@ -234,6 +239,7 @@ py2_osx_build_engine: &py2_osx_build_engine <<: *base_osx_build_engine name: "Build OSX native engine and Python 2 pants.pex" env: + - <<: *base_osx_build_engine_env - CACHE_NAME=osxpexbuild_py2 - BOOTSTRAPPED_PEX_KEY_SUFFIX="py2.osx" @@ -242,6 +248,7 @@ py3_osx_build_engine: &py3_osx_build_engine <<: *base_osx_build_engine name: "Build OSX native engine and Python 3 pants.pex" env: + - <<: *base_osx_build_engine_env - CACHE_NAME=osxpexbuild_py3 - BOOTSTRAPPED_PEX_KEY_SUFFIX="py3.osx" From a3e8b5d0d9458a427e08e7f59e3128e78b703cef Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Thu, 10 Jan 2019 20:05:46 -0700 Subject: [PATCH 019/105] Setup cron shards with python 2 pex --- .travis.yml | 480 ++++++++++++++++------- build-support/travis/travis.yml.mustache | 430 ++++++++++++++------ 2 files changed, 652 insertions(+), 258 deletions(-) diff --git a/.travis.yml b/.travis.yml index f1d3319d3b7..89ebdb350b9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,10 @@ # Conditions are documented here: https://docs.travis-ci.com/user/conditions-v1 conditions: v1 +# ------------------------------------------------------------------------- +# Global setup +# ------------------------------------------------------------------------- + env: global: - PANTS_CONFIG_FILES="${TRAVIS_BUILD_DIR}/pants.travis-ci.ini" @@ -28,6 +32,10 @@ stages: - name: &build_unstable Deploy Pants Pex Unstable if: tag IS NOT present AND type NOT IN (pull_request, cron) +# ------------------------------------------------------------------------- +# Cache config +# ------------------------------------------------------------------------- + # Travis cache config for jobs that build the native engine. native_engine_cache_config: &native_engine_cache_config before_cache: @@ -79,6 +87,10 @@ pants_run_cache_config: &pants_run_cache_config # https://github.com/pantsbuild/pants/issues/2485 - ${HOME}/.npm +# ------------------------------------------------------------------------- +# Generic shard setups +# ------------------------------------------------------------------------- + base_linux_config: &base_linux_config os: linux dist: xenial @@ -112,7 +124,6 @@ py3_linux_config: &py3_linux_config base_linux_test_config: &base_linux_test_config <<: *base_linux_config <<: *pants_run_cache_config - stage: *test before_install: - PATH="/usr/lib/jvm/java-8-openjdk-amd64/jre/bin":$PATH - JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 @@ -126,13 +137,15 @@ base_linux_test_config: &base_linux_test_config py2_linux_test_config: &py2_linux_test_config <<: *py2_linux_config <<: *base_linux_test_config - env: + stage: *cron + env: &py2_linux_test_config_env - BOOTSTRAPPED_PEX_KEY_SUFFIX="py2.linux" py3_linux_test_config: &py3_linux_test_config <<: *py3_linux_config <<: *base_linux_test_config - env: + stage: *test + env: &py3_linux_test_config_env - BOOTSTRAPPED_PEX_KEY_SUFFIX="py3.linux" base_osx_config: &base_osx_config @@ -170,17 +183,34 @@ base_osx_test_config: &base_osx_test_config py2_osx_test_config: &py2_osx_test_config <<: *py2_osx_config <<: *base_osx_test_config - env: + env: &py2_osx_test_config_env - <<: *base_osx_config_env - BOOTSTRAPPED_PEX_KEY_SUFFIX="py2.osx" py3_osx_test_config: &py3_osx_test_config <<: *py3_osx_config <<: *base_osx_test_config - env: + env: &py3_osx_test_config_env - <<: *base_osx_config_env - BOOTSTRAPPED_PEX_KEY_SUFFIX="py3.osx" +linux_with_fuse: &linux_with_fuse + before_install: + - PATH="/usr/lib/jvm/java-8-openjdk-amd64/jre/bin":$PATH + - JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 + # Increase the max number of user watches to ensure that watchman is able to watch all + # files in the working copy. + - sudo sysctl fs.inotify.max_user_watches=524288 + - ./build-support/bin/install_aws_cli_for_ci.sh + - sudo apt-get install -y pkg-config fuse libfuse-dev + - sudo modprobe fuse + - sudo chmod 666 /dev/fuse + - sudo chown root:$USER /etc/fuse.conf + +# ------------------------------------------------------------------------- +# Bootstrap engine shards +# ------------------------------------------------------------------------- + base_linux_build_engine: &base_linux_build_engine <<: *base_linux_config <<: *native_engine_cache_config @@ -206,7 +236,7 @@ base_linux_build_engine: &base_linux_build_engine -v "${HOME}:/travis/home" -v "${TRAVIS_BUILD_DIR}:/travis/workdir" travis_ci:latest - sh -c "./build-support/bin/ci.sh && ./build-support/bin/release.sh -f" + sh -c "./build-support/bin/travis-ci.sh ${BOOTSTRAP_ARGS} && ./build-support/bin/release.sh -f" - aws --no-sign-request --region us-east-1 s3 cp ${TRAVIS_BUILD_DIR}/pants.pex ${BOOTSTRAPPED_PEX_URL_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX} py2_linux_build_engine: &py2_linux_build_engine @@ -215,8 +245,9 @@ py2_linux_build_engine: &py2_linux_build_engine name: "Build Linux native engine and Python 2 pants.pex" env: - <<: *base_linux_build_engine_env - - CACHE_NAME=linuxpexbuild_py2 + - CACHE_NAME=linuxpexbuild.py2 - BOOTSTRAPPED_PEX_KEY_SUFFIX="py2.linux" + - BOOTSTRAP_ARGS="-2" py3_linux_build_engine: &py3_linux_build_engine <<: *py3_linux_config @@ -224,8 +255,9 @@ py3_linux_build_engine: &py3_linux_build_engine name: "Build Linux native engine and Python 3 pants.pex" env: - <<: *base_linux_build_engine_env - - CACHE_NAME=linuxpexbuild_py3 + - CACHE_NAME=linuxpexbuild.py3 - BOOTSTRAPPED_PEX_KEY_SUFFIX="py3.linux" + - BOOTSTRAP_ARGS="" base_osx_build_engine: &base_osx_build_engine <<: *base_osx_config @@ -240,7 +272,7 @@ base_osx_build_engine: &base_osx_build_engine script: # With no args ci.sh just bootstraps a pants.pex. We also build fs_util, to take advantage # of the rust code built during bootstrapping. - - ./build-support/bin/ci.sh && ./build-support/bin/release.sh -f + - ./build-support/bin/ci.sh ${BOOTSTRAP_ARGS} && ./build-support/bin/release.sh -f - aws --no-sign-request --region us-east-1 s3 cp ${TRAVIS_BUILD_DIR}/pants.pex ${BOOTSTRAPPED_PEX_URL_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX} after_failure: - ./build-support/bin/ci-failure.sh @@ -251,8 +283,9 @@ py2_osx_build_engine: &py2_osx_build_engine name: "Build OSX native engine and Python 2 pants.pex" env: - <<: *base_osx_build_engine_env - - CACHE_NAME=osxpexbuild_py2 + - CACHE_NAME=osxpexbuild.py2 - BOOTSTRAPPED_PEX_KEY_SUFFIX="py2.osx" + - BOOTSTRAP_ARGS="-2" py3_osx_build_engine: &py3_osx_build_engine <<: *py3_osx_config @@ -260,114 +293,281 @@ py3_osx_build_engine: &py3_osx_build_engine name: "Build OSX native engine and Python 3 pants.pex" env: - <<: *base_osx_build_engine_env - - CACHE_NAME=osxpexbuild_py3 + - CACHE_NAME=osxpexbuild.py3 - BOOTSTRAPPED_PEX_KEY_SUFFIX="py3.osx" + - BOOTSTRAP_ARGS="" -linux_with_fuse: &linux_with_fuse +# ------------------------------------------------------------------------- +# Build wheels +# ------------------------------------------------------------------------- + +base_build_wheels: &base_build_wheels + env: &base_build_wheels_env + - RUN_PANTS_FROM_PEX=1 + - PREPARE_DEPLOY=1 + script: + - ./build-support/bin/release.sh -n + +py2_linux_build_wheels: &py2_linux_build_wheels + <<: *py2_linux_test_config + <<: *base_build_wheels + stage: *cron + name: "Build Linux wheels (Python 2 PEX)" + env: + - <<: *py2_linux_test_config_env + - <<: *base_build_wheels_env + - CACHE_NAME=linuxwheelsbuild.py2 + +py3_linux_build_wheels: &py3_linux_build_wheels + <<: *py3_linux_test_config + <<: *base_build_wheels + name: "Build Linux wheels (Python 3 PEX)" + env: + - <<: *py3_linux_test_config_env + - <<: *base_build_wheels_env + - CACHE_NAME=linuxwheelsbuild.py3 + +py2_osx_build_wheels: &py2_osx_build_wheels + <<: *py2_osx_test_config + <<: *base_build_wheels + name: "Build OSX wheels (Python 2 PEX)" + stage: *cron + env: + - <<: *py2_osx_test_config_env + - <<: *base_build_wheels_env + - CACHE_NAME=osxwheelsbuild.py2 + +py3_osx_build_wheels: &py3_osx_build_wheels + <<: *py3_osx_test_config + <<: *base_osx_build_wheels + name: "Build OSX wheels (Python 3 PEX)" + env: + - <<: *py3_osx_test_config_env + - <<: *base_build_wheels_env + - CACHE_NAME=osxwheelsbuild.py3 + +# ------------------------------------------------------------------------- +# OSX sanity checks +# ------------------------------------------------------------------------- + +base_osx_sanity_check: &base_osx_sanity_check + script: + - MODE=debug ./build-support/bin/travis-ci.sh -bm + +# TODO: Update this to use 10.14 once it is available +base_osx_10_12_sanity_check: &base_osx_10_12_sanity_check + <<: *base_osx_sanity_check + osx_image: xcode9.2 + +py2_osx_10_12_sanity_check: &py2_osx_10_12_sanity_check + <<: *py2_osx_test_config + <<: *base_osx_10_12_sanity_check + name: "OSX 10.12 sanity check (Python 2 PEX)" + stage: *cron + env: + - <<: *py2_osx_test_config_env + - CACHE_NAME=macos10.12sanity.py2 + +py3_osx_10_12_sanity_check: &py3_osx_10_12_sanity_check + <<: *py3_osx_test_config + <<: *base_osx_10_12_sanity_check + name: "OSX 10.12 sanity check (Python 3 PEX)" + env: + - <<: *py3_osx_test_config_env + - CACHE_NAME=macos10.12sanity.py3 + +base_osx_10_13_sanity_check: &base_osx_10_13_sanity_check + <<: *base_osx_sanity_check + osx_image: xcode10.1 + +py2_osx_10_13_sanity_check: &py2_osx_10_13_sanity_check + <<: *py2_osx_test_config + <<: *base_osx_10_13_sanity_check + name: "OSX 10.13 sanity check (Python 2 PEX)" + stage: *cron + env: + - <<: *py2_osx_test_config_env + - CACHE_NAME=macos10.13sanity.py2 + +py3_osx_10_13_sanity_check: &py3_osx_10_13_sanity_check + <<: *py3_osx_test_config + <<: *base_osx_10_13_sanity_check + name: "OSX 10.13 sanity check (Python 3 PEX)" + env: + - <<: *py3_osx_test_config_env + - CACHE_NAME=macos10.13sanity.py3 + +# ------------------------------------------------------------------------- +# Lint +# ------------------------------------------------------------------------- + +base_lint: &base_lint + <<: *linux_with_fuse + script: + - ./build-support/bin/travis-ci.sh -fbmrjt + +py2_lint: &py2_lint + <<: *py2_linux_test_config + <<: *base_lint + name: "Self-checks, lint, and JVM tests (Python 2 PEX)" + env: + - <<: *py2_linux_test_config_env + - CACHE_NAME=linuxselfchecks.py2 + +py3_lint: &py3_lint + <<: *py3_linux_test_config + <<: *base_lint + name: "Self-checks, lint, and JVM tests (Python 3 PEX)" + env: + - <<: *py3_linux_test_config_env + - CACHE_NAME=linuxselfchecks.py3 + +# ------------------------------------------------------------------------- +# Deploy +# ------------------------------------------------------------------------- + +deploy_stable_multiplatform_pex: &deploy_stable_multiplatform_pex + name: "Deploy stable multiplatform pants.pex" + os: linux + language: python + stage: *build_stable + env: + - CACHE_NAME=linuxpexdeploystable + - RUN_PANTS_FROM_PEX=1 + - PANTS_PEX_RELEASE=stable + script: + - ./build-support/bin/release.sh -p + deploy: + # See https://docs.travis-ci.com/user/deployment/releases/ + provider: releases + # The pantsbuild-ci-bot OAuth token, see the pantsbuild vault for details. + api_key: + secure: "u0aCsiuVGOg28YxG0sQUovuUm29kKwQfFgHbNz2TT5L+cGoHxGl4aoVOCtuwWYEtbNGmYc8/3WRS3C/jOiqQj6JEgHUzWOsnfKUObEqNhisAmXbzBbKc0wPQTL8WNK+DKFh32sD3yPYcw+a5PTLO56+o7rqlI25LK7A17WesHC4=" + file_glob: true + file: dist/deploy/pex/* + skip_cleanup: true + on: + # We only release a pex for Pants releases, which are tagged. + tags: true + repo: pantsbuild/pants + +deploy_unstable_multiplatform_pex: &deploy_unstable_multiplatform_pex + name: "Deploy unstable multiplatform pants.pex" + os: linux + language: python + stage: *build_unstable + env: + - CACHE_NAME=linuxpexdeployunstable + - RUN_PANTS_FROM_PEX=1 + - PREPARE_DEPLOY=1 + script: + - ./build-support/bin/release.sh -p && mkdir -p dist/deploy/pex/ && mv dist/pants*.pex dist/deploy/pex/ + +# ------------------------------------------------------------------------- +# Rust +# ------------------------------------------------------------------------- + +base_linux_rust_tests: &base_linux_rust_tests + <<: *linux_with_fuse + <<: *native_engine_cache_config + name: "Linux Rust tests" + env: + - CACHE_NAME=linuxrusttests + os: linux + dist: xenial + sudo: required + language: python + before_script: + - ulimit -c unlimited + - ulimit -n 8192 + +py2_linux_rust_tests: &py2_linux_rust_tests + <<: *base_linux_rust_tests + name: "Linux Rust tests (Python 2 PEX)" + python: 2.7 + stage: *cron + env: + - <<: *py2_linux_test_config_env + - CACHE_NAME=linuxrusttests.py2 + script: + - ./build-support/bin/travis-ci.sh -be2 + +py2_linux_rust_tests: &py2_linux_rust_tests + <<: *base_linux_rust_tests + name: "Linux Rust tests (Python 3 PEX)" + python: 3.6 + stage: *test + env: + - <<: *py3_linux_test_config_env + - CACHE_NAME=linuxrusttests.py3 + script: + - ./build-support/bin/travis-ci.sh -be + +base_osx_rust_tests: &base_osx_rust_tests + <<: *base_osx_test_config + # Fuse actually works on this image. It hangs on many others. + osx_image: xcode8.3 before_install: - - PATH="/usr/lib/jvm/java-8-openjdk-amd64/jre/bin":$PATH - - JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 - # Increase the max number of user watches to ensure that watchman is able to watch all - # files in the working copy. - - sudo sysctl fs.inotify.max_user_watches=524288 + - curl -L https://github.com/stedolan/jq/releases/download/jq-1.5/jq-osx-amd64 -o /usr/local/bin/jq + - chmod 755 /usr/local/bin/jq - ./build-support/bin/install_aws_cli_for_ci.sh - - sudo apt-get install -y pkg-config fuse libfuse-dev - - sudo modprobe fuse - - sudo chmod 666 /dev/fuse - - sudo chown root:$USER /etc/fuse.conf + - brew tap caskroom/cask && brew update && brew cask install osxfuse + +py2_osx_rust_tests: &py2_osx_rust_tests + <<: *base_osx_rust_tests + <<: py2_osx_test_config + name: "OSX Rust + platform-specific tests (Python 2 PEX)" + env: + - <<: *py2_osx_test_config_env + - CACHE_NAME=macosrusttests.py2 + script: + - ./build-support/bin/travis-ci.sh -bez2 + +py3_osx_rust_tests: &py3_osx_rust_tests + <<: *base_osx_rust_tests + <<: py3_osx_test_config + name: "OSX Rust + platform-specific tests (Python 3 PEX)" + env: + - <<: *py3_osx_test_config_env + - CACHE_NAME=macosrusttests.py3 + script: + - ./build-support/bin/travis-ci.sh -bez + +# ------------------------------------------------------------------------- +# Test matrix +# ------------------------------------------------------------------------- matrix: include: - <<: *py2_linux_build_engine - <<: *py3_linux_build_engine + - <<: *py2_osx_build_engine - <<: *py3_osx_build_engine - - <<: *py3_linux_test_config - name: "Build Linux wheels" - stage: *test - env: - - CACHE_NAME=linuxwheelsbuild - - RUN_PANTS_FROM_PEX=1 - - PREPARE_DEPLOY=1 - script: - - ./build-support/bin/release.sh -n + - <<: *py2_linux_build_wheels + - <<: *py3_linux_build_wheels - - <<: *py3_osx_test_config - name: "Build OSX wheels" - osx_image: xcode8 - env: - - CACHE_NAME=osxwheelsbuild - - RUN_PANTS_FROM_PEX=1 - - PREPARE_DEPLOY=1 - script: - - ./build-support/bin/release.sh -n + - <<: *py2_osx_build_wheels + - <<: *py3_osx_build_wheels - # TODO: Update this to use 10.14 once it is available - - <<: *py3_osx_test_config - name: "OSX 10.12 sanity check" - osx_image: xcode9.2 - env: - - CACHE_NAME=macos10.12sanity - script: - - MODE=debug ./build-support/bin/travis-ci.sh -bm + - <<: *py2_osx_10_12_sanity_check + - <<: *py3_osx_10_12_sanity_check - - <<: *py3_osx_test_config - name: "OSX 10.13 sanity check" - osx_image: xcode10.1 - env: - - CACHE_NAME=macos10.13sanity - script: - - MODE=debug ./build-support/bin/travis-ci.sh -bm + - <<: *py2_osx_10_13_sanity_check + - <<: *py3_osx_10_13_sanity_check - # Deploy Pex Stable - - name: "Deploy stable multiplatform pants.pex" - os: linux - language: python - stage: *build_stable - env: - - CACHE_NAME=linuxpexdeploystable - - RUN_PANTS_FROM_PEX=1 - - PANTS_PEX_RELEASE=stable - script: - - ./build-support/bin/release.sh -p - deploy: - # See https://docs.travis-ci.com/user/deployment/releases/ - provider: releases - # The pantsbuild-ci-bot OAuth token, see the pantsbuild vault for details. - api_key: - secure: "u0aCsiuVGOg28YxG0sQUovuUm29kKwQfFgHbNz2TT5L+cGoHxGl4aoVOCtuwWYEtbNGmYc8/3WRS3C/jOiqQj6JEgHUzWOsnfKUObEqNhisAmXbzBbKc0wPQTL8WNK+DKFh32sD3yPYcw+a5PTLO56+o7rqlI25LK7A17WesHC4=" - file_glob: true - file: dist/deploy/pex/* - skip_cleanup: true - on: - # We only release a pex for Pants releases, which are tagged. - tags: true - repo: pantsbuild/pants - - # Deploy Pex Unstable to s3 - - name: "Deploy unstable multiplatform pants.pex" - os: linux - language: python - stage: *build_unstable - env: - - CACHE_NAME=linuxpexdeployunstable - - RUN_PANTS_FROM_PEX=1 - - PREPARE_DEPLOY=1 - script: - - ./build-support/bin/release.sh -p && mkdir -p dist/deploy/pex/ && mv dist/pants*.pex dist/deploy/pex/ + - <<: *deploy_stable_multiplatform_pex + - <<: *deploy_unstable_multiplatform_pex - - <<: *py3_linux_test_config - <<: *linux_with_fuse - name: "Self-checks, lint, and JVM tests" - env: - - CACHE_NAME=linuxselfchecks - script: - - ./build-support/bin/travis-ci.sh -fbmrjt + - <<: *py2_lint + - <<: *py3_lint - <<: *py2_linux_test_config name: "Py2 - Unit tests for pants and pants-plugins" + stage: *test env: + - <<: *py2_linux_test_config_env - CACHE_NAME=linuxunittests.py2 script: - ./build-support/bin/travis-ci.sh -2blp @@ -375,6 +575,7 @@ matrix: - <<: *py3_linux_test_config name: "Py3 - Unit tests for pants and pants-plugins" env: + - <<: *py3_linux_test_config_env - CACHE_NAME=linuxunittests.py3 script: - ./build-support/bin/travis-ci.sh -blp @@ -382,6 +583,7 @@ matrix: - <<: *py2_linux_test_config name: "Py2 - Python contrib tests" env: + - <<: *py2_linux_test_config_env - CACHE_NAME=linuxcontribtests.py2 script: - ./build-support/bin/travis-ci.sh -2bn @@ -389,6 +591,7 @@ matrix: - <<: *py3_linux_test_config name: "Py3 - Python contrib tests" env: + - <<: *py3_linux_test_config_env - CACHE_NAME=linuxcontribtests.py3 script: - ./build-support/bin/travis-ci.sh -bn @@ -396,6 +599,7 @@ matrix: - <<: *py3_linux_test_config name: "Integration tests for pants - shard 0" env: + - <<: *py3_linux_test_config_env - CACHE_NAME=integrationshard0 script: - ./build-support/bin/travis-ci.sh -bc -i 0/20 @@ -403,6 +607,7 @@ matrix: - <<: *py3_linux_test_config name: "Integration tests for pants - shard 1" env: + - <<: *py3_linux_test_config_env - CACHE_NAME=integrationshard1 script: - ./build-support/bin/travis-ci.sh -bc -i 1/20 @@ -410,6 +615,7 @@ matrix: - <<: *py3_linux_test_config name: "Integration tests for pants - shard 2" env: + - <<: *py3_linux_test_config_env - CACHE_NAME=integrationshard2 script: - ./build-support/bin/travis-ci.sh -bc -i 2/20 @@ -417,6 +623,7 @@ matrix: - <<: *py3_linux_test_config name: "Integration tests for pants - shard 3" env: + - <<: *py3_linux_test_config_env - CACHE_NAME=integrationshard3 script: - ./build-support/bin/travis-ci.sh -bc -i 3/20 @@ -424,6 +631,7 @@ matrix: - <<: *py3_linux_test_config name: "Integration tests for pants - shard 4" env: + - <<: *py3_linux_test_config_env - CACHE_NAME=integrationshard4 script: - ./build-support/bin/travis-ci.sh -bc -i 4/20 @@ -431,6 +639,7 @@ matrix: - <<: *py3_linux_test_config name: "Integration tests for pants - shard 5" env: + - <<: *py3_linux_test_config_env - CACHE_NAME=integrationshard5 script: - ./build-support/bin/travis-ci.sh -bc -i 5/20 @@ -438,6 +647,7 @@ matrix: - <<: *py3_linux_test_config name: "Integration tests for pants - shard 6" env: + - <<: *py3_linux_test_config_env - CACHE_NAME=integrationshard6 script: - ./build-support/bin/travis-ci.sh -bc -i 6/20 @@ -445,6 +655,7 @@ matrix: - <<: *py3_linux_test_config name: "Integration tests for pants - shard 7" env: + - <<: *py3_linux_test_config_env - CACHE_NAME=integrationshard7 script: - ./build-support/bin/travis-ci.sh -bc -i 7/20 @@ -452,6 +663,7 @@ matrix: - <<: *py3_linux_test_config name: "Integration tests for pants - shard 8" env: + - <<: *py3_linux_test_config_env - CACHE_NAME=integrationshard8 script: - ./build-support/bin/travis-ci.sh -bc -i 8/20 @@ -459,6 +671,7 @@ matrix: - <<: *py3_linux_test_config name: "Integration tests for pants - shard 9" env: + - <<: *py3_linux_test_config_env - CACHE_NAME=integrationshard9 script: - ./build-support/bin/travis-ci.sh -bc -i 9/20 @@ -466,6 +679,7 @@ matrix: - <<: *py3_linux_test_config name: "Integration tests for pants - shard 10" env: + - <<: *py3_linux_test_config_env - CACHE_NAME=integrationshard10 script: - ./build-support/bin/travis-ci.sh -bc -i 10/20 @@ -473,6 +687,7 @@ matrix: - <<: *py3_linux_test_config name: "Integration tests for pants - shard 11" env: + - <<: *py3_linux_test_config_env - CACHE_NAME=integrationshard11 script: - ./build-support/bin/travis-ci.sh -bc -i 11/20 @@ -480,6 +695,7 @@ matrix: - <<: *py3_linux_test_config name: "Integration tests for pants - shard 12" env: + - <<: *py3_linux_test_config_env - CACHE_NAME=integrationshard12 script: - ./build-support/bin/travis-ci.sh -bc -i 12/20 @@ -487,6 +703,7 @@ matrix: - <<: *py3_linux_test_config name: "Integration tests for pants - shard 13" env: + - <<: *py3_linux_test_config_env - CACHE_NAME=integrationshard13 script: - ./build-support/bin/travis-ci.sh -bc -i 13/20 @@ -494,6 +711,7 @@ matrix: - <<: *py3_linux_test_config name: "Integration tests for pants - shard 14" env: + - <<: *py3_linux_test_config_env - CACHE_NAME=integrationshard14 script: - ./build-support/bin/travis-ci.sh -bc -i 14/20 @@ -501,6 +719,7 @@ matrix: - <<: *py3_linux_test_config name: "Integration tests for pants - shard 15" env: + - <<: *py3_linux_test_config_env - CACHE_NAME=integrationshard15 script: - ./build-support/bin/travis-ci.sh -bc -i 15/20 @@ -508,6 +727,7 @@ matrix: - <<: *py3_linux_test_config name: "Integration tests for pants - shard 16" env: + - <<: *py3_linux_test_config_env - CACHE_NAME=integrationshard16 script: - ./build-support/bin/travis-ci.sh -bc -i 16/20 @@ -515,6 +735,7 @@ matrix: - <<: *py3_linux_test_config name: "Integration tests for pants - shard 17" env: + - <<: *py3_linux_test_config_env - CACHE_NAME=integrationshard17 script: - ./build-support/bin/travis-ci.sh -bc -i 17/20 @@ -522,6 +743,7 @@ matrix: - <<: *py3_linux_test_config name: "Integration tests for pants - shard 18" env: + - <<: *py3_linux_test_config_env - CACHE_NAME=integrationshard18 script: - ./build-support/bin/travis-ci.sh -bc -i 18/20 @@ -529,6 +751,7 @@ matrix: - <<: *py3_linux_test_config name: "Integration tests for pants - shard 19" env: + - <<: *py3_linux_test_config_env - CACHE_NAME=integrationshard19 script: - ./build-support/bin/travis-ci.sh -bc -i 19/20 @@ -537,6 +760,7 @@ matrix: name: "Integration tests for pants (Python 2) - shard 0" stage: *cron env: + - <<: *py2_linux_test_config_env - CACHE_NAME=cronshard0 script: - ./build-support/bin/travis-ci.sh -bc2 -i 0/20 @@ -545,6 +769,7 @@ matrix: name: "Integration tests for pants (Python 2) - shard 1" stage: *cron env: + - <<: *py2_linux_test_config_env - CACHE_NAME=cronshard1 script: - ./build-support/bin/travis-ci.sh -bc2 -i 1/20 @@ -553,6 +778,7 @@ matrix: name: "Integration tests for pants (Python 2) - shard 2" stage: *cron env: + - <<: *py2_linux_test_config_env - CACHE_NAME=cronshard2 script: - ./build-support/bin/travis-ci.sh -bc2 -i 2/20 @@ -561,6 +787,7 @@ matrix: name: "Integration tests for pants (Python 2) - shard 3" stage: *cron env: + - <<: *py2_linux_test_config_env - CACHE_NAME=cronshard3 script: - ./build-support/bin/travis-ci.sh -bc2 -i 3/20 @@ -569,6 +796,7 @@ matrix: name: "Integration tests for pants (Python 2) - shard 4" stage: *cron env: + - <<: *py2_linux_test_config_env - CACHE_NAME=cronshard4 script: - ./build-support/bin/travis-ci.sh -bc2 -i 4/20 @@ -577,6 +805,7 @@ matrix: name: "Integration tests for pants (Python 2) - shard 5" stage: *cron env: + - <<: *py2_linux_test_config_env - CACHE_NAME=cronshard5 script: - ./build-support/bin/travis-ci.sh -bc2 -i 5/20 @@ -585,6 +814,7 @@ matrix: name: "Integration tests for pants (Python 2) - shard 6" stage: *cron env: + - <<: *py2_linux_test_config_env - CACHE_NAME=cronshard6 script: - ./build-support/bin/travis-ci.sh -bc2 -i 6/20 @@ -593,6 +823,7 @@ matrix: name: "Integration tests for pants (Python 2) - shard 7" stage: *cron env: + - <<: *py2_linux_test_config_env - CACHE_NAME=cronshard7 script: - ./build-support/bin/travis-ci.sh -bc2 -i 7/20 @@ -601,6 +832,7 @@ matrix: name: "Integration tests for pants (Python 2) - shard 8" stage: *cron env: + - <<: *py2_linux_test_config_env - CACHE_NAME=cronshard8 script: - ./build-support/bin/travis-ci.sh -bc2 -i 8/20 @@ -609,6 +841,7 @@ matrix: name: "Integration tests for pants (Python 2) - shard 9" stage: *cron env: + - <<: *py2_linux_test_config_env - CACHE_NAME=cronshard9 script: - ./build-support/bin/travis-ci.sh -bc2 -i 9/20 @@ -617,6 +850,7 @@ matrix: name: "Integration tests for pants (Python 2) - shard 10" stage: *cron env: + - <<: *py2_linux_test_config_env - CACHE_NAME=cronshard10 script: - ./build-support/bin/travis-ci.sh -bc2 -i 10/20 @@ -625,6 +859,7 @@ matrix: name: "Integration tests for pants (Python 2) - shard 11" stage: *cron env: + - <<: *py2_linux_test_config_env - CACHE_NAME=cronshard11 script: - ./build-support/bin/travis-ci.sh -bc2 -i 11/20 @@ -633,6 +868,7 @@ matrix: name: "Integration tests for pants (Python 2) - shard 12" stage: *cron env: + - <<: *py2_linux_test_config_env - CACHE_NAME=cronshard12 script: - ./build-support/bin/travis-ci.sh -bc2 -i 12/20 @@ -641,6 +877,7 @@ matrix: name: "Integration tests for pants (Python 2) - shard 13" stage: *cron env: + - <<: *py2_linux_test_config_env - CACHE_NAME=cronshard13 script: - ./build-support/bin/travis-ci.sh -bc2 -i 13/20 @@ -649,6 +886,7 @@ matrix: name: "Integration tests for pants (Python 2) - shard 14" stage: *cron env: + - <<: *py2_linux_test_config_env - CACHE_NAME=cronshard14 script: - ./build-support/bin/travis-ci.sh -bc2 -i 14/20 @@ -657,6 +895,7 @@ matrix: name: "Integration tests for pants (Python 2) - shard 15" stage: *cron env: + - <<: *py2_linux_test_config_env - CACHE_NAME=cronshard15 script: - ./build-support/bin/travis-ci.sh -bc2 -i 15/20 @@ -665,6 +904,7 @@ matrix: name: "Integration tests for pants (Python 2) - shard 16" stage: *cron env: + - <<: *py2_linux_test_config_env - CACHE_NAME=cronshard16 script: - ./build-support/bin/travis-ci.sh -bc2 -i 16/20 @@ -673,6 +913,7 @@ matrix: name: "Integration tests for pants (Python 2) - shard 17" stage: *cron env: + - <<: *py2_linux_test_config_env - CACHE_NAME=cronshard17 script: - ./build-support/bin/travis-ci.sh -bc2 -i 17/20 @@ -681,6 +922,7 @@ matrix: name: "Integration tests for pants (Python 2) - shard 18" stage: *cron env: + - <<: *py2_linux_test_config_env - CACHE_NAME=cronshard18 script: - ./build-support/bin/travis-ci.sh -bc2 -i 18/20 @@ -689,49 +931,23 @@ matrix: name: "Integration tests for pants (Python 2) - shard 19" stage: *cron env: + - <<: *py2_linux_test_config_env - CACHE_NAME=cronshard19 script: - ./build-support/bin/travis-ci.sh -bc2 -i 19/20 - # Rust on linux - - <<: *linux_with_fuse - <<: *native_engine_cache_config - name: "Linux Rust tests" - env: - - CACHE_NAME=linuxrusttests - os: linux - dist: xenial - sudo: required - stage: *test - language: python - python: *python_version - before_script: - - ulimit -c unlimited - - ulimit -n 8192 - script: - - ./build-support/bin/travis-ci.sh -be + - <<: *py2_linux_rust_tests + - <<: *py3_linux_rust_tests - # Rust on macOS - - <<: *py3_osx_test_config - name: "OSX Rust + platform-specific tests" - env: - - CACHE_NAME=macosrusttests - # Fuse actually works on this image. It hangs on many others. - osx_image: xcode8.3 - before_install: - - curl -L https://github.com/stedolan/jq/releases/download/jq-1.5/jq-osx-amd64 -o /usr/local/bin/jq - - chmod 755 /usr/local/bin/jq - - ./build-support/bin/install_aws_cli_for_ci.sh - - brew tap caskroom/cask && brew update && brew cask install osxfuse - script: - - ./build-support/bin/travis-ci.sh -bez + - <<: *py2_osx_rust_tests + - <<: *py3_osx_rust_tests - # Rust Clippy on Linux - <<: *linux_with_fuse <<: *native_engine_cache_config name: "Linux Rust Clippy" env: + - <<: *py2_linux_test_config_env - CACHE_NAME=linuxclippy os: linux dist: xenial @@ -743,19 +959,19 @@ matrix: - ulimit -c unlimited - ulimit -n 8192 script: - - ./build-support/bin/travis-ci.sh -bs + - ./build-support/bin/travis-ci.sh -bs2 - # Cargo audit - <<: *linux_with_fuse name: "Cargo audit" env: + - <<: *py2_linux_test_config_env - CACHE_NAME=linuxcargoaudit os: linux dist: xenial sudo: required stage: *cron script: - - ./build-support/bin/travis-ci.sh -ba + - ./build-support/bin/travis-ci.sh -ba2 deploy: # Deploy whatever a previous stage has left in dist/deploy. diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index e50905c3c9b..5a77520744d 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -3,6 +3,10 @@ # Conditions are documented here: https://docs.travis-ci.com/user/conditions-v1 conditions: v1 +# ------------------------------------------------------------------------- +# Global setup +# ------------------------------------------------------------------------- + env: global: - PANTS_CONFIG_FILES="${TRAVIS_BUILD_DIR}/pants.travis-ci.ini" @@ -24,6 +28,10 @@ stages: - name: &build_unstable Deploy Pants Pex Unstable if: tag IS NOT present AND type NOT IN (pull_request, cron) +# ------------------------------------------------------------------------- +# Cache config +# ------------------------------------------------------------------------- + # Travis cache config for jobs that build the native engine. native_engine_cache_config: &native_engine_cache_config before_cache: @@ -75,6 +83,10 @@ pants_run_cache_config: &pants_run_cache_config # https://github.com/pantsbuild/pants/issues/2485 - ${HOME}/.npm +# ------------------------------------------------------------------------- +# Generic shard setups +# ------------------------------------------------------------------------- + base_linux_config: &base_linux_config os: linux dist: xenial @@ -108,7 +120,6 @@ py3_linux_config: &py3_linux_config base_linux_test_config: &base_linux_test_config <<: *base_linux_config <<: *pants_run_cache_config - stage: *test before_install: {{>before_install_linux}} before_script: @@ -117,13 +128,15 @@ base_linux_test_config: &base_linux_test_config py2_linux_test_config: &py2_linux_test_config <<: *py2_linux_config <<: *base_linux_test_config - env: + stage: *cron + env: &py2_linux_test_config_env - BOOTSTRAPPED_PEX_KEY_SUFFIX="py2.linux" py3_linux_test_config: &py3_linux_test_config <<: *py3_linux_config <<: *base_linux_test_config - env: + stage: *test + env: &py3_linux_test_config_env - BOOTSTRAPPED_PEX_KEY_SUFFIX="py3.linux" base_osx_config: &base_osx_config @@ -159,17 +172,29 @@ base_osx_test_config: &base_osx_test_config py2_osx_test_config: &py2_osx_test_config <<: *py2_osx_config <<: *base_osx_test_config - env: + env: &py2_osx_test_config_env - <<: *base_osx_config_env - BOOTSTRAPPED_PEX_KEY_SUFFIX="py2.osx" py3_osx_test_config: &py3_osx_test_config <<: *py3_osx_config <<: *base_osx_test_config - env: + env: &py3_osx_test_config_env - <<: *base_osx_config_env - BOOTSTRAPPED_PEX_KEY_SUFFIX="py3.osx" +linux_with_fuse: &linux_with_fuse + before_install: + {{>before_install_linux}} + - sudo apt-get install -y pkg-config fuse libfuse-dev + - sudo modprobe fuse + - sudo chmod 666 /dev/fuse + - sudo chown root:$USER /etc/fuse.conf + +# ------------------------------------------------------------------------- +# Bootstrap engine shards +# ------------------------------------------------------------------------- + base_linux_build_engine: &base_linux_build_engine <<: *base_linux_config <<: *native_engine_cache_config @@ -195,7 +220,7 @@ base_linux_build_engine: &base_linux_build_engine -v "${HOME}:/travis/home" -v "${TRAVIS_BUILD_DIR}:/travis/workdir" travis_ci:latest - sh -c "./build-support/bin/ci.sh && ./build-support/bin/release.sh -f" + sh -c "./build-support/bin/travis-ci.sh ${BOOTSTRAP_ARGS} && ./build-support/bin/release.sh -f" - aws --no-sign-request --region us-east-1 s3 cp ${TRAVIS_BUILD_DIR}/pants.pex ${BOOTSTRAPPED_PEX_URL_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX} py2_linux_build_engine: &py2_linux_build_engine @@ -204,8 +229,9 @@ py2_linux_build_engine: &py2_linux_build_engine name: "Build Linux native engine and Python 2 pants.pex" env: - <<: *base_linux_build_engine_env - - CACHE_NAME=linuxpexbuild_py2 + - CACHE_NAME=linuxpexbuild.py2 - BOOTSTRAPPED_PEX_KEY_SUFFIX="py2.linux" + - BOOTSTRAP_ARGS="-2" py3_linux_build_engine: &py3_linux_build_engine <<: *py3_linux_config @@ -213,8 +239,9 @@ py3_linux_build_engine: &py3_linux_build_engine name: "Build Linux native engine and Python 3 pants.pex" env: - <<: *base_linux_build_engine_env - - CACHE_NAME=linuxpexbuild_py3 + - CACHE_NAME=linuxpexbuild.py3 - BOOTSTRAPPED_PEX_KEY_SUFFIX="py3.linux" + - BOOTSTRAP_ARGS="" base_osx_build_engine: &base_osx_build_engine <<: *base_osx_config @@ -229,7 +256,7 @@ base_osx_build_engine: &base_osx_build_engine script: # With no args ci.sh just bootstraps a pants.pex. We also build fs_util, to take advantage # of the rust code built during bootstrapping. - - ./build-support/bin/ci.sh && ./build-support/bin/release.sh -f + - ./build-support/bin/ci.sh ${BOOTSTRAP_ARGS} && ./build-support/bin/release.sh -f - aws --no-sign-request --region us-east-1 s3 cp ${TRAVIS_BUILD_DIR}/pants.pex ${BOOTSTRAPPED_PEX_URL_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX} after_failure: - ./build-support/bin/ci-failure.sh @@ -240,8 +267,9 @@ py2_osx_build_engine: &py2_osx_build_engine name: "Build OSX native engine and Python 2 pants.pex" env: - <<: *base_osx_build_engine_env - - CACHE_NAME=osxpexbuild_py2 + - CACHE_NAME=osxpexbuild.py2 - BOOTSTRAPPED_PEX_KEY_SUFFIX="py2.osx" + - BOOTSTRAP_ARGS="-2" py3_osx_build_engine: &py3_osx_build_engine <<: *py3_osx_config @@ -249,109 +277,279 @@ py3_osx_build_engine: &py3_osx_build_engine name: "Build OSX native engine and Python 3 pants.pex" env: - <<: *base_osx_build_engine_env - - CACHE_NAME=osxpexbuild_py3 + - CACHE_NAME=osxpexbuild.py3 - BOOTSTRAPPED_PEX_KEY_SUFFIX="py3.osx" + - BOOTSTRAP_ARGS="" -linux_with_fuse: &linux_with_fuse +# ------------------------------------------------------------------------- +# Build wheels +# ------------------------------------------------------------------------- + +base_build_wheels: &base_build_wheels + env: &base_build_wheels_env + - RUN_PANTS_FROM_PEX=1 + - PREPARE_DEPLOY=1 + script: + - ./build-support/bin/release.sh -n + +py2_linux_build_wheels: &py2_linux_build_wheels + <<: *py2_linux_test_config + <<: *base_build_wheels + stage: *cron + name: "Build Linux wheels (Python 2 PEX)" + env: + - <<: *py2_linux_test_config_env + - <<: *base_build_wheels_env + - CACHE_NAME=linuxwheelsbuild.py2 + +py3_linux_build_wheels: &py3_linux_build_wheels + <<: *py3_linux_test_config + <<: *base_build_wheels + name: "Build Linux wheels (Python 3 PEX)" + env: + - <<: *py3_linux_test_config_env + - <<: *base_build_wheels_env + - CACHE_NAME=linuxwheelsbuild.py3 + +py2_osx_build_wheels: &py2_osx_build_wheels + <<: *py2_osx_test_config + <<: *base_build_wheels + name: "Build OSX wheels (Python 2 PEX)" + stage: *cron + env: + - <<: *py2_osx_test_config_env + - <<: *base_build_wheels_env + - CACHE_NAME=osxwheelsbuild.py2 + +py3_osx_build_wheels: &py3_osx_build_wheels + <<: *py3_osx_test_config + <<: *base_osx_build_wheels + name: "Build OSX wheels (Python 3 PEX)" + env: + - <<: *py3_osx_test_config_env + - <<: *base_build_wheels_env + - CACHE_NAME=osxwheelsbuild.py3 + +# ------------------------------------------------------------------------- +# OSX sanity checks +# ------------------------------------------------------------------------- + +base_osx_sanity_check: &base_osx_sanity_check + script: + - MODE=debug ./build-support/bin/travis-ci.sh -bm + +# TODO: Update this to use 10.14 once it is available +base_osx_10_12_sanity_check: &base_osx_10_12_sanity_check + <<: *base_osx_sanity_check + osx_image: xcode9.2 + +py2_osx_10_12_sanity_check: &py2_osx_10_12_sanity_check + <<: *py2_osx_test_config + <<: *base_osx_10_12_sanity_check + name: "OSX 10.12 sanity check (Python 2 PEX)" + stage: *cron + env: + - <<: *py2_osx_test_config_env + - CACHE_NAME=macos10.12sanity.py2 + +py3_osx_10_12_sanity_check: &py3_osx_10_12_sanity_check + <<: *py3_osx_test_config + <<: *base_osx_10_12_sanity_check + name: "OSX 10.12 sanity check (Python 3 PEX)" + env: + - <<: *py3_osx_test_config_env + - CACHE_NAME=macos10.12sanity.py3 + +base_osx_10_13_sanity_check: &base_osx_10_13_sanity_check + <<: *base_osx_sanity_check + osx_image: xcode10.1 + +py2_osx_10_13_sanity_check: &py2_osx_10_13_sanity_check + <<: *py2_osx_test_config + <<: *base_osx_10_13_sanity_check + name: "OSX 10.13 sanity check (Python 2 PEX)" + stage: *cron + env: + - <<: *py2_osx_test_config_env + - CACHE_NAME=macos10.13sanity.py2 + +py3_osx_10_13_sanity_check: &py3_osx_10_13_sanity_check + <<: *py3_osx_test_config + <<: *base_osx_10_13_sanity_check + name: "OSX 10.13 sanity check (Python 3 PEX)" + env: + - <<: *py3_osx_test_config_env + - CACHE_NAME=macos10.13sanity.py3 + +# ------------------------------------------------------------------------- +# Lint +# ------------------------------------------------------------------------- + +base_lint: &base_lint + <<: *linux_with_fuse + script: + - ./build-support/bin/travis-ci.sh -fbmrjt + +py2_lint: &py2_lint + <<: *py2_linux_test_config + <<: *base_lint + name: "Self-checks, lint, and JVM tests (Python 2 PEX)" + env: + - <<: *py2_linux_test_config_env + - CACHE_NAME=linuxselfchecks.py2 + +py3_lint: &py3_lint + <<: *py3_linux_test_config + <<: *base_lint + name: "Self-checks, lint, and JVM tests (Python 3 PEX)" + env: + - <<: *py3_linux_test_config_env + - CACHE_NAME=linuxselfchecks.py3 + +# ------------------------------------------------------------------------- +# Deploy +# ------------------------------------------------------------------------- + +deploy_stable_multiplatform_pex: &deploy_stable_multiplatform_pex + name: "Deploy stable multiplatform pants.pex" + os: linux + language: python + stage: *build_stable + env: + - CACHE_NAME=linuxpexdeploystable + - RUN_PANTS_FROM_PEX=1 + - PANTS_PEX_RELEASE=stable + script: + - ./build-support/bin/release.sh -p + deploy: + # See https://docs.travis-ci.com/user/deployment/releases/ + provider: releases + # The pantsbuild-ci-bot OAuth token, see the pantsbuild vault for details. + api_key: + secure: "u0aCsiuVGOg28YxG0sQUovuUm29kKwQfFgHbNz2TT5L+cGoHxGl4aoVOCtuwWYEtbNGmYc8/3WRS3C/jOiqQj6JEgHUzWOsnfKUObEqNhisAmXbzBbKc0wPQTL8WNK+DKFh32sD3yPYcw+a5PTLO56+o7rqlI25LK7A17WesHC4=" + file_glob: true + file: dist/deploy/pex/* + skip_cleanup: true + on: + # We only release a pex for Pants releases, which are tagged. + tags: true + repo: pantsbuild/pants + +deploy_unstable_multiplatform_pex: &deploy_unstable_multiplatform_pex + name: "Deploy unstable multiplatform pants.pex" + os: linux + language: python + stage: *build_unstable + env: + - CACHE_NAME=linuxpexdeployunstable + - RUN_PANTS_FROM_PEX=1 + - PREPARE_DEPLOY=1 + script: + - ./build-support/bin/release.sh -p && mkdir -p dist/deploy/pex/ && mv dist/pants*.pex dist/deploy/pex/ + +# ------------------------------------------------------------------------- +# Rust +# ------------------------------------------------------------------------- + +base_linux_rust_tests: &base_linux_rust_tests + <<: *linux_with_fuse + <<: *native_engine_cache_config + name: "Linux Rust tests" + env: + - CACHE_NAME=linuxrusttests + os: linux + dist: xenial + sudo: required + language: python + before_script: + - ulimit -c unlimited + - ulimit -n 8192 + +py2_linux_rust_tests: &py2_linux_rust_tests + <<: *base_linux_rust_tests + name: "Linux Rust tests (Python 2 PEX)" + python: 2.7 + stage: *cron + env: + - <<: *py2_linux_test_config_env + - CACHE_NAME=linuxrusttests.py2 + script: + - ./build-support/bin/travis-ci.sh -be2 + +py2_linux_rust_tests: &py2_linux_rust_tests + <<: *base_linux_rust_tests + name: "Linux Rust tests (Python 3 PEX)" + python: 3.6 + stage: *test + env: + - <<: *py3_linux_test_config_env + - CACHE_NAME=linuxrusttests.py3 + script: + - ./build-support/bin/travis-ci.sh -be + +base_osx_rust_tests: &base_osx_rust_tests + <<: *base_osx_test_config + # Fuse actually works on this image. It hangs on many others. + osx_image: xcode8.3 before_install: - {{>before_install_linux}} - - sudo apt-get install -y pkg-config fuse libfuse-dev - - sudo modprobe fuse - - sudo chmod 666 /dev/fuse - - sudo chown root:$USER /etc/fuse.conf + {{>before_install_osx}} + - brew tap caskroom/cask && brew update && brew cask install osxfuse + +py2_osx_rust_tests: &py2_osx_rust_tests + <<: *base_osx_rust_tests + <<: py2_osx_test_config + name: "OSX Rust + platform-specific tests (Python 2 PEX)" + env: + - <<: *py2_osx_test_config_env + - CACHE_NAME=macosrusttests.py2 + script: + - ./build-support/bin/travis-ci.sh -bez2 + +py3_osx_rust_tests: &py3_osx_rust_tests + <<: *base_osx_rust_tests + <<: py3_osx_test_config + name: "OSX Rust + platform-specific tests (Python 3 PEX)" + env: + - <<: *py3_osx_test_config_env + - CACHE_NAME=macosrusttests.py3 + script: + - ./build-support/bin/travis-ci.sh -bez + +# ------------------------------------------------------------------------- +# Test matrix +# ------------------------------------------------------------------------- matrix: include: - <<: *py2_linux_build_engine - <<: *py3_linux_build_engine + - <<: *py2_osx_build_engine - <<: *py3_osx_build_engine - - <<: *py3_linux_test_config - name: "Build Linux wheels" - stage: *test - env: - - CACHE_NAME=linuxwheelsbuild - - RUN_PANTS_FROM_PEX=1 - - PREPARE_DEPLOY=1 - script: - - ./build-support/bin/release.sh -n + - <<: *py2_linux_build_wheels + - <<: *py3_linux_build_wheels - - <<: *py3_osx_test_config - name: "Build OSX wheels" - osx_image: xcode8 - env: - - CACHE_NAME=osxwheelsbuild - - RUN_PANTS_FROM_PEX=1 - - PREPARE_DEPLOY=1 - script: - - ./build-support/bin/release.sh -n + - <<: *py2_osx_build_wheels + - <<: *py3_osx_build_wheels - # TODO: Update this to use 10.14 once it is available - - <<: *py3_osx_test_config - name: "OSX 10.12 sanity check" - osx_image: xcode9.2 - env: - - CACHE_NAME=macos10.12sanity - script: - - MODE=debug ./build-support/bin/travis-ci.sh -bm + - <<: *py2_osx_10_12_sanity_check + - <<: *py3_osx_10_12_sanity_check - - <<: *py3_osx_test_config - name: "OSX 10.13 sanity check" - osx_image: xcode10.1 - env: - - CACHE_NAME=macos10.13sanity - script: - - MODE=debug ./build-support/bin/travis-ci.sh -bm + - <<: *py2_osx_10_13_sanity_check + - <<: *py3_osx_10_13_sanity_check - # Deploy Pex Stable - - name: "Deploy stable multiplatform pants.pex" - os: linux - language: python - stage: *build_stable - env: - - CACHE_NAME=linuxpexdeploystable - - RUN_PANTS_FROM_PEX=1 - - PANTS_PEX_RELEASE=stable - script: - - ./build-support/bin/release.sh -p - deploy: - # See https://docs.travis-ci.com/user/deployment/releases/ - provider: releases - # The pantsbuild-ci-bot OAuth token, see the pantsbuild vault for details. - api_key: - secure: "u0aCsiuVGOg28YxG0sQUovuUm29kKwQfFgHbNz2TT5L+cGoHxGl4aoVOCtuwWYEtbNGmYc8/3WRS3C/jOiqQj6JEgHUzWOsnfKUObEqNhisAmXbzBbKc0wPQTL8WNK+DKFh32sD3yPYcw+a5PTLO56+o7rqlI25LK7A17WesHC4=" - file_glob: true - file: dist/deploy/pex/* - skip_cleanup: true - on: - # We only release a pex for Pants releases, which are tagged. - tags: true - repo: pantsbuild/pants - - # Deploy Pex Unstable to s3 - - name: "Deploy unstable multiplatform pants.pex" - os: linux - language: python - stage: *build_unstable - env: - - CACHE_NAME=linuxpexdeployunstable - - RUN_PANTS_FROM_PEX=1 - - PREPARE_DEPLOY=1 - script: - - ./build-support/bin/release.sh -p && mkdir -p dist/deploy/pex/ && mv dist/pants*.pex dist/deploy/pex/ + - <<: *deploy_stable_multiplatform_pex + - <<: *deploy_unstable_multiplatform_pex - - <<: *py3_linux_test_config - <<: *linux_with_fuse - name: "Self-checks, lint, and JVM tests" - env: - - CACHE_NAME=linuxselfchecks - script: - - ./build-support/bin/travis-ci.sh -fbmrjt + - <<: *py2_lint + - <<: *py3_lint - <<: *py2_linux_test_config name: "Py2 - Unit tests for pants and pants-plugins" + stage: *test env: + - <<: *py2_linux_test_config_env - CACHE_NAME=linuxunittests.py2 script: - ./build-support/bin/travis-ci.sh -2blp @@ -359,6 +557,7 @@ matrix: - <<: *py3_linux_test_config name: "Py3 - Unit tests for pants and pants-plugins" env: + - <<: *py3_linux_test_config_env - CACHE_NAME=linuxunittests.py3 script: - ./build-support/bin/travis-ci.sh -blp @@ -366,6 +565,7 @@ matrix: - <<: *py2_linux_test_config name: "Py2 - Python contrib tests" env: + - <<: *py2_linux_test_config_env - CACHE_NAME=linuxcontribtests.py2 script: - ./build-support/bin/travis-ci.sh -2bn @@ -373,6 +573,7 @@ matrix: - <<: *py3_linux_test_config name: "Py3 - Python contrib tests" env: + - <<: *py3_linux_test_config_env - CACHE_NAME=linuxcontribtests.py3 script: - ./build-support/bin/travis-ci.sh -bn @@ -381,6 +582,7 @@ matrix: - <<: *py3_linux_test_config name: "Integration tests for pants - shard {{.}}" env: + - <<: *py3_linux_test_config_env - CACHE_NAME=integrationshard{{.}} script: - ./build-support/bin/travis-ci.sh -bc -i {{.}}/{{integration_shards_length}} @@ -391,48 +593,24 @@ matrix: name: "Integration tests for pants (Python 2) - shard {{.}}" stage: *cron env: + - <<: *py2_linux_test_config_env - CACHE_NAME=cronshard{{.}} script: - ./build-support/bin/travis-ci.sh -bc2 -i {{.}}/{{integration_shards_length}} {{/integration_shards}} - # Rust on linux - - <<: *linux_with_fuse - <<: *native_engine_cache_config - name: "Linux Rust tests" - env: - - CACHE_NAME=linuxrusttests - os: linux - dist: xenial - sudo: required - stage: *test - language: python - python: *python_version - before_script: - - ulimit -c unlimited - - ulimit -n 8192 - script: - - ./build-support/bin/travis-ci.sh -be + - <<: *py2_linux_rust_tests + - <<: *py3_linux_rust_tests - # Rust on macOS - - <<: *py3_osx_test_config - name: "OSX Rust + platform-specific tests" - env: - - CACHE_NAME=macosrusttests - # Fuse actually works on this image. It hangs on many others. - osx_image: xcode8.3 - before_install: - {{>before_install_osx}} - - brew tap caskroom/cask && brew update && brew cask install osxfuse - script: - - ./build-support/bin/travis-ci.sh -bez + - <<: *py2_osx_rust_tests + - <<: *py3_osx_rust_tests - # Rust Clippy on Linux - <<: *linux_with_fuse <<: *native_engine_cache_config name: "Linux Rust Clippy" env: + - <<: *py2_linux_test_config_env - CACHE_NAME=linuxclippy os: linux dist: xenial @@ -444,19 +622,19 @@ matrix: - ulimit -c unlimited - ulimit -n 8192 script: - - ./build-support/bin/travis-ci.sh -bs + - ./build-support/bin/travis-ci.sh -bs2 - # Cargo audit - <<: *linux_with_fuse name: "Cargo audit" env: + - <<: *py2_linux_test_config_env - CACHE_NAME=linuxcargoaudit os: linux dist: xenial sudo: required stage: *cron script: - - ./build-support/bin/travis-ci.sh -ba + - ./build-support/bin/travis-ci.sh -ba2 deploy: # Deploy whatever a previous stage has left in dist/deploy. From b3c837ac5cd04b5f12133553154f3a1ac74a1c8f Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Thu, 10 Jan 2019 20:56:06 -0700 Subject: [PATCH 020/105] Fix sharing env values & other yml issues YAML has no way to merge lists. Instead, we exploit Travis's ability to define multiple environment variables in the same line. So, we simply move the variables to one line and assign that to an anchor. FYI inputting `.travis.yml` into https://yamlvalidator.com is extremely helpful to debug things and see the corresponding JSON. --- .travis.yml | 172 +++++++++++------------ build-support/travis/travis.yml.mustache | 94 ++++++------- 2 files changed, 132 insertions(+), 134 deletions(-) diff --git a/.travis.yml b/.travis.yml index 89ebdb350b9..7e162bfd83d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -138,25 +138,24 @@ py2_linux_test_config: &py2_linux_test_config <<: *py2_linux_config <<: *base_linux_test_config stage: *cron - env: &py2_linux_test_config_env - - BOOTSTRAPPED_PEX_KEY_SUFFIX="py2.linux" + env: + - &py2_linux_test_config_env BOOTSTRAPPED_PEX_KEY_SUFFIX="py2.linux" py3_linux_test_config: &py3_linux_test_config <<: *py3_linux_config <<: *base_linux_test_config stage: *test - env: &py3_linux_test_config_env - - BOOTSTRAPPED_PEX_KEY_SUFFIX="py3.linux" + env: + - &py3_linux_test_config_env BOOTSTRAPPED_PEX_KEY_SUFFIX="py3.linux" base_osx_config: &base_osx_config os: osx language: generic # TODO: workaround for Homebrew bug triggered by running `brew update`. # Follow at https://github.com/Homebrew/brew/issues/5513 and remove this workaround once its fixed. - env: &base_osx_config_env - - HOMEBREW_LOGS=~/homebrew-logs - - HOMEBREW_TEMP=~/homebrew-temp - before_install: &base_osx_config_before_install + env: + - &base_osx_config_env HOMEBREW_LOGS=~/homebrew-log HOMEBREW_TEMP=~/homebrew-temp + before_install: - curl -L https://github.com/stedolan/jq/releases/download/jq-1.5/jq-osx-amd64 -o /usr/local/bin/jq - chmod 755 /usr/local/bin/jq - ./build-support/bin/install_aws_cli_for_ci.sh @@ -167,7 +166,9 @@ py2_osx_config: &py2_osx_config py3_osx_config: &py3_osx_config <<: *base_osx_config before_install: - - <<: *base_osx_config_before_install + - curl -L https://github.com/stedolan/jq/releases/download/jq-1.5/jq-osx-amd64 -o /usr/local/bin/jq + - chmod 755 /usr/local/bin/jq + - ./build-support/bin/install_aws_cli_for_ci.sh # Upgrade to Python 3 - brew upgrade python @@ -184,14 +185,14 @@ py2_osx_test_config: &py2_osx_test_config <<: *py2_osx_config <<: *base_osx_test_config env: &py2_osx_test_config_env - - <<: *base_osx_config_env + - *base_osx_config_env - BOOTSTRAPPED_PEX_KEY_SUFFIX="py2.osx" py3_osx_test_config: &py3_osx_test_config <<: *py3_osx_config <<: *base_osx_test_config env: &py3_osx_test_config_env - - <<: *base_osx_config_env + - *base_osx_config_env - BOOTSTRAPPED_PEX_KEY_SUFFIX="py3.osx" linux_with_fuse: &linux_with_fuse @@ -244,7 +245,7 @@ py2_linux_build_engine: &py2_linux_build_engine <<: *base_linux_build_engine name: "Build Linux native engine and Python 2 pants.pex" env: - - <<: *base_linux_build_engine_env + - *base_linux_build_engine_env - CACHE_NAME=linuxpexbuild.py2 - BOOTSTRAPPED_PEX_KEY_SUFFIX="py2.linux" - BOOTSTRAP_ARGS="-2" @@ -254,7 +255,7 @@ py3_linux_build_engine: &py3_linux_build_engine <<: *base_linux_build_engine name: "Build Linux native engine and Python 3 pants.pex" env: - - <<: *base_linux_build_engine_env + - *base_linux_build_engine_env - CACHE_NAME=linuxpexbuild.py3 - BOOTSTRAPPED_PEX_KEY_SUFFIX="py3.linux" - BOOTSTRAP_ARGS="" @@ -282,7 +283,7 @@ py2_osx_build_engine: &py2_osx_build_engine <<: *base_osx_build_engine name: "Build OSX native engine and Python 2 pants.pex" env: - - <<: *base_osx_build_engine_env + - *base_osx_build_engine_env - CACHE_NAME=osxpexbuild.py2 - BOOTSTRAPPED_PEX_KEY_SUFFIX="py2.osx" - BOOTSTRAP_ARGS="-2" @@ -292,7 +293,7 @@ py3_osx_build_engine: &py3_osx_build_engine <<: *base_osx_build_engine name: "Build OSX native engine and Python 3 pants.pex" env: - - <<: *base_osx_build_engine_env + - *base_osx_build_engine_env - CACHE_NAME=osxpexbuild.py3 - BOOTSTRAPPED_PEX_KEY_SUFFIX="py3.osx" - BOOTSTRAP_ARGS="" @@ -302,9 +303,8 @@ py3_osx_build_engine: &py3_osx_build_engine # ------------------------------------------------------------------------- base_build_wheels: &base_build_wheels - env: &base_build_wheels_env - - RUN_PANTS_FROM_PEX=1 - - PREPARE_DEPLOY=1 + env: + - &base_build_wheels_env RUN_PANTS_FROM_PEX=1 PREPARE_DEPLOY=1 script: - ./build-support/bin/release.sh -n @@ -314,8 +314,8 @@ py2_linux_build_wheels: &py2_linux_build_wheels stage: *cron name: "Build Linux wheels (Python 2 PEX)" env: - - <<: *py2_linux_test_config_env - - <<: *base_build_wheels_env + - *py2_linux_test_config_env + - *base_build_wheels_env - CACHE_NAME=linuxwheelsbuild.py2 py3_linux_build_wheels: &py3_linux_build_wheels @@ -323,8 +323,8 @@ py3_linux_build_wheels: &py3_linux_build_wheels <<: *base_build_wheels name: "Build Linux wheels (Python 3 PEX)" env: - - <<: *py3_linux_test_config_env - - <<: *base_build_wheels_env + - *py3_linux_test_config_env + - *base_build_wheels_env - CACHE_NAME=linuxwheelsbuild.py3 py2_osx_build_wheels: &py2_osx_build_wheels @@ -333,17 +333,17 @@ py2_osx_build_wheels: &py2_osx_build_wheels name: "Build OSX wheels (Python 2 PEX)" stage: *cron env: - - <<: *py2_osx_test_config_env - - <<: *base_build_wheels_env + - *py2_osx_test_config_env + - *base_build_wheels_env - CACHE_NAME=osxwheelsbuild.py2 py3_osx_build_wheels: &py3_osx_build_wheels <<: *py3_osx_test_config - <<: *base_osx_build_wheels + <<: *base_build_wheels name: "Build OSX wheels (Python 3 PEX)" env: - - <<: *py3_osx_test_config_env - - <<: *base_build_wheels_env + - *py3_osx_test_config_env + - *base_build_wheels_env - CACHE_NAME=osxwheelsbuild.py3 # ------------------------------------------------------------------------- @@ -365,7 +365,7 @@ py2_osx_10_12_sanity_check: &py2_osx_10_12_sanity_check name: "OSX 10.12 sanity check (Python 2 PEX)" stage: *cron env: - - <<: *py2_osx_test_config_env + - *py2_osx_test_config_env - CACHE_NAME=macos10.12sanity.py2 py3_osx_10_12_sanity_check: &py3_osx_10_12_sanity_check @@ -373,7 +373,7 @@ py3_osx_10_12_sanity_check: &py3_osx_10_12_sanity_check <<: *base_osx_10_12_sanity_check name: "OSX 10.12 sanity check (Python 3 PEX)" env: - - <<: *py3_osx_test_config_env + - *py3_osx_test_config_env - CACHE_NAME=macos10.12sanity.py3 base_osx_10_13_sanity_check: &base_osx_10_13_sanity_check @@ -386,7 +386,7 @@ py2_osx_10_13_sanity_check: &py2_osx_10_13_sanity_check name: "OSX 10.13 sanity check (Python 2 PEX)" stage: *cron env: - - <<: *py2_osx_test_config_env + - *py2_osx_test_config_env - CACHE_NAME=macos10.13sanity.py2 py3_osx_10_13_sanity_check: &py3_osx_10_13_sanity_check @@ -394,7 +394,7 @@ py3_osx_10_13_sanity_check: &py3_osx_10_13_sanity_check <<: *base_osx_10_13_sanity_check name: "OSX 10.13 sanity check (Python 3 PEX)" env: - - <<: *py3_osx_test_config_env + - *py3_osx_test_config_env - CACHE_NAME=macos10.13sanity.py3 # ------------------------------------------------------------------------- @@ -411,7 +411,7 @@ py2_lint: &py2_lint <<: *base_lint name: "Self-checks, lint, and JVM tests (Python 2 PEX)" env: - - <<: *py2_linux_test_config_env + - *py2_linux_test_config_env - CACHE_NAME=linuxselfchecks.py2 py3_lint: &py3_lint @@ -419,7 +419,7 @@ py3_lint: &py3_lint <<: *base_lint name: "Self-checks, lint, and JVM tests (Python 3 PEX)" env: - - <<: *py3_linux_test_config_env + - *py3_linux_test_config_env - CACHE_NAME=linuxselfchecks.py3 # ------------------------------------------------------------------------- @@ -487,18 +487,18 @@ py2_linux_rust_tests: &py2_linux_rust_tests python: 2.7 stage: *cron env: - - <<: *py2_linux_test_config_env + - *py2_linux_test_config_env - CACHE_NAME=linuxrusttests.py2 script: - ./build-support/bin/travis-ci.sh -be2 -py2_linux_rust_tests: &py2_linux_rust_tests +py3_linux_rust_tests: &py3_linux_rust_tests <<: *base_linux_rust_tests name: "Linux Rust tests (Python 3 PEX)" python: 3.6 stage: *test env: - - <<: *py3_linux_test_config_env + - *py3_linux_test_config_env - CACHE_NAME=linuxrusttests.py3 script: - ./build-support/bin/travis-ci.sh -be @@ -515,20 +515,20 @@ base_osx_rust_tests: &base_osx_rust_tests py2_osx_rust_tests: &py2_osx_rust_tests <<: *base_osx_rust_tests - <<: py2_osx_test_config + <<: *py2_osx_test_config name: "OSX Rust + platform-specific tests (Python 2 PEX)" env: - - <<: *py2_osx_test_config_env + - *py2_osx_test_config_env - CACHE_NAME=macosrusttests.py2 script: - ./build-support/bin/travis-ci.sh -bez2 py3_osx_rust_tests: &py3_osx_rust_tests <<: *base_osx_rust_tests - <<: py3_osx_test_config + <<: *py3_osx_test_config name: "OSX Rust + platform-specific tests (Python 3 PEX)" env: - - <<: *py3_osx_test_config_env + - *py3_osx_test_config_env - CACHE_NAME=macosrusttests.py3 script: - ./build-support/bin/travis-ci.sh -bez @@ -567,7 +567,7 @@ matrix: name: "Py2 - Unit tests for pants and pants-plugins" stage: *test env: - - <<: *py2_linux_test_config_env + - *py2_linux_test_config_env - CACHE_NAME=linuxunittests.py2 script: - ./build-support/bin/travis-ci.sh -2blp @@ -575,7 +575,7 @@ matrix: - <<: *py3_linux_test_config name: "Py3 - Unit tests for pants and pants-plugins" env: - - <<: *py3_linux_test_config_env + - *py3_linux_test_config_env - CACHE_NAME=linuxunittests.py3 script: - ./build-support/bin/travis-ci.sh -blp @@ -583,7 +583,7 @@ matrix: - <<: *py2_linux_test_config name: "Py2 - Python contrib tests" env: - - <<: *py2_linux_test_config_env + - *py2_linux_test_config_env - CACHE_NAME=linuxcontribtests.py2 script: - ./build-support/bin/travis-ci.sh -2bn @@ -591,7 +591,7 @@ matrix: - <<: *py3_linux_test_config name: "Py3 - Python contrib tests" env: - - <<: *py3_linux_test_config_env + - *py3_linux_test_config_env - CACHE_NAME=linuxcontribtests.py3 script: - ./build-support/bin/travis-ci.sh -bn @@ -599,7 +599,7 @@ matrix: - <<: *py3_linux_test_config name: "Integration tests for pants - shard 0" env: - - <<: *py3_linux_test_config_env + - *py3_linux_test_config_env - CACHE_NAME=integrationshard0 script: - ./build-support/bin/travis-ci.sh -bc -i 0/20 @@ -607,7 +607,7 @@ matrix: - <<: *py3_linux_test_config name: "Integration tests for pants - shard 1" env: - - <<: *py3_linux_test_config_env + - *py3_linux_test_config_env - CACHE_NAME=integrationshard1 script: - ./build-support/bin/travis-ci.sh -bc -i 1/20 @@ -615,7 +615,7 @@ matrix: - <<: *py3_linux_test_config name: "Integration tests for pants - shard 2" env: - - <<: *py3_linux_test_config_env + - *py3_linux_test_config_env - CACHE_NAME=integrationshard2 script: - ./build-support/bin/travis-ci.sh -bc -i 2/20 @@ -623,7 +623,7 @@ matrix: - <<: *py3_linux_test_config name: "Integration tests for pants - shard 3" env: - - <<: *py3_linux_test_config_env + - *py3_linux_test_config_env - CACHE_NAME=integrationshard3 script: - ./build-support/bin/travis-ci.sh -bc -i 3/20 @@ -631,7 +631,7 @@ matrix: - <<: *py3_linux_test_config name: "Integration tests for pants - shard 4" env: - - <<: *py3_linux_test_config_env + - *py3_linux_test_config_env - CACHE_NAME=integrationshard4 script: - ./build-support/bin/travis-ci.sh -bc -i 4/20 @@ -639,7 +639,7 @@ matrix: - <<: *py3_linux_test_config name: "Integration tests for pants - shard 5" env: - - <<: *py3_linux_test_config_env + - *py3_linux_test_config_env - CACHE_NAME=integrationshard5 script: - ./build-support/bin/travis-ci.sh -bc -i 5/20 @@ -647,7 +647,7 @@ matrix: - <<: *py3_linux_test_config name: "Integration tests for pants - shard 6" env: - - <<: *py3_linux_test_config_env + - *py3_linux_test_config_env - CACHE_NAME=integrationshard6 script: - ./build-support/bin/travis-ci.sh -bc -i 6/20 @@ -655,7 +655,7 @@ matrix: - <<: *py3_linux_test_config name: "Integration tests for pants - shard 7" env: - - <<: *py3_linux_test_config_env + - *py3_linux_test_config_env - CACHE_NAME=integrationshard7 script: - ./build-support/bin/travis-ci.sh -bc -i 7/20 @@ -663,7 +663,7 @@ matrix: - <<: *py3_linux_test_config name: "Integration tests for pants - shard 8" env: - - <<: *py3_linux_test_config_env + - *py3_linux_test_config_env - CACHE_NAME=integrationshard8 script: - ./build-support/bin/travis-ci.sh -bc -i 8/20 @@ -671,7 +671,7 @@ matrix: - <<: *py3_linux_test_config name: "Integration tests for pants - shard 9" env: - - <<: *py3_linux_test_config_env + - *py3_linux_test_config_env - CACHE_NAME=integrationshard9 script: - ./build-support/bin/travis-ci.sh -bc -i 9/20 @@ -679,7 +679,7 @@ matrix: - <<: *py3_linux_test_config name: "Integration tests for pants - shard 10" env: - - <<: *py3_linux_test_config_env + - *py3_linux_test_config_env - CACHE_NAME=integrationshard10 script: - ./build-support/bin/travis-ci.sh -bc -i 10/20 @@ -687,7 +687,7 @@ matrix: - <<: *py3_linux_test_config name: "Integration tests for pants - shard 11" env: - - <<: *py3_linux_test_config_env + - *py3_linux_test_config_env - CACHE_NAME=integrationshard11 script: - ./build-support/bin/travis-ci.sh -bc -i 11/20 @@ -695,7 +695,7 @@ matrix: - <<: *py3_linux_test_config name: "Integration tests for pants - shard 12" env: - - <<: *py3_linux_test_config_env + - *py3_linux_test_config_env - CACHE_NAME=integrationshard12 script: - ./build-support/bin/travis-ci.sh -bc -i 12/20 @@ -703,7 +703,7 @@ matrix: - <<: *py3_linux_test_config name: "Integration tests for pants - shard 13" env: - - <<: *py3_linux_test_config_env + - *py3_linux_test_config_env - CACHE_NAME=integrationshard13 script: - ./build-support/bin/travis-ci.sh -bc -i 13/20 @@ -711,7 +711,7 @@ matrix: - <<: *py3_linux_test_config name: "Integration tests for pants - shard 14" env: - - <<: *py3_linux_test_config_env + - *py3_linux_test_config_env - CACHE_NAME=integrationshard14 script: - ./build-support/bin/travis-ci.sh -bc -i 14/20 @@ -719,7 +719,7 @@ matrix: - <<: *py3_linux_test_config name: "Integration tests for pants - shard 15" env: - - <<: *py3_linux_test_config_env + - *py3_linux_test_config_env - CACHE_NAME=integrationshard15 script: - ./build-support/bin/travis-ci.sh -bc -i 15/20 @@ -727,7 +727,7 @@ matrix: - <<: *py3_linux_test_config name: "Integration tests for pants - shard 16" env: - - <<: *py3_linux_test_config_env + - *py3_linux_test_config_env - CACHE_NAME=integrationshard16 script: - ./build-support/bin/travis-ci.sh -bc -i 16/20 @@ -735,7 +735,7 @@ matrix: - <<: *py3_linux_test_config name: "Integration tests for pants - shard 17" env: - - <<: *py3_linux_test_config_env + - *py3_linux_test_config_env - CACHE_NAME=integrationshard17 script: - ./build-support/bin/travis-ci.sh -bc -i 17/20 @@ -743,7 +743,7 @@ matrix: - <<: *py3_linux_test_config name: "Integration tests for pants - shard 18" env: - - <<: *py3_linux_test_config_env + - *py3_linux_test_config_env - CACHE_NAME=integrationshard18 script: - ./build-support/bin/travis-ci.sh -bc -i 18/20 @@ -751,7 +751,7 @@ matrix: - <<: *py3_linux_test_config name: "Integration tests for pants - shard 19" env: - - <<: *py3_linux_test_config_env + - *py3_linux_test_config_env - CACHE_NAME=integrationshard19 script: - ./build-support/bin/travis-ci.sh -bc -i 19/20 @@ -760,7 +760,7 @@ matrix: name: "Integration tests for pants (Python 2) - shard 0" stage: *cron env: - - <<: *py2_linux_test_config_env + - *py2_linux_test_config_env - CACHE_NAME=cronshard0 script: - ./build-support/bin/travis-ci.sh -bc2 -i 0/20 @@ -769,7 +769,7 @@ matrix: name: "Integration tests for pants (Python 2) - shard 1" stage: *cron env: - - <<: *py2_linux_test_config_env + - *py2_linux_test_config_env - CACHE_NAME=cronshard1 script: - ./build-support/bin/travis-ci.sh -bc2 -i 1/20 @@ -778,7 +778,7 @@ matrix: name: "Integration tests for pants (Python 2) - shard 2" stage: *cron env: - - <<: *py2_linux_test_config_env + - *py2_linux_test_config_env - CACHE_NAME=cronshard2 script: - ./build-support/bin/travis-ci.sh -bc2 -i 2/20 @@ -787,7 +787,7 @@ matrix: name: "Integration tests for pants (Python 2) - shard 3" stage: *cron env: - - <<: *py2_linux_test_config_env + - *py2_linux_test_config_env - CACHE_NAME=cronshard3 script: - ./build-support/bin/travis-ci.sh -bc2 -i 3/20 @@ -796,7 +796,7 @@ matrix: name: "Integration tests for pants (Python 2) - shard 4" stage: *cron env: - - <<: *py2_linux_test_config_env + - *py2_linux_test_config_env - CACHE_NAME=cronshard4 script: - ./build-support/bin/travis-ci.sh -bc2 -i 4/20 @@ -805,7 +805,7 @@ matrix: name: "Integration tests for pants (Python 2) - shard 5" stage: *cron env: - - <<: *py2_linux_test_config_env + - *py2_linux_test_config_env - CACHE_NAME=cronshard5 script: - ./build-support/bin/travis-ci.sh -bc2 -i 5/20 @@ -814,7 +814,7 @@ matrix: name: "Integration tests for pants (Python 2) - shard 6" stage: *cron env: - - <<: *py2_linux_test_config_env + - *py2_linux_test_config_env - CACHE_NAME=cronshard6 script: - ./build-support/bin/travis-ci.sh -bc2 -i 6/20 @@ -823,7 +823,7 @@ matrix: name: "Integration tests for pants (Python 2) - shard 7" stage: *cron env: - - <<: *py2_linux_test_config_env + - *py2_linux_test_config_env - CACHE_NAME=cronshard7 script: - ./build-support/bin/travis-ci.sh -bc2 -i 7/20 @@ -832,7 +832,7 @@ matrix: name: "Integration tests for pants (Python 2) - shard 8" stage: *cron env: - - <<: *py2_linux_test_config_env + - *py2_linux_test_config_env - CACHE_NAME=cronshard8 script: - ./build-support/bin/travis-ci.sh -bc2 -i 8/20 @@ -841,7 +841,7 @@ matrix: name: "Integration tests for pants (Python 2) - shard 9" stage: *cron env: - - <<: *py2_linux_test_config_env + - *py2_linux_test_config_env - CACHE_NAME=cronshard9 script: - ./build-support/bin/travis-ci.sh -bc2 -i 9/20 @@ -850,7 +850,7 @@ matrix: name: "Integration tests for pants (Python 2) - shard 10" stage: *cron env: - - <<: *py2_linux_test_config_env + - *py2_linux_test_config_env - CACHE_NAME=cronshard10 script: - ./build-support/bin/travis-ci.sh -bc2 -i 10/20 @@ -859,7 +859,7 @@ matrix: name: "Integration tests for pants (Python 2) - shard 11" stage: *cron env: - - <<: *py2_linux_test_config_env + - *py2_linux_test_config_env - CACHE_NAME=cronshard11 script: - ./build-support/bin/travis-ci.sh -bc2 -i 11/20 @@ -868,7 +868,7 @@ matrix: name: "Integration tests for pants (Python 2) - shard 12" stage: *cron env: - - <<: *py2_linux_test_config_env + - *py2_linux_test_config_env - CACHE_NAME=cronshard12 script: - ./build-support/bin/travis-ci.sh -bc2 -i 12/20 @@ -877,7 +877,7 @@ matrix: name: "Integration tests for pants (Python 2) - shard 13" stage: *cron env: - - <<: *py2_linux_test_config_env + - *py2_linux_test_config_env - CACHE_NAME=cronshard13 script: - ./build-support/bin/travis-ci.sh -bc2 -i 13/20 @@ -886,7 +886,7 @@ matrix: name: "Integration tests for pants (Python 2) - shard 14" stage: *cron env: - - <<: *py2_linux_test_config_env + - *py2_linux_test_config_env - CACHE_NAME=cronshard14 script: - ./build-support/bin/travis-ci.sh -bc2 -i 14/20 @@ -895,7 +895,7 @@ matrix: name: "Integration tests for pants (Python 2) - shard 15" stage: *cron env: - - <<: *py2_linux_test_config_env + - *py2_linux_test_config_env - CACHE_NAME=cronshard15 script: - ./build-support/bin/travis-ci.sh -bc2 -i 15/20 @@ -904,7 +904,7 @@ matrix: name: "Integration tests for pants (Python 2) - shard 16" stage: *cron env: - - <<: *py2_linux_test_config_env + - *py2_linux_test_config_env - CACHE_NAME=cronshard16 script: - ./build-support/bin/travis-ci.sh -bc2 -i 16/20 @@ -913,7 +913,7 @@ matrix: name: "Integration tests for pants (Python 2) - shard 17" stage: *cron env: - - <<: *py2_linux_test_config_env + - *py2_linux_test_config_env - CACHE_NAME=cronshard17 script: - ./build-support/bin/travis-ci.sh -bc2 -i 17/20 @@ -922,7 +922,7 @@ matrix: name: "Integration tests for pants (Python 2) - shard 18" stage: *cron env: - - <<: *py2_linux_test_config_env + - *py2_linux_test_config_env - CACHE_NAME=cronshard18 script: - ./build-support/bin/travis-ci.sh -bc2 -i 18/20 @@ -931,7 +931,7 @@ matrix: name: "Integration tests for pants (Python 2) - shard 19" stage: *cron env: - - <<: *py2_linux_test_config_env + - *py2_linux_test_config_env - CACHE_NAME=cronshard19 script: - ./build-support/bin/travis-ci.sh -bc2 -i 19/20 @@ -947,7 +947,7 @@ matrix: <<: *native_engine_cache_config name: "Linux Rust Clippy" env: - - <<: *py2_linux_test_config_env + - *py2_linux_test_config_env - CACHE_NAME=linuxclippy os: linux dist: xenial @@ -964,7 +964,7 @@ matrix: - <<: *linux_with_fuse name: "Cargo audit" env: - - <<: *py2_linux_test_config_env + - *py2_linux_test_config_env - CACHE_NAME=linuxcargoaudit os: linux dist: xenial diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index 5a77520744d..0c0f03e67fb 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -129,25 +129,24 @@ py2_linux_test_config: &py2_linux_test_config <<: *py2_linux_config <<: *base_linux_test_config stage: *cron - env: &py2_linux_test_config_env - - BOOTSTRAPPED_PEX_KEY_SUFFIX="py2.linux" + env: + - &py2_linux_test_config_env BOOTSTRAPPED_PEX_KEY_SUFFIX="py2.linux" py3_linux_test_config: &py3_linux_test_config <<: *py3_linux_config <<: *base_linux_test_config stage: *test - env: &py3_linux_test_config_env - - BOOTSTRAPPED_PEX_KEY_SUFFIX="py3.linux" + env: + - &py3_linux_test_config_env BOOTSTRAPPED_PEX_KEY_SUFFIX="py3.linux" base_osx_config: &base_osx_config os: osx language: generic # TODO: workaround for Homebrew bug triggered by running `brew update`. # Follow at https://github.com/Homebrew/brew/issues/5513 and remove this workaround once its fixed. - env: &base_osx_config_env - - HOMEBREW_LOGS=~/homebrew-logs - - HOMEBREW_TEMP=~/homebrew-temp - before_install: &base_osx_config_before_install + env: + - &base_osx_config_env HOMEBREW_LOGS=~/homebrew-log HOMEBREW_TEMP=~/homebrew-temp + before_install: {{>before_install_osx}} py2_osx_config: &py2_osx_config @@ -156,7 +155,7 @@ py2_osx_config: &py2_osx_config py3_osx_config: &py3_osx_config <<: *base_osx_config before_install: - - <<: *base_osx_config_before_install + {{>before_install_osx}} # Upgrade to Python 3 - brew upgrade python @@ -173,14 +172,14 @@ py2_osx_test_config: &py2_osx_test_config <<: *py2_osx_config <<: *base_osx_test_config env: &py2_osx_test_config_env - - <<: *base_osx_config_env + - *base_osx_config_env - BOOTSTRAPPED_PEX_KEY_SUFFIX="py2.osx" py3_osx_test_config: &py3_osx_test_config <<: *py3_osx_config <<: *base_osx_test_config env: &py3_osx_test_config_env - - <<: *base_osx_config_env + - *base_osx_config_env - BOOTSTRAPPED_PEX_KEY_SUFFIX="py3.osx" linux_with_fuse: &linux_with_fuse @@ -228,7 +227,7 @@ py2_linux_build_engine: &py2_linux_build_engine <<: *base_linux_build_engine name: "Build Linux native engine and Python 2 pants.pex" env: - - <<: *base_linux_build_engine_env + - *base_linux_build_engine_env - CACHE_NAME=linuxpexbuild.py2 - BOOTSTRAPPED_PEX_KEY_SUFFIX="py2.linux" - BOOTSTRAP_ARGS="-2" @@ -238,7 +237,7 @@ py3_linux_build_engine: &py3_linux_build_engine <<: *base_linux_build_engine name: "Build Linux native engine and Python 3 pants.pex" env: - - <<: *base_linux_build_engine_env + - *base_linux_build_engine_env - CACHE_NAME=linuxpexbuild.py3 - BOOTSTRAPPED_PEX_KEY_SUFFIX="py3.linux" - BOOTSTRAP_ARGS="" @@ -266,7 +265,7 @@ py2_osx_build_engine: &py2_osx_build_engine <<: *base_osx_build_engine name: "Build OSX native engine and Python 2 pants.pex" env: - - <<: *base_osx_build_engine_env + - *base_osx_build_engine_env - CACHE_NAME=osxpexbuild.py2 - BOOTSTRAPPED_PEX_KEY_SUFFIX="py2.osx" - BOOTSTRAP_ARGS="-2" @@ -276,7 +275,7 @@ py3_osx_build_engine: &py3_osx_build_engine <<: *base_osx_build_engine name: "Build OSX native engine and Python 3 pants.pex" env: - - <<: *base_osx_build_engine_env + - *base_osx_build_engine_env - CACHE_NAME=osxpexbuild.py3 - BOOTSTRAPPED_PEX_KEY_SUFFIX="py3.osx" - BOOTSTRAP_ARGS="" @@ -286,9 +285,8 @@ py3_osx_build_engine: &py3_osx_build_engine # ------------------------------------------------------------------------- base_build_wheels: &base_build_wheels - env: &base_build_wheels_env - - RUN_PANTS_FROM_PEX=1 - - PREPARE_DEPLOY=1 + env: + - &base_build_wheels_env RUN_PANTS_FROM_PEX=1 PREPARE_DEPLOY=1 script: - ./build-support/bin/release.sh -n @@ -298,8 +296,8 @@ py2_linux_build_wheels: &py2_linux_build_wheels stage: *cron name: "Build Linux wheels (Python 2 PEX)" env: - - <<: *py2_linux_test_config_env - - <<: *base_build_wheels_env + - *py2_linux_test_config_env + - *base_build_wheels_env - CACHE_NAME=linuxwheelsbuild.py2 py3_linux_build_wheels: &py3_linux_build_wheels @@ -307,8 +305,8 @@ py3_linux_build_wheels: &py3_linux_build_wheels <<: *base_build_wheels name: "Build Linux wheels (Python 3 PEX)" env: - - <<: *py3_linux_test_config_env - - <<: *base_build_wheels_env + - *py3_linux_test_config_env + - *base_build_wheels_env - CACHE_NAME=linuxwheelsbuild.py3 py2_osx_build_wheels: &py2_osx_build_wheels @@ -317,17 +315,17 @@ py2_osx_build_wheels: &py2_osx_build_wheels name: "Build OSX wheels (Python 2 PEX)" stage: *cron env: - - <<: *py2_osx_test_config_env - - <<: *base_build_wheels_env + - *py2_osx_test_config_env + - *base_build_wheels_env - CACHE_NAME=osxwheelsbuild.py2 py3_osx_build_wheels: &py3_osx_build_wheels <<: *py3_osx_test_config - <<: *base_osx_build_wheels + <<: *base_build_wheels name: "Build OSX wheels (Python 3 PEX)" env: - - <<: *py3_osx_test_config_env - - <<: *base_build_wheels_env + - *py3_osx_test_config_env + - *base_build_wheels_env - CACHE_NAME=osxwheelsbuild.py3 # ------------------------------------------------------------------------- @@ -349,7 +347,7 @@ py2_osx_10_12_sanity_check: &py2_osx_10_12_sanity_check name: "OSX 10.12 sanity check (Python 2 PEX)" stage: *cron env: - - <<: *py2_osx_test_config_env + - *py2_osx_test_config_env - CACHE_NAME=macos10.12sanity.py2 py3_osx_10_12_sanity_check: &py3_osx_10_12_sanity_check @@ -357,7 +355,7 @@ py3_osx_10_12_sanity_check: &py3_osx_10_12_sanity_check <<: *base_osx_10_12_sanity_check name: "OSX 10.12 sanity check (Python 3 PEX)" env: - - <<: *py3_osx_test_config_env + - *py3_osx_test_config_env - CACHE_NAME=macos10.12sanity.py3 base_osx_10_13_sanity_check: &base_osx_10_13_sanity_check @@ -370,7 +368,7 @@ py2_osx_10_13_sanity_check: &py2_osx_10_13_sanity_check name: "OSX 10.13 sanity check (Python 2 PEX)" stage: *cron env: - - <<: *py2_osx_test_config_env + - *py2_osx_test_config_env - CACHE_NAME=macos10.13sanity.py2 py3_osx_10_13_sanity_check: &py3_osx_10_13_sanity_check @@ -378,7 +376,7 @@ py3_osx_10_13_sanity_check: &py3_osx_10_13_sanity_check <<: *base_osx_10_13_sanity_check name: "OSX 10.13 sanity check (Python 3 PEX)" env: - - <<: *py3_osx_test_config_env + - *py3_osx_test_config_env - CACHE_NAME=macos10.13sanity.py3 # ------------------------------------------------------------------------- @@ -395,7 +393,7 @@ py2_lint: &py2_lint <<: *base_lint name: "Self-checks, lint, and JVM tests (Python 2 PEX)" env: - - <<: *py2_linux_test_config_env + - *py2_linux_test_config_env - CACHE_NAME=linuxselfchecks.py2 py3_lint: &py3_lint @@ -403,7 +401,7 @@ py3_lint: &py3_lint <<: *base_lint name: "Self-checks, lint, and JVM tests (Python 3 PEX)" env: - - <<: *py3_linux_test_config_env + - *py3_linux_test_config_env - CACHE_NAME=linuxselfchecks.py3 # ------------------------------------------------------------------------- @@ -471,18 +469,18 @@ py2_linux_rust_tests: &py2_linux_rust_tests python: 2.7 stage: *cron env: - - <<: *py2_linux_test_config_env + - *py2_linux_test_config_env - CACHE_NAME=linuxrusttests.py2 script: - ./build-support/bin/travis-ci.sh -be2 -py2_linux_rust_tests: &py2_linux_rust_tests +py3_linux_rust_tests: &py3_linux_rust_tests <<: *base_linux_rust_tests name: "Linux Rust tests (Python 3 PEX)" python: 3.6 stage: *test env: - - <<: *py3_linux_test_config_env + - *py3_linux_test_config_env - CACHE_NAME=linuxrusttests.py3 script: - ./build-support/bin/travis-ci.sh -be @@ -497,20 +495,20 @@ base_osx_rust_tests: &base_osx_rust_tests py2_osx_rust_tests: &py2_osx_rust_tests <<: *base_osx_rust_tests - <<: py2_osx_test_config + <<: *py2_osx_test_config name: "OSX Rust + platform-specific tests (Python 2 PEX)" env: - - <<: *py2_osx_test_config_env + - *py2_osx_test_config_env - CACHE_NAME=macosrusttests.py2 script: - ./build-support/bin/travis-ci.sh -bez2 py3_osx_rust_tests: &py3_osx_rust_tests <<: *base_osx_rust_tests - <<: py3_osx_test_config + <<: *py3_osx_test_config name: "OSX Rust + platform-specific tests (Python 3 PEX)" env: - - <<: *py3_osx_test_config_env + - *py3_osx_test_config_env - CACHE_NAME=macosrusttests.py3 script: - ./build-support/bin/travis-ci.sh -bez @@ -549,7 +547,7 @@ matrix: name: "Py2 - Unit tests for pants and pants-plugins" stage: *test env: - - <<: *py2_linux_test_config_env + - *py2_linux_test_config_env - CACHE_NAME=linuxunittests.py2 script: - ./build-support/bin/travis-ci.sh -2blp @@ -557,7 +555,7 @@ matrix: - <<: *py3_linux_test_config name: "Py3 - Unit tests for pants and pants-plugins" env: - - <<: *py3_linux_test_config_env + - *py3_linux_test_config_env - CACHE_NAME=linuxunittests.py3 script: - ./build-support/bin/travis-ci.sh -blp @@ -565,7 +563,7 @@ matrix: - <<: *py2_linux_test_config name: "Py2 - Python contrib tests" env: - - <<: *py2_linux_test_config_env + - *py2_linux_test_config_env - CACHE_NAME=linuxcontribtests.py2 script: - ./build-support/bin/travis-ci.sh -2bn @@ -573,7 +571,7 @@ matrix: - <<: *py3_linux_test_config name: "Py3 - Python contrib tests" env: - - <<: *py3_linux_test_config_env + - *py3_linux_test_config_env - CACHE_NAME=linuxcontribtests.py3 script: - ./build-support/bin/travis-ci.sh -bn @@ -582,7 +580,7 @@ matrix: - <<: *py3_linux_test_config name: "Integration tests for pants - shard {{.}}" env: - - <<: *py3_linux_test_config_env + - *py3_linux_test_config_env - CACHE_NAME=integrationshard{{.}} script: - ./build-support/bin/travis-ci.sh -bc -i {{.}}/{{integration_shards_length}} @@ -593,7 +591,7 @@ matrix: name: "Integration tests for pants (Python 2) - shard {{.}}" stage: *cron env: - - <<: *py2_linux_test_config_env + - *py2_linux_test_config_env - CACHE_NAME=cronshard{{.}} script: - ./build-support/bin/travis-ci.sh -bc2 -i {{.}}/{{integration_shards_length}} @@ -610,7 +608,7 @@ matrix: <<: *native_engine_cache_config name: "Linux Rust Clippy" env: - - <<: *py2_linux_test_config_env + - *py2_linux_test_config_env - CACHE_NAME=linuxclippy os: linux dist: xenial @@ -627,7 +625,7 @@ matrix: - <<: *linux_with_fuse name: "Cargo audit" env: - - <<: *py2_linux_test_config_env + - *py2_linux_test_config_env - CACHE_NAME=linuxcargoaudit os: linux dist: xenial From 8450dcd149291587eff6397ac9b476006547668e Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Thu, 10 Jan 2019 21:24:33 -0700 Subject: [PATCH 021/105] Fix additional bad anchors with env Note that you can't mix an anchor with a property. `&new_var *old_var "test"` is invalid. So, we have to duplicate the Homebrew workaround in two places. --- .travis.yml | 18 ++++++++---------- build-support/travis/travis.yml.mustache | 18 ++++++++---------- 2 files changed, 16 insertions(+), 20 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7e162bfd83d..61e1333974b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -184,16 +184,14 @@ base_osx_test_config: &base_osx_test_config py2_osx_test_config: &py2_osx_test_config <<: *py2_osx_config <<: *base_osx_test_config - env: &py2_osx_test_config_env - - *base_osx_config_env - - BOOTSTRAPPED_PEX_KEY_SUFFIX="py2.osx" + env: + - &py2_osx_test_config_env HOMEBREW_LOGS=~/homebrew-log HOMEBREW_TEMP=~/homebrew-temp BOOTSTRAPPED_PEX_KEY_SUFFIX="py2.osx" py3_osx_test_config: &py3_osx_test_config <<: *py3_osx_config <<: *base_osx_test_config - env: &py3_osx_test_config_env - - *base_osx_config_env - - BOOTSTRAPPED_PEX_KEY_SUFFIX="py3.osx" + env: + - &py3_osx_test_config_env HOMEBREW_LOGS=~/homebrew-log HOMEBREW_TEMP=~/homebrew-temp BOOTSTRAPPED_PEX_KEY_SUFFIX="py3.osx" linux_with_fuse: &linux_with_fuse before_install: @@ -218,8 +216,8 @@ base_linux_build_engine: &base_linux_build_engine stage: *bootstrap services: - docker - env: &base_linux_build_engine_env - - PREPARE_DEPLOY=1 # To deploy fs_util. + env: + - &base_linux_build_engine_env PREPARE_DEPLOY=1 # To deploy fs_util. before_script: - ulimit -c unlimited script: @@ -268,8 +266,8 @@ base_osx_build_engine: &base_osx_build_engine # We use 10.11 as a minimum to avoid https://github.com/rust-lang/regex/issues/489. # See: https://docs.travis-ci.com/user/reference/osx/#OS-X-Version osx_image: xcode8 - env: &base_osx_build_engine_env - - PREPARE_DEPLOY=1 # To deploy fs_util. + env: + - &base_osx_build_engine_env PREPARE_DEPLOY=1 # To deploy fs_util. script: # With no args ci.sh just bootstraps a pants.pex. We also build fs_util, to take advantage # of the rust code built during bootstrapping. diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index 0c0f03e67fb..8e890c826b6 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -171,16 +171,14 @@ base_osx_test_config: &base_osx_test_config py2_osx_test_config: &py2_osx_test_config <<: *py2_osx_config <<: *base_osx_test_config - env: &py2_osx_test_config_env - - *base_osx_config_env - - BOOTSTRAPPED_PEX_KEY_SUFFIX="py2.osx" + env: + - &py2_osx_test_config_env HOMEBREW_LOGS=~/homebrew-log HOMEBREW_TEMP=~/homebrew-temp BOOTSTRAPPED_PEX_KEY_SUFFIX="py2.osx" py3_osx_test_config: &py3_osx_test_config <<: *py3_osx_config <<: *base_osx_test_config - env: &py3_osx_test_config_env - - *base_osx_config_env - - BOOTSTRAPPED_PEX_KEY_SUFFIX="py3.osx" + env: + - &py3_osx_test_config_env HOMEBREW_LOGS=~/homebrew-log HOMEBREW_TEMP=~/homebrew-temp BOOTSTRAPPED_PEX_KEY_SUFFIX="py3.osx" linux_with_fuse: &linux_with_fuse before_install: @@ -200,8 +198,8 @@ base_linux_build_engine: &base_linux_build_engine stage: *bootstrap services: - docker - env: &base_linux_build_engine_env - - PREPARE_DEPLOY=1 # To deploy fs_util. + env: + - &base_linux_build_engine_env PREPARE_DEPLOY=1 # To deploy fs_util. before_script: - ulimit -c unlimited script: @@ -250,8 +248,8 @@ base_osx_build_engine: &base_osx_build_engine # We use 10.11 as a minimum to avoid https://github.com/rust-lang/regex/issues/489. # See: https://docs.travis-ci.com/user/reference/osx/#OS-X-Version osx_image: xcode8 - env: &base_osx_build_engine_env - - PREPARE_DEPLOY=1 # To deploy fs_util. + env: + - &base_osx_build_engine_env PREPARE_DEPLOY=1 # To deploy fs_util. script: # With no args ci.sh just bootstraps a pants.pex. We also build fs_util, to take advantage # of the rust code built during bootstrapping. From e5cba84e5492f303f2d8e8f79943fd1b8bb2cb43 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Fri, 11 Jan 2019 13:23:23 -0700 Subject: [PATCH 022/105] Use consistent naming scheme for py2 vs py3 shards --- .travis.yml | 125 +++++++++++------------ build-support/travis/travis.yml.mustache | 49 +++++---- 2 files changed, 86 insertions(+), 88 deletions(-) diff --git a/.travis.yml b/.travis.yml index 61e1333974b..b3e8bbc0cbc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -241,7 +241,7 @@ base_linux_build_engine: &base_linux_build_engine py2_linux_build_engine: &py2_linux_build_engine <<: *py2_linux_config <<: *base_linux_build_engine - name: "Build Linux native engine and Python 2 pants.pex" + name: "Build Linux native engine and pants.pex (Py2 PEX)" env: - *base_linux_build_engine_env - CACHE_NAME=linuxpexbuild.py2 @@ -251,7 +251,7 @@ py2_linux_build_engine: &py2_linux_build_engine py3_linux_build_engine: &py3_linux_build_engine <<: *py3_linux_config <<: *base_linux_build_engine - name: "Build Linux native engine and Python 3 pants.pex" + name: "Build Linux native engine and pants.pex (Py3 PEX)" env: - *base_linux_build_engine_env - CACHE_NAME=linuxpexbuild.py3 @@ -279,7 +279,7 @@ base_osx_build_engine: &base_osx_build_engine py2_osx_build_engine: &py2_osx_build_engine <<: *py2_osx_config <<: *base_osx_build_engine - name: "Build OSX native engine and Python 2 pants.pex" + name: "Build OSX native engine and pants.pex (Py2 PEX)" env: - *base_osx_build_engine_env - CACHE_NAME=osxpexbuild.py2 @@ -289,7 +289,7 @@ py2_osx_build_engine: &py2_osx_build_engine py3_osx_build_engine: &py3_osx_build_engine <<: *py3_osx_config <<: *base_osx_build_engine - name: "Build OSX native engine and Python 3 pants.pex" + name: "Build OSX native engine and pants.pex (Py3 PEX)" env: - *base_osx_build_engine_env - CACHE_NAME=osxpexbuild.py3 @@ -310,7 +310,7 @@ py2_linux_build_wheels: &py2_linux_build_wheels <<: *py2_linux_test_config <<: *base_build_wheels stage: *cron - name: "Build Linux wheels (Python 2 PEX)" + name: "Build Linux wheels (Py2 PEX)" env: - *py2_linux_test_config_env - *base_build_wheels_env @@ -319,7 +319,7 @@ py2_linux_build_wheels: &py2_linux_build_wheels py3_linux_build_wheels: &py3_linux_build_wheels <<: *py3_linux_test_config <<: *base_build_wheels - name: "Build Linux wheels (Python 3 PEX)" + name: "Build Linux wheels (Py3 PEX)" env: - *py3_linux_test_config_env - *base_build_wheels_env @@ -328,7 +328,7 @@ py3_linux_build_wheels: &py3_linux_build_wheels py2_osx_build_wheels: &py2_osx_build_wheels <<: *py2_osx_test_config <<: *base_build_wheels - name: "Build OSX wheels (Python 2 PEX)" + name: "Build OSX wheels (Py2 PEX)" stage: *cron env: - *py2_osx_test_config_env @@ -338,7 +338,7 @@ py2_osx_build_wheels: &py2_osx_build_wheels py3_osx_build_wheels: &py3_osx_build_wheels <<: *py3_osx_test_config <<: *base_build_wheels - name: "Build OSX wheels (Python 3 PEX)" + name: "Build OSX wheels (Py3 PEX)" env: - *py3_osx_test_config_env - *base_build_wheels_env @@ -360,7 +360,7 @@ base_osx_10_12_sanity_check: &base_osx_10_12_sanity_check py2_osx_10_12_sanity_check: &py2_osx_10_12_sanity_check <<: *py2_osx_test_config <<: *base_osx_10_12_sanity_check - name: "OSX 10.12 sanity check (Python 2 PEX)" + name: "OSX 10.12 sanity check (Py2 PEX)" stage: *cron env: - *py2_osx_test_config_env @@ -369,7 +369,7 @@ py2_osx_10_12_sanity_check: &py2_osx_10_12_sanity_check py3_osx_10_12_sanity_check: &py3_osx_10_12_sanity_check <<: *py3_osx_test_config <<: *base_osx_10_12_sanity_check - name: "OSX 10.12 sanity check (Python 3 PEX)" + name: "OSX 10.12 sanity check (Py3 PEX)" env: - *py3_osx_test_config_env - CACHE_NAME=macos10.12sanity.py3 @@ -381,7 +381,7 @@ base_osx_10_13_sanity_check: &base_osx_10_13_sanity_check py2_osx_10_13_sanity_check: &py2_osx_10_13_sanity_check <<: *py2_osx_test_config <<: *base_osx_10_13_sanity_check - name: "OSX 10.13 sanity check (Python 2 PEX)" + name: "OSX 10.13 sanity check (Py2 PEX)" stage: *cron env: - *py2_osx_test_config_env @@ -390,7 +390,7 @@ py2_osx_10_13_sanity_check: &py2_osx_10_13_sanity_check py3_osx_10_13_sanity_check: &py3_osx_10_13_sanity_check <<: *py3_osx_test_config <<: *base_osx_10_13_sanity_check - name: "OSX 10.13 sanity check (Python 3 PEX)" + name: "OSX 10.13 sanity check (Py3 PEX)" env: - *py3_osx_test_config_env - CACHE_NAME=macos10.13sanity.py3 @@ -407,7 +407,7 @@ base_lint: &base_lint py2_lint: &py2_lint <<: *py2_linux_test_config <<: *base_lint - name: "Self-checks, lint, and JVM tests (Python 2 PEX)" + name: "Self-checks, lint, and JVM tests (Py2 PEX)" env: - *py2_linux_test_config_env - CACHE_NAME=linuxselfchecks.py2 @@ -415,7 +415,7 @@ py2_lint: &py2_lint py3_lint: &py3_lint <<: *py3_linux_test_config <<: *base_lint - name: "Self-checks, lint, and JVM tests (Python 3 PEX)" + name: "Self-checks, lint, and JVM tests (Py3 PEX)" env: - *py3_linux_test_config_env - CACHE_NAME=linuxselfchecks.py3 @@ -468,7 +468,6 @@ deploy_unstable_multiplatform_pex: &deploy_unstable_multiplatform_pex base_linux_rust_tests: &base_linux_rust_tests <<: *linux_with_fuse <<: *native_engine_cache_config - name: "Linux Rust tests" env: - CACHE_NAME=linuxrusttests os: linux @@ -481,7 +480,7 @@ base_linux_rust_tests: &base_linux_rust_tests py2_linux_rust_tests: &py2_linux_rust_tests <<: *base_linux_rust_tests - name: "Linux Rust tests (Python 2 PEX)" + name: "Linux Rust tests (Py2 PEX)" python: 2.7 stage: *cron env: @@ -492,7 +491,7 @@ py2_linux_rust_tests: &py2_linux_rust_tests py3_linux_rust_tests: &py3_linux_rust_tests <<: *base_linux_rust_tests - name: "Linux Rust tests (Python 3 PEX)" + name: "Linux Rust tests (Py3 PEX)" python: 3.6 stage: *test env: @@ -514,7 +513,7 @@ base_osx_rust_tests: &base_osx_rust_tests py2_osx_rust_tests: &py2_osx_rust_tests <<: *base_osx_rust_tests <<: *py2_osx_test_config - name: "OSX Rust + platform-specific tests (Python 2 PEX)" + name: "OSX Rust + platform-specific tests (Py2 PEX)" env: - *py2_osx_test_config_env - CACHE_NAME=macosrusttests.py2 @@ -524,7 +523,7 @@ py2_osx_rust_tests: &py2_osx_rust_tests py3_osx_rust_tests: &py3_osx_rust_tests <<: *base_osx_rust_tests <<: *py3_osx_test_config - name: "OSX Rust + platform-specific tests (Python 3 PEX)" + name: "OSX Rust + platform-specific tests (Py3 PEX)" env: - *py3_osx_test_config_env - CACHE_NAME=macosrusttests.py3 @@ -562,7 +561,7 @@ matrix: - <<: *py3_lint - <<: *py2_linux_test_config - name: "Py2 - Unit tests for pants and pants-plugins" + name: "Unit tests for pants and pants-plugins (Py2 PEX)" stage: *test env: - *py2_linux_test_config_env @@ -571,7 +570,7 @@ matrix: - ./build-support/bin/travis-ci.sh -2blp - <<: *py3_linux_test_config - name: "Py3 - Unit tests for pants and pants-plugins" + name: "Unit tests for pants and pants-plugins (Py3 PEX)" env: - *py3_linux_test_config_env - CACHE_NAME=linuxunittests.py3 @@ -579,7 +578,7 @@ matrix: - ./build-support/bin/travis-ci.sh -blp - <<: *py2_linux_test_config - name: "Py2 - Python contrib tests" + name: "Python contrib tests (Py2 PEX)" env: - *py2_linux_test_config_env - CACHE_NAME=linuxcontribtests.py2 @@ -587,7 +586,7 @@ matrix: - ./build-support/bin/travis-ci.sh -2bn - <<: *py3_linux_test_config - name: "Py3 - Python contrib tests" + name: "Python contrib tests (Py3 PEX)" env: - *py3_linux_test_config_env - CACHE_NAME=linuxcontribtests.py3 @@ -595,7 +594,7 @@ matrix: - ./build-support/bin/travis-ci.sh -bn - <<: *py3_linux_test_config - name: "Integration tests for pants - shard 0" + name: "Integration tests for pants - shard 0 (Py3 PEX)" env: - *py3_linux_test_config_env - CACHE_NAME=integrationshard0 @@ -603,7 +602,7 @@ matrix: - ./build-support/bin/travis-ci.sh -bc -i 0/20 - <<: *py3_linux_test_config - name: "Integration tests for pants - shard 1" + name: "Integration tests for pants - shard 1 (Py3 PEX)" env: - *py3_linux_test_config_env - CACHE_NAME=integrationshard1 @@ -611,7 +610,7 @@ matrix: - ./build-support/bin/travis-ci.sh -bc -i 1/20 - <<: *py3_linux_test_config - name: "Integration tests for pants - shard 2" + name: "Integration tests for pants - shard 2 (Py3 PEX)" env: - *py3_linux_test_config_env - CACHE_NAME=integrationshard2 @@ -619,7 +618,7 @@ matrix: - ./build-support/bin/travis-ci.sh -bc -i 2/20 - <<: *py3_linux_test_config - name: "Integration tests for pants - shard 3" + name: "Integration tests for pants - shard 3 (Py3 PEX)" env: - *py3_linux_test_config_env - CACHE_NAME=integrationshard3 @@ -627,7 +626,7 @@ matrix: - ./build-support/bin/travis-ci.sh -bc -i 3/20 - <<: *py3_linux_test_config - name: "Integration tests for pants - shard 4" + name: "Integration tests for pants - shard 4 (Py3 PEX)" env: - *py3_linux_test_config_env - CACHE_NAME=integrationshard4 @@ -635,7 +634,7 @@ matrix: - ./build-support/bin/travis-ci.sh -bc -i 4/20 - <<: *py3_linux_test_config - name: "Integration tests for pants - shard 5" + name: "Integration tests for pants - shard 5 (Py3 PEX)" env: - *py3_linux_test_config_env - CACHE_NAME=integrationshard5 @@ -643,7 +642,7 @@ matrix: - ./build-support/bin/travis-ci.sh -bc -i 5/20 - <<: *py3_linux_test_config - name: "Integration tests for pants - shard 6" + name: "Integration tests for pants - shard 6 (Py3 PEX)" env: - *py3_linux_test_config_env - CACHE_NAME=integrationshard6 @@ -651,7 +650,7 @@ matrix: - ./build-support/bin/travis-ci.sh -bc -i 6/20 - <<: *py3_linux_test_config - name: "Integration tests for pants - shard 7" + name: "Integration tests for pants - shard 7 (Py3 PEX)" env: - *py3_linux_test_config_env - CACHE_NAME=integrationshard7 @@ -659,7 +658,7 @@ matrix: - ./build-support/bin/travis-ci.sh -bc -i 7/20 - <<: *py3_linux_test_config - name: "Integration tests for pants - shard 8" + name: "Integration tests for pants - shard 8 (Py3 PEX)" env: - *py3_linux_test_config_env - CACHE_NAME=integrationshard8 @@ -667,7 +666,7 @@ matrix: - ./build-support/bin/travis-ci.sh -bc -i 8/20 - <<: *py3_linux_test_config - name: "Integration tests for pants - shard 9" + name: "Integration tests for pants - shard 9 (Py3 PEX)" env: - *py3_linux_test_config_env - CACHE_NAME=integrationshard9 @@ -675,7 +674,7 @@ matrix: - ./build-support/bin/travis-ci.sh -bc -i 9/20 - <<: *py3_linux_test_config - name: "Integration tests for pants - shard 10" + name: "Integration tests for pants - shard 10 (Py3 PEX)" env: - *py3_linux_test_config_env - CACHE_NAME=integrationshard10 @@ -683,7 +682,7 @@ matrix: - ./build-support/bin/travis-ci.sh -bc -i 10/20 - <<: *py3_linux_test_config - name: "Integration tests for pants - shard 11" + name: "Integration tests for pants - shard 11 (Py3 PEX)" env: - *py3_linux_test_config_env - CACHE_NAME=integrationshard11 @@ -691,7 +690,7 @@ matrix: - ./build-support/bin/travis-ci.sh -bc -i 11/20 - <<: *py3_linux_test_config - name: "Integration tests for pants - shard 12" + name: "Integration tests for pants - shard 12 (Py3 PEX)" env: - *py3_linux_test_config_env - CACHE_NAME=integrationshard12 @@ -699,7 +698,7 @@ matrix: - ./build-support/bin/travis-ci.sh -bc -i 12/20 - <<: *py3_linux_test_config - name: "Integration tests for pants - shard 13" + name: "Integration tests for pants - shard 13 (Py3 PEX)" env: - *py3_linux_test_config_env - CACHE_NAME=integrationshard13 @@ -707,7 +706,7 @@ matrix: - ./build-support/bin/travis-ci.sh -bc -i 13/20 - <<: *py3_linux_test_config - name: "Integration tests for pants - shard 14" + name: "Integration tests for pants - shard 14 (Py3 PEX)" env: - *py3_linux_test_config_env - CACHE_NAME=integrationshard14 @@ -715,7 +714,7 @@ matrix: - ./build-support/bin/travis-ci.sh -bc -i 14/20 - <<: *py3_linux_test_config - name: "Integration tests for pants - shard 15" + name: "Integration tests for pants - shard 15 (Py3 PEX)" env: - *py3_linux_test_config_env - CACHE_NAME=integrationshard15 @@ -723,7 +722,7 @@ matrix: - ./build-support/bin/travis-ci.sh -bc -i 15/20 - <<: *py3_linux_test_config - name: "Integration tests for pants - shard 16" + name: "Integration tests for pants - shard 16 (Py3 PEX)" env: - *py3_linux_test_config_env - CACHE_NAME=integrationshard16 @@ -731,7 +730,7 @@ matrix: - ./build-support/bin/travis-ci.sh -bc -i 16/20 - <<: *py3_linux_test_config - name: "Integration tests for pants - shard 17" + name: "Integration tests for pants - shard 17 (Py3 PEX)" env: - *py3_linux_test_config_env - CACHE_NAME=integrationshard17 @@ -739,7 +738,7 @@ matrix: - ./build-support/bin/travis-ci.sh -bc -i 17/20 - <<: *py3_linux_test_config - name: "Integration tests for pants - shard 18" + name: "Integration tests for pants - shard 18 (Py3 PEX)" env: - *py3_linux_test_config_env - CACHE_NAME=integrationshard18 @@ -747,7 +746,7 @@ matrix: - ./build-support/bin/travis-ci.sh -bc -i 18/20 - <<: *py3_linux_test_config - name: "Integration tests for pants - shard 19" + name: "Integration tests for pants - shard 19 (Py3 PEX)" env: - *py3_linux_test_config_env - CACHE_NAME=integrationshard19 @@ -755,7 +754,7 @@ matrix: - ./build-support/bin/travis-ci.sh -bc -i 19/20 - <<: *py2_linux_test_config - name: "Integration tests for pants (Python 2) - shard 0" + name: "Integration tests for pants - shard 0 (Py2 PEX)" stage: *cron env: - *py2_linux_test_config_env @@ -764,7 +763,7 @@ matrix: - ./build-support/bin/travis-ci.sh -bc2 -i 0/20 - <<: *py2_linux_test_config - name: "Integration tests for pants (Python 2) - shard 1" + name: "Integration tests for pants - shard 1 (Py2 PEX)" stage: *cron env: - *py2_linux_test_config_env @@ -773,7 +772,7 @@ matrix: - ./build-support/bin/travis-ci.sh -bc2 -i 1/20 - <<: *py2_linux_test_config - name: "Integration tests for pants (Python 2) - shard 2" + name: "Integration tests for pants - shard 2 (Py2 PEX)" stage: *cron env: - *py2_linux_test_config_env @@ -782,7 +781,7 @@ matrix: - ./build-support/bin/travis-ci.sh -bc2 -i 2/20 - <<: *py2_linux_test_config - name: "Integration tests for pants (Python 2) - shard 3" + name: "Integration tests for pants - shard 3 (Py2 PEX)" stage: *cron env: - *py2_linux_test_config_env @@ -791,7 +790,7 @@ matrix: - ./build-support/bin/travis-ci.sh -bc2 -i 3/20 - <<: *py2_linux_test_config - name: "Integration tests for pants (Python 2) - shard 4" + name: "Integration tests for pants - shard 4 (Py2 PEX)" stage: *cron env: - *py2_linux_test_config_env @@ -800,7 +799,7 @@ matrix: - ./build-support/bin/travis-ci.sh -bc2 -i 4/20 - <<: *py2_linux_test_config - name: "Integration tests for pants (Python 2) - shard 5" + name: "Integration tests for pants - shard 5 (Py2 PEX)" stage: *cron env: - *py2_linux_test_config_env @@ -809,7 +808,7 @@ matrix: - ./build-support/bin/travis-ci.sh -bc2 -i 5/20 - <<: *py2_linux_test_config - name: "Integration tests for pants (Python 2) - shard 6" + name: "Integration tests for pants - shard 6 (Py2 PEX)" stage: *cron env: - *py2_linux_test_config_env @@ -818,7 +817,7 @@ matrix: - ./build-support/bin/travis-ci.sh -bc2 -i 6/20 - <<: *py2_linux_test_config - name: "Integration tests for pants (Python 2) - shard 7" + name: "Integration tests for pants - shard 7 (Py2 PEX)" stage: *cron env: - *py2_linux_test_config_env @@ -827,7 +826,7 @@ matrix: - ./build-support/bin/travis-ci.sh -bc2 -i 7/20 - <<: *py2_linux_test_config - name: "Integration tests for pants (Python 2) - shard 8" + name: "Integration tests for pants - shard 8 (Py2 PEX)" stage: *cron env: - *py2_linux_test_config_env @@ -836,7 +835,7 @@ matrix: - ./build-support/bin/travis-ci.sh -bc2 -i 8/20 - <<: *py2_linux_test_config - name: "Integration tests for pants (Python 2) - shard 9" + name: "Integration tests for pants - shard 9 (Py2 PEX)" stage: *cron env: - *py2_linux_test_config_env @@ -845,7 +844,7 @@ matrix: - ./build-support/bin/travis-ci.sh -bc2 -i 9/20 - <<: *py2_linux_test_config - name: "Integration tests for pants (Python 2) - shard 10" + name: "Integration tests for pants - shard 10 (Py2 PEX)" stage: *cron env: - *py2_linux_test_config_env @@ -854,7 +853,7 @@ matrix: - ./build-support/bin/travis-ci.sh -bc2 -i 10/20 - <<: *py2_linux_test_config - name: "Integration tests for pants (Python 2) - shard 11" + name: "Integration tests for pants - shard 11 (Py2 PEX)" stage: *cron env: - *py2_linux_test_config_env @@ -863,7 +862,7 @@ matrix: - ./build-support/bin/travis-ci.sh -bc2 -i 11/20 - <<: *py2_linux_test_config - name: "Integration tests for pants (Python 2) - shard 12" + name: "Integration tests for pants - shard 12 (Py2 PEX)" stage: *cron env: - *py2_linux_test_config_env @@ -872,7 +871,7 @@ matrix: - ./build-support/bin/travis-ci.sh -bc2 -i 12/20 - <<: *py2_linux_test_config - name: "Integration tests for pants (Python 2) - shard 13" + name: "Integration tests for pants - shard 13 (Py2 PEX)" stage: *cron env: - *py2_linux_test_config_env @@ -881,7 +880,7 @@ matrix: - ./build-support/bin/travis-ci.sh -bc2 -i 13/20 - <<: *py2_linux_test_config - name: "Integration tests for pants (Python 2) - shard 14" + name: "Integration tests for pants - shard 14 (Py2 PEX)" stage: *cron env: - *py2_linux_test_config_env @@ -890,7 +889,7 @@ matrix: - ./build-support/bin/travis-ci.sh -bc2 -i 14/20 - <<: *py2_linux_test_config - name: "Integration tests for pants (Python 2) - shard 15" + name: "Integration tests for pants - shard 15 (Py2 PEX)" stage: *cron env: - *py2_linux_test_config_env @@ -899,7 +898,7 @@ matrix: - ./build-support/bin/travis-ci.sh -bc2 -i 15/20 - <<: *py2_linux_test_config - name: "Integration tests for pants (Python 2) - shard 16" + name: "Integration tests for pants - shard 16 (Py2 PEX)" stage: *cron env: - *py2_linux_test_config_env @@ -908,7 +907,7 @@ matrix: - ./build-support/bin/travis-ci.sh -bc2 -i 16/20 - <<: *py2_linux_test_config - name: "Integration tests for pants (Python 2) - shard 17" + name: "Integration tests for pants - shard 17 (Py2 PEX)" stage: *cron env: - *py2_linux_test_config_env @@ -917,7 +916,7 @@ matrix: - ./build-support/bin/travis-ci.sh -bc2 -i 17/20 - <<: *py2_linux_test_config - name: "Integration tests for pants (Python 2) - shard 18" + name: "Integration tests for pants - shard 18 (Py2 PEX)" stage: *cron env: - *py2_linux_test_config_env @@ -926,7 +925,7 @@ matrix: - ./build-support/bin/travis-ci.sh -bc2 -i 18/20 - <<: *py2_linux_test_config - name: "Integration tests for pants (Python 2) - shard 19" + name: "Integration tests for pants - shard 19 (Py2 PEX)" stage: *cron env: - *py2_linux_test_config_env diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index 8e890c826b6..626250c009d 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -223,7 +223,7 @@ base_linux_build_engine: &base_linux_build_engine py2_linux_build_engine: &py2_linux_build_engine <<: *py2_linux_config <<: *base_linux_build_engine - name: "Build Linux native engine and Python 2 pants.pex" + name: "Build Linux native engine and pants.pex (Py2 PEX)" env: - *base_linux_build_engine_env - CACHE_NAME=linuxpexbuild.py2 @@ -233,7 +233,7 @@ py2_linux_build_engine: &py2_linux_build_engine py3_linux_build_engine: &py3_linux_build_engine <<: *py3_linux_config <<: *base_linux_build_engine - name: "Build Linux native engine and Python 3 pants.pex" + name: "Build Linux native engine and pants.pex (Py3 PEX)" env: - *base_linux_build_engine_env - CACHE_NAME=linuxpexbuild.py3 @@ -261,7 +261,7 @@ base_osx_build_engine: &base_osx_build_engine py2_osx_build_engine: &py2_osx_build_engine <<: *py2_osx_config <<: *base_osx_build_engine - name: "Build OSX native engine and Python 2 pants.pex" + name: "Build OSX native engine and pants.pex (Py2 PEX)" env: - *base_osx_build_engine_env - CACHE_NAME=osxpexbuild.py2 @@ -271,7 +271,7 @@ py2_osx_build_engine: &py2_osx_build_engine py3_osx_build_engine: &py3_osx_build_engine <<: *py3_osx_config <<: *base_osx_build_engine - name: "Build OSX native engine and Python 3 pants.pex" + name: "Build OSX native engine and pants.pex (Py3 PEX)" env: - *base_osx_build_engine_env - CACHE_NAME=osxpexbuild.py3 @@ -292,7 +292,7 @@ py2_linux_build_wheels: &py2_linux_build_wheels <<: *py2_linux_test_config <<: *base_build_wheels stage: *cron - name: "Build Linux wheels (Python 2 PEX)" + name: "Build Linux wheels (Py2 PEX)" env: - *py2_linux_test_config_env - *base_build_wheels_env @@ -301,7 +301,7 @@ py2_linux_build_wheels: &py2_linux_build_wheels py3_linux_build_wheels: &py3_linux_build_wheels <<: *py3_linux_test_config <<: *base_build_wheels - name: "Build Linux wheels (Python 3 PEX)" + name: "Build Linux wheels (Py3 PEX)" env: - *py3_linux_test_config_env - *base_build_wheels_env @@ -310,7 +310,7 @@ py3_linux_build_wheels: &py3_linux_build_wheels py2_osx_build_wheels: &py2_osx_build_wheels <<: *py2_osx_test_config <<: *base_build_wheels - name: "Build OSX wheels (Python 2 PEX)" + name: "Build OSX wheels (Py2 PEX)" stage: *cron env: - *py2_osx_test_config_env @@ -320,7 +320,7 @@ py2_osx_build_wheels: &py2_osx_build_wheels py3_osx_build_wheels: &py3_osx_build_wheels <<: *py3_osx_test_config <<: *base_build_wheels - name: "Build OSX wheels (Python 3 PEX)" + name: "Build OSX wheels (Py3 PEX)" env: - *py3_osx_test_config_env - *base_build_wheels_env @@ -342,7 +342,7 @@ base_osx_10_12_sanity_check: &base_osx_10_12_sanity_check py2_osx_10_12_sanity_check: &py2_osx_10_12_sanity_check <<: *py2_osx_test_config <<: *base_osx_10_12_sanity_check - name: "OSX 10.12 sanity check (Python 2 PEX)" + name: "OSX 10.12 sanity check (Py2 PEX)" stage: *cron env: - *py2_osx_test_config_env @@ -351,7 +351,7 @@ py2_osx_10_12_sanity_check: &py2_osx_10_12_sanity_check py3_osx_10_12_sanity_check: &py3_osx_10_12_sanity_check <<: *py3_osx_test_config <<: *base_osx_10_12_sanity_check - name: "OSX 10.12 sanity check (Python 3 PEX)" + name: "OSX 10.12 sanity check (Py3 PEX)" env: - *py3_osx_test_config_env - CACHE_NAME=macos10.12sanity.py3 @@ -363,7 +363,7 @@ base_osx_10_13_sanity_check: &base_osx_10_13_sanity_check py2_osx_10_13_sanity_check: &py2_osx_10_13_sanity_check <<: *py2_osx_test_config <<: *base_osx_10_13_sanity_check - name: "OSX 10.13 sanity check (Python 2 PEX)" + name: "OSX 10.13 sanity check (Py2 PEX)" stage: *cron env: - *py2_osx_test_config_env @@ -372,7 +372,7 @@ py2_osx_10_13_sanity_check: &py2_osx_10_13_sanity_check py3_osx_10_13_sanity_check: &py3_osx_10_13_sanity_check <<: *py3_osx_test_config <<: *base_osx_10_13_sanity_check - name: "OSX 10.13 sanity check (Python 3 PEX)" + name: "OSX 10.13 sanity check (Py3 PEX)" env: - *py3_osx_test_config_env - CACHE_NAME=macos10.13sanity.py3 @@ -389,7 +389,7 @@ base_lint: &base_lint py2_lint: &py2_lint <<: *py2_linux_test_config <<: *base_lint - name: "Self-checks, lint, and JVM tests (Python 2 PEX)" + name: "Self-checks, lint, and JVM tests (Py2 PEX)" env: - *py2_linux_test_config_env - CACHE_NAME=linuxselfchecks.py2 @@ -397,7 +397,7 @@ py2_lint: &py2_lint py3_lint: &py3_lint <<: *py3_linux_test_config <<: *base_lint - name: "Self-checks, lint, and JVM tests (Python 3 PEX)" + name: "Self-checks, lint, and JVM tests (Py3 PEX)" env: - *py3_linux_test_config_env - CACHE_NAME=linuxselfchecks.py3 @@ -450,7 +450,6 @@ deploy_unstable_multiplatform_pex: &deploy_unstable_multiplatform_pex base_linux_rust_tests: &base_linux_rust_tests <<: *linux_with_fuse <<: *native_engine_cache_config - name: "Linux Rust tests" env: - CACHE_NAME=linuxrusttests os: linux @@ -463,7 +462,7 @@ base_linux_rust_tests: &base_linux_rust_tests py2_linux_rust_tests: &py2_linux_rust_tests <<: *base_linux_rust_tests - name: "Linux Rust tests (Python 2 PEX)" + name: "Linux Rust tests (Py2 PEX)" python: 2.7 stage: *cron env: @@ -474,7 +473,7 @@ py2_linux_rust_tests: &py2_linux_rust_tests py3_linux_rust_tests: &py3_linux_rust_tests <<: *base_linux_rust_tests - name: "Linux Rust tests (Python 3 PEX)" + name: "Linux Rust tests (Py3 PEX)" python: 3.6 stage: *test env: @@ -494,7 +493,7 @@ base_osx_rust_tests: &base_osx_rust_tests py2_osx_rust_tests: &py2_osx_rust_tests <<: *base_osx_rust_tests <<: *py2_osx_test_config - name: "OSX Rust + platform-specific tests (Python 2 PEX)" + name: "OSX Rust + platform-specific tests (Py2 PEX)" env: - *py2_osx_test_config_env - CACHE_NAME=macosrusttests.py2 @@ -504,7 +503,7 @@ py2_osx_rust_tests: &py2_osx_rust_tests py3_osx_rust_tests: &py3_osx_rust_tests <<: *base_osx_rust_tests <<: *py3_osx_test_config - name: "OSX Rust + platform-specific tests (Python 3 PEX)" + name: "OSX Rust + platform-specific tests (Py3 PEX)" env: - *py3_osx_test_config_env - CACHE_NAME=macosrusttests.py3 @@ -542,7 +541,7 @@ matrix: - <<: *py3_lint - <<: *py2_linux_test_config - name: "Py2 - Unit tests for pants and pants-plugins" + name: "Unit tests for pants and pants-plugins (Py2 PEX)" stage: *test env: - *py2_linux_test_config_env @@ -551,7 +550,7 @@ matrix: - ./build-support/bin/travis-ci.sh -2blp - <<: *py3_linux_test_config - name: "Py3 - Unit tests for pants and pants-plugins" + name: "Unit tests for pants and pants-plugins (Py3 PEX)" env: - *py3_linux_test_config_env - CACHE_NAME=linuxunittests.py3 @@ -559,7 +558,7 @@ matrix: - ./build-support/bin/travis-ci.sh -blp - <<: *py2_linux_test_config - name: "Py2 - Python contrib tests" + name: "Python contrib tests (Py2 PEX)" env: - *py2_linux_test_config_env - CACHE_NAME=linuxcontribtests.py2 @@ -567,7 +566,7 @@ matrix: - ./build-support/bin/travis-ci.sh -2bn - <<: *py3_linux_test_config - name: "Py3 - Python contrib tests" + name: "Python contrib tests (Py3 PEX)" env: - *py3_linux_test_config_env - CACHE_NAME=linuxcontribtests.py3 @@ -576,7 +575,7 @@ matrix: {{#integration_shards}} - <<: *py3_linux_test_config - name: "Integration tests for pants - shard {{.}}" + name: "Integration tests for pants - shard {{.}} (Py3 PEX)" env: - *py3_linux_test_config_env - CACHE_NAME=integrationshard{{.}} @@ -586,7 +585,7 @@ matrix: {{/integration_shards}} {{#integration_shards}} - <<: *py2_linux_test_config - name: "Integration tests for pants (Python 2) - shard {{.}}" + name: "Integration tests for pants - shard {{.}} (Py2 PEX)" stage: *cron env: - *py2_linux_test_config_env From 6a355c7e70fe9a0610af26494789e85936befec3 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Fri, 11 Jan 2019 13:30:50 -0700 Subject: [PATCH 023/105] Add tip to use yamlvalidator.com & change py2_osx_test_config to default to cron --- .travis.yml | 6 +++++- build-support/travis/generate_travis_yml.py | 3 +++ build-support/travis/travis.yml.mustache | 3 ++- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index b3e8bbc0cbc..6fb33d4b6c0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,9 @@ # GENERATED, DO NOT EDIT! # To change, edit build-support/travis/travis.yml.mustache and run # ./pants --quiet run build-support/travis:generate_travis_yml > .travis.yml +# +# Tip: Copy the generated `.travis.yml` into https://yamlvalidator.com to validate the YAML +# and see how the entries resolve to normalized JSON (helpful to debug anchors). # Conditions are documented here: https://docs.travis-ci.com/user/conditions-v1 @@ -175,7 +178,6 @@ py3_osx_config: &py3_osx_config base_osx_test_config: &base_osx_test_config <<: *base_osx_config <<: *pants_run_cache_config - stage: *test before_script: - ulimit -c unlimited - ulimit -n 8192 @@ -184,12 +186,14 @@ base_osx_test_config: &base_osx_test_config py2_osx_test_config: &py2_osx_test_config <<: *py2_osx_config <<: *base_osx_test_config + stage: *cron env: - &py2_osx_test_config_env HOMEBREW_LOGS=~/homebrew-log HOMEBREW_TEMP=~/homebrew-temp BOOTSTRAPPED_PEX_KEY_SUFFIX="py2.osx" py3_osx_test_config: &py3_osx_test_config <<: *py3_osx_config <<: *base_osx_test_config + stage: *test env: - &py3_osx_test_config_env HOMEBREW_LOGS=~/homebrew-log HOMEBREW_TEMP=~/homebrew-temp BOOTSTRAPPED_PEX_KEY_SUFFIX="py3.osx" diff --git a/build-support/travis/generate_travis_yml.py b/build-support/travis/generate_travis_yml.py index ee9e822e5bc..2cc19cd9914 100644 --- a/build-support/travis/generate_travis_yml.py +++ b/build-support/travis/generate_travis_yml.py @@ -15,6 +15,9 @@ # GENERATED, DO NOT EDIT! # To change, edit build-support/travis/travis.yml.mustache and run # ./pants --quiet run build-support/travis:generate_travis_yml > .travis.yml +# +# Tip: Copy the generated `.travis.yml` into https://yamlvalidator.com to validate the YAML +# and see how the entries resolve to normalized JSON (helpful to debug anchors). """ diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index 626250c009d..3f60693dc6e 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -162,7 +162,6 @@ py3_osx_config: &py3_osx_config base_osx_test_config: &base_osx_test_config <<: *base_osx_config <<: *pants_run_cache_config - stage: *test before_script: - ulimit -c unlimited - ulimit -n 8192 @@ -171,12 +170,14 @@ base_osx_test_config: &base_osx_test_config py2_osx_test_config: &py2_osx_test_config <<: *py2_osx_config <<: *base_osx_test_config + stage: *cron env: - &py2_osx_test_config_env HOMEBREW_LOGS=~/homebrew-log HOMEBREW_TEMP=~/homebrew-temp BOOTSTRAPPED_PEX_KEY_SUFFIX="py2.osx" py3_osx_test_config: &py3_osx_test_config <<: *py3_osx_config <<: *base_osx_test_config + stage: *test env: - &py3_osx_test_config_env HOMEBREW_LOGS=~/homebrew-log HOMEBREW_TEMP=~/homebrew-temp BOOTSTRAPPED_PEX_KEY_SUFFIX="py3.osx" From 75cae7b6f32a863391805f20d91cf22112427bea Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Fri, 11 Jan 2019 13:39:11 -0700 Subject: [PATCH 024/105] use no quotes & single quotes --- .travis.yml | 24 ++++++++++++------------ build-support/travis/travis.yml.mustache | 24 ++++++++++++------------ 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6fb33d4b6c0..d4f78ace1ed 100644 --- a/.travis.yml +++ b/.travis.yml @@ -142,14 +142,14 @@ py2_linux_test_config: &py2_linux_test_config <<: *base_linux_test_config stage: *cron env: - - &py2_linux_test_config_env BOOTSTRAPPED_PEX_KEY_SUFFIX="py2.linux" + - &py2_linux_test_config_env BOOTSTRAPPED_PEX_KEY_SUFFIX=py2.linux py3_linux_test_config: &py3_linux_test_config <<: *py3_linux_config <<: *base_linux_test_config stage: *test env: - - &py3_linux_test_config_env BOOTSTRAPPED_PEX_KEY_SUFFIX="py3.linux" + - &py3_linux_test_config_env BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.linux base_osx_config: &base_osx_config os: osx @@ -188,14 +188,14 @@ py2_osx_test_config: &py2_osx_test_config <<: *base_osx_test_config stage: *cron env: - - &py2_osx_test_config_env HOMEBREW_LOGS=~/homebrew-log HOMEBREW_TEMP=~/homebrew-temp BOOTSTRAPPED_PEX_KEY_SUFFIX="py2.osx" + - &py2_osx_test_config_env HOMEBREW_LOGS=~/homebrew-log HOMEBREW_TEMP=~/homebrew-temp BOOTSTRAPPED_PEX_KEY_SUFFIX=py2.osx py3_osx_test_config: &py3_osx_test_config <<: *py3_osx_config <<: *base_osx_test_config stage: *test env: - - &py3_osx_test_config_env HOMEBREW_LOGS=~/homebrew-log HOMEBREW_TEMP=~/homebrew-temp BOOTSTRAPPED_PEX_KEY_SUFFIX="py3.osx" + - &py3_osx_test_config_env HOMEBREW_LOGS=~/homebrew-log HOMEBREW_TEMP=~/homebrew-temp BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.osx linux_with_fuse: &linux_with_fuse before_install: @@ -249,8 +249,8 @@ py2_linux_build_engine: &py2_linux_build_engine env: - *base_linux_build_engine_env - CACHE_NAME=linuxpexbuild.py2 - - BOOTSTRAPPED_PEX_KEY_SUFFIX="py2.linux" - - BOOTSTRAP_ARGS="-2" + - BOOTSTRAPPED_PEX_KEY_SUFFIX=py2.linux + - BOOTSTRAP_ARGS='-2' py3_linux_build_engine: &py3_linux_build_engine <<: *py3_linux_config @@ -259,8 +259,8 @@ py3_linux_build_engine: &py3_linux_build_engine env: - *base_linux_build_engine_env - CACHE_NAME=linuxpexbuild.py3 - - BOOTSTRAPPED_PEX_KEY_SUFFIX="py3.linux" - - BOOTSTRAP_ARGS="" + - BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.linux + - BOOTSTRAP_ARGS='' base_osx_build_engine: &base_osx_build_engine <<: *base_osx_config @@ -287,8 +287,8 @@ py2_osx_build_engine: &py2_osx_build_engine env: - *base_osx_build_engine_env - CACHE_NAME=osxpexbuild.py2 - - BOOTSTRAPPED_PEX_KEY_SUFFIX="py2.osx" - - BOOTSTRAP_ARGS="-2" + - BOOTSTRAPPED_PEX_KEY_SUFFIX=py2.osx + - BOOTSTRAP_ARGS='-2' py3_osx_build_engine: &py3_osx_build_engine <<: *py3_osx_config @@ -297,8 +297,8 @@ py3_osx_build_engine: &py3_osx_build_engine env: - *base_osx_build_engine_env - CACHE_NAME=osxpexbuild.py3 - - BOOTSTRAPPED_PEX_KEY_SUFFIX="py3.osx" - - BOOTSTRAP_ARGS="" + - BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.osx + - BOOTSTRAP_ARGS='' # ------------------------------------------------------------------------- # Build wheels diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index 3f60693dc6e..114caa1e631 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -130,14 +130,14 @@ py2_linux_test_config: &py2_linux_test_config <<: *base_linux_test_config stage: *cron env: - - &py2_linux_test_config_env BOOTSTRAPPED_PEX_KEY_SUFFIX="py2.linux" + - &py2_linux_test_config_env BOOTSTRAPPED_PEX_KEY_SUFFIX=py2.linux py3_linux_test_config: &py3_linux_test_config <<: *py3_linux_config <<: *base_linux_test_config stage: *test env: - - &py3_linux_test_config_env BOOTSTRAPPED_PEX_KEY_SUFFIX="py3.linux" + - &py3_linux_test_config_env BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.linux base_osx_config: &base_osx_config os: osx @@ -172,14 +172,14 @@ py2_osx_test_config: &py2_osx_test_config <<: *base_osx_test_config stage: *cron env: - - &py2_osx_test_config_env HOMEBREW_LOGS=~/homebrew-log HOMEBREW_TEMP=~/homebrew-temp BOOTSTRAPPED_PEX_KEY_SUFFIX="py2.osx" + - &py2_osx_test_config_env HOMEBREW_LOGS=~/homebrew-log HOMEBREW_TEMP=~/homebrew-temp BOOTSTRAPPED_PEX_KEY_SUFFIX=py2.osx py3_osx_test_config: &py3_osx_test_config <<: *py3_osx_config <<: *base_osx_test_config stage: *test env: - - &py3_osx_test_config_env HOMEBREW_LOGS=~/homebrew-log HOMEBREW_TEMP=~/homebrew-temp BOOTSTRAPPED_PEX_KEY_SUFFIX="py3.osx" + - &py3_osx_test_config_env HOMEBREW_LOGS=~/homebrew-log HOMEBREW_TEMP=~/homebrew-temp BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.osx linux_with_fuse: &linux_with_fuse before_install: @@ -228,8 +228,8 @@ py2_linux_build_engine: &py2_linux_build_engine env: - *base_linux_build_engine_env - CACHE_NAME=linuxpexbuild.py2 - - BOOTSTRAPPED_PEX_KEY_SUFFIX="py2.linux" - - BOOTSTRAP_ARGS="-2" + - BOOTSTRAPPED_PEX_KEY_SUFFIX=py2.linux + - BOOTSTRAP_ARGS='-2' py3_linux_build_engine: &py3_linux_build_engine <<: *py3_linux_config @@ -238,8 +238,8 @@ py3_linux_build_engine: &py3_linux_build_engine env: - *base_linux_build_engine_env - CACHE_NAME=linuxpexbuild.py3 - - BOOTSTRAPPED_PEX_KEY_SUFFIX="py3.linux" - - BOOTSTRAP_ARGS="" + - BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.linux + - BOOTSTRAP_ARGS='' base_osx_build_engine: &base_osx_build_engine <<: *base_osx_config @@ -266,8 +266,8 @@ py2_osx_build_engine: &py2_osx_build_engine env: - *base_osx_build_engine_env - CACHE_NAME=osxpexbuild.py2 - - BOOTSTRAPPED_PEX_KEY_SUFFIX="py2.osx" - - BOOTSTRAP_ARGS="-2" + - BOOTSTRAPPED_PEX_KEY_SUFFIX=py2.osx + - BOOTSTRAP_ARGS='-2' py3_osx_build_engine: &py3_osx_build_engine <<: *py3_osx_config @@ -276,8 +276,8 @@ py3_osx_build_engine: &py3_osx_build_engine env: - *base_osx_build_engine_env - CACHE_NAME=osxpexbuild.py3 - - BOOTSTRAPPED_PEX_KEY_SUFFIX="py3.osx" - - BOOTSTRAP_ARGS="" + - BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.osx + - BOOTSTRAP_ARGS='' # ------------------------------------------------------------------------- # Build wheels From 8ab35e30050d6a6b183ae491a7fbba6163115d92 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Fri, 11 Jan 2019 13:53:35 -0700 Subject: [PATCH 025/105] try debugging why engine build fails by adding -x --- build-support/bin/ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-support/bin/ci.sh b/build-support/bin/ci.sh index d0171460914..f24494d923a 100755 --- a/build-support/bin/ci.sh +++ b/build-support/bin/ci.sh @@ -2,7 +2,7 @@ # We use some subshell pipelines to collect target lists, make sure target collection failing # fails the build. -set -o pipefail +set -ox pipefail REPO_ROOT=$(cd $(dirname "${BASH_SOURCE[0]}") && cd "$(git rev-parse --show-toplevel)" && pwd) cd ${REPO_ROOT} From d3709eecd9eb25994d63817ca1f10ec78b0f3e70 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Fri, 11 Jan 2019 14:07:41 -0700 Subject: [PATCH 026/105] try debugging round 2 with -x in travis-ci --- build-support/bin/travis-ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-support/bin/travis-ci.sh b/build-support/bin/travis-ci.sh index 6401113d2d9..2d1b679b2a8 100755 --- a/build-support/bin/travis-ci.sh +++ b/build-support/bin/travis-ci.sh @@ -5,7 +5,7 @@ # TravisCI-specific environment fixups can live in this script which forwards to the generic ci # script post fixups. -set -euo pipefail +set -euox pipefail cd "$(git rev-parse --show-toplevel)" From b63ff0e47a4b50e5200d8e0f39b7ae968ec49cb1 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Fri, 11 Jan 2019 14:34:35 -0700 Subject: [PATCH 027/105] Use ci.sh, not travis-ci.sh, to bootstrap linux engine Why did I revert back to travis-ci.sh to begin with? It has a Pyenv issue that makes the shard fail within 2 minutes, regardless of Py2 vs Py3. --- .travis.yml | 2 +- build-support/travis/travis.yml.mustache | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index d4f78ace1ed..30a8019d202 100644 --- a/.travis.yml +++ b/.travis.yml @@ -239,7 +239,7 @@ base_linux_build_engine: &base_linux_build_engine -v "${HOME}:/travis/home" -v "${TRAVIS_BUILD_DIR}:/travis/workdir" travis_ci:latest - sh -c "./build-support/bin/travis-ci.sh ${BOOTSTRAP_ARGS} && ./build-support/bin/release.sh -f" + sh -c "./build-support/bin/ci.sh ${BOOTSTRAP_ARGS} && ./build-support/bin/release.sh -f" - aws --no-sign-request --region us-east-1 s3 cp ${TRAVIS_BUILD_DIR}/pants.pex ${BOOTSTRAPPED_PEX_URL_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX} py2_linux_build_engine: &py2_linux_build_engine diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index 114caa1e631..c7fdee30ea3 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -218,7 +218,7 @@ base_linux_build_engine: &base_linux_build_engine -v "${HOME}:/travis/home" -v "${TRAVIS_BUILD_DIR}:/travis/workdir" travis_ci:latest - sh -c "./build-support/bin/travis-ci.sh ${BOOTSTRAP_ARGS} && ./build-support/bin/release.sh -f" + sh -c "./build-support/bin/ci.sh ${BOOTSTRAP_ARGS} && ./build-support/bin/release.sh -f" - aws --no-sign-request --region us-east-1 s3 cp ${TRAVIS_BUILD_DIR}/pants.pex ${BOOTSTRAPPED_PEX_URL_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX} py2_linux_build_engine: &py2_linux_build_engine From ed26f27991665ed2c494432ef02cef93a4958494 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Fri, 11 Jan 2019 14:41:46 -0700 Subject: [PATCH 028/105] Reword ci.sh comment now that it's physically separated from related code --- build-support/bin/ci.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build-support/bin/ci.sh b/build-support/bin/ci.sh index f24494d923a..88ea039cd03 100755 --- a/build-support/bin/ci.sh +++ b/build-support/bin/ci.sh @@ -136,8 +136,8 @@ fi # integration tests that shell out to `./pants`, so we set this env var for those cases. export RUN_PANTS_FROM_PEX=1 -# TODO: Clear interpreters, otherwise this constraint does not end up applying due to a cache -# bug between the `./pants binary` and further runs. +# TODO: Clear interpreters, otherwise the above subprocess interpreter constraint does not end up +# applying due to a cache bug between the `./pants binary` and further runs. ./pants.pex clean-all if [[ "${run_pre_commit_checks:-false}" == "true" ]]; then From 088593258ec7e0dd323da59f838dc11bbfc666f1 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Fri, 11 Jan 2019 14:55:34 -0700 Subject: [PATCH 029/105] Base records only add their new info This is an attempt to get the OSX Py3 shard to work. Instead of records like base_osx_test_config extending base_osx_config, we leave it to the py2 and py3 versions to choose which base setup to extend. This is an attepmt to fix a suspected merge conflict for the Py3 OSX engine shard. I suspect py3_osx_config is being overriden by base_osx_config. We want py3_osx_config to win out. More philosophically, because the `base` records are not adequate to stand on their own, they should solely include new information, and not be extending other records, where possible. --- .travis.yml | 4 ---- build-support/travis/travis.yml.mustache | 4 ---- 2 files changed, 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 30a8019d202..b180f4a4716 100644 --- a/.travis.yml +++ b/.travis.yml @@ -176,7 +176,6 @@ py3_osx_config: &py3_osx_config - brew upgrade python base_osx_test_config: &base_osx_test_config - <<: *base_osx_config <<: *pants_run_cache_config before_script: - ulimit -c unlimited @@ -215,7 +214,6 @@ linux_with_fuse: &linux_with_fuse # ------------------------------------------------------------------------- base_linux_build_engine: &base_linux_build_engine - <<: *base_linux_config <<: *native_engine_cache_config stage: *bootstrap services: @@ -263,7 +261,6 @@ py3_linux_build_engine: &py3_linux_build_engine - BOOTSTRAP_ARGS='' base_osx_build_engine: &base_osx_build_engine - <<: *base_osx_config <<: *native_engine_cache_config stage: *bootstrap # We request the oldest image we can (corresponding to OSX 10.11) for maximum compatibility. @@ -505,7 +502,6 @@ py3_linux_rust_tests: &py3_linux_rust_tests - ./build-support/bin/travis-ci.sh -be base_osx_rust_tests: &base_osx_rust_tests - <<: *base_osx_test_config # Fuse actually works on this image. It hangs on many others. osx_image: xcode8.3 before_install: diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index c7fdee30ea3..f775cf09f85 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -160,7 +160,6 @@ py3_osx_config: &py3_osx_config - brew upgrade python base_osx_test_config: &base_osx_test_config - <<: *base_osx_config <<: *pants_run_cache_config before_script: - ulimit -c unlimited @@ -194,7 +193,6 @@ linux_with_fuse: &linux_with_fuse # ------------------------------------------------------------------------- base_linux_build_engine: &base_linux_build_engine - <<: *base_linux_config <<: *native_engine_cache_config stage: *bootstrap services: @@ -242,7 +240,6 @@ py3_linux_build_engine: &py3_linux_build_engine - BOOTSTRAP_ARGS='' base_osx_build_engine: &base_osx_build_engine - <<: *base_osx_config <<: *native_engine_cache_config stage: *bootstrap # We request the oldest image we can (corresponding to OSX 10.11) for maximum compatibility. @@ -484,7 +481,6 @@ py3_linux_rust_tests: &py3_linux_rust_tests - ./build-support/bin/travis-ci.sh -be base_osx_rust_tests: &base_osx_rust_tests - <<: *base_osx_test_config # Fuse actually works on this image. It hangs on many others. osx_image: xcode8.3 before_install: From 99873095e0dd28de244f6854b5f5659d175f14f1 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Fri, 11 Jan 2019 15:33:41 -0700 Subject: [PATCH 030/105] Try to set up Docker to build Py3 TIL that Centos6 does not by default have python3, let alone 3.6 or 3.7. Nor can you download it from Yum. So, we must build from source. This is to fix an issue with the Py3 build engine shard not being able to discover python3 on its path. Also remove PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS entry because we only want this constrained in `ci.sh`, not in Docker! --- build-support/docker/centos6/Dockerfile | 9 +++++++++ build-support/docker/travis_ci/Dockerfile | 3 --- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/build-support/docker/centos6/Dockerfile b/build-support/docker/centos6/Dockerfile index a733da334f9..fa6b960cf64 100644 --- a/build-support/docker/centos6/Dockerfile +++ b/build-support/docker/centos6/Dockerfile @@ -17,5 +17,14 @@ RUN yum install -y \ libffi-devel \ openssl-devel +# Install Python 3.6, which we must build because it's not available through yum +RUN cd /usr/src && \ + wget https://www.python.org/ftp/python/3.6.8/Python-3.6.8.tgz && \ + tar -xzf Python-3-6-8.tgz && \ + cd Python-3-6-8 && \ + ./configure --enable-optimizations && \ + make altinstall && \ + rm /usr/src/Python-3-6-8.tgz + # By default, execute in an environment with python27 enabled. ENTRYPOINT ["/usr/bin/scl", "enable", "python27", "devtoolset-7", "--"] diff --git a/build-support/docker/travis_ci/Dockerfile b/build-support/docker/travis_ci/Dockerfile index 28364413077..de807f4e52d 100644 --- a/build-support/docker/travis_ci/Dockerfile +++ b/build-support/docker/travis_ci/Dockerfile @@ -18,9 +18,6 @@ RUN groupadd --gid ${TRAVIS_GID} ${TRAVIS_GROUP} || true RUN useradd -d /travis/home -g ${TRAVIS_GROUP} --uid ${TRAVIS_UID} ${TRAVIS_USER} USER ${TRAVIS_USER}:${TRAVIS_GROUP} -# Ensure Pants runs under the 2.7.13 interpreter. -ENV PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="['CPython==2.7.13']" - # Our newly created user is unlikely to have a sane environment: set a locale at least. ENV LC_ALL="en_US.UTF-8" From ea545d63e1a077ef39cab0db1382d02ae07933c2 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Fri, 11 Jan 2019 16:20:31 -0700 Subject: [PATCH 031/105] Build Py3 in travis_ci/Dockerfile We'll want to actually build this in centos6, but that needs to be published to Docker. In the meantime, we can build it in travis_ci/Dockerfile to test it all works and not be blocked by anything. --- build-support/docker/travis_ci/Dockerfile | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/build-support/docker/travis_ci/Dockerfile b/build-support/docker/travis_ci/Dockerfile index de807f4e52d..691ee4846a3 100644 --- a/build-support/docker/travis_ci/Dockerfile +++ b/build-support/docker/travis_ci/Dockerfile @@ -21,4 +21,13 @@ USER ${TRAVIS_USER}:${TRAVIS_GROUP} # Our newly created user is unlikely to have a sane environment: set a locale at least. ENV LC_ALL="en_US.UTF-8" +# Install Python 3.6, which we must build because it's not available through yum +RUN cd /usr/src && \ + wget https://www.python.org/ftp/python/3.6.8/Python-3.6.8.tgz && \ + tar -xzf Python-3-6-8.tgz && \ + cd Python-3-6-8 && \ + ./configure --enable-optimizations && \ + make altinstall && \ + rm /usr/src/Python-3-6-8.tgz + WORKDIR /travis/workdir From 5b8d251b30fe083a615c44cd9bce432588cc7376 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Fri, 11 Jan 2019 16:31:25 -0700 Subject: [PATCH 032/105] Try to get OSX to work with Py3 using brew addon --- .travis.yml | 26 +++++++++++++----------- build-support/travis/travis.yml.mustache | 24 +++++++++++++--------- 2 files changed, 28 insertions(+), 22 deletions(-) diff --git a/.travis.yml b/.travis.yml index b180f4a4716..fcb3c770112 100644 --- a/.travis.yml +++ b/.travis.yml @@ -154,10 +154,6 @@ py3_linux_test_config: &py3_linux_test_config base_osx_config: &base_osx_config os: osx language: generic - # TODO: workaround for Homebrew bug triggered by running `brew update`. - # Follow at https://github.com/Homebrew/brew/issues/5513 and remove this workaround once its fixed. - env: - - &base_osx_config_env HOMEBREW_LOGS=~/homebrew-log HOMEBREW_TEMP=~/homebrew-temp before_install: - curl -L https://github.com/stedolan/jq/releases/download/jq-1.5/jq-osx-amd64 -o /usr/local/bin/jq - chmod 755 /usr/local/bin/jq @@ -168,12 +164,10 @@ py2_osx_config: &py2_osx_config py3_osx_config: &py3_osx_config <<: *base_osx_config - before_install: - - curl -L https://github.com/stedolan/jq/releases/download/jq-1.5/jq-osx-amd64 -o /usr/local/bin/jq - - chmod 755 /usr/local/bin/jq - - ./build-support/bin/install_aws_cli_for_ci.sh - # Upgrade to Python 3 - - brew upgrade python + addons: + homebrew: + packages: &py3_osx_config_brew_packages + - python3 base_osx_test_config: &base_osx_test_config <<: *pants_run_cache_config @@ -187,14 +181,14 @@ py2_osx_test_config: &py2_osx_test_config <<: *base_osx_test_config stage: *cron env: - - &py2_osx_test_config_env HOMEBREW_LOGS=~/homebrew-log HOMEBREW_TEMP=~/homebrew-temp BOOTSTRAPPED_PEX_KEY_SUFFIX=py2.osx + - &py2_osx_test_config_env BOOTSTRAPPED_PEX_KEY_SUFFIX=py2.osx py3_osx_test_config: &py3_osx_test_config <<: *py3_osx_config <<: *base_osx_test_config stage: *test env: - - &py3_osx_test_config_env HOMEBREW_LOGS=~/homebrew-log HOMEBREW_TEMP=~/homebrew-temp BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.osx + - &py3_osx_test_config_env BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.osx linux_with_fuse: &linux_with_fuse before_install: @@ -509,6 +503,10 @@ base_osx_rust_tests: &base_osx_rust_tests - chmod 755 /usr/local/bin/jq - ./build-support/bin/install_aws_cli_for_ci.sh - brew tap caskroom/cask && brew update && brew cask install osxfuse + addons: + homebrew: + casks: &base_osx_rust_tests_brew_casks + - osxfuse py2_osx_rust_tests: &py2_osx_rust_tests <<: *base_osx_rust_tests @@ -527,6 +525,10 @@ py3_osx_rust_tests: &py3_osx_rust_tests env: - *py3_osx_test_config_env - CACHE_NAME=macosrusttests.py3 + addons: + homebrew: + packages: *py3_osx_config_brew_packages + casks: *base_osx_rust_tests_brew_casks script: - ./build-support/bin/travis-ci.sh -bez diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index f775cf09f85..bdf3fa0cc43 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -142,10 +142,6 @@ py3_linux_test_config: &py3_linux_test_config base_osx_config: &base_osx_config os: osx language: generic - # TODO: workaround for Homebrew bug triggered by running `brew update`. - # Follow at https://github.com/Homebrew/brew/issues/5513 and remove this workaround once its fixed. - env: - - &base_osx_config_env HOMEBREW_LOGS=~/homebrew-log HOMEBREW_TEMP=~/homebrew-temp before_install: {{>before_install_osx}} @@ -154,10 +150,10 @@ py2_osx_config: &py2_osx_config py3_osx_config: &py3_osx_config <<: *base_osx_config - before_install: - {{>before_install_osx}} - # Upgrade to Python 3 - - brew upgrade python + addons: + homebrew: + packages: &py3_osx_config_brew_packages + - python3 base_osx_test_config: &base_osx_test_config <<: *pants_run_cache_config @@ -171,14 +167,14 @@ py2_osx_test_config: &py2_osx_test_config <<: *base_osx_test_config stage: *cron env: - - &py2_osx_test_config_env HOMEBREW_LOGS=~/homebrew-log HOMEBREW_TEMP=~/homebrew-temp BOOTSTRAPPED_PEX_KEY_SUFFIX=py2.osx + - &py2_osx_test_config_env BOOTSTRAPPED_PEX_KEY_SUFFIX=py2.osx py3_osx_test_config: &py3_osx_test_config <<: *py3_osx_config <<: *base_osx_test_config stage: *test env: - - &py3_osx_test_config_env HOMEBREW_LOGS=~/homebrew-log HOMEBREW_TEMP=~/homebrew-temp BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.osx + - &py3_osx_test_config_env BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.osx linux_with_fuse: &linux_with_fuse before_install: @@ -486,6 +482,10 @@ base_osx_rust_tests: &base_osx_rust_tests before_install: {{>before_install_osx}} - brew tap caskroom/cask && brew update && brew cask install osxfuse + addons: + homebrew: + casks: &base_osx_rust_tests_brew_casks + - osxfuse py2_osx_rust_tests: &py2_osx_rust_tests <<: *base_osx_rust_tests @@ -504,6 +504,10 @@ py3_osx_rust_tests: &py3_osx_rust_tests env: - *py3_osx_test_config_env - CACHE_NAME=macosrusttests.py3 + addons: + homebrew: + packages: *py3_osx_config_brew_packages + casks: *base_osx_rust_tests_brew_casks script: - ./build-support/bin/travis-ci.sh -bez From dbcf750fb09720155619c4416371f6e0a7c756b0 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Fri, 11 Jan 2019 16:46:08 -0700 Subject: [PATCH 033/105] Temporarily install wget in travis Docker --- build-support/docker/centos6/Dockerfile | 3 ++- build-support/docker/travis_ci/Dockerfile | 11 ++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/build-support/docker/centos6/Dockerfile b/build-support/docker/centos6/Dockerfile index fa6b960cf64..bf53d6daee6 100644 --- a/build-support/docker/centos6/Dockerfile +++ b/build-support/docker/centos6/Dockerfile @@ -15,7 +15,8 @@ RUN yum install -y \ python27 \ libffi \ libffi-devel \ - openssl-devel + openssl-devel \ + wget # Install Python 3.6, which we must build because it's not available through yum RUN cd /usr/src && \ diff --git a/build-support/docker/travis_ci/Dockerfile b/build-support/docker/travis_ci/Dockerfile index 691ee4846a3..05dea63f131 100644 --- a/build-support/docker/travis_ci/Dockerfile +++ b/build-support/docker/travis_ci/Dockerfile @@ -21,7 +21,14 @@ USER ${TRAVIS_USER}:${TRAVIS_GROUP} # Our newly created user is unlikely to have a sane environment: set a locale at least. ENV LC_ALL="en_US.UTF-8" -# Install Python 3.6, which we must build because it's not available through yum +# ----------------------------------------------------------------------- +# This section belongs in centos6, but is here temporarily because +# centos6's build must first be published to Docker. + +RUN yum -y update +RUN yum install -y centos-release-scl +RUN yum install -y wget + RUN cd /usr/src && \ wget https://www.python.org/ftp/python/3.6.8/Python-3.6.8.tgz && \ tar -xzf Python-3-6-8.tgz && \ @@ -30,4 +37,6 @@ RUN cd /usr/src && \ make altinstall && \ rm /usr/src/Python-3-6-8.tgz +# ----------------------------------------------------------------------- + WORKDIR /travis/workdir From 6e49aa9d7e4ae63486e766533d0d2fd8b9379ec6 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Fri, 11 Jan 2019 16:53:47 -0700 Subject: [PATCH 034/105] Remove unnecessary before_install thanks to using brew addon --- .travis.yml | 5 ----- build-support/travis/travis.yml.mustache | 3 --- 2 files changed, 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index fcb3c770112..7411e907508 100644 --- a/.travis.yml +++ b/.travis.yml @@ -498,11 +498,6 @@ py3_linux_rust_tests: &py3_linux_rust_tests base_osx_rust_tests: &base_osx_rust_tests # Fuse actually works on this image. It hangs on many others. osx_image: xcode8.3 - before_install: - - curl -L https://github.com/stedolan/jq/releases/download/jq-1.5/jq-osx-amd64 -o /usr/local/bin/jq - - chmod 755 /usr/local/bin/jq - - ./build-support/bin/install_aws_cli_for_ci.sh - - brew tap caskroom/cask && brew update && brew cask install osxfuse addons: homebrew: casks: &base_osx_rust_tests_brew_casks diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index bdf3fa0cc43..c46fdc9c07b 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -479,9 +479,6 @@ py3_linux_rust_tests: &py3_linux_rust_tests base_osx_rust_tests: &base_osx_rust_tests # Fuse actually works on this image. It hangs on many others. osx_image: xcode8.3 - before_install: - {{>before_install_osx}} - - brew tap caskroom/cask && brew update && brew cask install osxfuse addons: homebrew: casks: &base_osx_rust_tests_brew_casks From c38939b796aa7ae052f73a1b49c8b49f3f52630a Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Fri, 11 Jan 2019 16:56:53 -0700 Subject: [PATCH 035/105] See if brew update=true fixes install error The py3 osx engine shard fails to install anything with brew, complaining brew is too outdated. Lets see if updating brew makes it all work.. --- .travis.yml | 1 + build-support/travis/travis.yml.mustache | 1 + 2 files changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 7411e907508..b986329a726 100644 --- a/.travis.yml +++ b/.travis.yml @@ -168,6 +168,7 @@ py3_osx_config: &py3_osx_config homebrew: packages: &py3_osx_config_brew_packages - python3 + update: true base_osx_test_config: &base_osx_test_config <<: *pants_run_cache_config diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index c46fdc9c07b..508d3cbf7ae 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -154,6 +154,7 @@ py3_osx_config: &py3_osx_config homebrew: packages: &py3_osx_config_brew_packages - python3 + update: true base_osx_test_config: &base_osx_test_config <<: *pants_run_cache_config From 70eade97b747d2122f256a41d8d5bb3af53de3dd Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Fri, 11 Jan 2019 17:03:40 -0700 Subject: [PATCH 036/105] Try to fix root user docker issue --- build-support/docker/travis_ci/Dockerfile | 34 +++++++++++------------ 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/build-support/docker/travis_ci/Dockerfile b/build-support/docker/travis_ci/Dockerfile index 05dea63f131..a77bffbf6a0 100644 --- a/build-support/docker/travis_ci/Dockerfile +++ b/build-support/docker/travis_ci/Dockerfile @@ -4,23 +4,6 @@ # Use our custom Centos6 image for binary compatibility with old linux distros. FROM pantsbuild/centos6:latest -# Setup mount points for the travis ci user & workdir. -VOLUME /travis/home -VOLUME /travis/workdir - -# Setup a non-root user to execute the build under (avoids problems with npm install). -ARG TRAVIS_USER=travis_ci -ARG TRAVIS_UID=1000 -ARG TRAVIS_GROUP=root -ARG TRAVIS_GID=0 - -RUN groupadd --gid ${TRAVIS_GID} ${TRAVIS_GROUP} || true -RUN useradd -d /travis/home -g ${TRAVIS_GROUP} --uid ${TRAVIS_UID} ${TRAVIS_USER} -USER ${TRAVIS_USER}:${TRAVIS_GROUP} - -# Our newly created user is unlikely to have a sane environment: set a locale at least. -ENV LC_ALL="en_US.UTF-8" - # ----------------------------------------------------------------------- # This section belongs in centos6, but is here temporarily because # centos6's build must first be published to Docker. @@ -39,4 +22,21 @@ RUN cd /usr/src && \ # ----------------------------------------------------------------------- +# Setup mount points for the travis ci user & workdir. +VOLUME /travis/home +VOLUME /travis/workdir + +# Setup a non-root user to execute the build under (avoids problems with npm install). +ARG TRAVIS_USER=travis_ci +ARG TRAVIS_UID=1000 +ARG TRAVIS_GROUP=root +ARG TRAVIS_GID=0 + +RUN groupadd --gid ${TRAVIS_GID} ${TRAVIS_GROUP} || true +RUN useradd -d /travis/home -g ${TRAVIS_GROUP} --uid ${TRAVIS_UID} ${TRAVIS_USER} +USER ${TRAVIS_USER}:${TRAVIS_GROUP} + +# Our newly created user is unlikely to have a sane environment: set a locale at least. +ENV LC_ALL="en_US.UTF-8" + WORKDIR /travis/workdir From d957c37b928e4f04e3d7749215577f32662dacb7 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Fri, 11 Jan 2019 17:37:03 -0700 Subject: [PATCH 037/105] Only update brew for building engine shard I think we only have to update brew for this shard because we're using such an old version of OSX for it. In normal test shards, we use a more recent OSX so I think we can avoid this very very costly line (takes 16 minutes to setup Py3 on OSX py3 engine shard!!) --- build-support/travis/travis.yml.mustache | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index 508d3cbf7ae..753d65706e5 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -154,7 +154,6 @@ py3_osx_config: &py3_osx_config homebrew: packages: &py3_osx_config_brew_packages - python3 - update: true base_osx_test_config: &base_osx_test_config <<: *pants_run_cache_config @@ -267,6 +266,11 @@ py3_osx_build_engine: &py3_osx_build_engine <<: *py3_osx_config <<: *base_osx_build_engine name: "Build OSX native engine and pants.pex (Py3 PEX)" + addons: + homebrew: + packages: *py3_osx_config_brew_packages + update: true + env: - *base_osx_build_engine_env - CACHE_NAME=osxpexbuild.py3 From 2253a602c274dd372cf9e0ac0bc10f23b541c3f4 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Fri, 11 Jan 2019 17:40:14 -0700 Subject: [PATCH 038/105] Oops, forgot to regenerate travis.yml --- .travis.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b986329a726..da6e3a8ea26 100644 --- a/.travis.yml +++ b/.travis.yml @@ -168,7 +168,6 @@ py3_osx_config: &py3_osx_config homebrew: packages: &py3_osx_config_brew_packages - python3 - update: true base_osx_test_config: &base_osx_test_config <<: *pants_run_cache_config @@ -286,6 +285,11 @@ py3_osx_build_engine: &py3_osx_build_engine <<: *py3_osx_config <<: *base_osx_build_engine name: "Build OSX native engine and pants.pex (Py3 PEX)" + addons: + homebrew: + packages: *py3_osx_config_brew_packages + update: true + env: - *base_osx_build_engine_env - CACHE_NAME=osxpexbuild.py3 From a1c2e447f6c53136b67fcc05f233d87777ad9bb1 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Fri, 11 Jan 2019 17:47:45 -0700 Subject: [PATCH 039/105] Fix typo in Dockerfiles --- build-support/docker/centos6/Dockerfile | 6 +++--- build-support/docker/travis_ci/Dockerfile | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/build-support/docker/centos6/Dockerfile b/build-support/docker/centos6/Dockerfile index bf53d6daee6..c3d82dbf138 100644 --- a/build-support/docker/centos6/Dockerfile +++ b/build-support/docker/centos6/Dockerfile @@ -21,11 +21,11 @@ RUN yum install -y \ # Install Python 3.6, which we must build because it's not available through yum RUN cd /usr/src && \ wget https://www.python.org/ftp/python/3.6.8/Python-3.6.8.tgz && \ - tar -xzf Python-3-6-8.tgz && \ - cd Python-3-6-8 && \ + tar -xzf Python-3.6.8.tgz && \ + cd Python-3.6.8 && \ ./configure --enable-optimizations && \ make altinstall && \ - rm /usr/src/Python-3-6-8.tgz + rm /usr/src/Python-3.6.8.tgz # By default, execute in an environment with python27 enabled. ENTRYPOINT ["/usr/bin/scl", "enable", "python27", "devtoolset-7", "--"] diff --git a/build-support/docker/travis_ci/Dockerfile b/build-support/docker/travis_ci/Dockerfile index a77bffbf6a0..e25327e5f30 100644 --- a/build-support/docker/travis_ci/Dockerfile +++ b/build-support/docker/travis_ci/Dockerfile @@ -14,11 +14,11 @@ RUN yum install -y wget RUN cd /usr/src && \ wget https://www.python.org/ftp/python/3.6.8/Python-3.6.8.tgz && \ - tar -xzf Python-3-6-8.tgz && \ - cd Python-3-6-8 && \ + tar -xzf Python-3.6.8.tgz && \ + cd Python-3.6.8 && \ ./configure --enable-optimizations && \ make altinstall && \ - rm /usr/src/Python-3-6-8.tgz + rm /usr/src/Python-3.6.8.tgz # ----------------------------------------------------------------------- From 74c7cb0acb6c7541bab6dde380b6b6f893ab0374 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Fri, 11 Jan 2019 18:09:36 -0700 Subject: [PATCH 040/105] Try out pyenv to install 3.6 on osx --- .travis.yml | 9 +++++---- build-support/travis/travis.yml.mustache | 7 +++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index da6e3a8ea26..2185979bcec 100644 --- a/.travis.yml +++ b/.travis.yml @@ -164,10 +164,11 @@ py2_osx_config: &py2_osx_config py3_osx_config: &py3_osx_config <<: *base_osx_config - addons: - homebrew: - packages: &py3_osx_config_brew_packages - - python3 + before_install: + - curl -L https://github.com/stedolan/jq/releases/download/jq-1.5/jq-osx-amd64 -o /usr/local/bin/jq + - chmod 755 /usr/local/bin/jq + - ./build-support/bin/install_aws_cli_for_ci.sh + - pyenv install 3.6.8 && pyenv global 3.6.8 base_osx_test_config: &base_osx_test_config <<: *pants_run_cache_config diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index 753d65706e5..2bb774f4d8b 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -150,10 +150,9 @@ py2_osx_config: &py2_osx_config py3_osx_config: &py3_osx_config <<: *base_osx_config - addons: - homebrew: - packages: &py3_osx_config_brew_packages - - python3 + before_install: + {{>before_install_osx}} + - pyenv install 3.6.8 && pyenv global 3.6.8 base_osx_test_config: &base_osx_test_config <<: *pants_run_cache_config From bd616eca4d08f23d7e7035417a7c86fe5425fc42 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Fri, 11 Jan 2019 18:12:52 -0700 Subject: [PATCH 041/105] Oops, left bad references to brew python3 --- .travis.yml | 5 ----- build-support/travis/travis.yml.mustache | 5 ----- 2 files changed, 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2185979bcec..662b2c2df81 100644 --- a/.travis.yml +++ b/.travis.yml @@ -286,10 +286,6 @@ py3_osx_build_engine: &py3_osx_build_engine <<: *py3_osx_config <<: *base_osx_build_engine name: "Build OSX native engine and pants.pex (Py3 PEX)" - addons: - homebrew: - packages: *py3_osx_config_brew_packages - update: true env: - *base_osx_build_engine_env @@ -528,7 +524,6 @@ py3_osx_rust_tests: &py3_osx_rust_tests - CACHE_NAME=macosrusttests.py3 addons: homebrew: - packages: *py3_osx_config_brew_packages casks: *base_osx_rust_tests_brew_casks script: - ./build-support/bin/travis-ci.sh -bez diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index 2bb774f4d8b..f0945f2cbcc 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -265,10 +265,6 @@ py3_osx_build_engine: &py3_osx_build_engine <<: *py3_osx_config <<: *base_osx_build_engine name: "Build OSX native engine and pants.pex (Py3 PEX)" - addons: - homebrew: - packages: *py3_osx_config_brew_packages - update: true env: - *base_osx_build_engine_env @@ -507,7 +503,6 @@ py3_osx_rust_tests: &py3_osx_rust_tests - CACHE_NAME=macosrusttests.py3 addons: homebrew: - packages: *py3_osx_config_brew_packages casks: *base_osx_rust_tests_brew_casks script: - ./build-support/bin/travis-ci.sh -bez From 71aa8f42fa42c3705e997fdfeb7261465972e0f1 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Fri, 11 Jan 2019 18:16:24 -0700 Subject: [PATCH 042/105] Oops again.. didn't regen travis --- .travis.yml | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index dc4dd65040c..662b2c2df81 100644 --- a/.travis.yml +++ b/.travis.yml @@ -935,24 +935,8 @@ matrix: - <<: *py2_linux_rust_tests - <<: *py3_linux_rust_tests -<<<<<<< HEAD - <<: *py2_osx_rust_tests - <<: *py3_osx_rust_tests -======= - # Rust on macOS - - <<: *default_osx_test_config - name: "OSX Rust + platform-specific tests" - env: - - CACHE_NAME=macosrusttests - # Fuse actually works on this image. It hangs on many others. - osx_image: xcode8.3 - addons: - homebrew: - casks: - - osxfuse - script: - - ./build-support/bin/travis-ci.sh -bez ->>>>>>> 60f3c93cad6cf05f27403d0473f6d4741e5db2cb - <<: *linux_with_fuse <<: *native_engine_cache_config From e3a98041e9bba3d5279158008307152972924c00 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Fri, 11 Jan 2019 18:28:58 -0700 Subject: [PATCH 043/105] brew install pyenv --- .travis.yml | 5 +++++ build-support/travis/travis.yml.mustache | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/.travis.yml b/.travis.yml index 662b2c2df81..5943eb82e2d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -164,6 +164,10 @@ py2_osx_config: &py2_osx_config py3_osx_config: &py3_osx_config <<: *base_osx_config + addons: + brew: + packages: &py3_osx_config_brew_packages + - pyenv before_install: - curl -L https://github.com/stedolan/jq/releases/download/jq-1.5/jq-osx-amd64 -o /usr/local/bin/jq - chmod 755 /usr/local/bin/jq @@ -524,6 +528,7 @@ py3_osx_rust_tests: &py3_osx_rust_tests - CACHE_NAME=macosrusttests.py3 addons: homebrew: + packages: *py3_osx_config_brew_packages casks: *base_osx_rust_tests_brew_casks script: - ./build-support/bin/travis-ci.sh -bez diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index f0945f2cbcc..1db3c4ba8bc 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -150,6 +150,10 @@ py2_osx_config: &py2_osx_config py3_osx_config: &py3_osx_config <<: *base_osx_config + addons: + brew: + packages: &py3_osx_config_brew_packages + - pyenv before_install: {{>before_install_osx}} - pyenv install 3.6.8 && pyenv global 3.6.8 @@ -503,6 +507,7 @@ py3_osx_rust_tests: &py3_osx_rust_tests - CACHE_NAME=macosrusttests.py3 addons: homebrew: + packages: *py3_osx_config_brew_packages casks: *base_osx_rust_tests_brew_casks script: - ./build-support/bin/travis-ci.sh -bez From 61e6c14ae69f10b3d934fe67ee078bc3d22c4d1e Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Fri, 11 Jan 2019 18:32:40 -0700 Subject: [PATCH 044/105] Try out openssl fix that works for me locally --- .travis.yml | 7 ++++++- build-support/travis/travis.yml.mustache | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5943eb82e2d..dd373cef332 100644 --- a/.travis.yml +++ b/.travis.yml @@ -168,6 +168,10 @@ py3_osx_config: &py3_osx_config brew: packages: &py3_osx_config_brew_packages - pyenv + - openssl + env: + # Fix Python 3 issue linking to OpenSSL + - PATH="/usr/local/opt/openssl/bin:$PATH" LDFLAGS="-L/usr/local/opt/openssl/lib" CPPFLAGS="-I/usr/local/opt/openssl/include" before_install: - curl -L https://github.com/stedolan/jq/releases/download/jq-1.5/jq-osx-amd64 -o /usr/local/bin/jq - chmod 755 /usr/local/bin/jq @@ -193,7 +197,8 @@ py3_osx_test_config: &py3_osx_test_config <<: *base_osx_test_config stage: *test env: - - &py3_osx_test_config_env BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.osx + # Must duplicate OpenSSL config from py3_osx_config's env because it cannot be merged into a new anchor + - &py3_osx_test_config_env PATH="/usr/local/opt/openssl/bin:$PATH" LDFLAGS="-L/usr/local/opt/openssl/lib" CPPFLAGS="-I/usr/local/opt/openssl/include" BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.osx linux_with_fuse: &linux_with_fuse before_install: diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index 1db3c4ba8bc..1d2a9b96591 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -154,6 +154,10 @@ py3_osx_config: &py3_osx_config brew: packages: &py3_osx_config_brew_packages - pyenv + - openssl + env: + # Fix Python 3 issue linking to OpenSSL + - PATH="/usr/local/opt/openssl/bin:$PATH" LDFLAGS="-L/usr/local/opt/openssl/lib" CPPFLAGS="-I/usr/local/opt/openssl/include" before_install: {{>before_install_osx}} - pyenv install 3.6.8 && pyenv global 3.6.8 @@ -177,7 +181,8 @@ py3_osx_test_config: &py3_osx_test_config <<: *base_osx_test_config stage: *test env: - - &py3_osx_test_config_env BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.osx + # Must duplicate OpenSSL config from py3_osx_config's env because it cannot be merged into a new anchor + - &py3_osx_test_config_env PATH="/usr/local/opt/openssl/bin:$PATH" LDFLAGS="-L/usr/local/opt/openssl/lib" CPPFLAGS="-I/usr/local/opt/openssl/include" BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.osx linux_with_fuse: &linux_with_fuse before_install: From cc2da27ce257607eb33cef77fc8e086acc54530b Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Fri, 11 Jan 2019 18:42:02 -0700 Subject: [PATCH 045/105] Try to explicitly turn on modern gcc before py3 build Thanks Danny for the code snippet! --- build-support/docker/travis_ci/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/build-support/docker/travis_ci/Dockerfile b/build-support/docker/travis_ci/Dockerfile index e25327e5f30..2ce44ebb04e 100644 --- a/build-support/docker/travis_ci/Dockerfile +++ b/build-support/docker/travis_ci/Dockerfile @@ -12,6 +12,7 @@ RUN yum -y update RUN yum install -y centos-release-scl RUN yum install -y wget +RUN /usr/bin/scl enable devtoolset-7 RUN cd /usr/src && \ wget https://www.python.org/ftp/python/3.6.8/Python-3.6.8.tgz && \ tar -xzf Python-3.6.8.tgz && \ From d9a303c8ff6d4f389005414160b370b38e689ccd Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Fri, 11 Jan 2019 18:54:30 -0700 Subject: [PATCH 046/105] Try installing pyenv normal brew way --- .travis.yml | 2 ++ build-support/travis/travis.yml.mustache | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index dd373cef332..b74fe027e25 100644 --- a/.travis.yml +++ b/.travis.yml @@ -176,6 +176,8 @@ py3_osx_config: &py3_osx_config - curl -L https://github.com/stedolan/jq/releases/download/jq-1.5/jq-osx-amd64 -o /usr/local/bin/jq - chmod 755 /usr/local/bin/jq - ./build-support/bin/install_aws_cli_for_ci.sh + # Install pyenv the conventional way, instead of through addon, due to execution order + - brew install pyenv - pyenv install 3.6.8 && pyenv global 3.6.8 base_osx_test_config: &base_osx_test_config diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index 1d2a9b96591..c178ebe284c 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -160,6 +160,8 @@ py3_osx_config: &py3_osx_config - PATH="/usr/local/opt/openssl/bin:$PATH" LDFLAGS="-L/usr/local/opt/openssl/lib" CPPFLAGS="-I/usr/local/opt/openssl/include" before_install: {{>before_install_osx}} + # Install pyenv the conventional way, instead of through addon, due to execution order + - brew install pyenv - pyenv install 3.6.8 && pyenv global 3.6.8 base_osx_test_config: &base_osx_test_config From 5ae50c9fb0c26f03db080a4c40e7b1948d51d98f Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Fri, 11 Jan 2019 18:55:44 -0700 Subject: [PATCH 047/105] Fix using gcc & running build python command --- build-support/docker/travis_ci/Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/build-support/docker/travis_ci/Dockerfile b/build-support/docker/travis_ci/Dockerfile index 2ce44ebb04e..c89635586d3 100644 --- a/build-support/docker/travis_ci/Dockerfile +++ b/build-support/docker/travis_ci/Dockerfile @@ -12,14 +12,13 @@ RUN yum -y update RUN yum install -y centos-release-scl RUN yum install -y wget -RUN /usr/bin/scl enable devtoolset-7 -RUN cd /usr/src && \ +RUN /usr/bin/scl enable devtoolset-7 -- bash -c 'cd /usr/src && \ wget https://www.python.org/ftp/python/3.6.8/Python-3.6.8.tgz && \ tar -xzf Python-3.6.8.tgz && \ cd Python-3.6.8 && \ ./configure --enable-optimizations && \ make altinstall && \ - rm /usr/src/Python-3.6.8.tgz + rm /usr/src/Python-3.6.8.tgz' # ----------------------------------------------------------------------- From 136906f3c4ea0fa0b28143b45e22db170e3476d5 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Fri, 11 Jan 2019 19:04:17 -0700 Subject: [PATCH 048/105] Don't let brew auto-update --- build-support/travis/travis.yml.mustache | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index c178ebe284c..6b3e602383f 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -161,7 +161,7 @@ py3_osx_config: &py3_osx_config before_install: {{>before_install_osx}} # Install pyenv the conventional way, instead of through addon, due to execution order - - brew install pyenv + - HOMEBREW_NO_AUTO_UPDATE=1 brew install pyenv - pyenv install 3.6.8 && pyenv global 3.6.8 base_osx_test_config: &base_osx_test_config From 517452963ec615a0ecc787389f48088d14d77bc0 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Fri, 11 Jan 2019 23:42:38 -0700 Subject: [PATCH 049/105] Use Danny's docker setup + add sqlite-devel to it --- build-support/docker/travis_ci/Dockerfile | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/build-support/docker/travis_ci/Dockerfile b/build-support/docker/travis_ci/Dockerfile index c89635586d3..8c5d08df424 100644 --- a/build-support/docker/travis_ci/Dockerfile +++ b/build-support/docker/travis_ci/Dockerfile @@ -8,17 +8,16 @@ FROM pantsbuild/centos6:latest # This section belongs in centos6, but is here temporarily because # centos6's build must first be published to Docker. -RUN yum -y update -RUN yum install -y centos-release-scl -RUN yum install -y wget - -RUN /usr/bin/scl enable devtoolset-7 -- bash -c 'cd /usr/src && \ - wget https://www.python.org/ftp/python/3.6.8/Python-3.6.8.tgz && \ - tar -xzf Python-3.6.8.tgz && \ - cd Python-3.6.8 && \ - ./configure --enable-optimizations && \ - make altinstall && \ - rm /usr/src/Python-3.6.8.tgz' +RUN yum install sqlite-devel -y +RUN mkdir -p /python-3-docker-build/python-3.6.8-{source,install} +RUN curl -L -v https://www.python.org/ftp/python/3.6.8/Python-3.6.8.tgz \ + | tar -zxvC /python-3-docker-build/python-3.6.8-source +RUN /usr/bin/scl enable devtoolset-7 -- \ + bash -c 'pushd /python-3-docker-build/python-3.6.8-source/Python-3.6.8 \ + && ./configure --prefix=/python-3-docker-build/python-3.6.8-install \ + --enable-optimizations \ + && make install' +ENV PATH "${PATH}:/python-3-docker-build/python-3.6.8-install/bin" # ----------------------------------------------------------------------- From 7220e42d51177a60a6861e04b9910376befc1c1a Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Fri, 11 Jan 2019 23:47:21 -0700 Subject: [PATCH 050/105] add back contrib py2 to test stage, not cron --- .travis.yml | 3 ++- build-support/travis/travis.yml.mustache | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b74fe027e25..816bfed6738 100644 --- a/.travis.yml +++ b/.travis.yml @@ -177,7 +177,7 @@ py3_osx_config: &py3_osx_config - chmod 755 /usr/local/bin/jq - ./build-support/bin/install_aws_cli_for_ci.sh # Install pyenv the conventional way, instead of through addon, due to execution order - - brew install pyenv + - HOMEBREW_NO_AUTO_UPDATE=1 brew install pyenv - pyenv install 3.6.8 && pyenv global 3.6.8 base_osx_test_config: &base_osx_test_config @@ -589,6 +589,7 @@ matrix: - <<: *py2_linux_test_config name: "Python contrib tests (Py2 PEX)" + stage: *test env: - *py2_linux_test_config_env - CACHE_NAME=linuxcontribtests.py2 diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index 6b3e602383f..4461aabd16a 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -568,6 +568,7 @@ matrix: - <<: *py2_linux_test_config name: "Python contrib tests (Py2 PEX)" + stage: *test env: - *py2_linux_test_config_env - CACHE_NAME=linuxcontribtests.py2 From acfdf3b4fdb252ecfd750b769fd7af1be999db5b Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Fri, 11 Jan 2019 23:53:34 -0700 Subject: [PATCH 051/105] Pyenv doesn't have 3.6.8 So, try 3.6.5 + list what is available to see the max possible version we can use --- build-support/travis/travis.yml.mustache | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index 4461aabd16a..89bc67eac72 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -162,7 +162,7 @@ py3_osx_config: &py3_osx_config {{>before_install_osx}} # Install pyenv the conventional way, instead of through addon, due to execution order - HOMEBREW_NO_AUTO_UPDATE=1 brew install pyenv - - pyenv install 3.6.8 && pyenv global 3.6.8 + - pyenv list && pyenv install 3.6.5 && pyenv global 3.6.5 base_osx_test_config: &base_osx_test_config <<: *pants_run_cache_config From 8a9c106011295f0478d9dc4189bb68827b1b0d2e Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Sat, 12 Jan 2019 00:32:19 -0700 Subject: [PATCH 052/105] oops, forgot to generate travis again --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 816bfed6738..56b533692e5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -178,7 +178,7 @@ py3_osx_config: &py3_osx_config - ./build-support/bin/install_aws_cli_for_ci.sh # Install pyenv the conventional way, instead of through addon, due to execution order - HOMEBREW_NO_AUTO_UPDATE=1 brew install pyenv - - pyenv install 3.6.8 && pyenv global 3.6.8 + - pyenv list && pyenv install 3.6.5 && pyenv global 3.6.5 base_osx_test_config: &base_osx_test_config <<: *pants_run_cache_config From 1790add5a8cb0caa6f256c1424391ae20777dc9a Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Sat, 12 Jan 2019 00:40:27 -0700 Subject: [PATCH 053/105] messed up the pyenv list command --- .travis.yml | 2 +- build-support/travis/travis.yml.mustache | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 56b533692e5..6c7cccf95f2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -178,7 +178,7 @@ py3_osx_config: &py3_osx_config - ./build-support/bin/install_aws_cli_for_ci.sh # Install pyenv the conventional way, instead of through addon, due to execution order - HOMEBREW_NO_AUTO_UPDATE=1 brew install pyenv - - pyenv list && pyenv install 3.6.5 && pyenv global 3.6.5 + - pyenv install --list && pyenv install 3.6.5 && pyenv global 3.6.5 base_osx_test_config: &base_osx_test_config <<: *pants_run_cache_config diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index 89bc67eac72..889988ed343 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -162,7 +162,7 @@ py3_osx_config: &py3_osx_config {{>before_install_osx}} # Install pyenv the conventional way, instead of through addon, due to execution order - HOMEBREW_NO_AUTO_UPDATE=1 brew install pyenv - - pyenv list && pyenv install 3.6.5 && pyenv global 3.6.5 + - pyenv install --list && pyenv install 3.6.5 && pyenv global 3.6.5 base_osx_test_config: &base_osx_test_config <<: *pants_run_cache_config From 6c5d01e0d09f48761b82366d73f356f4f733d829 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Sat, 12 Jan 2019 09:22:02 -0700 Subject: [PATCH 054/105] Try using beta release of Py3.6 on the build engine shard This shard uses an old OSX image, so its pyenv does not have more modern 3.6 releases. --- .travis.yml | 11 ++++++++++- build-support/travis/travis.yml.mustache | 9 ++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6c7cccf95f2..5033e4b6927 100644 --- a/.travis.yml +++ b/.travis.yml @@ -177,7 +177,7 @@ py3_osx_config: &py3_osx_config - chmod 755 /usr/local/bin/jq - ./build-support/bin/install_aws_cli_for_ci.sh # Install pyenv the conventional way, instead of through addon, due to execution order - - HOMEBREW_NO_AUTO_UPDATE=1 brew install pyenv + - &install_pyenv HOMEBREW_NO_AUTO_UPDATE=1 brew install pyenv - pyenv install --list && pyenv install 3.6.5 && pyenv global 3.6.5 base_osx_test_config: &base_osx_test_config @@ -280,6 +280,15 @@ base_osx_build_engine: &base_osx_build_engine # of the rust code built during bootstrapping. - ./build-support/bin/ci.sh ${BOOTSTRAP_ARGS} && ./build-support/bin/release.sh -f - aws --no-sign-request --region us-east-1 s3 cp ${TRAVIS_BUILD_DIR}/pants.pex ${BOOTSTRAPPED_PEX_URL_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX} + before_install: + - curl -L https://github.com/stedolan/jq/releases/download/jq-1.5/jq-osx-amd64 -o /usr/local/bin/jq + - chmod 755 /usr/local/bin/jq + - ./build-support/bin/install_aws_cli_for_ci.sh + - *install_pyenv + # We must use a beta release of Py3.6 because this is the most recent release available through + # the preinstalled brew's pyenv version. If we want a more recent Python version, we must call + # `brew update && brew upgrade pyenv`, which is costly on such an old osx image (~5 minutes). + - pyenv install 3.6.0b3 && pyenv global 3.6.0b3 after_failure: - ./build-support/bin/ci-failure.sh diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index 889988ed343..23921afbe1d 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -161,7 +161,7 @@ py3_osx_config: &py3_osx_config before_install: {{>before_install_osx}} # Install pyenv the conventional way, instead of through addon, due to execution order - - HOMEBREW_NO_AUTO_UPDATE=1 brew install pyenv + - &install_pyenv HOMEBREW_NO_AUTO_UPDATE=1 brew install pyenv - pyenv install --list && pyenv install 3.6.5 && pyenv global 3.6.5 base_osx_test_config: &base_osx_test_config @@ -259,6 +259,13 @@ base_osx_build_engine: &base_osx_build_engine # of the rust code built during bootstrapping. - ./build-support/bin/ci.sh ${BOOTSTRAP_ARGS} && ./build-support/bin/release.sh -f - aws --no-sign-request --region us-east-1 s3 cp ${TRAVIS_BUILD_DIR}/pants.pex ${BOOTSTRAPPED_PEX_URL_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX} + before_install: + {{>before_install_osx}} + - *install_pyenv + # We must use a beta release of Py3.6 because this is the most recent release available through + # the preinstalled brew's pyenv version. If we want a more recent Python version, we must call + # `brew update && brew upgrade pyenv`, which is costly on such an old osx image (~5 minutes). + - pyenv install 3.6.0b3 && pyenv global 3.6.0b3 after_failure: - ./build-support/bin/ci-failure.sh From 51dce87b589ceb9a0c34ac27ec8ce56e270644af Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Sat, 12 Jan 2019 14:15:34 -0700 Subject: [PATCH 055/105] Clone pyenv directly on osx engine shard Allows us to get a modern version of Py3.6 without having to call `brew update`. Note we only do this for the OSX engine shard, and not the default py3_osx_config, because those have a more modern osx_image so can use the default brew-installed pyenv; this allows us to avoid the cost of having to git clone pyenv for those shards. Thanks John for the suggestion! --- .travis.yml | 14 +++++++------- build-support/travis/travis.yml.mustache | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5033e4b6927..c3397595b07 100644 --- a/.travis.yml +++ b/.travis.yml @@ -176,8 +176,8 @@ py3_osx_config: &py3_osx_config - curl -L https://github.com/stedolan/jq/releases/download/jq-1.5/jq-osx-amd64 -o /usr/local/bin/jq - chmod 755 /usr/local/bin/jq - ./build-support/bin/install_aws_cli_for_ci.sh - # Install pyenv the conventional way, instead of through addon, due to execution order - - &install_pyenv HOMEBREW_NO_AUTO_UPDATE=1 brew install pyenv + # Install pyenv the conventional brew way, instead of through addon, due to execution order + - HOMEBREW_NO_AUTO_UPDATE=1 brew install pyenv - pyenv install --list && pyenv install 3.6.5 && pyenv global 3.6.5 base_osx_test_config: &base_osx_test_config @@ -284,11 +284,11 @@ base_osx_build_engine: &base_osx_build_engine - curl -L https://github.com/stedolan/jq/releases/download/jq-1.5/jq-osx-amd64 -o /usr/local/bin/jq - chmod 755 /usr/local/bin/jq - ./build-support/bin/install_aws_cli_for_ci.sh - - *install_pyenv - # We must use a beta release of Py3.6 because this is the most recent release available through - # the preinstalled brew's pyenv version. If we want a more recent Python version, we must call - # `brew update && brew upgrade pyenv`, which is costly on such an old osx image (~5 minutes). - - pyenv install 3.6.0b3 && pyenv global 3.6.0b3 + # Clone pyenv directly from GitHub. Brew's version of pyenv on this osx_image is too old and only has + # a beta version of Python 3.6 that does not work properly. We could `brew update && brew upgrade pyenv`, + # but this would substantially slow down CI, so we instead circumvent Brew and download pyenv directly. + - git clone https://github.com/pyenv/pyenv ./pyenv + - ./pyenv/bin/pyenv install 3.6.5 && ./pyenv/bin/pyenv global 3.6.5 after_failure: - ./build-support/bin/ci-failure.sh diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index 23921afbe1d..1912e74ba7d 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -160,8 +160,8 @@ py3_osx_config: &py3_osx_config - PATH="/usr/local/opt/openssl/bin:$PATH" LDFLAGS="-L/usr/local/opt/openssl/lib" CPPFLAGS="-I/usr/local/opt/openssl/include" before_install: {{>before_install_osx}} - # Install pyenv the conventional way, instead of through addon, due to execution order - - &install_pyenv HOMEBREW_NO_AUTO_UPDATE=1 brew install pyenv + # Install pyenv the conventional brew way, instead of through addon, due to execution order + - HOMEBREW_NO_AUTO_UPDATE=1 brew install pyenv - pyenv install --list && pyenv install 3.6.5 && pyenv global 3.6.5 base_osx_test_config: &base_osx_test_config @@ -261,11 +261,11 @@ base_osx_build_engine: &base_osx_build_engine - aws --no-sign-request --region us-east-1 s3 cp ${TRAVIS_BUILD_DIR}/pants.pex ${BOOTSTRAPPED_PEX_URL_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX} before_install: {{>before_install_osx}} - - *install_pyenv - # We must use a beta release of Py3.6 because this is the most recent release available through - # the preinstalled brew's pyenv version. If we want a more recent Python version, we must call - # `brew update && brew upgrade pyenv`, which is costly on such an old osx image (~5 minutes). - - pyenv install 3.6.0b3 && pyenv global 3.6.0b3 + # Clone pyenv directly from GitHub. Brew's version of pyenv on this osx_image is too old and only has + # a beta version of Python 3.6 that does not work properly. We could `brew update && brew upgrade pyenv`, + # but this would substantially slow down CI, so we instead circumvent Brew and download pyenv directly. + - git clone https://github.com/pyenv/pyenv ./pyenv + - ./pyenv/bin/pyenv install 3.6.5 && ./pyenv/bin/pyenv global 3.6.5 after_failure: - ./build-support/bin/ci-failure.sh From fdff72405a284d44947b33a7058178fcefd41182 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Sat, 12 Jan 2019 14:22:57 -0700 Subject: [PATCH 056/105] Add back blacklist logic to ci.sh --- build-support/bin/ci.sh | 18 +++++++++++++++--- build-support/known_py3_pex_failures.txt | 0 2 files changed, 15 insertions(+), 3 deletions(-) create mode 100644 build-support/known_py3_pex_failures.txt diff --git a/build-support/bin/ci.sh b/build-support/bin/ci.sh index 88ea039cd03..2a148c459c8 100755 --- a/build-support/bin/ci.sh +++ b/build-support/bin/ci.sh @@ -13,7 +13,7 @@ function usage() { cat < 0 )); then die "$@" @@ -59,7 +60,7 @@ python_contrib_shard="0/1" python_intg_shard="0/1" python_two="false" -while getopts "h2fxbmrjlpeasu:ny:ci:tz" opt; do +while getopts "h2fxbmrjlpeasu:ny:ci:tzw" opt; do case ${opt} in h) usage ;; 2) python_two="true" ;; @@ -81,6 +82,7 @@ while getopts "h2fxbmrjlpeasu:ny:ci:tz" opt; do i) python_intg_shard=${OPTARG} ;; t) run_lint="true" ;; z) test_platform_specific_behavior="true" ;; + w) run_blacklisted_tests_only="true" ;; *) usage "Invalid option: -${OPTARG}" ;; esac done @@ -274,9 +276,19 @@ if [[ "${run_integration:-false}" == "true" ]]; then fi start_travis_section "IntegrationTests" "Running Pants Integration tests${shard_desc}" ( + known_py3_pex_failures_file="${REPO_ROOT}/build-support/known_py3_pex_failures.txt" + if [[ "${python_two:-false}" == "false" ]]; then + targets="$(comm -23 <(./pants.pex --tag='+integration' list tests/python:: | grep '.' | sort) <(sort "${known_py3_pex_failures_file}"))" + else + if [[ "${run_blacklisted_tests_only:-false}" == "true" ]]; then + targets="$(cat ${known_py3_pex_failures_file})" + else + targets="tests/python::" + fi + fi ./pants.pex --tag='+integration' test.pytest \ --test-pytest-test-shard=${python_intg_shard} \ - tests/python:: -- ${PYTEST_PASSTHRU_ARGS} + $targets -- ${PYTEST_PASSTHRU_ARGS} ) || die "Pants Integration test failure" end_travis_section fi diff --git a/build-support/known_py3_pex_failures.txt b/build-support/known_py3_pex_failures.txt new file mode 100644 index 00000000000..e69de29bb2d From 1f07703d655c5ed5d91b67b4a1ba4b2931e7121e Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Sat, 12 Jan 2019 14:30:57 -0700 Subject: [PATCH 057/105] Add blacklisted shards to Travis Thanks to Benjy's mustache change, it's extremely easy for us to tweak the number of blacklisted vs normal py3 shards. I'll fill out the actual blacklist and tweak the # of shards in a bit, after rerunning the test suite. But this should get the setup down. --- .travis.yml | 8 ++++++++ build-support/travis/generate_travis_yml.py | 12 ++++++++--- build-support/travis/travis.yml.mustache | 22 +++++++++++++++------ 3 files changed, 33 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index c3397595b07..5691a5ef5fc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -773,6 +773,14 @@ matrix: script: - ./build-support/bin/travis-ci.sh -bc -i 19/20 + - <<: *py3_linux_test_config + name: "Blacklisted integration tests for pants - shard 0 (Py2 PEX)" + env: + - *py2_linux_test_config_env + - CACHE_NAME=integrationshard0.py2blacklist + script: + - ./build-support/bin/travis-ci.sh -bc2 -i 0/1 + - <<: *py2_linux_test_config name: "Integration tests for pants - shard 0 (Py2 PEX)" stage: *cron diff --git a/build-support/travis/generate_travis_yml.py b/build-support/travis/generate_travis_yml.py index 2cc19cd9914..d3888bdff34 100644 --- a/build-support/travis/generate_travis_yml.py +++ b/build-support/travis/generate_travis_yml.py @@ -8,7 +8,9 @@ import pystache -num_integration_shards = 20 +num_py3_integration_shards = 20 +num_py2_blacklist_integration_shards = 1 +num_cron_integration_shards = 20 HEADER = """ @@ -31,8 +33,12 @@ def generate_travis_yml(): __name__, 'before_install_osx.mustache').decode('utf-8') context = { 'header': HEADER, - 'integration_shards': range(0, num_integration_shards), - 'integration_shards_length': num_integration_shards, + 'py3_integration_shards': range(0, num_py3_integration_shards), + 'py3_integration_shards_length': num_py3_integration_shards, + 'py2_blacklist_integration_shards': range(0, num_py2_blacklist_integration_shards), + 'py2_blacklist_integration_shards_length': num_py2_blacklist_integration_shards, + 'cron_integration_shards': range(0, num_cron_integration_shards), + 'cron_integration_shards_length': num_cron_integration_shards, } renderer = pystache.Renderer(partials={ 'before_install_linux': before_install_linux, diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index 1912e74ba7d..643d467f3e6 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -590,17 +590,27 @@ matrix: script: - ./build-support/bin/travis-ci.sh -bn -{{#integration_shards}} +{{#py3_integration_shards}} - <<: *py3_linux_test_config name: "Integration tests for pants - shard {{.}} (Py3 PEX)" env: - *py3_linux_test_config_env - CACHE_NAME=integrationshard{{.}} script: - - ./build-support/bin/travis-ci.sh -bc -i {{.}}/{{integration_shards_length}} + - ./build-support/bin/travis-ci.sh -bc -i {{.}}/{{py3_integration_shards_length}} -{{/integration_shards}} -{{#integration_shards}} +{{/py3_integration_shards}} +{{#py2_blacklist_integration_shards}} + - <<: *py3_linux_test_config + name: "Blacklisted integration tests for pants - shard {{.}} (Py2 PEX)" + env: + - *py2_linux_test_config_env + - CACHE_NAME=integrationshard{{.}}.py2blacklist + script: + - ./build-support/bin/travis-ci.sh -bc2 -i {{.}}/{{py2_blacklist_integration_shards_length}} + +{{/py2_blacklist_integration_shards}} +{{#cron_integration_shards}} - <<: *py2_linux_test_config name: "Integration tests for pants - shard {{.}} (Py2 PEX)" stage: *cron @@ -608,9 +618,9 @@ matrix: - *py2_linux_test_config_env - CACHE_NAME=cronshard{{.}} script: - - ./build-support/bin/travis-ci.sh -bc2 -i {{.}}/{{integration_shards_length}} + - ./build-support/bin/travis-ci.sh -bc2 -i {{.}}/{{cron_integration_shards_length}} -{{/integration_shards}} +{{/cron_integration_shards}} - <<: *py2_linux_rust_tests - <<: *py3_linux_rust_tests From 511c81c5f7554e16c6f3c483aa1077b7729bf83b Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Sat, 12 Jan 2019 14:49:06 -0700 Subject: [PATCH 058/105] Try different pyenv path to fix unrecognized command issue --- .travis.yml | 20 ++++++++++---------- build-support/travis/travis.yml.mustache | 16 ++++++++-------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5691a5ef5fc..69e9b9f6a36 100644 --- a/.travis.yml +++ b/.travis.yml @@ -280,15 +280,6 @@ base_osx_build_engine: &base_osx_build_engine # of the rust code built during bootstrapping. - ./build-support/bin/ci.sh ${BOOTSTRAP_ARGS} && ./build-support/bin/release.sh -f - aws --no-sign-request --region us-east-1 s3 cp ${TRAVIS_BUILD_DIR}/pants.pex ${BOOTSTRAPPED_PEX_URL_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX} - before_install: - - curl -L https://github.com/stedolan/jq/releases/download/jq-1.5/jq-osx-amd64 -o /usr/local/bin/jq - - chmod 755 /usr/local/bin/jq - - ./build-support/bin/install_aws_cli_for_ci.sh - # Clone pyenv directly from GitHub. Brew's version of pyenv on this osx_image is too old and only has - # a beta version of Python 3.6 that does not work properly. We could `brew update && brew upgrade pyenv`, - # but this would substantially slow down CI, so we instead circumvent Brew and download pyenv directly. - - git clone https://github.com/pyenv/pyenv ./pyenv - - ./pyenv/bin/pyenv install 3.6.5 && ./pyenv/bin/pyenv global 3.6.5 after_failure: - ./build-support/bin/ci-failure.sh @@ -306,12 +297,21 @@ py3_osx_build_engine: &py3_osx_build_engine <<: *py3_osx_config <<: *base_osx_build_engine name: "Build OSX native engine and pants.pex (Py3 PEX)" - env: - *base_osx_build_engine_env - CACHE_NAME=osxpexbuild.py3 - BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.osx - BOOTSTRAP_ARGS='' + before_install: + - curl -L https://github.com/stedolan/jq/releases/download/jq-1.5/jq-osx-amd64 -o /usr/local/bin/jq + - chmod 755 /usr/local/bin/jq + - ./build-support/bin/install_aws_cli_for_ci.sh + # Clone pyenv directly from GitHub. Brew's version of pyenv on this osx_image is too old and only has + # a beta version of Python 3.6 that does not work properly. We could `brew update && brew upgrade pyenv`, + # but this would substantially slow down CI, so we instead circumvent Brew and download pyenv directly. + - git clone https://github.com/pyenv/pyenv ~/.pyenv + - ~/.pyenv/bin/pyenv + - ~/.pyenv/bin/pyenv install 3.6.5 && ~/.pyenv/bin/pyenv global 3.6.5 # ------------------------------------------------------------------------- # Build wheels diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index 643d467f3e6..ee7bfe8bb00 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -259,13 +259,6 @@ base_osx_build_engine: &base_osx_build_engine # of the rust code built during bootstrapping. - ./build-support/bin/ci.sh ${BOOTSTRAP_ARGS} && ./build-support/bin/release.sh -f - aws --no-sign-request --region us-east-1 s3 cp ${TRAVIS_BUILD_DIR}/pants.pex ${BOOTSTRAPPED_PEX_URL_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX} - before_install: - {{>before_install_osx}} - # Clone pyenv directly from GitHub. Brew's version of pyenv on this osx_image is too old and only has - # a beta version of Python 3.6 that does not work properly. We could `brew update && brew upgrade pyenv`, - # but this would substantially slow down CI, so we instead circumvent Brew and download pyenv directly. - - git clone https://github.com/pyenv/pyenv ./pyenv - - ./pyenv/bin/pyenv install 3.6.5 && ./pyenv/bin/pyenv global 3.6.5 after_failure: - ./build-support/bin/ci-failure.sh @@ -283,12 +276,19 @@ py3_osx_build_engine: &py3_osx_build_engine <<: *py3_osx_config <<: *base_osx_build_engine name: "Build OSX native engine and pants.pex (Py3 PEX)" - env: - *base_osx_build_engine_env - CACHE_NAME=osxpexbuild.py3 - BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.osx - BOOTSTRAP_ARGS='' + before_install: + {{>before_install_osx}} + # Clone pyenv directly from GitHub. Brew's version of pyenv on this osx_image is too old and only has + # a beta version of Python 3.6 that does not work properly. We could `brew update && brew upgrade pyenv`, + # but this would substantially slow down CI, so we instead circumvent Brew and download pyenv directly. + - git clone https://github.com/pyenv/pyenv ~/.pyenv + - ~/.pyenv/bin/pyenv + - ~/.pyenv/bin/pyenv install 3.6.5 && ~/.pyenv/bin/pyenv global 3.6.5 # ------------------------------------------------------------------------- # Build wheels From 8b762316fcfbeebefc28fed016aae43982a48c6c Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Sat, 12 Jan 2019 15:46:51 -0700 Subject: [PATCH 059/105] Set PYENV_ROOT Danny found that this is why pyenv is not working as intended. --- .travis.yml | 6 +++--- build-support/travis/travis.yml.mustache | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 69e9b9f6a36..44feb65424e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -302,6 +302,7 @@ py3_osx_build_engine: &py3_osx_build_engine - CACHE_NAME=osxpexbuild.py3 - BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.osx - BOOTSTRAP_ARGS='' + - PYENV_ROOT="~/.pyenv" before_install: - curl -L https://github.com/stedolan/jq/releases/download/jq-1.5/jq-osx-amd64 -o /usr/local/bin/jq - chmod 755 /usr/local/bin/jq @@ -309,9 +310,8 @@ py3_osx_build_engine: &py3_osx_build_engine # Clone pyenv directly from GitHub. Brew's version of pyenv on this osx_image is too old and only has # a beta version of Python 3.6 that does not work properly. We could `brew update && brew upgrade pyenv`, # but this would substantially slow down CI, so we instead circumvent Brew and download pyenv directly. - - git clone https://github.com/pyenv/pyenv ~/.pyenv - - ~/.pyenv/bin/pyenv - - ~/.pyenv/bin/pyenv install 3.6.5 && ~/.pyenv/bin/pyenv global 3.6.5 + - git clone https://github.com/pyenv/pyenv ${PYENV_ROOT} + - ${PYENV_ROOT}/bin/pyenv install 3.6.5 && ${PYENV_ROOT}/bin/pyenv global 3.6.5 # ------------------------------------------------------------------------- # Build wheels diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index ee7bfe8bb00..41714d525d4 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -281,14 +281,14 @@ py3_osx_build_engine: &py3_osx_build_engine - CACHE_NAME=osxpexbuild.py3 - BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.osx - BOOTSTRAP_ARGS='' + - PYENV_ROOT="~/.pyenv" before_install: {{>before_install_osx}} # Clone pyenv directly from GitHub. Brew's version of pyenv on this osx_image is too old and only has # a beta version of Python 3.6 that does not work properly. We could `brew update && brew upgrade pyenv`, # but this would substantially slow down CI, so we instead circumvent Brew and download pyenv directly. - - git clone https://github.com/pyenv/pyenv ~/.pyenv - - ~/.pyenv/bin/pyenv - - ~/.pyenv/bin/pyenv install 3.6.5 && ~/.pyenv/bin/pyenv global 3.6.5 + - git clone https://github.com/pyenv/pyenv ${PYENV_ROOT} + - ${PYENV_ROOT}/bin/pyenv install 3.6.5 && ${PYENV_ROOT}/bin/pyenv global 3.6.5 # ------------------------------------------------------------------------- # Build wheels From b8158cd75dc7b15cfdf46cb855da720bfb231dfe Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Sat, 12 Jan 2019 15:59:16 -0700 Subject: [PATCH 060/105] Also modify PATH to see if that fixes pyenv --- .travis.yml | 8 ++++++-- build-support/travis/travis.yml.mustache | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 44feb65424e..671444e15f6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -178,7 +178,9 @@ py3_osx_config: &py3_osx_config - ./build-support/bin/install_aws_cli_for_ci.sh # Install pyenv the conventional brew way, instead of through addon, due to execution order - HOMEBREW_NO_AUTO_UPDATE=1 brew install pyenv - - pyenv install --list && pyenv install 3.6.5 && pyenv global 3.6.5 + - pyenv install --list + - pyenv install 3.6.5 + = pyenv global 3.6.5 base_osx_test_config: &base_osx_test_config <<: *pants_run_cache_config @@ -303,6 +305,7 @@ py3_osx_build_engine: &py3_osx_build_engine - BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.osx - BOOTSTRAP_ARGS='' - PYENV_ROOT="~/.pyenv" + - PATH="${PYENV_ROOT}/shims:${PATH}" before_install: - curl -L https://github.com/stedolan/jq/releases/download/jq-1.5/jq-osx-amd64 -o /usr/local/bin/jq - chmod 755 /usr/local/bin/jq @@ -311,7 +314,8 @@ py3_osx_build_engine: &py3_osx_build_engine # a beta version of Python 3.6 that does not work properly. We could `brew update && brew upgrade pyenv`, # but this would substantially slow down CI, so we instead circumvent Brew and download pyenv directly. - git clone https://github.com/pyenv/pyenv ${PYENV_ROOT} - - ${PYENV_ROOT}/bin/pyenv install 3.6.5 && ${PYENV_ROOT}/bin/pyenv global 3.6.5 + - ${PYENV_ROOT}/bin/pyenv install 3.6.5 + - ${PYENV_ROOT}/bin/pyenv global 3.6.5 # ------------------------------------------------------------------------- # Build wheels diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index 41714d525d4..5578a525e6c 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -162,7 +162,9 @@ py3_osx_config: &py3_osx_config {{>before_install_osx}} # Install pyenv the conventional brew way, instead of through addon, due to execution order - HOMEBREW_NO_AUTO_UPDATE=1 brew install pyenv - - pyenv install --list && pyenv install 3.6.5 && pyenv global 3.6.5 + - pyenv install --list + - pyenv install 3.6.5 + = pyenv global 3.6.5 base_osx_test_config: &base_osx_test_config <<: *pants_run_cache_config @@ -282,13 +284,15 @@ py3_osx_build_engine: &py3_osx_build_engine - BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.osx - BOOTSTRAP_ARGS='' - PYENV_ROOT="~/.pyenv" + - PATH="${PYENV_ROOT}/shims:${PATH}" before_install: {{>before_install_osx}} # Clone pyenv directly from GitHub. Brew's version of pyenv on this osx_image is too old and only has # a beta version of Python 3.6 that does not work properly. We could `brew update && brew upgrade pyenv`, # but this would substantially slow down CI, so we instead circumvent Brew and download pyenv directly. - git clone https://github.com/pyenv/pyenv ${PYENV_ROOT} - - ${PYENV_ROOT}/bin/pyenv install 3.6.5 && ${PYENV_ROOT}/bin/pyenv global 3.6.5 + - ${PYENV_ROOT}/bin/pyenv install 3.6.5 + - ${PYENV_ROOT}/bin/pyenv global 3.6.5 # ------------------------------------------------------------------------- # Build wheels From 04da4aae52ca768610e3c7a88702365aa9699142 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Sat, 12 Jan 2019 16:01:15 -0700 Subject: [PATCH 061/105] typo... used = instead of - --- .travis.yml | 2 +- build-support/travis/travis.yml.mustache | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 671444e15f6..00c68245b32 100644 --- a/.travis.yml +++ b/.travis.yml @@ -180,7 +180,7 @@ py3_osx_config: &py3_osx_config - HOMEBREW_NO_AUTO_UPDATE=1 brew install pyenv - pyenv install --list - pyenv install 3.6.5 - = pyenv global 3.6.5 + - pyenv global 3.6.5 base_osx_test_config: &base_osx_test_config <<: *pants_run_cache_config diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index 5578a525e6c..aea07135b18 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -164,7 +164,7 @@ py3_osx_config: &py3_osx_config - HOMEBREW_NO_AUTO_UPDATE=1 brew install pyenv - pyenv install --list - pyenv install 3.6.5 - = pyenv global 3.6.5 + - pyenv global 3.6.5 base_osx_test_config: &base_osx_test_config <<: *pants_run_cache_config From c1192c468f8a5c5ebbcdb590d014ae76775c6daa Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Sat, 12 Jan 2019 16:07:08 -0700 Subject: [PATCH 062/105] try Danny's suggestion of making PYENV_ROOT not belong to home --- .travis.yml | 2 +- build-support/travis/travis.yml.mustache | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 00c68245b32..e97748856ac 100644 --- a/.travis.yml +++ b/.travis.yml @@ -304,7 +304,7 @@ py3_osx_build_engine: &py3_osx_build_engine - CACHE_NAME=osxpexbuild.py3 - BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.osx - BOOTSTRAP_ARGS='' - - PYENV_ROOT="~/.pyenv" + - PYENV_ROOT="/.pyenv" - PATH="${PYENV_ROOT}/shims:${PATH}" before_install: - curl -L https://github.com/stedolan/jq/releases/download/jq-1.5/jq-osx-amd64 -o /usr/local/bin/jq diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index aea07135b18..56caa4ae6b3 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -283,7 +283,7 @@ py3_osx_build_engine: &py3_osx_build_engine - CACHE_NAME=osxpexbuild.py3 - BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.osx - BOOTSTRAP_ARGS='' - - PYENV_ROOT="~/.pyenv" + - PYENV_ROOT="/.pyenv" - PATH="${PYENV_ROOT}/shims:${PATH}" before_install: {{>before_install_osx}} From b06b9bb9d685c112adabaafe5f4494704cdd1368 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Sat, 12 Jan 2019 16:17:58 -0700 Subject: [PATCH 063/105] Try Pex's PYENV_ROOT If this doesn't work, I'm taking a break.. --- .travis.yml | 2 +- build-support/travis/travis.yml.mustache | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index e97748856ac..08f861e3803 100644 --- a/.travis.yml +++ b/.travis.yml @@ -304,7 +304,7 @@ py3_osx_build_engine: &py3_osx_build_engine - CACHE_NAME=osxpexbuild.py3 - BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.osx - BOOTSTRAP_ARGS='' - - PYENV_ROOT="/.pyenv" + - PYENV_ROOT="${HOME}/.pyenv" - PATH="${PYENV_ROOT}/shims:${PATH}" before_install: - curl -L https://github.com/stedolan/jq/releases/download/jq-1.5/jq-osx-amd64 -o /usr/local/bin/jq diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index 56caa4ae6b3..89978c46cb2 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -283,7 +283,7 @@ py3_osx_build_engine: &py3_osx_build_engine - CACHE_NAME=osxpexbuild.py3 - BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.osx - BOOTSTRAP_ARGS='' - - PYENV_ROOT="/.pyenv" + - PYENV_ROOT="${HOME}/.pyenv" - PATH="${PYENV_ROOT}/shims:${PATH}" before_install: {{>before_install_osx}} From b6378f73b4f33dc616f4b127424a1a64d9ac167b Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Sat, 12 Jan 2019 16:49:19 -0700 Subject: [PATCH 064/105] Add openssl env vars to osx engine shard --- .travis.yml | 5 ++--- build-support/travis/travis.yml.mustache | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 08f861e3803..e8de349560f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -167,18 +167,16 @@ py3_osx_config: &py3_osx_config addons: brew: packages: &py3_osx_config_brew_packages - - pyenv - openssl env: # Fix Python 3 issue linking to OpenSSL - - PATH="/usr/local/opt/openssl/bin:$PATH" LDFLAGS="-L/usr/local/opt/openssl/lib" CPPFLAGS="-I/usr/local/opt/openssl/include" + - &py3_osx_config_env PATH="/usr/local/opt/openssl/bin:$PATH" LDFLAGS="-L/usr/local/opt/openssl/lib" CPPFLAGS="-I/usr/local/opt/openssl/include" before_install: - curl -L https://github.com/stedolan/jq/releases/download/jq-1.5/jq-osx-amd64 -o /usr/local/bin/jq - chmod 755 /usr/local/bin/jq - ./build-support/bin/install_aws_cli_for_ci.sh # Install pyenv the conventional brew way, instead of through addon, due to execution order - HOMEBREW_NO_AUTO_UPDATE=1 brew install pyenv - - pyenv install --list - pyenv install 3.6.5 - pyenv global 3.6.5 @@ -301,6 +299,7 @@ py3_osx_build_engine: &py3_osx_build_engine name: "Build OSX native engine and pants.pex (Py3 PEX)" env: - *base_osx_build_engine_env + - *py3_osx_config_env - CACHE_NAME=osxpexbuild.py3 - BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.osx - BOOTSTRAP_ARGS='' diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index 89978c46cb2..06c676a4825 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -153,16 +153,14 @@ py3_osx_config: &py3_osx_config addons: brew: packages: &py3_osx_config_brew_packages - - pyenv - openssl env: # Fix Python 3 issue linking to OpenSSL - - PATH="/usr/local/opt/openssl/bin:$PATH" LDFLAGS="-L/usr/local/opt/openssl/lib" CPPFLAGS="-I/usr/local/opt/openssl/include" + - &py3_osx_config_env PATH="/usr/local/opt/openssl/bin:$PATH" LDFLAGS="-L/usr/local/opt/openssl/lib" CPPFLAGS="-I/usr/local/opt/openssl/include" before_install: {{>before_install_osx}} # Install pyenv the conventional brew way, instead of through addon, due to execution order - HOMEBREW_NO_AUTO_UPDATE=1 brew install pyenv - - pyenv install --list - pyenv install 3.6.5 - pyenv global 3.6.5 @@ -280,6 +278,7 @@ py3_osx_build_engine: &py3_osx_build_engine name: "Build OSX native engine and pants.pex (Py3 PEX)" env: - *base_osx_build_engine_env + - *py3_osx_config_env - CACHE_NAME=osxpexbuild.py3 - BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.osx - BOOTSTRAP_ARGS='' From e5973c8a12ea9f3005b575797cedb36560a4295a Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Sat, 12 Jan 2019 17:44:35 -0700 Subject: [PATCH 065/105] Revert -x debug in ci scripts --- build-support/bin/ci.sh | 2 +- build-support/bin/travis-ci.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build-support/bin/ci.sh b/build-support/bin/ci.sh index 2a148c459c8..2bb6e43bd39 100755 --- a/build-support/bin/ci.sh +++ b/build-support/bin/ci.sh @@ -2,7 +2,7 @@ # We use some subshell pipelines to collect target lists, make sure target collection failing # fails the build. -set -ox pipefail +set -o pipefail REPO_ROOT=$(cd $(dirname "${BASH_SOURCE[0]}") && cd "$(git rev-parse --show-toplevel)" && pwd) cd ${REPO_ROOT} diff --git a/build-support/bin/travis-ci.sh b/build-support/bin/travis-ci.sh index 2d1b679b2a8..6401113d2d9 100755 --- a/build-support/bin/travis-ci.sh +++ b/build-support/bin/travis-ci.sh @@ -5,7 +5,7 @@ # TravisCI-specific environment fixups can live in this script which forwards to the generic ci # script post fixups. -set -euox pipefail +set -euo pipefail cd "$(git rev-parse --show-toplevel)" From 0a901e0c6297d192e70a0da49cea2ed576af1ecc Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Sat, 12 Jan 2019 17:50:02 -0700 Subject: [PATCH 066/105] Build wheels with Py2 They fail due to unicode issue. That can be addressed in a followup PR. This PR is focused on setting up the infrastructure for us to use Py3 PEX in CI. --- .travis.yml | 9 ++++++--- build-support/travis/travis.yml.mustache | 9 ++++++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index e8de349560f..5fbb3b8c541 100644 --- a/.travis.yml +++ b/.travis.yml @@ -564,11 +564,14 @@ matrix: - <<: *py2_osx_build_engine - <<: *py3_osx_build_engine + # TODO: the build wheels shards fail with Py3 pex due to unicode issues. + # Until fixing this, simply run shards with Py2. - <<: *py2_linux_build_wheels - - <<: *py3_linux_build_wheels - + stage: *test + # - <<: *py3_linux_build_wheels - <<: *py2_osx_build_wheels - - <<: *py3_osx_build_wheels + stage: *test + # - <<: *py3_osx_build_wheels - <<: *py2_osx_10_12_sanity_check - <<: *py3_osx_10_12_sanity_check diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index 06c676a4825..68d8b6a0d86 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -541,11 +541,14 @@ matrix: - <<: *py2_osx_build_engine - <<: *py3_osx_build_engine + # TODO: the build wheels shards fail with Py3 pex due to unicode issues. + # Until fixing this, simply run shards with Py2. - <<: *py2_linux_build_wheels - - <<: *py3_linux_build_wheels - + stage: *test + # - <<: *py3_linux_build_wheels - <<: *py2_osx_build_wheels - - <<: *py3_osx_build_wheels + stage: *test + # - <<: *py3_osx_build_wheels - <<: *py2_osx_10_12_sanity_check - <<: *py3_osx_10_12_sanity_check From c5967af324157c26199c5738ee816312d97eba0c Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Sat, 12 Jan 2019 18:18:28 -0700 Subject: [PATCH 067/105] Use older Py3.6 for osx 10.12 sanity check --- .travis.yml | 10 +++++++++- build-support/travis/travis.yml.mustache | 8 +++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5fbb3b8c541..9932c85e39f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -176,7 +176,7 @@ py3_osx_config: &py3_osx_config - chmod 755 /usr/local/bin/jq - ./build-support/bin/install_aws_cli_for_ci.sh # Install pyenv the conventional brew way, instead of through addon, due to execution order - - HOMEBREW_NO_AUTO_UPDATE=1 brew install pyenv + - &install_pyenv HOMEBREW_NO_AUTO_UPDATE=1 brew install pyenv - pyenv install 3.6.5 - pyenv global 3.6.5 @@ -393,6 +393,14 @@ py3_osx_10_12_sanity_check: &py3_osx_10_12_sanity_check env: - *py3_osx_test_config_env - CACHE_NAME=macos10.12sanity.py3 + # Install earlier version of Python 3.6 than other shards, due to older osx_image + before_install: + - curl -L https://github.com/stedolan/jq/releases/download/jq-1.5/jq-osx-amd64 -o /usr/local/bin/jq + - chmod 755 /usr/local/bin/jq + - ./build-support/bin/install_aws_cli_for_ci.sh + - *install_pyenv + - pyenv install 3.6.3 + - pyenv global 3.6.3 base_osx_10_13_sanity_check: &base_osx_10_13_sanity_check <<: *base_osx_sanity_check diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index 68d8b6a0d86..cadd32004fc 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -160,7 +160,7 @@ py3_osx_config: &py3_osx_config before_install: {{>before_install_osx}} # Install pyenv the conventional brew way, instead of through addon, due to execution order - - HOMEBREW_NO_AUTO_UPDATE=1 brew install pyenv + - &install_pyenv HOMEBREW_NO_AUTO_UPDATE=1 brew install pyenv - pyenv install 3.6.5 - pyenv global 3.6.5 @@ -370,6 +370,12 @@ py3_osx_10_12_sanity_check: &py3_osx_10_12_sanity_check env: - *py3_osx_test_config_env - CACHE_NAME=macos10.12sanity.py3 + # Install earlier version of Python 3.6 than other shards, due to older osx_image + before_install: + {{>before_install_osx}} + - *install_pyenv + - pyenv install 3.6.3 + - pyenv global 3.6.3 base_osx_10_13_sanity_check: &base_osx_10_13_sanity_check <<: *base_osx_sanity_check From 6c9ee3857deafb9f7217436077925f53873ac922 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Sat, 12 Jan 2019 18:41:28 -0700 Subject: [PATCH 068/105] Always on OSX py3 clone pyenv Earlier we were trying to use brew pyenv in some places, if it was recent enough. I found this led to several different cases where we had to add workarounds. Instead, we simplify to always clone from pyenv and use 3.6.8 no matter what. --- .travis.yml | 46 ++++++++++-------------- build-support/travis/travis.yml.mustache | 42 ++++++++++------------ 2 files changed, 38 insertions(+), 50 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9932c85e39f..8f90cbb790c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -170,15 +170,21 @@ py3_osx_config: &py3_osx_config - openssl env: # Fix Python 3 issue linking to OpenSSL - - &py3_osx_config_env PATH="/usr/local/opt/openssl/bin:$PATH" LDFLAGS="-L/usr/local/opt/openssl/lib" CPPFLAGS="-I/usr/local/opt/openssl/include" + - &py3_osx_config_env > + PATH="/usr/local/opt/openssl/bin:$PATH" + LDFLAGS="-L/usr/local/opt/openssl/lib" + CPPFLAGS="-I/usr/local/opt/openssl/include" + PYENV_ROOT="${HOME}/.pyenv" + PATH="${PYENV_ROOT}/shims:${PATH}" before_install: - curl -L https://github.com/stedolan/jq/releases/download/jq-1.5/jq-osx-amd64 -o /usr/local/bin/jq - chmod 755 /usr/local/bin/jq - ./build-support/bin/install_aws_cli_for_ci.sh - # Install pyenv the conventional brew way, instead of through addon, due to execution order - - &install_pyenv HOMEBREW_NO_AUTO_UPDATE=1 brew install pyenv - - pyenv install 3.6.5 - - pyenv global 3.6.5 + # Clone pyenv directly from GitHub. For multiple osx images, brew's version of pyenv is too old to get + # modern Python 3.6. Pulling from GitHub instead allows us to use the same Python 3 version accross every OSX shard. + - git clone https://github.com/pyenv/pyenv ${PYENV_ROOT} + - ${PYENV_ROOT}/bin/pyenv install 3.6.8 + - ${PYENV_ROOT}/bin/pyenv global 3.6.8 base_osx_test_config: &base_osx_test_config <<: *pants_run_cache_config @@ -199,8 +205,14 @@ py3_osx_test_config: &py3_osx_test_config <<: *base_osx_test_config stage: *test env: - # Must duplicate OpenSSL config from py3_osx_config's env because it cannot be merged into a new anchor - - &py3_osx_test_config_env PATH="/usr/local/opt/openssl/bin:$PATH" LDFLAGS="-L/usr/local/opt/openssl/lib" CPPFLAGS="-I/usr/local/opt/openssl/include" BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.osx + # Must duplicate py3_osx_config's env because it cannot be merged into a new anchor + - &py3_osx_test_config_env > + PATH="/usr/local/opt/openssl/bin:$PATH" + LDFLAGS="-L/usr/local/opt/openssl/lib" + CPPFLAGS="-I/usr/local/opt/openssl/include" + PYENV_ROOT="${HOME}/.pyenv" + PATH="${PYENV_ROOT}/shims:${PATH}" + BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.osx linux_with_fuse: &linux_with_fuse before_install: @@ -303,18 +315,6 @@ py3_osx_build_engine: &py3_osx_build_engine - CACHE_NAME=osxpexbuild.py3 - BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.osx - BOOTSTRAP_ARGS='' - - PYENV_ROOT="${HOME}/.pyenv" - - PATH="${PYENV_ROOT}/shims:${PATH}" - before_install: - - curl -L https://github.com/stedolan/jq/releases/download/jq-1.5/jq-osx-amd64 -o /usr/local/bin/jq - - chmod 755 /usr/local/bin/jq - - ./build-support/bin/install_aws_cli_for_ci.sh - # Clone pyenv directly from GitHub. Brew's version of pyenv on this osx_image is too old and only has - # a beta version of Python 3.6 that does not work properly. We could `brew update && brew upgrade pyenv`, - # but this would substantially slow down CI, so we instead circumvent Brew and download pyenv directly. - - git clone https://github.com/pyenv/pyenv ${PYENV_ROOT} - - ${PYENV_ROOT}/bin/pyenv install 3.6.5 - - ${PYENV_ROOT}/bin/pyenv global 3.6.5 # ------------------------------------------------------------------------- # Build wheels @@ -393,14 +393,6 @@ py3_osx_10_12_sanity_check: &py3_osx_10_12_sanity_check env: - *py3_osx_test_config_env - CACHE_NAME=macos10.12sanity.py3 - # Install earlier version of Python 3.6 than other shards, due to older osx_image - before_install: - - curl -L https://github.com/stedolan/jq/releases/download/jq-1.5/jq-osx-amd64 -o /usr/local/bin/jq - - chmod 755 /usr/local/bin/jq - - ./build-support/bin/install_aws_cli_for_ci.sh - - *install_pyenv - - pyenv install 3.6.3 - - pyenv global 3.6.3 base_osx_10_13_sanity_check: &base_osx_10_13_sanity_check <<: *base_osx_sanity_check diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index cadd32004fc..ede0b34b734 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -156,13 +156,19 @@ py3_osx_config: &py3_osx_config - openssl env: # Fix Python 3 issue linking to OpenSSL - - &py3_osx_config_env PATH="/usr/local/opt/openssl/bin:$PATH" LDFLAGS="-L/usr/local/opt/openssl/lib" CPPFLAGS="-I/usr/local/opt/openssl/include" + - &py3_osx_config_env > + PATH="/usr/local/opt/openssl/bin:$PATH" + LDFLAGS="-L/usr/local/opt/openssl/lib" + CPPFLAGS="-I/usr/local/opt/openssl/include" + PYENV_ROOT="${HOME}/.pyenv" + PATH="${PYENV_ROOT}/shims:${PATH}" before_install: {{>before_install_osx}} - # Install pyenv the conventional brew way, instead of through addon, due to execution order - - &install_pyenv HOMEBREW_NO_AUTO_UPDATE=1 brew install pyenv - - pyenv install 3.6.5 - - pyenv global 3.6.5 + # Clone pyenv directly from GitHub. For multiple osx images, brew's version of pyenv is too old to get + # modern Python 3.6. Pulling from GitHub instead allows us to use the same Python 3 version accross every OSX shard. + - git clone https://github.com/pyenv/pyenv ${PYENV_ROOT} + - ${PYENV_ROOT}/bin/pyenv install 3.6.8 + - ${PYENV_ROOT}/bin/pyenv global 3.6.8 base_osx_test_config: &base_osx_test_config <<: *pants_run_cache_config @@ -183,8 +189,14 @@ py3_osx_test_config: &py3_osx_test_config <<: *base_osx_test_config stage: *test env: - # Must duplicate OpenSSL config from py3_osx_config's env because it cannot be merged into a new anchor - - &py3_osx_test_config_env PATH="/usr/local/opt/openssl/bin:$PATH" LDFLAGS="-L/usr/local/opt/openssl/lib" CPPFLAGS="-I/usr/local/opt/openssl/include" BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.osx + # Must duplicate py3_osx_config's env because it cannot be merged into a new anchor + - &py3_osx_test_config_env > + PATH="/usr/local/opt/openssl/bin:$PATH" + LDFLAGS="-L/usr/local/opt/openssl/lib" + CPPFLAGS="-I/usr/local/opt/openssl/include" + PYENV_ROOT="${HOME}/.pyenv" + PATH="${PYENV_ROOT}/shims:${PATH}" + BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.osx linux_with_fuse: &linux_with_fuse before_install: @@ -282,16 +294,6 @@ py3_osx_build_engine: &py3_osx_build_engine - CACHE_NAME=osxpexbuild.py3 - BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.osx - BOOTSTRAP_ARGS='' - - PYENV_ROOT="${HOME}/.pyenv" - - PATH="${PYENV_ROOT}/shims:${PATH}" - before_install: - {{>before_install_osx}} - # Clone pyenv directly from GitHub. Brew's version of pyenv on this osx_image is too old and only has - # a beta version of Python 3.6 that does not work properly. We could `brew update && brew upgrade pyenv`, - # but this would substantially slow down CI, so we instead circumvent Brew and download pyenv directly. - - git clone https://github.com/pyenv/pyenv ${PYENV_ROOT} - - ${PYENV_ROOT}/bin/pyenv install 3.6.5 - - ${PYENV_ROOT}/bin/pyenv global 3.6.5 # ------------------------------------------------------------------------- # Build wheels @@ -370,12 +372,6 @@ py3_osx_10_12_sanity_check: &py3_osx_10_12_sanity_check env: - *py3_osx_test_config_env - CACHE_NAME=macos10.12sanity.py3 - # Install earlier version of Python 3.6 than other shards, due to older osx_image - before_install: - {{>before_install_osx}} - - *install_pyenv - - pyenv install 3.6.3 - - pyenv global 3.6.3 base_osx_10_13_sanity_check: &base_osx_10_13_sanity_check <<: *base_osx_sanity_check From 6141260c96da5805afa79a8cd271471f5495bfc2 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Sat, 12 Jan 2019 19:14:20 -0700 Subject: [PATCH 069/105] Forgot -w flag for blacklist shards --- .travis.yml | 2 +- build-support/travis/travis.yml.mustache | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8f90cbb790c..5dede7c9222 100644 --- a/.travis.yml +++ b/.travis.yml @@ -785,7 +785,7 @@ matrix: - *py2_linux_test_config_env - CACHE_NAME=integrationshard0.py2blacklist script: - - ./build-support/bin/travis-ci.sh -bc2 -i 0/1 + - ./build-support/bin/travis-ci.sh -bc2w -i 0/1 - <<: *py2_linux_test_config name: "Integration tests for pants - shard 0 (Py2 PEX)" diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index ede0b34b734..f71f936127c 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -615,7 +615,7 @@ matrix: - *py2_linux_test_config_env - CACHE_NAME=integrationshard{{.}}.py2blacklist script: - - ./build-support/bin/travis-ci.sh -bc2 -i {{.}}/{{py2_blacklist_integration_shards_length}} + - ./build-support/bin/travis-ci.sh -bc2w -i {{.}}/{{py2_blacklist_integration_shards_length}} {{/py2_blacklist_integration_shards}} {{#cron_integration_shards}} From 9e3b4530e0211e701c44f970cd5e9f6d8759f317 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Sat, 12 Jan 2019 19:55:36 -0700 Subject: [PATCH 070/105] Add all failing targets to blacklist --- build-support/known_py3_pex_failures.txt | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/build-support/known_py3_pex_failures.txt b/build-support/known_py3_pex_failures.txt index e69de29bb2d..b9b50e65cb9 100644 --- a/build-support/known_py3_pex_failures.txt +++ b/build-support/known_py3_pex_failures.txt @@ -0,0 +1,20 @@ +tests/python/pants_test/backend/jvm/tasks/jvm_compile/java:zinc_compile_integration +tests/python/pants_test/backend/jvm/tasks/jvm_compile/rsc:rsc_compile_integration +tests/python/pants_test/backend/jvm/tasks/jvm_compile:declared_deps_integration +tests/python/pants_test/backend/jvm/tasks:classmap_integration +tests/python/pants_test/backend/jvm/tasks:jar_publish_integration +tests/python/pants_test/backend/jvm/tasks:jvm_platform_analysis_integration +tests/python/pants_test/backend/jvm/tasks:scala_repl_integration +tests/python/pants_test/backend/project_info/tasks:idea_plugin_integration +tests/python/pants_test/backend/python/tasks:isort_run_integration +tests/python/pants_test/backend/python/tasks:python_native_code_testing_1 +tests/python/pants_test/backend/python:integration +tests/python/pants_test/base:exception_sink_integration +tests/python/pants_test/base:exiter_integration +tests/python/pants_test/engine/legacy:console_rule_integration +tests/python/pants_test/engine/legacy:graph_integration +tests/python/pants_test/engine:scheduler_integration +tests/python/pants_test/option:options_integration +tests/python/pants_test/pantsd:pantsd_integration +tests/python/pants_test/projects:testprojects_integration +tests/python/pants_test/rules:test_integration From 3d01e8fdc3fc6d6e98a5281710ac28c1c2df7d3e Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Sat, 12 Jan 2019 20:05:08 -0700 Subject: [PATCH 071/105] Revert "Build wheels with Py2" This reverts commit 0a901e0c6297d192e70a0da49cea2ed576af1ecc. Contrib py3 is failing for the same reason now, and I don't want to skip contrib py3, because then we would be getting less test coverage than we currently do. Instead, I'll try to fix the setup_py.py unicode issue and land it as a separate PR. --- .travis.yml | 9 +++------ build-support/travis/travis.yml.mustache | 9 +++------ 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5dede7c9222..9d973d2f2fe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -564,14 +564,11 @@ matrix: - <<: *py2_osx_build_engine - <<: *py3_osx_build_engine - # TODO: the build wheels shards fail with Py3 pex due to unicode issues. - # Until fixing this, simply run shards with Py2. - <<: *py2_linux_build_wheels - stage: *test - # - <<: *py3_linux_build_wheels + - <<: *py3_linux_build_wheels + - <<: *py2_osx_build_wheels - stage: *test - # - <<: *py3_osx_build_wheels + - <<: *py3_osx_build_wheels - <<: *py2_osx_10_12_sanity_check - <<: *py3_osx_10_12_sanity_check diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index f71f936127c..28e265e6b86 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -543,14 +543,11 @@ matrix: - <<: *py2_osx_build_engine - <<: *py3_osx_build_engine - # TODO: the build wheels shards fail with Py3 pex due to unicode issues. - # Until fixing this, simply run shards with Py2. - <<: *py2_linux_build_wheels - stage: *test - # - <<: *py3_linux_build_wheels + - <<: *py3_linux_build_wheels + - <<: *py2_osx_build_wheels - stage: *test - # - <<: *py3_osx_build_wheels + - <<: *py3_osx_build_wheels - <<: *py2_osx_10_12_sanity_check - <<: *py3_osx_10_12_sanity_check From d91744bef66534ee0540b82ec61352d94db616e5 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Sat, 12 Jan 2019 20:15:37 -0700 Subject: [PATCH 072/105] Fix setup_py.py unicode issue We must always write to the outstream with bytes, because `rwbuf` uses bytes and we decided to try to unify our treatment of outstreams (e.g. use sys.stdout in Py2 and equivalent sys.stdout.buffer in Py3). This issue was causing the build wheels jobs to fail and the contrib tests to fail. This will be broken out into a separate PR, but is being tested here that it works. --- src/python/pants/backend/python/tasks/setup_py.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/python/pants/backend/python/tasks/setup_py.py b/src/python/pants/backend/python/tasks/setup_py.py index d84acee9a84..169663457a1 100644 --- a/src/python/pants/backend/python/tasks/setup_py.py +++ b/src/python/pants/backend/python/tasks/setup_py.py @@ -14,7 +14,6 @@ from builtins import bytes, map, object, str, zip from collections import defaultdict -from future.utils import PY2 from pex.installer import InstallerBase, Packager from pex.interpreter import PythonInterpreter from pex.pex import PEX @@ -501,7 +500,7 @@ def filter_namespace_packages(self, root_target, inits): # and then writing it out after the process has finished like we do here. def write(stream_name, data): stream = workunit.output(stream_name) - stream.write(ensure_binary(data) if PY2 else ensure_text(data)) + stream.write(ensure_binary(data)) stream.flush() write('stdout', stdout) From a3c73ae938c890092e7ee2081573eb51e38dce2a Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Sat, 12 Jan 2019 20:20:40 -0700 Subject: [PATCH 073/105] Use py2 pex for lint shard --- .travis.yml | 5 ++++- build-support/travis/travis.yml.mustache | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9d973d2f2fe..4ad599f8d3a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -580,7 +580,10 @@ matrix: - <<: *deploy_unstable_multiplatform_pex - <<: *py2_lint - - <<: *py3_lint + stage: *test + # TODO: lint shard fails with Py3 pex (see https://travis-ci.org/pantsbuild/pants/jobs/478884818#L3355) + # Temporarily use Py2 pex and fix this in a later PR. + # - <<: *py3_lint - <<: *py2_linux_test_config name: "Unit tests for pants and pants-plugins (Py2 PEX)" diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index 28e265e6b86..2d2cbb855bf 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -559,7 +559,10 @@ matrix: - <<: *deploy_unstable_multiplatform_pex - <<: *py2_lint - - <<: *py3_lint + stage: *test + # TODO: lint shard fails with Py3 pex (see https://travis-ci.org/pantsbuild/pants/jobs/478884818#L3355) + # Temporarily use Py2 pex and fix this in a later PR. + # - <<: *py3_lint - <<: *py2_linux_test_config name: "Unit tests for pants and pants-plugins (Py2 PEX)" From 5f878905acf046459314a777add311adf3481e08 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Sat, 12 Jan 2019 20:22:25 -0700 Subject: [PATCH 074/105] Redistribute blacklist shards --- .travis.yml | 70 +++++++++------------ build-support/travis/generate_travis_yml.py | 4 +- 2 files changed, 33 insertions(+), 41 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4ad599f8d3a..4796647f1fe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -625,7 +625,7 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard0 script: - - ./build-support/bin/travis-ci.sh -bc -i 0/20 + - ./build-support/bin/travis-ci.sh -bc -i 0/16 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 1 (Py3 PEX)" @@ -633,7 +633,7 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard1 script: - - ./build-support/bin/travis-ci.sh -bc -i 1/20 + - ./build-support/bin/travis-ci.sh -bc -i 1/16 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 2 (Py3 PEX)" @@ -641,7 +641,7 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard2 script: - - ./build-support/bin/travis-ci.sh -bc -i 2/20 + - ./build-support/bin/travis-ci.sh -bc -i 2/16 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 3 (Py3 PEX)" @@ -649,7 +649,7 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard3 script: - - ./build-support/bin/travis-ci.sh -bc -i 3/20 + - ./build-support/bin/travis-ci.sh -bc -i 3/16 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 4 (Py3 PEX)" @@ -657,7 +657,7 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard4 script: - - ./build-support/bin/travis-ci.sh -bc -i 4/20 + - ./build-support/bin/travis-ci.sh -bc -i 4/16 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 5 (Py3 PEX)" @@ -665,7 +665,7 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard5 script: - - ./build-support/bin/travis-ci.sh -bc -i 5/20 + - ./build-support/bin/travis-ci.sh -bc -i 5/16 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 6 (Py3 PEX)" @@ -673,7 +673,7 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard6 script: - - ./build-support/bin/travis-ci.sh -bc -i 6/20 + - ./build-support/bin/travis-ci.sh -bc -i 6/16 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 7 (Py3 PEX)" @@ -681,7 +681,7 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard7 script: - - ./build-support/bin/travis-ci.sh -bc -i 7/20 + - ./build-support/bin/travis-ci.sh -bc -i 7/16 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 8 (Py3 PEX)" @@ -689,7 +689,7 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard8 script: - - ./build-support/bin/travis-ci.sh -bc -i 8/20 + - ./build-support/bin/travis-ci.sh -bc -i 8/16 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 9 (Py3 PEX)" @@ -697,7 +697,7 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard9 script: - - ./build-support/bin/travis-ci.sh -bc -i 9/20 + - ./build-support/bin/travis-ci.sh -bc -i 9/16 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 10 (Py3 PEX)" @@ -705,7 +705,7 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard10 script: - - ./build-support/bin/travis-ci.sh -bc -i 10/20 + - ./build-support/bin/travis-ci.sh -bc -i 10/16 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 11 (Py3 PEX)" @@ -713,7 +713,7 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard11 script: - - ./build-support/bin/travis-ci.sh -bc -i 11/20 + - ./build-support/bin/travis-ci.sh -bc -i 11/16 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 12 (Py3 PEX)" @@ -721,7 +721,7 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard12 script: - - ./build-support/bin/travis-ci.sh -bc -i 12/20 + - ./build-support/bin/travis-ci.sh -bc -i 12/16 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 13 (Py3 PEX)" @@ -729,7 +729,7 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard13 script: - - ./build-support/bin/travis-ci.sh -bc -i 13/20 + - ./build-support/bin/travis-ci.sh -bc -i 13/16 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 14 (Py3 PEX)" @@ -737,7 +737,7 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard14 script: - - ./build-support/bin/travis-ci.sh -bc -i 14/20 + - ./build-support/bin/travis-ci.sh -bc -i 14/16 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 15 (Py3 PEX)" @@ -745,47 +745,39 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard15 script: - - ./build-support/bin/travis-ci.sh -bc -i 15/20 + - ./build-support/bin/travis-ci.sh -bc -i 15/16 - <<: *py3_linux_test_config - name: "Integration tests for pants - shard 16 (Py3 PEX)" - env: - - *py3_linux_test_config_env - - CACHE_NAME=integrationshard16 - script: - - ./build-support/bin/travis-ci.sh -bc -i 16/20 - - - <<: *py3_linux_test_config - name: "Integration tests for pants - shard 17 (Py3 PEX)" + name: "Blacklisted integration tests for pants - shard 0 (Py2 PEX)" env: - - *py3_linux_test_config_env - - CACHE_NAME=integrationshard17 + - *py2_linux_test_config_env + - CACHE_NAME=integrationshard0.py2blacklist script: - - ./build-support/bin/travis-ci.sh -bc -i 17/20 + - ./build-support/bin/travis-ci.sh -bc2w -i 0/4 - <<: *py3_linux_test_config - name: "Integration tests for pants - shard 18 (Py3 PEX)" + name: "Blacklisted integration tests for pants - shard 1 (Py2 PEX)" env: - - *py3_linux_test_config_env - - CACHE_NAME=integrationshard18 + - *py2_linux_test_config_env + - CACHE_NAME=integrationshard1.py2blacklist script: - - ./build-support/bin/travis-ci.sh -bc -i 18/20 + - ./build-support/bin/travis-ci.sh -bc2w -i 1/4 - <<: *py3_linux_test_config - name: "Integration tests for pants - shard 19 (Py3 PEX)" + name: "Blacklisted integration tests for pants - shard 2 (Py2 PEX)" env: - - *py3_linux_test_config_env - - CACHE_NAME=integrationshard19 + - *py2_linux_test_config_env + - CACHE_NAME=integrationshard2.py2blacklist script: - - ./build-support/bin/travis-ci.sh -bc -i 19/20 + - ./build-support/bin/travis-ci.sh -bc2w -i 2/4 - <<: *py3_linux_test_config - name: "Blacklisted integration tests for pants - shard 0 (Py2 PEX)" + name: "Blacklisted integration tests for pants - shard 3 (Py2 PEX)" env: - *py2_linux_test_config_env - - CACHE_NAME=integrationshard0.py2blacklist + - CACHE_NAME=integrationshard3.py2blacklist script: - - ./build-support/bin/travis-ci.sh -bc2w -i 0/1 + - ./build-support/bin/travis-ci.sh -bc2w -i 3/4 - <<: *py2_linux_test_config name: "Integration tests for pants - shard 0 (Py2 PEX)" diff --git a/build-support/travis/generate_travis_yml.py b/build-support/travis/generate_travis_yml.py index d3888bdff34..eac486b8c37 100644 --- a/build-support/travis/generate_travis_yml.py +++ b/build-support/travis/generate_travis_yml.py @@ -8,8 +8,8 @@ import pystache -num_py3_integration_shards = 20 -num_py2_blacklist_integration_shards = 1 +num_py3_integration_shards = 16 +num_py2_blacklist_integration_shards = 4 num_cron_integration_shards = 20 From b3d030e223943797f4601de1a60134aaf5c83dd3 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Sat, 12 Jan 2019 22:58:28 -0700 Subject: [PATCH 075/105] Add 3 missed targets to blacklist --- build-support/known_py3_pex_failures.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build-support/known_py3_pex_failures.txt b/build-support/known_py3_pex_failures.txt index b9b50e65cb9..14c68e92337 100644 --- a/build-support/known_py3_pex_failures.txt +++ b/build-support/known_py3_pex_failures.txt @@ -6,13 +6,16 @@ tests/python/pants_test/backend/jvm/tasks:jar_publish_integration tests/python/pants_test/backend/jvm/tasks:jvm_platform_analysis_integration tests/python/pants_test/backend/jvm/tasks:scala_repl_integration tests/python/pants_test/backend/project_info/tasks:idea_plugin_integration +tests/python/pants_test/backend/python/tasks:integration tests/python/pants_test/backend/python/tasks:isort_run_integration tests/python/pants_test/backend/python/tasks:python_native_code_testing_1 tests/python/pants_test/backend/python:integration tests/python/pants_test/base:exception_sink_integration tests/python/pants_test/base:exiter_integration +tests/python/pants_test/engine/legacy:changed_integration tests/python/pants_test/engine/legacy:console_rule_integration tests/python/pants_test/engine/legacy:graph_integration +tests/python/pants_test/engine/legacy:owners_integration tests/python/pants_test/engine:scheduler_integration tests/python/pants_test/option:options_integration tests/python/pants_test/pantsd:pantsd_integration From 59efa23f042dfdd86c3714ed3404ebc5d8bf2923 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Sat, 12 Jan 2019 23:05:18 -0700 Subject: [PATCH 076/105] Use py2 for wheel building shards again Realized we can't use py3 until we add py3 support to release.sh, which we are not ready to do yet. --- .travis.yml | 9 +++++++-- build-support/travis/travis.yml.mustache | 9 +++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4796647f1fe..0287bae5bd6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -564,11 +564,16 @@ matrix: - <<: *py2_osx_build_engine - <<: *py3_osx_build_engine + # TODO(6450): add support for Py3 in the release script. This can't be done until + # Pants is ready to be built and released as a Py3 wheel. - <<: *py2_linux_build_wheels - - <<: *py3_linux_build_wheels + stage: *test + # - <<: *py3_linux_build_wheels + # TODO(6450): see above about adding support for Py3 in release script. - <<: *py2_osx_build_wheels - - <<: *py3_osx_build_wheels + stage: *test + # - <<: *py3_osx_build_wheels - <<: *py2_osx_10_12_sanity_check - <<: *py3_osx_10_12_sanity_check diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index 2d2cbb855bf..729ff5b0d4a 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -543,11 +543,16 @@ matrix: - <<: *py2_osx_build_engine - <<: *py3_osx_build_engine + # TODO(6450): add support for Py3 in the release script. This can't be done until + # Pants is ready to be built and released as a Py3 wheel. - <<: *py2_linux_build_wheels - - <<: *py3_linux_build_wheels + stage: *test + # - <<: *py3_linux_build_wheels + # TODO(6450): see above about adding support for Py3 in release script. - <<: *py2_osx_build_wheels - - <<: *py3_osx_build_wheels + stage: *test + # - <<: *py3_osx_build_wheels - <<: *py2_osx_10_12_sanity_check - <<: *py3_osx_10_12_sanity_check From 5c8de8e5b61a94739b9207ce7793820715a58b2a Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Sat, 12 Jan 2019 23:14:58 -0700 Subject: [PATCH 077/105] Use py2 pex w/ py3 interpreter constraints for contrib py3 I can't get Py3 to be green, and would rather punt on it. So, use py2 pex and constraint subprocesses to py3, which is identical to our setup on master. --- .travis.yml | 12 ++++++++---- build-support/travis/travis.yml.mustache | 12 ++++++++---- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0287bae5bd6..894a34fa695 100644 --- a/.travis.yml +++ b/.travis.yml @@ -616,13 +616,15 @@ matrix: script: - ./build-support/bin/travis-ci.sh -2bn + # TODO: get this passing with a Python 3 PEX. - <<: *py3_linux_test_config - name: "Python contrib tests (Py3 PEX)" + name: "Python contrib tests (Py2 PEX w/ Py3 constraints)" env: - - *py3_linux_test_config_env + - *py2_linux_test_config_env + - PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="['CPython>=3.6']" - CACHE_NAME=linuxcontribtests.py3 script: - - ./build-support/bin/travis-ci.sh -bn + - ./build-support/bin/travis-ci.sh -bn2 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 0 (Py3 PEX)" @@ -968,8 +970,10 @@ matrix: - <<: *py2_linux_rust_tests - <<: *py3_linux_rust_tests + # TODO: get this to pass using Python 3. In the meantime, just use Python 2. - <<: *py2_osx_rust_tests - - <<: *py3_osx_rust_tests + stage: *test + # - <<: *py3_osx_rust_tests - <<: *linux_with_fuse <<: *native_engine_cache_config diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index 729ff5b0d4a..bc219c6ac6b 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -595,13 +595,15 @@ matrix: script: - ./build-support/bin/travis-ci.sh -2bn + # TODO: get this passing with a Python 3 PEX. - <<: *py3_linux_test_config - name: "Python contrib tests (Py3 PEX)" + name: "Python contrib tests (Py2 PEX w/ Py3 constraints)" env: - - *py3_linux_test_config_env + - *py2_linux_test_config_env + - PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="['CPython>=3.6']" - CACHE_NAME=linuxcontribtests.py3 script: - - ./build-support/bin/travis-ci.sh -bn + - ./build-support/bin/travis-ci.sh -bn2 {{#py3_integration_shards}} - <<: *py3_linux_test_config @@ -638,8 +640,10 @@ matrix: - <<: *py2_linux_rust_tests - <<: *py3_linux_rust_tests + # TODO: get this to pass using Python 3. In the meantime, just use Python 2. - <<: *py2_osx_rust_tests - - <<: *py3_osx_rust_tests + stage: *test + # - <<: *py3_osx_rust_tests - <<: *linux_with_fuse <<: *native_engine_cache_config From 0d43d594c74a7d27ca5bd952f8cf2cb9fcde6a1d Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Sat, 12 Jan 2019 23:16:53 -0700 Subject: [PATCH 078/105] Redistribute blacklist shards to favor blacklist --- .travis.yml | 74 ++++++++++----------- build-support/travis/generate_travis_yml.py | 4 +- 2 files changed, 39 insertions(+), 39 deletions(-) diff --git a/.travis.yml b/.travis.yml index 894a34fa695..eae81b3efc7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -632,7 +632,7 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard0 script: - - ./build-support/bin/travis-ci.sh -bc -i 0/16 + - ./build-support/bin/travis-ci.sh -bc -i 0/13 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 1 (Py3 PEX)" @@ -640,7 +640,7 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard1 script: - - ./build-support/bin/travis-ci.sh -bc -i 1/16 + - ./build-support/bin/travis-ci.sh -bc -i 1/13 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 2 (Py3 PEX)" @@ -648,7 +648,7 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard2 script: - - ./build-support/bin/travis-ci.sh -bc -i 2/16 + - ./build-support/bin/travis-ci.sh -bc -i 2/13 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 3 (Py3 PEX)" @@ -656,7 +656,7 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard3 script: - - ./build-support/bin/travis-ci.sh -bc -i 3/16 + - ./build-support/bin/travis-ci.sh -bc -i 3/13 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 4 (Py3 PEX)" @@ -664,7 +664,7 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard4 script: - - ./build-support/bin/travis-ci.sh -bc -i 4/16 + - ./build-support/bin/travis-ci.sh -bc -i 4/13 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 5 (Py3 PEX)" @@ -672,7 +672,7 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard5 script: - - ./build-support/bin/travis-ci.sh -bc -i 5/16 + - ./build-support/bin/travis-ci.sh -bc -i 5/13 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 6 (Py3 PEX)" @@ -680,7 +680,7 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard6 script: - - ./build-support/bin/travis-ci.sh -bc -i 6/16 + - ./build-support/bin/travis-ci.sh -bc -i 6/13 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 7 (Py3 PEX)" @@ -688,7 +688,7 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard7 script: - - ./build-support/bin/travis-ci.sh -bc -i 7/16 + - ./build-support/bin/travis-ci.sh -bc -i 7/13 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 8 (Py3 PEX)" @@ -696,7 +696,7 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard8 script: - - ./build-support/bin/travis-ci.sh -bc -i 8/16 + - ./build-support/bin/travis-ci.sh -bc -i 8/13 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 9 (Py3 PEX)" @@ -704,7 +704,7 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard9 script: - - ./build-support/bin/travis-ci.sh -bc -i 9/16 + - ./build-support/bin/travis-ci.sh -bc -i 9/13 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 10 (Py3 PEX)" @@ -712,7 +712,7 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard10 script: - - ./build-support/bin/travis-ci.sh -bc -i 10/16 + - ./build-support/bin/travis-ci.sh -bc -i 10/13 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 11 (Py3 PEX)" @@ -720,7 +720,7 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard11 script: - - ./build-support/bin/travis-ci.sh -bc -i 11/16 + - ./build-support/bin/travis-ci.sh -bc -i 11/13 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 12 (Py3 PEX)" @@ -728,63 +728,63 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard12 script: - - ./build-support/bin/travis-ci.sh -bc -i 12/16 + - ./build-support/bin/travis-ci.sh -bc -i 12/13 - <<: *py3_linux_test_config - name: "Integration tests for pants - shard 13 (Py3 PEX)" + name: "Blacklisted integration tests for pants - shard 0 (Py2 PEX)" env: - - *py3_linux_test_config_env - - CACHE_NAME=integrationshard13 + - *py2_linux_test_config_env + - CACHE_NAME=integrationshard0.py2blacklist script: - - ./build-support/bin/travis-ci.sh -bc -i 13/16 + - ./build-support/bin/travis-ci.sh -bc2w -i 0/7 - <<: *py3_linux_test_config - name: "Integration tests for pants - shard 14 (Py3 PEX)" + name: "Blacklisted integration tests for pants - shard 1 (Py2 PEX)" env: - - *py3_linux_test_config_env - - CACHE_NAME=integrationshard14 + - *py2_linux_test_config_env + - CACHE_NAME=integrationshard1.py2blacklist script: - - ./build-support/bin/travis-ci.sh -bc -i 14/16 + - ./build-support/bin/travis-ci.sh -bc2w -i 1/7 - <<: *py3_linux_test_config - name: "Integration tests for pants - shard 15 (Py3 PEX)" + name: "Blacklisted integration tests for pants - shard 2 (Py2 PEX)" env: - - *py3_linux_test_config_env - - CACHE_NAME=integrationshard15 + - *py2_linux_test_config_env + - CACHE_NAME=integrationshard2.py2blacklist script: - - ./build-support/bin/travis-ci.sh -bc -i 15/16 + - ./build-support/bin/travis-ci.sh -bc2w -i 2/7 - <<: *py3_linux_test_config - name: "Blacklisted integration tests for pants - shard 0 (Py2 PEX)" + name: "Blacklisted integration tests for pants - shard 3 (Py2 PEX)" env: - *py2_linux_test_config_env - - CACHE_NAME=integrationshard0.py2blacklist + - CACHE_NAME=integrationshard3.py2blacklist script: - - ./build-support/bin/travis-ci.sh -bc2w -i 0/4 + - ./build-support/bin/travis-ci.sh -bc2w -i 3/7 - <<: *py3_linux_test_config - name: "Blacklisted integration tests for pants - shard 1 (Py2 PEX)" + name: "Blacklisted integration tests for pants - shard 4 (Py2 PEX)" env: - *py2_linux_test_config_env - - CACHE_NAME=integrationshard1.py2blacklist + - CACHE_NAME=integrationshard4.py2blacklist script: - - ./build-support/bin/travis-ci.sh -bc2w -i 1/4 + - ./build-support/bin/travis-ci.sh -bc2w -i 4/7 - <<: *py3_linux_test_config - name: "Blacklisted integration tests for pants - shard 2 (Py2 PEX)" + name: "Blacklisted integration tests for pants - shard 5 (Py2 PEX)" env: - *py2_linux_test_config_env - - CACHE_NAME=integrationshard2.py2blacklist + - CACHE_NAME=integrationshard5.py2blacklist script: - - ./build-support/bin/travis-ci.sh -bc2w -i 2/4 + - ./build-support/bin/travis-ci.sh -bc2w -i 5/7 - <<: *py3_linux_test_config - name: "Blacklisted integration tests for pants - shard 3 (Py2 PEX)" + name: "Blacklisted integration tests for pants - shard 6 (Py2 PEX)" env: - *py2_linux_test_config_env - - CACHE_NAME=integrationshard3.py2blacklist + - CACHE_NAME=integrationshard6.py2blacklist script: - - ./build-support/bin/travis-ci.sh -bc2w -i 3/4 + - ./build-support/bin/travis-ci.sh -bc2w -i 6/7 - <<: *py2_linux_test_config name: "Integration tests for pants - shard 0 (Py2 PEX)" diff --git a/build-support/travis/generate_travis_yml.py b/build-support/travis/generate_travis_yml.py index eac486b8c37..08e12ddac74 100644 --- a/build-support/travis/generate_travis_yml.py +++ b/build-support/travis/generate_travis_yml.py @@ -8,8 +8,8 @@ import pystache -num_py3_integration_shards = 16 -num_py2_blacklist_integration_shards = 4 +num_py3_integration_shards = 13 +num_py2_blacklist_integration_shards = 7 num_cron_integration_shards = 20 From 3b3a107ef540df7a0d73b6a0c5784d3795b64db4 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Sun, 13 Jan 2019 08:19:04 -0700 Subject: [PATCH 079/105] Revert contrib fixes. Broken out into #7067. --- .../python/pants/contrib/confluence/util/confluence_util.py | 2 +- .../pants/contrib/go/tasks/go_binary_fingerprint_strategy.py | 2 +- .../scrooge/tasks/java_thrift_library_fingerprint_strategy.py | 2 +- src/python/pants/backend/python/tasks/setup_py.py | 3 ++- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/contrib/confluence/src/python/pants/contrib/confluence/util/confluence_util.py b/contrib/confluence/src/python/pants/contrib/confluence/util/confluence_util.py index 1d40c573fef..fa35978048d 100644 --- a/contrib/confluence/src/python/pants/contrib/confluence/util/confluence_util.py +++ b/contrib/confluence/src/python/pants/contrib/confluence/util/confluence_util.py @@ -171,7 +171,7 @@ def addattachment(self, page, filename): except (IOError, OSError) as e: log.error('Failed to read data from file %s: %s' % (filename, str(e))) return None - except XMLRPCError: + except XMLRPCError as e: log.error('Failed to add file attachment %s to page: %s' % (filename, page.get('title', '[unknown title]'))) return None diff --git a/contrib/go/src/python/pants/contrib/go/tasks/go_binary_fingerprint_strategy.py b/contrib/go/src/python/pants/contrib/go/tasks/go_binary_fingerprint_strategy.py index 56cc75129a3..4cb3a0bbb17 100644 --- a/contrib/go/src/python/pants/contrib/go/tasks/go_binary_fingerprint_strategy.py +++ b/contrib/go/src/python/pants/contrib/go/tasks/go_binary_fingerprint_strategy.py @@ -32,7 +32,7 @@ def compute_fingerprint(self, target): return fp hasher = hashlib.sha1() - hasher.update(fp.encode('utf-8')) + hasher.update(fp) hasher.update(str(self._get_build_flags_func(target)).encode('utf-8')) return hasher.hexdigest() if PY3 else hasher.hexdigest().decode('utf-8') diff --git a/contrib/scrooge/src/python/pants/contrib/scrooge/tasks/java_thrift_library_fingerprint_strategy.py b/contrib/scrooge/src/python/pants/contrib/scrooge/tasks/java_thrift_library_fingerprint_strategy.py index d78b20435e0..de5c5ff3f61 100644 --- a/contrib/scrooge/src/python/pants/contrib/scrooge/tasks/java_thrift_library_fingerprint_strategy.py +++ b/contrib/scrooge/src/python/pants/contrib/scrooge/tasks/java_thrift_library_fingerprint_strategy.py @@ -39,7 +39,7 @@ def compute_fingerprint(self, target): default_java_namespace = self._thrift_defaults.default_java_namespace(target) if default_java_namespace: - hasher.update(default_java_namespace.encode('utf-8')) + hasher.update(default_java_namespace) if target.include_paths: hasher.update(str(target.include_paths).encode('utf-8')) diff --git a/src/python/pants/backend/python/tasks/setup_py.py b/src/python/pants/backend/python/tasks/setup_py.py index 169663457a1..d84acee9a84 100644 --- a/src/python/pants/backend/python/tasks/setup_py.py +++ b/src/python/pants/backend/python/tasks/setup_py.py @@ -14,6 +14,7 @@ from builtins import bytes, map, object, str, zip from collections import defaultdict +from future.utils import PY2 from pex.installer import InstallerBase, Packager from pex.interpreter import PythonInterpreter from pex.pex import PEX @@ -500,7 +501,7 @@ def filter_namespace_packages(self, root_target, inits): # and then writing it out after the process has finished like we do here. def write(stream_name, data): stream = workunit.output(stream_name) - stream.write(ensure_binary(data)) + stream.write(ensure_binary(data) if PY2 else ensure_text(data)) stream.flush() write('stdout', stdout) From e0bb83e09b15f1f87401c99b69216a1792f25f86 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Sun, 13 Jan 2019 08:21:37 -0700 Subject: [PATCH 080/105] Constrain blacklist shards interpreter to Py3 The underlying PEX is Py2, but subprocesses Py3. This is exactly how we have CI on master setup. Without this commit, we'd be losing some of the Py3 compatibility CI gains we had made before. The cron job will run purely Py2 pex with Py2 subprocesses to check for regressions. --- .travis.yml | 7 +++++++ build-support/travis/travis.yml.mustache | 1 + 2 files changed, 8 insertions(+) diff --git a/.travis.yml b/.travis.yml index eae81b3efc7..7079cb225bb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -734,6 +734,7 @@ matrix: name: "Blacklisted integration tests for pants - shard 0 (Py2 PEX)" env: - *py2_linux_test_config_env + - PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="['CPython>=3.6']" - CACHE_NAME=integrationshard0.py2blacklist script: - ./build-support/bin/travis-ci.sh -bc2w -i 0/7 @@ -742,6 +743,7 @@ matrix: name: "Blacklisted integration tests for pants - shard 1 (Py2 PEX)" env: - *py2_linux_test_config_env + - PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="['CPython>=3.6']" - CACHE_NAME=integrationshard1.py2blacklist script: - ./build-support/bin/travis-ci.sh -bc2w -i 1/7 @@ -750,6 +752,7 @@ matrix: name: "Blacklisted integration tests for pants - shard 2 (Py2 PEX)" env: - *py2_linux_test_config_env + - PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="['CPython>=3.6']" - CACHE_NAME=integrationshard2.py2blacklist script: - ./build-support/bin/travis-ci.sh -bc2w -i 2/7 @@ -758,6 +761,7 @@ matrix: name: "Blacklisted integration tests for pants - shard 3 (Py2 PEX)" env: - *py2_linux_test_config_env + - PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="['CPython>=3.6']" - CACHE_NAME=integrationshard3.py2blacklist script: - ./build-support/bin/travis-ci.sh -bc2w -i 3/7 @@ -766,6 +770,7 @@ matrix: name: "Blacklisted integration tests for pants - shard 4 (Py2 PEX)" env: - *py2_linux_test_config_env + - PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="['CPython>=3.6']" - CACHE_NAME=integrationshard4.py2blacklist script: - ./build-support/bin/travis-ci.sh -bc2w -i 4/7 @@ -774,6 +779,7 @@ matrix: name: "Blacklisted integration tests for pants - shard 5 (Py2 PEX)" env: - *py2_linux_test_config_env + - PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="['CPython>=3.6']" - CACHE_NAME=integrationshard5.py2blacklist script: - ./build-support/bin/travis-ci.sh -bc2w -i 5/7 @@ -782,6 +788,7 @@ matrix: name: "Blacklisted integration tests for pants - shard 6 (Py2 PEX)" env: - *py2_linux_test_config_env + - PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="['CPython>=3.6']" - CACHE_NAME=integrationshard6.py2blacklist script: - ./build-support/bin/travis-ci.sh -bc2w -i 6/7 diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index bc219c6ac6b..5b10047d84f 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -620,6 +620,7 @@ matrix: name: "Blacklisted integration tests for pants - shard {{.}} (Py2 PEX)" env: - *py2_linux_test_config_env + - PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="['CPython>=3.6']" - CACHE_NAME=integrationshard{{.}}.py2blacklist script: - ./build-support/bin/travis-ci.sh -bc2w -i {{.}}/{{py2_blacklist_integration_shards_length}} From f6c4d55c512e9c0f787c07d9ff2589d4b0e3cc82 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Sun, 13 Jan 2019 08:38:47 -0700 Subject: [PATCH 081/105] Deduplicate py3_osx_config via new mustache file Thanks Benjy for making this possible with your mustache changes! This is much cleaner. --- .travis.yml | 26 +++++++++------------ build-support/travis/generate_travis_yml.py | 18 ++++++++------ build-support/travis/py3_osx_env.mustache | 5 ++++ build-support/travis/travis.yml.mustache | 18 +++----------- 4 files changed, 30 insertions(+), 37 deletions(-) create mode 100644 build-support/travis/py3_osx_env.mustache diff --git a/.travis.yml b/.travis.yml index 7079cb225bb..342d9e7a4ba 100644 --- a/.travis.yml +++ b/.travis.yml @@ -169,13 +169,11 @@ py3_osx_config: &py3_osx_config packages: &py3_osx_config_brew_packages - openssl env: - # Fix Python 3 issue linking to OpenSSL - - &py3_osx_config_env > - PATH="/usr/local/opt/openssl/bin:$PATH" - LDFLAGS="-L/usr/local/opt/openssl/lib" - CPPFLAGS="-I/usr/local/opt/openssl/include" - PYENV_ROOT="${HOME}/.pyenv" - PATH="${PYENV_ROOT}/shims:${PATH}" + - PATH="/usr/local/opt/openssl/bin:$PATH" + - LDFLAGS="-L/usr/local/opt/openssl/lib" + - CPPFLAGS="-I/usr/local/opt/openssl/include" + - PYENV_ROOT="${HOME}/.pyenv" + - PATH="${PYENV_ROOT}/shims:${PATH}" before_install: - curl -L https://github.com/stedolan/jq/releases/download/jq-1.5/jq-osx-amd64 -o /usr/local/bin/jq - chmod 755 /usr/local/bin/jq @@ -205,14 +203,12 @@ py3_osx_test_config: &py3_osx_test_config <<: *base_osx_test_config stage: *test env: - # Must duplicate py3_osx_config's env because it cannot be merged into a new anchor - - &py3_osx_test_config_env > - PATH="/usr/local/opt/openssl/bin:$PATH" - LDFLAGS="-L/usr/local/opt/openssl/lib" - CPPFLAGS="-I/usr/local/opt/openssl/include" - PYENV_ROOT="${HOME}/.pyenv" - PATH="${PYENV_ROOT}/shims:${PATH}" - BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.osx + - PATH="/usr/local/opt/openssl/bin:$PATH" + - LDFLAGS="-L/usr/local/opt/openssl/lib" + - CPPFLAGS="-I/usr/local/opt/openssl/include" + - PYENV_ROOT="${HOME}/.pyenv" + - PATH="${PYENV_ROOT}/shims:${PATH}" + - BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.osx linux_with_fuse: &linux_with_fuse before_install: diff --git a/build-support/travis/generate_travis_yml.py b/build-support/travis/generate_travis_yml.py index 08e12ddac74..8b62786b3aa 100644 --- a/build-support/travis/generate_travis_yml.py +++ b/build-support/travis/generate_travis_yml.py @@ -25,12 +25,15 @@ def generate_travis_yml(): """Generates content for a .travis.yml file from templates.""" - template = pkg_resources.resource_string( - __name__, 'travis.yml.mustache').decode('utf-8') - before_install_linux = pkg_resources.resource_string( - __name__, 'before_install_linux.mustache').decode('utf-8') - before_install_osx = pkg_resources.resource_string( - __name__, 'before_install_osx.mustache').decode('utf-8') + + def get_mustache_resource(file_name): + return pkg_resources.resource_string(__name__, "{}.mustache".format(file_name)).decode('utf-8') + + template = get_mustache_resource('travis.yml') + before_install_linux = get_mustache_resource('before_install_linux') + before_install_osx = get_mustache_resource('before_install_osx') + py3_osx_env = get_mustache_resource('py3_osx_env') + context = { 'header': HEADER, 'py3_integration_shards': range(0, num_py3_integration_shards), @@ -42,6 +45,7 @@ def generate_travis_yml(): } renderer = pystache.Renderer(partials={ 'before_install_linux': before_install_linux, - 'before_install_osx': before_install_osx + 'before_install_osx': before_install_osx, + 'py3_osx_env': py3_osx_env }) print(renderer.render(template, context)) diff --git a/build-support/travis/py3_osx_env.mustache b/build-support/travis/py3_osx_env.mustache new file mode 100644 index 00000000000..66585940b18 --- /dev/null +++ b/build-support/travis/py3_osx_env.mustache @@ -0,0 +1,5 @@ +- PATH="/usr/local/opt/openssl/bin:$PATH" +- LDFLAGS="-L/usr/local/opt/openssl/lib" +- CPPFLAGS="-I/usr/local/opt/openssl/include" +- PYENV_ROOT="${HOME}/.pyenv" +- PATH="${PYENV_ROOT}/shims:${PATH}" diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index 5b10047d84f..3e6d27c8ab5 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -155,13 +155,7 @@ py3_osx_config: &py3_osx_config packages: &py3_osx_config_brew_packages - openssl env: - # Fix Python 3 issue linking to OpenSSL - - &py3_osx_config_env > - PATH="/usr/local/opt/openssl/bin:$PATH" - LDFLAGS="-L/usr/local/opt/openssl/lib" - CPPFLAGS="-I/usr/local/opt/openssl/include" - PYENV_ROOT="${HOME}/.pyenv" - PATH="${PYENV_ROOT}/shims:${PATH}" + {{>py3_osx_env}} before_install: {{>before_install_osx}} # Clone pyenv directly from GitHub. For multiple osx images, brew's version of pyenv is too old to get @@ -189,14 +183,8 @@ py3_osx_test_config: &py3_osx_test_config <<: *base_osx_test_config stage: *test env: - # Must duplicate py3_osx_config's env because it cannot be merged into a new anchor - - &py3_osx_test_config_env > - PATH="/usr/local/opt/openssl/bin:$PATH" - LDFLAGS="-L/usr/local/opt/openssl/lib" - CPPFLAGS="-I/usr/local/opt/openssl/include" - PYENV_ROOT="${HOME}/.pyenv" - PATH="${PYENV_ROOT}/shims:${PATH}" - BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.osx + {{>py3_osx_env}} + - BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.osx linux_with_fuse: &linux_with_fuse before_install: From 73ec683205a16d4e136ffca7252210c5101d23f9 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Sun, 13 Jan 2019 08:43:21 -0700 Subject: [PATCH 082/105] Revert "Deduplicate py3_osx_config via new mustache file" Sad, actually we can't deduplicate this :( The reason is that multiple records extend py3_osx_config and py3_osx_test_config. We need some way for osx_test_config to refer to one single entry in the list, as it's impossible to merge lists, so we do need that to all be on one line. --- .travis.yml | 26 ++++++++++++--------- build-support/travis/generate_travis_yml.py | 18 ++++++-------- build-support/travis/py3_osx_env.mustache | 5 ---- build-support/travis/travis.yml.mustache | 18 +++++++++++--- 4 files changed, 37 insertions(+), 30 deletions(-) delete mode 100644 build-support/travis/py3_osx_env.mustache diff --git a/.travis.yml b/.travis.yml index 342d9e7a4ba..7079cb225bb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -169,11 +169,13 @@ py3_osx_config: &py3_osx_config packages: &py3_osx_config_brew_packages - openssl env: - - PATH="/usr/local/opt/openssl/bin:$PATH" - - LDFLAGS="-L/usr/local/opt/openssl/lib" - - CPPFLAGS="-I/usr/local/opt/openssl/include" - - PYENV_ROOT="${HOME}/.pyenv" - - PATH="${PYENV_ROOT}/shims:${PATH}" + # Fix Python 3 issue linking to OpenSSL + - &py3_osx_config_env > + PATH="/usr/local/opt/openssl/bin:$PATH" + LDFLAGS="-L/usr/local/opt/openssl/lib" + CPPFLAGS="-I/usr/local/opt/openssl/include" + PYENV_ROOT="${HOME}/.pyenv" + PATH="${PYENV_ROOT}/shims:${PATH}" before_install: - curl -L https://github.com/stedolan/jq/releases/download/jq-1.5/jq-osx-amd64 -o /usr/local/bin/jq - chmod 755 /usr/local/bin/jq @@ -203,12 +205,14 @@ py3_osx_test_config: &py3_osx_test_config <<: *base_osx_test_config stage: *test env: - - PATH="/usr/local/opt/openssl/bin:$PATH" - - LDFLAGS="-L/usr/local/opt/openssl/lib" - - CPPFLAGS="-I/usr/local/opt/openssl/include" - - PYENV_ROOT="${HOME}/.pyenv" - - PATH="${PYENV_ROOT}/shims:${PATH}" - - BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.osx + # Must duplicate py3_osx_config's env because it cannot be merged into a new anchor + - &py3_osx_test_config_env > + PATH="/usr/local/opt/openssl/bin:$PATH" + LDFLAGS="-L/usr/local/opt/openssl/lib" + CPPFLAGS="-I/usr/local/opt/openssl/include" + PYENV_ROOT="${HOME}/.pyenv" + PATH="${PYENV_ROOT}/shims:${PATH}" + BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.osx linux_with_fuse: &linux_with_fuse before_install: diff --git a/build-support/travis/generate_travis_yml.py b/build-support/travis/generate_travis_yml.py index 8b62786b3aa..08e12ddac74 100644 --- a/build-support/travis/generate_travis_yml.py +++ b/build-support/travis/generate_travis_yml.py @@ -25,15 +25,12 @@ def generate_travis_yml(): """Generates content for a .travis.yml file from templates.""" - - def get_mustache_resource(file_name): - return pkg_resources.resource_string(__name__, "{}.mustache".format(file_name)).decode('utf-8') - - template = get_mustache_resource('travis.yml') - before_install_linux = get_mustache_resource('before_install_linux') - before_install_osx = get_mustache_resource('before_install_osx') - py3_osx_env = get_mustache_resource('py3_osx_env') - + template = pkg_resources.resource_string( + __name__, 'travis.yml.mustache').decode('utf-8') + before_install_linux = pkg_resources.resource_string( + __name__, 'before_install_linux.mustache').decode('utf-8') + before_install_osx = pkg_resources.resource_string( + __name__, 'before_install_osx.mustache').decode('utf-8') context = { 'header': HEADER, 'py3_integration_shards': range(0, num_py3_integration_shards), @@ -45,7 +42,6 @@ def get_mustache_resource(file_name): } renderer = pystache.Renderer(partials={ 'before_install_linux': before_install_linux, - 'before_install_osx': before_install_osx, - 'py3_osx_env': py3_osx_env + 'before_install_osx': before_install_osx }) print(renderer.render(template, context)) diff --git a/build-support/travis/py3_osx_env.mustache b/build-support/travis/py3_osx_env.mustache deleted file mode 100644 index 66585940b18..00000000000 --- a/build-support/travis/py3_osx_env.mustache +++ /dev/null @@ -1,5 +0,0 @@ -- PATH="/usr/local/opt/openssl/bin:$PATH" -- LDFLAGS="-L/usr/local/opt/openssl/lib" -- CPPFLAGS="-I/usr/local/opt/openssl/include" -- PYENV_ROOT="${HOME}/.pyenv" -- PATH="${PYENV_ROOT}/shims:${PATH}" diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index 3e6d27c8ab5..5b10047d84f 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -155,7 +155,13 @@ py3_osx_config: &py3_osx_config packages: &py3_osx_config_brew_packages - openssl env: - {{>py3_osx_env}} + # Fix Python 3 issue linking to OpenSSL + - &py3_osx_config_env > + PATH="/usr/local/opt/openssl/bin:$PATH" + LDFLAGS="-L/usr/local/opt/openssl/lib" + CPPFLAGS="-I/usr/local/opt/openssl/include" + PYENV_ROOT="${HOME}/.pyenv" + PATH="${PYENV_ROOT}/shims:${PATH}" before_install: {{>before_install_osx}} # Clone pyenv directly from GitHub. For multiple osx images, brew's version of pyenv is too old to get @@ -183,8 +189,14 @@ py3_osx_test_config: &py3_osx_test_config <<: *base_osx_test_config stage: *test env: - {{>py3_osx_env}} - - BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.osx + # Must duplicate py3_osx_config's env because it cannot be merged into a new anchor + - &py3_osx_test_config_env > + PATH="/usr/local/opt/openssl/bin:$PATH" + LDFLAGS="-L/usr/local/opt/openssl/lib" + CPPFLAGS="-I/usr/local/opt/openssl/include" + PYENV_ROOT="${HOME}/.pyenv" + PATH="${PYENV_ROOT}/shims:${PATH}" + BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.osx linux_with_fuse: &linux_with_fuse before_install: From f3af5ad7270819a708dbda744a056086d71742c3 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Sun, 13 Jan 2019 08:53:24 -0700 Subject: [PATCH 083/105] Deduplicate `stage; *cron` entries Because py2_{linux,osx}_test_config defaults to cron, we no longer need to hardcode this in tests that extend these records. --- .travis.yml | 48 ++++++++---------------- build-support/travis/travis.yml.mustache | 11 ++---- 2 files changed, 18 insertions(+), 41 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7079cb225bb..5c8dba9df12 100644 --- a/.travis.yml +++ b/.travis.yml @@ -329,7 +329,6 @@ base_build_wheels: &base_build_wheels py2_linux_build_wheels: &py2_linux_build_wheels <<: *py2_linux_test_config <<: *base_build_wheels - stage: *cron name: "Build Linux wheels (Py2 PEX)" env: - *py2_linux_test_config_env @@ -349,7 +348,6 @@ py2_osx_build_wheels: &py2_osx_build_wheels <<: *py2_osx_test_config <<: *base_build_wheels name: "Build OSX wheels (Py2 PEX)" - stage: *cron env: - *py2_osx_test_config_env - *base_build_wheels_env @@ -381,7 +379,6 @@ py2_osx_10_12_sanity_check: &py2_osx_10_12_sanity_check <<: *py2_osx_test_config <<: *base_osx_10_12_sanity_check name: "OSX 10.12 sanity check (Py2 PEX)" - stage: *cron env: - *py2_osx_test_config_env - CACHE_NAME=macos10.12sanity.py2 @@ -402,7 +399,6 @@ py2_osx_10_13_sanity_check: &py2_osx_10_13_sanity_check <<: *py2_osx_test_config <<: *base_osx_10_13_sanity_check name: "OSX 10.13 sanity check (Py2 PEX)" - stage: *cron env: - *py2_osx_test_config_env - CACHE_NAME=macos10.13sanity.py2 @@ -502,7 +498,6 @@ py2_linux_rust_tests: &py2_linux_rust_tests <<: *base_linux_rust_tests name: "Linux Rust tests (Py2 PEX)" python: 2.7 - stage: *cron env: - *py2_linux_test_config_env - CACHE_NAME=linuxrusttests.py2 @@ -617,7 +612,7 @@ matrix: - ./build-support/bin/travis-ci.sh -2bn # TODO: get this passing with a Python 3 PEX. - - <<: *py3_linux_test_config + - <<: *py2_linux_test_config name: "Python contrib tests (Py2 PEX w/ Py3 constraints)" env: - *py2_linux_test_config_env @@ -730,8 +725,9 @@ matrix: script: - ./build-support/bin/travis-ci.sh -bc -i 12/13 - - <<: *py3_linux_test_config + - <<: *py2_linux_test_config name: "Blacklisted integration tests for pants - shard 0 (Py2 PEX)" + stage: *test env: - *py2_linux_test_config_env - PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="['CPython>=3.6']" @@ -739,8 +735,9 @@ matrix: script: - ./build-support/bin/travis-ci.sh -bc2w -i 0/7 - - <<: *py3_linux_test_config + - <<: *py2_linux_test_config name: "Blacklisted integration tests for pants - shard 1 (Py2 PEX)" + stage: *test env: - *py2_linux_test_config_env - PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="['CPython>=3.6']" @@ -748,8 +745,9 @@ matrix: script: - ./build-support/bin/travis-ci.sh -bc2w -i 1/7 - - <<: *py3_linux_test_config + - <<: *py2_linux_test_config name: "Blacklisted integration tests for pants - shard 2 (Py2 PEX)" + stage: *test env: - *py2_linux_test_config_env - PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="['CPython>=3.6']" @@ -757,8 +755,9 @@ matrix: script: - ./build-support/bin/travis-ci.sh -bc2w -i 2/7 - - <<: *py3_linux_test_config + - <<: *py2_linux_test_config name: "Blacklisted integration tests for pants - shard 3 (Py2 PEX)" + stage: *test env: - *py2_linux_test_config_env - PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="['CPython>=3.6']" @@ -766,8 +765,9 @@ matrix: script: - ./build-support/bin/travis-ci.sh -bc2w -i 3/7 - - <<: *py3_linux_test_config + - <<: *py2_linux_test_config name: "Blacklisted integration tests for pants - shard 4 (Py2 PEX)" + stage: *test env: - *py2_linux_test_config_env - PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="['CPython>=3.6']" @@ -775,8 +775,9 @@ matrix: script: - ./build-support/bin/travis-ci.sh -bc2w -i 4/7 - - <<: *py3_linux_test_config + - <<: *py2_linux_test_config name: "Blacklisted integration tests for pants - shard 5 (Py2 PEX)" + stage: *test env: - *py2_linux_test_config_env - PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="['CPython>=3.6']" @@ -784,8 +785,9 @@ matrix: script: - ./build-support/bin/travis-ci.sh -bc2w -i 5/7 - - <<: *py3_linux_test_config + - <<: *py2_linux_test_config name: "Blacklisted integration tests for pants - shard 6 (Py2 PEX)" + stage: *test env: - *py2_linux_test_config_env - PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="['CPython>=3.6']" @@ -795,7 +797,6 @@ matrix: - <<: *py2_linux_test_config name: "Integration tests for pants - shard 0 (Py2 PEX)" - stage: *cron env: - *py2_linux_test_config_env - CACHE_NAME=cronshard0 @@ -804,7 +805,6 @@ matrix: - <<: *py2_linux_test_config name: "Integration tests for pants - shard 1 (Py2 PEX)" - stage: *cron env: - *py2_linux_test_config_env - CACHE_NAME=cronshard1 @@ -813,7 +813,6 @@ matrix: - <<: *py2_linux_test_config name: "Integration tests for pants - shard 2 (Py2 PEX)" - stage: *cron env: - *py2_linux_test_config_env - CACHE_NAME=cronshard2 @@ -822,7 +821,6 @@ matrix: - <<: *py2_linux_test_config name: "Integration tests for pants - shard 3 (Py2 PEX)" - stage: *cron env: - *py2_linux_test_config_env - CACHE_NAME=cronshard3 @@ -831,7 +829,6 @@ matrix: - <<: *py2_linux_test_config name: "Integration tests for pants - shard 4 (Py2 PEX)" - stage: *cron env: - *py2_linux_test_config_env - CACHE_NAME=cronshard4 @@ -840,7 +837,6 @@ matrix: - <<: *py2_linux_test_config name: "Integration tests for pants - shard 5 (Py2 PEX)" - stage: *cron env: - *py2_linux_test_config_env - CACHE_NAME=cronshard5 @@ -849,7 +845,6 @@ matrix: - <<: *py2_linux_test_config name: "Integration tests for pants - shard 6 (Py2 PEX)" - stage: *cron env: - *py2_linux_test_config_env - CACHE_NAME=cronshard6 @@ -858,7 +853,6 @@ matrix: - <<: *py2_linux_test_config name: "Integration tests for pants - shard 7 (Py2 PEX)" - stage: *cron env: - *py2_linux_test_config_env - CACHE_NAME=cronshard7 @@ -867,7 +861,6 @@ matrix: - <<: *py2_linux_test_config name: "Integration tests for pants - shard 8 (Py2 PEX)" - stage: *cron env: - *py2_linux_test_config_env - CACHE_NAME=cronshard8 @@ -876,7 +869,6 @@ matrix: - <<: *py2_linux_test_config name: "Integration tests for pants - shard 9 (Py2 PEX)" - stage: *cron env: - *py2_linux_test_config_env - CACHE_NAME=cronshard9 @@ -885,7 +877,6 @@ matrix: - <<: *py2_linux_test_config name: "Integration tests for pants - shard 10 (Py2 PEX)" - stage: *cron env: - *py2_linux_test_config_env - CACHE_NAME=cronshard10 @@ -894,7 +885,6 @@ matrix: - <<: *py2_linux_test_config name: "Integration tests for pants - shard 11 (Py2 PEX)" - stage: *cron env: - *py2_linux_test_config_env - CACHE_NAME=cronshard11 @@ -903,7 +893,6 @@ matrix: - <<: *py2_linux_test_config name: "Integration tests for pants - shard 12 (Py2 PEX)" - stage: *cron env: - *py2_linux_test_config_env - CACHE_NAME=cronshard12 @@ -912,7 +901,6 @@ matrix: - <<: *py2_linux_test_config name: "Integration tests for pants - shard 13 (Py2 PEX)" - stage: *cron env: - *py2_linux_test_config_env - CACHE_NAME=cronshard13 @@ -921,7 +909,6 @@ matrix: - <<: *py2_linux_test_config name: "Integration tests for pants - shard 14 (Py2 PEX)" - stage: *cron env: - *py2_linux_test_config_env - CACHE_NAME=cronshard14 @@ -930,7 +917,6 @@ matrix: - <<: *py2_linux_test_config name: "Integration tests for pants - shard 15 (Py2 PEX)" - stage: *cron env: - *py2_linux_test_config_env - CACHE_NAME=cronshard15 @@ -939,7 +925,6 @@ matrix: - <<: *py2_linux_test_config name: "Integration tests for pants - shard 16 (Py2 PEX)" - stage: *cron env: - *py2_linux_test_config_env - CACHE_NAME=cronshard16 @@ -948,7 +933,6 @@ matrix: - <<: *py2_linux_test_config name: "Integration tests for pants - shard 17 (Py2 PEX)" - stage: *cron env: - *py2_linux_test_config_env - CACHE_NAME=cronshard17 @@ -957,7 +941,6 @@ matrix: - <<: *py2_linux_test_config name: "Integration tests for pants - shard 18 (Py2 PEX)" - stage: *cron env: - *py2_linux_test_config_env - CACHE_NAME=cronshard18 @@ -966,7 +949,6 @@ matrix: - <<: *py2_linux_test_config name: "Integration tests for pants - shard 19 (Py2 PEX)" - stage: *cron env: - *py2_linux_test_config_env - CACHE_NAME=cronshard19 diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index 5b10047d84f..6e99a04c16c 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -308,7 +308,6 @@ base_build_wheels: &base_build_wheels py2_linux_build_wheels: &py2_linux_build_wheels <<: *py2_linux_test_config <<: *base_build_wheels - stage: *cron name: "Build Linux wheels (Py2 PEX)" env: - *py2_linux_test_config_env @@ -328,7 +327,6 @@ py2_osx_build_wheels: &py2_osx_build_wheels <<: *py2_osx_test_config <<: *base_build_wheels name: "Build OSX wheels (Py2 PEX)" - stage: *cron env: - *py2_osx_test_config_env - *base_build_wheels_env @@ -360,7 +358,6 @@ py2_osx_10_12_sanity_check: &py2_osx_10_12_sanity_check <<: *py2_osx_test_config <<: *base_osx_10_12_sanity_check name: "OSX 10.12 sanity check (Py2 PEX)" - stage: *cron env: - *py2_osx_test_config_env - CACHE_NAME=macos10.12sanity.py2 @@ -381,7 +378,6 @@ py2_osx_10_13_sanity_check: &py2_osx_10_13_sanity_check <<: *py2_osx_test_config <<: *base_osx_10_13_sanity_check name: "OSX 10.13 sanity check (Py2 PEX)" - stage: *cron env: - *py2_osx_test_config_env - CACHE_NAME=macos10.13sanity.py2 @@ -481,7 +477,6 @@ py2_linux_rust_tests: &py2_linux_rust_tests <<: *base_linux_rust_tests name: "Linux Rust tests (Py2 PEX)" python: 2.7 - stage: *cron env: - *py2_linux_test_config_env - CACHE_NAME=linuxrusttests.py2 @@ -596,7 +591,7 @@ matrix: - ./build-support/bin/travis-ci.sh -2bn # TODO: get this passing with a Python 3 PEX. - - <<: *py3_linux_test_config + - <<: *py2_linux_test_config name: "Python contrib tests (Py2 PEX w/ Py3 constraints)" env: - *py2_linux_test_config_env @@ -616,8 +611,9 @@ matrix: {{/py3_integration_shards}} {{#py2_blacklist_integration_shards}} - - <<: *py3_linux_test_config + - <<: *py2_linux_test_config name: "Blacklisted integration tests for pants - shard {{.}} (Py2 PEX)" + stage: *test env: - *py2_linux_test_config_env - PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="['CPython>=3.6']" @@ -629,7 +625,6 @@ matrix: {{#cron_integration_shards}} - <<: *py2_linux_test_config name: "Integration tests for pants - shard {{.}} (Py2 PEX)" - stage: *cron env: - *py2_linux_test_config_env - CACHE_NAME=cronshard{{.}} From 2cb7f835a22028b0177d15c5368d1ce210a569aa Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Sun, 13 Jan 2019 09:07:06 -0700 Subject: [PATCH 084/105] Remove rust audit config to reflect they don't grab pex from aws --- .travis.yml | 7 +++---- build-support/travis/travis.yml.mustache | 7 +++---- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5c8dba9df12..98667e6596b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -478,7 +478,7 @@ deploy_unstable_multiplatform_pex: &deploy_unstable_multiplatform_pex - ./build-support/bin/release.sh -p && mkdir -p dist/deploy/pex/ && mv dist/pants*.pex dist/deploy/pex/ # ------------------------------------------------------------------------- -# Rust +# Rust tests # ------------------------------------------------------------------------- base_linux_rust_tests: &base_linux_rust_tests @@ -966,9 +966,8 @@ matrix: - <<: *linux_with_fuse <<: *native_engine_cache_config - name: "Linux Rust Clippy" + name: "Linux Rust Clippy (No PEX)" env: - - *py2_linux_test_config_env - CACHE_NAME=linuxclippy os: linux dist: xenial @@ -983,7 +982,7 @@ matrix: - ./build-support/bin/travis-ci.sh -bs2 - <<: *linux_with_fuse - name: "Cargo audit" + name: "Cargo audit (No PEX)" env: - *py2_linux_test_config_env - CACHE_NAME=linuxcargoaudit diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index 6e99a04c16c..3267dd34c1b 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -457,7 +457,7 @@ deploy_unstable_multiplatform_pex: &deploy_unstable_multiplatform_pex - ./build-support/bin/release.sh -p && mkdir -p dist/deploy/pex/ && mv dist/pants*.pex dist/deploy/pex/ # ------------------------------------------------------------------------- -# Rust +# Rust tests # ------------------------------------------------------------------------- base_linux_rust_tests: &base_linux_rust_tests @@ -643,9 +643,8 @@ matrix: - <<: *linux_with_fuse <<: *native_engine_cache_config - name: "Linux Rust Clippy" + name: "Linux Rust Clippy (No PEX)" env: - - *py2_linux_test_config_env - CACHE_NAME=linuxclippy os: linux dist: xenial @@ -660,7 +659,7 @@ matrix: - ./build-support/bin/travis-ci.sh -bs2 - <<: *linux_with_fuse - name: "Cargo audit" + name: "Cargo audit (No PEX)" env: - *py2_linux_test_config_env - CACHE_NAME=linuxcargoaudit From 9feef95fe61449cab2ed7418e55a4b4f3e9ab5f1 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Sun, 13 Jan 2019 09:08:18 -0700 Subject: [PATCH 085/105] Clarify blacklist is using Py3 constraints --- .travis.yml | 14 +++++++------- build-support/travis/travis.yml.mustache | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 98667e6596b..838219abd89 100644 --- a/.travis.yml +++ b/.travis.yml @@ -726,7 +726,7 @@ matrix: - ./build-support/bin/travis-ci.sh -bc -i 12/13 - <<: *py2_linux_test_config - name: "Blacklisted integration tests for pants - shard 0 (Py2 PEX)" + name: "Blacklisted integration tests for pants - shard 0 (Py2 PEX w/ Py3 constraints)" stage: *test env: - *py2_linux_test_config_env @@ -736,7 +736,7 @@ matrix: - ./build-support/bin/travis-ci.sh -bc2w -i 0/7 - <<: *py2_linux_test_config - name: "Blacklisted integration tests for pants - shard 1 (Py2 PEX)" + name: "Blacklisted integration tests for pants - shard 1 (Py2 PEX w/ Py3 constraints)" stage: *test env: - *py2_linux_test_config_env @@ -746,7 +746,7 @@ matrix: - ./build-support/bin/travis-ci.sh -bc2w -i 1/7 - <<: *py2_linux_test_config - name: "Blacklisted integration tests for pants - shard 2 (Py2 PEX)" + name: "Blacklisted integration tests for pants - shard 2 (Py2 PEX w/ Py3 constraints)" stage: *test env: - *py2_linux_test_config_env @@ -756,7 +756,7 @@ matrix: - ./build-support/bin/travis-ci.sh -bc2w -i 2/7 - <<: *py2_linux_test_config - name: "Blacklisted integration tests for pants - shard 3 (Py2 PEX)" + name: "Blacklisted integration tests for pants - shard 3 (Py2 PEX w/ Py3 constraints)" stage: *test env: - *py2_linux_test_config_env @@ -766,7 +766,7 @@ matrix: - ./build-support/bin/travis-ci.sh -bc2w -i 3/7 - <<: *py2_linux_test_config - name: "Blacklisted integration tests for pants - shard 4 (Py2 PEX)" + name: "Blacklisted integration tests for pants - shard 4 (Py2 PEX w/ Py3 constraints)" stage: *test env: - *py2_linux_test_config_env @@ -776,7 +776,7 @@ matrix: - ./build-support/bin/travis-ci.sh -bc2w -i 4/7 - <<: *py2_linux_test_config - name: "Blacklisted integration tests for pants - shard 5 (Py2 PEX)" + name: "Blacklisted integration tests for pants - shard 5 (Py2 PEX w/ Py3 constraints)" stage: *test env: - *py2_linux_test_config_env @@ -786,7 +786,7 @@ matrix: - ./build-support/bin/travis-ci.sh -bc2w -i 5/7 - <<: *py2_linux_test_config - name: "Blacklisted integration tests for pants - shard 6 (Py2 PEX)" + name: "Blacklisted integration tests for pants - shard 6 (Py2 PEX w/ Py3 constraints)" stage: *test env: - *py2_linux_test_config_env diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index 3267dd34c1b..03b9c2e4ffd 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -612,7 +612,7 @@ matrix: {{/py3_integration_shards}} {{#py2_blacklist_integration_shards}} - <<: *py2_linux_test_config - name: "Blacklisted integration tests for pants - shard {{.}} (Py2 PEX)" + name: "Blacklisted integration tests for pants - shard {{.}} (Py2 PEX w/ Py3 constraints)" stage: *test env: - *py2_linux_test_config_env From d616a0ec8b6826de5106de50677d1e5f945153fc Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Sun, 13 Jan 2019 10:23:19 -0700 Subject: [PATCH 086/105] Squash merge newest py3 changes --- pants3 | 2 +- src/rust/engine/src/cffi_build.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pants3 b/pants3 index cd5d1f5f068..750b829ad09 100755 --- a/pants3 +++ b/pants3 @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). # This bootstrap script invokes Pants using a Python 3 interpreter. diff --git a/src/rust/engine/src/cffi_build.rs b/src/rust/engine/src/cffi_build.rs index a865ddbe30c..9a6e67124e9 100644 --- a/src/rust/engine/src/cffi_build.rs +++ b/src/rust/engine/src/cffi_build.rs @@ -114,7 +114,7 @@ fn main() -> Result<(), CffiBuildError> { // When built with Python 2, it works with both Python 2 and Python 3. // So, we check to see if the under-the-hood interpreter has changed and rebuild the native engine // when needed. - println!("rerun-if-env-changed=PANTS_USE_PYTHON3"); + println!("cargo:rerun-if-env-changed=PANTS_USE_PYTHON3"); if cfg!(target_os = "linux") { println!("cargo:rustc-link-lib=static=stdc++"); From 99a160e30dad95cc1ad5f75261e9807f96f04d23 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Sun, 13 Jan 2019 15:00:23 -0700 Subject: [PATCH 087/105] Squash merge 7073. This one will need to be merged before we can merge this PR, as it fixes the \n exception issue, which is a major quality of life issue we must solve. 7073 can't run with a Py3 pex in CI, so I'm not 100% certain it's solved until we run against this PR here. Followup commit will intentionally make failing targets so that we can test this out. --- src/python/pants/base/exception_sink.py | 3 ++- src/python/pants/base/exiter.py | 7 ++++--- src/python/pants/bin/remote_pants_runner.py | 6 +++--- src/python/pants/java/nailgun_client.py | 6 ++++-- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/python/pants/base/exception_sink.py b/src/python/pants/base/exception_sink.py index 9e885984a7f..af75dff5131 100644 --- a/src/python/pants/base/exception_sink.py +++ b/src/python/pants/base/exception_sink.py @@ -14,6 +14,7 @@ from builtins import object, str import setproctitle +from future.utils import PY3 from pants.base.build_environment import get_buildroot from pants.base.exiter import Exiter @@ -381,4 +382,4 @@ def handle_signal_gracefully(cls, signum, frame): # Sets except hook. ExceptionSink.reset_exiter(Exiter(exiter=sys.exit)) # Sets a SIGUSR2 handler. -ExceptionSink.reset_interactive_output_stream(sys.stderr) +ExceptionSink.reset_interactive_output_stream(sys.stderr.buffer if PY3 else sys.stderr) diff --git a/src/python/pants/base/exiter.py b/src/python/pants/base/exiter.py index 6b04fcecbb6..a7d450455ee 100644 --- a/src/python/pants/base/exiter.py +++ b/src/python/pants/base/exiter.py @@ -8,6 +8,8 @@ import sys from builtins import object +from future.utils import PY3 + from pants.util.strutil import ensure_binary @@ -51,11 +53,10 @@ def exit(self, result=0, msg=None, out=None): :param out: The file descriptor to emit `msg` to. (Optional) """ if msg: - out = out or sys.stderr - # sys.stderr expects bytes in Py2, unicode in Py3 + out = out or (sys.stderr.buffer if PY3 else sys.stderr) msg = ensure_binary(msg) try: - print(msg, file=out) + out.write(msg + b'\n') # TODO: Determine whether this call is a no-op because the stream gets flushed on exit, or # if we could lose what we just printed, e.g. if we get interrupted by a signal while # exiting and the stream is buffered like stdout. diff --git a/src/python/pants/bin/remote_pants_runner.py b/src/python/pants/bin/remote_pants_runner.py index 5942785f0be..17052c15486 100644 --- a/src/python/pants/bin/remote_pants_runner.py +++ b/src/python/pants/bin/remote_pants_runner.py @@ -11,7 +11,7 @@ from builtins import object, str from contextlib import contextmanager -from future.utils import raise_with_traceback +from future.utils import PY3, raise_with_traceback from pants.base.exception_sink import ExceptionSink from pants.console.stty_utils import STTYSettings @@ -57,8 +57,8 @@ def __init__(self, exiter, args, env, options_bootstrapper, stdin=None, stdout=N self._options_bootstrapper = options_bootstrapper self._bootstrap_options = options_bootstrapper.bootstrap_options self._stdin = stdin or sys.stdin - self._stdout = stdout or sys.stdout - self._stderr = stderr or sys.stderr + self._stdout = stdout or (sys.stdout.buffer if PY3 else sys.stdout) + self._stderr = stderr or (sys.stderr.buffer if PY3 else sys.stdout) @contextmanager def _trapped_signals(self, client): diff --git a/src/python/pants/java/nailgun_client.py b/src/python/pants/java/nailgun_client.py index 8956b823286..88cdd84570b 100644 --- a/src/python/pants/java/nailgun_client.py +++ b/src/python/pants/java/nailgun_client.py @@ -11,6 +11,8 @@ import sys from builtins import object, str +from future.utils import PY3 + from pants.java.nailgun_io import NailgunStreamWriter from pants.java.nailgun_protocol import ChunkType, NailgunProtocol from pants.util.osutil import safe_kill @@ -179,8 +181,8 @@ def __init__(self, host=DEFAULT_NG_HOST, port=DEFAULT_NG_PORT, ins=sys.stdin, ou self._address = (host, port) self._address_string = ':'.join(str(i) for i in self._address) self._stdin = ins - self._stdout = out or sys.stdout - self._stderr = err or sys.stderr + self._stdout = out or (sys.stdout.buffer if PY3 else sys.stdout) + self._stderr = err or (sys.stderr.buffer if PY3 else sys.stderr) self._workdir = workdir or os.path.abspath(os.path.curdir) self._exit_on_broken_pipe = exit_on_broken_pipe self._expects_pid = expects_pid From 902175e624378f65739ec7aacaa967fc3a75a5a4 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Sun, 13 Jan 2019 15:10:46 -0700 Subject: [PATCH 088/105] Run failing things with Py3 to inspect if 7073 worked --- .travis.yml | 119 +++++++++----------- build-support/known_py3_pex_failures.txt | 23 ---- build-support/travis/generate_travis_yml.py | 4 +- build-support/travis/travis.yml.mustache | 9 +- 4 files changed, 58 insertions(+), 97 deletions(-) diff --git a/.travis.yml b/.travis.yml index 838219abd89..e0f4123858b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -612,14 +612,13 @@ matrix: - ./build-support/bin/travis-ci.sh -2bn # TODO: get this passing with a Python 3 PEX. - - <<: *py2_linux_test_config - name: "Python contrib tests (Py2 PEX w/ Py3 constraints)" + - <<: *py3_linux_test_config + name: "Python contrib tests (Py3 PEX)" env: - - *py2_linux_test_config_env - - PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="['CPython>=3.6']" + - *py3_linux_test_config_env - CACHE_NAME=linuxcontribtests.py3 script: - - ./build-support/bin/travis-ci.sh -bn2 + - ./build-support/bin/travis-ci.sh -bn - <<: *py3_linux_test_config name: "Integration tests for pants - shard 0 (Py3 PEX)" @@ -627,7 +626,7 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard0 script: - - ./build-support/bin/travis-ci.sh -bc -i 0/13 + - ./build-support/bin/travis-ci.sh -bc -i 0/20 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 1 (Py3 PEX)" @@ -635,7 +634,7 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard1 script: - - ./build-support/bin/travis-ci.sh -bc -i 1/13 + - ./build-support/bin/travis-ci.sh -bc -i 1/20 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 2 (Py3 PEX)" @@ -643,7 +642,7 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard2 script: - - ./build-support/bin/travis-ci.sh -bc -i 2/13 + - ./build-support/bin/travis-ci.sh -bc -i 2/20 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 3 (Py3 PEX)" @@ -651,7 +650,7 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard3 script: - - ./build-support/bin/travis-ci.sh -bc -i 3/13 + - ./build-support/bin/travis-ci.sh -bc -i 3/20 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 4 (Py3 PEX)" @@ -659,7 +658,7 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard4 script: - - ./build-support/bin/travis-ci.sh -bc -i 4/13 + - ./build-support/bin/travis-ci.sh -bc -i 4/20 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 5 (Py3 PEX)" @@ -667,7 +666,7 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard5 script: - - ./build-support/bin/travis-ci.sh -bc -i 5/13 + - ./build-support/bin/travis-ci.sh -bc -i 5/20 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 6 (Py3 PEX)" @@ -675,7 +674,7 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard6 script: - - ./build-support/bin/travis-ci.sh -bc -i 6/13 + - ./build-support/bin/travis-ci.sh -bc -i 6/20 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 7 (Py3 PEX)" @@ -683,7 +682,7 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard7 script: - - ./build-support/bin/travis-ci.sh -bc -i 7/13 + - ./build-support/bin/travis-ci.sh -bc -i 7/20 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 8 (Py3 PEX)" @@ -691,7 +690,7 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard8 script: - - ./build-support/bin/travis-ci.sh -bc -i 8/13 + - ./build-support/bin/travis-ci.sh -bc -i 8/20 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 9 (Py3 PEX)" @@ -699,7 +698,7 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard9 script: - - ./build-support/bin/travis-ci.sh -bc -i 9/13 + - ./build-support/bin/travis-ci.sh -bc -i 9/20 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 10 (Py3 PEX)" @@ -707,7 +706,7 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard10 script: - - ./build-support/bin/travis-ci.sh -bc -i 10/13 + - ./build-support/bin/travis-ci.sh -bc -i 10/20 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 11 (Py3 PEX)" @@ -715,7 +714,7 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard11 script: - - ./build-support/bin/travis-ci.sh -bc -i 11/13 + - ./build-support/bin/travis-ci.sh -bc -i 11/20 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 12 (Py3 PEX)" @@ -723,77 +722,63 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard12 script: - - ./build-support/bin/travis-ci.sh -bc -i 12/13 + - ./build-support/bin/travis-ci.sh -bc -i 12/20 - - <<: *py2_linux_test_config - name: "Blacklisted integration tests for pants - shard 0 (Py2 PEX w/ Py3 constraints)" - stage: *test + - <<: *py3_linux_test_config + name: "Integration tests for pants - shard 13 (Py3 PEX)" env: - - *py2_linux_test_config_env - - PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="['CPython>=3.6']" - - CACHE_NAME=integrationshard0.py2blacklist + - *py3_linux_test_config_env + - CACHE_NAME=integrationshard13 script: - - ./build-support/bin/travis-ci.sh -bc2w -i 0/7 + - ./build-support/bin/travis-ci.sh -bc -i 13/20 - - <<: *py2_linux_test_config - name: "Blacklisted integration tests for pants - shard 1 (Py2 PEX w/ Py3 constraints)" - stage: *test + - <<: *py3_linux_test_config + name: "Integration tests for pants - shard 14 (Py3 PEX)" env: - - *py2_linux_test_config_env - - PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="['CPython>=3.6']" - - CACHE_NAME=integrationshard1.py2blacklist + - *py3_linux_test_config_env + - CACHE_NAME=integrationshard14 script: - - ./build-support/bin/travis-ci.sh -bc2w -i 1/7 + - ./build-support/bin/travis-ci.sh -bc -i 14/20 - - <<: *py2_linux_test_config - name: "Blacklisted integration tests for pants - shard 2 (Py2 PEX w/ Py3 constraints)" - stage: *test + - <<: *py3_linux_test_config + name: "Integration tests for pants - shard 15 (Py3 PEX)" env: - - *py2_linux_test_config_env - - PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="['CPython>=3.6']" - - CACHE_NAME=integrationshard2.py2blacklist + - *py3_linux_test_config_env + - CACHE_NAME=integrationshard15 script: - - ./build-support/bin/travis-ci.sh -bc2w -i 2/7 + - ./build-support/bin/travis-ci.sh -bc -i 15/20 - - <<: *py2_linux_test_config - name: "Blacklisted integration tests for pants - shard 3 (Py2 PEX w/ Py3 constraints)" - stage: *test + - <<: *py3_linux_test_config + name: "Integration tests for pants - shard 16 (Py3 PEX)" env: - - *py2_linux_test_config_env - - PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="['CPython>=3.6']" - - CACHE_NAME=integrationshard3.py2blacklist + - *py3_linux_test_config_env + - CACHE_NAME=integrationshard16 script: - - ./build-support/bin/travis-ci.sh -bc2w -i 3/7 + - ./build-support/bin/travis-ci.sh -bc -i 16/20 - - <<: *py2_linux_test_config - name: "Blacklisted integration tests for pants - shard 4 (Py2 PEX w/ Py3 constraints)" - stage: *test + - <<: *py3_linux_test_config + name: "Integration tests for pants - shard 17 (Py3 PEX)" env: - - *py2_linux_test_config_env - - PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="['CPython>=3.6']" - - CACHE_NAME=integrationshard4.py2blacklist + - *py3_linux_test_config_env + - CACHE_NAME=integrationshard17 script: - - ./build-support/bin/travis-ci.sh -bc2w -i 4/7 + - ./build-support/bin/travis-ci.sh -bc -i 17/20 - - <<: *py2_linux_test_config - name: "Blacklisted integration tests for pants - shard 5 (Py2 PEX w/ Py3 constraints)" - stage: *test + - <<: *py3_linux_test_config + name: "Integration tests for pants - shard 18 (Py3 PEX)" env: - - *py2_linux_test_config_env - - PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="['CPython>=3.6']" - - CACHE_NAME=integrationshard5.py2blacklist + - *py3_linux_test_config_env + - CACHE_NAME=integrationshard18 script: - - ./build-support/bin/travis-ci.sh -bc2w -i 5/7 + - ./build-support/bin/travis-ci.sh -bc -i 18/20 - - <<: *py2_linux_test_config - name: "Blacklisted integration tests for pants - shard 6 (Py2 PEX w/ Py3 constraints)" - stage: *test + - <<: *py3_linux_test_config + name: "Integration tests for pants - shard 19 (Py3 PEX)" env: - - *py2_linux_test_config_env - - PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="['CPython>=3.6']" - - CACHE_NAME=integrationshard6.py2blacklist + - *py3_linux_test_config_env + - CACHE_NAME=integrationshard19 script: - - ./build-support/bin/travis-ci.sh -bc2w -i 6/7 + - ./build-support/bin/travis-ci.sh -bc -i 19/20 - <<: *py2_linux_test_config name: "Integration tests for pants - shard 0 (Py2 PEX)" diff --git a/build-support/known_py3_pex_failures.txt b/build-support/known_py3_pex_failures.txt index 14c68e92337..e69de29bb2d 100644 --- a/build-support/known_py3_pex_failures.txt +++ b/build-support/known_py3_pex_failures.txt @@ -1,23 +0,0 @@ -tests/python/pants_test/backend/jvm/tasks/jvm_compile/java:zinc_compile_integration -tests/python/pants_test/backend/jvm/tasks/jvm_compile/rsc:rsc_compile_integration -tests/python/pants_test/backend/jvm/tasks/jvm_compile:declared_deps_integration -tests/python/pants_test/backend/jvm/tasks:classmap_integration -tests/python/pants_test/backend/jvm/tasks:jar_publish_integration -tests/python/pants_test/backend/jvm/tasks:jvm_platform_analysis_integration -tests/python/pants_test/backend/jvm/tasks:scala_repl_integration -tests/python/pants_test/backend/project_info/tasks:idea_plugin_integration -tests/python/pants_test/backend/python/tasks:integration -tests/python/pants_test/backend/python/tasks:isort_run_integration -tests/python/pants_test/backend/python/tasks:python_native_code_testing_1 -tests/python/pants_test/backend/python:integration -tests/python/pants_test/base:exception_sink_integration -tests/python/pants_test/base:exiter_integration -tests/python/pants_test/engine/legacy:changed_integration -tests/python/pants_test/engine/legacy:console_rule_integration -tests/python/pants_test/engine/legacy:graph_integration -tests/python/pants_test/engine/legacy:owners_integration -tests/python/pants_test/engine:scheduler_integration -tests/python/pants_test/option:options_integration -tests/python/pants_test/pantsd:pantsd_integration -tests/python/pants_test/projects:testprojects_integration -tests/python/pants_test/rules:test_integration diff --git a/build-support/travis/generate_travis_yml.py b/build-support/travis/generate_travis_yml.py index 08e12ddac74..03e89040fd5 100644 --- a/build-support/travis/generate_travis_yml.py +++ b/build-support/travis/generate_travis_yml.py @@ -8,8 +8,8 @@ import pystache -num_py3_integration_shards = 13 -num_py2_blacklist_integration_shards = 7 +num_py3_integration_shards = 20 +num_py2_blacklist_integration_shards = 0 num_cron_integration_shards = 20 diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index 03b9c2e4ffd..3768b9884e6 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -591,14 +591,13 @@ matrix: - ./build-support/bin/travis-ci.sh -2bn # TODO: get this passing with a Python 3 PEX. - - <<: *py2_linux_test_config - name: "Python contrib tests (Py2 PEX w/ Py3 constraints)" + - <<: *py3_linux_test_config + name: "Python contrib tests (Py3 PEX)" env: - - *py2_linux_test_config_env - - PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="['CPython>=3.6']" + - *py3_linux_test_config_env - CACHE_NAME=linuxcontribtests.py3 script: - - ./build-support/bin/travis-ci.sh -bn2 + - ./build-support/bin/travis-ci.sh -bn {{#py3_integration_shards}} - <<: *py3_linux_test_config From f180f885930e7b9da380bc7088db443874150f80 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Sun, 13 Jan 2019 16:15:35 -0700 Subject: [PATCH 089/105] Rerun \n pr with its reverted changes Check if the solution still works even without reverted commit. --- src/python/pants/bin/remote_pants_runner.py | 6 +++--- src/python/pants/java/nailgun_client.py | 6 ++---- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/python/pants/bin/remote_pants_runner.py b/src/python/pants/bin/remote_pants_runner.py index 17052c15486..5942785f0be 100644 --- a/src/python/pants/bin/remote_pants_runner.py +++ b/src/python/pants/bin/remote_pants_runner.py @@ -11,7 +11,7 @@ from builtins import object, str from contextlib import contextmanager -from future.utils import PY3, raise_with_traceback +from future.utils import raise_with_traceback from pants.base.exception_sink import ExceptionSink from pants.console.stty_utils import STTYSettings @@ -57,8 +57,8 @@ def __init__(self, exiter, args, env, options_bootstrapper, stdin=None, stdout=N self._options_bootstrapper = options_bootstrapper self._bootstrap_options = options_bootstrapper.bootstrap_options self._stdin = stdin or sys.stdin - self._stdout = stdout or (sys.stdout.buffer if PY3 else sys.stdout) - self._stderr = stderr or (sys.stderr.buffer if PY3 else sys.stdout) + self._stdout = stdout or sys.stdout + self._stderr = stderr or sys.stderr @contextmanager def _trapped_signals(self, client): diff --git a/src/python/pants/java/nailgun_client.py b/src/python/pants/java/nailgun_client.py index 88cdd84570b..8956b823286 100644 --- a/src/python/pants/java/nailgun_client.py +++ b/src/python/pants/java/nailgun_client.py @@ -11,8 +11,6 @@ import sys from builtins import object, str -from future.utils import PY3 - from pants.java.nailgun_io import NailgunStreamWriter from pants.java.nailgun_protocol import ChunkType, NailgunProtocol from pants.util.osutil import safe_kill @@ -181,8 +179,8 @@ def __init__(self, host=DEFAULT_NG_HOST, port=DEFAULT_NG_PORT, ins=sys.stdin, ou self._address = (host, port) self._address_string = ':'.join(str(i) for i in self._address) self._stdin = ins - self._stdout = out or (sys.stdout.buffer if PY3 else sys.stdout) - self._stderr = err or (sys.stderr.buffer if PY3 else sys.stderr) + self._stdout = out or sys.stdout + self._stderr = err or sys.stderr self._workdir = workdir or os.path.abspath(os.path.curdir) self._exit_on_broken_pipe = exit_on_broken_pipe self._expects_pid = expects_pid From 3cb45624e2db1b01125c1ee8c6b9fcf9def49402 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Sun, 13 Jan 2019 17:45:39 -0700 Subject: [PATCH 090/105] Refix bad lint Originally I was going to wait to have this fix be a part of #7067, but we need to include it now to get the linting shard to pass. I have no idea why the linter passed all this time on master, yet now fails when I didn't even touch source code.. Alas. --- .../src/python/pants/contrib/confluence/util/confluence_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/confluence/src/python/pants/contrib/confluence/util/confluence_util.py b/contrib/confluence/src/python/pants/contrib/confluence/util/confluence_util.py index fa35978048d..1d40c573fef 100644 --- a/contrib/confluence/src/python/pants/contrib/confluence/util/confluence_util.py +++ b/contrib/confluence/src/python/pants/contrib/confluence/util/confluence_util.py @@ -171,7 +171,7 @@ def addattachment(self, page, filename): except (IOError, OSError) as e: log.error('Failed to read data from file %s: %s' % (filename, str(e))) return None - except XMLRPCError as e: + except XMLRPCError: log.error('Failed to add file attachment %s to page: %s' % (filename, page.get('title', '[unknown title]'))) return None From ce27ff6c9651cf71993060cc38881fc617566dab Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Sun, 13 Jan 2019 17:48:22 -0700 Subject: [PATCH 091/105] Revert commits from 7073 Now that we tested 7073 properly fixes \n rendering, we can restore this PR to its original purpose and add back the blacklist. --- .travis.yml | 119 +++++++++++--------- build-support/known_py3_pex_failures.txt | 23 ++++ build-support/travis/generate_travis_yml.py | 4 +- build-support/travis/travis.yml.mustache | 9 +- src/python/pants/base/exception_sink.py | 3 +- src/python/pants/base/exiter.py | 7 +- 6 files changed, 101 insertions(+), 64 deletions(-) diff --git a/.travis.yml b/.travis.yml index e0f4123858b..838219abd89 100644 --- a/.travis.yml +++ b/.travis.yml @@ -612,13 +612,14 @@ matrix: - ./build-support/bin/travis-ci.sh -2bn # TODO: get this passing with a Python 3 PEX. - - <<: *py3_linux_test_config - name: "Python contrib tests (Py3 PEX)" + - <<: *py2_linux_test_config + name: "Python contrib tests (Py2 PEX w/ Py3 constraints)" env: - - *py3_linux_test_config_env + - *py2_linux_test_config_env + - PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="['CPython>=3.6']" - CACHE_NAME=linuxcontribtests.py3 script: - - ./build-support/bin/travis-ci.sh -bn + - ./build-support/bin/travis-ci.sh -bn2 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 0 (Py3 PEX)" @@ -626,7 +627,7 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard0 script: - - ./build-support/bin/travis-ci.sh -bc -i 0/20 + - ./build-support/bin/travis-ci.sh -bc -i 0/13 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 1 (Py3 PEX)" @@ -634,7 +635,7 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard1 script: - - ./build-support/bin/travis-ci.sh -bc -i 1/20 + - ./build-support/bin/travis-ci.sh -bc -i 1/13 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 2 (Py3 PEX)" @@ -642,7 +643,7 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard2 script: - - ./build-support/bin/travis-ci.sh -bc -i 2/20 + - ./build-support/bin/travis-ci.sh -bc -i 2/13 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 3 (Py3 PEX)" @@ -650,7 +651,7 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard3 script: - - ./build-support/bin/travis-ci.sh -bc -i 3/20 + - ./build-support/bin/travis-ci.sh -bc -i 3/13 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 4 (Py3 PEX)" @@ -658,7 +659,7 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard4 script: - - ./build-support/bin/travis-ci.sh -bc -i 4/20 + - ./build-support/bin/travis-ci.sh -bc -i 4/13 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 5 (Py3 PEX)" @@ -666,7 +667,7 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard5 script: - - ./build-support/bin/travis-ci.sh -bc -i 5/20 + - ./build-support/bin/travis-ci.sh -bc -i 5/13 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 6 (Py3 PEX)" @@ -674,7 +675,7 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard6 script: - - ./build-support/bin/travis-ci.sh -bc -i 6/20 + - ./build-support/bin/travis-ci.sh -bc -i 6/13 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 7 (Py3 PEX)" @@ -682,7 +683,7 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard7 script: - - ./build-support/bin/travis-ci.sh -bc -i 7/20 + - ./build-support/bin/travis-ci.sh -bc -i 7/13 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 8 (Py3 PEX)" @@ -690,7 +691,7 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard8 script: - - ./build-support/bin/travis-ci.sh -bc -i 8/20 + - ./build-support/bin/travis-ci.sh -bc -i 8/13 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 9 (Py3 PEX)" @@ -698,7 +699,7 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard9 script: - - ./build-support/bin/travis-ci.sh -bc -i 9/20 + - ./build-support/bin/travis-ci.sh -bc -i 9/13 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 10 (Py3 PEX)" @@ -706,7 +707,7 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard10 script: - - ./build-support/bin/travis-ci.sh -bc -i 10/20 + - ./build-support/bin/travis-ci.sh -bc -i 10/13 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 11 (Py3 PEX)" @@ -714,7 +715,7 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard11 script: - - ./build-support/bin/travis-ci.sh -bc -i 11/20 + - ./build-support/bin/travis-ci.sh -bc -i 11/13 - <<: *py3_linux_test_config name: "Integration tests for pants - shard 12 (Py3 PEX)" @@ -722,63 +723,77 @@ matrix: - *py3_linux_test_config_env - CACHE_NAME=integrationshard12 script: - - ./build-support/bin/travis-ci.sh -bc -i 12/20 + - ./build-support/bin/travis-ci.sh -bc -i 12/13 - - <<: *py3_linux_test_config - name: "Integration tests for pants - shard 13 (Py3 PEX)" + - <<: *py2_linux_test_config + name: "Blacklisted integration tests for pants - shard 0 (Py2 PEX w/ Py3 constraints)" + stage: *test env: - - *py3_linux_test_config_env - - CACHE_NAME=integrationshard13 + - *py2_linux_test_config_env + - PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="['CPython>=3.6']" + - CACHE_NAME=integrationshard0.py2blacklist script: - - ./build-support/bin/travis-ci.sh -bc -i 13/20 + - ./build-support/bin/travis-ci.sh -bc2w -i 0/7 - - <<: *py3_linux_test_config - name: "Integration tests for pants - shard 14 (Py3 PEX)" + - <<: *py2_linux_test_config + name: "Blacklisted integration tests for pants - shard 1 (Py2 PEX w/ Py3 constraints)" + stage: *test env: - - *py3_linux_test_config_env - - CACHE_NAME=integrationshard14 + - *py2_linux_test_config_env + - PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="['CPython>=3.6']" + - CACHE_NAME=integrationshard1.py2blacklist script: - - ./build-support/bin/travis-ci.sh -bc -i 14/20 + - ./build-support/bin/travis-ci.sh -bc2w -i 1/7 - - <<: *py3_linux_test_config - name: "Integration tests for pants - shard 15 (Py3 PEX)" + - <<: *py2_linux_test_config + name: "Blacklisted integration tests for pants - shard 2 (Py2 PEX w/ Py3 constraints)" + stage: *test env: - - *py3_linux_test_config_env - - CACHE_NAME=integrationshard15 + - *py2_linux_test_config_env + - PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="['CPython>=3.6']" + - CACHE_NAME=integrationshard2.py2blacklist script: - - ./build-support/bin/travis-ci.sh -bc -i 15/20 + - ./build-support/bin/travis-ci.sh -bc2w -i 2/7 - - <<: *py3_linux_test_config - name: "Integration tests for pants - shard 16 (Py3 PEX)" + - <<: *py2_linux_test_config + name: "Blacklisted integration tests for pants - shard 3 (Py2 PEX w/ Py3 constraints)" + stage: *test env: - - *py3_linux_test_config_env - - CACHE_NAME=integrationshard16 + - *py2_linux_test_config_env + - PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="['CPython>=3.6']" + - CACHE_NAME=integrationshard3.py2blacklist script: - - ./build-support/bin/travis-ci.sh -bc -i 16/20 + - ./build-support/bin/travis-ci.sh -bc2w -i 3/7 - - <<: *py3_linux_test_config - name: "Integration tests for pants - shard 17 (Py3 PEX)" + - <<: *py2_linux_test_config + name: "Blacklisted integration tests for pants - shard 4 (Py2 PEX w/ Py3 constraints)" + stage: *test env: - - *py3_linux_test_config_env - - CACHE_NAME=integrationshard17 + - *py2_linux_test_config_env + - PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="['CPython>=3.6']" + - CACHE_NAME=integrationshard4.py2blacklist script: - - ./build-support/bin/travis-ci.sh -bc -i 17/20 + - ./build-support/bin/travis-ci.sh -bc2w -i 4/7 - - <<: *py3_linux_test_config - name: "Integration tests for pants - shard 18 (Py3 PEX)" + - <<: *py2_linux_test_config + name: "Blacklisted integration tests for pants - shard 5 (Py2 PEX w/ Py3 constraints)" + stage: *test env: - - *py3_linux_test_config_env - - CACHE_NAME=integrationshard18 + - *py2_linux_test_config_env + - PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="['CPython>=3.6']" + - CACHE_NAME=integrationshard5.py2blacklist script: - - ./build-support/bin/travis-ci.sh -bc -i 18/20 + - ./build-support/bin/travis-ci.sh -bc2w -i 5/7 - - <<: *py3_linux_test_config - name: "Integration tests for pants - shard 19 (Py3 PEX)" + - <<: *py2_linux_test_config + name: "Blacklisted integration tests for pants - shard 6 (Py2 PEX w/ Py3 constraints)" + stage: *test env: - - *py3_linux_test_config_env - - CACHE_NAME=integrationshard19 + - *py2_linux_test_config_env + - PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="['CPython>=3.6']" + - CACHE_NAME=integrationshard6.py2blacklist script: - - ./build-support/bin/travis-ci.sh -bc -i 19/20 + - ./build-support/bin/travis-ci.sh -bc2w -i 6/7 - <<: *py2_linux_test_config name: "Integration tests for pants - shard 0 (Py2 PEX)" diff --git a/build-support/known_py3_pex_failures.txt b/build-support/known_py3_pex_failures.txt index e69de29bb2d..14c68e92337 100644 --- a/build-support/known_py3_pex_failures.txt +++ b/build-support/known_py3_pex_failures.txt @@ -0,0 +1,23 @@ +tests/python/pants_test/backend/jvm/tasks/jvm_compile/java:zinc_compile_integration +tests/python/pants_test/backend/jvm/tasks/jvm_compile/rsc:rsc_compile_integration +tests/python/pants_test/backend/jvm/tasks/jvm_compile:declared_deps_integration +tests/python/pants_test/backend/jvm/tasks:classmap_integration +tests/python/pants_test/backend/jvm/tasks:jar_publish_integration +tests/python/pants_test/backend/jvm/tasks:jvm_platform_analysis_integration +tests/python/pants_test/backend/jvm/tasks:scala_repl_integration +tests/python/pants_test/backend/project_info/tasks:idea_plugin_integration +tests/python/pants_test/backend/python/tasks:integration +tests/python/pants_test/backend/python/tasks:isort_run_integration +tests/python/pants_test/backend/python/tasks:python_native_code_testing_1 +tests/python/pants_test/backend/python:integration +tests/python/pants_test/base:exception_sink_integration +tests/python/pants_test/base:exiter_integration +tests/python/pants_test/engine/legacy:changed_integration +tests/python/pants_test/engine/legacy:console_rule_integration +tests/python/pants_test/engine/legacy:graph_integration +tests/python/pants_test/engine/legacy:owners_integration +tests/python/pants_test/engine:scheduler_integration +tests/python/pants_test/option:options_integration +tests/python/pants_test/pantsd:pantsd_integration +tests/python/pants_test/projects:testprojects_integration +tests/python/pants_test/rules:test_integration diff --git a/build-support/travis/generate_travis_yml.py b/build-support/travis/generate_travis_yml.py index 03e89040fd5..08e12ddac74 100644 --- a/build-support/travis/generate_travis_yml.py +++ b/build-support/travis/generate_travis_yml.py @@ -8,8 +8,8 @@ import pystache -num_py3_integration_shards = 20 -num_py2_blacklist_integration_shards = 0 +num_py3_integration_shards = 13 +num_py2_blacklist_integration_shards = 7 num_cron_integration_shards = 20 diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index 3768b9884e6..03b9c2e4ffd 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -591,13 +591,14 @@ matrix: - ./build-support/bin/travis-ci.sh -2bn # TODO: get this passing with a Python 3 PEX. - - <<: *py3_linux_test_config - name: "Python contrib tests (Py3 PEX)" + - <<: *py2_linux_test_config + name: "Python contrib tests (Py2 PEX w/ Py3 constraints)" env: - - *py3_linux_test_config_env + - *py2_linux_test_config_env + - PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="['CPython>=3.6']" - CACHE_NAME=linuxcontribtests.py3 script: - - ./build-support/bin/travis-ci.sh -bn + - ./build-support/bin/travis-ci.sh -bn2 {{#py3_integration_shards}} - <<: *py3_linux_test_config diff --git a/src/python/pants/base/exception_sink.py b/src/python/pants/base/exception_sink.py index af75dff5131..9e885984a7f 100644 --- a/src/python/pants/base/exception_sink.py +++ b/src/python/pants/base/exception_sink.py @@ -14,7 +14,6 @@ from builtins import object, str import setproctitle -from future.utils import PY3 from pants.base.build_environment import get_buildroot from pants.base.exiter import Exiter @@ -382,4 +381,4 @@ def handle_signal_gracefully(cls, signum, frame): # Sets except hook. ExceptionSink.reset_exiter(Exiter(exiter=sys.exit)) # Sets a SIGUSR2 handler. -ExceptionSink.reset_interactive_output_stream(sys.stderr.buffer if PY3 else sys.stderr) +ExceptionSink.reset_interactive_output_stream(sys.stderr) diff --git a/src/python/pants/base/exiter.py b/src/python/pants/base/exiter.py index a7d450455ee..6b04fcecbb6 100644 --- a/src/python/pants/base/exiter.py +++ b/src/python/pants/base/exiter.py @@ -8,8 +8,6 @@ import sys from builtins import object -from future.utils import PY3 - from pants.util.strutil import ensure_binary @@ -53,10 +51,11 @@ def exit(self, result=0, msg=None, out=None): :param out: The file descriptor to emit `msg` to. (Optional) """ if msg: - out = out or (sys.stderr.buffer if PY3 else sys.stderr) + out = out or sys.stderr + # sys.stderr expects bytes in Py2, unicode in Py3 msg = ensure_binary(msg) try: - out.write(msg + b'\n') + print(msg, file=out) # TODO: Determine whether this call is a no-op because the stream gets flushed on exit, or # if we could lose what we just printed, e.g. if we get interrupted by a signal while # exiting and the stream is buffered like stdout. From 71eeb9af8d1f136c166734ef6c67b276953e9dd2 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Mon, 14 Jan 2019 09:02:42 -0700 Subject: [PATCH 092/105] Remove ./pants clean-all from ci.sh Because we set PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS before running any Pants code now, we no longer need the call `./pants clean-all` because it will properly constrain the subprocess interpreter the very first time we call `./pants`. --- build-support/bin/ci.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/build-support/bin/ci.sh b/build-support/bin/ci.sh index 2bb6e43bd39..cb301b91916 100755 --- a/build-support/bin/ci.sh +++ b/build-support/bin/ci.sh @@ -138,10 +138,6 @@ fi # integration tests that shell out to `./pants`, so we set this env var for those cases. export RUN_PANTS_FROM_PEX=1 -# TODO: Clear interpreters, otherwise the above subprocess interpreter constraint does not end up -# applying due to a cache bug between the `./pants binary` and further runs. -./pants.pex clean-all - if [[ "${run_pre_commit_checks:-false}" == "true" ]]; then start_travis_section "PreCommit" "Running pre-commit checks" FULL_CHECK=1 ./build-support/bin/pre-commit.sh || exit 1 From 50e0ce64076d1f7eeca52a609a3970c5e12ddc2a Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Mon, 14 Jan 2019 11:15:09 -0700 Subject: [PATCH 093/105] Remove changes to centos6/Dockerfile and copy travis_ci/Dockerfile Realized we don't need to be blocked by https://github.com/pantsbuild/pants/pull/7064, although it should certainly be addressed soon after by a followup PR. --- build-support/docker/centos6/Dockerfile | 12 +----------- build-support/docker/travis_ci/Dockerfile | 24 +++++++++-------------- 2 files changed, 10 insertions(+), 26 deletions(-) diff --git a/build-support/docker/centos6/Dockerfile b/build-support/docker/centos6/Dockerfile index c3d82dbf138..a733da334f9 100644 --- a/build-support/docker/centos6/Dockerfile +++ b/build-support/docker/centos6/Dockerfile @@ -15,17 +15,7 @@ RUN yum install -y \ python27 \ libffi \ libffi-devel \ - openssl-devel \ - wget - -# Install Python 3.6, which we must build because it's not available through yum -RUN cd /usr/src && \ - wget https://www.python.org/ftp/python/3.6.8/Python-3.6.8.tgz && \ - tar -xzf Python-3.6.8.tgz && \ - cd Python-3.6.8 && \ - ./configure --enable-optimizations && \ - make altinstall && \ - rm /usr/src/Python-3.6.8.tgz + openssl-devel # By default, execute in an environment with python27 enabled. ENTRYPOINT ["/usr/bin/scl", "enable", "python27", "devtoolset-7", "--"] diff --git a/build-support/docker/travis_ci/Dockerfile b/build-support/docker/travis_ci/Dockerfile index 8c5d08df424..1f273e3bb6b 100644 --- a/build-support/docker/travis_ci/Dockerfile +++ b/build-support/docker/travis_ci/Dockerfile @@ -4,22 +4,16 @@ # Use our custom Centos6 image for binary compatibility with old linux distros. FROM pantsbuild/centos6:latest -# ----------------------------------------------------------------------- -# This section belongs in centos6, but is here temporarily because -# centos6's build must first be published to Docker. - +# TODO(7064) These changes belong in centos6/Dockerfile, because we always want that image to +# have Py3 available and also avoid the cost of rebuilding Python 3 every time in CI. However, +# this requires publishing an updated image to Docker hub, so we copy the code here until then. +ARG PYTHON_3_VERSION=3.6.8 RUN yum install sqlite-devel -y -RUN mkdir -p /python-3-docker-build/python-3.6.8-{source,install} -RUN curl -L -v https://www.python.org/ftp/python/3.6.8/Python-3.6.8.tgz \ - | tar -zxvC /python-3-docker-build/python-3.6.8-source -RUN /usr/bin/scl enable devtoolset-7 -- \ - bash -c 'pushd /python-3-docker-build/python-3.6.8-source/Python-3.6.8 \ - && ./configure --prefix=/python-3-docker-build/python-3.6.8-install \ - --enable-optimizations \ - && make install' -ENV PATH "${PATH}:/python-3-docker-build/python-3.6.8-install/bin" - -# ----------------------------------------------------------------------- +ENV PYENV_ROOT /pyenv-docker-build +RUN mkdir ${PYENV_ROOT} +RUN git clone https://github.com/pyenv/pyenv ${PYENV_ROOT} +RUN /usr/bin/scl enable devtoolset-7 -- bash -c '${PYENV_ROOT}/bin/pyenv install ${PYTHON_3_VERSION}' +ENV PATH "${PYENV_ROOT}/shims:${PATH}" # Setup mount points for the travis ci user & workdir. VOLUME /travis/home From 0164c36017eb2e45efcdae35a5509f140182e153 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Mon, 14 Jan 2019 11:35:06 -0700 Subject: [PATCH 094/105] Remove pyenv PATH hack for osx shards Because we're using `pyenv global`, the line isn't necessary. --- .travis.yml | 2 -- build-support/travis/travis.yml.mustache | 2 -- 2 files changed, 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 838219abd89..d03e7ae8a09 100644 --- a/.travis.yml +++ b/.travis.yml @@ -175,7 +175,6 @@ py3_osx_config: &py3_osx_config LDFLAGS="-L/usr/local/opt/openssl/lib" CPPFLAGS="-I/usr/local/opt/openssl/include" PYENV_ROOT="${HOME}/.pyenv" - PATH="${PYENV_ROOT}/shims:${PATH}" before_install: - curl -L https://github.com/stedolan/jq/releases/download/jq-1.5/jq-osx-amd64 -o /usr/local/bin/jq - chmod 755 /usr/local/bin/jq @@ -211,7 +210,6 @@ py3_osx_test_config: &py3_osx_test_config LDFLAGS="-L/usr/local/opt/openssl/lib" CPPFLAGS="-I/usr/local/opt/openssl/include" PYENV_ROOT="${HOME}/.pyenv" - PATH="${PYENV_ROOT}/shims:${PATH}" BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.osx linux_with_fuse: &linux_with_fuse diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index 03b9c2e4ffd..4d9adb62e31 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -161,7 +161,6 @@ py3_osx_config: &py3_osx_config LDFLAGS="-L/usr/local/opt/openssl/lib" CPPFLAGS="-I/usr/local/opt/openssl/include" PYENV_ROOT="${HOME}/.pyenv" - PATH="${PYENV_ROOT}/shims:${PATH}" before_install: {{>before_install_osx}} # Clone pyenv directly from GitHub. For multiple osx images, brew's version of pyenv is too old to get @@ -195,7 +194,6 @@ py3_osx_test_config: &py3_osx_test_config LDFLAGS="-L/usr/local/opt/openssl/lib" CPPFLAGS="-I/usr/local/opt/openssl/include" PYENV_ROOT="${HOME}/.pyenv" - PATH="${PYENV_ROOT}/shims:${PATH}" BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.osx linux_with_fuse: &linux_with_fuse From 775bd4ee6a4a0c4ac4470d428e934ea680e7c707 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Mon, 14 Jan 2019 11:55:47 -0700 Subject: [PATCH 095/105] Add stage: *test to contrib py2 w/ py3 subprocesses Otherwise it will only run in cron job --- .travis.yml | 1 + build-support/travis/travis.yml.mustache | 1 + 2 files changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index d03e7ae8a09..bd59dc75825 100644 --- a/.travis.yml +++ b/.travis.yml @@ -612,6 +612,7 @@ matrix: # TODO: get this passing with a Python 3 PEX. - <<: *py2_linux_test_config name: "Python contrib tests (Py2 PEX w/ Py3 constraints)" + stage: *test env: - *py2_linux_test_config_env - PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="['CPython>=3.6']" diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index 4d9adb62e31..5ca710f662d 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -591,6 +591,7 @@ matrix: # TODO: get this passing with a Python 3 PEX. - <<: *py2_linux_test_config name: "Python contrib tests (Py2 PEX w/ Py3 constraints)" + stage: *test env: - *py2_linux_test_config_env - PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="['CPython>=3.6']" From 606c09d9a680aaf876140111dc7d651ef7688609 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Mon, 14 Jan 2019 12:18:46 -0700 Subject: [PATCH 096/105] Revert "Remove pyenv PATH hack for osx shards" This reverts commit 0164c36017eb2e45efcdae35a5509f140182e153. Apparently `pyenv global` doesn't work as intended, and we need to explicitly add Pyenv to the PATH. This also means can remove the `pyenv global` line. --- .travis.yml | 3 ++- build-support/travis/travis.yml.mustache | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index bd59dc75825..dcc9f09ddb3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -175,6 +175,7 @@ py3_osx_config: &py3_osx_config LDFLAGS="-L/usr/local/opt/openssl/lib" CPPFLAGS="-I/usr/local/opt/openssl/include" PYENV_ROOT="${HOME}/.pyenv" + PATH="${PYENV_ROOT}/shims:${PATH}" before_install: - curl -L https://github.com/stedolan/jq/releases/download/jq-1.5/jq-osx-amd64 -o /usr/local/bin/jq - chmod 755 /usr/local/bin/jq @@ -183,7 +184,6 @@ py3_osx_config: &py3_osx_config # modern Python 3.6. Pulling from GitHub instead allows us to use the same Python 3 version accross every OSX shard. - git clone https://github.com/pyenv/pyenv ${PYENV_ROOT} - ${PYENV_ROOT}/bin/pyenv install 3.6.8 - - ${PYENV_ROOT}/bin/pyenv global 3.6.8 base_osx_test_config: &base_osx_test_config <<: *pants_run_cache_config @@ -210,6 +210,7 @@ py3_osx_test_config: &py3_osx_test_config LDFLAGS="-L/usr/local/opt/openssl/lib" CPPFLAGS="-I/usr/local/opt/openssl/include" PYENV_ROOT="${HOME}/.pyenv" + PATH="${PYENV_ROOT}/shims:${PATH}" BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.osx linux_with_fuse: &linux_with_fuse diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index 5ca710f662d..fcf86bd31e1 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -161,13 +161,13 @@ py3_osx_config: &py3_osx_config LDFLAGS="-L/usr/local/opt/openssl/lib" CPPFLAGS="-I/usr/local/opt/openssl/include" PYENV_ROOT="${HOME}/.pyenv" + PATH="${PYENV_ROOT}/shims:${PATH}" before_install: {{>before_install_osx}} # Clone pyenv directly from GitHub. For multiple osx images, brew's version of pyenv is too old to get # modern Python 3.6. Pulling from GitHub instead allows us to use the same Python 3 version accross every OSX shard. - git clone https://github.com/pyenv/pyenv ${PYENV_ROOT} - ${PYENV_ROOT}/bin/pyenv install 3.6.8 - - ${PYENV_ROOT}/bin/pyenv global 3.6.8 base_osx_test_config: &base_osx_test_config <<: *pants_run_cache_config @@ -194,6 +194,7 @@ py3_osx_test_config: &py3_osx_test_config LDFLAGS="-L/usr/local/opt/openssl/lib" CPPFLAGS="-I/usr/local/opt/openssl/include" PYENV_ROOT="${HOME}/.pyenv" + PATH="${PYENV_ROOT}/shims:${PATH}" BOOTSTRAPPED_PEX_KEY_SUFFIX=py3.osx linux_with_fuse: &linux_with_fuse From d847facf072e9285514cd2dc836882c09a59fba8 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Mon, 14 Jan 2019 13:21:31 -0700 Subject: [PATCH 097/105] Make pyenv-installed py3 valid for linux engine shard We're hitting this issue https://github.com/pyenv/pyenv/issues/34. While it's discoverable, pyenv complains that we must be more explicit. Note in the centos6 image, we'll need `pyenv global 2.7.13 3.6.8` so that both interpreters are available. We only do py3 for this one. --- build-support/docker/travis_ci/Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/build-support/docker/travis_ci/Dockerfile b/build-support/docker/travis_ci/Dockerfile index 1f273e3bb6b..78f32109a8a 100644 --- a/build-support/docker/travis_ci/Dockerfile +++ b/build-support/docker/travis_ci/Dockerfile @@ -12,8 +12,9 @@ RUN yum install sqlite-devel -y ENV PYENV_ROOT /pyenv-docker-build RUN mkdir ${PYENV_ROOT} RUN git clone https://github.com/pyenv/pyenv ${PYENV_ROOT} -RUN /usr/bin/scl enable devtoolset-7 -- bash -c '${PYENV_ROOT}/bin/pyenv install ${PYTHON_3_VERSION}' -ENV PATH "${PYENV_ROOT}/shims:${PATH}" +RUN /usr/bin/scl enable devtoolset-7 -- bash -c '\ + ${PYENV_ROOT}/bin/pyenv install ${PYTHON_3_VERSION} \ + ${PYENV_ROOT}/bin/pyenv global ${PYTHON_3_VERSION}' # Setup mount points for the travis ci user & workdir. VOLUME /travis/home From ee83c8bbdb5d7c10391581ef61d3c7ebdd97dcd4 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Mon, 14 Jan 2019 13:30:57 -0700 Subject: [PATCH 098/105] Left off && in dockerfile --- build-support/docker/travis_ci/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-support/docker/travis_ci/Dockerfile b/build-support/docker/travis_ci/Dockerfile index 78f32109a8a..99e228968b3 100644 --- a/build-support/docker/travis_ci/Dockerfile +++ b/build-support/docker/travis_ci/Dockerfile @@ -14,7 +14,7 @@ RUN mkdir ${PYENV_ROOT} RUN git clone https://github.com/pyenv/pyenv ${PYENV_ROOT} RUN /usr/bin/scl enable devtoolset-7 -- bash -c '\ ${PYENV_ROOT}/bin/pyenv install ${PYTHON_3_VERSION} \ - ${PYENV_ROOT}/bin/pyenv global ${PYTHON_3_VERSION}' + && ${PYENV_ROOT}/bin/pyenv global ${PYTHON_3_VERSION}' # Setup mount points for the travis ci user & workdir. VOLUME /travis/home From 5d8a3400b34dcb6b46c647505798957c87734a8b Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Mon, 14 Jan 2019 13:34:17 -0700 Subject: [PATCH 099/105] Add back pyenv global to osx shard Same issue we were having on Linux. Even when exposing the path to pyenv, we must call pyenv global. --- .travis.yml | 1 + build-support/travis/travis.yml.mustache | 1 + 2 files changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index dcc9f09ddb3..645811cf73c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -184,6 +184,7 @@ py3_osx_config: &py3_osx_config # modern Python 3.6. Pulling from GitHub instead allows us to use the same Python 3 version accross every OSX shard. - git clone https://github.com/pyenv/pyenv ${PYENV_ROOT} - ${PYENV_ROOT}/bin/pyenv install 3.6.8 + - ${PYENV_ROOT}/bin/pyenv global 3.6.8 base_osx_test_config: &base_osx_test_config <<: *pants_run_cache_config diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index fcf86bd31e1..645bc903ba2 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -168,6 +168,7 @@ py3_osx_config: &py3_osx_config # modern Python 3.6. Pulling from GitHub instead allows us to use the same Python 3 version accross every OSX shard. - git clone https://github.com/pyenv/pyenv ${PYENV_ROOT} - ${PYENV_ROOT}/bin/pyenv install 3.6.8 + - ${PYENV_ROOT}/bin/pyenv global 3.6.8 base_osx_test_config: &base_osx_test_config <<: *pants_run_cache_config From 683e5b6925eb13f6b69d9e4f28e24f47c4708f40 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Mon, 14 Jan 2019 14:08:43 -0700 Subject: [PATCH 100/105] Add back pyenv path to linux docker We need it because we aren't running `eval "$(pyenv init -)"`, so we need to manually expose the PATH to work properly. --- build-support/docker/travis_ci/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/build-support/docker/travis_ci/Dockerfile b/build-support/docker/travis_ci/Dockerfile index 99e228968b3..8f6e63f04e1 100644 --- a/build-support/docker/travis_ci/Dockerfile +++ b/build-support/docker/travis_ci/Dockerfile @@ -15,6 +15,7 @@ RUN git clone https://github.com/pyenv/pyenv ${PYENV_ROOT} RUN /usr/bin/scl enable devtoolset-7 -- bash -c '\ ${PYENV_ROOT}/bin/pyenv install ${PYTHON_3_VERSION} \ && ${PYENV_ROOT}/bin/pyenv global ${PYTHON_3_VERSION}' +ENV PATH "${PYENV_ROOT}/shims:${PATH}" # Setup mount points for the travis ci user & workdir. VOLUME /travis/home From bcd6f97671791980185a01a43ba77b23bc0379e7 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Tue, 15 Jan 2019 09:30:05 -0700 Subject: [PATCH 101/105] Rename in ci.sh `pants_script` to `boostrap_pants_script` Good suggestion Daniel! --- build-support/bin/ci.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build-support/bin/ci.sh b/build-support/bin/ci.sh index cb301b91916..0fc6ce470fb 100755 --- a/build-support/bin/ci.sh +++ b/build-support/bin/ci.sh @@ -110,11 +110,11 @@ export PANTS_DEV=1 # or we will encounter the _Py_Dealloc error when running `./pants.pex -V` with Python 3 under-the-hood. if [[ "${python_two:-false}" == "false" ]]; then py_version_number="3" - pants_script="./pants3" + bootstrap_pants_script="./pants3" export PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS='["CPython>=3.6,<4"]' else py_version_number="2" - pants_script="./pants" + bootstrap_pants_script="./pants" fi banner "Using Python ${py_version_number} to execute subprocesses." @@ -124,7 +124,7 @@ if [[ "${run_bootstrap:-true}" == "true" ]]; then if [[ "${run_bootstrap_clean:-false}" == "true" ]]; then ./build-support/python/clean.sh || die "Failed to clean before bootstrapping pants." fi - ${pants_script} binary \ + ${bootstrap_pants_script} binary \ src/python/pants/bin:pants_local_binary && \ mv dist/pants_local_binary.pex pants.pex && \ ./pants.pex -V From 0e07e72f964399b4c209ad903ced883ee088c4dd Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Tue, 15 Jan 2019 09:43:35 -0700 Subject: [PATCH 102/105] Remove py2 vs py3 distinction for linux_rust_tests It doesn't use a PEX, so we don't need py2 vs py3 setup. Instead, we specify it has no pex and run it in daily CI (no cron). --- .travis.yml | 32 ++++++------------------ build-support/travis/travis.yml.mustache | 32 ++++++------------------ 2 files changed, 14 insertions(+), 50 deletions(-) diff --git a/.travis.yml b/.travis.yml index 645811cf73c..b6e62b7f186 100644 --- a/.travis.yml +++ b/.travis.yml @@ -118,11 +118,11 @@ base_linux_config: &base_linux_config py2_linux_config: &py2_linux_config <<: *base_linux_config - python: &python_version "2.7" + python: &python2_version "2.7" py3_linux_config: &py3_linux_config <<: *base_linux_config - python: &python_version "3.6" + python: &python3_version "3.6" base_linux_test_config: &base_linux_test_config <<: *base_linux_config @@ -481,38 +481,21 @@ deploy_unstable_multiplatform_pex: &deploy_unstable_multiplatform_pex # Rust tests # ------------------------------------------------------------------------- -base_linux_rust_tests: &base_linux_rust_tests +linux_rust_tests: &linux_rust_tests <<: *linux_with_fuse <<: *native_engine_cache_config + name: "Linux Rust tests (No PEX)" env: - CACHE_NAME=linuxrusttests os: linux dist: xenial sudo: required language: python + python: *python3_version + stage: *test before_script: - ulimit -c unlimited - ulimit -n 8192 - -py2_linux_rust_tests: &py2_linux_rust_tests - <<: *base_linux_rust_tests - name: "Linux Rust tests (Py2 PEX)" - python: 2.7 - env: - - *py2_linux_test_config_env - - CACHE_NAME=linuxrusttests.py2 - script: - - ./build-support/bin/travis-ci.sh -be2 - -py3_linux_rust_tests: &py3_linux_rust_tests - <<: *base_linux_rust_tests - name: "Linux Rust tests (Py3 PEX)" - python: 3.6 - stage: *test - env: - - *py3_linux_test_config_env - - CACHE_NAME=linuxrusttests.py3 - script: - ./build-support/bin/travis-ci.sh -be base_osx_rust_tests: &base_osx_rust_tests @@ -957,8 +940,7 @@ matrix: - ./build-support/bin/travis-ci.sh -bc2 -i 19/20 - - <<: *py2_linux_rust_tests - - <<: *py3_linux_rust_tests + - <<: *linux_rust_tests # TODO: get this to pass using Python 3. In the meantime, just use Python 2. - <<: *py2_osx_rust_tests diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index 645bc903ba2..090c843e7c6 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -111,11 +111,11 @@ base_linux_config: &base_linux_config py2_linux_config: &py2_linux_config <<: *base_linux_config - python: &python_version "2.7" + python: &python2_version "2.7" py3_linux_config: &py3_linux_config <<: *base_linux_config - python: &python_version "3.6" + python: &python3_version "3.6" base_linux_test_config: &base_linux_test_config <<: *base_linux_config @@ -460,38 +460,21 @@ deploy_unstable_multiplatform_pex: &deploy_unstable_multiplatform_pex # Rust tests # ------------------------------------------------------------------------- -base_linux_rust_tests: &base_linux_rust_tests +linux_rust_tests: &linux_rust_tests <<: *linux_with_fuse <<: *native_engine_cache_config + name: "Linux Rust tests (No PEX)" env: - CACHE_NAME=linuxrusttests os: linux dist: xenial sudo: required language: python + python: *python3_version + stage: *test before_script: - ulimit -c unlimited - ulimit -n 8192 - -py2_linux_rust_tests: &py2_linux_rust_tests - <<: *base_linux_rust_tests - name: "Linux Rust tests (Py2 PEX)" - python: 2.7 - env: - - *py2_linux_test_config_env - - CACHE_NAME=linuxrusttests.py2 - script: - - ./build-support/bin/travis-ci.sh -be2 - -py3_linux_rust_tests: &py3_linux_rust_tests - <<: *base_linux_rust_tests - name: "Linux Rust tests (Py3 PEX)" - python: 3.6 - stage: *test - env: - - *py3_linux_test_config_env - - CACHE_NAME=linuxrusttests.py3 - script: - ./build-support/bin/travis-ci.sh -be base_osx_rust_tests: &base_osx_rust_tests @@ -634,8 +617,7 @@ matrix: {{/cron_integration_shards}} - - <<: *py2_linux_rust_tests - - <<: *py3_linux_rust_tests + - <<: *linux_rust_tests # TODO: get this to pass using Python 3. In the meantime, just use Python 2. - <<: *py2_osx_rust_tests From 6d35aea0cb2122f995bc652dce2cd706a4edee44 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Tue, 15 Jan 2019 09:48:22 -0700 Subject: [PATCH 103/105] Move linux_rust_clippy and cargo_audit out of test matrix Cleaner to define them at the top root level, and simply extend in test matrix. This matches how we do it for every other shard (except for integration shard because there we need mustache magic). --- .travis.yml | 64 +++++++++++++----------- build-support/travis/travis.yml.mustache | 64 +++++++++++++----------- 2 files changed, 72 insertions(+), 56 deletions(-) diff --git a/.travis.yml b/.travis.yml index b6e62b7f186..19f0419a660 100644 --- a/.travis.yml +++ b/.travis.yml @@ -530,6 +530,40 @@ py3_osx_rust_tests: &py3_osx_rust_tests script: - ./build-support/bin/travis-ci.sh -bez +# ------------------------------------------------------------------------- +# Rust audits +# ------------------------------------------------------------------------- + +linux_rust_clippy: &linux_rust_clippy + <<: *linux_with_fuse + <<: *native_engine_cache_config + name: "Linux Rust Clippy (No PEX)" + env: + - CACHE_NAME=linuxclippy + os: linux + dist: xenial + sudo: required + stage: *test + language: python + python: *python3_version + before_script: + - ulimit -c unlimited + - ulimit -n 8192 + script: + - ./build-support/bin/travis-ci.sh -bs + +cargo_audit: &cargo_audit + <<: *linux_with_fuse + name: "Cargo audit (No PEX)" + env: + - CACHE_NAME=linuxcargoaudit + os: linux + dist: xenial + sudo: required + stage: *cron + script: + - ./build-support/bin/travis-ci.sh -ba + # ------------------------------------------------------------------------- # Test matrix # ------------------------------------------------------------------------- @@ -947,34 +981,8 @@ matrix: stage: *test # - <<: *py3_osx_rust_tests - - <<: *linux_with_fuse - <<: *native_engine_cache_config - name: "Linux Rust Clippy (No PEX)" - env: - - CACHE_NAME=linuxclippy - os: linux - dist: xenial - sudo: required - stage: *test - language: python - python: *python_version - before_script: - - ulimit -c unlimited - - ulimit -n 8192 - script: - - ./build-support/bin/travis-ci.sh -bs2 - - - <<: *linux_with_fuse - name: "Cargo audit (No PEX)" - env: - - *py2_linux_test_config_env - - CACHE_NAME=linuxcargoaudit - os: linux - dist: xenial - sudo: required - stage: *cron - script: - - ./build-support/bin/travis-ci.sh -ba2 + - <<: *linux_rust_clippy + - <<: *cargo_audit deploy: # Deploy whatever a previous stage has left in dist/deploy. diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index 090c843e7c6..f9bc81a113a 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -509,6 +509,40 @@ py3_osx_rust_tests: &py3_osx_rust_tests script: - ./build-support/bin/travis-ci.sh -bez +# ------------------------------------------------------------------------- +# Rust audits +# ------------------------------------------------------------------------- + +linux_rust_clippy: &linux_rust_clippy + <<: *linux_with_fuse + <<: *native_engine_cache_config + name: "Linux Rust Clippy (No PEX)" + env: + - CACHE_NAME=linuxclippy + os: linux + dist: xenial + sudo: required + stage: *test + language: python + python: *python3_version + before_script: + - ulimit -c unlimited + - ulimit -n 8192 + script: + - ./build-support/bin/travis-ci.sh -bs + +cargo_audit: &cargo_audit + <<: *linux_with_fuse + name: "Cargo audit (No PEX)" + env: + - CACHE_NAME=linuxcargoaudit + os: linux + dist: xenial + sudo: required + stage: *cron + script: + - ./build-support/bin/travis-ci.sh -ba + # ------------------------------------------------------------------------- # Test matrix # ------------------------------------------------------------------------- @@ -624,34 +658,8 @@ matrix: stage: *test # - <<: *py3_osx_rust_tests - - <<: *linux_with_fuse - <<: *native_engine_cache_config - name: "Linux Rust Clippy (No PEX)" - env: - - CACHE_NAME=linuxclippy - os: linux - dist: xenial - sudo: required - stage: *test - language: python - python: *python_version - before_script: - - ulimit -c unlimited - - ulimit -n 8192 - script: - - ./build-support/bin/travis-ci.sh -bs2 - - - <<: *linux_with_fuse - name: "Cargo audit (No PEX)" - env: - - *py2_linux_test_config_env - - CACHE_NAME=linuxcargoaudit - os: linux - dist: xenial - sudo: required - stage: *cron - script: - - ./build-support/bin/travis-ci.sh -ba2 + - <<: *linux_rust_clippy + - <<: *cargo_audit deploy: # Deploy whatever a previous stage has left in dist/deploy. From 74a33e28923470108a7375a50cc9092621c53fe9 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Wed, 16 Jan 2019 09:47:10 -0700 Subject: [PATCH 104/105] Fix linux rust tests not using `script` section I accidently had the ci.sh command in `before_script`, rather than `script`. While everything had an exit code of 0, Travis didn't know what we're trying to test so would fail. --- .travis.yml | 1 + build-support/travis/travis.yml.mustache | 1 + 2 files changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 19f0419a660..b2d20898c93 100644 --- a/.travis.yml +++ b/.travis.yml @@ -496,6 +496,7 @@ linux_rust_tests: &linux_rust_tests before_script: - ulimit -c unlimited - ulimit -n 8192 + script: - ./build-support/bin/travis-ci.sh -be base_osx_rust_tests: &base_osx_rust_tests diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index f9bc81a113a..58ba3677823 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -475,6 +475,7 @@ linux_rust_tests: &linux_rust_tests before_script: - ulimit -c unlimited - ulimit -n 8192 + script: - ./build-support/bin/travis-ci.sh -be base_osx_rust_tests: &base_osx_rust_tests From 8e3d28302070f6cbecdd4ae959062b35e29a4d01 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Wed, 16 Jan 2019 11:44:53 -0700 Subject: [PATCH 105/105] Make ci banners more explicit / avoid phrase "under-the-hood" Thanks Benjy for pointing this out and working with me on a few iterations! This rewrite tries to be more explicit with what is still actually happening, while keeping things fairly concise as they're Travis banners. --- build-support/bin/ci.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build-support/bin/ci.sh b/build-support/bin/ci.sh index 0fc6ce470fb..832f74edaa1 100755 --- a/build-support/bin/ci.sh +++ b/build-support/bin/ci.sh @@ -105,9 +105,9 @@ esac # We're running against a Pants clone. export PANTS_DEV=1 -# Determine interpreter for both under-the-hood and for subprocesses. +# Determine which Python interpreter to use for bootstrapping pex and for executing subprocesses. # Order matters here. We must constrain subprocesses before running the bootstrap stage, -# or we will encounter the _Py_Dealloc error when running `./pants.pex -V` with Python 3 under-the-hood. +# or we will encounter the _Py_Dealloc error when bootstrapping a Python 3 PEX. if [[ "${python_two:-false}" == "false" ]]; then py_version_number="3" bootstrap_pants_script="./pants3" @@ -116,10 +116,10 @@ else py_version_number="2" bootstrap_pants_script="./pants" fi -banner "Using Python ${py_version_number} to execute subprocesses." +banner "Using Python ${py_version_number} to execute spawned subprocesses (e.g. tests)" if [[ "${run_bootstrap:-true}" == "true" ]]; then - start_travis_section "Bootstrap" "Bootstrapping pants with Python ${py_version_number} under-the-hood." + start_travis_section "Bootstrap" "Bootstrapping pants as a Python ${py_version_number} PEX" ( if [[ "${run_bootstrap_clean:-false}" == "true" ]]; then ./build-support/python/clean.sh || die "Failed to clean before bootstrapping pants."