From c835267b55bb0f7c7dc287eba750a6efcf30276e Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Mon, 11 Mar 2019 07:24:42 -0700 Subject: [PATCH] Refactor .travis.yml to deduplicate uses of Docker and use Mustache comments (#7351) ### Problem #7235 introduced an initial deduplication of launching docker images, notably allowing the image name to be parametrized. While working on #7197, it became evident how frequently the same code was being repeated for calling `docker run`. The only thing that changes between these invocations is what goes inside `sh -c "my bash command"`. This duplication makes our Travis code harder to understand, along with the usual arguments for DRY. ### Solution Introduce a new `docker_run_image` mustache template that takes the parameters `$docker_image_name` and `$docker_run_command`. Also changed: * rename `launch_docker_image` -> `docker_build_image` * remove script entry for `&travis_docker_image`, as it gets overridden every time by its caller so is unnecessary and confusing. * introduce Mustache comments to some of our templates. These don't get copied over into `.travis.yml`, which is good for reducing noise, while still allowing us to explain the concept in the template. ### Result CI should perform the same with the benefit of a shorter and better abstracted `travis.yml.mustache`. --- .travis.yml | 78 ++++++++----------- .../travis/before_install_linux.mustache | 4 +- ...e.mustache => docker_build_image.mustache} | 2 + .../travis/docker_run_image.mustache | 11 +++ .../travis/env_osx_with_pyenv.mustache | 5 ++ build-support/travis/generate_travis_yml.py | 6 +- build-support/travis/travis.yml.mustache | 52 +++++-------- 7 files changed, 76 insertions(+), 82 deletions(-) rename build-support/travis/{launch_docker_image.mustache => docker_build_image.mustache} (63%) create mode 100644 build-support/travis/docker_run_image.mustache diff --git a/.travis.yml b/.travis.yml index 4d7db0fd187..48fb659fece 100644 --- a/.travis.yml +++ b/.travis.yml @@ -131,8 +131,6 @@ base_linux_test_config: &base_linux_test_config before_install: - PATH="/usr/lib/jvm/java-8-openjdk-amd64/jre/bin":$PATH - JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 - # Increase the max number of user watches to ensure that watchman is able to watch all - # files in the working copy. - sudo sysctl fs.inotify.max_user_watches=524288 - ./build-support/bin/install_aws_cli_for_ci.sh before_script: @@ -218,8 +216,6 @@ linux_with_fuse: &linux_with_fuse before_install: - PATH="/usr/lib/jvm/java-8-openjdk-amd64/jre/bin":$PATH - JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 - # Increase the max number of user watches to ensure that watchman is able to watch all - # files in the working copy. - sudo sysctl fs.inotify.max_user_watches=524288 - ./build-support/bin/install_aws_cli_for_ci.sh - sudo apt-get install -y pkg-config fuse libfuse-dev @@ -232,15 +228,6 @@ travis_docker_image: &travis_docker_image - docker before_script: - ulimit -c unlimited - script: - - > - docker build - --rm -t ${docker_image_name} - --build-arg "TRAVIS_USER=$(id -un)" - --build-arg "TRAVIS_UID=$(id -u)" - --build-arg "TRAVIS_GROUP=$(id -gn)" - --build-arg "TRAVIS_GID=$(id -g)" - build-support/docker/${docker_image_name}/ # ------------------------------------------------------------------------- # Bootstrap engine shards @@ -250,6 +237,8 @@ base_linux_build_engine: &base_linux_build_engine <<: *native_engine_cache_config <<: *travis_docker_image stage: *bootstrap + # Callers of this anchor are expected to provide values in their `env` for + # `docker_image_name` and `docker_run_command` (i.e. the bash command(s) to run). script: - > docker build @@ -259,14 +248,13 @@ base_linux_build_engine: &base_linux_build_engine --build-arg "TRAVIS_GROUP=$(id -gn)" --build-arg "TRAVIS_GID=$(id -g)" build-support/docker/${docker_image_name}/ - # Note that: - # * We mount ${HOME} to cache the ${HOME}/.cache/pants/rust-toolchain. - # * We also build fs_util, to take advantage of the rust code built during bootstrapping. - - docker run --rm -t + - > + docker run + --rm -t -v "${HOME}:/travis/home" -v "${TRAVIS_BUILD_DIR}:/travis/workdir" ${docker_image_name}:latest - sh -c "./build-support/bin/ci.sh ${BOOTSTRAP_ARGS} && ./build-support/bin/release.sh -f" + sh -c "${docker_run_command}" - aws --no-sign-request --region us-east-1 s3 cp ${TRAVIS_BUILD_DIR}/pants.pex ${BOOTSTRAPPED_PEX_URL_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX} py27_linux_build_engine: &py27_linux_build_engine @@ -275,6 +263,8 @@ py27_linux_build_engine: &py27_linux_build_engine name: "Build Linux native engine and pants.pex (Py2.7 PEX)" env: - docker_image_name=travis_ci + # Note that we also build fs_util, to take advantage of the rust code built during bootstrapping. + - docker_run_command="./build-support/bin/ci.sh -2b && ./build-support/bin/release.sh -f" # NB: Only the Py2.7 shard sets PREPARE_DEPLOY to cause the fs_util binary to be uploaded to S3: # either linux shard could upload this binary, since it does not depend on python at all. - PREPARE_DEPLOY=1 @@ -288,6 +278,8 @@ py36_linux_build_engine: &py36_linux_build_engine name: "Build Linux native engine and pants.pex (Py3.6 PEX)" env: - docker_image_name=travis_ci + # Note that we also build fs_util, to take advantage of the rust code built during bootstrapping. + - docker_run_command="./build-support/bin/ci.sh -b && ./build-support/bin/release.sh -f" - CACHE_NAME=linuxpexbuild.py36 - BOOTSTRAPPED_PEX_KEY_SUFFIX=py36.linux - BOOTSTRAP_ARGS='-b' @@ -409,16 +401,8 @@ py27_linux_build_wheels_no_ucs: &py27_linux_build_wheels_no_ucs # compatibility. This is a Py2.7 shard, so it is not subject to #6985. <<: *travis_docker_image <<: *base_build_wheels - -py27_linux_build_wheels_ucs2: &py27_linux_build_wheels_ucs2 - <<: *py27_linux_config - <<: *py27_linux_build_wheels_no_ucs - <<: *native_engine_cache_config - name: "Build wheels - Linux and cp27m (UCS2)" - env: - - *base_build_wheels_env - - docker_image_name=travis_ci_py27_ucs2 - - CACHE_NAME=linuxwheelsbuild.ucs2 + # Callers of this anchor are expected to provide values in their `env` for + # `docker_image_name` and `docker_run_command` (i.e. the bash command(s) to run). script: - > docker build @@ -428,13 +412,26 @@ py27_linux_build_wheels_ucs2: &py27_linux_build_wheels_ucs2 --build-arg "TRAVIS_GROUP=$(id -gn)" --build-arg "TRAVIS_GID=$(id -g)" build-support/docker/${docker_image_name}/ - - docker run --rm -t + - > + docker run + --rm -t -v "${HOME}:/travis/home" -v "${TRAVIS_BUILD_DIR}:/travis/workdir" ${docker_image_name}:latest - sh -c "./build-support/bin/ci.sh -2b - && ./build-support/bin/check_pants_pex_abi.py cp27m - && RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -n" + sh -c "${docker_run_command}" + +py27_linux_build_wheels_ucs2: &py27_linux_build_wheels_ucs2 + <<: *py27_linux_config + <<: *py27_linux_build_wheels_no_ucs + <<: *native_engine_cache_config + name: "Build wheels - Linux and cp27m (UCS2)" + env: + - *base_build_wheels_env + - docker_image_name=travis_ci_py27_ucs2 + - docker_run_command="./build-support/bin/ci.sh -2b + && ./build-support/bin/check_pants_pex_abi.py cp27m + && RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -n" + - CACHE_NAME=linuxwheelsbuild.ucs2 py27_linux_build_wheels_ucs4: &py27_linux_build_wheels_ucs4 <<: *py27_linux_build_wheels_no_ucs @@ -446,22 +443,9 @@ py27_linux_build_wheels_ucs4: &py27_linux_build_wheels_ucs4 - *py27_linux_test_config_env - *base_build_wheels_env - docker_image_name=travis_ci + - docker_run_command="./build-support/bin/check_pants_pex_abi.py cp27mu + && RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -n" - CACHE_NAME=linuxwheelsbuild.ucs4 - script: - - > - docker build - --rm -t ${docker_image_name} - --build-arg "TRAVIS_USER=$(id -un)" - --build-arg "TRAVIS_UID=$(id -u)" - --build-arg "TRAVIS_GROUP=$(id -gn)" - --build-arg "TRAVIS_GID=$(id -g)" - build-support/docker/${docker_image_name}/ - - docker run --rm -t - -v "${HOME}:/travis/home" - -v "${TRAVIS_BUILD_DIR}:/travis/workdir" - ${docker_image_name}:latest - sh -c "./build-support/bin/check_pants_pex_abi.py cp27mu - && RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -n" py27_osx_build_wheels_no_ucs: &py27_osx_build_wheels_no_ucs <<: *base_build_wheels diff --git a/build-support/travis/before_install_linux.mustache b/build-support/travis/before_install_linux.mustache index fa160b114ba..07e1d9fb33b 100644 --- a/build-support/travis/before_install_linux.mustache +++ b/build-support/travis/before_install_linux.mustache @@ -1,6 +1,6 @@ - PATH="/usr/lib/jvm/java-8-openjdk-amd64/jre/bin":$PATH - JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 -# Increase the max number of user watches to ensure that watchman is able to watch all -# files in the working copy. +{{! Increase the max number of user watches to ensure that watchman is able to watch all +# files in the working copy. }} - sudo sysctl fs.inotify.max_user_watches=524288 - ./build-support/bin/install_aws_cli_for_ci.sh diff --git a/build-support/travis/launch_docker_image.mustache b/build-support/travis/docker_build_image.mustache similarity index 63% rename from build-support/travis/launch_docker_image.mustache rename to build-support/travis/docker_build_image.mustache index 9a8ce0e1080..6b7cb06d7bf 100644 --- a/build-support/travis/launch_docker_image.mustache +++ b/build-support/travis/docker_build_image.mustache @@ -4,4 +4,6 @@ docker build --build-arg "TRAVIS_UID=$(id -u)" --build-arg "TRAVIS_GROUP=$(id -gn)" --build-arg "TRAVIS_GID=$(id -g)" +{{! $docker_image_name is an environment variable with an expected value like `travis_ci`. +It should be set in the env section.}} build-support/docker/${docker_image_name}/ diff --git a/build-support/travis/docker_run_image.mustache b/build-support/travis/docker_run_image.mustache new file mode 100644 index 00000000000..55f0fa0c7a8 --- /dev/null +++ b/build-support/travis/docker_run_image.mustache @@ -0,0 +1,11 @@ +{{! Note we mount ${HOME} to cache the ${HOME}/.cache/pants/rust-toolchain. }} +docker run +--rm -t +-v "${HOME}:/travis/home" +-v "${TRAVIS_BUILD_DIR}:/travis/workdir" +{{! $docker_image_name is an environment variable with an expected value like `travis_ci`. +It should be set in the env section.}} +${docker_image_name}:latest +{{! $docker_run_command is an environment variable with an expected string value like +`"echo 'hello' && exit 1"`. It should be set in the env section.}} +sh -c "${docker_run_command}" diff --git a/build-support/travis/env_osx_with_pyenv.mustache b/build-support/travis/env_osx_with_pyenv.mustache index df639a77a9f..0f729bef150 100644 --- a/build-support/travis/env_osx_with_pyenv.mustache +++ b/build-support/travis/env_osx_with_pyenv.mustache @@ -1,5 +1,10 @@ +{{! These flags are necessary to get OpenSSL working. +See https://github.com/pyenv/pyenv/wiki/Common-build-problems#error-the-python-ssl-extension-was-not-compiled-missing-the-openssl-lib. }} PATH="/usr/local/opt/openssl/bin:$PATH" LDFLAGS="-L/usr/local/opt/openssl/lib" CPPFLAGS="-I/usr/local/opt/openssl/include" +{{! OSX shards that use Pyenv do so by directly cloning the repository, as several OSX images +have outdated versions of pyenv that don't include the Python versions we need. +So, we set these environment variables here to get this solution working. }} PYENV_ROOT="${HOME}/.pyenv" PATH="${PYENV_ROOT}/shims:${PATH}" diff --git a/build-support/travis/generate_travis_yml.py b/build-support/travis/generate_travis_yml.py index a029d2c6166..eb830e529e9 100644 --- a/build-support/travis/generate_travis_yml.py +++ b/build-support/travis/generate_travis_yml.py @@ -32,7 +32,8 @@ def get_mustache_file(file_name): before_install_linux = get_mustache_file('before_install_linux.mustache') before_install_osx = get_mustache_file('before_install_osx.mustache') env_osx_with_pyenv = get_mustache_file('env_osx_with_pyenv.mustache') - launch_docker_image = get_mustache_file('launch_docker_image.mustache') + docker_build_image = get_mustache_file('docker_build_image.mustache') + docker_run_image = get_mustache_file('docker_run_image.mustache') context = { 'header': HEADER, @@ -47,6 +48,7 @@ def get_mustache_file(file_name): 'before_install_linux': before_install_linux, 'before_install_osx': before_install_osx, 'env_osx_with_pyenv': env_osx_with_pyenv, - 'launch_docker_image': launch_docker_image + 'docker_build_image': docker_build_image, + 'docker_run_image': docker_run_image }) print(renderer.render(template, context)) diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index b2c1374abe1..0d8cf93e5e1 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -203,9 +203,6 @@ travis_docker_image: &travis_docker_image - docker before_script: - ulimit -c unlimited - script: - - > - {{>launch_docker_image}} # ------------------------------------------------------------------------- # Bootstrap engine shards @@ -215,17 +212,13 @@ base_linux_build_engine: &base_linux_build_engine <<: *native_engine_cache_config <<: *travis_docker_image stage: *bootstrap + # Callers of this anchor are expected to provide values in their `env` for + # `docker_image_name` and `docker_run_command` (i.e. the bash command(s) to run). script: - > - {{>launch_docker_image}} - # Note that: - # * We mount ${HOME} to cache the ${HOME}/.cache/pants/rust-toolchain. - # * We also build fs_util, to take advantage of the rust code built during bootstrapping. - - docker run --rm -t - -v "${HOME}:/travis/home" - -v "${TRAVIS_BUILD_DIR}:/travis/workdir" - ${docker_image_name}:latest - sh -c "./build-support/bin/ci.sh ${BOOTSTRAP_ARGS} && ./build-support/bin/release.sh -f" + {{>docker_build_image}} + - > + {{>docker_run_image}} - aws --no-sign-request --region us-east-1 s3 cp ${TRAVIS_BUILD_DIR}/pants.pex ${BOOTSTRAPPED_PEX_URL_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX} py27_linux_build_engine: &py27_linux_build_engine @@ -234,6 +227,8 @@ py27_linux_build_engine: &py27_linux_build_engine name: "Build Linux native engine and pants.pex (Py2.7 PEX)" env: - docker_image_name=travis_ci + # Note that we also build fs_util, to take advantage of the rust code built during bootstrapping. + - docker_run_command="./build-support/bin/ci.sh -2b && ./build-support/bin/release.sh -f" # NB: Only the Py2.7 shard sets PREPARE_DEPLOY to cause the fs_util binary to be uploaded to S3: # either linux shard could upload this binary, since it does not depend on python at all. - PREPARE_DEPLOY=1 @@ -247,6 +242,8 @@ py36_linux_build_engine: &py36_linux_build_engine name: "Build Linux native engine and pants.pex (Py3.6 PEX)" env: - docker_image_name=travis_ci + # Note that we also build fs_util, to take advantage of the rust code built during bootstrapping. + - docker_run_command="./build-support/bin/ci.sh -b && ./build-support/bin/release.sh -f" - CACHE_NAME=linuxpexbuild.py36 - BOOTSTRAPPED_PEX_KEY_SUFFIX=py36.linux - BOOTSTRAP_ARGS='-b' @@ -368,6 +365,13 @@ py27_linux_build_wheels_no_ucs: &py27_linux_build_wheels_no_ucs # compatibility. This is a Py2.7 shard, so it is not subject to #6985. <<: *travis_docker_image <<: *base_build_wheels + # Callers of this anchor are expected to provide values in their `env` for + # `docker_image_name` and `docker_run_command` (i.e. the bash command(s) to run). + script: + - > + {{>docker_build_image}} + - > + {{>docker_run_image}} py27_linux_build_wheels_ucs2: &py27_linux_build_wheels_ucs2 <<: *py27_linux_config @@ -377,17 +381,10 @@ py27_linux_build_wheels_ucs2: &py27_linux_build_wheels_ucs2 env: - *base_build_wheels_env - docker_image_name=travis_ci_py27_ucs2 + - docker_run_command="./build-support/bin/ci.sh -2b + && ./build-support/bin/check_pants_pex_abi.py cp27m + && RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -n" - CACHE_NAME=linuxwheelsbuild.ucs2 - script: - - > - {{>launch_docker_image}} - - docker run --rm -t - -v "${HOME}:/travis/home" - -v "${TRAVIS_BUILD_DIR}:/travis/workdir" - ${docker_image_name}:latest - sh -c "./build-support/bin/ci.sh -2b - && ./build-support/bin/check_pants_pex_abi.py cp27m - && RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -n" py27_linux_build_wheels_ucs4: &py27_linux_build_wheels_ucs4 <<: *py27_linux_build_wheels_no_ucs @@ -399,16 +396,9 @@ py27_linux_build_wheels_ucs4: &py27_linux_build_wheels_ucs4 - *py27_linux_test_config_env - *base_build_wheels_env - docker_image_name=travis_ci + - docker_run_command="./build-support/bin/check_pants_pex_abi.py cp27mu + && RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -n" - CACHE_NAME=linuxwheelsbuild.ucs4 - script: - - > - {{>launch_docker_image}} - - docker run --rm -t - -v "${HOME}:/travis/home" - -v "${TRAVIS_BUILD_DIR}:/travis/workdir" - ${docker_image_name}:latest - sh -c "./build-support/bin/check_pants_pex_abi.py cp27mu - && RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -n" py27_osx_build_wheels_no_ucs: &py27_osx_build_wheels_no_ucs <<: *base_build_wheels