Refactor .travis.yml to deduplicate uses of Docker and use Mustache c…

…omments (#7351) ### Problem #7235 introduced an initial deduplication of launching docker images, notably allowing the image name to be parametrized. While working on #7197, it became evident how frequently the same code was being repeated for calling `docker run`. The only thing that changes between these invocations is what goes inside `sh -c "my bash command"`. This duplication makes our Travis code harder to understand, along with the usual arguments for DRY. ### Solution Introduce a new `docker_run_image` mustache template that takes the parameters `$docker_image_name` and `$docker_run_command`. Also changed: * rename `launch_docker_image` -> `docker_build_image` * remove script entry for `&travis_docker_image`, as it gets overridden every time by its caller so is unnecessary and confusing. * introduce Mustache comments to some of our templates. These don't get copied over into `.travis.yml`, which is good for reducing noise, while still allowing us to explain the concept in the template. ### Result CI should perform the same with the benefit of a shorter and better abstracted `travis.yml.mustache`.
pantsbuild · Mar 11, 2019 · c835267 · c835267
1 parent 63d430e
commit c835267
Show file tree

Hide file tree

Showing 7 changed files with 76 additions and 82 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -131,8 +131,6 @@ base_linux_test_config: &base_linux_test_config
   before_install:
     - PATH="/usr/lib/jvm/java-8-openjdk-amd64/jre/bin":$PATH
     - JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
-    # Increase the max number of user watches to ensure that watchman is able to watch all
-    # files in the working copy.
     - sudo sysctl fs.inotify.max_user_watches=524288
     - ./build-support/bin/install_aws_cli_for_ci.sh
   before_script:
@@ -218,8 +216,6 @@ linux_with_fuse: &linux_with_fuse
   before_install:
     - PATH="/usr/lib/jvm/java-8-openjdk-amd64/jre/bin":$PATH
     - JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
-    # Increase the max number of user watches to ensure that watchman is able to watch all
-    # files in the working copy.
     - sudo sysctl fs.inotify.max_user_watches=524288
     - ./build-support/bin/install_aws_cli_for_ci.sh
     - sudo apt-get install -y pkg-config fuse libfuse-dev
@@ -232,15 +228,6 @@ travis_docker_image: &travis_docker_image
     - docker
   before_script:
     - ulimit -c unlimited
-  script:
-    - >
-      docker build
-      --rm -t ${docker_image_name}
-      --build-arg "TRAVIS_USER=$(id -un)"
-      --build-arg "TRAVIS_UID=$(id -u)"
-      --build-arg "TRAVIS_GROUP=$(id -gn)"
-      --build-arg "TRAVIS_GID=$(id -g)"
-      build-support/docker/${docker_image_name}/
 
 # -------------------------------------------------------------------------
 # Bootstrap engine shards
@@ -250,6 +237,8 @@ base_linux_build_engine: &base_linux_build_engine
   <<: *native_engine_cache_config
   <<: *travis_docker_image
   stage: *bootstrap
+  # Callers of this anchor are expected to provide values in their `env` for
+  # `docker_image_name` and `docker_run_command` (i.e. the bash command(s) to run).
   script:
     - >
       docker build
@@ -259,14 +248,13 @@ base_linux_build_engine: &base_linux_build_engine
       --build-arg "TRAVIS_GROUP=$(id -gn)"
       --build-arg "TRAVIS_GID=$(id -g)"
       build-support/docker/${docker_image_name}/
-    # Note that:
-    # * We mount ${HOME} to cache the ${HOME}/.cache/pants/rust-toolchain.
-    # * We also build fs_util, to take advantage of the rust code built during bootstrapping.
-    - docker run --rm -t
+    - >
+      docker run
+      --rm -t
       -v "${HOME}:/travis/home"
       -v "${TRAVIS_BUILD_DIR}:/travis/workdir"
       ${docker_image_name}:latest
-      sh -c "./build-support/bin/ci.sh ${BOOTSTRAP_ARGS} && ./build-support/bin/release.sh -f"
+      sh -c "${docker_run_command}"
     - aws --no-sign-request --region us-east-1 s3 cp ${TRAVIS_BUILD_DIR}/pants.pex ${BOOTSTRAPPED_PEX_URL_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX}
 
 py27_linux_build_engine: &py27_linux_build_engine
@@ -275,6 +263,8 @@ py27_linux_build_engine: &py27_linux_build_engine
   name: "Build Linux native engine and pants.pex (Py2.7 PEX)"
   env:
     - docker_image_name=travis_ci
+    # Note that we also build fs_util, to take advantage of the rust code built during bootstrapping.
+    - docker_run_command="./build-support/bin/ci.sh -2b && ./build-support/bin/release.sh -f"
     # NB: Only the Py2.7 shard sets PREPARE_DEPLOY to cause the fs_util binary to be uploaded to S3:
     # either linux shard could upload this binary, since it does not depend on python at all.
     - PREPARE_DEPLOY=1
@@ -288,6 +278,8 @@ py36_linux_build_engine: &py36_linux_build_engine
   name: "Build Linux native engine and pants.pex (Py3.6 PEX)"
   env:
     - docker_image_name=travis_ci
+    # Note that we also build fs_util, to take advantage of the rust code built during bootstrapping.
+    - docker_run_command="./build-support/bin/ci.sh -b && ./build-support/bin/release.sh -f"
     - CACHE_NAME=linuxpexbuild.py36
     - BOOTSTRAPPED_PEX_KEY_SUFFIX=py36.linux
     - BOOTSTRAP_ARGS='-b'
@@ -409,16 +401,8 @@ py27_linux_build_wheels_no_ucs: &py27_linux_build_wheels_no_ucs
   # compatibility. This is a Py2.7 shard, so it is not subject to #6985.
   <<: *travis_docker_image
   <<: *base_build_wheels
-
-py27_linux_build_wheels_ucs2: &py27_linux_build_wheels_ucs2
-  <<: *py27_linux_config
-  <<: *py27_linux_build_wheels_no_ucs
-  <<: *native_engine_cache_config
-  name: "Build wheels - Linux and cp27m (UCS2)"
-  env:
-    - *base_build_wheels_env
-    - docker_image_name=travis_ci_py27_ucs2
-    - CACHE_NAME=linuxwheelsbuild.ucs2
+  # Callers of this anchor are expected to provide values in their `env` for
+  # `docker_image_name` and `docker_run_command` (i.e. the bash command(s) to run).
   script:
     - >
       docker build
@@ -428,13 +412,26 @@ py27_linux_build_wheels_ucs2: &py27_linux_build_wheels_ucs2
       --build-arg "TRAVIS_GROUP=$(id -gn)"
       --build-arg "TRAVIS_GID=$(id -g)"
       build-support/docker/${docker_image_name}/
-    - docker run --rm -t
+    - >
+      docker run
+      --rm -t
       -v "${HOME}:/travis/home"
       -v "${TRAVIS_BUILD_DIR}:/travis/workdir"
       ${docker_image_name}:latest
-      sh -c "./build-support/bin/ci.sh -2b
-             && ./build-support/bin/check_pants_pex_abi.py cp27m
-             && RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -n"
+      sh -c "${docker_run_command}"
+
+py27_linux_build_wheels_ucs2: &py27_linux_build_wheels_ucs2
+  <<: *py27_linux_config
+  <<: *py27_linux_build_wheels_no_ucs
+  <<: *native_engine_cache_config
+  name: "Build wheels - Linux and cp27m (UCS2)"
+  env:
+    - *base_build_wheels_env
+    - docker_image_name=travis_ci_py27_ucs2
+    - docker_run_command="./build-support/bin/ci.sh -2b
+                          && ./build-support/bin/check_pants_pex_abi.py cp27m
+                          && RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -n"
+    - CACHE_NAME=linuxwheelsbuild.ucs2
 
 py27_linux_build_wheels_ucs4: &py27_linux_build_wheels_ucs4
   <<: *py27_linux_build_wheels_no_ucs
@@ -446,22 +443,9 @@ py27_linux_build_wheels_ucs4: &py27_linux_build_wheels_ucs4
     - *py27_linux_test_config_env
     - *base_build_wheels_env
     - docker_image_name=travis_ci
+    - docker_run_command="./build-support/bin/check_pants_pex_abi.py cp27mu
+                          && RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -n"
     - CACHE_NAME=linuxwheelsbuild.ucs4
-  script:
-    - >
-      docker build
-      --rm -t ${docker_image_name}
-      --build-arg "TRAVIS_USER=$(id -un)"
-      --build-arg "TRAVIS_UID=$(id -u)"
-      --build-arg "TRAVIS_GROUP=$(id -gn)"
-      --build-arg "TRAVIS_GID=$(id -g)"
-      build-support/docker/${docker_image_name}/
-    - docker run --rm -t
-      -v "${HOME}:/travis/home"
-      -v "${TRAVIS_BUILD_DIR}:/travis/workdir"
-      ${docker_image_name}:latest
-      sh -c "./build-support/bin/check_pants_pex_abi.py cp27mu
-             && RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -n"
 
 py27_osx_build_wheels_no_ucs: &py27_osx_build_wheels_no_ucs
   <<: *base_build_wheels

diff --git a/build-support/travis/before_install_linux.mustache b/build-support/travis/before_install_linux.mustache
@@ -1,6 +1,6 @@
 - PATH="/usr/lib/jvm/java-8-openjdk-amd64/jre/bin":$PATH
 - JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
-# Increase the max number of user watches to ensure that watchman is able to watch all
-# files in the working copy.
+{{! Increase the max number of user watches to ensure that watchman is able to watch all
+# files in the working copy. }}
 - sudo sysctl fs.inotify.max_user_watches=524288
 - ./build-support/bin/install_aws_cli_for_ci.sh
diff --git a/...pport/travis/launch_docker_image.mustache → ...upport/travis/docker_build_image.mustache b/...pport/travis/launch_docker_image.mustache → ...upport/travis/docker_build_image.mustache
@@ -4,4 +4,6 @@ docker build
 --build-arg "TRAVIS_UID=$(id -u)"
 --build-arg "TRAVIS_GROUP=$(id -gn)"
 --build-arg "TRAVIS_GID=$(id -g)"
+{{! $docker_image_name is an environment variable with an expected value like `travis_ci`.
+It should be set in the env section.}}
 build-support/docker/${docker_image_name}/
diff --git a/build-support/travis/docker_run_image.mustache b/build-support/travis/docker_run_image.mustache
@@ -0,0 +1,11 @@
+{{! Note we mount ${HOME} to cache the ${HOME}/.cache/pants/rust-toolchain. }}
+docker run
+--rm -t
+-v "${HOME}:/travis/home"
+-v "${TRAVIS_BUILD_DIR}:/travis/workdir"
+{{! $docker_image_name is an environment variable with an expected value like `travis_ci`.
+It should be set in the env section.}}
+${docker_image_name}:latest
+{{! $docker_run_command is an environment variable with an expected string value like
+`"echo 'hello' && exit 1"`. It should be set in the env section.}}
+sh -c "${docker_run_command}"
diff --git a/build-support/travis/env_osx_with_pyenv.mustache b/build-support/travis/env_osx_with_pyenv.mustache
@@ -1,5 +1,10 @@
+{{! These flags are necessary to get OpenSSL working.
+See https://github.com/pyenv/pyenv/wiki/Common-build-problems#error-the-python-ssl-extension-was-not-compiled-missing-the-openssl-lib. }}
 PATH="/usr/local/opt/openssl/bin:$PATH"
 LDFLAGS="-L/usr/local/opt/openssl/lib"
 CPPFLAGS="-I/usr/local/opt/openssl/include"
+{{! OSX shards that use Pyenv do so by directly cloning the repository, as several OSX images
+have outdated versions of pyenv that don't include the Python versions we need.
+So, we set these environment variables here to get this solution working. }}
 PYENV_ROOT="${HOME}/.pyenv"
 PATH="${PYENV_ROOT}/shims:${PATH}"
diff --git a/build-support/travis/generate_travis_yml.py b/build-support/travis/generate_travis_yml.py
@@ -32,7 +32,8 @@ def get_mustache_file(file_name):
   before_install_linux = get_mustache_file('before_install_linux.mustache')
   before_install_osx = get_mustache_file('before_install_osx.mustache')
   env_osx_with_pyenv = get_mustache_file('env_osx_with_pyenv.mustache')
-  launch_docker_image = get_mustache_file('launch_docker_image.mustache')
+  docker_build_image = get_mustache_file('docker_build_image.mustache')
+  docker_run_image = get_mustache_file('docker_run_image.mustache')
 
   context = {
     'header': HEADER,
@@ -47,6 +48,7 @@ def get_mustache_file(file_name):
     'before_install_linux': before_install_linux,
     'before_install_osx': before_install_osx,
     'env_osx_with_pyenv': env_osx_with_pyenv,
-    'launch_docker_image': launch_docker_image
+    'docker_build_image': docker_build_image,
+    'docker_run_image': docker_run_image
   })
   print(renderer.render(template, context))
diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache
@@ -203,9 +203,6 @@ travis_docker_image: &travis_docker_image
     - docker
   before_script:
     - ulimit -c unlimited
-  script:
-    - >
-      {{>launch_docker_image}}
 
 # -------------------------------------------------------------------------
 # Bootstrap engine shards
@@ -215,17 +212,13 @@ base_linux_build_engine: &base_linux_build_engine
   <<: *native_engine_cache_config
   <<: *travis_docker_image
   stage: *bootstrap
+  # Callers of this anchor are expected to provide values in their `env` for
+  # `docker_image_name` and `docker_run_command` (i.e. the bash command(s) to run).
   script:
     - >
-      {{>launch_docker_image}}
-    # Note that:
-    # * We mount ${HOME} to cache the ${HOME}/.cache/pants/rust-toolchain.
-    # * We also build fs_util, to take advantage of the rust code built during bootstrapping.
-    - docker run --rm -t
-      -v "${HOME}:/travis/home"
-      -v "${TRAVIS_BUILD_DIR}:/travis/workdir"
-      ${docker_image_name}:latest
-      sh -c "./build-support/bin/ci.sh ${BOOTSTRAP_ARGS} && ./build-support/bin/release.sh -f"
+      {{>docker_build_image}}
+    - >
+      {{>docker_run_image}}
     - aws --no-sign-request --region us-east-1 s3 cp ${TRAVIS_BUILD_DIR}/pants.pex ${BOOTSTRAPPED_PEX_URL_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX}
 
 py27_linux_build_engine: &py27_linux_build_engine
@@ -234,6 +227,8 @@ py27_linux_build_engine: &py27_linux_build_engine
   name: "Build Linux native engine and pants.pex (Py2.7 PEX)"
   env:
     - docker_image_name=travis_ci
+    # Note that we also build fs_util, to take advantage of the rust code built during bootstrapping.
+    - docker_run_command="./build-support/bin/ci.sh -2b && ./build-support/bin/release.sh -f"
     # NB: Only the Py2.7 shard sets PREPARE_DEPLOY to cause the fs_util binary to be uploaded to S3:
     # either linux shard could upload this binary, since it does not depend on python at all.
     - PREPARE_DEPLOY=1
@@ -247,6 +242,8 @@ py36_linux_build_engine: &py36_linux_build_engine
   name: "Build Linux native engine and pants.pex (Py3.6 PEX)"
   env:
     - docker_image_name=travis_ci
+    # Note that we also build fs_util, to take advantage of the rust code built during bootstrapping.
+    - docker_run_command="./build-support/bin/ci.sh -b && ./build-support/bin/release.sh -f"
     - CACHE_NAME=linuxpexbuild.py36
     - BOOTSTRAPPED_PEX_KEY_SUFFIX=py36.linux
     - BOOTSTRAP_ARGS='-b'
@@ -368,6 +365,13 @@ py27_linux_build_wheels_no_ucs: &py27_linux_build_wheels_no_ucs
   # compatibility. This is a Py2.7 shard, so it is not subject to #6985.
   <<: *travis_docker_image
   <<: *base_build_wheels
+  # Callers of this anchor are expected to provide values in their `env` for
+  # `docker_image_name` and `docker_run_command` (i.e. the bash command(s) to run).
+  script:
+    - >
+      {{>docker_build_image}}
+    - >
+      {{>docker_run_image}}
 
 py27_linux_build_wheels_ucs2: &py27_linux_build_wheels_ucs2
   <<: *py27_linux_config
@@ -377,17 +381,10 @@ py27_linux_build_wheels_ucs2: &py27_linux_build_wheels_ucs2
   env:
     - *base_build_wheels_env
     - docker_image_name=travis_ci_py27_ucs2
+    - docker_run_command="./build-support/bin/ci.sh -2b
+                          && ./build-support/bin/check_pants_pex_abi.py cp27m
+                          && RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -n"
     - CACHE_NAME=linuxwheelsbuild.ucs2
-  script:
-    - >
-      {{>launch_docker_image}}
-    - docker run --rm -t
-      -v "${HOME}:/travis/home"
-      -v "${TRAVIS_BUILD_DIR}:/travis/workdir"
-      ${docker_image_name}:latest
-      sh -c "./build-support/bin/ci.sh -2b
-             && ./build-support/bin/check_pants_pex_abi.py cp27m
-             && RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -n"
 
 py27_linux_build_wheels_ucs4: &py27_linux_build_wheels_ucs4
   <<: *py27_linux_build_wheels_no_ucs
@@ -399,16 +396,9 @@ py27_linux_build_wheels_ucs4: &py27_linux_build_wheels_ucs4
     - *py27_linux_test_config_env
     - *base_build_wheels_env
     - docker_image_name=travis_ci
+    - docker_run_command="./build-support/bin/check_pants_pex_abi.py cp27mu
+                          && RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -n"
     - CACHE_NAME=linuxwheelsbuild.ucs4
-  script:
-    - >
-      {{>launch_docker_image}}
-    - docker run --rm -t
-      -v "${HOME}:/travis/home"
-      -v "${TRAVIS_BUILD_DIR}:/travis/workdir"
-      ${docker_image_name}:latest
-      sh -c "./build-support/bin/check_pants_pex_abi.py cp27mu
-             && RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -n"
 
 py27_osx_build_wheels_no_ucs: &py27_osx_build_wheels_no_ucs
   <<: *base_build_wheels