Use remote execution with the lint CI shard (#9156)

Now that we are using V2 isort and docformatter, and will soon use Black, we are running into the atrocious performance Pants currently has when running with a completely cold cache due to per-target caching: https://docs.google.com/document/d/1Tdof6jx9aVaOGeIQeI9Gn-8x7nd6LfjnJ0QrbLFBbgc/edit#. We no longer store `.lmbd_store` in CI, so this means that we always have a cold cache and our lint shard was frequently timing out. While we can fix this by simply using `travis-wait`, we would still have extremely slow results. Instead, we can use remote caching to have this shard only run over targets that have actually changed.
pantsbuild · Feb 21, 2020 · d690ab2 · d690ab2
1 parent 9a18727
commit d690ab2
Show file tree

Hide file tree

Showing 3 changed files with 38 additions and 19 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -381,8 +381,9 @@ matrix:
     - '3.6'
     - '3.7'
     script:
-    - travis-wait-enhanced --timeout 40m --interval 9m ./build-support/bin/ci.py --githooks
-      --sanity-checks --doc-gen --lint --python-version 3.6
+    - travis-wait-enhanced --timeout 40m --interval 9m -- ./build-support/bin/ci.py
+      --remote-execution-enabled --githooks --sanity-checks --doc-gen --lint --python-version
+      3.6
     stage: Test Pants
     sudo: required
   - addons:
@@ -442,8 +443,9 @@ matrix:
     - '3.6'
     - '3.7'
     script:
-    - travis-wait-enhanced --timeout 40m --interval 9m ./build-support/bin/ci.py --githooks
-      --sanity-checks --doc-gen --lint --python-version 3.7
+    - travis-wait-enhanced --timeout 40m --interval 9m -- ./build-support/bin/ci.py
+      --remote-execution-enabled --githooks --sanity-checks --doc-gen --lint --python-version
+      3.7
     stage: Test Pants (Cron)
     sudo: required
   - before_cache:

diff --git a/build-support/bin/ci.py b/build-support/bin/ci.py
@@ -34,7 +34,7 @@ def main() -> None:
     if args.sanity_checks:
       run_sanity_checks()
     if args.lint:
-      run_lint()
+      run_lint(oauth_token_path=remote_execution_oauth_token_path)
     if args.doc_gen:
       run_doc_gen_tests()
     if args.clippy:
@@ -219,14 +219,8 @@ def pants_command(
       self.v1_chroot: ["./pants.pex", "test.pytest", *sorted(targets), *PYTEST_PASSTHRU_ARGS],
       self.v2_local: ["./pants.pex", "--no-v1", "--v2", "test", *sorted(targets)],
       self.v2_remote: [
-                        "./pants.pex",
-                        "--no-v1",
-                        "--v2",
-                        "--pants-config-files=pants.remote.toml",
-                        f"--remote-oauth-bearer-token-path={oauth_token_path}",
-                        "test",
-                        *sorted(targets),
-                      ]
+        "./pants.pex", *_use_remote_execution(oauth_token_path or ""), "test", *sorted(targets)
+      ]
     }[self]  # type: ignore[index]  # issues with understanding `self`
     if shard is not None and self in [self.v1_no_chroot, self.v1_chroot]:  # type: ignore[comparison-overlap]  # issues with understanding `self`
       result.insert(2, f"--test-pytest-test-shard={shard}")
@@ -321,6 +315,15 @@ def check_pants_pex_exists() -> None:
 PYTEST_PASSTHRU_ARGS = ["--", "-q", "-rfa"]
 
 
+def _use_remote_execution(oauth_token_path: str) -> List[str]:
+  return [
+    "--no-v1",
+    "--v2",
+    "--pants-config-files=pants.remote.toml",
+    f"--remote-oauth-bearer-token-path={oauth_token_path}",
+  ]
+
+
 def _run_command(
   command: List[str], *, slug: str, start_message: str, die_message: str, requires_pex: bool = True
 ) -> None:
@@ -370,12 +373,25 @@ def run_check(command: List[str]) -> None:
       run_check(check)
 
 
-def run_lint() -> None:
+def run_lint(*, oauth_token_path: Optional[str] = None) -> None:
   targets = ["contrib::", "examples::", "src::", "tests::", "zinc::"]
+  command_prefix = ["./pants.pex", "--tag=-nolint"]
+
+  v2_command = (
+    [*command_prefix, "lint2", *targets]
+    if oauth_token_path is None
+    else [*command_prefix, *_use_remote_execution(oauth_token_path), "lint2", *targets]
+  )
+  _run_command(
+    v2_command,
+    slug="Lint (V2)",
+    start_message="Running V2 lint checks",
+    die_message="Lint check failure.",
+  )
   _run_command(
-    ["./pants.pex", "--tag=-nolint", "lint", "lint2", *targets],
-    slug="Lint",
-    start_message="Running lint checks",
+    [*command_prefix, "lint", *targets],
+    slug="Lint (V1)",
+    start_message="Running V1 lint checks",
     die_message="Lint check failure.",
   )
 

diff --git a/build-support/bin/generate_travis_yml.py b/build-support/bin/generate_travis_yml.py
@@ -436,8 +436,9 @@ def lint(python_version: PythonVersion) -> Dict:
     **linux_shard(python_version=python_version, install_travis_wait=True),
     "name": f"Self-checks and lint (Python {python_version.decimal})",
     "script": [
-      f"travis-wait-enhanced --timeout 40m --interval 9m ./build-support/bin/ci.py --githooks "
-      f"--sanity-checks --doc-gen --lint --python-version {python_version.decimal}"
+      "travis-wait-enhanced --timeout 40m --interval 9m -- ./build-support/bin/ci.py "
+      "--remote-execution-enabled --githooks --sanity-checks --doc-gen --lint "
+      f"--python-version {python_version.decimal}"
     ]
   }
   safe_append(shard, "env", f"CACHE_NAME=lint.py{python_version.number}")