Upgrade to Pex 2.1.48 and leverage packed layout. (pantsbuild#12808)

Pex 2.1.48 brings `--layout {packed,loose}` alternate layouts for PEXes.
These are both more friendly to remote caching, leading to smaller
artifacts to cache and greater cache hit ratios in the face of
requirement changes. Since the loose layout still does not perform well
with the local CAS scheme, we use packed for now.

Fixes pantsbuild#12548
Fixes pantsbuild#12688
Fixes pantsbuild#12803

[ci skip-rust]

[ci skip-build-wheels]

3rdparty/python/lockfiles/user_reqs.txt

@@ -5,7 +5,7 @@
 # --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE ---
 # {
 #   "version": 1,
-#   "requirements_invalidation_digest": "19770133e0608f747845bc429b61942d2f79b3cad46790962ade28db07e2b4fd",
+#   "requirements_invalidation_digest": "6adf6599ccc90c6aad4cc62333b089eea937311f47ae9eab025203d7a0e0eb2c",
 #   "valid_for_interpreter_constraints": [
 #     "CPython<3.10,>=3.7"
 #   ]
@@ -113,9 +113,9 @@ iniconfig==1.1.1; python_version >= "3.6" \
 packaging==21.0; python_version >= "3.6" \
     --hash=sha256:c86254f9220d55e31cc94d69bade760f0847da8000def4dfe1c6b872fd14ff14 \
     --hash=sha256:7dc96269f53a4ccec5c0670940a4281106dd0bb343f47b7471f779df49c2fbe7
-pex==2.1.47; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0" and python_version < "3.10") \
-    --hash=sha256:2341bad1146136237f2d7a7440fe706f010879febe0d8a714e6cd92b8ba47ce9 \
-    --hash=sha256:0928d0316caac840db528030fc741930e8be22a3fa6a8635308fb8443a0a0c6a
+pex==2.1.48; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0" and python_version < "3.10") \
+    --hash=sha256:903ab1e2781dbf077e9936d6a5dada9fa868c5416423065eee7bd00d94f2ea06 \
+    --hash=sha256:5f6a489075c5bbecdb36a42249cd52cfd882e205242f80a1f1e2294951ab46e7
 pluggy==1.0.0; python_version >= "3.6" \
     --hash=sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3 \
     --hash=sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159

3rdparty/python/requirements.txt

@@ -8,7 +8,7 @@ humbug==0.2.6
 
 ijson==3.1.4
 packaging==21.0
-pex==2.1.47
+pex==2.1.48
 psutil==5.8.0
 pystache==0.5.4
 # This should be kept in sync with `pytest.py`.

build-support/bin/_release_helper.py

@@ -598,7 +598,6 @@ def build_pex(fetch: bool) -> None:
                 str(CONSTANTS.deploy_3rdparty_wheel_dir / CONSTANTS.pants_unstable_version),
                 "--no-strip-pex-env",
                 "--console-script=pants",
-                "--unzip",
                 *extra_pex_args,
                 f"pantsbuild.pants=={CONSTANTS.pants_unstable_version}",
             ],

src/python/pants/backend/python/goals/package_pex_binary.py

@@ -71,8 +71,6 @@ def _execution_mode(self) -> PexExecutionMode:
 
     def generate_additional_args(self, pex_binary_defaults: PexBinaryDefaults) -> Tuple[str, ...]:
         args = []
-        if self.always_write_cache.value is True:
-            args.append("--always-write-cache")
         if self.emit_warnings.value_or_global_default(pex_binary_defaults) is False:
             args.append("--no-emit-warnings")
         if self.ignore_errors.value is True:
@@ -81,12 +79,8 @@ def generate_additional_args(self, pex_binary_defaults: PexBinaryDefaults) -> Tu
             args.append(f"--inherit-path={self.inherit_path.value}")
         if self.shebang.value is not None:
             args.append(f"--python-shebang={self.shebang.value}")
-        if self.zip_safe.value is False:
-            args.append("--not-zip-safe")
         if self.strip_env.value is False:
             args.append("--no-strip-pex-env")
-        if self._execution_mode is PexExecutionMode.UNZIP:
-            args.append("--unzip")
         if self._execution_mode is PexExecutionMode.VENV:
             args.extend(("--venv", "prepend"))
         if self.include_tools.value is True:

src/python/pants/backend/python/goals/pytest_runner.py

@@ -13,13 +13,7 @@
 )
 from pants.backend.python.subsystems.pytest import PyTest, PythonTestFieldSet
 from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints
-from pants.backend.python.util_rules.pex import (
-    Pex,
-    PexRequest,
-    VenvPex,
-    VenvPexProcess,
-    pex_path_closure,
-)
+from pants.backend.python.util_rules.pex import Pex, PexRequest, VenvPex, VenvPexProcess
 from pants.backend.python.util_rules.pex_from_targets import PexFromTargetsRequest
 from pants.backend.python.util_rules.python_sources import (
     PythonSourceFiles,
@@ -222,7 +216,7 @@ async def setup_pytest_for_target(
             interpreter_constraints=interpreter_constraints,
             main=pytest.main,
             internal_only=True,
-            pex_path=pex_path_closure([pytest_pex, requirements_pex]),
+            pex_path=[pytest_pex, requirements_pex],
         ),
     )
     config_files_get = Get(

src/python/pants/backend/python/goals/repl.py

@@ -2,7 +2,7 @@
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
 from pants.backend.python.subsystems.ipython import IPython
-from pants.backend.python.util_rules.pex import Pex, PexRequest, pex_path_closure
+from pants.backend.python.util_rules.pex import Pex, PexRequest
 from pants.backend.python.util_rules.pex_environment import PexEnvironment
 from pants.backend.python.util_rules.pex_from_targets import PexFromTargetsRequest
 from pants.backend.python.util_rules.python_sources import (
@@ -106,7 +106,7 @@ async def create_ipython_repl_request(
     chrooted_source_roots = [repl.in_chroot(sr) for sr in sources.source_roots]
     extra_env = {
         **complete_pex_env.environment_dict(python_configured=ipython_pex.python is not None),
-        "PEX_PATH": ":".join(repl.in_chroot(p.name) for p in pex_path_closure([requirements_pex])),
+        "PEX_PATH": repl.in_chroot(requirements_pex_request.output_filename),
         "PEX_EXTRA_SYS_PATH": ":".join(chrooted_source_roots),
     }
 

src/python/pants/backend/python/goals/run_pex_binary.py

@@ -9,7 +9,7 @@
     ResolvedPexEntryPoint,
     ResolvePexEntryPointRequest,
 )
-from pants.backend.python.util_rules.pex import Pex, PexRequest, pex_path_closure
+from pants.backend.python.util_rules.pex import Pex, PexRequest
 from pants.backend.python.util_rules.pex_environment import PexEnvironment
 from pants.backend.python.util_rules.pex_from_targets import PexFromTargetsRequest
 from pants.backend.python.util_rules.python_sources import (
@@ -94,7 +94,7 @@ def in_chroot(relpath: str) -> str:
     chrooted_source_roots = [in_chroot(sr) for sr in sources.source_roots]
     extra_env = {
         **complete_pex_env.environment_dict(python_configured=runner_pex.python is not None),
-        "PEX_PATH": ":".join(in_chroot(p.name) for p in pex_path_closure([requirements])),
+        "PEX_PATH": in_chroot(requirements_pex_request.output_filename),
         "PEX_EXTRA_SYS_PATH": ":".join(chrooted_source_roots),
     }
 

src/python/pants/backend/python/goals/run_pex_binary_integration_test.py

@@ -88,7 +88,6 @@ def run(*extra_args: str, **extra_env: str) -> PantsResult:
         result = run("--", "info", PEX_TOOLS="1")
         assert result.exit_code == 0
         pex_info = json.loads(result.stdout)
-        assert (execution_mode is PexExecutionMode.UNZIP) == pex_info["unzip"]
         assert (execution_mode is PexExecutionMode.VENV) == pex_info["venv"]
         assert ("prepend" if execution_mode is PexExecutionMode.VENV else "false") == pex_info[
             "venv_bin_path"

src/python/pants/backend/python/lint/pylint/rules.py

@@ -13,13 +13,7 @@
 from pants.backend.python.target_types import InterpreterConstraintsField
 from pants.backend.python.util_rules import pex_from_targets
 from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints
-from pants.backend.python.util_rules.pex import (
-    Pex,
-    PexRequest,
-    VenvPex,
-    VenvPexProcess,
-    pex_path_closure,
-)
+from pants.backend.python.util_rules.pex import Pex, PexRequest, VenvPex, VenvPexProcess
 from pants.backend.python.util_rules.pex_from_targets import PexFromTargetsRequest
 from pants.backend.python.util_rules.python_sources import (
     PythonSourceFiles,
@@ -134,7 +128,7 @@ async def pylint_lint_partition(
                 interpreter_constraints=partition.interpreter_constraints,
                 main=pylint.main,
                 internal_only=True,
-                pex_path=pex_path_closure([pylint_pex, requirements_pex]),
+                pex_path=[pylint_pex, requirements_pex],
             ),
         ),
         Get(

src/python/pants/backend/python/target_types.py

@@ -10,7 +10,7 @@
 from dataclasses import dataclass
 from enum import Enum
 from textwrap import dedent
-from typing import TYPE_CHECKING, Dict, Iterable, Iterator, Optional, Tuple, Union, cast
+from typing import TYPE_CHECKING, Any, Dict, Iterable, Iterator, Optional, Tuple, Union, cast
 
 from packaging.utils import canonicalize_name as canonicalize_project_name
 from pkg_resources import Requirement
@@ -20,6 +20,7 @@
     DEFAULT_TYPE_STUB_MODULE_MAPPING,
 )
 from pants.backend.python.macros.python_artifact import PythonArtifact
+from pants.base.deprecated import warn_or_error
 from pants.core.goals.package import OutputPathField
 from pants.core.goals.test import RuntimePackageDependenciesField
 from pants.engine.addresses import Address, Addresses
@@ -281,6 +282,7 @@ def compute_value(
         return super().compute_value(raw_value, address)
 
 
+# TODO(John Sirois): Deprecate: https://github.com/pantsbuild/pants/issues/12803
 class PexZipSafeField(BoolField):
     alias = "zip_safe"
     default = True
@@ -289,6 +291,11 @@ class PexZipSafeField(BoolField):
         "not zip safe, it will be written to disk prior to execution. You may need to mark "
         "`zip_safe=False` if you're having issues loading your code."
     )
+    removal_version = "2.9.0.dev0"
+    removal_hint = (
+        "All PEX binaries now unpack your code to disk prior to first execution; so this option no "
+        "longer needs to be specified."
+    )
 
 
 class PexStripEnvField(BoolField):
@@ -311,6 +318,12 @@ class PexAlwaysWriteCacheField(BoolField):
         "Whether PEX should always write the .deps cache of the .pex file to disk or not. This "
         "can use less memory in RAM-constrained environments."
     )
+    removal_version = "2.9.0.dev0"
+    removal_hint = (
+        "This option never had any effect when passed to Pex and the Pex option is now removed "
+        "altogether. PEXes always write all their internal dependencies out to disk as part of "
+        "first execution bootstrapping."
+    )
 
 
 class PexIgnoreErrorsField(BoolField):
@@ -354,18 +367,33 @@ class PexExecutionModeField(StringField):
     expected_type = str
     default = PexExecutionMode.ZIPAPP.value
     help = (
-        "The mode the generated PEX file will run in.\n\nThe traditional PEX file runs in "
-        f"{PexExecutionMode.ZIPAPP.value!r} mode (See: https://www.python.org/dev/peps/pep-0441/). "
-        f"In general, faster cold start times can be attained using the "
-        f"{PexExecutionMode.UNZIP.value!r} mode which also has the benefit of allowing standard "
-        "use of `__file__` and filesystem APIs to access code and resources in the PEX.\n\nThe "
-        f"fastest execution mode in the steady state is {PexExecutionMode.VENV.value!r}, which "
-        "generates a virtual environment from the PEX file on first run, but then achieves near "
-        "native virtual environment start times. This mode also benefits from a traditional "
-        "virtual environment `sys.path`, giving maximum compatibility with stdlib and third party "
-        "APIs."
+        "The mode the generated PEX file will run in.\n\nThe traditional PEX file runs in a "
+        f"modified {PexExecutionMode.ZIPAPP.value!r} mode (See: "
+        "https://www.python.org/dev/peps/pep-0441/) where zipped internal code and dependencies "
+        "are first unpacked to disk. This mode achieves the fastest cold start times and may, for "
+        "example be the best choice for cloud lambda functions.\n\nThe fastest execution mode in "
+        f"the steady state is {PexExecutionMode.VENV.value!r}, which generates a virtual "
+        "environment from the PEX file on first run, but then achieves near native virtual "
+        "environment start times. This mode also benefits from a traditional virtual environment "
+        "`sys.path`, giving maximum compatibility with stdlib and third party APIs.\n\nThe "
+        f"{PexExecutionMode.UNZIP.value!r} mode is deprecated since the default "
+        f"{PexExecutionMode.ZIPAPP.value!r} mode now executes this way."
     )
 
+    @classmethod
+    def _check_deprecated(cls, raw_value: Optional[Any], address_: Address) -> None:
+        if PexExecutionMode.UNZIP.value == raw_value:
+            warn_or_error(
+                removal_version="2.9.0.dev0",
+                entity=f"the {cls.alias!r} field {PexExecutionMode.UNZIP.value!r} value",
+                hint=(
+                    f"The {PexExecutionMode.UNZIP.value!r} mode is now the default PEX execution "
+                    "mode; so you can remove this field setting or explicitly choose the default "
+                    f"of {PexExecutionMode.ZIPAPP.value!r} and get the same benefits you already "
+                    "enjoy from this mode."
+                ),
+            )
+
 
 class PexIncludeToolsField(BoolField):
     alias = "include_tools"

src/python/pants/backend/python/typecheck/mypy/rules.py

@@ -15,13 +15,7 @@
 )
 from pants.backend.python.util_rules import pex_from_targets
 from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints
-from pants.backend.python.util_rules.pex import (
-    Pex,
-    PexRequest,
-    VenvPex,
-    VenvPexProcess,
-    pex_path_closure,
-)
+from pants.backend.python.util_rules.pex import Pex, PexRequest, VenvPex, VenvPexProcess
 from pants.backend.python.util_rules.pex_from_targets import PexFromTargetsRequest
 from pants.backend.python.util_rules.python_sources import (
     PythonSourceFiles,
@@ -178,7 +172,7 @@ async def mypy_typecheck_partition(
         PexRequest(
             output_filename="requirements_venv.pex",
             internal_only=True,
-            pex_path=pex_path_closure([requirements_pex]),
+            pex_path=[requirements_pex],
             interpreter_constraints=partition.interpreter_constraints,
         ),
     )