Upgrade to Pex 2.1.48 and leverage packed layout.

3rdparty/python/lockfiles/pytest.txt

@@ -99,9 +99,9 @@ ipython==7.27.0; python_version >= "3.7" \
 jedi==0.18.0; python_version >= "3.7" \
     --hash=sha256:18456d83f65f400ab0c2d3319e48520420ef43b23a086fdc05dff34132f0fb93 \
     --hash=sha256:92550a404bad8afed881a137ec9a461fed49eca661414be45059329614ed0707
-matplotlib-inline==0.1.2; python_version >= "3.7" \
-    --hash=sha256:f41d5ff73c9f5385775d5c0bc13b424535c8402fe70ea8210f93e11f3683993e \
-    --hash=sha256:5cf1176f554abb4fa98cb362aa2b55c500147e4bdbb07e3fda359143e1da0811
+matplotlib-inline==0.1.3; python_version >= "3.7" \
+    --hash=sha256:a04bfba22e0d1395479f866853ec1ee28eea1485c1d69a6faf00dc3e24ff34ee \
+    --hash=sha256:aed605ba3b72462d64d475a21a9296f400a19c4f74a31b59103d2a99ffd5aa5c
 packaging==21.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" \
     --hash=sha256:c86254f9220d55e31cc94d69bade760f0847da8000def4dfe1c6b872fd14ff14 \
     --hash=sha256:7dc96269f53a4ccec5c0670940a4281106dd0bb343f47b7471f779df49c2fbe7
@@ -149,9 +149,9 @@ pytest-metadata==1.11.0; python_version >= "3.6" and python_full_version < "3.0.
 pytest==6.2.5; python_version >= "3.6" \
     --hash=sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134 \
     --hash=sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89
-setuptools==57.4.0; python_version >= "3.7" \
-    --hash=sha256:a49230977aa6cfb9d933614d2f7b79036e9945c4cdd7583163f4e920b83418d6 \
-    --hash=sha256:6bac238ffdf24e8806c61440e755192470352850f3419a52f26ffe0a1a64f465
+setuptools==58.0.4; python_version >= "3.7" \
+    --hash=sha256:69cc739bc2662098a68a9bc575cd974a57969e70c1d58ade89d104ab73d79770 \
+    --hash=sha256:f10059f0152e0b7fb6b2edd77bcb1ecd4c9ed7048a826eb2d79f72fd2e6e237b
 toml==0.10.2; python_version > "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version > "3.6" \
     --hash=sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b \
     --hash=sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f

3rdparty/python/lockfiles/user_reqs.txt

@@ -5,7 +5,7 @@
 # --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE ---
 # {
 #   "version": 1,
-#   "requirements_invalidation_digest": "19770133e0608f747845bc429b61942d2f79b3cad46790962ade28db07e2b4fd",
+#   "requirements_invalidation_digest": "6adf6599ccc90c6aad4cc62333b089eea937311f47ae9eab025203d7a0e0eb2c",
 #   "valid_for_interpreter_constraints": [
 #     "CPython<3.10,>=3.7"
 #   ]
@@ -113,9 +113,9 @@ iniconfig==1.1.1; python_version >= "3.6" \
 packaging==21.0; python_version >= "3.6" \
     --hash=sha256:c86254f9220d55e31cc94d69bade760f0847da8000def4dfe1c6b872fd14ff14 \
     --hash=sha256:7dc96269f53a4ccec5c0670940a4281106dd0bb343f47b7471f779df49c2fbe7
-pex==2.1.47; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0" and python_version < "3.10") \
-    --hash=sha256:2341bad1146136237f2d7a7440fe706f010879febe0d8a714e6cd92b8ba47ce9 \
-    --hash=sha256:0928d0316caac840db528030fc741930e8be22a3fa6a8635308fb8443a0a0c6a
+pex==2.1.48; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0" and python_version < "3.10") \
+    --hash=sha256:903ab1e2781dbf077e9936d6a5dada9fa868c5416423065eee7bd00d94f2ea06 \
+    --hash=sha256:5f6a489075c5bbecdb36a42249cd52cfd882e205242f80a1f1e2294951ab46e7
 pluggy==1.0.0; python_version >= "3.6" \
     --hash=sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3 \
     --hash=sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159
@@ -217,9 +217,9 @@ setproctitle==1.2.2; python_version >= "3.6" \
     --hash=sha256:249526a06f16d493a2cb632abc1b1fdfaaa05776339a50dd9f27c941f6ff1383 \
     --hash=sha256:4fc5bebd34f451dc87d2772ae6093adea1ea1dc29afc24641b250140decd23bb \
     --hash=sha256:7dfb472c8852403d34007e01d6e3c68c57eb66433fb8a5c77b13b89a160d97df
-setuptools==57.4.0; python_version >= "3.6" \
-    --hash=sha256:a49230977aa6cfb9d933614d2f7b79036e9945c4cdd7583163f4e920b83418d6 \
-    --hash=sha256:6bac238ffdf24e8806c61440e755192470352850f3419a52f26ffe0a1a64f465
+setuptools==57.5.0; python_version >= "3.6" \
+    --hash=sha256:60d78588f15b048f86e35cdab73003d8b21dd45108ee61a6693881a427f22073 \
+    --hash=sha256:d9d3266d50f59c6967b9312844470babbdb26304fe740833a5f8d89829ba3a24
 six==1.16.0; python_version >= "3.5" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.5" \
     --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 \
     --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926

3rdparty/python/requirements.txt

@@ -8,7 +8,7 @@ humbug==0.2.6
 
 ijson==3.1.4
 packaging==21.0
-pex==2.1.47
+pex==2.1.48
 psutil==5.8.0
 pystache==0.5.4
 # This should be kept in sync with `pytest.py`.

build-support/bin/_release_helper.py

@@ -598,7 +598,6 @@ def build_pex(fetch: bool) -> None:
                 str(CONSTANTS.deploy_3rdparty_wheel_dir / CONSTANTS.pants_unstable_version),
                 "--no-strip-pex-env",
                 "--console-script=pants",
-                "--unzip",
                 *extra_pex_args,
                 f"pantsbuild.pants=={CONSTANTS.pants_unstable_version}",
             ],

src/python/pants/backend/python/goals/package_pex_binary.py

@@ -74,8 +74,6 @@ def _execution_mode(self) -> PexExecutionMode:
 
     def generate_additional_args(self, pex_binary_defaults: PexBinaryDefaults) -> Tuple[str, ...]:
         args = []
-        if self.always_write_cache.value is True:
-            args.append("--always-write-cache")
         if self.emit_warnings.value_or_global_default(pex_binary_defaults) is False:
             args.append("--no-emit-warnings")
         if self.ignore_errors.value is True:
@@ -84,12 +82,8 @@ def generate_additional_args(self, pex_binary_defaults: PexBinaryDefaults) -> Tu
             args.append(f"--inherit-path={self.inherit_path.value}")
         if self.shebang.value is not None:
             args.append(f"--python-shebang={self.shebang.value}")
-        if self.zip_safe.value is False:
-            args.append("--not-zip-safe")
         if self.strip_env.value is False:
             args.append("--no-strip-pex-env")
-        if self._execution_mode is PexExecutionMode.UNZIP:
-            args.append("--unzip")
         if self._execution_mode is PexExecutionMode.VENV:
             args.extend(("--venv", "prepend"))
         if self.include_tools.value is True:

src/python/pants/backend/python/goals/pytest_runner.py

@@ -13,13 +13,7 @@
 )
 from pants.backend.python.subsystems.pytest import PyTest, PythonTestFieldSet
 from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints
-from pants.backend.python.util_rules.pex import (
-    Pex,
-    PexRequest,
-    VenvPex,
-    VenvPexProcess,
-    pex_path_closure,
-)
+from pants.backend.python.util_rules.pex import Pex, PexRequest, VenvPex, VenvPexProcess
 from pants.backend.python.util_rules.pex_from_targets import PexFromTargetsRequest
 from pants.backend.python.util_rules.python_sources import (
     PythonSourceFiles,
@@ -226,7 +220,7 @@ async def setup_pytest_for_target(
             interpreter_constraints=interpreter_constraints,
             main=pytest.main,
             internal_only=True,
-            pex_path=pex_path_closure([pytest_pex, requirements_pex]),
+            pex_path=[pytest_pex, requirements_pex],
         ),
     )
     config_files_get = Get(

src/python/pants/backend/python/goals/repl.py

@@ -2,7 +2,7 @@
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
 from pants.backend.python.subsystems.ipython import IPython
-from pants.backend.python.util_rules.pex import Pex, PexRequest, pex_path_closure
+from pants.backend.python.util_rules.pex import Pex, PexRequest
 from pants.backend.python.util_rules.pex_environment import PexEnvironment
 from pants.backend.python.util_rules.pex_from_targets import PexFromTargetsRequest
 from pants.backend.python.util_rules.python_sources import (
@@ -106,7 +106,7 @@ async def create_ipython_repl_request(
     chrooted_source_roots = [repl.in_chroot(sr) for sr in sources.source_roots]
     extra_env = {
         **complete_pex_env.environment_dict(python_configured=ipython_pex.python is not None),
-        "PEX_PATH": ":".join(repl.in_chroot(p.name) for p in pex_path_closure([requirements_pex])),
+        "PEX_PATH": repl.in_chroot(requirements_pex_request.output_filename),
         "PEX_EXTRA_SYS_PATH": ":".join(chrooted_source_roots),
     }
 

src/python/pants/backend/python/goals/run_pex_binary.py

@@ -9,7 +9,7 @@
     ResolvedPexEntryPoint,
     ResolvePexEntryPointRequest,
 )
-from pants.backend.python.util_rules.pex import Pex, PexRequest, pex_path_closure
+from pants.backend.python.util_rules.pex import Pex, PexRequest
 from pants.backend.python.util_rules.pex_environment import PexEnvironment
 from pants.backend.python.util_rules.pex_from_targets import PexFromTargetsRequest
 from pants.backend.python.util_rules.python_sources import (
@@ -100,7 +100,7 @@ def in_chroot(relpath: str) -> str:
     chrooted_source_roots = [in_chroot(sr) for sr in sources.source_roots]
     extra_env = {
         **complete_pex_env.environment_dict(python_configured=runner_pex.python is not None),
-        "PEX_PATH": ":".join(in_chroot(p.name) for p in pex_path_closure([requirements])),
+        "PEX_PATH": in_chroot(requirements_pex_request.output_filename),
         "PEX_EXTRA_SYS_PATH": ":".join(chrooted_source_roots),
     }
 

src/python/pants/backend/python/goals/run_pex_binary_integration_test.py

@@ -88,7 +88,6 @@ def run(*extra_args: str, **extra_env: str) -> PantsResult:
         result = run("--", "info", PEX_TOOLS="1")
         assert result.exit_code == 0
         pex_info = json.loads(result.stdout)
-        assert (execution_mode is PexExecutionMode.UNZIP) == pex_info["unzip"]
         assert (execution_mode is PexExecutionMode.VENV) == pex_info["venv"]
         assert ("prepend" if execution_mode is PexExecutionMode.VENV else "false") == pex_info[
             "venv_bin_path"

src/python/pants/backend/python/lint/bandit/lockfile.txt

@@ -60,9 +60,9 @@ pyyaml==5.4.1; python_version >= "3.5" and python_full_version < "3.0.0" or pyth
     --hash=sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10 \
     --hash=sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db \
     --hash=sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e
-setuptools==58.0.3; python_version >= "3.6" \
-    --hash=sha256:1ceadf3ea9a821ef305505db995f2e21550ea62500900164278c4b23109204f3 \
-    --hash=sha256:5e4c36f55012a46c1b3e4b67a8236d1d73856a90fc7b3207d29bedb7d2bac417
+setuptools==58.0.4; python_version >= "3.6" \
+    --hash=sha256:69cc739bc2662098a68a9bc575cd974a57969e70c1d58ade89d104ab73d79770 \
+    --hash=sha256:f10059f0152e0b7fb6b2edd77bcb1ecd4c9ed7048a826eb2d79f72fd2e6e237b
 six==1.16.0; python_version >= "3.5" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.5" \
     --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 \
     --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926

src/python/pants/backend/python/lint/pylint/rules.py

@@ -13,13 +13,7 @@
 from pants.backend.python.target_types import InterpreterConstraintsField
 from pants.backend.python.util_rules import pex_from_targets
 from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints
-from pants.backend.python.util_rules.pex import (
-    Pex,
-    PexRequest,
-    VenvPex,
-    VenvPexProcess,
-    pex_path_closure,
-)
+from pants.backend.python.util_rules.pex import Pex, PexRequest, VenvPex, VenvPexProcess
 from pants.backend.python.util_rules.pex_from_targets import PexFromTargetsRequest
 from pants.backend.python.util_rules.python_sources import (
     PythonSourceFiles,
@@ -134,7 +128,7 @@ async def pylint_lint_partition(
                 interpreter_constraints=partition.interpreter_constraints,
                 main=pylint.main,
                 internal_only=True,
-                pex_path=pex_path_closure([pylint_pex, requirements_pex]),
+                pex_path=[pylint_pex, requirements_pex],
             ),
         ),
         Get(

src/python/pants/backend/python/subsystems/ipython_lockfile.txt

@@ -51,9 +51,9 @@ ptyprocess==0.7.0; sys_platform != "win32" and python_version >= "3.6" \
 pygments==2.10.0; python_version >= "3.6" \
     --hash=sha256:b8e67fe6af78f492b3c4b3e2970c0624cbf08beb1e493b2c99b9fa1b67a20380 \
     --hash=sha256:f398865f7eb6874156579fdf36bc840a03cab64d1cde9e93d68f46a425ec52c6
-setuptools==58.0.3; python_version >= "3.6" \
-    --hash=sha256:1ceadf3ea9a821ef305505db995f2e21550ea62500900164278c4b23109204f3 \
-    --hash=sha256:5e4c36f55012a46c1b3e4b67a8236d1d73856a90fc7b3207d29bedb7d2bac417
+setuptools==58.0.4; python_version >= "3.6" \
+    --hash=sha256:69cc739bc2662098a68a9bc575cd974a57969e70c1d58ade89d104ab73d79770 \
+    --hash=sha256:f10059f0152e0b7fb6b2edd77bcb1ecd4c9ed7048a826eb2d79f72fd2e6e237b
 six==1.16.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" \
     --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 \
     --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926

src/python/pants/backend/python/subsystems/lambdex_lockfile.txt

@@ -15,6 +15,6 @@
 lambdex==0.1.6; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.6.0" and python_version < "3.10") \
     --hash=sha256:cb685b106617fbd1afd26d6e9472b2e0c99df8574c6d358aee4e6c13aeef8eb1 \
     --hash=sha256:6d1a95c8a31baa703edece8e36a705045b0203c7e886812c27a4dd945aa694e0
-pex==2.1.47; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version < "3.10" \
-    --hash=sha256:2341bad1146136237f2d7a7440fe706f010879febe0d8a714e6cd92b8ba47ce9 \
-    --hash=sha256:0928d0316caac840db528030fc741930e8be22a3fa6a8635308fb8443a0a0c6a
+pex==2.1.48; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version < "3.10" \
+    --hash=sha256:903ab1e2781dbf077e9936d6a5dada9fa868c5416423065eee7bd00d94f2ea06 \
+    --hash=sha256:5f6a489075c5bbecdb36a42249cd52cfd882e205242f80a1f1e2294951ab46e7

src/python/pants/backend/python/target_types.py

@@ -10,7 +10,7 @@
 from dataclasses import dataclass
 from enum import Enum
 from textwrap import dedent
-from typing import TYPE_CHECKING, Dict, Iterable, Iterator, Optional, Tuple, Union, cast
+from typing import TYPE_CHECKING, Any, Dict, Iterable, Iterator, Optional, Tuple, Union, cast
 
 from packaging.utils import canonicalize_name as canonicalize_project_name
 from pkg_resources import Requirement
@@ -20,6 +20,7 @@
     DEFAULT_TYPE_STUB_MODULE_MAPPING,
 )
 from pants.backend.python.macros.python_artifact import PythonArtifact
+from pants.base.deprecated import warn_or_error
 from pants.core.goals.package import OutputPathField
 from pants.core.goals.test import RuntimePackageDependenciesField
 from pants.engine.addresses import Address, Addresses
@@ -335,6 +336,7 @@ def compute_value(
         return super().compute_value(raw_value, address)
 
 
+# TODO(John Sirois): Deprecate: https://github.com/pantsbuild/pants/issues/12803
 class PexZipSafeField(BoolField):
     alias = "zip_safe"
     default = True
@@ -343,6 +345,11 @@ class PexZipSafeField(BoolField):
         "not zip safe, it will be written to disk prior to execution. You may need to mark "
         "`zip_safe=False` if you're having issues loading your code."
     )
+    removal_version = "2.9.0.dev0"
+    removal_hint = (
+        "All PEX binaries now unpack your code to disk prior to first execution; so this option no "
+        "longer needs to be specified."
+    )
 
 
 class PexStripEnvField(BoolField):
@@ -365,6 +372,12 @@ class PexAlwaysWriteCacheField(BoolField):
         "Whether PEX should always write the .deps cache of the .pex file to disk or not. This "
         "can use less memory in RAM-constrained environments."
     )
+    removal_version = "2.9.0.dev0"
+    removal_hint = (
+        "This option never had any effect when passed to Pex and the Pex option is now removed "
+        "altogether. PEXes always write all their internal dependencies out to disk as part of "
+        "first execution bootstrapping."
+    )
 
 
 class PexIgnoreErrorsField(BoolField):
@@ -408,18 +421,33 @@ class PexExecutionModeField(StringField):
     expected_type = str
     default = PexExecutionMode.ZIPAPP.value
     help = (
-        "The mode the generated PEX file will run in.\n\nThe traditional PEX file runs in "
-        f"{PexExecutionMode.ZIPAPP.value!r} mode (See: https://www.python.org/dev/peps/pep-0441/). "
-        f"In general, faster cold start times can be attained using the "
-        f"{PexExecutionMode.UNZIP.value!r} mode which also has the benefit of allowing standard "
-        "use of `__file__` and filesystem APIs to access code and resources in the PEX.\n\nThe "
-        f"fastest execution mode in the steady state is {PexExecutionMode.VENV.value!r}, which "
-        "generates a virtual environment from the PEX file on first run, but then achieves near "
-        "native virtual environment start times. This mode also benefits from a traditional "
-        "virtual environment `sys.path`, giving maximum compatibility with stdlib and third party "
-        "APIs."
+        "The mode the generated PEX file will run in.\n\nThe traditional PEX file runs in a "
+        f"modified {PexExecutionMode.ZIPAPP.value!r} mode (See: "
+        "https://www.python.org/dev/peps/pep-0441/) where zipped internal code and dependencies "
+        "are first unpacked to disk. This mode achieves the fasted cold start times and may, for "
+        "example be the best choice for cloud lambda functions.\n\nThe fastest execution mode in "
+        f"the steady state is {PexExecutionMode.VENV.value!r}, which generates a virtual "
+        "environment from the PEX file on first run, but then achieves near native virtual "
+        "environment start times. This mode also benefits from a traditional virtual environment "
+        "`sys.path`, giving maximum compatibility with stdlib and third party APIs.\n\nThe "
+        f"{PexExecutionMode.UNZIP.value!r} mode is deprecated since the default "
+        f"{PexExecutionMode.ZIPAPP.value!r} mode now executes this way."
     )
 
+    @classmethod
+    def _check_deprecated(cls, raw_value: Optional[Any], address_: Address) -> None:
+        if PexExecutionMode.UNZIP.value == raw_value:
+            warn_or_error(
+                removal_version="2.9.0.dev0",
+                entity=f"the {cls.alias!r} field {PexExecutionMode.UNZIP.value!r} value",
+                hint=(
+                    f"The {PexExecutionMode.UNZIP.value!r} mode is now the default PEX execution "
+                    "mode; so you can remove this field setting or explicitly choose the default "
+                    f"of {PexExecutionMode.ZIPAPP.value!r} and get the same benefits you already "
+                    "enjoy from this mode."
+                ),
+            )
+
 
 class PexIncludeToolsField(BoolField):
     alias = "include_tools"

src/python/pants/backend/python/typecheck/mypy/rules.py

@@ -15,13 +15,7 @@
 )
 from pants.backend.python.util_rules import pex_from_targets
 from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints
-from pants.backend.python.util_rules.pex import (
-    Pex,
-    PexRequest,
-    VenvPex,
-    VenvPexProcess,
-    pex_path_closure,
-)
+from pants.backend.python.util_rules.pex import Pex, PexRequest, VenvPex, VenvPexProcess
 from pants.backend.python.util_rules.pex_from_targets import PexFromTargetsRequest
 from pants.backend.python.util_rules.python_sources import (
     PythonSourceFiles,
@@ -178,7 +172,7 @@ async def mypy_typecheck_partition(
         PexRequest(
             output_filename="requirements_venv.pex",
             internal_only=True,
-            pex_path=pex_path_closure([requirements_pex]),
+            pex_path=[requirements_pex],
             interpreter_constraints=partition.interpreter_constraints,
         ),
     )