Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Reduce # of resolves when packaging pex_binary and python_awslambda targets #12343

Merged
merged 1 commit into from
Jul 14, 2021
Merged

Reduce # of resolves when packaging pex_binary and python_awslambda targets #12343

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 18 additions & 22 deletions src/python/pants/backend/awslambda/python/rules.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,17 +14,14 @@
from pants.backend.python.util_rules import pex_from_targets
from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints
from pants.backend.python.util_rules.pex import (
Pex,
PexPlatforms,
PexRequest,
PexRequirements,
TwoStepPex,
VenvPex,
VenvPexProcess,
)
from pants.backend.python.util_rules.pex_from_targets import (
PexFromTargetsRequest,
TwoStepPexFromTargetsRequest,
)
from pants.backend.python.util_rules.pex_from_targets import PexFromTargetsRequest
from pants.core.goals.package import (
BuiltPackage,
BuiltPackageArtifact,
Expand Down Expand Up @@ -75,21 +72,20 @@ async def package_python_awslambda(
platform += "m"
if (py_major, py_minor) == (2, 7):
platform += "u"
pex_request = TwoStepPexFromTargetsRequest(
PexFromTargetsRequest(
addresses=[field_set.address],
internal_only=False,
main=None,
output_filename=output_filename,
platforms=PexPlatforms([platform]),
additional_args=[
# Ensure we can resolve manylinux wheels in addition to any AMI-specific wheels.
"--manylinux=manylinux2014",
# When we're executing Pex on Linux, allow a local interpreter to be resolved if
# available and matching the AMI platform.
"--resolve-local-platforms",
],
)

pex_request = PexFromTargetsRequest(
addresses=[field_set.address],
internal_only=False,
main=None,
output_filename=output_filename,
platforms=PexPlatforms([platform]),
additional_args=[
# Ensure we can resolve manylinux wheels in addition to any AMI-specific wheels.
"--manylinux=manylinux2014",
# When we're executing Pex on Linux, allow a local interpreter to be resolved if
# available and matching the AMI platform.
"--resolve-local-platforms",
],
)

lambdex_request = PexRequest(
Expand All @@ -102,7 +98,7 @@ async def package_python_awslambda(

lambdex_pex, pex_result, handler, transitive_targets = await MultiGet(
Get(VenvPex, PexRequest, lambdex_request),
Get(TwoStepPex, TwoStepPexFromTargetsRequest, pex_request),
Get(Pex, PexFromTargetsRequest, pex_request),
Get(ResolvedPythonAwsHandler, ResolvePythonAwsHandlerRequest(field_set.handler)),
Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])),
)
Expand All @@ -129,7 +125,7 @@ async def package_python_awslambda(
VenvPexProcess(
lambdex_pex,
argv=("build", "-e", handler.val, output_filename),
input_digest=pex_result.pex.digest,
input_digest=pex_result.digest,
output_files=(output_filename,),
description=f"Setting up handler in {output_filename}",
),
Expand Down
33 changes: 14 additions & 19 deletions src/python/pants/backend/python/goals/package_pex_binary.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,8 @@
ResolvedPexEntryPoint,
ResolvePexEntryPointRequest,
)
from pants.backend.python.util_rules.pex import PexPlatforms, TwoStepPex
from pants.backend.python.util_rules.pex_from_targets import (
PexFromTargetsRequest,
TwoStepPexFromTargetsRequest,
)
from pants.backend.python.util_rules.pex import Pex, PexPlatforms
from pants.backend.python.util_rules.pex_from_targets import PexFromTargetsRequest
from pants.core.goals.package import (
BuiltPackage,
BuiltPackageArtifact,
Expand Down Expand Up @@ -126,22 +123,20 @@ async def package_pex_binary(
)

output_filename = field_set.output_path.value_or_default(field_set.address, file_ending="pex")
two_step_pex = await Get(
TwoStepPex,
TwoStepPexFromTargetsRequest(
PexFromTargetsRequest(
addresses=[field_set.address],
internal_only=False,
# TODO(John Sirois): Support ConsoleScript in PexBinary targets:
# https://github.com/pantsbuild/pants/issues/11619
main=resolved_entry_point.val,
platforms=PexPlatforms.create_from_platforms_field(field_set.platforms),
output_filename=output_filename,
additional_args=field_set.generate_additional_args(pex_binary_defaults),
)
pex = await Get(
Pex,
PexFromTargetsRequest(
addresses=[field_set.address],
internal_only=False,
# TODO(John Sirois): Support ConsoleScript in PexBinary targets:
# https://github.com/pantsbuild/pants/issues/11619
main=resolved_entry_point.val,
platforms=PexPlatforms.create_from_platforms_field(field_set.platforms),
output_filename=output_filename,
additional_args=field_set.generate_additional_args(pex_binary_defaults),
),
)
return BuiltPackage(two_step_pex.pex.digest, (BuiltPackageArtifact(output_filename),))
return BuiltPackage(pex.digest, (BuiltPackageArtifact(output_filename),))


def rules():
Expand Down
61 changes: 0 additions & 61 deletions src/python/pants/backend/python/util_rules/pex.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -180,21 +180,6 @@ def debug_hint(self) -> str:
return self.output_filename


@dataclass(frozen=True)
class TwoStepPexRequest:
"""A request to create a PEX in two steps.

First we create a requirements-only pex. Then we create the full pex on top of that
requirements pex, instead of having the full pex directly resolve its requirements.

This allows us to re-use the requirements-only pex when no requirements have changed (which is
the overwhelmingly common case), thus avoiding spurious re-resolves of the same requirements
over and over again.
"""

pex_request: PexRequest


@dataclass(frozen=True)
class Pex:
"""Wrapper for a digest containing a pex file created with some filename."""
Expand All @@ -204,17 +189,6 @@ class Pex:
python: PythonExecutable | None


@dataclass(frozen=True)
class TwoStepPex:
"""The result of creating a PEX in two steps.

TODO(9320): A workaround for https://github.com/pantsbuild/pants/issues/9320. Really we
just want the rules to directly return a Pex.
"""

pex: Pex


logger = logging.getLogger(__name__)


Expand Down Expand Up @@ -655,41 +629,6 @@ async def create_venv_pex(
)


@rule(level=LogLevel.DEBUG)
async def two_step_create_pex(two_step_pex_request: TwoStepPexRequest) -> TwoStepPex:
"""Create a PEX in two steps: a requirements-only PEX and then a full PEX from it."""
request = two_step_pex_request.pex_request
req_pex_name = "__requirements.pex"

repository_pex: Pex | None = None

# Create a pex containing just the requirements.
if request.requirements:
repository_pex = await Get(
Pex,
PexRequest(
output_filename=req_pex_name,
internal_only=request.internal_only,
requirements=request.requirements,
interpreter_constraints=request.interpreter_constraints,
platforms=request.platforms,
# TODO: Do we need to pass all the additional args to the requirements pex creation?
# Some of them may affect resolution behavior, but others may be irrelevant.
# For now we err on the side of caution.
additional_args=request.additional_args,
description=(
f"Resolving {pluralize(len(request.requirements), 'requirement')}: "
f"{', '.join(request.requirements)}"
),
),
)

# Now create a full PEX on top of the requirements PEX.
full_pex_request = dataclasses.replace(request, repository_pex=repository_pex)
full_pex = await Get(Pex, PexRequest, full_pex_request)
return TwoStepPex(pex=full_pex)


@frozen_after_init
@dataclass(unsafe_hash=True)
class PexProcess:
Expand Down
29 changes: 1 addition & 28 deletions src/python/pants/backend/python/util_rules/pex_from_targets.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,7 @@
parse_requirements_file,
)
from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints
from pants.backend.python.util_rules.pex import (
Pex,
PexPlatforms,
PexRequest,
PexRequirements,
TwoStepPexRequest,
)
from pants.backend.python.util_rules.pex import Pex, PexPlatforms, PexRequest, PexRequirements
from pants.backend.python.util_rules.pex import rules as pex_rules
from pants.backend.python.util_rules.python_sources import (
PythonSourceFilesRequest,
Expand Down Expand Up @@ -163,21 +157,6 @@ def for_requirements(
)


@dataclass(frozen=True)
class TwoStepPexFromTargetsRequest:
"""Request to create a PEX from the closure of a set of targets, in two steps.

First we create a requirements-only pex. Then we create the full pex on top of that
requirements pex, instead of having the full pex directly resolve its requirements.

This allows us to re-use the requirements-only pex when no requirements have changed (which is
the overwhelmingly common case), thus avoiding spurious re-resolves of the same requirements
over and over again.
"""

pex_from_targets_request: PexFromTargetsRequest


@rule(level=LogLevel.DEBUG)
async def pex_from_targets(request: PexFromTargetsRequest, python_setup: PythonSetup) -> PexRequest:
if request.direct_deps_only:
Expand Down Expand Up @@ -349,11 +328,5 @@ async def pex_from_targets(request: PexFromTargetsRequest, python_setup: PythonS
)


@rule
async def two_step_pex_from_targets(req: TwoStepPexFromTargetsRequest) -> TwoStepPexRequest:
pex_request = await Get(PexRequest, PexFromTargetsRequest, req.pex_from_targets_request)
return TwoStepPexRequest(pex_request=pex_request)


def rules():
return (*collect_rules(), *pex_rules(), *python_sources_rules())