Skip to content

Commit

Permalink
WIP: --elide-unused-requires-dist
Browse files Browse the repository at this point in the history
  • Loading branch information
@jsirois
jsirois committed Dec 8, 2024
1 parent 50d84a9 commit a9f0792
Show file tree
Hide file tree
Showing 13 changed files with 545 additions and 105 deletions.
85 changes: 31 additions & 54 deletions package/pex-scie.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,25 +12,19 @@
"artifacts": [
{
"algorithm": "sha256",
"hash": "c588a7e9b1173b6e866756dde596fd4cad94f9399daf99ad8c3258b3cb2b47a0",
"url": "https://files.pythonhosted.org/packages/0b/37/f8da2fbd29690b3557cca414c1949f92162981920699cd62095a984983bf/psutil-6.0.0-cp36-abi3-macosx_10_9_x86_64.whl"
"hash": "6e2dcd475ce8b80522e51d923d10c7871e45f20918e027ab682f94f1c6351688",
"url": "https://files.pythonhosted.org/packages/01/9e/8be43078a171381953cfee33c07c0d628594b5dbfc5157847b85022c2c1b/psutil-6.1.0-cp36-abi3-macosx_10_9_x86_64.whl"
}
],
"project_name": "psutil",
"requires_dists": [
"enum34; python_version <= \"3.4\" and extra == \"test\"",
"ipaddress; python_version < \"3.0\" and extra == \"test\"",
"mock; python_version < \"3.0\" and extra == \"test\"",
"pywin32; sys_platform == \"win32\" and extra == \"test\"",
"wmi; sys_platform == \"win32\" and extra == \"test\""
],
"requires_dists": [],
"requires_python": "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7",
"version": "6.0.0"
"version": "6.1.0"
}
],
"platform_tag": [
"cp312",
"cp312",
"cp313",
"cp313",
"macosx_13_0_x86_64"
]
},
Expand All @@ -40,26 +34,20 @@
"artifacts": [
{
"algorithm": "sha256",
"hash": "5fd9a97c8e94059b0ef54a7d4baf13b405011176c3b6ff257c247cae0d560ecd",
"url": "https://files.pythonhosted.org/packages/19/74/f59e7e0d392bc1070e9a70e2f9190d652487ac115bb16e2eff6b22ad1d24/psutil-6.0.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl"
"hash": "0895b8414afafc526712c498bd9de2b063deaac4021a3b3c34566283464aff8e",
"url": "https://files.pythonhosted.org/packages/1d/cb/313e80644ea407f04f6602a9e23096540d9dc1878755f3952ea8d3d104be/psutil-6.1.0-cp36-abi3-macosx_11_0_arm64.whl"
}
],
"project_name": "psutil",
"requires_dists": [
"enum34; python_version <= \"3.4\" and extra == \"test\"",
"ipaddress; python_version < \"3.0\" and extra == \"test\"",
"mock; python_version < \"3.0\" and extra == \"test\"",
"pywin32; sys_platform == \"win32\" and extra == \"test\"",
"wmi; sys_platform == \"win32\" and extra == \"test\""
],
"requires_dists": [],
"requires_python": "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7",
"version": "6.0.0"
"version": "6.1.0"
}
],
"platform_tag": [
"cp312",
"cp312",
"manylinux_2_31_x86_64"
"cp313",
"cp313",
"macosx_14_0_arm64"
]
},
{
Expand All @@ -68,26 +56,20 @@
"artifacts": [
{
"algorithm": "sha256",
"hash": "ffe7fc9b6b36beadc8c322f84e1caff51e8703b88eee1da46d1e3a6ae11b4fd0",
"url": "https://files.pythonhosted.org/packages/7c/06/63872a64c312a24fb9b4af123ee7007a306617da63ff13bcc1432386ead7/psutil-6.0.0-cp38-abi3-macosx_11_0_arm64.whl"
"hash": "d905186d647b16755a800e7263d43df08b790d709d575105d419f8b6ef65423a",
"url": "https://files.pythonhosted.org/packages/27/c2/d034856ac47e3b3cdfa9720d0e113902e615f4190d5d1bdb8df4b2015fb2/psutil-6.1.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl"
}
],
"project_name": "psutil",
"requires_dists": [
"enum34; python_version <= \"3.4\" and extra == \"test\"",
"ipaddress; python_version < \"3.0\" and extra == \"test\"",
"mock; python_version < \"3.0\" and extra == \"test\"",
"pywin32; sys_platform == \"win32\" and extra == \"test\"",
"wmi; sys_platform == \"win32\" and extra == \"test\""
],
"requires_dists": [],
"requires_python": "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7",
"version": "6.0.0"
"version": "6.1.0"
}
],
"platform_tag": [
"cp312",
"cp312",
"macosx_14_0_arm64"
"cp313",
"cp313",
"manylinux_2_31_aarch64"
]
},
{
Expand All @@ -96,35 +78,29 @@
"artifacts": [
{
"algorithm": "sha256",
"hash": "e2e8d0054fc88153ca0544f5c4d554d42e33df2e009c4ff42284ac9ebdef4132",
"url": "https://files.pythonhosted.org/packages/cd/5f/60038e277ff0a9cc8f0c9ea3d0c5eb6ee1d2470ea3f9389d776432888e47/psutil-6.0.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl"
"hash": "498c6979f9c6637ebc3a73b3f87f9eb1ec24e1ce53a7c5173b8508981614a90b",
"url": "https://files.pythonhosted.org/packages/58/4d/8245e6f76a93c98aab285a43ea71ff1b171bcd90c9d238bf81f7021fb233/psutil-6.1.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl"
}
],
"project_name": "psutil",
"requires_dists": [
"enum34; python_version <= \"3.4\" and extra == \"test\"",
"ipaddress; python_version < \"3.0\" and extra == \"test\"",
"mock; python_version < \"3.0\" and extra == \"test\"",
"pywin32; sys_platform == \"win32\" and extra == \"test\"",
"wmi; sys_platform == \"win32\" and extra == \"test\""
],
"requires_dists": [],
"requires_python": "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7",
"version": "6.0.0"
"version": "6.1.0"
}
],
"platform_tag": [
"cp312",
"cp312",
"manylinux_2_31_aarch64"
"cp313",
"cp313",
"manylinux_2_31_x86_64"
]
}
],
"only_builds": [],
"only_wheels": [],
"overridden": [],
"path_mappings": {},
"pex_version": "2.17.0",
"pip_version": "24.2",
"pex_version": "2.24.3",
"pip_version": "24.3.1",
"prefer_older_binary": false,
"requirements": [
"psutil>=5.3"
Expand All @@ -134,5 +110,6 @@
"style": "strict",
"target_systems": [],
"transitive": true,
"use_pep517": null
"use_pep517": null,
"use_system_time": false
}
27 changes: 25 additions & 2 deletions pex/cli/commands/lock.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -526,6 +526,20 @@ def add_create_lock_options(cls, create_parser):
)
),
)
create_parser.add_argument(
"--elide-unused-requires-dist",
"--no-elide-unused-requires-dist",
dest="elide_unused_requires_dist",
type=bool,
default=False,
action=HandleBoolAction,
help=(
"When creating the lock, elide dependencies from the 'requires_dists' lists that "
"can never be active due to markers. This does not change the reachable content of "
"the lock, but it does cut down on lock file size. This currently only elides "
"extras deps that are never activated, but may trim more in the future."
),
)
cls._add_lock_options(create_parser)
cls._add_resolve_options(create_parser)
cls.add_json_options(create_parser, entity="lock", include_switch=False)
Expand Down Expand Up @@ -899,6 +913,7 @@ def _create(self):
for interpreter_constraint in target_configuration.interpreter_constraints
),
target_systems=tuple(self.options.target_systems),
elide_unused_requires_dist=self.options.elide_unused_requires_dist,
)
elif self.options.target_systems:
return Error(
Expand All @@ -907,7 +922,10 @@ def _create(self):
)
)
else:
lock_configuration = LockConfiguration(style=self.options.style)
lock_configuration = LockConfiguration(
style=self.options.style,
elide_unused_requires_dist=self.options.elide_unused_requires_dist,
)

targets = try_(
self._resolve_targets(
Expand Down Expand Up @@ -1547,6 +1565,7 @@ def _sync(self):
for interpreter_constraint in target_configuration.interpreter_constraints
),
target_systems=tuple(self.options.target_systems),
elide_unused_requires_dist=self.options.elide_unused_requires_dist,
)
elif self.options.target_systems:
return Error(
Expand All @@ -1555,7 +1574,10 @@ def _sync(self):
)
)
else:
lock_configuration = LockConfiguration(style=self.options.style)
lock_configuration = LockConfiguration(
style=self.options.style,
elide_unused_requires_dist=self.options.elide_unused_requires_dist,
)

lock_file_path = self.options.lock
if os.path.exists(lock_file_path):
Expand All @@ -1566,6 +1588,7 @@ def _sync(self):
style=lock_configuration.style,
requires_python=SortedTuple(lock_configuration.requires_python),
target_systems=SortedTuple(lock_configuration.target_systems),
elide_unused_requires_dist=lock_configuration.elide_unused_requires_dist,
pip_version=pip_configuration.version,
resolver_version=pip_configuration.resolver_version,
allow_prereleases=pip_configuration.allow_prereleases,
Expand Down
7 changes: 1 addition & 6 deletions pex/resolve/lock_downloader.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@
DownloadableArtifact,
FileArtifact,
LocalProjectArtifact,
LockConfiguration,
VCSArtifact,
)
from pex.resolve.lockfile.download_manager import DownloadedArtifact, DownloadManager
Expand Down Expand Up @@ -233,11 +232,7 @@ def create(
file_lock_style=file_lock_style,
downloader=ArtifactDownloader(
resolver=resolver,
lock_configuration=LockConfiguration(
style=lock.style,
requires_python=lock.requires_python,
target_systems=lock.target_systems,
),
lock_configuration=lock.lock_configuration(),
target=target,
package_index_configuration=PackageIndexConfiguration.create(
pip_version=pip_version,
Expand Down
1 change: 1 addition & 0 deletions pex/resolve/locked_resolve.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ class LockConfiguration(object):
style = attr.ib() # type: LockStyle.Value
requires_python = attr.ib(default=()) # type: Tuple[str, ...]
target_systems = attr.ib(default=()) # type: Tuple[TargetSystem.Value, ...]
elide_unused_requires_dist = attr.ib(default=False) # type: bool

@requires_python.validator
@target_systems.validator
Expand Down
9 changes: 6 additions & 3 deletions pex/resolve/lockfile/create.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -341,9 +341,11 @@ def lock(self, downloaded):
package_index_configuration=self.package_index_configuration,
max_parallel_jobs=self.max_parallel_jobs,
),
platform_tag=None
if self.lock_configuration.style == LockStyle.UNIVERSAL
else target.platform.tag,
platform_tag=(
None
if self.lock_configuration.style == LockStyle.UNIVERSAL
else target.platform.tag
),
)
for target, resolved_requirements in resolved_requirements_by_target.items()
)
Expand Down Expand Up @@ -456,6 +458,7 @@ def create(
excluded=dependency_configuration.excluded,
overridden=dependency_configuration.all_overrides(),
locked_resolves=locked_resolves,
elide_unused_requires_dist=lock_configuration.elide_unused_requires_dist,
)

if lock_configuration.style is LockStyle.UNIVERSAL and (
Expand Down
3 changes: 3 additions & 0 deletions pex/resolve/lockfile/json_codec.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,8 @@ def parse_version_specifier(
for index, target_system in enumerate(get("target_systems", list, optional=True) or ())
]

elide_unused_requires_dist = get("elide_unused_requires_dist", bool, optional=True) or False

only_wheels = [
parse_project_name(project_name, path=".only_wheels[{index}]".format(index=index))
for index, project_name in enumerate(get("only_wheels", list, optional=True) or ())
Expand Down Expand Up @@ -337,6 +339,7 @@ def assemble_tag(
style=get_enum_value(LockStyle, "style"),
requires_python=get("requires_python", list),
target_systems=target_systems,
elide_unused_requires_dist=elide_unused_requires_dist,
pip_version=get_enum_value(
PipVersion,
"pip_version",
Expand Down
30 changes: 28 additions & 2 deletions pex/resolve/lockfile/model.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,14 @@
from pex.pep_503 import ProjectName
from pex.pip.version import PipVersion, PipVersionValue
from pex.requirements import LocalProjectRequirement
from pex.resolve.locked_resolve import LocalProjectArtifact, LockedResolve, LockStyle, TargetSystem
from pex.resolve.locked_resolve import (
LocalProjectArtifact,
LockConfiguration,
LockedResolve,
LockStyle,
TargetSystem,
)
from pex.resolve.lockfile import requires_dist
from pex.resolve.resolved_requirement import Pin
from pex.resolve.resolver_configuration import BuildConfiguration, ResolverVersion
from pex.sorted_tuple import SortedTuple
Expand Down Expand Up @@ -47,6 +54,7 @@ def create(
source=None, # type: Optional[str]
pip_version=None, # type: Optional[PipVersionValue]
resolver_version=None, # type: Optional[ResolverVersion.Value]
elide_unused_requires_dist=False, # type: bool
):
# type: (...) -> Lockfile

Expand Down Expand Up @@ -94,6 +102,7 @@ def extract_requirement(req):
style=style,
requires_python=SortedTuple(requires_python),
target_systems=SortedTuple(target_systems),
elide_unused_requires_dist=elide_unused_requires_dist,
pip_version=pip_ver,
resolver_version=resolver_version or ResolverVersion.default(pip_ver),
requirements=SortedTuple(resolve_requirements, key=str),
Expand All @@ -110,7 +119,14 @@ def extract_requirement(req):
transitive=transitive,
excluded=SortedTuple(excluded),
overridden=SortedTuple(overridden),
locked_resolves=SortedTuple(locked_resolves),
locked_resolves=SortedTuple(
(
requires_dist.remove_unused_requires_dist(resolve_requirements, locked_resolve)
if elide_unused_requires_dist
else locked_resolve
)
for locked_resolve in locked_resolves
),
local_project_requirement_mapping=requirement_by_local_project_directory,
source=source,
)
Expand All @@ -119,6 +135,7 @@ def extract_requirement(req):
style = attr.ib() # type: LockStyle.Value
requires_python = attr.ib() # type: SortedTuple[str]
target_systems = attr.ib() # type: SortedTuple[TargetSystem.Value]
elide_unused_requires_dist = attr.ib() # type: bool
pip_version = attr.ib() # type: PipVersionValue
resolver_version = attr.ib() # type: ResolverVersion.Value
requirements = attr.ib() # type: SortedTuple[Requirement]
Expand All @@ -139,6 +156,15 @@ def extract_requirement(req):
local_project_requirement_mapping = attr.ib(eq=False) # type: Mapping[str, Requirement]
source = attr.ib(default=None, eq=False) # type: Optional[str]

def lock_configuration(self):
# type: () -> LockConfiguration
return LockConfiguration(
style=self.style,
requires_python=self.requires_python,
target_systems=self.target_systems,
elide_unused_requires_dist=self.elide_unused_requires_dist,
)

def build_configuration(self):
# type: () -> BuildConfiguration
return BuildConfiguration.create(
Expand Down
Loading

0 comments on commit a9f0792

Please sign in to comment.