From c81de55f16f9cfc629c1e078ecbb532809cf6875 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Tue, 29 Oct 2024 00:34:01 -0700 Subject: [PATCH] Add support for PEP-735 dependency groups. (#2584) You can now specify one or more `--group @` as sources of requirements when building a PEX or creating a lock. See: https://peps.python.org/pep-0735 --- CHANGES.md | 12 + pex/bin/pex.py | 10 +- pex/build_system/pep_518.py | 7 +- pex/cli/commands/lock.py | 33 +- pex/pep_723.py | 4 +- pex/resolve/project.py | 182 ++++- pex/scie/science.py | 13 +- pex/toml.py | 60 ++ pex/vendor/__init__.py | 9 +- pex/vendor/_vendored/tomli-w/.layout.json | 1 + pex/vendor/_vendored/tomli-w/__init__.py | 0 pex/vendor/_vendored/tomli-w/constraints.txt | 1 + .../tomli-w/tomli_w-1.0.0.dist-info/INSTALLER | 1 + .../tomli-w/tomli_w-1.0.0.dist-info/LICENSE | 21 + .../tomli-w/tomli_w-1.0.0.dist-info/METADATA | 144 ++++ .../tomli-w/tomli_w-1.0.0.dist-info/WHEEL | 4 + .../_vendored/tomli-w/tomli_w/__init__.py | 8 + .../_vendored/tomli-w/tomli_w/_writer.py | 199 +++++ pex/vendor/_vendored/tomli-w/tomli_w/py.typed | 1 + pex/vendor/_vendored/tomli/.layout.json | 1 + pex/vendor/_vendored/tomli/__init__.py | 0 pex/vendor/_vendored/tomli/constraints.txt | 1 + .../tomli/tomli-2.0.1.dist-info/INSTALLER | 1 + .../tomli/tomli-2.0.1.dist-info/LICENSE | 21 + .../tomli/tomli-2.0.1.dist-info/METADATA | 206 ++++++ .../tomli/tomli-2.0.1.dist-info/WHEEL | 4 + pex/vendor/_vendored/tomli/tomli/__init__.py | 11 + pex/vendor/_vendored/tomli/tomli/_parser.py | 691 ++++++++++++++++++ pex/vendor/_vendored/tomli/tomli/_re.py | 107 +++ pex/vendor/_vendored/tomli/tomli/_types.py | 10 + pex/vendor/_vendored/tomli/tomli/py.typed | 1 + pex/version.py | 2 +- tests/bin/test_sh_boot.py | 6 +- .../commands/test_lock_dependency_groups.py | 59 ++ .../resolve/test_dependency_groups.py | 62 ++ tests/integration/test_issue_1560.py | 5 +- tests/resolve/test_dependency_groups.py | 218 ++++++ tests/test_pep_723.py | 11 +- 38 files changed, 2081 insertions(+), 46 deletions(-) create mode 100644 pex/toml.py create mode 100644 pex/vendor/_vendored/tomli-w/.layout.json create mode 100644 pex/vendor/_vendored/tomli-w/__init__.py create mode 100644 pex/vendor/_vendored/tomli-w/constraints.txt create mode 100644 pex/vendor/_vendored/tomli-w/tomli_w-1.0.0.dist-info/INSTALLER create mode 100644 pex/vendor/_vendored/tomli-w/tomli_w-1.0.0.dist-info/LICENSE create mode 100644 pex/vendor/_vendored/tomli-w/tomli_w-1.0.0.dist-info/METADATA create mode 100644 pex/vendor/_vendored/tomli-w/tomli_w-1.0.0.dist-info/WHEEL create mode 100644 pex/vendor/_vendored/tomli-w/tomli_w/__init__.py create mode 100644 pex/vendor/_vendored/tomli-w/tomli_w/_writer.py create mode 100644 pex/vendor/_vendored/tomli-w/tomli_w/py.typed create mode 100644 pex/vendor/_vendored/tomli/.layout.json create mode 100644 pex/vendor/_vendored/tomli/__init__.py create mode 100644 pex/vendor/_vendored/tomli/constraints.txt create mode 100644 pex/vendor/_vendored/tomli/tomli-2.0.1.dist-info/INSTALLER create mode 100644 pex/vendor/_vendored/tomli/tomli-2.0.1.dist-info/LICENSE create mode 100644 pex/vendor/_vendored/tomli/tomli-2.0.1.dist-info/METADATA create mode 100644 pex/vendor/_vendored/tomli/tomli-2.0.1.dist-info/WHEEL create mode 100644 pex/vendor/_vendored/tomli/tomli/__init__.py create mode 100644 pex/vendor/_vendored/tomli/tomli/_parser.py create mode 100644 pex/vendor/_vendored/tomli/tomli/_re.py create mode 100644 pex/vendor/_vendored/tomli/tomli/_types.py create mode 100644 pex/vendor/_vendored/tomli/tomli/py.typed create mode 100644 tests/integration/cli/commands/test_lock_dependency_groups.py create mode 100644 tests/integration/resolve/test_dependency_groups.py create mode 100644 tests/resolve/test_dependency_groups.py diff --git a/CHANGES.md b/CHANGES.md index 4bb7266d2..1a5c9ca17 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,17 @@ # Release Notes +## 2.23.0 + +This release adds support for drawing requirements from +[PEP-735][PEP-735] dependency groups when creating PEXes or lock files. +Groups are requested via `--group @` or just +`--group ` if the project directory is the current working +directory. + +* Add support for PEP-735 dependency groups. (#2584) + +[PEP-735]: https://peps.python.org/pep-0735/ + ## 2.22.0 This release adds support for `--pip-version 24.3.1`. diff --git a/pex/bin/pex.py b/pex/bin/pex.py index b0e7f02a7..3ef43ecb4 100644 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -746,7 +746,7 @@ def configure_clp_sources(parser): project.register_options( parser, - help=( + project_help=( "Add the local project at the specified path to the generated .pex file along with " "its transitive dependencies." ), @@ -1016,6 +1016,14 @@ def build_pex( else resolver_configuration.pip_configuration ) + group_requirements = project.get_group_requirements(options) + if group_requirements: + requirements = OrderedSet(requirement_configuration.requirements) + requirements.update(str(req) for req in group_requirements) + requirement_configuration = attr.evolve( + requirement_configuration, requirements=requirements + ) + project_dependencies = OrderedSet() # type: OrderedSet[Requirement] with TRACER.timed( "Adding distributions built from local projects and collecting their requirements: " diff --git a/pex/build_system/pep_518.py b/pex/build_system/pep_518.py index ebbcb6a57..09387dfce 100644 --- a/pex/build_system/pep_518.py +++ b/pex/build_system/pep_518.py @@ -6,6 +6,7 @@ import os.path import subprocess +from pex import toml from pex.build_system import DEFAULT_BUILD_BACKEND from pex.common import REPRODUCIBLE_BUILDS_ENV, CopyMode from pex.dist_metadata import Distribution @@ -26,9 +27,8 @@ from typing import Iterable, Mapping, Optional, Tuple, Union import attr # vendor:skip - import toml # vendor:skip else: - from pex.third_party import attr, toml + from pex.third_party import attr @attr.s(frozen=True) @@ -43,8 +43,7 @@ def _read_build_system_table( ): # type: (...) -> Union[Optional[BuildSystemTable], Error] try: - with open(pyproject_toml) as fp: - data = toml.load(fp) + data = toml.load(pyproject_toml) except toml.TomlDecodeError as e: return Error( "Problem parsing toml in {pyproject_toml}: {err}".format( diff --git a/pex/cli/commands/lock.py b/pex/cli/commands/lock.py index 621d74457..7d96aa8a3 100644 --- a/pex/cli/commands/lock.py +++ b/pex/cli/commands/lock.py @@ -416,7 +416,7 @@ def _add_resolve_options(cls, parser): requirement_options.register(options_group) project.register_options( options_group, - help=( + project_help=( "Add the transitive dependencies of the local project at the specified path to " "the lock but do not lock project itself." ), @@ -846,25 +846,28 @@ def _gather_requirements( ): # type: (...) -> RequirementConfiguration requirement_configuration = requirement_options.configure(self.options) + group_requirements = project.get_group_requirements(self.options) projects = project.get_projects(self.options) - if not projects: + if not projects and not group_requirements: return requirement_configuration requirements = OrderedSet(requirement_configuration.requirements) - with TRACER.timed( - "Collecting requirements from {count} local {projects}".format( - count=len(projects), projects=pluralize(projects, "project") - ) - ): - requirements.update( - str(req) - for req in projects.collect_requirements( - resolver=ConfiguredResolver(pip_configuration), - interpreter=targets.interpreter, - pip_version=pip_configuration.version, - max_jobs=pip_configuration.max_jobs, + requirements.update(str(req) for req in group_requirements) + if projects: + with TRACER.timed( + "Collecting requirements from {count} local {projects}".format( + count=len(projects), projects=pluralize(projects, "project") + ) + ): + requirements.update( + str(req) + for req in projects.collect_requirements( + resolver=ConfiguredResolver(pip_configuration), + interpreter=targets.interpreter, + pip_version=pip_configuration.version, + max_jobs=pip_configuration.max_jobs, + ) ) - ) return attr.evolve(requirement_configuration, requirements=requirements) def _create(self): diff --git a/pex/pep_723.py b/pex/pep_723.py index 229b74419..d4b316e92 100644 --- a/pex/pep_723.py +++ b/pex/pep_723.py @@ -7,6 +7,7 @@ import re from collections import OrderedDict +from pex import toml from pex.common import pluralize from pex.compatibility import string from pex.dist_metadata import Requirement, RequirementParseError @@ -17,9 +18,8 @@ from typing import Any, List, Mapping, Tuple import attr # vendor:skip - import toml # vendor:skip else: - from pex.third_party import attr, toml + from pex.third_party import attr _UNSPECIFIED_SOURCE = "" diff --git a/pex/resolve/project.py b/pex/resolve/project.py index a144475c6..903c8c425 100644 --- a/pex/resolve/project.py +++ b/pex/resolve/project.py @@ -3,17 +3,21 @@ from __future__ import absolute_import +import os.path from argparse import Namespace, _ActionsContainer -from pex import requirements +from pex import requirements, toml from pex.build_system import pep_517 from pex.common import pluralize +from pex.compatibility import string from pex.dependency_configuration import DependencyConfiguration -from pex.dist_metadata import DistMetadata, Requirement +from pex.dist_metadata import DistMetadata, Requirement, RequirementParseError from pex.fingerprinted_distribution import FingerprintedDistribution from pex.interpreter import PythonInterpreter from pex.jobs import Raise, SpawnedJob, execute_parallel +from pex.orderedset import OrderedSet from pex.pep_427 import InstallableType +from pex.pep_503 import ProjectName from pex.pip.version import PipVersionValue from pex.requirements import LocalProjectRequirement, ParseError from pex.resolve.configured_resolve import resolve @@ -25,7 +29,7 @@ from pex.typing import TYPE_CHECKING if TYPE_CHECKING: - from typing import Iterable, Iterator, List, Optional, Set, Tuple + from typing import Any, Iterable, Iterator, List, Mapping, Optional, Set, Tuple, Union import attr # vendor:skip else: @@ -148,9 +152,147 @@ def __len__(self): return len(self.projects) +@attr.s(frozen=True) +class GroupName(ProjectName): + # N.B.: A dependency group name follows the same rules, including canonicalization, as project + # names. + pass + + +@attr.s(frozen=True) +class DependencyGroup(object): + @classmethod + def parse(cls, spec): + # type: (str) -> DependencyGroup + + group, sep, project_dir = spec.partition("@") + abs_project_dir = os.path.realpath(project_dir) + if not os.path.isdir(abs_project_dir): + raise ValueError( + "The project directory specified by '{spec}' is not a directory".format(spec=spec) + ) + + pyproject_toml = os.path.join(abs_project_dir, "pyproject.toml") + if not os.path.isfile(pyproject_toml): + raise ValueError( + "The project directory specified by '{spec}' does not contain a pyproject.toml " + "file".format(spec=spec) + ) + + group_name = GroupName(group) + try: + dependency_groups = { + GroupName(name): group + for name, group in toml.load(pyproject_toml)["dependency-groups"].items() + } # type: Mapping[GroupName, Any] + except (IOError, OSError, KeyError, ValueError, AttributeError) as e: + raise ValueError( + "Failed to read `[dependency-groups]` metadata from {pyproject_toml} when parsing " + "dependency group spec '{spec}': {err}".format( + pyproject_toml=pyproject_toml, spec=spec, err=e + ) + ) + if group_name not in dependency_groups: + raise KeyError( + "The dependency group '{group}' specified by '{spec}' does not exist in " + "{pyproject_toml}".format(group=group, spec=spec, pyproject_toml=pyproject_toml) + ) + + return cls(project_dir=abs_project_dir, name=group_name, groups=dependency_groups) + + project_dir = attr.ib() # type: str + name = attr.ib() # type: GroupName + _groups = attr.ib() # type: Mapping[GroupName, Any] + + def _parse_group_items( + self, + group, # type: GroupName + required_by=None, # type: Optional[GroupName] + ): + # type: (...) -> Iterator[Union[GroupName, Requirement]] + + members = self._groups.get(group) + if not members: + if not required_by: + raise KeyError( + "The dependency group '{group}' does not exist in the project at " + "{project_dir}.".format(group=group, project_dir=self.project_dir) + ) + else: + raise KeyError( + "The dependency group '{group}' required by dependency group '{required_by}' " + "does not exist in the project at {project_dir}.".format( + group=group, required_by=required_by, project_dir=self.project_dir + ) + ) + + if not isinstance(members, list): + raise ValueError( + "Invalid dependency group '{group}' in the project at {project_dir}.\n" + "The value must be a list containing dependency specifiers or dependency group " + "includes.\n" + "See https://peps.python.org/pep-0735/#specification for the specification " + "of [dependency-groups] syntax." + ) + + for index, item in enumerate(members, start=1): + if isinstance(item, string): + try: + yield Requirement.parse(item) + except RequirementParseError as e: + raise ValueError( + "Invalid [dependency-group] entry '{name}'.\n" + "Item {index}: '{req}', is an invalid dependency specifier: {err}".format( + name=group.raw, index=index, req=item, err=e + ) + ) + elif isinstance(item, dict): + try: + yield GroupName(item["include-group"]) + except KeyError: + raise ValueError( + "Invalid [dependency-group] entry '{name}'.\n" + "Item {index} is a non 'include-group' table and only dependency " + "specifiers and single entry 'include-group' tables are allowed in group " + "dependency lists.\n" + "See https://peps.python.org/pep-0735/#specification for the specification " + "of [dependency-groups] syntax.\n" + "Given: {item}".format(name=group.raw, index=index, item=item) + ) + else: + raise ValueError( + "Invalid [dependency-group] entry '{name}'.\n" + "Item {index} is not a dependency specifier or a dependency group include.\n" + "See https://peps.python.org/pep-0735/#specification for the specification " + "of [dependency-groups] syntax.\n" + "Given: {item}".format(name=group.raw, index=index, item=item) + ) + + def iter_requirements(self): + # type: () -> Iterator[Requirement] + + visited_groups = set() # type: Set[GroupName] + + def iter_group( + group, # type: GroupName + required_by=None, # type: Optional[GroupName] + ): + # type: (...) -> Iterator[Requirement] + if group not in visited_groups: + visited_groups.add(group) + for item in self._parse_group_items(group, required_by=required_by): + if isinstance(item, Requirement): + yield item + else: + for req in iter_group(item, required_by=group): + yield req + + return iter_group(self.name) + + def register_options( parser, # type: _ActionsContainer - help, # type: str + project_help, # type: str ): # type: (...) -> None @@ -161,7 +303,27 @@ def register_options( default=[], type=str, action="append", - help=help, + help=project_help, + ) + + parser.add_argument( + "--group", + "--dependency-group", + dest="dependency_groups", + metavar="GROUP[@DIR]", + default=[], + type=DependencyGroup.parse, + action="append", + help=( + "Pull requirements from the specified PEP-735 dependency group. Dependency groups are " + "specified by referencing the group name in a given project's pyproject.toml in the " + "form `@`; e.g.: `test@local/project/directory`. If " + "either the `@` suffix is not present or the suffix is just `@`, " + "the current working directory is assumed to be the project directory to read the " + "dependency group information from. Multiple dependency groups across any number of " + "projects can be specified. Read more about dependency groups at " + "https://peps.python.org/pep-0735/." + ), ) @@ -207,3 +369,13 @@ def get_projects(options): ) return Projects(projects=tuple(projects)) + + +def get_group_requirements(options): + # type: (Namespace) -> Iterable[Requirement] + + group_requirements = OrderedSet() # type: OrderedSet[Requirement] + for dependency_group in options.dependency_groups: + for requirement in dependency_group.iter_requirements(): + group_requirements.add(requirement) + return group_requirements diff --git a/pex/scie/science.py b/pex/scie/science.py index 62be4207c..63fd8b313 100644 --- a/pex/scie/science.py +++ b/pex/scie/science.py @@ -10,6 +10,7 @@ from collections import OrderedDict from subprocess import CalledProcessError +from pex import toml from pex.atomic_directory import atomic_directory from pex.cache.dirs import CacheDir from pex.common import chmod_plus_x, is_exe, pluralize, safe_mkdtemp, safe_open @@ -44,9 +45,8 @@ from typing import Any, Dict, Iterator, List, Optional, Union, cast import attr # vendor:skip - import toml # vendor:skip else: - from pex.third_party import attr, toml + from pex.third_party import attr @attr.s(frozen=True) @@ -235,7 +235,7 @@ def create_cmd(named_entry_point): }, }, "name": "configure", - "exe": "#{cpython:python}", + "exe": "#{python-distribution:python}", "args": configure_binding_args, } ], @@ -248,12 +248,13 @@ def create_cmd(named_entry_point): ) interpreter_config = { - "id": "cpython", + "id": "python-distribution", "provider": interpreter.provider.value, - "release": interpreter.release, "version": interpreter.version_str, "lazy": configuration.options.style is ScieStyle.LAZY, } + if interpreter.release: + interpreter_config["release"] = interpreter.release if Provider.PythonBuildStandalone is interpreter.provider: interpreter_config.update( flavor=( @@ -263,7 +264,7 @@ def create_cmd(named_entry_point): ) ) - with safe_open(manifest_path, "w") as fp: + with safe_open(manifest_path, "wb") as fp: toml.dump( { "lift": dict( diff --git a/pex/toml.py b/pex/toml.py new file mode 100644 index 000000000..7b04a1f5c --- /dev/null +++ b/pex/toml.py @@ -0,0 +1,60 @@ +# Copyright 2024 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +import sys +from io import BytesIO + +from pex.typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any, Dict, Union + +if sys.version_info[:2] < (3, 7): + from pex.third_party.toml import TomlDecodeError as _TomlDecodeError + from pex.third_party.toml import dump as _dump + from pex.third_party.toml import dumps as _dumps + from pex.third_party.toml import load as _load + from pex.third_party.toml import loads as _loads + + def load(source): + # type: (Union[str, BytesIO]) -> Any + if isinstance(source, str): + return _load(source) + else: + return _loads(source.read().decode("utf-8")) + + def dump( + data, # type: Dict[str, Any] + fp, # type: BytesIO + ): + # type: (...) -> None + fp.write(_dumps(data).decode("utf-8")) + +else: + from pex.third_party.tomli import TOMLDecodeError as _TomlDecodeError + from pex.third_party.tomli import load as _load + from pex.third_party.tomli import loads as _loads + from pex.third_party.tomli_w import dump as _dump + from pex.third_party.tomli_w import dumps as _dumps + + def load(source): + # type: (Union[str, BytesIO]) -> Any + if isinstance(source, str): + with open(source, "rb") as fp: + return _load(fp) + else: + return _load(source) + + def dump( + data, # type: Dict[str, Any] + fp, # type: BytesIO + ): + # type: (...) -> None + _dump(data, fp) + + +loads = _loads +TomlDecodeError = _TomlDecodeError +dumps = _dumps diff --git a/pex/vendor/__init__.py b/pex/vendor/__init__.py index 077ad59b8..36dceae2d 100644 --- a/pex/vendor/__init__.py +++ b/pex/vendor/__init__.py @@ -235,7 +235,14 @@ def iter_vendor_specs(filter_requires_python=None): yield VendorSpec.pinned("packaging", "23.1", import_path="packaging_23_1") # We use toml to read pyproject.toml when building sdists from local source projects. - yield VendorSpec.pinned("toml", "0.10.2") + # The toml project provides compatibility back to Python 2.7, but is frozen in time in 2020 + # with bugs - notably no support for heterogeneous lists. We add the more modern tomli/tomli-w + # for other Pythons. + if not python_major_minor or python_major_minor < (3, 7): + yield VendorSpec.pinned("toml", "0.10.2") + if not python_major_minor or python_major_minor >= (3, 7): + yield VendorSpec.pinned("tomli", "2.0.1") + yield VendorSpec.pinned("tomli-w", "1.0.0") # We shell out to pip at buildtime to resolve and install dependencies. yield PIP_SPEC diff --git a/pex/vendor/_vendored/tomli-w/.layout.json b/pex/vendor/_vendored/tomli-w/.layout.json new file mode 100644 index 000000000..e4af80167 --- /dev/null +++ b/pex/vendor/_vendored/tomli-w/.layout.json @@ -0,0 +1 @@ +{"fingerprint": "aa3678564fc2104aae0d694c677d3ea26701821134153749cc0074077c75e70e", "record_relpath": "tomli_w-1.0.0.dist-info/RECORD", "root_is_purelib": true, "stash_dir": ".prefix"} \ No newline at end of file diff --git a/pex/vendor/_vendored/tomli-w/__init__.py b/pex/vendor/_vendored/tomli-w/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pex/vendor/_vendored/tomli-w/constraints.txt b/pex/vendor/_vendored/tomli-w/constraints.txt new file mode 100644 index 000000000..6b0fcf8b9 --- /dev/null +++ b/pex/vendor/_vendored/tomli-w/constraints.txt @@ -0,0 +1 @@ +tomli_w==1.0.0 diff --git a/pex/vendor/_vendored/tomli-w/tomli_w-1.0.0.dist-info/INSTALLER b/pex/vendor/_vendored/tomli-w/tomli_w-1.0.0.dist-info/INSTALLER new file mode 100644 index 000000000..923be60db --- /dev/null +++ b/pex/vendor/_vendored/tomli-w/tomli_w-1.0.0.dist-info/INSTALLER @@ -0,0 +1 @@ +pex diff --git a/pex/vendor/_vendored/tomli-w/tomli_w-1.0.0.dist-info/LICENSE b/pex/vendor/_vendored/tomli-w/tomli_w-1.0.0.dist-info/LICENSE new file mode 100644 index 000000000..e859590f8 --- /dev/null +++ b/pex/vendor/_vendored/tomli-w/tomli_w-1.0.0.dist-info/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 Taneli Hukkinen + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/pex/vendor/_vendored/tomli-w/tomli_w-1.0.0.dist-info/METADATA b/pex/vendor/_vendored/tomli-w/tomli_w-1.0.0.dist-info/METADATA new file mode 100644 index 000000000..5b309e399 --- /dev/null +++ b/pex/vendor/_vendored/tomli-w/tomli_w-1.0.0.dist-info/METADATA @@ -0,0 +1,144 @@ +Metadata-Version: 2.1 +Name: tomli_w +Version: 1.0.0 +Summary: A lil' TOML writer +Keywords: toml,tomli +Author-email: Taneli Hukkinen +Requires-Python: >=3.7 +Description-Content-Type: text/markdown +Classifier: License :: OSI Approved :: MIT License +Classifier: Operating System :: MacOS +Classifier: Operating System :: Microsoft :: Windows +Classifier: Operating System :: POSIX :: Linux +Classifier: Programming Language :: Python :: 3 :: Only +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Classifier: Typing :: Typed +Project-URL: Changelog, https://github.com/hukkin/tomli-w/blob/master/CHANGELOG.md +Project-URL: Homepage, https://github.com/hukkin/tomli-w + +[![Build Status](https://github.com/hukkin/tomli-w/workflows/Tests/badge.svg?branch=master)](https://github.com/hukkin/tomli-w/actions?query=workflow%3ATests+branch%3Amaster+event%3Apush) +[![codecov.io](https://codecov.io/gh/hukkin/tomli-w/branch/master/graph/badge.svg)](https://codecov.io/gh/hukkin/tomli-w) +[![PyPI version](https://img.shields.io/pypi/v/tomli-w)](https://pypi.org/project/tomli-w) + +# Tomli-W + +> A lil' TOML writer + +**Table of Contents** *generated with [mdformat-toc](https://github.com/hukkin/mdformat-toc)* + + + +- [Intro](#intro) +- [Installation](#installation) +- [Usage](#usage) + - [Write to string](#write-to-string) + - [Write to file](#write-to-file) +- [FAQ](#faq) + - [Does Tomli-W sort the document?](#does-tomli-w-sort-the-document) + - [Does Tomli-W support writing documents with comments or custom whitespace?](#does-tomli-w-support-writing-documents-with-comments-or-custom-whitespace) + - [Why does Tomli-W not write a multi-line string if the string value contains newlines?](#why-does-tomli-w-not-write-a-multi-line-string-if-the-string-value-contains-newlines) + - [Is Tomli-W output guaranteed to be valid TOML?](#is-tomli-w-output-guaranteed-to-be-valid-toml) + + + +## Intro + +Tomli-W is a Python library for writing [TOML](https://toml.io). +It is a write-only counterpart to [Tomli](https://github.com/hukkin/tomli), +which is a read-only TOML parser. +Tomli-W is fully compatible with [TOML v1.0.0](https://toml.io/en/v1.0.0). + +## Installation + +```bash +pip install tomli-w +``` + +## Usage + +### Write to string + +```python +import tomli_w + +doc = {"table": {"nested": {}, "val3": 3}, "val2": 2, "val1": 1} +expected_toml = """\ +val2 = 2 +val1 = 1 + +[table] +val3 = 3 + +[table.nested] +""" +assert tomli_w.dumps(doc) == expected_toml +``` + +### Write to file + +```python +import tomli_w + +doc = {"one": 1, "two": 2, "pi": 3} +with open("path_to_file/conf.toml", "wb") as f: + tomli_w.dump(doc, f) +``` + +## FAQ + +### Does Tomli-W sort the document? + +No, but it respects sort order of the input data, +so one could sort the content of the `dict` (recursively) before calling `tomli_w.dumps`. + +### Does Tomli-W support writing documents with comments or custom whitespace? + +No. + +### Why does Tomli-W not write a multi-line string if the string value contains newlines? + +This default was chosen to achieve lossless parse/write round-trips. + +TOML strings can contain newlines where exact bytes matter, e.g. + +```toml +s = "here's a newline\r\n" +``` + +TOML strings also can contain newlines where exact byte representation is not relevant, e.g. + +```toml +s = """here's a newline +""" +``` + +A parse/write round-trip that converts the former example to the latter does not preserve the original newline byte sequence. +This is why Tomli-W avoids writing multi-line strings. + +A keyword argument is provided for users who do not need newline bytes to be preserved: + +```python +import tomli_w + +doc = {"s": "here's a newline\r\n"} +expected_toml = '''\ +s = """ +here's a newline +""" +''' +assert tomli_w.dumps(doc, multiline_strings=True) == expected_toml +``` + +### Is Tomli-W output guaranteed to be valid TOML? + +No. +If there's a chance that your input data is bad and you need output validation, +parse the output string once with `tomli.loads`. +If the parse is successful (does not raise `tomli.TOMLDecodeError`) then the string is valid TOML. + diff --git a/pex/vendor/_vendored/tomli-w/tomli_w-1.0.0.dist-info/WHEEL b/pex/vendor/_vendored/tomli-w/tomli_w-1.0.0.dist-info/WHEEL new file mode 100644 index 000000000..884ceb565 --- /dev/null +++ b/pex/vendor/_vendored/tomli-w/tomli_w-1.0.0.dist-info/WHEEL @@ -0,0 +1,4 @@ +Wheel-Version: 1.0 +Generator: flit 3.5.1 +Root-Is-Purelib: true +Tag: py3-none-any diff --git a/pex/vendor/_vendored/tomli-w/tomli_w/__init__.py b/pex/vendor/_vendored/tomli-w/tomli_w/__init__.py new file mode 100644 index 000000000..a7726c94b --- /dev/null +++ b/pex/vendor/_vendored/tomli-w/tomli_w/__init__.py @@ -0,0 +1,8 @@ +__all__ = ("dumps", "dump") +__version__ = "1.0.0" # DO NOT EDIT THIS LINE MANUALLY. LET bump2version UTILITY DO IT + +if "tomli-w" in __import__("os").environ.get("__PEX_UNVENDORED__", ""): + from tomli_w._writer import dump, dumps # vendor:skip +else: + from pex.third_party.tomli_w._writer import dump, dumps + diff --git a/pex/vendor/_vendored/tomli-w/tomli_w/_writer.py b/pex/vendor/_vendored/tomli-w/tomli_w/_writer.py new file mode 100644 index 000000000..45efcf782 --- /dev/null +++ b/pex/vendor/_vendored/tomli-w/tomli_w/_writer.py @@ -0,0 +1,199 @@ +from __future__ import annotations + +from collections.abc import Generator, Mapping +from datetime import date, datetime, time +from decimal import Decimal +import string +from types import MappingProxyType +from typing import Any, BinaryIO, NamedTuple + +ASCII_CTRL = frozenset(chr(i) for i in range(32)) | frozenset(chr(127)) +ILLEGAL_BASIC_STR_CHARS = frozenset('"\\') | ASCII_CTRL - frozenset("\t") +BARE_KEY_CHARS = frozenset(string.ascii_letters + string.digits + "-_") +ARRAY_TYPES = (list, tuple) +ARRAY_INDENT = " " * 4 +MAX_LINE_LENGTH = 100 + +COMPACT_ESCAPES = MappingProxyType( + { + "\u0008": "\\b", # backspace + "\u000A": "\\n", # linefeed + "\u000C": "\\f", # form feed + "\u000D": "\\r", # carriage return + "\u0022": '\\"', # quote + "\u005C": "\\\\", # backslash + } +) + + +def dump( + __obj: dict[str, Any], __fp: BinaryIO, *, multiline_strings: bool = False +) -> None: + ctx = Context(multiline_strings, {}) + for chunk in gen_table_chunks(__obj, ctx, name=""): + __fp.write(chunk.encode()) + + +def dumps(__obj: dict[str, Any], *, multiline_strings: bool = False) -> str: + ctx = Context(multiline_strings, {}) + return "".join(gen_table_chunks(__obj, ctx, name="")) + + +class Context(NamedTuple): + allow_multiline: bool + # cache rendered inline tables (mapping from object id to rendered inline table) + inline_table_cache: dict[int, str] + + +def gen_table_chunks( + table: Mapping[str, Any], + ctx: Context, + *, + name: str, + inside_aot: bool = False, +) -> Generator[str, None, None]: + yielded = False + literals = [] + tables: list[tuple[str, Any, bool]] = [] # => [(key, value, inside_aot)] + for k, v in table.items(): + if isinstance(v, dict): + tables.append((k, v, False)) + elif is_aot(v) and not all(is_suitable_inline_table(t, ctx) for t in v): + tables.extend((k, t, True) for t in v) + else: + literals.append((k, v)) + + if inside_aot or name and (literals or not tables): + yielded = True + yield f"[[{name}]]\n" if inside_aot else f"[{name}]\n" + + if literals: + yielded = True + for k, v in literals: + yield f"{format_key_part(k)} = {format_literal(v, ctx)}\n" + + for k, v, in_aot in tables: + if yielded: + yield "\n" + else: + yielded = True + key_part = format_key_part(k) + display_name = f"{name}.{key_part}" if name else key_part + yield from gen_table_chunks(v, ctx, name=display_name, inside_aot=in_aot) + + +def format_literal(obj: object, ctx: Context, *, nest_level: int = 0) -> str: + if isinstance(obj, bool): + return "true" if obj else "false" + if isinstance(obj, (int, float, date, datetime)): + return str(obj) + if isinstance(obj, Decimal): + return format_decimal(obj) + if isinstance(obj, time): + if obj.tzinfo: + raise ValueError("TOML does not support offset times") + return str(obj) + if isinstance(obj, str): + return format_string(obj, allow_multiline=ctx.allow_multiline) + if isinstance(obj, ARRAY_TYPES): + return format_inline_array(obj, ctx, nest_level) + if isinstance(obj, dict): + return format_inline_table(obj, ctx) + raise TypeError(f"Object of type {type(obj)} is not TOML serializable") + + +def format_decimal(obj: Decimal) -> str: + if obj.is_nan(): + return "nan" + if obj == Decimal("inf"): + return "inf" + if obj == Decimal("-inf"): + return "-inf" + return str(obj) + + +def format_inline_table(obj: dict, ctx: Context) -> str: + # check cache first + obj_id = id(obj) + if obj_id in ctx.inline_table_cache: + return ctx.inline_table_cache[obj_id] + + if not obj: + rendered = "{}" + else: + rendered = ( + "{ " + + ", ".join( + f"{format_key_part(k)} = {format_literal(v, ctx)}" + for k, v in obj.items() + ) + + " }" + ) + ctx.inline_table_cache[obj_id] = rendered + return rendered + + +def format_inline_array(obj: tuple | list, ctx: Context, nest_level: int) -> str: + if not obj: + return "[]" + item_indent = ARRAY_INDENT * (1 + nest_level) + closing_bracket_indent = ARRAY_INDENT * nest_level + return ( + "[\n" + + ",\n".join( + item_indent + format_literal(item, ctx, nest_level=nest_level + 1) + for item in obj + ) + + f",\n{closing_bracket_indent}]" + ) + + +def format_key_part(part: str) -> str: + if part and BARE_KEY_CHARS.issuperset(part): + return part + return format_string(part, allow_multiline=False) + + +def format_string(s: str, *, allow_multiline: bool) -> str: + do_multiline = allow_multiline and "\n" in s + if do_multiline: + result = '"""\n' + s = s.replace("\r\n", "\n") + else: + result = '"' + + pos = seq_start = 0 + while True: + try: + char = s[pos] + except IndexError: + result += s[seq_start:pos] + if do_multiline: + return result + '"""' + return result + '"' + if char in ILLEGAL_BASIC_STR_CHARS: + result += s[seq_start:pos] + if char in COMPACT_ESCAPES: + if do_multiline and char == "\n": + result += "\n" + else: + result += COMPACT_ESCAPES[char] + else: + result += "\\u" + hex(ord(char))[2:].rjust(4, "0") + seq_start = pos + 1 + pos += 1 + + +def is_aot(obj: Any) -> bool: + """Decides if an object behaves as an array of tables (i.e. a nonempty list + of dicts).""" + return bool( + isinstance(obj, ARRAY_TYPES) and obj and all(isinstance(v, dict) for v in obj) + ) + + +def is_suitable_inline_table(obj: dict, ctx: Context) -> bool: + """Use heuristics to decide if the inline-style representation is a good + choice for a given table.""" + rendered_inline = f"{ARRAY_INDENT}{format_inline_table(obj, ctx)}," + return len(rendered_inline) <= MAX_LINE_LENGTH and "\n" not in rendered_inline diff --git a/pex/vendor/_vendored/tomli-w/tomli_w/py.typed b/pex/vendor/_vendored/tomli-w/tomli_w/py.typed new file mode 100644 index 000000000..7632ecf77 --- /dev/null +++ b/pex/vendor/_vendored/tomli-w/tomli_w/py.typed @@ -0,0 +1 @@ +# Marker file for PEP 561 diff --git a/pex/vendor/_vendored/tomli/.layout.json b/pex/vendor/_vendored/tomli/.layout.json new file mode 100644 index 000000000..c09f2042f --- /dev/null +++ b/pex/vendor/_vendored/tomli/.layout.json @@ -0,0 +1 @@ +{"fingerprint": "0454d05ff5986998a7ace56edd25253452f464fd365f7ab4295edba84b58162f", "record_relpath": "tomli-2.0.1.dist-info/RECORD", "root_is_purelib": true, "stash_dir": ".prefix"} \ No newline at end of file diff --git a/pex/vendor/_vendored/tomli/__init__.py b/pex/vendor/_vendored/tomli/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pex/vendor/_vendored/tomli/constraints.txt b/pex/vendor/_vendored/tomli/constraints.txt new file mode 100644 index 000000000..1d0cc1486 --- /dev/null +++ b/pex/vendor/_vendored/tomli/constraints.txt @@ -0,0 +1 @@ +tomli==2.0.1 diff --git a/pex/vendor/_vendored/tomli/tomli-2.0.1.dist-info/INSTALLER b/pex/vendor/_vendored/tomli/tomli-2.0.1.dist-info/INSTALLER new file mode 100644 index 000000000..923be60db --- /dev/null +++ b/pex/vendor/_vendored/tomli/tomli-2.0.1.dist-info/INSTALLER @@ -0,0 +1 @@ +pex diff --git a/pex/vendor/_vendored/tomli/tomli-2.0.1.dist-info/LICENSE b/pex/vendor/_vendored/tomli/tomli-2.0.1.dist-info/LICENSE new file mode 100644 index 000000000..e859590f8 --- /dev/null +++ b/pex/vendor/_vendored/tomli/tomli-2.0.1.dist-info/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 Taneli Hukkinen + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/pex/vendor/_vendored/tomli/tomli-2.0.1.dist-info/METADATA b/pex/vendor/_vendored/tomli/tomli-2.0.1.dist-info/METADATA new file mode 100644 index 000000000..efd87ecc1 --- /dev/null +++ b/pex/vendor/_vendored/tomli/tomli-2.0.1.dist-info/METADATA @@ -0,0 +1,206 @@ +Metadata-Version: 2.1 +Name: tomli +Version: 2.0.1 +Summary: A lil' TOML parser +Keywords: toml +Author-email: Taneli Hukkinen +Requires-Python: >=3.7 +Description-Content-Type: text/markdown +Classifier: License :: OSI Approved :: MIT License +Classifier: Operating System :: MacOS +Classifier: Operating System :: Microsoft :: Windows +Classifier: Operating System :: POSIX :: Linux +Classifier: Programming Language :: Python :: 3 :: Only +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Classifier: Typing :: Typed +Project-URL: Changelog, https://github.com/hukkin/tomli/blob/master/CHANGELOG.md +Project-URL: Homepage, https://github.com/hukkin/tomli + +[![Build Status](https://github.com/hukkin/tomli/workflows/Tests/badge.svg?branch=master)](https://github.com/hukkin/tomli/actions?query=workflow%3ATests+branch%3Amaster+event%3Apush) +[![codecov.io](https://codecov.io/gh/hukkin/tomli/branch/master/graph/badge.svg)](https://codecov.io/gh/hukkin/tomli) +[![PyPI version](https://img.shields.io/pypi/v/tomli)](https://pypi.org/project/tomli) + +# Tomli + +> A lil' TOML parser + +**Table of Contents** *generated with [mdformat-toc](https://github.com/hukkin/mdformat-toc)* + + + +- [Intro](#intro) +- [Installation](#installation) +- [Usage](#usage) + - [Parse a TOML string](#parse-a-toml-string) + - [Parse a TOML file](#parse-a-toml-file) + - [Handle invalid TOML](#handle-invalid-toml) + - [Construct `decimal.Decimal`s from TOML floats](#construct-decimaldecimals-from-toml-floats) +- [FAQ](#faq) + - [Why this parser?](#why-this-parser) + - [Is comment preserving round-trip parsing supported?](#is-comment-preserving-round-trip-parsing-supported) + - [Is there a `dumps`, `write` or `encode` function?](#is-there-a-dumps-write-or-encode-function) + - [How do TOML types map into Python types?](#how-do-toml-types-map-into-python-types) +- [Performance](#performance) + + + +## Intro + +Tomli is a Python library for parsing [TOML](https://toml.io). +Tomli is fully compatible with [TOML v1.0.0](https://toml.io/en/v1.0.0). + +## Installation + +```bash +pip install tomli +``` + +## Usage + +### Parse a TOML string + +```python +import tomli + +toml_str = """ + gretzky = 99 + + [kurri] + jari = 17 + """ + +toml_dict = tomli.loads(toml_str) +assert toml_dict == {"gretzky": 99, "kurri": {"jari": 17}} +``` + +### Parse a TOML file + +```python +import tomli + +with open("path_to_file/conf.toml", "rb") as f: + toml_dict = tomli.load(f) +``` + +The file must be opened in binary mode (with the `"rb"` flag). +Binary mode will enforce decoding the file as UTF-8 with universal newlines disabled, +both of which are required to correctly parse TOML. + +### Handle invalid TOML + +```python +import tomli + +try: + toml_dict = tomli.loads("]] this is invalid TOML [[") +except tomli.TOMLDecodeError: + print("Yep, definitely not valid.") +``` + +Note that error messages are considered informational only. +They should not be assumed to stay constant across Tomli versions. + +### Construct `decimal.Decimal`s from TOML floats + +```python +from decimal import Decimal +import tomli + +toml_dict = tomli.loads("precision-matters = 0.982492", parse_float=Decimal) +assert toml_dict["precision-matters"] == Decimal("0.982492") +``` + +Note that `decimal.Decimal` can be replaced with another callable that converts a TOML float from string to a Python type. +The `decimal.Decimal` is, however, a practical choice for use cases where float inaccuracies can not be tolerated. + +Illegal types are `dict` and `list`, and their subtypes. +A `ValueError` will be raised if `parse_float` produces illegal types. + +## FAQ + +### Why this parser? + +- it's lil' +- pure Python with zero dependencies +- the fastest pure Python parser [\*](#performance): + 15x as fast as [tomlkit](https://pypi.org/project/tomlkit/), + 2.4x as fast as [toml](https://pypi.org/project/toml/) +- outputs [basic data types](#how-do-toml-types-map-into-python-types) only +- 100% spec compliant: passes all tests in + [a test set](https://github.com/toml-lang/compliance/pull/8) + soon to be merged to the official + [compliance tests for TOML](https://github.com/toml-lang/compliance) + repository +- thoroughly tested: 100% branch coverage + +### Is comment preserving round-trip parsing supported? + +No. + +The `tomli.loads` function returns a plain `dict` that is populated with builtin types and types from the standard library only. +Preserving comments requires a custom type to be returned so will not be supported, +at least not by the `tomli.loads` and `tomli.load` functions. + +Look into [TOML Kit](https://github.com/sdispater/tomlkit) if preservation of style is what you need. + +### Is there a `dumps`, `write` or `encode` function? + +[Tomli-W](https://github.com/hukkin/tomli-w) is the write-only counterpart of Tomli, providing `dump` and `dumps` functions. + +The core library does not include write capability, as most TOML use cases are read-only, and Tomli intends to be minimal. + +### How do TOML types map into Python types? + +| TOML type | Python type | Details | +| ---------------- | ------------------- | ------------------------------------------------------------ | +| Document Root | `dict` | | +| Key | `str` | | +| String | `str` | | +| Integer | `int` | | +| Float | `float` | | +| Boolean | `bool` | | +| Offset Date-Time | `datetime.datetime` | `tzinfo` attribute set to an instance of `datetime.timezone` | +| Local Date-Time | `datetime.datetime` | `tzinfo` attribute set to `None` | +| Local Date | `datetime.date` | | +| Local Time | `datetime.time` | | +| Array | `list` | | +| Table | `dict` | | +| Inline Table | `dict` | | + +## Performance + +The `benchmark/` folder in this repository contains a performance benchmark for comparing the various Python TOML parsers. +The benchmark can be run with `tox -e benchmark-pypi`. +Running the benchmark on my personal computer output the following: + +```console +foo@bar:~/dev/tomli$ tox -e benchmark-pypi +benchmark-pypi installed: attrs==19.3.0,click==7.1.2,pytomlpp==1.0.2,qtoml==0.3.0,rtoml==0.7.0,toml==0.10.2,tomli==1.1.0,tomlkit==0.7.2 +benchmark-pypi run-test-pre: PYTHONHASHSEED='2658546909' +benchmark-pypi run-test: commands[0] | python -c 'import datetime; print(datetime.date.today())' +2021-07-23 +benchmark-pypi run-test: commands[1] | python --version +Python 3.8.10 +benchmark-pypi run-test: commands[2] | python benchmark/run.py +Parsing data.toml 5000 times: +------------------------------------------------------ + parser | exec time | performance (more is better) +-----------+------------+----------------------------- + rtoml | 0.901 s | baseline (100%) + pytomlpp | 1.08 s | 83.15% + tomli | 3.89 s | 23.15% + toml | 9.36 s | 9.63% + qtoml | 11.5 s | 7.82% + tomlkit | 56.8 s | 1.59% +``` + +The parsers are ordered from fastest to slowest, using the fastest parser as baseline. +Tomli performed the best out of all pure Python TOML parsers, +losing only to pytomlpp (wraps C++) and rtoml (wraps Rust). + diff --git a/pex/vendor/_vendored/tomli/tomli-2.0.1.dist-info/WHEEL b/pex/vendor/_vendored/tomli/tomli-2.0.1.dist-info/WHEEL new file mode 100644 index 000000000..c727d1482 --- /dev/null +++ b/pex/vendor/_vendored/tomli/tomli-2.0.1.dist-info/WHEEL @@ -0,0 +1,4 @@ +Wheel-Version: 1.0 +Generator: flit 3.6.0 +Root-Is-Purelib: true +Tag: py3-none-any diff --git a/pex/vendor/_vendored/tomli/tomli/__init__.py b/pex/vendor/_vendored/tomli/tomli/__init__.py new file mode 100644 index 000000000..4c6ec97ec --- /dev/null +++ b/pex/vendor/_vendored/tomli/tomli/__init__.py @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: MIT +# SPDX-FileCopyrightText: 2021 Taneli Hukkinen +# Licensed to PSF under a Contributor Agreement. + +__all__ = ("loads", "load", "TOMLDecodeError") +__version__ = "2.0.1" # DO NOT EDIT THIS LINE MANUALLY. LET bump2version UTILITY DO IT + +from ._parser import TOMLDecodeError, load, loads + +# Pretend this exception was created here. +TOMLDecodeError.__module__ = __name__ diff --git a/pex/vendor/_vendored/tomli/tomli/_parser.py b/pex/vendor/_vendored/tomli/tomli/_parser.py new file mode 100644 index 000000000..f1bb0aa19 --- /dev/null +++ b/pex/vendor/_vendored/tomli/tomli/_parser.py @@ -0,0 +1,691 @@ +# SPDX-License-Identifier: MIT +# SPDX-FileCopyrightText: 2021 Taneli Hukkinen +# Licensed to PSF under a Contributor Agreement. + +from __future__ import annotations + +from collections.abc import Iterable +import string +from types import MappingProxyType +from typing import Any, BinaryIO, NamedTuple + +from ._re import ( + RE_DATETIME, + RE_LOCALTIME, + RE_NUMBER, + match_to_datetime, + match_to_localtime, + match_to_number, +) +from ._types import Key, ParseFloat, Pos + +ASCII_CTRL = frozenset(chr(i) for i in range(32)) | frozenset(chr(127)) + +# Neither of these sets include quotation mark or backslash. They are +# currently handled as separate cases in the parser functions. +ILLEGAL_BASIC_STR_CHARS = ASCII_CTRL - frozenset("\t") +ILLEGAL_MULTILINE_BASIC_STR_CHARS = ASCII_CTRL - frozenset("\t\n") + +ILLEGAL_LITERAL_STR_CHARS = ILLEGAL_BASIC_STR_CHARS +ILLEGAL_MULTILINE_LITERAL_STR_CHARS = ILLEGAL_MULTILINE_BASIC_STR_CHARS + +ILLEGAL_COMMENT_CHARS = ILLEGAL_BASIC_STR_CHARS + +TOML_WS = frozenset(" \t") +TOML_WS_AND_NEWLINE = TOML_WS | frozenset("\n") +BARE_KEY_CHARS = frozenset(string.ascii_letters + string.digits + "-_") +KEY_INITIAL_CHARS = BARE_KEY_CHARS | frozenset("\"'") +HEXDIGIT_CHARS = frozenset(string.hexdigits) + +BASIC_STR_ESCAPE_REPLACEMENTS = MappingProxyType( + { + "\\b": "\u0008", # backspace + "\\t": "\u0009", # tab + "\\n": "\u000A", # linefeed + "\\f": "\u000C", # form feed + "\\r": "\u000D", # carriage return + '\\"': "\u0022", # quote + "\\\\": "\u005C", # backslash + } +) + + +class TOMLDecodeError(ValueError): + """An error raised if a document is not valid TOML.""" + + +def load(__fp: BinaryIO, *, parse_float: ParseFloat = float) -> dict[str, Any]: + """Parse TOML from a binary file object.""" + b = __fp.read() + try: + s = b.decode() + except AttributeError: + raise TypeError( + "File must be opened in binary mode, e.g. use `open('foo.toml', 'rb')`" + ) from None + return loads(s, parse_float=parse_float) + + +def loads(__s: str, *, parse_float: ParseFloat = float) -> dict[str, Any]: # noqa: C901 + """Parse TOML from a string.""" + + # The spec allows converting "\r\n" to "\n", even in string + # literals. Let's do so to simplify parsing. + src = __s.replace("\r\n", "\n") + pos = 0 + out = Output(NestedDict(), Flags()) + header: Key = () + parse_float = make_safe_parse_float(parse_float) + + # Parse one statement at a time + # (typically means one line in TOML source) + while True: + # 1. Skip line leading whitespace + pos = skip_chars(src, pos, TOML_WS) + + # 2. Parse rules. Expect one of the following: + # - end of file + # - end of line + # - comment + # - key/value pair + # - append dict to list (and move to its namespace) + # - create dict (and move to its namespace) + # Skip trailing whitespace when applicable. + try: + char = src[pos] + except IndexError: + break + if char == "\n": + pos += 1 + continue + if char in KEY_INITIAL_CHARS: + pos = key_value_rule(src, pos, out, header, parse_float) + pos = skip_chars(src, pos, TOML_WS) + elif char == "[": + try: + second_char: str | None = src[pos + 1] + except IndexError: + second_char = None + out.flags.finalize_pending() + if second_char == "[": + pos, header = create_list_rule(src, pos, out) + else: + pos, header = create_dict_rule(src, pos, out) + pos = skip_chars(src, pos, TOML_WS) + elif char != "#": + raise suffixed_err(src, pos, "Invalid statement") + + # 3. Skip comment + pos = skip_comment(src, pos) + + # 4. Expect end of line or end of file + try: + char = src[pos] + except IndexError: + break + if char != "\n": + raise suffixed_err( + src, pos, "Expected newline or end of document after a statement" + ) + pos += 1 + + return out.data.dict + + +class Flags: + """Flags that map to parsed keys/namespaces.""" + + # Marks an immutable namespace (inline array or inline table). + FROZEN = 0 + # Marks a nest that has been explicitly created and can no longer + # be opened using the "[table]" syntax. + EXPLICIT_NEST = 1 + + def __init__(self) -> None: + self._flags: dict[str, dict] = {} + self._pending_flags: set[tuple[Key, int]] = set() + + def add_pending(self, key: Key, flag: int) -> None: + self._pending_flags.add((key, flag)) + + def finalize_pending(self) -> None: + for key, flag in self._pending_flags: + self.set(key, flag, recursive=False) + self._pending_flags.clear() + + def unset_all(self, key: Key) -> None: + cont = self._flags + for k in key[:-1]: + if k not in cont: + return + cont = cont[k]["nested"] + cont.pop(key[-1], None) + + def set(self, key: Key, flag: int, *, recursive: bool) -> None: # noqa: A003 + cont = self._flags + key_parent, key_stem = key[:-1], key[-1] + for k in key_parent: + if k not in cont: + cont[k] = {"flags": set(), "recursive_flags": set(), "nested": {}} + cont = cont[k]["nested"] + if key_stem not in cont: + cont[key_stem] = {"flags": set(), "recursive_flags": set(), "nested": {}} + cont[key_stem]["recursive_flags" if recursive else "flags"].add(flag) + + def is_(self, key: Key, flag: int) -> bool: + if not key: + return False # document root has no flags + cont = self._flags + for k in key[:-1]: + if k not in cont: + return False + inner_cont = cont[k] + if flag in inner_cont["recursive_flags"]: + return True + cont = inner_cont["nested"] + key_stem = key[-1] + if key_stem in cont: + cont = cont[key_stem] + return flag in cont["flags"] or flag in cont["recursive_flags"] + return False + + +class NestedDict: + def __init__(self) -> None: + # The parsed content of the TOML document + self.dict: dict[str, Any] = {} + + def get_or_create_nest( + self, + key: Key, + *, + access_lists: bool = True, + ) -> dict: + cont: Any = self.dict + for k in key: + if k not in cont: + cont[k] = {} + cont = cont[k] + if access_lists and isinstance(cont, list): + cont = cont[-1] + if not isinstance(cont, dict): + raise KeyError("There is no nest behind this key") + return cont + + def append_nest_to_list(self, key: Key) -> None: + cont = self.get_or_create_nest(key[:-1]) + last_key = key[-1] + if last_key in cont: + list_ = cont[last_key] + if not isinstance(list_, list): + raise KeyError("An object other than list found behind this key") + list_.append({}) + else: + cont[last_key] = [{}] + + +class Output(NamedTuple): + data: NestedDict + flags: Flags + + +def skip_chars(src: str, pos: Pos, chars: Iterable[str]) -> Pos: + try: + while src[pos] in chars: + pos += 1 + except IndexError: + pass + return pos + + +def skip_until( + src: str, + pos: Pos, + expect: str, + *, + error_on: frozenset[str], + error_on_eof: bool, +) -> Pos: + try: + new_pos = src.index(expect, pos) + except ValueError: + new_pos = len(src) + if error_on_eof: + raise suffixed_err(src, new_pos, f"Expected {expect!r}") from None + + if not error_on.isdisjoint(src[pos:new_pos]): + while src[pos] not in error_on: + pos += 1 + raise suffixed_err(src, pos, f"Found invalid character {src[pos]!r}") + return new_pos + + +def skip_comment(src: str, pos: Pos) -> Pos: + try: + char: str | None = src[pos] + except IndexError: + char = None + if char == "#": + return skip_until( + src, pos + 1, "\n", error_on=ILLEGAL_COMMENT_CHARS, error_on_eof=False + ) + return pos + + +def skip_comments_and_array_ws(src: str, pos: Pos) -> Pos: + while True: + pos_before_skip = pos + pos = skip_chars(src, pos, TOML_WS_AND_NEWLINE) + pos = skip_comment(src, pos) + if pos == pos_before_skip: + return pos + + +def create_dict_rule(src: str, pos: Pos, out: Output) -> tuple[Pos, Key]: + pos += 1 # Skip "[" + pos = skip_chars(src, pos, TOML_WS) + pos, key = parse_key(src, pos) + + if out.flags.is_(key, Flags.EXPLICIT_NEST) or out.flags.is_(key, Flags.FROZEN): + raise suffixed_err(src, pos, f"Cannot declare {key} twice") + out.flags.set(key, Flags.EXPLICIT_NEST, recursive=False) + try: + out.data.get_or_create_nest(key) + except KeyError: + raise suffixed_err(src, pos, "Cannot overwrite a value") from None + + if not src.startswith("]", pos): + raise suffixed_err(src, pos, "Expected ']' at the end of a table declaration") + return pos + 1, key + + +def create_list_rule(src: str, pos: Pos, out: Output) -> tuple[Pos, Key]: + pos += 2 # Skip "[[" + pos = skip_chars(src, pos, TOML_WS) + pos, key = parse_key(src, pos) + + if out.flags.is_(key, Flags.FROZEN): + raise suffixed_err(src, pos, f"Cannot mutate immutable namespace {key}") + # Free the namespace now that it points to another empty list item... + out.flags.unset_all(key) + # ...but this key precisely is still prohibited from table declaration + out.flags.set(key, Flags.EXPLICIT_NEST, recursive=False) + try: + out.data.append_nest_to_list(key) + except KeyError: + raise suffixed_err(src, pos, "Cannot overwrite a value") from None + + if not src.startswith("]]", pos): + raise suffixed_err(src, pos, "Expected ']]' at the end of an array declaration") + return pos + 2, key + + +def key_value_rule( + src: str, pos: Pos, out: Output, header: Key, parse_float: ParseFloat +) -> Pos: + pos, key, value = parse_key_value_pair(src, pos, parse_float) + key_parent, key_stem = key[:-1], key[-1] + abs_key_parent = header + key_parent + + relative_path_cont_keys = (header + key[:i] for i in range(1, len(key))) + for cont_key in relative_path_cont_keys: + # Check that dotted key syntax does not redefine an existing table + if out.flags.is_(cont_key, Flags.EXPLICIT_NEST): + raise suffixed_err(src, pos, f"Cannot redefine namespace {cont_key}") + # Containers in the relative path can't be opened with the table syntax or + # dotted key/value syntax in following table sections. + out.flags.add_pending(cont_key, Flags.EXPLICIT_NEST) + + if out.flags.is_(abs_key_parent, Flags.FROZEN): + raise suffixed_err( + src, pos, f"Cannot mutate immutable namespace {abs_key_parent}" + ) + + try: + nest = out.data.get_or_create_nest(abs_key_parent) + except KeyError: + raise suffixed_err(src, pos, "Cannot overwrite a value") from None + if key_stem in nest: + raise suffixed_err(src, pos, "Cannot overwrite a value") + # Mark inline table and array namespaces recursively immutable + if isinstance(value, (dict, list)): + out.flags.set(header + key, Flags.FROZEN, recursive=True) + nest[key_stem] = value + return pos + + +def parse_key_value_pair( + src: str, pos: Pos, parse_float: ParseFloat +) -> tuple[Pos, Key, Any]: + pos, key = parse_key(src, pos) + try: + char: str | None = src[pos] + except IndexError: + char = None + if char != "=": + raise suffixed_err(src, pos, "Expected '=' after a key in a key/value pair") + pos += 1 + pos = skip_chars(src, pos, TOML_WS) + pos, value = parse_value(src, pos, parse_float) + return pos, key, value + + +def parse_key(src: str, pos: Pos) -> tuple[Pos, Key]: + pos, key_part = parse_key_part(src, pos) + key: Key = (key_part,) + pos = skip_chars(src, pos, TOML_WS) + while True: + try: + char: str | None = src[pos] + except IndexError: + char = None + if char != ".": + return pos, key + pos += 1 + pos = skip_chars(src, pos, TOML_WS) + pos, key_part = parse_key_part(src, pos) + key += (key_part,) + pos = skip_chars(src, pos, TOML_WS) + + +def parse_key_part(src: str, pos: Pos) -> tuple[Pos, str]: + try: + char: str | None = src[pos] + except IndexError: + char = None + if char in BARE_KEY_CHARS: + start_pos = pos + pos = skip_chars(src, pos, BARE_KEY_CHARS) + return pos, src[start_pos:pos] + if char == "'": + return parse_literal_str(src, pos) + if char == '"': + return parse_one_line_basic_str(src, pos) + raise suffixed_err(src, pos, "Invalid initial character for a key part") + + +def parse_one_line_basic_str(src: str, pos: Pos) -> tuple[Pos, str]: + pos += 1 + return parse_basic_str(src, pos, multiline=False) + + +def parse_array(src: str, pos: Pos, parse_float: ParseFloat) -> tuple[Pos, list]: + pos += 1 + array: list = [] + + pos = skip_comments_and_array_ws(src, pos) + if src.startswith("]", pos): + return pos + 1, array + while True: + pos, val = parse_value(src, pos, parse_float) + array.append(val) + pos = skip_comments_and_array_ws(src, pos) + + c = src[pos : pos + 1] + if c == "]": + return pos + 1, array + if c != ",": + raise suffixed_err(src, pos, "Unclosed array") + pos += 1 + + pos = skip_comments_and_array_ws(src, pos) + if src.startswith("]", pos): + return pos + 1, array + + +def parse_inline_table(src: str, pos: Pos, parse_float: ParseFloat) -> tuple[Pos, dict]: + pos += 1 + nested_dict = NestedDict() + flags = Flags() + + pos = skip_chars(src, pos, TOML_WS) + if src.startswith("}", pos): + return pos + 1, nested_dict.dict + while True: + pos, key, value = parse_key_value_pair(src, pos, parse_float) + key_parent, key_stem = key[:-1], key[-1] + if flags.is_(key, Flags.FROZEN): + raise suffixed_err(src, pos, f"Cannot mutate immutable namespace {key}") + try: + nest = nested_dict.get_or_create_nest(key_parent, access_lists=False) + except KeyError: + raise suffixed_err(src, pos, "Cannot overwrite a value") from None + if key_stem in nest: + raise suffixed_err(src, pos, f"Duplicate inline table key {key_stem!r}") + nest[key_stem] = value + pos = skip_chars(src, pos, TOML_WS) + c = src[pos : pos + 1] + if c == "}": + return pos + 1, nested_dict.dict + if c != ",": + raise suffixed_err(src, pos, "Unclosed inline table") + if isinstance(value, (dict, list)): + flags.set(key, Flags.FROZEN, recursive=True) + pos += 1 + pos = skip_chars(src, pos, TOML_WS) + + +def parse_basic_str_escape( + src: str, pos: Pos, *, multiline: bool = False +) -> tuple[Pos, str]: + escape_id = src[pos : pos + 2] + pos += 2 + if multiline and escape_id in {"\\ ", "\\\t", "\\\n"}: + # Skip whitespace until next non-whitespace character or end of + # the doc. Error if non-whitespace is found before newline. + if escape_id != "\\\n": + pos = skip_chars(src, pos, TOML_WS) + try: + char = src[pos] + except IndexError: + return pos, "" + if char != "\n": + raise suffixed_err(src, pos, "Unescaped '\\' in a string") + pos += 1 + pos = skip_chars(src, pos, TOML_WS_AND_NEWLINE) + return pos, "" + if escape_id == "\\u": + return parse_hex_char(src, pos, 4) + if escape_id == "\\U": + return parse_hex_char(src, pos, 8) + try: + return pos, BASIC_STR_ESCAPE_REPLACEMENTS[escape_id] + except KeyError: + raise suffixed_err(src, pos, "Unescaped '\\' in a string") from None + + +def parse_basic_str_escape_multiline(src: str, pos: Pos) -> tuple[Pos, str]: + return parse_basic_str_escape(src, pos, multiline=True) + + +def parse_hex_char(src: str, pos: Pos, hex_len: int) -> tuple[Pos, str]: + hex_str = src[pos : pos + hex_len] + if len(hex_str) != hex_len or not HEXDIGIT_CHARS.issuperset(hex_str): + raise suffixed_err(src, pos, "Invalid hex value") + pos += hex_len + hex_int = int(hex_str, 16) + if not is_unicode_scalar_value(hex_int): + raise suffixed_err(src, pos, "Escaped character is not a Unicode scalar value") + return pos, chr(hex_int) + + +def parse_literal_str(src: str, pos: Pos) -> tuple[Pos, str]: + pos += 1 # Skip starting apostrophe + start_pos = pos + pos = skip_until( + src, pos, "'", error_on=ILLEGAL_LITERAL_STR_CHARS, error_on_eof=True + ) + return pos + 1, src[start_pos:pos] # Skip ending apostrophe + + +def parse_multiline_str(src: str, pos: Pos, *, literal: bool) -> tuple[Pos, str]: + pos += 3 + if src.startswith("\n", pos): + pos += 1 + + if literal: + delim = "'" + end_pos = skip_until( + src, + pos, + "'''", + error_on=ILLEGAL_MULTILINE_LITERAL_STR_CHARS, + error_on_eof=True, + ) + result = src[pos:end_pos] + pos = end_pos + 3 + else: + delim = '"' + pos, result = parse_basic_str(src, pos, multiline=True) + + # Add at maximum two extra apostrophes/quotes if the end sequence + # is 4 or 5 chars long instead of just 3. + if not src.startswith(delim, pos): + return pos, result + pos += 1 + if not src.startswith(delim, pos): + return pos, result + delim + pos += 1 + return pos, result + (delim * 2) + + +def parse_basic_str(src: str, pos: Pos, *, multiline: bool) -> tuple[Pos, str]: + if multiline: + error_on = ILLEGAL_MULTILINE_BASIC_STR_CHARS + parse_escapes = parse_basic_str_escape_multiline + else: + error_on = ILLEGAL_BASIC_STR_CHARS + parse_escapes = parse_basic_str_escape + result = "" + start_pos = pos + while True: + try: + char = src[pos] + except IndexError: + raise suffixed_err(src, pos, "Unterminated string") from None + if char == '"': + if not multiline: + return pos + 1, result + src[start_pos:pos] + if src.startswith('"""', pos): + return pos + 3, result + src[start_pos:pos] + pos += 1 + continue + if char == "\\": + result += src[start_pos:pos] + pos, parsed_escape = parse_escapes(src, pos) + result += parsed_escape + start_pos = pos + continue + if char in error_on: + raise suffixed_err(src, pos, f"Illegal character {char!r}") + pos += 1 + + +def parse_value( # noqa: C901 + src: str, pos: Pos, parse_float: ParseFloat +) -> tuple[Pos, Any]: + try: + char: str | None = src[pos] + except IndexError: + char = None + + # IMPORTANT: order conditions based on speed of checking and likelihood + + # Basic strings + if char == '"': + if src.startswith('"""', pos): + return parse_multiline_str(src, pos, literal=False) + return parse_one_line_basic_str(src, pos) + + # Literal strings + if char == "'": + if src.startswith("'''", pos): + return parse_multiline_str(src, pos, literal=True) + return parse_literal_str(src, pos) + + # Booleans + if char == "t": + if src.startswith("true", pos): + return pos + 4, True + if char == "f": + if src.startswith("false", pos): + return pos + 5, False + + # Arrays + if char == "[": + return parse_array(src, pos, parse_float) + + # Inline tables + if char == "{": + return parse_inline_table(src, pos, parse_float) + + # Dates and times + datetime_match = RE_DATETIME.match(src, pos) + if datetime_match: + try: + datetime_obj = match_to_datetime(datetime_match) + except ValueError as e: + raise suffixed_err(src, pos, "Invalid date or datetime") from e + return datetime_match.end(), datetime_obj + localtime_match = RE_LOCALTIME.match(src, pos) + if localtime_match: + return localtime_match.end(), match_to_localtime(localtime_match) + + # Integers and "normal" floats. + # The regex will greedily match any type starting with a decimal + # char, so needs to be located after handling of dates and times. + number_match = RE_NUMBER.match(src, pos) + if number_match: + return number_match.end(), match_to_number(number_match, parse_float) + + # Special floats + first_three = src[pos : pos + 3] + if first_three in {"inf", "nan"}: + return pos + 3, parse_float(first_three) + first_four = src[pos : pos + 4] + if first_four in {"-inf", "+inf", "-nan", "+nan"}: + return pos + 4, parse_float(first_four) + + raise suffixed_err(src, pos, "Invalid value") + + +def suffixed_err(src: str, pos: Pos, msg: str) -> TOMLDecodeError: + """Return a `TOMLDecodeError` where error message is suffixed with + coordinates in source.""" + + def coord_repr(src: str, pos: Pos) -> str: + if pos >= len(src): + return "end of document" + line = src.count("\n", 0, pos) + 1 + if line == 1: + column = pos + 1 + else: + column = pos - src.rindex("\n", 0, pos) + return f"line {line}, column {column}" + + return TOMLDecodeError(f"{msg} (at {coord_repr(src, pos)})") + + +def is_unicode_scalar_value(codepoint: int) -> bool: + return (0 <= codepoint <= 55295) or (57344 <= codepoint <= 1114111) + + +def make_safe_parse_float(parse_float: ParseFloat) -> ParseFloat: + """A decorator to make `parse_float` safe. + + `parse_float` must not return dicts or lists, because these types + would be mixed with parsed TOML tables and arrays, thus confusing + the parser. The returned decorated callable raises `ValueError` + instead of returning illegal types. + """ + # The default `float` callable never returns illegal types. Optimize it. + if parse_float is float: # type: ignore[comparison-overlap] + return float + + def safe_parse_float(float_str: str) -> Any: + float_value = parse_float(float_str) + if isinstance(float_value, (dict, list)): + raise ValueError("parse_float must not return dicts or lists") + return float_value + + return safe_parse_float diff --git a/pex/vendor/_vendored/tomli/tomli/_re.py b/pex/vendor/_vendored/tomli/tomli/_re.py new file mode 100644 index 000000000..994bb7493 --- /dev/null +++ b/pex/vendor/_vendored/tomli/tomli/_re.py @@ -0,0 +1,107 @@ +# SPDX-License-Identifier: MIT +# SPDX-FileCopyrightText: 2021 Taneli Hukkinen +# Licensed to PSF under a Contributor Agreement. + +from __future__ import annotations + +from datetime import date, datetime, time, timedelta, timezone, tzinfo +from functools import lru_cache +import re +from typing import Any + +from ._types import ParseFloat + +# E.g. +# - 00:32:00.999999 +# - 00:32:00 +_TIME_RE_STR = r"([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9])(?:\.([0-9]{1,6})[0-9]*)?" + +RE_NUMBER = re.compile( + r""" +0 +(?: + x[0-9A-Fa-f](?:_?[0-9A-Fa-f])* # hex + | + b[01](?:_?[01])* # bin + | + o[0-7](?:_?[0-7])* # oct +) +| +[+-]?(?:0|[1-9](?:_?[0-9])*) # dec, integer part +(?P + (?:\.[0-9](?:_?[0-9])*)? # optional fractional part + (?:[eE][+-]?[0-9](?:_?[0-9])*)? # optional exponent part +) +""", + flags=re.VERBOSE, +) +RE_LOCALTIME = re.compile(_TIME_RE_STR) +RE_DATETIME = re.compile( + rf""" +([0-9]{{4}})-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01]) # date, e.g. 1988-10-27 +(?: + [Tt ] + {_TIME_RE_STR} + (?:([Zz])|([+-])([01][0-9]|2[0-3]):([0-5][0-9]))? # optional time offset +)? +""", + flags=re.VERBOSE, +) + + +def match_to_datetime(match: re.Match) -> datetime | date: + """Convert a `RE_DATETIME` match to `datetime.datetime` or `datetime.date`. + + Raises ValueError if the match does not correspond to a valid date + or datetime. + """ + ( + year_str, + month_str, + day_str, + hour_str, + minute_str, + sec_str, + micros_str, + zulu_time, + offset_sign_str, + offset_hour_str, + offset_minute_str, + ) = match.groups() + year, month, day = int(year_str), int(month_str), int(day_str) + if hour_str is None: + return date(year, month, day) + hour, minute, sec = int(hour_str), int(minute_str), int(sec_str) + micros = int(micros_str.ljust(6, "0")) if micros_str else 0 + if offset_sign_str: + tz: tzinfo | None = cached_tz( + offset_hour_str, offset_minute_str, offset_sign_str + ) + elif zulu_time: + tz = timezone.utc + else: # local date-time + tz = None + return datetime(year, month, day, hour, minute, sec, micros, tzinfo=tz) + + +@lru_cache(maxsize=None) +def cached_tz(hour_str: str, minute_str: str, sign_str: str) -> timezone: + sign = 1 if sign_str == "+" else -1 + return timezone( + timedelta( + hours=sign * int(hour_str), + minutes=sign * int(minute_str), + ) + ) + + +def match_to_localtime(match: re.Match) -> time: + hour_str, minute_str, sec_str, micros_str = match.groups() + micros = int(micros_str.ljust(6, "0")) if micros_str else 0 + return time(int(hour_str), int(minute_str), int(sec_str), micros) + + +def match_to_number(match: re.Match, parse_float: ParseFloat) -> Any: + if match.group("floatpart"): + return parse_float(match.group()) + return int(match.group(), 0) diff --git a/pex/vendor/_vendored/tomli/tomli/_types.py b/pex/vendor/_vendored/tomli/tomli/_types.py new file mode 100644 index 000000000..d949412e0 --- /dev/null +++ b/pex/vendor/_vendored/tomli/tomli/_types.py @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: MIT +# SPDX-FileCopyrightText: 2021 Taneli Hukkinen +# Licensed to PSF under a Contributor Agreement. + +from typing import Any, Callable, Tuple + +# Type annotations +ParseFloat = Callable[[str], Any] +Key = Tuple[str, ...] +Pos = int diff --git a/pex/vendor/_vendored/tomli/tomli/py.typed b/pex/vendor/_vendored/tomli/tomli/py.typed new file mode 100644 index 000000000..7632ecf77 --- /dev/null +++ b/pex/vendor/_vendored/tomli/tomli/py.typed @@ -0,0 +1 @@ +# Marker file for PEP 561 diff --git a/pex/version.py b/pex/version.py index 1d2fc6ac6..d82a4583f 100644 --- a/pex/version.py +++ b/pex/version.py @@ -1,4 +1,4 @@ # Copyright 2015 Pex project contributors. # Licensed under the Apache License, Version 2.0 (see LICENSE). -__version__ = "2.22.0" +__version__ = "2.23.0" diff --git a/tests/bin/test_sh_boot.py b/tests/bin/test_sh_boot.py index 5c080e01f..c4e46744a 100644 --- a/tests/bin/test_sh_boot.py +++ b/tests/bin/test_sh_boot.py @@ -5,7 +5,7 @@ import pytest -from pex import sh_boot +from pex import sh_boot, toml from pex.interpreter import PythonInterpreter from pex.interpreter_constraints import InterpreterConstraints, iter_compatible_versions from pex.orderedset import OrderedSet @@ -19,10 +19,6 @@ if TYPE_CHECKING: from typing import Iterable, List - import toml # vendor:skip -else: - from pex.third_party import toml - def calculate_binary_names( targets=Targets(), # type: Targets diff --git a/tests/integration/cli/commands/test_lock_dependency_groups.py b/tests/integration/cli/commands/test_lock_dependency_groups.py new file mode 100644 index 000000000..3c414d883 --- /dev/null +++ b/tests/integration/cli/commands/test_lock_dependency_groups.py @@ -0,0 +1,59 @@ +# Copyright 2024 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +import os +from textwrap import dedent + +from pex.common import safe_open +from pex.dist_metadata import Requirement +from pex.pep_440 import Version +from pex.pep_503 import ProjectName +from pex.resolve.lockfile import json_codec +from pex.sorted_tuple import SortedTuple +from testing.cli import run_pex3 +from testing.pytest.tmp import Tempdir + +req = Requirement.parse + + +def test_lock_dependency_groups(tmpdir): + # type: (Tempdir) -> None + + project_dir = tmpdir.join("project") + with safe_open(os.path.join(project_dir, "pyproject.toml"), "w") as fp: + fp.write( + dedent( + """\ + [dependency-groups] + speak = ["cowsay==5.0"] + """ + ) + ) + + lock = tmpdir.join("lock.json") + run_pex3( + "lock", + "create", + "--group", + "speak@{project}".format(project=project_dir), + "ansicolors==1.1.8", + "-o", + lock, + "--indent", + "2", + ).assert_success() + + lockfile = json_codec.load(lock) + assert ( + SortedTuple((req("cowsay==5.0"), req("ansicolors==1.1.8")), key=str) + == lockfile.requirements + ) + assert 1 == len(lockfile.locked_resolves) + locked_requirements = lockfile.locked_resolves[0].locked_requirements + assert sorted( + ((ProjectName("cowsay"), Version("5.0")), (ProjectName("ansicolors"), Version("1.1.8"))) + ) == sorted( + (locked_req.pin.project_name, locked_req.pin.version) for locked_req in locked_requirements + ) diff --git a/tests/integration/resolve/test_dependency_groups.py b/tests/integration/resolve/test_dependency_groups.py new file mode 100644 index 000000000..1503c0653 --- /dev/null +++ b/tests/integration/resolve/test_dependency_groups.py @@ -0,0 +1,62 @@ +# Copyright 2024 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +import os.path +import subprocess +from textwrap import dedent + +import colors # vendor:skip + +from pex.common import safe_open +from pex.pep_440 import Version +from pex.pep_503 import ProjectName +from pex.pex import PEX +from testing import run_pex_command +from testing.pytest.tmp import Tempdir + + +def test_pex_from_dependency_groups(tmpdir): + # type: (Tempdir) -> None + + project_dir = tmpdir.join("project") + with safe_open(os.path.join(project_dir, "pyproject.toml"), "w") as fp: + fp.write( + dedent( + """\ + [project] + dependencies = [ + "does-not-exist", + "requests", + ] + + [dependency-groups] + colors = ["ansicolors==1.1.8"] + speak = ["cowsay==5.0"] + """ + ) + ) + + pex = tmpdir.join("pex") + run_pex_command( + args=[ + "--group", + "colors@.", + "--group", + "speak@{project}".format(project=project_dir), + "-c", + "cowsay", + "-o", + pex, + ], + cwd=project_dir, + ).assert_success() + + assert sorted( + ((ProjectName("cowsay"), Version("5.0")), (ProjectName("ansicolors"), Version("1.1.8"))) + ) == [(dist.metadata.project_name, dist.metadata.version) for dist in PEX(pex).resolve()] + + assert "| {moo} |".format(moo=colors.yellow("Moo!")) in subprocess.check_output( + args=[pex, colors.yellow("Moo!")] + ).decode("utf-8") diff --git a/tests/integration/test_issue_1560.py b/tests/integration/test_issue_1560.py index 8cb30e8df..cd6599fc8 100644 --- a/tests/integration/test_issue_1560.py +++ b/tests/integration/test_issue_1560.py @@ -6,6 +6,7 @@ import pytest +from pex import toml from pex.typing import TYPE_CHECKING from testing import IntegResults, VenvFactory, all_python_venvs, make_source_dir, run_pex_command from testing.pythonPI import skip_flit_core_39 @@ -13,10 +14,6 @@ if TYPE_CHECKING: from typing import Any - import toml # vendor:skip -else: - from pex.third_party import toml - @pytest.mark.parametrize( "venv_factory", diff --git a/tests/resolve/test_dependency_groups.py b/tests/resolve/test_dependency_groups.py new file mode 100644 index 000000000..119b523f6 --- /dev/null +++ b/tests/resolve/test_dependency_groups.py @@ -0,0 +1,218 @@ +# Copyright 2024 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +import os.path +import re +import sys +from argparse import ArgumentParser, Namespace +from textwrap import dedent +from typing import List, Optional, Sequence + +import pytest + +from pex.common import safe_open +from pex.dist_metadata import Requirement +from pex.resolve import project +from pex.typing import cast +from testing import pushd +from testing.pytest.tmp import Tempdir + + +def create_dependency_groups_project( + pyproject_toml_path, # type: str + contents, # type: str +): + # type: (...) -> str + with safe_open(pyproject_toml_path, "w") as fp: + fp.write(contents) + return cast(str, os.path.dirname(fp.name)) + + +@pytest.fixture +def project_dir1(tmpdir): + # type: (Tempdir) -> str + if sys.version_info[:2] < (3, 7): + pytest.skip("The toml library we use for old pythons cannot parse heterogeneous lists.") + + return create_dependency_groups_project( + tmpdir.join("project1", "pyproject.toml"), + dedent( + """\ + [dependency-groups] + basic = ["foo", "bar>2"] + include1 = [{include-group = "basic"}] + include2 = ["spam", {include-group = "include1"}, "bar", "foo"] + + # Per the spec, parsing should be lazy; so we should never "see" the bogus + # `set-phasers-to` inline table element. + bar = [{set-phasers-to = "stun"}] + + bad-req = ["meaning-of-life=42"] + missing-include = [{include-group = "does-not-exist"}] + """ + ), + ) + + +@pytest.fixture +def project_dir2(tmpdir): + # type: (Tempdir) -> str + return create_dependency_groups_project( + tmpdir.join("project2", "pyproject.toml"), + dedent( + """\ + [dependency-groups] + basic = [ + "baz<3; python_version < '3.9'", + "baz; python_version >= '3.9'", + ] + """ + ), + ) + + +def parse_args( + args, # type: Sequence[str] + cwd=None, # type: Optional[str] +): + # type: (...) -> Namespace + with pushd(cwd or os.getcwd()): + parser = ArgumentParser() + project.register_options(parser, project_help="test") + return parser.parse_args(args=args) + + +def parse_groups( + args, # type: Sequence[str] + cwd=None, # type: Optional[str] +): + # type: (...) -> List[Requirement] + return list(project.get_group_requirements(parse_args(args, cwd=cwd))) + + +req = Requirement.parse + + +def test_nominal(project_dir1): + # type: (str) -> None + expected_reqs = [req("foo"), req("bar>2")] + assert expected_reqs == parse_groups( + ["--group", "basic@{project_dir}".format(project_dir=project_dir1)] + ) + assert expected_reqs == parse_groups(["--group", "basic"], cwd=project_dir1) + + +def test_include(project_dir1): + # type: (str) -> None + expected_reqs = [req("foo"), req("bar>2")] + assert expected_reqs == parse_groups( + ["--group", "include1@{project_dir}".format(project_dir=project_dir1)] + ) + assert expected_reqs == parse_groups(["--group", "include1"], cwd=project_dir1) + + +def test_include_multi(project_dir1): + # type: (str) -> None + expected_reqs = [req("spam"), req("foo"), req("bar>2"), req("bar")] + assert expected_reqs == parse_groups( + ["--group", "include2@{project_dir}".format(project_dir=project_dir1)] + ) + assert expected_reqs == parse_groups(["--group", "include2@."], cwd=project_dir1) + + +def test_multiple_projects( + project_dir1, # type: str + project_dir2, # type: str +): + # type: (...) -> None + expected_reqs = [ + req("foo"), + req("bar>2"), + req("baz<3; python_version < '3.9'"), + req("baz; python_version >= '3.9'"), + ] + assert expected_reqs == parse_groups( + [ + "--group", + "include1@{project_dir}".format(project_dir=project_dir1), + "--group", + "basic@{project_dir}".format(project_dir=project_dir2), + "--group", + "basic@{project_dir}".format(project_dir=project_dir1), + ] + ) + assert expected_reqs == parse_groups( + [ + "--group", + "include1", + "--group", + "basic@{project_dir}".format(project_dir=project_dir2), + "--group", + "basic@", + ], + cwd=project_dir1, + ) + + +def test_missing_group(project_dir1): + # type: (str) -> None + + with pytest.raises( + KeyError, + match=re.escape( + "The dependency group 'does-not-exist' specified by 'does-not-exist@{project}' does " + "not exist in {project}".format(project=project_dir1) + ), + ): + parse_args(["--group", "does-not-exist@{project}".format(project=project_dir1)]) + + +def test_invalid_group_bad_req(project_dir1): + # type: (str) -> None + + options = parse_args(["--group", "bad-req"], cwd=project_dir1) + with pytest.raises( + ValueError, + match=re.escape( + "Invalid [dependency-group] entry 'bad-req'.\n" + "Item 1: 'meaning-of-life=42', is an invalid dependency specifier: Expected end or " + "semicolon (after name and no valid version specifier)\n" + " meaning-of-life=42\n" + " ^" + ), + ): + project.get_group_requirements(options) + + +def test_invalid_group_bad_inline_table(project_dir1): + # type: (str) -> None + + options = parse_args(["--group", "bar"], cwd=project_dir1) + with pytest.raises( + ValueError, + match=re.escape( + "Invalid [dependency-group] entry 'bar'.\n" + "Item 1 is a non 'include-group' table and only dependency specifiers and single entry " + "'include-group' tables are allowed in group dependency lists.\n" + "See https://peps.python.org/pep-0735/#specification for the specification of " + "[dependency-groups] syntax.\n" + "Given: {'set-phasers-to': 'stun'}" + ), + ): + project.get_group_requirements(options) + + +def test_invalid_group_missing_include(project_dir1): + # type: (str) -> None + + options = parse_args(["--group", "missing-include"], cwd=project_dir1) + with pytest.raises( + KeyError, + match=re.escape( + "The dependency group 'does-not-exist' required by dependency group 'missing-include' " + "does not exist in the project at {project}.".format(project=project_dir1) + ), + ): + project.get_group_requirements(options) diff --git a/tests/test_pep_723.py b/tests/test_pep_723.py index c8216085f..c381b7922 100644 --- a/tests/test_pep_723.py +++ b/tests/test_pep_723.py @@ -4,6 +4,7 @@ from __future__ import absolute_import import re +import sys from textwrap import dedent import pytest @@ -238,9 +239,15 @@ def test_parse_invalid_toml(): InvalidMetadataError, match=re.escape( "The script metadata found in exe.py starting at line 3 embeds malformed toml: " - "Empty value is invalid (line 1 column 1 char 0).\n" + "{toml_error}.\n" "See: https://packaging.python.org/specifications/" - "inline-script-metadata#inline-script-metadata" + "inline-script-metadata#inline-script-metadata".format( + toml_error=( + "Invalid value (at line 1, column 21)" # N.B.: tomli + if sys.version_info[:2] >= (3, 7) + else "Empty value is invalid (line 1 column 1 char 0)" # N.B.: toml + ) + ) ), ): ScriptMetadata.parse(