From c9e29873fa9c2bba9916c6215d2207e6cf84f960 Mon Sep 17 00:00:00 2001 From: q0w <43147888+q0w@users.noreply.github.com> Date: Sun, 11 Dec 2022 17:05:55 +0300 Subject: [PATCH 1/7] Filter out the user provided unsafe packages --- piptools/scripts/compile.py | 1 + piptools/writer.py | 20 ++++++++------ tests/test_cli_compile.py | 53 +++++++++++++++++++++++++++++++++++++ 3 files changed, 66 insertions(+), 8 deletions(-) diff --git a/piptools/scripts/compile.py b/piptools/scripts/compile.py index 8438e7f55..770a61ce9 100755 --- a/piptools/scripts/compile.py +++ b/piptools/scripts/compile.py @@ -599,6 +599,7 @@ def cli( ) writer.write( results=results, + unsafe_packages=resolver.unsafe_packages, unsafe_requirements=resolver.unsafe_constraints, markers={ key_from_ireq(ireq): ireq.markers for ireq in constraints if ireq.markers diff --git a/piptools/writer.py b/piptools/writer.py index b9f75a2f0..280f1db35 100644 --- a/piptools/writer.py +++ b/piptools/writer.py @@ -16,7 +16,6 @@ from .logging import log from .utils import ( - UNSAFE_PACKAGES, comment, dedup, format_requirement, @@ -177,11 +176,15 @@ def _iter_lines( self, results: set[InstallRequirement], unsafe_requirements: set[InstallRequirement] | None = None, + unsafe_packages: set[str] | None = None, markers: dict[str, Marker] | None = None, hashes: dict[InstallRequirement, set[str]] | None = None, ) -> Iterator[str]: # default values unsafe_requirements = unsafe_requirements or set() + if unsafe_packages is None: + unsafe_packages = set() + unsafe_packages = unsafe_packages if not self.allow_unsafe else set() markers = markers or {} hashes = hashes or {} @@ -199,12 +202,10 @@ def _iter_lines( yield line yielded = True - unsafe_requirements = ( - {r for r in results if r.name in UNSAFE_PACKAGES} - if not unsafe_requirements - else unsafe_requirements - ) - packages = {r for r in results if r.name not in UNSAFE_PACKAGES} + unsafe_requirements = unsafe_requirements or { + r for r in results if r.name in unsafe_packages + } + packages = {r for r in results if r.name not in unsafe_packages} if packages: for ireq in sorted(packages, key=self._sort_key): @@ -247,6 +248,7 @@ def write( self, results: set[InstallRequirement], unsafe_requirements: set[InstallRequirement], + unsafe_packages: set[str], markers: dict[str, Marker], hashes: dict[InstallRequirement, set[str]] | None, ) -> None: @@ -259,7 +261,9 @@ def write( line_buffering=True, ) try: - for line in self._iter_lines(results, unsafe_requirements, markers, hashes): + for line in self._iter_lines( + results, unsafe_requirements, unsafe_packages, markers, hashes + ): if self.dry_run: # Bypass the log level to always print this during a dry run log.log(line) diff --git a/tests/test_cli_compile.py b/tests/test_cli_compile.py index 69185354b..37eccfb60 100644 --- a/tests/test_cli_compile.py +++ b/tests/test_cli_compile.py @@ -1536,6 +1536,59 @@ def test_allow_unsafe_option(pip_conf, monkeypatch, runner, option, expected): assert out.stdout == expected +@pytest.mark.parametrize( + ("unsafe_package", "expected"), + ( + ( + "small-fake-with-deps", + dedent( + """\ + small-fake-a==0.1 + small-fake-b==0.3 + + # The following packages are considered to be unsafe in a requirements file: + # small-fake-with-deps + """ + ), + ), + ( + "small-fake-a", + dedent( + """\ + small-fake-b==0.3 + small-fake-with-deps==0.1 + + # The following packages are considered to be unsafe in a requirements file: + # small-fake-a + """ + ), + ), + ), +) +def test_unsafe_package_option(pip_conf, monkeypatch, runner, unsafe_package, expected): + monkeypatch.setattr("piptools.resolver.UNSAFE_PACKAGES", {"small-fake-with-deps"}) + with open("requirements.in", "w") as req_in: + req_in.write("small-fake-b\n") + req_in.write("small-fake-with-deps") + + out = runner.invoke( + cli, + [ + "--output-file", + "-", + "--quiet", + "--no-header", + "--no-emit-options", + "--no-annotate", + "--unsafe-package", + unsafe_package, + ], + ) + + assert out.exit_code == 0, out + assert out.stdout == expected + + @pytest.mark.parametrize( ("option", "attr", "expected"), (("--cert", "cert", "foo.crt"), ("--client-cert", "client_cert", "bar.pem")), From 6601154e9f4213b47b71ffdd9ab4a8b9af71b917 Mon Sep 17 00:00:00 2001 From: Sorin Sbarnea Date: Sun, 11 Dec 2022 11:53:07 +0000 Subject: [PATCH 2/7] Restore coloring for console and CI (#1765) As tox v4 included some side-effects, that will restore the lost colors. --- .github/workflows/ci.yml | 2 ++ tox.ini | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f14b6731f..daeeff819 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,6 +20,7 @@ env: PIP_DISABLE_PIP_VERSION_CHECK: 1 PIP_NO_PYTHON_VERSION_WARNING: 1 PIP_NO_WARN_SCRIPT_LOCATION: 1 + PRE_COMMIT_COLOR: 1 PY_COLORS: 1 # Recognized by the `py` package, dependency of `pytest` TOX_PARALLEL_NO_SPINNER: 1 TOX_TESTENV_PASSENV: >- @@ -29,6 +30,7 @@ env: PY_COLORS PYTEST_THEME PYTEST_THEME_MODE + PRE_COMMIT_COLOR jobs: test: diff --git a/tox.ini b/tox.ini index 98f3f285c..459a251fa 100644 --- a/tox.ini +++ b/tox.ini @@ -23,7 +23,11 @@ commands_pre = commands = pytest {posargs} passenv = CI + FORCE_COLOR GITHUB_ACTIONS + MYPY_FORCE_COLOR + PRE_COMMIT_COLOR + PY_COLORS pip_pre=True [testenv:checkqa] From e6ef58013bdb8862bee804afb40f7b8540a5db14 Mon Sep 17 00:00:00 2001 From: Albert Tugushev Date: Sun, 11 Dec 2022 13:23:44 +0100 Subject: [PATCH 3/7] Add `--no-index` flag to `pip-compile` (#1745) Co-authored-by: Sorin Sbarnea --- piptools/scripts/compile.py | 19 +++++++++++++++---- tests/test_cli_compile.py | 10 ++++++++++ 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/piptools/scripts/compile.py b/piptools/scripts/compile.py index 770a61ce9..08925daf3 100755 --- a/piptools/scripts/compile.py +++ b/piptools/scripts/compile.py @@ -126,6 +126,11 @@ def _determine_linesep( index_url=redact_auth_from_url(_get_default_option("index_url")) ), ) +@click.option( + "--no-index", + is_flag=True, + help="Ignore package index (only looking at --find-links URLs instead).", +) @click.option( "--extra-index-url", multiple=True, @@ -302,6 +307,7 @@ def cli( all_extras: bool, find_links: tuple[str, ...], index_url: str, + no_index: bool, extra_index_url: tuple[str, ...], cert: str | None, client_cert: str | None, @@ -392,6 +398,8 @@ def cli( pip_args.extend(["-f", link]) if index_url: pip_args.extend(["-i", index_url]) + if no_index: + pip_args.extend(["--no-index"]) for extra_index in extra_index_url: pip_args.extend(["--extra-index-url", extra_index]) if cert: @@ -526,10 +534,13 @@ def cli( for req in constraints: drop_extras(req) - log.debug("Using indexes:") - with log.indentation(): - for index_url in dedup(repository.finder.index_urls): - log.debug(redact_auth_from_url(index_url)) + if repository.finder.index_urls: + log.debug("Using indexes:") + with log.indentation(): + for index_url in dedup(repository.finder.index_urls): + log.debug(redact_auth_from_url(index_url)) + else: + log.debug("Ignoring indexes.") if repository.finder.find_links: log.debug("") diff --git a/tests/test_cli_compile.py b/tests/test_cli_compile.py index 37eccfb60..c5198c4d7 100644 --- a/tests/test_cli_compile.py +++ b/tests/test_cli_compile.py @@ -220,6 +220,16 @@ def test_setuptools_preserves_environment_markers( assert out.stdout == 'foo==1.0 ; python_version >= "1"\n' +def test_no_index_option(runner, tmp_path): + req_in = tmp_path / "requirements.in" + req_in.touch() + + out = runner.invoke(cli, [req_in.as_posix(), "--no-index", "--verbose"]) + + assert out.exit_code == 0 + assert "Ignoring indexes." in out.stderr + + def test_find_links_option(runner): with open("requirements.in", "w") as req_in: req_in.write("-f ./libs3") From 790bcbf3d066704cb0736c98640eac2084ea5fd0 Mon Sep 17 00:00:00 2001 From: Sorin Sbarnea Date: Sun, 11 Dec 2022 16:16:44 +0000 Subject: [PATCH 4/7] Adopt PEP-621 for packaging (#1763) --- .flake8 | 15 +++++++ pyproject.toml | 116 ++++++++++++++++++++++++++++++++++++++++++++++++- setup.cfg | 115 ------------------------------------------------ setup.py | 5 --- tox.ini | 2 +- 5 files changed, 131 insertions(+), 122 deletions(-) create mode 100644 .flake8 delete mode 100644 setup.cfg delete mode 100644 setup.py diff --git a/.flake8 b/.flake8 new file mode 100644 index 000000000..a6ed61924 --- /dev/null +++ b/.flake8 @@ -0,0 +1,15 @@ +[flake8] +max-line-length = 100 +# E203 conflicts with PEP8; see https://github.com/psf/black#slices +extend-ignore = E203 + +# flake8-pytest-style +# PT001: +pytest-fixture-no-parentheses = true +# PT006: +pytest-parametrize-names-type = tuple +# PT007: +pytest-parametrize-values-type = tuple +pytest-parametrize-values-row-type = tuple +# PT023: +pytest-mark-no-parentheses = true diff --git a/pyproject.toml b/pyproject.toml index a81638437..eeceabfee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,119 @@ [build-system] -requires = ["setuptools>=45", "setuptools_scm[toml]>=6.2"] +requires = ["setuptools>=63", "setuptools_scm[toml]>=7"] build-backend = "setuptools.build_meta" +[project] +# https://peps.python.org/pep-0621/#readme +requires-python = ">=3.7" +dynamic = ["version"] +name = "pip-tools" +description = "pip-tools keeps your pinned dependencies fresh." +readme = "README.rst" +authors = [{ "name" = "Vincent Driessen", "email" = "me@nvie.com" }] +license = { text = "BSD" } +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "License :: OSI Approved :: BSD License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Programming Language :: Python", + "Topic :: Software Development :: Quality Assurance", + "Topic :: Software Development :: Testing", + "Topic :: System :: Systems Administration", + "Topic :: Utilities", + "Typing :: Typed", +] +keywords = ["pip", "requirements", "packaging"] +dependencies = [ + # direct dependencies + "build", + "click >= 8", + "pip >= 22.2", + # indirect dependencies + "setuptools", # typically needed when pip-tools invokes setup.py + "wheel", # pip plugin needed by pip-tools +] + +[project.urls] +homepage = "https://github.com/jazzband/pip-tools/" +documentation = "https://pip-tools.readthedocs.io/en/latest/" +repository = "https://github.com/jazzband/pip-tools" +changelog = "https://github.com/jazzband/pip-tools/releases" + +[project.optional-dependencies] +testing = [ + "pytest >= 7.2.0", + "pytest-rerunfailures", + "pytest-xdist", + # build deps for tests + "flit_core >=2,<4", + "poetry_core>=1.0.0", +] +coverage = [ + "pytest-cov", +] + +[project.scripts] +pip-compile = "piptools.scripts.compile:cli" +pip-sync = "piptools.scripts.sync:cli" + +[tool.isort] +profile = "black" +add_imports = "from __future__ import annotations" + +[tool.mypy] +disallow_untyped_defs = true +disallow_any_generics = true +disallow_incomplete_defs = true +disallow_subclassing_any = true +disallow_untyped_calls = true +disallow_untyped_decorators = true +ignore_missing_imports = true +no_implicit_optional = true +no_implicit_reexport = true +strict_equality = true +warn_redundant_casts = true +warn_return_any = true +warn_unused_configs = true +warn_unused_ignores = true + +# Avoid error: Duplicate module named 'setup' +# https://github.com/python/mypy/issues/4008 +exclude = "^tests/test_data/" + +[tool.mypy.overrides] +modules = [ + "tests.*" +] +disallow_untyped_defs = false + +[tool.pytest.ini_options] +addopts = [ + # `pytest-xdist`: + "--numprocesses=auto" +] +norecursedirs = ".* build dist venv test_data piptools/_compat/*" +testpaths = "tests piptools" +filterwarnings = [ + "always" +] +markers = [ + "network: mark tests that require internet access" +] + +[tool.setuptools.packages.find] +# needed only because we did not adopt src layout yet +exclude = ["img"] + [tool.setuptools_scm] diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 6cde1d037..000000000 --- a/setup.cfg +++ /dev/null @@ -1,115 +0,0 @@ -[metadata] -name = pip-tools -url = https://github.com/jazzband/pip-tools/ -license = BSD 3 Clause -author = Vincent Driessen -author_email = me@nvie.com -description = pip-tools keeps your pinned dependencies fresh. -long_description = file: README.rst -classifiers = - Development Status :: 5 - Production/Stable - Intended Audience :: Developers - Intended Audience :: System Administrators - License :: OSI Approved :: BSD License - Operating System :: OS Independent - Programming Language :: Python - Programming Language :: Python :: 3 - Programming Language :: Python :: 3 :: Only - Programming Language :: Python :: 3.7 - Programming Language :: Python :: 3.8 - Programming Language :: Python :: 3.9 - Programming Language :: Python :: 3.10 - Programming Language :: Python :: 3.11 - Programming Language :: Python :: Implementation :: CPython - Programming Language :: Python :: Implementation :: PyPy - Topic :: System :: Systems Administration - Typing :: Typed - -[options] -python_requires = >=3.7 -setup_requires = setuptools_scm -packages = find: -zip_safe = false -install_requires = - # direct dependencies - build - click >= 8 - pip >= 22.2 - # indirect dependencies - setuptools # typically needed when pip-tools invokes setup.py - wheel # pip plugin needed by pip-tools - -[options.packages.find] -exclude = tests - -[options.package_data] -piptools = py.typed - -[options.extras_require] -testing = - pytest - pytest-rerunfailures - pytest-xdist - # build deps for tests - flit_core >=2,<4 - poetry_core>=1.0.0 -coverage = pytest-cov - -[options.entry_points] -console_scripts = - pip-compile = piptools.scripts.compile:cli - pip-sync = piptools.scripts.sync:cli - -[tool:pytest] -addopts = - # `pytest-xdist`: - --numprocesses=auto -norecursedirs = .* build dist venv test_data piptools/_compat/* -testpaths = tests piptools -filterwarnings = - always -markers = - network: mark tests that require internet access - -[flake8] -max-line-length = 100 -# E203 conflicts with PEP8; see https://github.com/psf/black#slices -extend-ignore = E203 - -# flake8-pytest-style -# PT001: -pytest-fixture-no-parentheses = true -# PT006: -pytest-parametrize-names-type = tuple -# PT007: -pytest-parametrize-values-type = tuple -pytest-parametrize-values-row-type = tuple -# PT023: -pytest-mark-no-parentheses = true - -[isort] -profile = black -add_imports = from __future__ import annotations - -[mypy] -disallow_untyped_defs = true -disallow_any_generics = true -disallow_incomplete_defs = true -disallow_subclassing_any = true -disallow_untyped_calls = true -disallow_untyped_decorators = true -ignore_missing_imports = true -no_implicit_optional = true -no_implicit_reexport = true -strict_equality = true -warn_redundant_casts = true -warn_return_any = true -warn_unused_configs = true -warn_unused_ignores = true - -# Avoid error: Duplicate module named 'setup' -# https://github.com/python/mypy/issues/4008 -exclude = ^tests/test_data/ - -[mypy-tests.*] -disallow_untyped_defs = false diff --git a/setup.py b/setup.py deleted file mode 100644 index 046d466e8..000000000 --- a/setup.py +++ /dev/null @@ -1,5 +0,0 @@ -from __future__ import annotations - -from setuptools import setup - -setup(use_scm_version=True) diff --git a/tox.ini b/tox.ini index 459a251fa..765e4ca19 100644 --- a/tox.ini +++ b/tox.ini @@ -45,7 +45,7 @@ deps = commands_pre = commands = python -m build --outdir {envtmpdir} --sdist {toxinidir} - twine check {envtmpdir}{/}* + twine check --strict {envtmpdir}{/}* skip_install = true [testenv:build-docs] From 8dec2a1dc747ac71edaf4cbe738a24c4adaeb89c Mon Sep 17 00:00:00 2001 From: q0w <43147888+q0w@users.noreply.github.com> Date: Sun, 11 Dec 2022 19:44:41 +0300 Subject: [PATCH 5/7] Remove default None for some args in writer._iter_lines --- piptools/writer.py | 10 +++------- tests/test_writer.py | 38 +++++++++++++++++++++++++++++++------- 2 files changed, 34 insertions(+), 14 deletions(-) diff --git a/piptools/writer.py b/piptools/writer.py index 280f1db35..ae42e5bf1 100644 --- a/piptools/writer.py +++ b/piptools/writer.py @@ -175,17 +175,13 @@ def write_flags(self) -> Iterator[str]: def _iter_lines( self, results: set[InstallRequirement], - unsafe_requirements: set[InstallRequirement] | None = None, - unsafe_packages: set[str] | None = None, - markers: dict[str, Marker] | None = None, + unsafe_requirements: set[InstallRequirement], + unsafe_packages: set[str], + markers: dict[str, Marker], hashes: dict[InstallRequirement, set[str]] | None = None, ) -> Iterator[str]: # default values - unsafe_requirements = unsafe_requirements or set() - if unsafe_packages is None: - unsafe_packages = set() unsafe_packages = unsafe_packages if not self.allow_unsafe else set() - markers = markers or {} hashes = hashes or {} # Check for unhashed or unpinned packages if at least one package does have diff --git a/tests/test_writer.py b/tests/test_writer.py index 91b7b195d..71d407419 100644 --- a/tests/test_writer.py +++ b/tests/test_writer.py @@ -113,7 +113,10 @@ def test_iter_lines__unsafe_dependencies(writer, from_line, allow_unsafe): writer.emit_header = False lines = writer._iter_lines( - [from_line("test==1.2")], [from_line("setuptools==1.10.0")] + {from_line("test==1.2")}, + {from_line("setuptools==1.10.0")}, + unsafe_packages=set(), + markers={}, ) expected_lines = ( @@ -132,7 +135,9 @@ def test_iter_lines__unsafe_with_hashes(capsys, writer, from_line): unsafe_ireqs = [from_line("setuptools==1.10.0")] hashes = {ireqs[0]: {"FAKEHASH"}, unsafe_ireqs[0]: set()} - lines = writer._iter_lines(ireqs, unsafe_ireqs, hashes=hashes) + lines = writer._iter_lines( + ireqs, unsafe_ireqs, unsafe_packages=set(), markers={}, hashes=hashes + ) expected_lines = ( "test==1.2 \\\n --hash=FAKEHASH", @@ -152,7 +157,13 @@ def test_iter_lines__hash_missing(capsys, writer, from_line): ireqs = [from_line("test==1.2"), from_line("file:///example/#egg=example")] hashes = {ireqs[0]: {"FAKEHASH"}, ireqs[1]: set()} - lines = writer._iter_lines(ireqs, hashes=hashes) + lines = writer._iter_lines( + ireqs, + hashes=hashes, + unsafe_requirements=set(), + unsafe_packages=set(), + markers={}, + ) expected_lines = ( MESSAGE_UNHASHED_PACKAGE, @@ -178,7 +189,13 @@ def test_iter_lines__no_warn_if_only_unhashable_packages(writer, from_line): ] hashes = {ireq: set() for ireq in ireqs} - lines = writer._iter_lines(ireqs, hashes=hashes) + lines = writer._iter_lines( + ireqs, + hashes=hashes, + unsafe_requirements=set(), + unsafe_packages=set(), + markers={}, + ) expected_lines = ( "unhashable-pkg1 @ file:///unhashable-pkg1/", @@ -389,16 +406,23 @@ def test_write_order(writer, from_line): """ writer.emit_header = False - packages = [ + packages = { from_line("package_a==0.1"), from_line("Package-b==2.3.4"), from_line("Package==5.6"), from_line("package2==7.8.9"), - ] + } expected_lines = [ "package==5.6", "package-a==0.1", "package-b==2.3.4", "package2==7.8.9", ] - assert list(writer._iter_lines(packages)) == expected_lines + assert ( + list( + writer._iter_lines( + packages, unsafe_requirements=set(), unsafe_packages=set(), markers={} + ) + ) + == expected_lines + ) From 2ccaa625067da53296bdbc76f4d6b0619d875ec8 Mon Sep 17 00:00:00 2001 From: q0w <43147888+q0w@users.noreply.github.com> Date: Sun, 11 Dec 2022 21:30:35 +0300 Subject: [PATCH 6/7] Prettify --- tests/test_writer.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/test_writer.py b/tests/test_writer.py index 71d407419..4dfeb0303 100644 --- a/tests/test_writer.py +++ b/tests/test_writer.py @@ -418,11 +418,9 @@ def test_write_order(writer, from_line): "package-b==2.3.4", "package2==7.8.9", ] - assert ( - list( - writer._iter_lines( - packages, unsafe_requirements=set(), unsafe_packages=set(), markers={} - ) + result = list( + writer._iter_lines( + packages, unsafe_requirements=set(), unsafe_packages=set(), markers={} ) - == expected_lines ) + assert result == expected_lines From aa243188e7ec26b14118fa83dc62cad4e14b6eab Mon Sep 17 00:00:00 2001 From: q0w <43147888+q0w@users.noreply.github.com> Date: Sun, 11 Dec 2022 21:35:04 +0300 Subject: [PATCH 7/7] Prettify --- tests/test_writer.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/test_writer.py b/tests/test_writer.py index 4dfeb0303..8a59269c7 100644 --- a/tests/test_writer.py +++ b/tests/test_writer.py @@ -418,9 +418,7 @@ def test_write_order(writer, from_line): "package-b==2.3.4", "package2==7.8.9", ] - result = list( - writer._iter_lines( - packages, unsafe_requirements=set(), unsafe_packages=set(), markers={} - ) + result = writer._iter_lines( + packages, unsafe_requirements=set(), unsafe_packages=set(), markers={} ) - assert result == expected_lines + assert list(result) == expected_lines