From 7059a26fbd35aee122322b4e223c375fdc714b39 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 29 Apr 2020 04:11:19 -0400 Subject: [PATCH 1/4] Float wheels to the top of the candidate sort order - `ignore_compatibility` is meant to resolve hashes into the lockfile after resolution happens - We still want compatible items to be the ones we actually tell pip to install - Fixes #4231 Signed-off-by: Dan Ryan --- news/4231.bugfix.rst | 1 + pipenv/patched/notpip/_internal/index/package_finder.py | 2 +- tasks/vendoring/patches/patched/pip20.patch | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 news/4231.bugfix.rst diff --git a/news/4231.bugfix.rst b/news/4231.bugfix.rst new file mode 100644 index 0000000000..225e66b682 --- /dev/null +++ b/news/4231.bugfix.rst @@ -0,0 +1 @@ +Fixed a bug which caused pipenv to prefer source distributions over wheels from ``PyPI`` during the dependency resolution phase. diff --git a/pipenv/patched/notpip/_internal/index/package_finder.py b/pipenv/patched/notpip/_internal/index/package_finder.py index e8a806a448..8c3e98957d 100644 --- a/pipenv/patched/notpip/_internal/index/package_finder.py +++ b/pipenv/patched/notpip/_internal/index/package_finder.py @@ -535,7 +535,7 @@ def _sort_key(self, candidate, ignore_compatibility=True): ) if self._prefer_binary: binary_preference = 1 - tags = self.valid_tags if not ignore_compatibility else None + tags = valid_tags try: pri = -(wheel.support_index_min(tags=tags)) except TypeError: diff --git a/tasks/vendoring/patches/patched/pip20.patch b/tasks/vendoring/patches/patched/pip20.patch index 9fa79b0fda..443a1975f0 100644 --- a/tasks/vendoring/patches/patched/pip20.patch +++ b/tasks/vendoring/patches/patched/pip20.patch @@ -114,7 +114,7 @@ index 02a187c8..f917e645 100644 modifying_pip=modifying_pip ) diff --git a/pipenv/patched/pip/_internal/index/package_finder.py b/pipenv/patched/pip/_internal/index/package_finder.py -index a74d78db..11128f4d 100644 +index a74d78db..7c9dc1be 100644 --- a/pipenv/patched/pip/_internal/index/package_finder.py +++ b/pipenv/patched/pip/_internal/index/package_finder.py @@ -121,6 +121,7 @@ class LinkEvaluator(object): @@ -201,7 +201,7 @@ index a74d78db..11128f4d 100644 if self._prefer_binary: binary_preference = 1 - pri = -(wheel.support_index_min(valid_tags)) -+ tags = self.valid_tags if not ignore_compatibility else None ++ tags = valid_tags + try: + pri = -(wheel.support_index_min(tags=tags)) + except TypeError: From 47aa2acc9078b032685c0459bf0c99238064aee5 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 8 May 2020 17:50:42 -0400 Subject: [PATCH 2/4] Set build_isolation and use_pep517 correctly - Fix how `use_pep517` and `build_isolation` are read from the environment -- introduce a new environment helper to detect `_` and `_NO_`, check for booleans and return appropriately boolean, str, or None types - Check for `False` values when adding `--no-use-pep517` and `--no-build-isolation` during resolution rather than falsey values - Change environment variable name from `PIP_PYTHON_VERSION` to `PIPENV_REQUESTED_PYTHON_VERSION` to avoid causing `pip` to fail due to accidentally percieving the `python_version` flag as being set -- this is an artifact from attempting to resolve outside of the virtualenv - Add `pipenv` to the path of patched `notpip.__main__` to accommodate updated import fully qualified module names - Update `pip` and `piptools` patches - Add test packages for each of two known failure modes: outdated `setuptools` with a missing `build-backend` (which `pip` forces to `build_meta:__legacy__` & which doesn't exist before `40.8`), and `import Cython` statements in `setup.py` in packages with properly defined `pyproject.toml` `build-backend` lines. - Fixes #4231 - Replaces, includes, and closes #4242 Signed-off-by: Dan Ryan Add integration tests for #4231 Signed-off-by: Dan Ryan --- news/4231.bugfix.rst | 1 + pipenv/environments.py | 30 +++++ pipenv/patched/notpip/__main__.py | 2 + pipenv/patched/piptools/utils.py | 2 +- pipenv/resolver.py | 2 +- pipenv/utils.py | 19 ++- tasks/vendoring/patches/patched/pip20.patch | 14 ++ .../vendoring/patches/patched/piptools.patch | 2 +- .../cython-import-package/pyproject.toml | 52 +++++++ .../fixtures/cython-import-package/setup.cfg | 58 ++++++++ tests/fixtures/cython-import-package/setup.py | 43 ++++++ .../src/cython_import_package/__init__.py | 1 + .../legacy-backend-package/pyproject.toml | 51 +++++++ .../fixtures/legacy-backend-package/setup.cfg | 127 ++++++++++++++++++ .../fixtures/legacy-backend-package/setup.py | 35 +++++ .../src/legacy_backend_package/__init__.py | 1 + tests/integration/test_lock.py | 49 +++++++ tests/pypi | 2 +- 18 files changed, 476 insertions(+), 15 deletions(-) create mode 100644 tests/fixtures/cython-import-package/pyproject.toml create mode 100644 tests/fixtures/cython-import-package/setup.cfg create mode 100644 tests/fixtures/cython-import-package/setup.py create mode 100644 tests/fixtures/cython-import-package/src/cython_import_package/__init__.py create mode 100644 tests/fixtures/legacy-backend-package/pyproject.toml create mode 100644 tests/fixtures/legacy-backend-package/setup.cfg create mode 100644 tests/fixtures/legacy-backend-package/setup.py create mode 100644 tests/fixtures/legacy-backend-package/src/legacy_backend_package/__init__.py diff --git a/news/4231.bugfix.rst b/news/4231.bugfix.rst index 225e66b682..b6f9bf8add 100644 --- a/news/4231.bugfix.rst +++ b/news/4231.bugfix.rst @@ -1 +1,2 @@ Fixed a bug which caused pipenv to prefer source distributions over wheels from ``PyPI`` during the dependency resolution phase. +Fixed an issue which prevented proper build isolation using ``pep517`` based builders during dependency resolution. diff --git a/pipenv/environments.py b/pipenv/environments.py index b0ba164d34..7154e80d3e 100644 --- a/pipenv/environments.py +++ b/pipenv/environments.py @@ -24,6 +24,36 @@ def _is_env_truthy(name): return os.environ.get(name).lower() not in ("0", "false", "no", "off") +def get_from_env(arg, prefix="PIPENV", check_for_negation=True): + """ + Check the environment for a variable, returning its truthy or stringified value + + For example, setting ``PIPENV_NO_RESOLVE_VCS=1`` would mean that + ``get_from_env("RESOLVE_VCS", prefix="PIPENV")`` would return ``False``. + + :param str arg: The name of the variable to look for + :param str prefix: The prefix to attach to the variable, defaults to "PIPENV" + :param bool check_for_negation: Whether to check for ``_NO_``, defaults + to True + :return: The value from the environment if available + :rtype: Optional[Union[str, bool]] + """ + negative_lookup = "NO_{0}".format(arg) + positive_lookup = arg + if prefix: + positive_lookup = "{0}_{1}".format(prefix, arg) + negative_lookup = "{0}_{1}".format(prefix, negative_lookup) + if positive_lookup in os.environ: + if _is_env_truthy(positive_lookup): + return bool(os.environ[positive_lookup]) + return os.environ[positive_lookup] + if negative_lookup in os.environ: + if _is_env_truthy(negative_lookup): + return not bool(os.environ[negative_lookup]) + return os.environ[negative_lookup] + return None + + PIPENV_IS_CI = bool("CI" in os.environ or "TF_BUILD" in os.environ) # HACK: Prevent invalid shebangs with Homebrew-installed Python: diff --git a/pipenv/patched/notpip/__main__.py b/pipenv/patched/notpip/__main__.py index 56f669fafa..3c2161897b 100644 --- a/pipenv/patched/notpip/__main__.py +++ b/pipenv/patched/notpip/__main__.py @@ -11,7 +11,9 @@ # Resulting path is the name of the wheel itself # Add that to sys.path so we can import pipenv.patched.notpip path = os.path.dirname(os.path.dirname(__file__)) + pipenv = os.path.dirname(os.path.dirname(path)) sys.path.insert(0, path) + sys.path.insert(0, pipenv) from pipenv.patched.notpip._internal.cli.main import main as _main # isort:skip # noqa diff --git a/pipenv/patched/piptools/utils.py b/pipenv/patched/piptools/utils.py index e6f232f698..1123fb64b6 100644 --- a/pipenv/patched/piptools/utils.py +++ b/pipenv/patched/piptools/utils.py @@ -76,7 +76,7 @@ def simplify_markers(ireq): def clean_requires_python(candidates): """Get a cleaned list of all the candidates with valid specifiers in the `requires_python` attributes.""" all_candidates = [] - py_version = parse_version(os.environ.get('PIP_PYTHON_VERSION', '.'.join(map(str, sys.version_info[:3])))) + py_version = parse_version(os.environ.get('PIPENV_REQUESTED_PYTHON_VERSION', '.'.join(map(str, sys.version_info[:3])))) for c in candidates: if getattr(c, "requires_python", None): # Old specifications had people setting this to single digits diff --git a/pipenv/resolver.py b/pipenv/resolver.py index cd04fccb0f..733b28d5da 100644 --- a/pipenv/resolver.py +++ b/pipenv/resolver.py @@ -771,7 +771,7 @@ def resolve(packages, pre, project, sources, clear, system, requirements_dir=Non def _main(pre, clear, verbose, system, write, requirements_dir, packages, parse_only=False): - os.environ["PIP_PYTHON_VERSION"] = ".".join([str(s) for s in sys.version_info[:3]]) + os.environ["PIPENV_REQUESTED_PYTHON_VERSION"] = ".".join([str(s) for s in sys.version_info[:3]]) os.environ["PIP_PYTHON_PATH"] = str(sys.executable) if parse_only: parse_packages( diff --git a/pipenv/utils.py b/pipenv/utils.py index eb819d5c34..ed8ed29661 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -236,14 +236,14 @@ def __init__(self, python_version, python_path): def __enter__(self): # Only inject when the value is valid if self.python_version: - os.environ["PIP_PYTHON_VERSION"] = str(self.python_version) + os.environ["PIPENV_REQUESTED_PYTHON_VERSION"] = str(self.python_version) if self.python_path: os.environ["PIP_PYTHON_PATH"] = str(self.python_path) def __exit__(self, *args): # Restore original Python version information. try: - del os.environ["PIP_PYTHON_VERSION"] + del os.environ["PIPENV_REQUESTED_PYTHON_VERSION"] except KeyError: pass @@ -682,25 +682,21 @@ def pip_command(self): self._pip_command = self._get_pip_command() return self._pip_command - def prepare_pip_args(self, use_pep517=True, build_isolation=True): + def prepare_pip_args(self, use_pep517=False, build_isolation=True): pip_args = [] if self.sources: pip_args = prepare_pip_source_args(self.sources, pip_args) - if not use_pep517: + if use_pep517 is False: pip_args.append("--no-use-pep517") - if not build_isolation: + if build_isolation is False: pip_args.append("--no-build-isolation") pip_args.extend(["--cache-dir", environments.PIPENV_CACHE_DIR]) return pip_args @property def pip_args(self): - use_pep517 = False if ( - os.environ.get("PIP_NO_USE_PEP517", None) is not None - ) else (True if os.environ.get("PIP_USE_PEP517", None) is not None else None) - build_isolation = False if ( - os.environ.get("PIP_NO_BUILD_ISOLATION", None) is not None - ) else (True if os.environ.get("PIP_BUILD_ISOLATION", None) is not None else None) + use_pep517 = environments.get_from_env("USE_PEP517", prefix="PIP") + build_isolation = environments.get_from_env("BUILD_ISOLATION", prefix="PIP") if self._pip_args is None: self._pip_args = self.prepare_pip_args( use_pep517=use_pep517, build_isolation=build_isolation @@ -790,6 +786,7 @@ def get_resolver(self, clear=False, pre=False): self._resolver = PiptoolsResolver( constraints=self.parsed_constraints, repository=self.repository, cache=DependencyCache(environments.PIPENV_CACHE_DIR), clear_caches=clear, + # TODO: allow users to toggle the 'allow unsafe' flag to resolve setuptools? prereleases=pre, allow_unsafe=False ) diff --git a/tasks/vendoring/patches/patched/pip20.patch b/tasks/vendoring/patches/patched/pip20.patch index 443a1975f0..c3dcd2a184 100644 --- a/tasks/vendoring/patches/patched/pip20.patch +++ b/tasks/vendoring/patches/patched/pip20.patch @@ -589,3 +589,17 @@ index 65e41bc7..9eabf28e 100644 class AdjacentTempDirectory(TempDirectory): +diff --git a/pipenv/patched/pip/__main__.py b/pipenv/patched/pip/__main__.py +index 56f669fa..3c216189 100644 +--- a/pipenv/patched/pip/__main__.py ++++ b/pipenv/patched/pip/__main__.py +@@ -11,7 +11,9 @@ if __package__ == '': + # Resulting path is the name of the wheel itself + # Add that to sys.path so we can import pip + path = os.path.dirname(os.path.dirname(__file__)) ++ pipenv = os.path.dirname(os.path.dirname(path)) + sys.path.insert(0, path) ++ sys.path.insert(0, pipenv) + + from pip._internal.cli.main import main as _main # isort:skip # noqa + diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index 06e480d96e..4eea7808f1 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -745,7 +745,7 @@ index 7733447..e6f232f 100644 +def clean_requires_python(candidates): + """Get a cleaned list of all the candidates with valid specifiers in the `requires_python` attributes.""" + all_candidates = [] -+ py_version = parse_version(os.environ.get('PIP_PYTHON_VERSION', '.'.join(map(str, sys.version_info[:3])))) ++ py_version = parse_version(os.environ.get('PIPENV_REQUESTED_PYTHON_VERSION', '.'.join(map(str, sys.version_info[:3])))) + for c in candidates: + if getattr(c, "requires_python", None): + # Old specifications had people setting this to single digits diff --git a/tests/fixtures/cython-import-package/pyproject.toml b/tests/fixtures/cython-import-package/pyproject.toml new file mode 100644 index 0000000000..661b63c5bf --- /dev/null +++ b/tests/fixtures/cython-import-package/pyproject.toml @@ -0,0 +1,52 @@ +[build-system] +requires = ["setuptools >= 40.6.0", "setuptools-scm", "cython"] +build-backend = "setuptools.build_meta" + +[tool.black] +line-length = 90 +target_version = ['py27', 'py35', 'py36', 'py37', 'py38'] +include = '\.pyi?$' +exclude = ''' +/( + \.eggs + | \.git + | \.hg + | \.mypy_cache + | \.tox + | \.pyre_configuration + | \.venv + | _build + | buck-out + | build + | dist +) +''' + +[tool.towncrier] +package = 'cython-import-package' +package_dir = 'src' +filename = 'CHANGELOG.rst' +directory = 'news/' +title_format = '{version} ({project_date})' +issue_format = '`#{issue} `_' +template = 'tasks/CHANGELOG.rst.jinja2' + + [[tool.towncrier.type]] + directory = 'feature' + name = 'Features' + showcontent = true + + [[tool.towncrier.type]] + directory = 'bugfix' + name = 'Bug Fixes' + showcontent = true + + [[tool.towncrier.type]] + directory = 'trivial' + name = 'Trivial Changes' + showcontent = false + + [[tool.towncrier.type]] + directory = 'removal' + name = 'Removals and Deprecations' + showcontent = true diff --git a/tests/fixtures/cython-import-package/setup.cfg b/tests/fixtures/cython-import-package/setup.cfg new file mode 100644 index 0000000000..a43ee22b9b --- /dev/null +++ b/tests/fixtures/cython-import-package/setup.cfg @@ -0,0 +1,58 @@ +[metadata] +name = cython_import_package +package_name = cython-import-package +description = A fake python package. +url = https://github.com/sarugaku/cython_import_package +author = Dan Ryan +author_email = dan@danryan.co +long_description = file: README.rst +license = ISC License +keywords = fake package test +classifier = + Development Status :: 1 - Planning + License :: OSI Approved :: ISC License (ISCL) + Operating System :: OS Independent + Programming Language :: Python :: 2 + Programming Language :: Python :: 2.6 + Programming Language :: Python :: 2.7 + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.4 + Programming Language :: Python :: 3.5 + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 + Topic :: Software Development :: Libraries :: Python Modules + +[options.extras_require] +tests = + pytest + pytest-xdist + pytest-cov + pytest-timeout + readme-renderer[md] + twine +dev = + black;python_version>="3.6" + flake8 + flake8-bugbear;python_version>="3.5" + invoke + isort + mypy;python_version>="3.5" + parver + pre-commit + rope + wheel + +[options] +zip_safe = true +python_requires = >=2.6,!=3.0,!=3.1,!=3.2,!=3.3 +install_requires = + attrs + vistir + +[bdist_wheel] +universal = 1 + +[egg_info] +tag_build = +tag_date = 0 + diff --git a/tests/fixtures/cython-import-package/setup.py b/tests/fixtures/cython-import-package/setup.py new file mode 100644 index 0000000000..78eeeeda17 --- /dev/null +++ b/tests/fixtures/cython-import-package/setup.py @@ -0,0 +1,43 @@ +import ast +import os + +from setuptools import setup, find_packages +from setuptools.command.test import test as TestCommand + +# ORDER MATTERS +# Import this after setuptools or it will fail +from Cython.Build import cythonize # noqa: I100 +import Cython.Distutils + + + +ROOT = os.path.dirname(__file__) + +PACKAGE_NAME = 'cython_import_package' + +VERSION = None + +with open(os.path.join(ROOT, 'src', PACKAGE_NAME.replace("-", "_"), '__init__.py')) as f: + for line in f: + if line.startswith('__version__ = '): + VERSION = ast.literal_eval(line[len('__version__ = '):].strip()) + break +if VERSION is None: + raise EnvironmentError('failed to read version') + + +# Put everything in setup.cfg, except those that don't actually work? +setup( + # These really don't work. + package_dir={'': 'src'}, + packages=find_packages('src'), + + # I don't know how to specify an empty key in setup.cfg. + package_data={ + '': ['LICENSE*', 'README*'], + }, + setup_requires=["setuptools_scm", "cython"], + + # I need this to be dynamic. + version=VERSION, +) diff --git a/tests/fixtures/cython-import-package/src/cython_import_package/__init__.py b/tests/fixtures/cython-import-package/src/cython_import_package/__init__.py new file mode 100644 index 0000000000..f102a9cadf --- /dev/null +++ b/tests/fixtures/cython-import-package/src/cython_import_package/__init__.py @@ -0,0 +1 @@ +__version__ = "0.0.1" diff --git a/tests/fixtures/legacy-backend-package/pyproject.toml b/tests/fixtures/legacy-backend-package/pyproject.toml new file mode 100644 index 0000000000..e646fb3c25 --- /dev/null +++ b/tests/fixtures/legacy-backend-package/pyproject.toml @@ -0,0 +1,51 @@ +[build-system] +requires = ["setuptools>=30.3.0", "wheel", "setuptools_scm>=3.3.1"] + +[tool.black] +line-length = 90 +target_version = ['py27', 'py35', 'py36', 'py37', 'py38'] +include = '\.pyi?$' +exclude = ''' +/( + \.eggs + | \.git + | \.hg + | \.mypy_cache + | \.tox + | \.pyre_configuration + | \.venv + | _build + | buck-out + | build + | dist +) +''' + +[tool.towncrier] +package = 'legacy-backend-package' +package_dir = 'src' +filename = 'CHANGELOG.rst' +directory = 'news/' +title_format = '{version} ({project_date})' +issue_format = '`#{issue} `_' +template = 'tasks/CHANGELOG.rst.jinja2' + + [[tool.towncrier.type]] + directory = 'feature' + name = 'Features' + showcontent = true + + [[tool.towncrier.type]] + directory = 'bugfix' + name = 'Bug Fixes' + showcontent = true + + [[tool.towncrier.type]] + directory = 'trivial' + name = 'Trivial Changes' + showcontent = false + + [[tool.towncrier.type]] + directory = 'removal' + name = 'Removals and Deprecations' + showcontent = true diff --git a/tests/fixtures/legacy-backend-package/setup.cfg b/tests/fixtures/legacy-backend-package/setup.cfg new file mode 100644 index 0000000000..1e5f1ed85d --- /dev/null +++ b/tests/fixtures/legacy-backend-package/setup.cfg @@ -0,0 +1,127 @@ +[metadata] +name = legacy_backend_package +package_name = legacy-backend-package +description = A fake python package. +url = https://github.com/sarugaku/legacy_backend_package +author = Dan Ryan +author_email = dan@danryan.co +long_description = file: README.rst +license = ISC License +keywords = fake package test +classifier = + Development Status :: 1 - Planning + License :: OSI Approved :: ISC License (ISCL) + Operating System :: OS Independent + Programming Language :: Python :: 2 + Programming Language :: Python :: 2.6 + Programming Language :: Python :: 2.7 + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.4 + Programming Language :: Python :: 3.5 + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 + Topic :: Software Development :: Libraries :: Python Modules + +[options.extras_require] +tests = + pytest + pytest-xdist + pytest-cov + pytest-timeout + readme-renderer[md] + twine +dev = + black;python_version>="3.6" + flake8 + flake8-bugbear;python_version>="3.5" + invoke + isort + mypy;python_version>="3.5" + parver + pre-commit + rope + wheel + +[options] +zip_safe = true +python_requires = >=2.6,!=3.0,!=3.1,!=3.2,!=3.3 +setup_requires = + setuptools_scm>=3.3.1 +install_requires = + attrs + vistir + +[bdist_wheel] +universal = 1 + +[egg_info] +tag_build = +tag_date = 0 + + +[tool:pytest] +strict = true +plugins = cov flake8 +addopts = -ra +testpaths = tests/ +norecursedirs = .* build dist news tasks docs +flake8-ignore = + docs/source/* ALL + tests/*.py ALL + setup.py ALL +filterwarnings = + ignore::DeprecationWarning + ignore::PendingDeprecationWarning + +[isort] +atomic = true +not_skip = __init__.py +line_length = 90 +indent = ' ' +multi_line_output = 3 +known_third_party = invoke,parver,pytest,setuptools,towncrier +known_first_party = + legacy_backend_package + tests +combine_as_imports=True +include_trailing_comma = True +force_grid_wrap=0 + +[flake8] +max-line-length = 90 +select = C,E,F,W,B,B950 +ignore = + # The default ignore list: + D203,F401,E123,E203,W503,E501,E402 + #E121,E123,E126,E226,E24,E704, + # Our additions: + # E123: closing bracket does not match indentation of opening bracket’s line + # E203: whitespace before ‘:’ + # E129: visually indented line with same indent as next logical line + # E222: multiple spaces after operator + # E231: missing whitespace after ',' + # D203: 1 blank line required before class docstring + # E402: module level import not at top of file + # E501: line too long (using B950 from flake8-bugbear) + # F401: Module imported but unused + # W503: line break before binary operator (not a pep8 issue, should be ignored) +exclude = + .tox, + .git, + __pycache__, + docs/source/*, + build, + dist, + tests/*, + *.pyc, + *.egg-info, + .cache, + .eggs, + setup.py, +max-complexity=13 + +[mypy] +ignore_missing_imports=true +follow_imports=skip +html_report=mypyhtml +python_version=2.7 diff --git a/tests/fixtures/legacy-backend-package/setup.py b/tests/fixtures/legacy-backend-package/setup.py new file mode 100644 index 0000000000..e41a3e36a6 --- /dev/null +++ b/tests/fixtures/legacy-backend-package/setup.py @@ -0,0 +1,35 @@ +import ast +import os + +from setuptools import setup, find_packages +from setuptools.command.test import test as TestCommand + +ROOT = os.path.dirname(__file__) + +PACKAGE_NAME = 'legacy_backend_package' + +VERSION = None + +with open(os.path.join(ROOT, 'src', PACKAGE_NAME.replace("-", "_"), '__init__.py')) as f: + for line in f: + if line.startswith('__version__ = '): + VERSION = ast.literal_eval(line[len('__version__ = '):].strip()) + break +if VERSION is None: + raise EnvironmentError('failed to read version') + + +# Put everything in setup.cfg, except those that don't actually work? +setup( + # These really don't work. + package_dir={'': 'src'}, + packages=find_packages('src'), + + # I don't know how to specify an empty key in setup.cfg. + package_data={ + '': ['LICENSE*', 'README*'], + }, + + # I need this to be dynamic. + version=VERSION, +) diff --git a/tests/fixtures/legacy-backend-package/src/legacy_backend_package/__init__.py b/tests/fixtures/legacy-backend-package/src/legacy_backend_package/__init__.py new file mode 100644 index 0000000000..f102a9cadf --- /dev/null +++ b/tests/fixtures/legacy-backend-package/src/legacy_backend_package/__init__.py @@ -0,0 +1 @@ +__version__ = "0.0.1" diff --git a/tests/integration/test_lock.py b/tests/integration/test_lock.py index d745d23711..cf9a3d5cb5 100644 --- a/tests/integration/test_lock.py +++ b/tests/integration/test_lock.py @@ -384,6 +384,55 @@ def test_private_index_mirror_lock_requirements(PipenvInstance_NoPyPI): assert '--extra-index-url {}'.format(mirror_url) not in c.out.strip() +@pytest.mark.lock +@pytest.mark.install +@pytest.mark.needs_internet +def test_outdated_setuptools_with_pep517_legacy_build_meta_is_updated(PipenvInstance): + """ + This test ensures we are using build isolation and a pep517 backend + because the package in question includes ``pyproject.toml`` but lacks + a ``build-backend`` declaration. In this case, ``pip`` defaults to using + ``setuptools.build_meta:__legacy__`` as a builder, but without ``pep517`` + enabled and with ``setuptools==40.2.0`` installed, this build backend was + not yet available. ``setuptools<40.8`` will not be aware of this backend. + + If pip is able to build in isolation with a pep517 backend, this will not + matter and the test will still pass as pip will by default install a more + recent version of ``setuptools``. + """ + with PipenvInstance(chdir=True) as p: + c = p.pipenv('run pip install "setuptools<=40.2"') + assert c.return_code == 0 + c = p.pipenv("run python -c 'import setuptools; print(setuptools.__version__)'") + assert c.return_code == 0 + assert c.out.strip() == "40.2.0" + c = p.pipenv("install legacy-backend-package") + assert c.return_code == 0 + assert "vistir" in p.lockfile["default"] + + +@pytest.mark.lock +@pytest.mark.install +@pytest.mark.needs_internet +def test_outdated_setuptools_with_pep517_cython_import_in_setuppy(PipenvInstance): + """ + This test ensures we are using build isolation and a pep517 backend + because the package in question declares 'cython' as a build dependency + in ``pyproject.toml``, then imports it in ``setup.py``. The pep517 + backend will have to install it first, so this will only pass if the + resolver is buliding with a proper backend. + """ + with PipenvInstance(chdir=True) as p: + c = p.pipenv('run pip install "setuptools<=40.2"') + assert c.return_code == 0 + c = p.pipenv("run python -c 'import setuptools; print(setuptools.__version__)'") + assert c.return_code == 0 + assert c.out.strip() == "40.2.0" + c = p.pipenv("install cython-import-package") + assert c.return_code == 0 + assert "vistir" in p.lockfile["default"] + + @pytest.mark.index @pytest.mark.install def test_lock_updated_source(PipenvInstance): diff --git a/tests/pypi b/tests/pypi index 59882cf200..776f8bfab1 160000 --- a/tests/pypi +++ b/tests/pypi @@ -1 +1 @@ -Subproject commit 59882cf2000f36a6644a883430df9dfeb35d2106 +Subproject commit 776f8bfab15bbbba410ecaa0516d164077087052 From 362a730facdf311e08e5a806c62ca8286774c0f3 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sat, 9 May 2020 00:41:49 -0400 Subject: [PATCH 3/4] Fix environment comparison code and add tests Signed-off-by: Dan Ryan --- pipenv/environments.py | 39 +++++++++++++++---- tests/unit/test_environments.py | 68 +++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 8 deletions(-) create mode 100644 tests/unit/test_environments.py diff --git a/pipenv/environments.py b/pipenv/environments.py index 7154e80d3e..887522312c 100644 --- a/pipenv/environments.py +++ b/pipenv/environments.py @@ -14,6 +14,25 @@ # HACK: avoid resolver.py uses the wrong byte code files. # I hope I can remove this one day. os.environ["PYTHONDONTWRITEBYTECODE"] = fs_str("1") +_false_values = ("0", "false", "no", "off") +_true_values = ("1", "true", "yes", "on") + + +def env_to_bool(val): + """ + Convert **val** to boolean, returning True if truthy or False if falsey + + :param Any val: The value to convert + :return: False if Falsey, True if truthy + :rtype: bool + """ + if isinstance(val, bool): + return val + if val.lower() in _false_values: + return False + if val.lower() in _true_values: + return True + raise ValueError("Value is not a valid boolean-like: {0}".format(val)) def _is_env_truthy(name): @@ -21,7 +40,7 @@ def _is_env_truthy(name): """ if name not in os.environ: return False - return os.environ.get(name).lower() not in ("0", "false", "no", "off") + return os.environ.get(name).lower() not in _false_values def get_from_env(arg, prefix="PIPENV", check_for_negation=True): @@ -44,13 +63,17 @@ def get_from_env(arg, prefix="PIPENV", check_for_negation=True): positive_lookup = "{0}_{1}".format(prefix, arg) negative_lookup = "{0}_{1}".format(prefix, negative_lookup) if positive_lookup in os.environ: - if _is_env_truthy(positive_lookup): - return bool(os.environ[positive_lookup]) - return os.environ[positive_lookup] - if negative_lookup in os.environ: - if _is_env_truthy(negative_lookup): - return not bool(os.environ[negative_lookup]) - return os.environ[negative_lookup] + value = os.environ[positive_lookup] + try: + return env_to_bool(value) + except ValueError: + return value + if check_for_negation and negative_lookup in os.environ: + value = os.environ[negative_lookup] + try: + return not env_to_bool(value) + except ValueError: + return value return None diff --git a/tests/unit/test_environments.py b/tests/unit/test_environments.py new file mode 100644 index 0000000000..6baf84c810 --- /dev/null +++ b/tests/unit/test_environments.py @@ -0,0 +1,68 @@ +import itertools +import pytest +import os +from pipenv import environments +from pipenv.utils import temp_environ + + +@pytest.mark.environments +@pytest.mark.parametrize( + "arg, prefix, use_negation", + list(itertools.product(("ENABLE_SOMETHING",), ("FAKEPREFIX", None), (True, False))), +) +def test_get_from_env(arg, prefix, use_negation): + negated_arg = "NO_{0}".format(arg) + positive_var = arg + negative_var = negated_arg + if prefix: + negative_var = "{0}_{1}".format(prefix, negative_var) + positive_var = "{0}_{1}".format(prefix, positive_var) + # set the positive first + for var_to_set, opposite_var in ((arg, negated_arg), (negated_arg, arg)): + os.environ.pop(var_to_set, None) + os.environ.pop(opposite_var, None) + with temp_environ(): + is_positive = var_to_set == arg + is_negative = not is_positive + envvar = positive_var if is_positive else negative_var + os.environ[envvar] = "true" + main_expected_value = True if is_positive else None + if use_negation and not is_positive: + main_expected_value = False + # use negation means if the normal variable isnt set we will check + # for the negated version + negative_expected_value = ( + True if is_negative else None + ) + if is_positive: + assert ( + environments.get_from_env( + var_to_set, prefix, check_for_negation=use_negation + ) + is main_expected_value + ) + assert ( + environments.get_from_env( + opposite_var, prefix, check_for_negation=use_negation + ) + is negative_expected_value + ) + else: + # var_to_set = negative version i.e. NO_xxxx + # opposite_var = positive_version i.e. XXXX + + # get NO_BLAH -- expecting this to be True + assert ( + environments.get_from_env( + var_to_set, prefix, check_for_negation=use_negation + ) + is negative_expected_value + ) + # get BLAH -- expecting False if checking for negation + # but otherwise should be None + assert ( + environments.get_from_env( + opposite_var, prefix, check_for_negation=use_negation + ) + is main_expected_value + ) From c7425e7508fe054a0f1ee7f37f925df679d19bf0 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sat, 9 May 2020 10:55:01 -0400 Subject: [PATCH 4/4] Skip tests that build extensions on windows Signed-off-by: Dan Ryan --- tests/integration/conftest.py | 2 ++ tests/integration/test_lock.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index ef91f5031f..4940fb57f4 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -133,6 +133,8 @@ def pytest_runtest_setup(item): sys.version_info[:2] == (3, 6) ): pytest.skip('test is skipped on python 3.6') + if item.get_closest_marker('skip_windows') is not None and (os.name == 'nt'): + pytest.skip('test does not run on windows') @pytest.fixture diff --git a/tests/integration/test_lock.py b/tests/integration/test_lock.py index cf9a3d5cb5..633e8f9371 100644 --- a/tests/integration/test_lock.py +++ b/tests/integration/test_lock.py @@ -386,6 +386,7 @@ def test_private_index_mirror_lock_requirements(PipenvInstance_NoPyPI): @pytest.mark.lock @pytest.mark.install +@pytest.mark.skip_windows @pytest.mark.needs_internet def test_outdated_setuptools_with_pep517_legacy_build_meta_is_updated(PipenvInstance): """ @@ -413,6 +414,7 @@ def test_outdated_setuptools_with_pep517_legacy_build_meta_is_updated(PipenvInst @pytest.mark.lock @pytest.mark.install +@pytest.mark.skip_windows @pytest.mark.needs_internet def test_outdated_setuptools_with_pep517_cython_import_in_setuppy(PipenvInstance): """