diff --git a/news/2828.feature.rst b/news/2828.feature.rst new file mode 100644 index 0000000000..688c47ee51 --- /dev/null +++ b/news/2828.feature.rst @@ -0,0 +1 @@ +Added additional output to ``pipenv update --outdated`` to indicate that the operation succeded and all packages were already up to date. diff --git a/news/3047.bugfix.rst b/news/3047.bugfix.rst new file mode 100644 index 0000000000..6c44bd420e --- /dev/null +++ b/news/3047.bugfix.rst @@ -0,0 +1 @@ +Fixed a virtualenv creation issue which could cause new virtualenvs to inadvertently attempt to read and write to global site packages. diff --git a/news/3055.bugfix.rst b/news/3055.bugfix.rst new file mode 100644 index 0000000000..7b2a0fa14f --- /dev/null +++ b/news/3055.bugfix.rst @@ -0,0 +1 @@ +Fixed an issue with virtualenv path derivation which could cause errors, particularly for users on WSL bash. diff --git a/news/3113.bugfix.rst b/news/3113.bugfix.rst index af43b87df8..75ee6de4f6 100644 --- a/news/3113.bugfix.rst +++ b/news/3113.bugfix.rst @@ -1 +1 @@ -Fixed an issue resolving virtualenv paths for users without ``platlib`` values on their systems. +Fixed an issue which caused ``pipenv clean`` to sometimes clean packages from the base ``site-packages`` folder or fail entirely. diff --git a/pipenv/__init__.py b/pipenv/__init__.py index 6b8ddf6664..f8a1a8b3e1 100644 --- a/pipenv/__init__.py +++ b/pipenv/__init__.py @@ -22,6 +22,7 @@ from pipenv.vendor.vistir.compat import ResourceWarning, fs_str warnings.filterwarnings("ignore", category=DependencyWarning) warnings.filterwarnings("ignore", category=ResourceWarning) +warnings.filterwarnings("ignore", category=UserWarning) if sys.version_info >= (3, 1) and sys.version_info <= (3, 6): if sys.stdout.isatty() and sys.stderr.isatty(): diff --git a/pipenv/core.py b/pipenv/core.py index 1b9c5ade65..8a6d21c25a 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -41,7 +41,8 @@ rmtree, clean_resolved_dep, parse_indexes, - escape_cmd + escape_cmd, + fix_venv_site ) from . import environments, pep508checker, progress from .environments import ( @@ -1705,6 +1706,7 @@ def do_py(system=False): def do_outdated(pypi_mirror=None): + # TODO: Allow --skip-lock here? from .vendor.requirementslib.models.requirements import Requirement packages = {} @@ -1729,6 +1731,9 @@ def do_outdated(pypi_mirror=None): outdated.append( (package, updated_packages[norm_name], packages[package]) ) + if not outdated: + click.echo(crayons.green("All packages are up to date!", bold=True)) + sys.exit(0) for package, new_version, old_version in outdated: click.echo( "Package {0!r} out-of-date: {1!r} installed, {2!r} available.".format( @@ -2062,6 +2067,7 @@ def do_uninstall( ): from .environments import PIPENV_USE_SYSTEM from .vendor.requirementslib.models.requirements import Requirement + from .vendor.packaging.utils import canonicalize_name # Automatically use an activated virtualenv. if PIPENV_USE_SYSTEM: @@ -2074,6 +2080,24 @@ def do_uninstall( Requirement.from_line("-e {0}".format(p)).name for p in editable_packages if p ] package_names = [p for p in packages if p] + editable_pkgs + installed_package_names = set([ + canonicalize_name(pkg.project_name) for pkg in project.get_installed_packages() + ]) + # Intelligently detect if --dev should be used or not. + if project.lockfile_exists: + develop = set( + [canonicalize_name(k) for k in project.lockfile_content["develop"].keys()] + ) + default = set( + [canonicalize_name(k) for k in project.lockfile_content["default"].keys()] + ) + else: + develop = set( + [canonicalize_name(k) for k in project.dev_packages.keys()] + ) + default = set( + [canonicalize_name(k) for k in project.packages.keys()] + ) pipfile_remove = True # Un-install all dependencies, if --all was provided. if all is True: @@ -2084,7 +2108,7 @@ def do_uninstall( return # Uninstall [dev-packages], if --dev was provided. if all_dev: - if "dev-packages" not in project.parsed_pipfile: + if "dev-packages" not in project.parsed_pipfile and not develop: click.echo( crayons.normal( "No {0} to uninstall.".format(crayons.red("[dev-packages]")), @@ -2097,40 +2121,64 @@ def do_uninstall( fix_utf8("Un-installing {0}…".format(crayons.red("[dev-packages]"))), bold=True ) ) - package_names = project.dev_packages.keys() if packages is False and editable_packages is False and not all_dev: click.echo(crayons.red("No package provided!"), err=True) return 1 - for package_name in package_names: - click.echo(fix_utf8("Un-installing {0}…".format(crayons.green(package_name)))) - cmd = "{0} uninstall {1} -y".format( - escape_grouped_arguments(which_pip(allow_global=system)), package_name + fix_venv_site(project.env_paths["lib"]) + # Remove known "bad packages" from the list. + for bad_package in BAD_PACKAGES: + if canonicalize_name(bad_package) in package_names: + if environments.is_verbose(): + click.echo("Ignoring {0}.".format(repr(bad_package)), err=True) + del package_names[package_names.index( + canonicalize_name(bad_package) + )] + used_packages = (develop | default) & installed_package_names + failure = False + packages_to_remove = set() + if all_dev: + packages_to_remove |= develop & installed_package_names + package_names = set([canonicalize_name(pkg_name) for pkg_name in package_names]) + packages_to_remove = package_names & used_packages + for package_name in packages_to_remove: + click.echo( + crayons.white( + fix_utf8("Uninstalling {0}…".format(repr(package_name))), bold=True + ) ) + # Uninstall the package. + cmd = "{0} uninstall {1} -y".format( + escape_grouped_arguments(which_pip()), package_name + ) if environments.is_verbose(): click.echo("$ {0}".format(cmd)) c = delegator.run(cmd) click.echo(crayons.blue(c.out)) - if pipfile_remove: - in_packages = project.get_package_name_in_pipfile(package_name, dev=False) - in_dev_packages = project.get_package_name_in_pipfile( - package_name, dev=True - ) - if not in_dev_packages and not in_packages: - click.echo( - "No package {0} to remove from Pipfile.".format( - crayons.green(package_name) - ) + if c.return_code != 0: + failure = True + else: + if pipfile_remove: + in_packages = project.get_package_name_in_pipfile(package_name, dev=False) + in_dev_packages = project.get_package_name_in_pipfile( + package_name, dev=True ) - continue + if not in_dev_packages and not in_packages: + click.echo( + "No package {0} to remove from Pipfile.".format( + crayons.green(package_name) + ) + ) + continue - click.echo( - fix_utf8("Removing {0} from Pipfile…".format(crayons.green(package_name))) - ) - # Remove package from both packages and dev-packages. - project.remove_package_from_pipfile(package_name, dev=True) - project.remove_package_from_pipfile(package_name, dev=False) + click.echo( + fix_utf8("Removing {0} from Pipfile…".format(crayons.green(package_name))) + ) + # Remove package from both packages and dev-packages. + project.remove_package_from_pipfile(package_name, dev=True) + project.remove_package_from_pipfile(package_name, dev=False) if lock: do_lock(system=system, keep_outdated=keep_outdated, pypi_mirror=pypi_mirror) + sys.exit(int(failure)) def do_shell(three=None, python=False, fancy=False, shell_args=None, pypi_mirror=None): @@ -2593,6 +2641,9 @@ def do_clean(ctx, three=None, python=None, dry_run=False, bare=False, pypi_mirro from packaging.utils import canonicalize_name ensure_project(three=three, python=python, validate=False, pypi_mirror=pypi_mirror) ensure_lockfile(pypi_mirror=pypi_mirror) + # Make sure that the virtualenv's site packages are configured correctly + # otherwise we may end up removing from the global site packages directory + fix_venv_site(project.env_paths["lib"]) installed_package_names = [ canonicalize_name(pkg.project_name) for pkg in project.get_installed_packages() ] diff --git a/pipenv/project.py b/pipenv/project.py index d25dba9a56..eabd7c8217 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -5,6 +5,7 @@ import re import sys import base64 +import itertools import fnmatch import hashlib import contoml @@ -33,6 +34,7 @@ get_workon_home, is_virtual_environment, looks_like_dir, + sys_version ) from .environments import ( PIPENV_MAX_DEPTH, @@ -42,6 +44,7 @@ PIPENV_TEST_INDEX, PIPENV_PYTHON, PIPENV_DEFAULT_PYTHON_VERSION, + PIPENV_CACHE_DIR ) from requirementslib.utils import is_vcs @@ -301,9 +304,9 @@ def find_egg(self, egg_dist): user_site = site.USER_SITE search_locations = [site_packages, user_site] for site_directory in search_locations: - egg = os.path.join(site_directory, search_filename) - if os.path.isfile(egg): - return egg + egg = os.path.join(site_directory, search_filename) + if os.path.isfile(egg): + return egg def locate_dist(self, dist): location = self.find_egg(dist) @@ -325,6 +328,71 @@ def get_installed_packages(self): packages = [pkg for pkg in packages] return packages + def get_package_info(self): + from .utils import prepare_pip_source_args + from .vendor.pip_shims import Command, cmdoptions, index_group, PackageFinder + index_urls = [source.get("url") for source in self.sources] + + class PipCommand(Command): + name = "PipCommand" + + dependency_links = [] + packages = self.get_installed_packages() + # This code is borrowed from pip's current implementation + for dist in packages: + if dist.has_metadata('dependency_links.txt'): + dependency_links.extend(dist.get_metadata_lines('dependency_links.txt')) + + pip_command = PipCommand() + index_opts = cmdoptions.make_option_group( + index_group, pip_command.parser + ) + cmd_opts = pip_command.cmd_opts + pip_command.parser.insert_option_group(0, index_opts) + pip_command.parser.insert_option_group(0, cmd_opts) + pip_args = prepare_pip_source_args(self.sources, []) + pip_options, _ = pip_command.parser.parse_args(pip_args) + pip_options.cache_dir = PIPENV_CACHE_DIR + pip_options.pre = self.settings.get("pre", False) + with pip_command._build_session(pip_options) as session: + finder = PackageFinder( + find_links=pip_options.find_links, + index_urls=index_urls, allow_all_prereleases=pip_options.pre, + trusted_hosts=pip_options.trusted_hosts, + process_dependency_links=pip_options.process_dependency_links, + session=session + ) + finder.add_dependency_links(dependency_links) + + for dist in packages: + typ = 'unknown' + all_candidates = finder.find_all_candidates(dist.key) + if not pip_options.pre: + # Remove prereleases + all_candidates = [ + candidate for candidate in all_candidates + if not candidate.version.is_prerelease + ] + + if not all_candidates: + continue + best_candidate = max(all_candidates, key=finder._candidate_sort_key) + remote_version = best_candidate.version + if best_candidate.location.is_wheel: + typ = 'wheel' + else: + typ = 'sdist' + # This is dirty but makes the rest of the code much cleaner + dist.latest_version = remote_version + dist.latest_filetype = typ + yield dist + + def get_outdated_packages(self): + return [ + pkg for pkg in self.get_package_info() + if pkg.latest_version._version > pkg.parsed_version._version + ] + @classmethod def _sanitize(cls, name): # Replace dangerous characters into '_'. The length of the sanitized @@ -975,13 +1043,19 @@ def proper_case_section(self, section): # Return whether or not values have been changed. return changed_values + @property + def py_version(self): + py_path = self.which("python") + version = python_version(py_path) + return version + @property def _pyversion(self): include_dir = vistir.compat.Path(self.virtualenv_location) / "include" python_path = next((x for x in include_dir.iterdir() if x.name.startswith("python")), None) if python_path: - python_version = python_path.name.replace("python", "") - py_version_short, abiflags = python_version[:3], python_version[3:] + py_version = python_path.name.replace("python", "") + py_version_short, abiflags = py_version[:3], py_version[3:] return {"py_version_short": py_version_short, "abiflags": abiflags} return {} @@ -990,8 +1064,10 @@ def env_paths(self): location = self.virtualenv_location if self.virtualenv_location else sys.prefix prefix = vistir.compat.Path(location) import importlib + py_version = tuple([int(v) for v in self.py_version.split(".")]) try: - _virtualenv = importlib.import_module("virtualenv") + with sys_version(py_version): + _virtualenv = importlib.import_module("virtualenv") except ImportError: with vistir.contextmanagers.temp_path(): from string import Formatter @@ -1015,9 +1091,11 @@ def env_paths(self): sys.path = [ os.path.join(sysconfig._INSTALL_SCHEMES[scheme][lib_key], "site-packages"), ] + sys.path - six.reload_module(importlib) - _virtualenv = importlib.import_module("virtualenv") - home, lib, inc, bin_ = _virtualenv.path_locations(prefix.absolute().as_posix()) + with sys_version(py_version): + six.reload_module(importlib) + _virtualenv = importlib.import_module("virtualenv") + with sys_version(py_version): + home, lib, inc, bin_ = _virtualenv.path_locations(prefix.absolute().as_posix()) paths = { "lib": lib, "include": inc, @@ -1031,8 +1109,10 @@ def env_paths(self): @cached_property def finders(self): from .vendor.pythonfinder import Finder + scripts_dirname = "Scripts" if os.name == "nt" else "bin" + scripts_dir = os.path.join(self.virtualenv_location, scripts_dirname) finders = [ - Finder(path=self.env_paths["scripts"], global_search=gs, system=False) + Finder(path=scripts_dir, global_search=gs, system=False) for gs in (False, True) ] return finders diff --git a/pipenv/utils.py b/pipenv/utils.py index c0979863c9..33e930d65f 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -372,7 +372,7 @@ def venv_resolve_deps( result = None while True: try: - result = c.expect(u"\n", timeout=-1) + result = c.expect(u"\n", timeout=environments.PIPENV_TIMEOUT) except (EOF, TIMEOUT): pass if result is None: @@ -1341,3 +1341,17 @@ def fix_venv_site(venv_lib_dir): fp.write(site_contents) # Make sure bytecode is up-to-date too. assert compileall.compile_file(str(site_py), quiet=1, force=True) + + +@contextmanager +def sys_version(version_tuple): + """ + Set a temporary sys.version_info tuple + + :param version_tuple: a fake sys.version_info tuple + """ + + old_version = sys.version_info + sys.version_info = version_tuple + yield + sys.version_info = old_version