diff --git a/Pipfile b/Pipfile index 18d80d1..25ed7ee 100644 --- a/Pipfile +++ b/Pipfile @@ -13,7 +13,6 @@ twine = '*' wheel = '*' [scripts] -passa = 'python -m passa' passa-add = 'python -m passa.cli.add' passa-remove = 'python -m passa.cli.remove' passa-upgrade = 'python -m passa.cli.upgrade' diff --git a/news/20.feature.rst b/news/20.feature.rst new file mode 100644 index 0000000..d6868c5 --- /dev/null +++ b/news/20.feature.rst @@ -0,0 +1 @@ +``sync`` is redisigned to be intergrated into ``add``, ``remove``, and ``upgrade``. Various ``clean`` operations are added to purge unneeded packages from the environment. ``install`` is added as a combination of ``lock`` and ``sync``. diff --git a/src/passa/_pip.py b/src/passa/_pip.py index 1715ec2..5cf1cea 100644 --- a/src/passa/_pip.py +++ b/src/passa/_pip.py @@ -2,14 +2,21 @@ from __future__ import absolute_import, unicode_literals +import contextlib +import distutils.log import os +import setuptools.dist + +import distlib.scripts +import distlib.wheel import pip_shims import six import vistir from ._pip_shims import VCS_SUPPORT, build_wheel as _build_wheel, unpack_url from .caches import CACHE_DIR +from .utils import filter_sources @vistir.path.ensure_mkdir_p(mode=0o775) @@ -129,7 +136,8 @@ def build_wheel(ireq, sources, hashes=None): If `hashes` is truthy, it is assumed to be a list of hashes (as formatted in Pipfile.lock) to be checked against the download. - Returns the wheel's path on disk, or None if the wheel cannot be built. + Returns a `distlib.wheel.Wheel` instance. Raises a `RuntimeError` if the + wheel cannot be built. """ kwargs = _prepare_wheel_building_kwargs(ireq) finder = _get_finder(sources) @@ -165,17 +173,19 @@ def build_wheel(ireq, sources, hashes=None): hashes=ireq.hashes(False), progress_bar=False, ) - # If this is a wheel, use the downloaded thing. if ireq.is_wheel: + # If this is a wheel, use the downloaded thing. output_dir = kwargs["wheel_download_dir"] - return os.path.join(output_dir, ireq.link.filename) - - # Othereise we need to build an ephemeral wheel. - wheel_path = _build_wheel( - ireq, vistir.path.create_tracked_tempdir(prefix="ephem"), - finder, _get_wheel_cache(), kwargs, - ) - return wheel_path + wheel_path = os.path.join(output_dir, ireq.link.filename) + else: + # Othereise we need to build an ephemeral wheel. + wheel_path = _build_wheel( + ireq, vistir.path.create_tracked_tempdir(prefix="ephem"), + finder, _get_wheel_cache(), kwargs, + ) + if wheel_path is None or not os.path.exists(wheel_path): + raise RuntimeError("failed to build wheel from {}".format(ireq)) + return distlib.wheel.Wheel(wheel_path) def _obtrain_ref(vcs_obj, src_dir, name, rev=None): @@ -201,3 +211,105 @@ def get_vcs_ref(requirement): def find_installation_candidates(ireq, sources): finder = _get_finder(sources) return finder.find_all_candidates(ireq.name) + + +class RequirementUninstallation(object): + """A context manager to remove a package for the inner block. + + This uses `UninstallPathSet` to control the workflow. If the inner block + exits correctly, the uninstallation is committed, otherwise rolled back. + """ + def __init__(self, ireq, auto_confirm, verbose): + self.ireq = ireq + self.pathset = None + self.auto_confirm = auto_confirm + self.verbose = verbose + + def __enter__(self): + self.pathset = self.ireq.uninstall( + auto_confirm=self.auto_confirm, + verbose=self.verbose, + ) + return self.pathset + + def __exit__(self, exc_type, exc_value, traceback): + if self.pathset is None: + return + if exc_type is None: + self.pathset.commit() + else: + self.pathset.rollback() + + +def uninstall_requirement(ireq, **kwargs): + return RequirementUninstallation(ireq, **kwargs) + + +@contextlib.contextmanager +def _suppress_distutils_logs(): + """Hack to hide noise generated by `setup.py develop`. + + There isn't a good way to suppress them now, so let's monky-patch. + See https://bugs.python.org/issue25392. + """ + f = distutils.log.Log._log + + def _log(log, level, msg, args): + if level >= distutils.log.ERROR: + f(log, level, msg, args) + + distutils.log.Log._log = _log + yield + distutils.log.Log._log = f + + +class NoopInstaller(object): + """An installer. + + This class is not designed to be instantiated by itself, but used as a + common interface for subclassing. + + An installer has two methods, `prepare()` and `install()`. Neither takes + arguments, and should be called in that order to prepare an installation + operation, and to actually install things. + """ + def prepare(self): + pass + + def install(self): + pass + + +class EditableInstaller(NoopInstaller): + """Installer to handle editable. + """ + def __init__(self, requirement): + ireq = requirement.as_ireq() + self.working_directory = ireq.setup_py_dir + self.setup_py = ireq.setup_py + + def install(self): + with vistir.cd(self.working_directory), _suppress_distutils_logs(): + # Access from Setuptools to ensure things are patched correctly. + setuptools.dist.distutils.core.run_setup( + self.setup_py, ["develop", "--no-deps"], + ) + + +class WheelInstaller(NoopInstaller): + """Installer by building a wheel. + + The wheel is built during `prepare()`, and installed in `install()`. + """ + def __init__(self, requirement, sources, paths): + self.ireq = requirement.as_ireq() + self.sources = filter_sources(requirement, sources) + self.hashes = requirement.hashes or None + self.paths = paths + self.wheel = None + + def prepare(self): + self.wheel = build_wheel(self.ireq, self.sources, self.hashes) + + def install(self): + self.wheel.install(self.paths, distlib.scripts.ScriptMaker(None, None)) diff --git a/src/passa/cli/add.py b/src/passa/cli/add.py index 3055f6e..1ea0f09 100644 --- a/src/passa/cli/add.py +++ b/src/passa/cli/add.py @@ -27,6 +27,8 @@ def main(options): ), file=sys.stderr) return 2 + prev_lockfile = project.lockfile + locker = PinReuseLocker(project) success = lock(locker) if not success: @@ -36,6 +38,26 @@ def main(options): project._l.write() print("Written to project at", project.root) + if not options.sync: + return + + from passa.operations.sync import sync + from passa.synchronizers import Synchronizer + + lockfile_diff = project.difference_lockfile(prev_lockfile) + default = bool(any(lockfile_diff.default)) + develop = bool(any(lockfile_diff.develop)) + + syncer = Synchronizer( + project, default=default, develop=develop, + clean_unneeded=False, + ) + success = sync(syncer) + if not success: + return 1 + + print("Synchronized project at", project.root) + class Command(BaseCommand): @@ -61,6 +83,11 @@ def add_arguments(self): action="store_true", help="add packages to [dev-packages]", ) + self.parser.add_argument( + "--no-sync", dest="sync", + action="store_false", default=True, + help="do not synchronize the environment", + ) def main(self, options): if not options.editable_lines and not options.requirement_lines: diff --git a/src/passa/cli/clean.py b/src/passa/cli/clean.py new file mode 100644 index 0000000..90dbe73 --- /dev/null +++ b/src/passa/cli/clean.py @@ -0,0 +1,38 @@ +# -*- coding=utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + +from ._base import BaseCommand + + +def main(options): + from passa.operations.sync import clean + from passa.synchronizers import Cleaner + + project = options.project + cleaner = Cleaner(project, default=True, develop=options.dev) + + success = clean(cleaner) + if not success: + return 1 + + print("Cleaned project at", project.root) + + +class Command(BaseCommand): + + name = "clean" + description = "Uninstall unlisted packages from the current environment." + parsed_main = main + + def add_arguments(self): + super(Command, self).add_arguments() + self.parser.add_argument( + "--no-dev", dest="dev", + action="store_false", default=True, + help="uninstall develop packages, only keep default ones", + ) + + +if __name__ == "__main__": + Command.run_current_module() diff --git a/src/passa/cli/install.py b/src/passa/cli/install.py new file mode 100644 index 0000000..47667e4 --- /dev/null +++ b/src/passa/cli/install.py @@ -0,0 +1,63 @@ +# -*- coding=utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + +from ._base import BaseCommand + + +def main(options): + from passa.lockers import BasicLocker + from passa.operations.lock import lock + + project = options.project + + if not options.check or not project.is_synced(): + locker = BasicLocker(project) + success = lock(locker) + if not success: + return 1 + project._l.write() + print("Written to project at", project.root) + + from passa.operations.sync import sync + from passa.synchronizers import Synchronizer + + syncer = Synchronizer( + project, default=True, develop=options.dev, + clean_unneeded=options.clean, + ) + + success = sync(syncer) + if not success: + return 1 + + print("Synchronized project at", project.root) + + +class Command(BaseCommand): + + name = "install" + description = "Generate Pipfile.lock to synchronize the environment." + parsed_main = main + + def add_arguments(self): + super(Command, self).add_arguments() + self.parser.add_argument( + "--no-check", dest="check", + action="store_false", default=True, + help="do not check if Pipfile.lock is update, always resolve", + ) + self.parser.add_argument( + "--dev", + action="store_true", + help="install develop packages", + ) + self.parser.add_argument( + "--no-clean", dest="clean", + action="store_false", default=True, + help="do not uninstall packages not specified in Pipfile.lock", + ) + + +if __name__ == "__main__": + Command.run_current_module() diff --git a/src/passa/cli/remove.py b/src/passa/cli/remove.py index a86f4f9..b2e0399 100644 --- a/src/passa/cli/remove.py +++ b/src/passa/cli/remove.py @@ -26,6 +26,19 @@ def main(options): project._l.write() print("Written to project at", project.root) + if not options.clean: + return + + from passa.operations.sync import clean + from passa.synchronizers import Cleaner + + cleaner = Cleaner(project, default=True, develop=True) + success = clean(cleaner) + if not success: + return 1 + + print("Cleaned project at", project.root) + class Command(BaseCommand): @@ -51,6 +64,11 @@ def add_arguments(self): action="store_const", const="default", help="only try to remove from [packages]", ) + self.parser.add_argument( + "--no-clean", dest="clean", + action="store_false", default=True, + help="do not uninstall packages not specified in Pipfile.lock", + ) if __name__ == "__main__": diff --git a/src/passa/cli/sync.py b/src/passa/cli/sync.py index a01138e..d2d9dcc 100644 --- a/src/passa/cli/sync.py +++ b/src/passa/cli/sync.py @@ -10,7 +10,10 @@ def main(options): from passa.synchronizers import Synchronizer project = options.project - syncer = Synchronizer(project, default=True, develop=options.dev) + syncer = Synchronizer( + project, default=True, develop=options.dev, + clean_unneeded=options.clean, + ) success = sync(syncer) if not success: @@ -32,6 +35,11 @@ def add_arguments(self): action="store_true", help="install develop packages", ) + self.parser.add_argument( + "--no-clean", dest="clean", + action="store_false", default=True, + help="do not uninstall packages not specified in Pipfile.lock", + ) if __name__ == "__main__": diff --git a/src/passa/cli/upgrade.py b/src/passa/cli/upgrade.py index 4ebe219..bfa342f 100644 --- a/src/passa/cli/upgrade.py +++ b/src/passa/cli/upgrade.py @@ -22,6 +22,8 @@ def main(options): project.remove_entries_from_lockfile(packages) + prev_lockfile = project.lockfile + if options.strategy == "eager": locker = EagerUpgradeLocker(project, packages) else: @@ -33,6 +35,26 @@ def main(options): project._l.write() print("Written to project at", project.root) + if not options.sync: + return + + from passa.operations.sync import sync + from passa.synchronizers import Synchronizer + + lockfile_diff = project.difference_lockfile(prev_lockfile) + default = bool(any(lockfile_diff.default)) + develop = bool(any(lockfile_diff.develop)) + + syncer = Synchronizer( + project, default=default, develop=develop, + clean_unneeded=False, + ) + success = sync(syncer) + if not success: + return 1 + + print("Synchronized project at", project.root) + class Command(BaseCommand): @@ -52,6 +74,16 @@ def add_arguments(self): default="only-if-needed", help="how dependency upgrading is handled", ) + self.parser.add_argument( + "--no-sync", dest="sync", + action="store_false", default=True, + help="do not synchronize the environment", + ) + self.parser.add_argument( + "--no-clean", dest="clean", + action="store_false", default=True, + help="do not uninstall packages not specified in Pipfile.lock", + ) if __name__ == "__main__": diff --git a/src/passa/dependencies.py b/src/passa/dependencies.py index a6dbdb5..8edf2fd 100644 --- a/src/passa/dependencies.py +++ b/src/passa/dependencies.py @@ -6,7 +6,6 @@ import os import sys -import distlib.wheel import packaging.specifiers import packaging.utils import packaging.version @@ -216,10 +215,7 @@ def _get_dependencies_from_pip(ireq, sources): The current strategy is to build a wheel out of the ireq, and read metadata out of it. """ - wheel_path = build_wheel(ireq, sources) - if not wheel_path or not os.path.exists(wheel_path): - raise RuntimeError("failed to build wheel from {}".format(ireq)) - wheel = distlib.wheel.Wheel(wheel_path) + wheel = build_wheel(ireq, sources) extras = ireq.extras or () requirements = _read_requirements(wheel.metadata, extras) requires_python = _read_requires_python(wheel.metadata) diff --git a/src/passa/lockers.py b/src/passa/lockers.py index 4dec70a..4ab4cc3 100644 --- a/src/passa/lockers.py +++ b/src/passa/lockers.py @@ -154,6 +154,8 @@ def __init__(self, project): super(PinReuseLocker, self).__init__(project) pins = _get_requirements(project.lockfile, "develop") pins.update(_get_requirements(project.lockfile, "default")) + for pin in pins.values(): + pin.markers = None self.preferred_pins = pins def get_provider(self): diff --git a/src/passa/operations/sync.py b/src/passa/operations/sync.py index 31ffe76..3014e8d 100644 --- a/src/passa/operations/sync.py +++ b/src/passa/operations/sync.py @@ -5,13 +5,19 @@ def sync(syncer): print("Starting synchronization") - installed, updated, skipped, cleaned = syncer.sync() + installed, updated, cleaned = syncer.sync() if cleaned: - print("Removed: {}".format(", ".join(sorted(cleaned)))) + print("Uninstalled: {}".format(", ".join(sorted(cleaned)))) if installed: print("Installed: {}".format(", ".join(sorted(installed)))) if updated: print("Updated: {}".format(", ".join(sorted(updated)))) - if skipped: - print("Skipped: {}".format(", ".join(sorted(skipped)))) + return True + + +def clean(cleaner): + print("Cleaning") + cleaned = cleaner.clean() + if cleaned: + print("Uninstalled: {}".format(", ".join(sorted(cleaned)))) return True diff --git a/src/passa/projects.py b/src/passa/projects.py index 8281625..79e71bb 100644 --- a/src/passa/projects.py +++ b/src/passa/projects.py @@ -1,7 +1,13 @@ +# -*- coding=utf-8 -*- + +from __future__ import absolute_import, unicode_literals + +import collections import io import os import attr +import packaging.markers import packaging.utils import plette import plette.models @@ -9,6 +15,30 @@ import tomlkit +SectionDifference = collections.namedtuple("SectionDifference", [ + "inthis", "inthat", +]) +FileDifference = collections.namedtuple("FileDifference", [ + "default", "develop", +]) + + +def _are_pipfile_entries_equal(a, b): + a = {k: v for k, v in a.items() if k not in ("markers", "hashes", "hash")} + b = {k: v for k, v in b.items() if k not in ("markers", "hashes", "hash")} + if a != b: + return False + try: + marker_eval_a = packaging.markers.Marker(a["markers"]).evaluate() + except (AttributeError, KeyError, TypeError, ValueError): + marker_eval_a = True + try: + marker_eval_b = packaging.markers.Marker(b["markers"]).evaluate() + except (AttributeError, KeyError, TypeError, ValueError): + marker_eval_b = True + return marker_eval_a == marker_eval_b + + DEFAULT_NEWLINES = "\n" @@ -169,3 +199,37 @@ def remove_keys_from_lockfile(self, keys): # HACK: The lock file no longer represents the Pipfile at this # point. Set the hash to an arbitrary invalid value. self.lockfile.meta.hash = plette.models.Hash({"__invalid__": ""}) + + def difference_lockfile(self, lockfile): + """Generate a difference between the current and given lockfiles. + + Returns a 2-tuple containing differences in default in develop + sections. + + Each element is a 2-tuple of dicts. The first, `inthis`, contains + entries only present in the current lockfile; the second, `inthat`, + contains entries only present in the given one. + + If a key exists in both this and that, but the values differ, the key + is present in both dicts, pointing to values from each file. + """ + diff_data = { + "default": SectionDifference({}, {}), + "develop": SectionDifference({}, {}), + } + for section_name, section_diff in diff_data.items(): + this = self.lockfile[section_name]._data + that = lockfile[section_name]._data + for key, this_value in this.items(): + try: + that_value = that[key] + except KeyError: + section_diff.inthis[key] = this_value + continue + if not _are_pipfile_entries_equal(this_value, that_value): + section_diff.inthis[key] = this_value + section_diff.inthat[key] = that_value + for key, that_value in that.items(): + if key not in this: + section_diff.inthat[key] = that_value + return FileDifference(**diff_data) diff --git a/src/passa/reporters.py b/src/passa/reporters.py index d9c60f0..4fe6c0b 100644 --- a/src/passa/reporters.py +++ b/src/passa/reporters.py @@ -84,6 +84,7 @@ def ending(self, state): continue print(' ', end='') for v in reversed(path[1:]): - print(' <=', state.mapping[v].as_line(), end='') + line = state.mapping[v].as_line(include_hashes=False) + print(' <=', line, end='') print() print() diff --git a/src/passa/synchronizers.py b/src/passa/synchronizers.py index 17bee71..7481e8d 100644 --- a/src/passa/synchronizers.py +++ b/src/passa/synchronizers.py @@ -2,132 +2,85 @@ from __future__ import absolute_import, unicode_literals +import collections import contextlib -import distutils.log import os import sys import sysconfig -import distlib.scripts -import distlib.wheel +import pkg_resources + import packaging.markers -import packaging.utils import packaging.version -import pkg_resources import requirementslib -import setuptools.dist -import vistir -from ._pip import build_wheel -from .utils import filter_sources - - -def _distutils_log_wrapped(log, level, msg, args): - if level < distutils.log.ERROR: - return - distutils.log.Log._log(log, level, msg, args) +from ._pip import uninstall_requirement, EditableInstaller, WheelInstaller -@contextlib.contextmanager -def _suppress_distutils_logs(): - f = distutils.log.Log._log - distutils.log.Log._log = _distutils_log_wrapped - yield - distutils.log.Log._log = f - +def _is_installation_local(name): + """Check whether the distribution is in the current Python installation. -def _build_paths(): - """Prepare paths for distlib.wheel.Wheel to install into. + This is used to distinguish packages seen by a virtual environment. A venv + may be able to see global packages, but we don't want to mess with them. """ - paths = sysconfig.get_paths() - return { - "prefix": sys.prefix, - "data": paths["data"], - "scripts": paths["scripts"], - "headers": paths["include"], - "purelib": paths["purelib"], - "platlib": paths["platlib"], - } - - -def _install_as_editable(requirement): - ireq = requirement.as_ireq() - with vistir.cd(ireq.setup_py_dir), _suppress_distutils_logs(): - # Access from Setuptools to make sure we have currect patches. - setuptools.dist.distutils.core.run_setup( - ireq.setup_py, ["--quiet", "develop", "--no-deps"], - ) - - -def _install_as_wheel(requirement, sources, paths): - ireq = requirement.as_ireq() - sources = filter_sources(requirement, sources) - hashes = requirement.hashes or None - # TODO: Provide some sort of cache so we don't need to build each ephemeral - # wheels twice if lock and sync is done in the same process. - wheel_path = build_wheel(ireq, sources, hashes) - if not wheel_path or not os.path.exists(wheel_path): - raise RuntimeError("failed to build wheel from {}".format(ireq)) - wheel = distlib.wheel.Wheel(wheel_path) - wheel.install(paths, distlib.scripts.ScriptMaker(None, None)) + location = pkg_resources.working_set.by_key[name].location + return os.path.commonprefix([location, sys.prefix]) == sys.prefix -PROTECTED_FROM_CLEAN = {"setuptools", "pip"} - +def _is_up_to_date(distro, version): + # This is done in strings to avoid type mismatches caused by vendering. + return str(version) == str(packaging.version.parse(distro.version)) -def _should_uninstall(name, distro, whitelist, for_sync): - if name in PROTECTED_FROM_CLEAN: - return False - try: - package = whitelist[name] - except KeyError: - return True - if not for_sync: - return False - r = requirementslib.Requirement.from_pipfile(name, package) +GroupCollection = collections.namedtuple("GroupCollection", [ + "uptodate", "outdated", "noremove", "unneeded", +]) - # Always remove and re-sync non-named requirements. pip does this? - if not r.is_named: - return True - # Remove packages with unmatched version. The comparison is done is - # strings to avoid type mismatching due to vendering. - if str(r.get_version()) != str(packaging.version.parse(distro.version)): - return True +def _group_installed_names(packages): + """Group locally installed packages based on given specifications. + `packages` is a name-package mapping that are used as baseline to + determine how the installed package should be grouped. -def _is_installation_local(distro): - """Check whether the distribution is in the current Python installation. + Returns a 3-tuple of disjoint sets, all containing names of installed + packages: - This is used to distinguish packages seen by a virtual environment. A venv - may be able to see global packages, but we don't want to mess with them. + * `uptodate`: These match the specifications. + * `outdated`: These installations are specified, but don't match the + specifications in `packages`. + * `unneeded`: These are installed, but not specified in `packages`. """ - return os.path.commonprefix([distro.location, sys.prefix]) == sys.prefix + groupcoll = GroupCollection(set(), set(), set(), set()) - -def _group_installed_names(whitelist, for_sync): - names_to_clean = set() - names_kept = set() for distro in pkg_resources.working_set: - name = packaging.utils.canonicalize_name(distro.project_name) - if (_should_uninstall(name, distro, whitelist, for_sync) and - _is_installation_local(distro)): - names_to_clean.add(name) + name = distro.key + try: + package = packages[name] + except KeyError: + groupcoll.unneeded.add(name) + continue + + r = requirementslib.Requirement.from_pipfile(name, package) + if not r.is_named: + # Always mark non-named. I think pip does something similar? + groupcoll.outdated.add(name) + elif not _is_up_to_date(distro, r.get_version()): + groupcoll.outdated.add(name) else: - names_kept.add(name) - return names_to_clean, names_kept + groupcoll.uptodate.add(name) + return groupcoll -def _clean(whitelist, for_sync): - names_to_clean, names_kept = _group_installed_names(whitelist, for_sync) - # TODO: Show a prompt to confirm cleaning. We will need to implement a - # reporter pattern for this as well. - for name in names_to_clean: - r = requirementslib.Requirement.from_line(name) - r.as_ireq().uninstall(auto_confirm=True, verbose=False) - return names_to_clean, names_kept +@contextlib.contextmanager +def _remove_package(name): + if name is None or not _is_installation_local(name): + yield + return + r = requirementslib.Requirement.from_line(name) + with uninstall_requirement(r.as_ireq(), auto_confirm=True, verbose=False): + yield def _get_packages(lockfile, default, develop): @@ -142,53 +95,116 @@ def _get_packages(lockfile, default, develop): return packages +def _build_paths(): + """Prepare paths for distlib.wheel.Wheel to install into. + """ + paths = sysconfig.get_paths() + return { + "prefix": sys.prefix, + "data": paths["data"], + "scripts": paths["scripts"], + "headers": paths["include"], + "purelib": paths["purelib"], + "platlib": paths["platlib"], + } + + +PROTECTED_FROM_CLEAN = {"setuptools", "pip"} + + +def _clean(names): + for name in names: + if name in PROTECTED_FROM_CLEAN: + continue + with _remove_package(name): + pass + + class Synchronizer(object): """Helper class to install packages from a project's lock file. """ - def __init__(self, project, default, develop): + def __init__(self, project, default, develop, clean_unneeded): self._root = project.root # Only for repr. self.packages = _get_packages(project.lockfile, default, develop) self.sources = project.lockfile.meta.sources._data self.paths = _build_paths() + self.clean_unneeded = clean_unneeded def __repr__(self): return "<{0} @ {1!r}>".format(type(self).__name__, self._root) def sync(self): - cleaned_names, installed_names = _clean(self.packages, for_sync=True) - # TODO: Specify installation order? (pypa/pipenv#2274) + groupcoll = _group_installed_names(self.packages) installed = set() updated = set() - skipped = set() + cleaned = set() + + # TODO: Show a prompt to confirm cleaning. We will need to implement a + # reporter pattern for this as well. + if self.clean_unneeded: + cleaned.update(groupcoll.unneeded) + _clean(cleaned) + + # TODO: Specify installation order? (pypa/pipenv#2274) + installers = [] for name, package in self.packages.items(): - if name in installed_names: - continue r = requirementslib.Requirement.from_pipfile(name, package) + name = r.normalized_name + if name in groupcoll.uptodate: + continue markers = r.markers if markers and not packaging.markers.Marker(markers).evaluate(): - skipped.add(r.normalized_name) continue + if r.editable: + installer = EditableInstaller(r) + else: + installer = WheelInstaller(r, self.sources, self.paths) try: - if r.editable: - _install_as_editable(r) - else: - _install_as_wheel(r, self.sources, self.paths) + installer.prepare() + except Exception as e: + if os.environ.get("PASSA_NO_SUPPRESS_EXCEPTIONS"): + raise + print("failed to prepare {0!r}: {1}".format( + r.as_line(include_hashes=False), e, + )) + else: + installers.append((name, installer)) + + for name, installer in installers: + if name in groupcoll.outdated: + name_to_remove = name + else: + name_to_remove = None + try: + with _remove_package(name_to_remove): + installer.install() except Exception as e: if os.environ.get("PASSA_NO_SUPPRESS_EXCEPTIONS"): raise print("failed to install {0!r}: {1}".format( r.as_line(include_hashes=False), e, )) - name = r.normalized_name - if name in cleaned_names: + continue + if name in groupcoll.outdated or name in groupcoll.noremove: updated.add(name) - cleaned_names.remove(name) else: installed.add(name) - return installed, updated, skipped, cleaned_names + return installed, updated, cleaned + + +class Cleaner(object): + """Helper class to clean packages not in a project's lock file. + """ + def __init__(self, project, default, develop): + self._root = project.root # Only for repr. + self.packages = _get_packages(project.lockfile, default, develop) + + def __repr__(self): + return "<{0} @ {1!r}>".format(type(self).__name__, self._root) def clean(self): - cleaned_names, kept_names = _clean(self.packages, for_sync=False) - return cleaned_names, kept_names + groupcoll = _group_installed_names(self.packages) + _clean(groupcoll.unneeded) + return groupcoll.unneeded