Redesign sync and add it to add/remove/upgrade

More pip internals are now abstracted away in _pip. build_wheel now
returns a wheel object directly. Installation and uninstallation
operations are both wrapped for easy access.

With the new abstraction, installations are now prepared before any
of them are actually applied, so we are less likely to end up with
broken environments due to build failures.

Pipfile

@@ -13,7 +13,6 @@ twine = '*'
 wheel = '*'
 
 [scripts]
-passa = 'python -m passa'
 passa-add = 'python -m passa.cli.add'
 passa-remove = 'python -m passa.cli.remove'
 passa-upgrade = 'python -m passa.cli.upgrade'

news/20.feature.rst

@@ -0,0 +1 @@
+``sync`` is redisigned to be intergrated into ``add``, ``remove``, and ``upgrade``. Various ``clean`` operations are added to purge unneeded packages from the environment. ``install`` is added as a combination of ``lock`` and ``sync``.

src/passa/_pip.py

@@ -2,14 +2,21 @@
 
 from __future__ import absolute_import, unicode_literals
 
+import contextlib
+import distutils.log
 import os
 
+import setuptools.dist
+
+import distlib.scripts
+import distlib.wheel
 import pip_shims
 import six
 import vistir
 
 from ._pip_shims import VCS_SUPPORT, build_wheel as _build_wheel, unpack_url
 from .caches import CACHE_DIR
+from .utils import filter_sources
 
 
 @vistir.path.ensure_mkdir_p(mode=0o775)
@@ -129,7 +136,8 @@ def build_wheel(ireq, sources, hashes=None):
     If `hashes` is truthy, it is assumed to be a list of hashes (as formatted
     in Pipfile.lock) to be checked against the download.
 
-    Returns the wheel's path on disk, or None if the wheel cannot be built.
+    Returns a `distlib.wheel.Wheel` instance. Raises a `RuntimeError` if the
+    wheel cannot be built.
     """
     kwargs = _prepare_wheel_building_kwargs(ireq)
     finder = _get_finder(sources)
@@ -165,17 +173,19 @@ def build_wheel(ireq, sources, hashes=None):
             hashes=ireq.hashes(False), progress_bar=False,
         )
 
-    # If this is a wheel, use the downloaded thing.
     if ireq.is_wheel:
+        # If this is a wheel, use the downloaded thing.
         output_dir = kwargs["wheel_download_dir"]
-        return os.path.join(output_dir, ireq.link.filename)
-
-    # Othereise we need to build an ephemeral wheel.
-    wheel_path = _build_wheel(
-        ireq, vistir.path.create_tracked_tempdir(prefix="ephem"),
-        finder, _get_wheel_cache(), kwargs,
-    )
-    return wheel_path
+        wheel_path = os.path.join(output_dir, ireq.link.filename)
+    else:
+        # Othereise we need to build an ephemeral wheel.
+        wheel_path = _build_wheel(
+            ireq, vistir.path.create_tracked_tempdir(prefix="ephem"),
+            finder, _get_wheel_cache(), kwargs,
+        )
+        if wheel_path is None or not os.path.exists(wheel_path):
+            raise RuntimeError("failed to build wheel from {}".format(ireq))
+    return distlib.wheel.Wheel(wheel_path)
 
 
 def _obtrain_ref(vcs_obj, src_dir, name, rev=None):
@@ -201,3 +211,105 @@ def get_vcs_ref(requirement):
 def find_installation_candidates(ireq, sources):
     finder = _get_finder(sources)
     return finder.find_all_candidates(ireq.name)
+
+
+class RequirementUninstallation(object):
+    """A context manager to remove a package for the inner block.
+
+    This uses `UninstallPathSet` to control the workflow. If the inner block
+    exits correctly, the uninstallation is committed, otherwise rolled back.
+    """
+    def __init__(self, ireq, auto_confirm, verbose):
+        self.ireq = ireq
+        self.pathset = None
+        self.auto_confirm = auto_confirm
+        self.verbose = verbose
+
+    def __enter__(self):
+        self.pathset = self.ireq.uninstall(
+            auto_confirm=self.auto_confirm,
+            verbose=self.verbose,
+        )
+        return self.pathset
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        if self.pathset is None:
+            return
+        if exc_type is None:
+            self.pathset.commit()
+        else:
+            self.pathset.rollback()
+
+
+def uninstall_requirement(ireq, **kwargs):
+    return RequirementUninstallation(ireq, **kwargs)
+
+
+@contextlib.contextmanager
+def _suppress_distutils_logs():
+    """Hack to hide noise generated by `setup.py develop`.
+
+    There isn't a good way to suppress them now, so let's monky-patch.
+    See https://bugs.python.org/issue25392.
+    """
+    f = distutils.log.Log._log
+
+    def _log(log, level, msg, args):
+        if level >= distutils.log.ERROR:
+            f(log, level, msg, args)
+
+    distutils.log.Log._log = _log
+    yield
+    distutils.log.Log._log = f
+
+
+class NoopInstaller(object):
+    """An installer.
+
+    This class is not designed to be instantiated by itself, but used as a
+    common interface for subclassing.
+
+    An installer has two methods, `prepare()` and `install()`. Neither takes
+    arguments, and should be called in that order to prepare an installation
+    operation, and to actually install things.
+    """
+    def prepare(self):
+        pass
+
+    def install(self):
+        pass
+
+
+class EditableInstaller(NoopInstaller):
+    """Installer to handle editable.
+    """
+    def __init__(self, requirement):
+        ireq = requirement.as_ireq()
+        self.working_directory = ireq.setup_py_dir
+        self.setup_py = ireq.setup_py
+
+    def install(self):
+        with vistir.cd(self.working_directory), _suppress_distutils_logs():
+            # Access from Setuptools to ensure things are patched correctly.
+            setuptools.dist.distutils.core.run_setup(
+                self.setup_py, ["develop", "--no-deps"],
+            )
+
+
+class WheelInstaller(NoopInstaller):
+    """Installer by building a wheel.
+
+    The wheel is built during `prepare()`, and installed in `install()`.
+    """
+    def __init__(self, requirement, sources, paths):
+        self.ireq = requirement.as_ireq()
+        self.sources = filter_sources(requirement, sources)
+        self.hashes = requirement.hashes or None
+        self.paths = paths
+        self.wheel = None
+
+    def prepare(self):
+        self.wheel = build_wheel(self.ireq, self.sources, self.hashes)
+
+    def install(self):
+        self.wheel.install(self.paths, distlib.scripts.ScriptMaker(None, None))

src/passa/cli/add.py

@@ -27,6 +27,8 @@ def main(options):
             ), file=sys.stderr)
             return 2
 
+    prev_lockfile = project.lockfile
+
     locker = PinReuseLocker(project)
     success = lock(locker)
     if not success:
@@ -36,6 +38,26 @@ def main(options):
     project._l.write()
     print("Written to project at", project.root)
 
+    if not options.sync:
+        return
+
+    from passa.operations.sync import sync
+    from passa.synchronizers import Synchronizer
+
+    lockfile_diff = project.difference_lockfile(prev_lockfile)
+    default = bool(any(lockfile_diff.default))
+    develop = bool(any(lockfile_diff.develop))
+
+    syncer = Synchronizer(
+        project, default=default, develop=develop,
+        clean_unneeded=False,
+    )
+    success = sync(syncer)
+    if not success:
+        return 1
+
+    print("Synchronized project at", project.root)
+
 
 class Command(BaseCommand):
 
@@ -61,6 +83,11 @@ def add_arguments(self):
             action="store_true",
             help="add packages to [dev-packages]",
         )
+        self.parser.add_argument(
+            "--no-sync", dest="sync",
+            action="store_false", default=True,
+            help="do not synchronize the environment",
+        )
 
     def main(self, options):
         if not options.editable_lines and not options.requirement_lines:

src/passa/cli/clean.py

@@ -0,0 +1,38 @@
+# -*- coding=utf-8 -*-
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from ._base import BaseCommand
+
+
+def main(options):
+    from passa.operations.sync import clean
+    from passa.synchronizers import Cleaner
+
+    project = options.project
+    cleaner = Cleaner(project, default=True, develop=options.dev)
+
+    success = clean(cleaner)
+    if not success:
+        return 1
+
+    print("Cleaned project at", project.root)
+
+
+class Command(BaseCommand):
+
+    name = "clean"
+    description = "Uninstall unlisted packages from the current environment."
+    parsed_main = main
+
+    def add_arguments(self):
+        super(Command, self).add_arguments()
+        self.parser.add_argument(
+            "--no-dev", dest="dev",
+            action="store_false", default=True,
+            help="uninstall develop packages, only keep default ones",
+        )
+
+
+if __name__ == "__main__":
+    Command.run_current_module()

src/passa/cli/install.py

@@ -0,0 +1,63 @@
+# -*- coding=utf-8 -*-
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from ._base import BaseCommand
+
+
+def main(options):
+    from passa.lockers import BasicLocker
+    from passa.operations.lock import lock
+
+    project = options.project
+
+    if not options.check or not project.is_synced():
+        locker = BasicLocker(project)
+        success = lock(locker)
+        if not success:
+            return 1
+        project._l.write()
+        print("Written to project at", project.root)
+
+    from passa.operations.sync import sync
+    from passa.synchronizers import Synchronizer
+
+    syncer = Synchronizer(
+        project, default=True, develop=options.dev,
+        clean_unneeded=options.clean,
+    )
+
+    success = sync(syncer)
+    if not success:
+        return 1
+
+    print("Synchronized project at", project.root)
+
+
+class Command(BaseCommand):
+
+    name = "install"
+    description = "Generate Pipfile.lock to synchronize the environment."
+    parsed_main = main
+
+    def add_arguments(self):
+        super(Command, self).add_arguments()
+        self.parser.add_argument(
+            "--no-check", dest="check",
+            action="store_false", default=True,
+            help="do not check if Pipfile.lock is update, always resolve",
+        )
+        self.parser.add_argument(
+            "--dev",
+            action="store_true",
+            help="install develop packages",
+        )
+        self.parser.add_argument(
+            "--no-clean", dest="clean",
+            action="store_false", default=True,
+            help="do not uninstall packages not specified in Pipfile.lock",
+        )
+
+
+if __name__ == "__main__":
+    Command.run_current_module()

src/passa/cli/remove.py

@@ -26,6 +26,19 @@ def main(options):
     project._l.write()
     print("Written to project at", project.root)
 
+    if not options.clean:
+        return
+
+    from passa.operations.sync import clean
+    from passa.synchronizers import Cleaner
+
+    cleaner = Cleaner(project, default=True, develop=True)
+    success = clean(cleaner)
+    if not success:
+        return 1
+
+    print("Cleaned project at", project.root)
+
 
 class Command(BaseCommand):
 
@@ -51,6 +64,11 @@ def add_arguments(self):
             action="store_const", const="default",
             help="only try to remove from [packages]",
         )
+        self.parser.add_argument(
+            "--no-clean", dest="clean",
+            action="store_false", default=True,
+            help="do not uninstall packages not specified in Pipfile.lock",
+        )
 
 
 if __name__ == "__main__":

src/passa/cli/sync.py

@@ -10,7 +10,10 @@ def main(options):
     from passa.synchronizers import Synchronizer
 
     project = options.project
-    syncer = Synchronizer(project, default=True, develop=options.dev)
+    syncer = Synchronizer(
+        project, default=True, develop=options.dev,
+        clean_unneeded=options.clean,
+    )
 
     success = sync(syncer)
     if not success:
@@ -32,6 +35,11 @@ def add_arguments(self):
             action="store_true",
             help="install develop packages",
         )
+        self.parser.add_argument(
+            "--no-clean", dest="clean",
+            action="store_false", default=True,
+            help="do not uninstall packages not specified in Pipfile.lock",
+        )
 
 
 if __name__ == "__main__":