Issue #2675: Granular control over wheels/sdists

With wheel autobuilding in place a release blocker is some granular
way to opt-out of wheels for known-bad packages. This patch introduces
two new options: --no-binary and --only-binary to control what
archives we are willing to use on both a global and per-package basis.

This also closes #2084

CHANGES.txt

@@ -25,6 +25,9 @@
 * Build Wheels prior to installing from sdist, caching them in the pip cache
   directory to speed up subsequent installs. (:pull:`2618`)
 
+* Allow fine grained control over the use of wheels and source builds.
+  (:pull:`2699`)
+
 **6.1.1 (2015-04-07)**
 
 * No longer ignore dependencies which have been added to the standard library,

docs/reference/pip_install.rst

@@ -112,7 +112,8 @@ Additionally, the following Package Index Options are supported:
   *  :ref:`--allow-external <--allow-external>`
   *  :ref:`--allow-all-external <--allow-external>`
   *  :ref:`--allow-unverified <--allow-unverified>`
-  *  :ref:`--no-use-wheel <install_--no-use-wheel>`
+  *  :ref:`--no-binary <install_--no-binary>`
+  *  :ref:`--only-binary <install_--only-binary>`
 
 For example, to specify :ref:`--no-index <--no-index>` and 2 :ref:`--find-links <--find-links>` locations:
 

docs/user_guide.rst

@@ -123,7 +123,7 @@ to building and installing from source archives. For more information, see the
 `PEP425 <http://www.python.org/dev/peps/pep-0425>`_
 
 Pip prefers Wheels where they are available. To disable this, use the
-:ref:`--no-use-wheel <install_--no-use-wheel>` flag for :ref:`pip install`.
+:ref:`--no-binary <install_--no-binary>` flag for :ref:`pip install`.
 
 If no satisfactory wheels are found, pip will default to finding source archives.
 

pip/cmdoptions.py

@@ -11,7 +11,9 @@
 
 from functools import partial
 from optparse import OptionGroup, SUPPRESS_HELP, Option
-from pip.index import PyPI
+
+from pip.index import (
+    PyPI, FormatControl, fmt_ctl_handle_mutual_exclude, fmt_ctl_no_use_wheel)
 from pip.locations import CA_BUNDLE_PATH, USER_CACHE_DIR, src_prefix
 
 
@@ -27,6 +29,12 @@ def make_option_group(group, parser):
     return option_group
 
 
+def resolve_wheel_no_use_binary(options):
+    if not options.use_wheel:
+        control = options.format_control
+        fmt_ctl_no_use_wheel(control)
+
+
 ###########
 # options #
 ###########
@@ -339,6 +347,7 @@ def editable():
     'The default for global installs is "<current dir>/src".'
 )
 
+# XXX: deprecated, remove in 9.0
 use_wheel = partial(
     Option,
     '--use-wheel',
@@ -354,9 +363,53 @@ def editable():
     action='store_false',
     default=True,
     help=('Do not Find and prefer wheel archives when searching indexes and '
-          'find-links locations.'),
+          'find-links locations. DEPRECATED in favour of --no-binary.'),
 )
 
+
+def _get_format_control(values, option):
+    """Get a format_control object."""
+    return getattr(values, option.dest)
+
+
+def _handle_no_binary(option, opt_str, value, parser):
+    existing = getattr(parser.values, option.dest)
+    fmt_ctl_handle_mutual_exclude(
+        value, existing.no_binary, existing.only_binary)
+
+
+def _handle_only_binary(option, opt_str, value, parser):
+    existing = getattr(parser.values, option.dest)
+    fmt_ctl_handle_mutual_exclude(
+        value, existing.only_binary, existing.no_binary)
+
+
+def no_binary():
+    return Option(
+        "--no-binary", dest="format_control", action="callback",
+        callback=_handle_no_binary, type="str",
+        default=FormatControl(set(), set()),
+        help="Do not use binary packages. Can be supplied multiple times, and "
+             "each time adds to the existing value. Accepts either :all: to "
+             "disable all binary packages, :none: to empty the set, or one or "
+             "more package names with commas between them. Note that some "
+             "packages are tricky to compile and may fail to install when "
+             "this option is used on them.")
+
+
+def only_binary():
+    return Option(
+        "--only-binary", dest="format_control", action="callback",
+        callback=_handle_only_binary, type="str",
+        default=FormatControl(set(), set()),
+        help="Do not use source packages. Can be supplied multiple times, and "
+             "each time adds to the existing value. Accepts either :all: to "
+             "disable all source packages, :none: to empty the set, or one or "
+             "more package names with commas between them. Packages without "
+             "binary distributions will fail to install when this option is "
+             "used on them.")
+
+
 cache_dir = partial(
     Option,
     "--cache-dir",

pip/commands/freeze.py

@@ -2,6 +2,7 @@
 
 import sys
 
+import pip
 from pip.basecommand import Command
 from pip.operations.freeze import freeze
 from pip.wheel import WheelCache
@@ -55,7 +56,8 @@ def __init__(self, *args, **kw):
         self.parser.insert_option_group(0, self.cmd_opts)
 
     def run(self, options, args):
-        wheel_cache = WheelCache(options.cache_dir)
+        format_control = pip.index.FormatControl(set(), set())
+        wheel_cache = WheelCache(options.cache_dir, format_control)
         freeze_kwargs = dict(
             requirement=options.requirement,
             find_links=options.find_links,

pip/commands/install.py

@@ -153,6 +153,8 @@ def __init__(self, *args, **kw):
 
         cmd_opts.add_option(cmdoptions.use_wheel())
         cmd_opts.add_option(cmdoptions.no_use_wheel())
+        cmd_opts.add_option(cmdoptions.no_binary())
+        cmd_opts.add_option(cmdoptions.only_binary())
 
         cmd_opts.add_option(
             '--pre',
@@ -179,8 +181,8 @@ def _build_package_finder(self, options, index_urls, session):
         """
         return PackageFinder(
             find_links=options.find_links,
+            format_control=options.format_control,
             index_urls=index_urls,
-            use_wheel=options.use_wheel,
             allow_external=options.allow_external,
             allow_unverified=options.allow_unverified,
             allow_all_external=options.allow_all_external,
@@ -191,6 +193,7 @@ def _build_package_finder(self, options, index_urls, session):
         )
 
     def run(self, options, args):
+        cmdoptions.resolve_wheel_no_use_binary(options)
 
         if options.download_dir:
             options.ignore_installed = True
@@ -239,7 +242,7 @@ def run(self, options, args):
 
             finder = self._build_package_finder(options, index_urls, session)
             build_delete = (not (options.no_clean or options.build_dir))
-            wheel_cache = WheelCache(options.cache_dir)
+            wheel_cache = WheelCache(options.cache_dir, options.format_control)
             with BuildDirectory(options.build_dir,
                                 delete=build_delete) as build_dir:
                 requirement_set = RequirementSet(

pip/commands/list.py

@@ -6,7 +6,7 @@
 
 from pip.basecommand import Command
 from pip.exceptions import DistributionNotFound
-from pip.index import PackageFinder, Search
+from pip.index import FormatControl, fmt_ctl_formats, PackageFinder, Search
 from pip.req import InstallRequirement
 from pip.utils import get_installed_distributions, dist_is_editable
 from pip.wheel import WheelCache
@@ -131,7 +131,8 @@ def find_packages_latest_versions(self, options):
                 user_only=options.user,
                 include_editables=False,
             )
-            wheel_cache = WheelCache(options.cache_dir)
+            format_control = FormatControl(set(), set())
+            wheel_cache = WheelCache(options.cache_dir, format_control)
             for dist in installed_packages:
                 req = InstallRequirement.from_line(
                     dist.key, None, isolated=options.isolated_mode,
@@ -148,10 +149,12 @@ def find_packages_latest_versions(self, options):
                 except DistributionNotFound:
                     continue
                 else:
+                    canonical_name = pkg_resources.safe_name(req.name).lower()
+                    formats = fmt_ctl_formats(format_control, canonical_name)
                     search = Search(
                         req.name,
-                        pkg_resources.safe_name(req.name).lower(),
-                        ["source", "binary"])
+                        canonical_name,
+                        formats)
                     remote_version = finder._link_package_versions(
                         link, search).version
                     if link.is_wheel:

pip/commands/uninstall.py

@@ -1,5 +1,6 @@
 from __future__ import absolute_import
 
+import pip
 from pip.wheel import WheelCache
 from pip.req import InstallRequirement, RequirementSet, parse_requirements
 from pip.basecommand import Command
@@ -43,7 +44,8 @@ def __init__(self, *args, **kw):
 
     def run(self, options, args):
         with self._build_session(options) as session:
-            wheel_cache = WheelCache(options.cache_dir)
+            format_control = pip.index.FormatControl(set(), set())
+            wheel_cache = WheelCache(options.cache_dir, format_control)
             requirement_set = RequirementSet(
                 build_dir=None,
                 src_dir=None,

pip/commands/wheel.py

@@ -61,6 +61,8 @@ def __init__(self, *args, **kw):
         )
         cmd_opts.add_option(cmdoptions.use_wheel())
         cmd_opts.add_option(cmdoptions.no_use_wheel())
+        cmd_opts.add_option(cmdoptions.no_binary())
+        cmd_opts.add_option(cmdoptions.only_binary())
         cmd_opts.add_option(
             '--build-option',
             dest='build_options',
@@ -122,6 +124,7 @@ def check_required_packages(self):
 
     def run(self, options, args):
         self.check_required_packages()
+        cmdoptions.resolve_wheel_no_use_binary(options)
 
         index_urls = [options.index_url] + options.extra_index_urls
         if options.no_index:
@@ -143,8 +146,8 @@ def run(self, options, args):
 
             finder = PackageFinder(
                 find_links=options.find_links,
+                format_control=options.format_control,
                 index_urls=index_urls,
-                use_wheel=options.use_wheel,
                 allow_external=options.allow_external,
                 allow_unverified=options.allow_unverified,
                 allow_all_external=options.allow_all_external,
@@ -155,7 +158,7 @@ def run(self, options, args):
             )
 
             build_delete = (not (options.no_clean or options.build_dir))
-            wheel_cache = WheelCache(options.cache_dir)
+            wheel_cache = WheelCache(options.cache_dir, options.format_control)
             with BuildDirectory(options.build_dir,
                                 delete=build_delete) as build_dir:
                 requirement_set = RequirementSet(

pip/index.py

@@ -34,7 +34,7 @@
 from pip._vendor.requests.exceptions import SSLError
 
 
-__all__ = ['PackageFinder']
+__all__ = ['FormatControl', 'fmt_ctl_handle_mutual_exclude', 'PackageFinder']
 
 
 # Taken from Chrome's list of secure origins (See: http://bit.ly/1qrySKC)
@@ -97,14 +97,20 @@ class PackageFinder(object):
     """This finds packages.
 
     This is meant to match easy_install's technique for looking for
-    packages, by reading pages and looking for appropriate links
+    packages, by reading pages and looking for appropriate links.
     """
 
     def __init__(self, find_links, index_urls,
-                 use_wheel=True, allow_external=(), allow_unverified=(),
+                 allow_external=(), allow_unverified=(),
                  allow_all_external=False, allow_all_prereleases=False,
                  trusted_hosts=None, process_dependency_links=False,
-                 session=None):
+                 session=None, format_control=None):
+        """Create a PackageFinder.
+
+        :param format_control: A FormatControl object or None. Used to control
+            the selection of source packages / binary packages when consulting
+            the index and links.
+        """
         if session is None:
             raise TypeError(
                 "PackageFinder() missing 1 required keyword argument: "
@@ -130,7 +136,7 @@ def __init__(self, find_links, index_urls,
         # These are boring links that have already been logged somehow:
         self.logged_links = set()
 
-        self.use_wheel = use_wheel
+        self.format_control = format_control or FormatControl(set(), set())
 
         # Do we allow (safe and verifiable) externally hosted files?
         self.allow_external = set(normalize_name(n) for n in allow_external)
@@ -413,13 +419,9 @@ def _find_all_versions(self, project_name):
         for location in url_locations:
             logger.debug('* %s', location)
 
-        formats = set(["source"])
-        if self.use_wheel:
-            formats.add("binary")
-        search = Search(
-            project_name.lower(),
-            pkg_resources.safe_name(project_name).lower(),
-            frozenset(formats))
+        canonical_name = pkg_resources.safe_name(project_name).lower()
+        formats = fmt_ctl_formats(self.format_control, canonical_name)
+        search = Search(project_name.lower(), canonical_name, formats)
         find_links_versions = self._package_versions(
             # We trust every directly linked archive in find_links
             (Link(url, '-f', trusted=True) for url in self.find_links),
@@ -686,6 +688,7 @@ def _link_package_versions(self, link, search):
         version = None
         if link.egg_fragment:
             egg_info = link.egg_fragment
+            ext = link.ext
         else:
             egg_info, ext = link.splitext()
             if not ext:
@@ -743,6 +746,12 @@ def _link_package_versions(self, link, search):
                         return
                 version = wheel.version
 
+        # This should be up by the search.ok_binary check, but see issue 2700.
+        if "source" not in search.formats and ext != wheel_ext:
+            self._log_skipped_link(
+                link, 'No sources permitted for %s' % search.supplied)
+            return
+
         if not version:
             version = egg_info_matches(egg_info, search.supplied, link)
         if version is None:
@@ -1192,6 +1201,56 @@ def is_wheel(self):
 INSTALLED_VERSION = Link(Inf)
 
 
+FormatControl = namedtuple('FormatControl', 'no_binary only_binary')
+"""This object has two fields, no_binary and only_binary.
+
+If a field is falsy, it isn't set. If it is {':all:'}, it should match all
+packages except those listed in the other field. Only one field can be set
+to {':all:'} at a time. The rest of the time exact package name matches
+are listed, with any given package only showing up in one field at a time.
+"""
+
+
+def fmt_ctl_handle_mutual_exclude(value, target, other):
+    new = value.split(',')
+    while ':all:' in new:
+        other.clear()
+        target.clear()
+        target.add(':all:')
+        del new[:new.index(':all:') + 1]
+        if ':none:' not in new:
+            # Without a none, we want to discard everything as :all: covers it
+            return
+    for name in new:
+        if name == ':none:':
+            target.clear()
+            continue
+        name = pkg_resources.safe_name(name).lower()
+        other.discard(name)
+        target.add(name)
+
+
+def fmt_ctl_formats(fmt_ctl, canonical_name):
+    result = set(["binary", "source"])
+    if canonical_name in fmt_ctl.only_binary:
+        result.discard('source')
+    elif canonical_name in fmt_ctl.no_binary:
+        result.discard('binary')
+    elif ':all:' in fmt_ctl.only_binary:
+        result.discard('source')
+    elif ':all:' in fmt_ctl.no_binary:
+        result.discard('binary')
+    return frozenset(result)
+
+
+def fmt_ctl_no_use_wheel(fmt_ctl):
+    fmt_ctl_handle_mutual_exclude(
+        ':all:', fmt_ctl.no_binary, fmt_ctl.only_binary)
+    warnings.warn(
+        '--no-use-wheel is deprecated and will be removed in the future. '
+        ' Please use --no-binary :all: instead.')
+
+
 Search = namedtuple('Search', 'supplied canonical formats')
 """Capture key aspects of a search.