Add --extra-index-url from all extra indexes

- Always add extra indexes when installing
- Look up indexes by key if key is given instead of url
- Fixes #1973, #1974, #1852

Signed-off-by: Dan Ryan <[email protected]>

pipenv/core.py

@@ -20,7 +20,7 @@
 import pipdeptree
 from pipreqs import pipreqs
 from blindspin import spinner
-
+from first import first
 from requests.packages import urllib3
 from requests.packages.urllib3.exceptions import InsecureRequestWarning
 
@@ -48,6 +48,8 @@
     is_star,
     TemporaryDirectory,
     rmtree,
+    split_index,
+    split_extra_index,
 )
 from .import pep508checker, progress
 from .environments import (
@@ -782,12 +784,8 @@ def cleanup_procs(procs, concurrent):
     for dep, ignore_hash, block in deps_list_bar:
         if len(procs) < PIPENV_MAX_SUBPROCESS:
             # Use a specific index, if specified.
-            index = None
-            if ' -i ' in dep:
-                dep, index = dep.split(' -i ')
-                dep = '{0} {1}'.format(dep, ' '.join(index.split()[1:])).strip(
-                )
-                index = index.split()[0]
+            dep, index = split_index(dep)
+            dep, extra_index = split_extra_index(dep)
             # Install the module.
             c = pip_install(
                 dep,
@@ -798,6 +796,7 @@ def cleanup_procs(procs, concurrent):
                 block=block,
                 index=index,
                 requirements_dir=requirements_dir,
+                extra_indexes=extra_index,
             )
             c.dep = dep
             c.ignore_hash = ignore_hash
@@ -816,12 +815,9 @@ def cleanup_procs(procs, concurrent):
         for dep, ignore_hash in progress.bar(
             failed_deps_list, label=INSTALL_LABEL2
         ):
-            index = None
-            if ' -i ' in dep:
-                dep, index = dep.split(' -i ')
-                dep = '{0} {1}'.format(dep, ' '.join(index.split()[1:])).strip(
-                )
-                index = index.split()[0]
+            # Use a specific index, if specified.
+            dep, index = split_index(dep)
+            dep, extra_index = split_extra_index(dep)
             # Install the module.
             c = pip_install(
                 dep,
@@ -831,6 +827,7 @@ def cleanup_procs(procs, concurrent):
                 verbose=verbose,
                 index=index,
                 requirements_dir=requirements_dir,
+                extra_index=extra_index,
             )
             # The Installation failed...
             if c.return_code != 0:
@@ -1358,6 +1355,7 @@ def pip_install(
     pre=False,
     selective_upgrade=False,
     requirements_dir=None,
+    extra_indexes=None,
 ):
     import pip9
 
@@ -1406,51 +1404,57 @@ def pip_install(
             src = ''
     else:
         src = ''
+
     # Try installing for each source in project.sources.
     if index:
+        valid_indexes = [p['name'] for p in project.parsed_pipfile['source']]
+        if not is_valid_url(index) and index in valid_indexes:
+            index = first([p['url'] for p in project.parsed_pipfile['source'] if p['name'] == index])
         sources = [{'url': index}]
+        if extra_indexes:
+            extra_indexes = [{'url': extra_src} for extra_src in extra_indexes if extra_src != index]
+        else:
+            extra_indexes = [{'url': s['url']} for s in project.parsed_pipfile['source'] if s['url'] != index]
+        sources = sources.extend(extra_indexes)
     else:
         sources = project.sources
-    for source in sources:
-        if package_name.startswith('-e '):
-            install_reqs = ' -e "{0}"'.format(package_name.split('-e ')[1])
-        elif r:
-            install_reqs = ' -r {0}'.format(r)
-        else:
-            install_reqs = ' "{0}"'.format(package_name)
-        # Skip hash-checking mode, when appropriate.
-        if r:
-            with open(r) as f:
-                if '--hash' not in f.read():
-                    ignore_hashes = True
-        else:
-            if '--hash' not in install_reqs:
+    if package_name.startswith('-e '):
+        install_reqs = ' -e "{0}"'.format(package_name.split('-e ')[1])
+    elif r:
+        install_reqs = ' -r {0}'.format(r)
+    else:
+        install_reqs = ' "{0}"'.format(package_name)
+    # Skip hash-checking mode, when appropriate.
+    if r:
+        with open(r) as f:
+            if '--hash' not in f.read():
                 ignore_hashes = True
-        verbose_flag = '--verbose' if verbose else ''
-        if not ignore_hashes:
-            install_reqs += ' --require-hashes'
-        no_deps = '--no-deps' if no_deps else ''
-        pre = '--pre' if pre else ''
-        quoted_pip = which_pip(allow_global=allow_global)
-        quoted_pip = escape_grouped_arguments(quoted_pip)
-        upgrade_strategy = '--upgrade --upgrade-strategy=only-if-needed' if selective_upgrade else ''
-        pip_command = '{0} install {4} {5} {6} {7} {3} {1} {2} --exists-action w'.format(
-            quoted_pip,
-            install_reqs,
-            ' '.join(prepare_pip_source_args([source])),
-            no_deps,
-            pre,
-            src,
-            verbose_flag,
-            upgrade_strategy,
-        )
-        if verbose:
-            click.echo('$ {0}'.format(pip_command), err=True)
-        c = delegator.run(pip_command, block=block)
-        if c.return_code == 0:
-            break
-
-    # Return the result of the first one that runs ok, or the last one that didn't work.
+    else:
+        if '--hash' not in install_reqs:
+            ignore_hashes = True
+    verbose_flag = '--verbose' if verbose else ''
+    if not ignore_hashes:
+        install_reqs += ' --require-hashes'
+    no_deps = '--no-deps' if no_deps else ''
+    pre = '--pre' if pre else ''
+    quoted_pip = which_pip(allow_global=allow_global)
+    quoted_pip = escape_grouped_arguments(quoted_pip)
+    sources = []
+    print(sources)
+    upgrade_strategy = '--upgrade --upgrade-strategy=only-if-needed' if selective_upgrade else ''
+    pip_command = '{0} install {4} {5} {6} {7} {3} {1} {2} --exists-action w'.format(
+        quoted_pip,
+        install_reqs,
+        ' '.join(prepare_pip_source_args(sources)),
+        no_deps,
+        pre,
+        src,
+        verbose_flag,
+        upgrade_strategy,
+    )
+    if verbose:
+        click.echo('$ {0}'.format(pip_command), err=True)
+    c = delegator.run(pip_command, block=block)
     return c
 
 

pipenv/project.py

@@ -7,7 +7,6 @@
 import shlex
 import base64
 import hashlib
-
 import contoml
 import delegator
 import pipfile

pipenv/utils.py

@@ -15,7 +15,7 @@
 import six
 import stat
 import warnings
-
+from first import first
 try:
     from weakref import finalize
 except ImportError:
@@ -1275,3 +1275,37 @@ def __exit__(self, exc, value, tb):
     def cleanup(self):
         if self._finalizer.detach():
             rmtree(self.name)
+
+
+def split_index(req):
+    """Split an index argument from a requirement (finds None if not present)
+
+    returns req, index
+    """
+    index_entries = [' -i ', ' --index ', ' --index=']
+    index = None
+    index_entry = first([entry for entry in index_entries if entry in req])
+    if index_entry:
+        req, index = req.split(index_entry)
+        remaining_line = index.split()
+        if len(remaining_line) > 1:
+            index, more_req = remaining_line[0], ' '.join(remaining_line[1:])
+            req = '{0} {1}'.format(req, more_req)
+    return req, index
+
+
+def split_extra_index(req):
+    """Split an extra index argument from a requirement (or None if not present)
+
+    returns req, index
+    """
+    index_entries = [' --extra-index-url ', ' --extra-index-url=']
+    index = None
+    index_entry = first([entry for entry in index_entries if entry in req])
+    if index_entry:
+        req, index = req.split(index_entry)
+        remaining_line = index.split()
+        if len(remaining_line) > 1:
+            index, more_req = remaining_line[0], ' '.join(remaining_line[1:])
+            req = '{0} {1}'.format(req, more_req)
+    return req, index