Support for installing virtual packages (Debian)

src/rosdep2/installers.py

@@ -556,22 +556,41 @@ def install_resolved(self, installer_key, resolved, simulate=False, interactive=
         if simulate:
             print("#[%s] Installation commands:"%(installer_key))
             for sub_command in command:
-                print('  '+' '.join(sub_command))
+                if isinstance(sub_command[0], list):
+                    sub_cmd_len = len(sub_command)
+                    for cmd, i in enumerate(sub_command):
+                        print("  '%s' (alternative %d/%d)" % (' '.join(cmd), i + 1, sub_cmd_len))
+                else:
+                    print('  ' + ' '.join(sub_command))
 
         # nothing left to do for simulation
         if simulate:
             return
 
+        def run_command(command, installer_key, failures, verbose):
+            # always echo commands to screen
+            print_bold("executing command [%s]" % ' '.join(command))
+            result = subprocess.call(command)
+            if verbose:
+                print("command return code [%s]: %s" % (' '.join(command), result))
+            if result != 0:
+                failures.append((installer_key, 'command [%s] failed' % (' '.join(command))))
+            return result
+
         # run each install command set and collect errors
         failures = []
         for sub_command in command:
-            # always echo commands to screen
-            print_bold("executing command [%s]"%' '.join(sub_command))
-            result = subprocess.call(sub_command)
-            if verbose:
-                print("command return code [%s]: %s"%(' '.join(sub_command), result))
+            if isinstance(sub_command[0], list): # list of alternatives
+                alt_failures = []
+                for alt_command in sub_command:
+                    result = run_command(alt_command, installer_key, alt_failures, verbose)
+                    if result == 0:  # one successsfull command is sufficient
+                        alt_failures = []  # clear failuers from other alternatives
+                        break
+                failures.extend(alt_failures)
+            else:
+                result = run_command(sub_command, installer_key, failures, verbose)
             if result != 0:
-                failures.append((installer_key, 'command [%s] failed'%(' '.join(sub_command))) )
                 if not continue_on_error:
                     raise InstallFailed(failures=failures)
 

src/rosdep2/platforms/debian.py

@@ -1,10 +1,10 @@
 #!/usr/bin/env python
 # Copyright (c) 2009, Willow Garage, Inc.
 # All rights reserved.
-# 
+#
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are met:
-# 
+#
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above copyright
@@ -13,7 +13,7 @@
 #     * Neither the name of the Willow Garage, Inc. nor the names of its
 #       contributors may be used to endorse or promote products derived from
 #       this software without specific prior written permission.
-# 
+#
 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
@@ -31,7 +31,6 @@
 from __future__ import print_function
 import subprocess
 import sys
-import re
 
 from rospkg.os_detect import OS_DEBIAN, OS_LINARO, OS_UBUNTU, OS_ELEMENTARY, OsDetect
 
@@ -42,40 +41,49 @@
 from ..shell_utils import read_stdout
 
 # apt package manager key
-APT_INSTALLER='apt'
+APT_INSTALLER = 'apt'
 
 
 def register_installers(context):
     context.set_installer(APT_INSTALLER, AptInstaller())
 
+
 def register_platforms(context):
     register_debian(context)
     register_linaro(context)
     register_ubuntu(context)
     register_elementary(context)
-
+
+
 def register_debian(context):
     context.add_os_installer_key(OS_DEBIAN, APT_INSTALLER)
     context.add_os_installer_key(OS_DEBIAN, PIP_INSTALLER)
     context.add_os_installer_key(OS_DEBIAN, GEM_INSTALLER)
     context.add_os_installer_key(OS_DEBIAN, SOURCE_INSTALLER)
     context.set_default_os_installer_key(OS_DEBIAN, lambda self: APT_INSTALLER)
     context.set_os_version_type(OS_DEBIAN, OsDetect.get_codename)
-
+
+
 def register_linaro(context):
-    # Linaro is an alias for Ubuntu. If linaro is detected and it's not set as an override force ubuntu.
+    # Linaro is an alias for Ubuntu. If linaro is detected and it's not set as
+    # an override force ubuntu.
     (os_name, os_version) = context.get_os_name_and_version()
     if os_name == OS_LINARO and not context.os_override:
-        print("rosdep detected OS: [%s] aliasing it to: [%s]" % (OS_LINARO, OS_UBUNTU), file=sys.stderr)
+        print("rosdep detected OS: [%s] aliasing it to: [%s]" %
+              (OS_LINARO, OS_UBUNTU), file=sys.stderr)
         context.set_os_override(OS_UBUNTU, context.os_detect.get_codename())
 
+
 def register_elementary(context):
-    # Elementary is an alias for Ubuntu. If elementary is detected and it's not set as an override force ubuntu.
+    # Elementary is an alias for Ubuntu. If elementary is detected and it's
+    # not set as an override force ubuntu.
     (os_name, os_version) = context.get_os_name_and_version()
     if os_name == OS_ELEMENTARY and not context.os_override:
-        print("rosdep detected OS: [%s] aliasing it to: [%s]" % (OS_ELEMENTARY, OS_UBUNTU), file=sys.stderr)
+        print("rosdep detected OS: [%s] aliasing it to: [%s]" %
+              (OS_ELEMENTARY, OS_UBUNTU), file=sys.stderr)
         context.set_os_override(OS_UBUNTU, context.os_detect.get_codename())
 
+
 def register_ubuntu(context):
     context.add_os_installer_key(OS_UBUNTU, APT_INSTALLER)
     context.add_os_installer_key(OS_UBUNTU, PIP_INSTALLER)
@@ -84,71 +92,56 @@ def register_ubuntu(context):
     context.set_default_os_installer_key(OS_UBUNTU, lambda self: APT_INSTALLER)
     context.set_os_version_type(OS_UBUNTU, OsDetect.get_codename)
 
-
-# detect that apt show indicates that the package is virtual
-APT_PURELY_VIRTUAL_RE = re.compile(
-        r'as it is purely virtual',
-        flags=re.DOTALL)
-# detect what lines in apt-cache showpkg show the packages providing a virtual
-# package
-APT_CACHE_REVERSE_PROVIDE_START_RE = re.compile(
-        r'^Reverse Provides:')
-# format of a 'Reverse Provides' line in the apt-cache showpkg output
-APT_CACHE_PROVIDER_RE = re.compile('^(.*?) (.*)$')
-
-
-def _is_installed_as_virtual_package(package, exec_fn=None):
+def _read_apt_cache_showpkg(packages, exec_fn=None):
     '''
-    Check whether this is a virtual package and a package providing this
-    virtual package is installed.
-
+    Output whether these packages are virtual package list providing package.
+    If one package was not found, it gets returned as non-virtual.
     :param exec_fn: see `dpkg_detect`; make sure that exec_fn supports a
     second, boolean, parameter.
     '''
-# Note: This can be done much more concise when adding python-apt as a dependency:
-#
-#    import apt
-#    cache = apt.Cache()
-#    if cache.is_virtual_package(package):
-#        for provider in cache.get_providing_packages(package):
-#            if cache[provider].is_installed:
-#                print('Virtual package {} is provided by {}'.format(
-#                    package, provider.name))
-#                return True
-#        return False
-#
-    # check output of `apt show package' for whether it's a virtual
-    # package and if so use `apt-cache showpkg package' to get the providing
-    # packages.  Then check if one of those is installed.
-    cmd = ['apt-cache', 'show', package]
+
+    cmd = ['apt-cache', 'showpkg'] + list(packages)
     if exec_fn is None:
         exec_fn = read_stdout
-    std_out, std_err = exec_fn(cmd, True) # use stderr as well to hide error message ... not too nice, but hopefully cautious
-    if APT_PURELY_VIRTUAL_RE.search(std_out):
-        print('Package {} seems to be virtual; try to specify a providing package in your rosdep config.'.format(package))
-        cmd = ['apt-cache', 'showpkg', package]
-        std_out = exec_fn(cmd)
-        is_provider = False # true when parsed line contains a povider
-        for line in std_out.split('\n'):
-            if is_provider:
-                match = APT_CACHE_PROVIDER_RE.match(line)
-                if not match:
-                    print('WARNING: The output of {} is strange; unable to determine providers of virtual package {}'.format(
-                        cmd[0] + ' ' + cmd[1], package))
-                else:
-                    provider_name, provider_version = match.groups()
-                    # now that we have the name of the provider, finaly check
-                    # whether the package is provided
-                    if dpkg_detect([provider_name]):
-                        print('Virtual package {} is provided by {}'.format(package, provider_name))
-                        return True
-            if APT_CACHE_REVERSE_PROVIDE_START_RE.match(line):
-                is_provider = True
-                # Note: Set this _after_ possibly parsing the current line to
-                #       not parse the line containing
-                #       APT_CACHE_REVERSE_PROVIDE_START_RE
-        return False # unable to find a provider that was installed
 
+    std_out = exec_fn(cmd).splitlines()
+
+    starts = []
+    notfound = set()
+    for p in packages:
+        last_start = starts[-1] if len(starts) > 0 else 0
+        try:
+            starts.append(std_out.index("Package: %s" % p, last_start))
+        except ValueError:
+            notfound.add(p)
+    starts.append(-1)
+
+    for p in packages:
+        if p in notfound:
+            yield p, False, None
+            continue
+        start = starts.pop(0)
+        lines = iter(std_out[start:starts[0]])
+
+        header = "Package: %s" % p
+        # proceed to Package header
+        while next(lines) != header:
+            pass
+
+        # proceed to versions section
+        while next(lines) != "Versions: ":
+            pass
+
+        # virtual packages don't have versions
+        if next(lines) != "":
+            yield p, False, None
+            continue
+
+        # proceed to reserve provides section
+        while next(lines) != "Reverse Provides: ":
+            pass
+
+        yield p, True, [line.split(' ', 2)[0] for line in lines]
 
 
 def dpkg_detect(pkgs, exec_fn=None):
@@ -175,30 +168,46 @@ def dpkg_detect(pkgs, exec_fn=None):
 
     if exec_fn is None:
         exec_fn = read_stdout
-    std_out = exec_fn(cmd)
-    std_out = std_out.replace('\'','')
+    std_out, std_err = exec_fn(cmd, True)
+    std_out = std_out.replace('\'', '')
     pkg_list = std_out.split('\n')
     for pkg in pkg_list:
         pkg_row = pkg.split()
-        if len(pkg_row) == 4 and (pkg_row[3] =='installed'):
-            ret_list.append( pkg_row[0])
+        if len(pkg_row) == 4 and (pkg_row[3] == 'installed'):
+            ret_list.append(pkg_row[0])
     installed_packages = [version_lock_map[r] for r in ret_list]
 
     # now for the remaining packages check, whether they are installed as
     # virtual packages
-    for rem in set(pkgs) - set(installed_packages):
-        if _is_installed_as_virtual_package(rem):
-            installed_packages.append(rem)
-
-    return installed_packages
-
+    remaining = _read_apt_cache_showpkg(p for p in pkgs if p not in installed_packages)
+    virtual = [ n for (n, v, pr) in remaining if v and len(dpkg_detect(pr)) > 0 ]
+
+    return installed_packages + virtual
+
+
+def _iterate_packages(packages, reinstall):
+    for entry in _read_apt_cache_showpkg(packages):
+        p, is_virtual, providers = entry
+        if is_virtual:
+            installed = []
+            if reinstall:
+                installed = dpkg_detect(providers)
+                if len(installed) > 0:
+                    for i in installed:
+                        yield i
+                    continue  # don't ouput providers
+            yield providers
+        else:
+            yield p
 
 
 class AptInstaller(PackageManagerInstaller):
+
     """
     An implementation of the Installer for use on debian style
     systems.
     """
+
     def __init__(self):
         super(AptInstaller, self).__init__(dpkg_detect)
 
@@ -207,15 +216,26 @@ def get_version_strings(self):
         version = output.splitlines()[0].split(' ')[1]
         return ['apt-get {}'.format(version)]
 
+    def _get_install_commands_for_package(self, base_cmd, package_or_list):
+        def pkg_command(p):
+            return self.elevate_priv(base_cmd + [p])
+
+        if isinstance(package_or_list, list):
+            return [pkg_command(p) for p in package_or_list]
+        else:
+            return pkg_command(package_or_list)
+
     def get_install_command(self, resolved, interactive=True, reinstall=False, quiet=False):
         packages = self.get_packages_to_install(resolved, reinstall=reinstall)
         if not packages:
             return []
         if not interactive and quiet:
-            return [self.elevate_priv(['apt-get', 'install', '-y', '-qq', p]) for p in packages]
+            base_cmd = ['apt-get', 'install', '-y', '-qq']
         elif quiet:
-            return [self.elevate_priv(['apt-get', 'install', '-qq', p]) for p in packages]
+            base_cmd = ['apt-get', 'install', '-qq']
         if not interactive:
-            return [self.elevate_priv(['apt-get', 'install', '-y', p]) for p in packages]
+            base_cmd = ['apt-get', 'install', '-y']
         else:
-            return [self.elevate_priv(['apt-get', 'install', p]) for p in packages]
+            base_cmd = ['apt-get', 'install']
+
+        return [self._get_install_commands_for_package(base_cmd, p) for p in _iterate_packages(packages, reinstall)]