From 6091c08c73feb9eef661c3058d53d7c9127bc126 Mon Sep 17 00:00:00 2001 From: kopp Date: Thu, 25 Aug 2016 01:39:10 +0200 Subject: [PATCH] Fix handle installed virtual packages (#468) * add/improve docstrings in installer.py * shell_utils: allow read_stdout to capture stderr * debian: detect installed virtual packages --- src/rosdep2/installers.py | 9 ++++ src/rosdep2/platforms/debian.py | 85 ++++++++++++++++++++++++++++++++- src/rosdep2/shell_utils.py | 33 ++++++++++--- 3 files changed, 119 insertions(+), 8 deletions(-) diff --git a/src/rosdep2/installers.py b/src/rosdep2/installers.py index 649e00fd4..91474e7d8 100644 --- a/src/rosdep2/installers.py +++ b/src/rosdep2/installers.py @@ -314,6 +314,8 @@ class PackageManagerInstaller(Installer): def __init__(self, detect_fn, supports_depends=False): """ :param supports_depends: package manager supports dependency key + :param detect_fn: function that for a given list of packages determines + the list of installed packages. """ self.detect_fn = detect_fn self.supports_depends = supports_depends @@ -356,6 +358,10 @@ def unique(self, *resolved_rules): return sorted(list(s)) def get_packages_to_install(self, resolved, reinstall=False): + ''' + Return a list of packages (out of *resolved*) that still need to get + installed. + ''' if reinstall: return resolved if not resolved: @@ -364,6 +370,9 @@ def get_packages_to_install(self, resolved, reinstall=False): return list(set(resolved) - set(self.detect_fn(resolved))) def is_installed(self, resolved_item): + ''' + Check if a given package was installed. + ''' return not self.get_packages_to_install([resolved_item]) def get_install_command(self, resolved, interactive=True, reinstall=False, quiet=False): diff --git a/src/rosdep2/platforms/debian.py b/src/rosdep2/platforms/debian.py index 81fd4f62a..c42b53fb5 100644 --- a/src/rosdep2/platforms/debian.py +++ b/src/rosdep2/platforms/debian.py @@ -30,6 +30,7 @@ from __future__ import print_function import sys +import re from rospkg.os_detect import OS_DEBIAN, OS_LINARO, OS_UBUNTU, OS_ELEMENTARY, OsDetect @@ -42,6 +43,7 @@ # apt package manager key APT_INSTALLER='apt' + def register_installers(context): context.set_installer(APT_INSTALLER, AptInstaller()) @@ -81,16 +83,86 @@ def register_ubuntu(context): context.set_default_os_installer_key(OS_UBUNTU, lambda self: APT_INSTALLER) context.set_os_version_type(OS_UBUNTU, OsDetect.get_codename) + +# detect that apt show indicates that the package is virtual +APT_PURELY_VIRTUAL_RE = re.compile( + r'State: not a real package \(virtual\)', + flags=re.DOTALL) +# detect what lines in apt-cache showpkg show the packages providing a virtual +# package +APT_CACHE_REVERSE_PROVIDE_START_RE = re.compile( + r'^Reverse Provides:') +# format of a 'Reverse Provides' line in the apt-cache showpkg output +APT_CACHE_PROVIDER_RE = re.compile('^(.*) (.*)$') + + +def _is_installed_as_virtual_package(package, exec_fn=None): + ''' + Check whether this is a virtual package and a package providing this + virtual package is installed. + + :param exec_fn: see `dpkg_detect`; make sure that exec_fn supports a + second, boolean, parameter. + ''' +# Note: This can be done much more concise when adding python-apt as a dependency: +# +# import apt +# cache = apt.Cache() +# if cache.is_virtual_package(package): +# for provider in cache.get_providing_packages(package): +# if cache[provider].is_installed: +# print('Virtual package {} is provided by {}'.format( +# package, provider.name)) +# return True +# return False +# + # check output of `apt show package' for whether it's a virtual + # package and if so use `apt-cache showpkg package' to get the providing + # packages. Then check if one of those is installed. + cmd = ['apt', 'show', package] + if exec_fn is None: + exec_fn = read_stdout + std_out, std_err = exec_fn(cmd, True) # use stderr as well to hide error message ... not too nice, but hopefully cautious + if APT_PURELY_VIRTUAL_RE.search(std_out): + print('Package {} seems to be virtual; try to specify a providing package in your rosdep config.'.format(package)) + cmd = ['apt-cache', 'showpkg', package] + std_out = exec_fn(cmd) + is_provider = False # true when parsed line contains a povider + for line in std_out.split('\n'): + if is_provider: + match = APT_CACHE_PROVIDER_RE.match(line) + if not match: + print('WARNING: The output of {} is strange; unable to determine providers of virtual package {}'.format( + cmd[0] + ' ' + cmd[1], package)) + else: + provider_name, provider_version = match.groups() + # now that we have the name of the provider, finaly check + # whether the package is provided + if dpkg_detect([provider_name]): + print('Virtual package {} is provided by {}'.format(package, provider_name)) + return True + if APT_CACHE_REVERSE_PROVIDE_START_RE.match(line): + is_provider = True + # Note: Set this _after_ possibly parsing the current line to + # not parse the line containing + # APT_CACHE_REVERSE_PROVIDE_START_RE + return False # unable to find a provider that was installed + + + def dpkg_detect(pkgs, exec_fn=None): - """ + """ Given a list of package, return the list of installed packages. + :param pkgs: list of package names, optionally followed by a fixed version (`foo=3.0`) :param exec_fn: function to execute Popen and read stdout (for testing) + :return: list elements in *pkgs* that were found installed on the system """ ret_list = [] # this is mainly a hack to support version locking for eigen. # we strip version-locking syntax, e.g. libeigen3-dev=3.0.1-*. # our query does not do the validation on the version itself. + # This is a map `package name -> package name optionally with version`. version_lock_map = {} for p in pkgs: if '=' in p: @@ -109,7 +181,16 @@ def dpkg_detect(pkgs, exec_fn=None): pkg_row = pkg.split() if len(pkg_row) == 4 and (pkg_row[3] =='installed'): ret_list.append( pkg_row[0]) - return [version_lock_map[r] for r in ret_list] + installed_packages = [version_lock_map[r] for r in ret_list] + + # now for the remaining packages check, whether they are installed as + # virtual packages + for rem in set(pkgs) - set(installed_packages): + if _is_installed_as_virtual_package(rem): + installed_packages.append(rem) + + return installed_packages + class AptInstaller(PackageManagerInstaller): diff --git a/src/rosdep2/shell_utils.py b/src/rosdep2/shell_utils.py index af04cd628..4b13bfb0f 100644 --- a/src/rosdep2/shell_utils.py +++ b/src/rosdep2/shell_utils.py @@ -42,13 +42,34 @@ else: python3 = False -def read_stdout(cmd): - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - std_out, std_err = p.communicate() - if python3: - return std_out.decode() + +def read_stdout(cmd, capture_stderr=False): + ''' + Execute given command and return stdout and if requested also stderr. + + :param cmd: command in a form that Popen understands (list of strings or one string) + :param suppress_stderr: If evaluates to True, capture output from stderr as + well and return it as well. + :return: if `capture_stderr` is evaluates to False, return the stdout of + the program as string (Note: stderr will be printed to the running + terminal). If it evaluates to True, tuple of strings: stdout output and + standard error output each as string. + ''' + if capture_stderr: + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + std_out, std_err = p.communicate() + if python3: + return std_out.decode(), std_err.decode() + else: + return std_out, std_err else: - return std_out + p = subprocess.Popen(cmd, stdout=subprocess.PIPE) + std_out, std_err = p.communicate() # ignore stderr + if python3: + return std_out.decode() + else: + return std_out + def create_tempfile_from_string_and_execute(string_script, path=None, exec_fn=None): """