Set ROS_PYTHON_VERSION if unset

setup.py

@@ -8,7 +8,7 @@
     version=__version__,  # noqa:F821
     packages=['rosdep2', 'rosdep2.ament_packages', 'rosdep2.platforms'],
     package_dir={'': 'src'},
-    install_requires=['catkin_pkg >= 0.4.0', 'rospkg >= 1.1.8', 'rosdistro >= 0.7.0', 'PyYAML >= 3.1'],
+    install_requires=['catkin_pkg >= 0.4.0', 'rospkg >= 1.1.8', 'rosdistro >= 0.7.5', 'PyYAML >= 3.1'],
     test_suite='nose.collector',
     test_requires=['mock', 'nose >= 1.0'],
     scripts=['scripts/rosdep', 'scripts/rosdep-source'],

src/rosdep2/cache_tools.py

@@ -0,0 +1,101 @@
+# Copyright (c) 2012, Willow Garage, Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#     * Neither the name of the Willow Garage, Inc. nor the names of its
+#       contributors may be used to endorse or promote products derived from
+#       this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+import hashlib
+import os
+import tempfile
+
+from .core import CachePermissionError
+
+try:
+    import cPickle as pickle
+except ImportError:
+    import pickle
+
+PICKLE_CACHE_EXT = '.pickle'
+
+
+def compute_filename_hash(key_filenames):
+    sha_hash = hashlib.sha1()
+    if isinstance(key_filenames, list):
+        for key in key_filenames:
+            sha_hash.update(key.encode())
+    else:
+        sha_hash.update(key_filenames.encode())
+    return sha_hash.hexdigest()
+
+
+def write_cache_file(source_cache_d, key_filenames, rosdep_data):
+    """
+    :param source_cache_d: directory to write cache file to
+    :param key_filenames: filename (or list of filenames) to be used in hashing
+    :param rosdep_data: dictionary of data to serialize as YAML
+    :returns: name of file where cache is stored
+    :raises: :exc:`OSError` if cannot write to cache file/directory
+    :raises: :exc:`IOError` if cannot write to cache file/directory
+    """
+    if not os.path.exists(source_cache_d):
+        os.makedirs(source_cache_d)
+    key_hash = compute_filename_hash(key_filenames)
+    filepath = os.path.join(source_cache_d, key_hash)
+    try:
+        write_atomic(filepath + PICKLE_CACHE_EXT, pickle.dumps(rosdep_data, 2), True)
+    except OSError as e:
+        raise CachePermissionError('Failed to write cache file: ' + str(e))
+    try:
+        os.unlink(filepath)
+    except OSError:
+        pass
+    return filepath
+
+
+def write_atomic(filepath, data, binary=False):
+    # write data to new file
+    fd, filepath_tmp = tempfile.mkstemp(prefix=os.path.basename(filepath) + '.tmp.', dir=os.path.dirname(filepath))
+
+    if (binary):
+        fmode = 'wb'
+    else:
+        fmode = 'w'
+
+    with os.fdopen(fd, fmode) as f:
+        f.write(data)
+        f.close()
+
+    try:
+        # switch file atomically (if supported)
+        os.rename(filepath_tmp, filepath)
+    except OSError:
+        # fall back to non-atomic operation
+        try:
+            os.unlink(filepath)
+        except OSError:
+            pass
+        try:
+            os.rename(filepath_tmp, filepath)
+        except OSError:
+            os.unlink(filepath_tmp)

src/rosdep2/main.py

@@ -62,6 +62,7 @@
 from .installers import normalize_uninstalled_to_list
 from .installers import RosdepInstaller
 from .lookup import RosdepLookup, ResolutionError, prune_catkin_packages
+from .meta import MetaDatabase
 from .rospkg_loader import DEFAULT_VIEW_KEY
 from .sources_list import update_sources_list, get_sources_cache_dir,\
     download_default_sources_list, SourcesListLoader, CACHE_INDEX,\
@@ -382,6 +383,13 @@ def _rosdep_main(args):
     args = args[1:]
 
     if options.ros_distro:
+        if 'ROS_DISTRO' in os.environ and os.environ['ROS_DISTRO'] != options.ros_distro:
+            # user has a different workspace sourced, use --rosdistro
+            print('WARNING: given --rosdistro {} but ROS_DISTRO is "{}". Ignoring environment.'.format(
+                options.ros_distro, os.environ['ROS_DISTRO']))
+            # Use python version from --rosdistro
+            if 'ROS_PYTHON_VERSION' in os.environ:
+                del os.environ['ROS_PYTHON_VERSION']
         os.environ['ROS_DISTRO'] = options.ros_distro
 
     # Convert list of keys to dictionary
@@ -391,6 +399,18 @@ def _rosdep_main(args):
         check_for_sources_list_init(options.sources_cache_dir)
     elif command not in ['fix-permissions']:
         setup_proxy_opener()
+
+    if 'ROS_PYTHON_VERSION' not in os.environ and 'ROS_DISTRO' in os.environ:
+        # Set python version to version used by ROS distro
+        python_versions = MetaDatabase().get('ROS_PYTHON_VERSION')
+        if os.environ['ROS_DISTRO'] in python_versions:
+            os.environ['ROS_PYTHON_VERSION'] = str(python_versions[os.environ['ROS_DISTRO']])
+
+    if 'ROS_PYTHON_VERSION' not in os.environ:
+        # Default to same python version used to invoke rosdep
+        print('WARNING: ROS_PYTHON_VERSION is unset. Defaulting to {}'.format(sys.version[0]), file=sys.stderr)
+        os.environ['ROS_PYTHON_VERSION'] = sys.version[0]
+
     if command in _command_rosdep_args:
         return _rosdep_args_handler(command, parser, options, args)
     elif command in _command_no_args:

src/rosdep2/meta.py

@@ -0,0 +1,119 @@
+# Copyright (c) 2019, Open Source Robotics Foundation, Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#     * Neither the name of the Willow Garage, Inc. nor the names of its
+#       contributors may be used to endorse or promote products derived from
+#       this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+import copy
+import os
+try:
+    import cPickle as pickle
+except ImportError:
+    import pickle
+
+try:
+    FileNotFoundError
+except NameError:
+    # Python 2 compatibility
+    # https://stackoverflow.com/questions/21367320/
+    FileNotFoundError = IOError
+
+import rospkg
+
+from ._version import __version__
+from .cache_tools import compute_filename_hash
+from .cache_tools import write_cache_file
+from .cache_tools import PICKLE_CACHE_EXT
+
+"""
+Rosdep needs to store data that isn't used to resolve rosdep keys, but needs to be cached during
+`rosdep update`.
+"""
+
+META_CACHE_DIR = 'meta.cache'
+
+
+def get_meta_cache_dir():
+    """Return storage location for cached meta data."""
+    ros_home = rospkg.get_ros_home()
+    return os.path.join(ros_home, 'rosdep', META_CACHE_DIR)
+
+
+class CacheWrapper(object):
+    """Make it possible to introspect cache in case some future bug needs to be worked around."""
+
+    def __init__(self, category, data):
+        # The version of rosdep that wrote the category
+        self.rosdep_version = __version__
+        # The un-hashed name of the category
+        self.category_name = category
+        # The stuff being stored
+        self.data = data
+
+    @property
+    def data(self):
+        # If cached data type is mutable, don't allow modifications to what's been loaded
+        return copy.deepcopy(self.__data)
+
+    @data.setter
+    def data(self, value):
+        self.__data = copy.deepcopy(value)
+
+
+class MetaDatabase:
+    """
+    Store and retrieve metadata from rosdep cache.
+
+    This data is fetched during `rosdep update`, but is not a source for resolving rosdep keys.
+    """
+
+    def __init__(self, cache_dir=None):
+        if cache_dir is None:
+            cache_dir = get_meta_cache_dir()
+
+        self._cache_dir = cache_dir
+        self._loaded = {}
+
+    def set(self, category, metadata):
+        """Add or overwrite metadata in the cache."""
+        wrapper = CacheWrapper(category, metadata)
+        # print(category, metadata)
+        write_cache_file(self._cache_dir, category, wrapper)
+        self._loaded[category] = wrapper
+
+    def get(self, category):
+        """Return metadata in the cache, or None if there is no cache entry."""
+        if category not in self._loaded:
+            self._load_from_cache(category, self._cache_dir)
+
+        if category in self._loaded:
+            return self._loaded[category].data
+
+    def _load_from_cache(self, category, cache_dir):
+        filename = compute_filename_hash(category) + PICKLE_CACHE_EXT
+        try:
+            with open(os.path.join(self._cache_dir, filename), 'rb') as cache_file:
+                self._loaded[category] = pickle.loads(cache_file.read())
+        except FileNotFoundError:
+            pass

src/rosdep2/sources_list.py

@@ -31,9 +31,7 @@
 
 import os
 import sys
-import tempfile
 import yaml
-import hashlib
 try:
     from urllib.request import urlopen
     from urllib.error import URLError
@@ -45,8 +43,10 @@
 except ImportError:
     import pickle
 
+from .cache_tools import compute_filename_hash, PICKLE_CACHE_EXT, write_atomic, write_cache_file
 from .core import InvalidData, DownloadFailure, CachePermissionError
 from .gbpdistro_support import get_gbprepo_as_rosdep_data, download_gbpdistro_as_rosdep_data
+from .meta import MetaDatabase
 
 try:
     import urlparse
@@ -78,7 +78,6 @@
 CACHE_INDEX = 'index'
 
 # extension for binary cache
-PICKLE_CACHE_EXT = '.pickle'
 SOURCE_PATH_ENV = 'ROSDEP_SOURCE_PATH'
 
 
@@ -485,6 +484,8 @@ def update_sources_list(sources_list_dir=None, sources_cache_dir=None,
 
     # Additional sources for ros distros
     # In compliance with REP137 and REP143
+    python_versions = {}
+
     print('Query rosdistro index %s' % get_index_url())
     for dist_name in sorted(get_index().distributions.keys()):
         distribution = get_index().distributions[dist_name]
@@ -495,12 +496,18 @@ def update_sources_list(sources_list_dir=None, sources_cache_dir=None,
         print('Add distro "%s"' % dist_name)
         rds = RosDistroSource(dist_name)
         rosdep_data = get_gbprepo_as_rosdep_data(dist_name)
+        # Store Python version from REP153
+        if distribution.get('python_version'):
+            python_versions[dist_name] = distribution.get('python_version')
         # dist_files can either be a string (single filename) or a list (list of filenames)
         dist_files = distribution['distribution']
         key = _generate_key_from_urls(dist_files)
         retval.append((rds, write_cache_file(sources_cache_dir, key, rosdep_data)))
         sources.append(rds)
 
+    # cache metadata that isn't a source list
+    MetaDatabase().set('ROS_PYTHON_VERSION', python_versions)
+
     # Create a combined index of *all* the sources.  We do all the
     # sources regardless of failures because a cache from a previous
     # attempt may still exist.  We have to do this cache index so that
@@ -540,68 +547,6 @@ def load_cached_sources_list(sources_cache_dir=None, verbose=False):
     return parse_sources_data(cache_data, origin=cache_index, model=model)
 
 
-def compute_filename_hash(key_filenames):
-    sha_hash = hashlib.sha1()
-    if isinstance(key_filenames, list):
-        for key in key_filenames:
-            sha_hash.update(key.encode())
-    else:
-        sha_hash.update(key_filenames.encode())
-    return sha_hash.hexdigest()
-
-
-def write_cache_file(source_cache_d, key_filenames, rosdep_data):
-    """
-    :param source_cache_d: directory to write cache file to
-    :param key_filenames: filename (or list of filenames) to be used in hashing
-    :param rosdep_data: dictionary of data to serialize as YAML
-    :returns: name of file where cache is stored
-    :raises: :exc:`OSError` if cannot write to cache file/directory
-    :raises: :exc:`IOError` if cannot write to cache file/directory
-    """
-    if not os.path.exists(source_cache_d):
-        os.makedirs(source_cache_d)
-    key_hash = compute_filename_hash(key_filenames)
-    filepath = os.path.join(source_cache_d, key_hash)
-    try:
-        write_atomic(filepath + PICKLE_CACHE_EXT, pickle.dumps(rosdep_data, 2), True)
-    except OSError as e:
-        raise CachePermissionError('Failed to write cache file: ' + str(e))
-    try:
-        os.unlink(filepath)
-    except OSError:
-        pass
-    return filepath
-
-
-def write_atomic(filepath, data, binary=False):
-    # write data to new file
-    fd, filepath_tmp = tempfile.mkstemp(prefix=os.path.basename(filepath) + '.tmp.', dir=os.path.dirname(filepath))
-
-    if (binary):
-        fmode = 'wb'
-    else:
-        fmode = 'w'
-
-    with os.fdopen(fd, fmode) as f:
-        f.write(data)
-        f.close()
-
-    try:
-        # switch file atomically (if supported)
-        os.rename(filepath_tmp, filepath)
-    except OSError:
-        # fall back to non-atomic operation
-        try:
-            os.unlink(filepath)
-        except OSError:
-            pass
-        try:
-            os.rename(filepath_tmp, filepath)
-        except OSError:
-            os.unlink(filepath_tmp)
-
-
 class SourcesListLoader(RosdepLoader):
     """
     SourcesList loader implements the general RosdepLoader API.  This