Initial implementation

README.rst

@@ -2,3 +2,31 @@ colcon-installed-package-information
 ====================================
 
 Extensions for `colcon-core <https://github.com/colcon/colcon-core>`_ to inspect packages which have already been installed.
+
+Overview
+--------
+
+These colcon extensions provide a mechanism which can be used for getting information about packages outside of the workspace, which have already been built and installed prior to the current operation.
+In general, it works similarly to and is based on the `PackageDiscoveryExtensionPoint <https://colcon.readthedocs.io/en/released/developer/extension-point.html#packagediscoveryextensionpoint>`_ and `PackageAugmentationExtensionPoint <https://colcon.readthedocs.io/en/released/developer/extension-point.html#packageaugmentationextensionpoint>`_ extensions provided by ``colcon-core``.
+
+Differences
+-----------
+
+Installed packages don't generally have a single directory which stores the package content and metadata.
+This set of extensions store the "prefix" under which the package resides rather than the package directory, meaning many packages will likely share the same ``path`` attribute value.
+
+Recursively crawling an entire system or even selective subdirectories to look for installed packages could be very slow, so this process also deviates from the Discover -> Identify -> Augment pipeline used in ``colcon-core``.
+Rather than attempting identification on perspective package locations, the discovery phase generally loads a list of installed packages from a database of some kind.
+In some cases, the database might already populate sufficient information on the descriptor to identify the package.
+For others, only presence can be known, and augmentation extensions must add additional information to the descriptor by searching for specific files throughout the prefix directory.
+
+The ``type`` attribute of an installed package works similarly to workspace packages, but must always start with ``installed.`` followed a more specific package type.
+If more information about a package cannot be determined and it is known only to exist under a certain prefix, the time should be set to ``installed``.
+
+Supported Package Types
+-----------------------
+
+This package provides extensions which are able to discover packages using the ``PrefixPathExtensionPoint`` to enumerate install prefixes, and ``FindInstalledPackagesExtensionPoint`` to enumerate names of packages installed under those prefixes.
+It can then use the colcon index in those prefixes as well as python eggs to determine dependency information and augment the packages appropriately.
+
+Support for more package databases for discovery and augmentation can be added by other packages by implementing and registering appropriate extensions in other packages.

colcon_installed_package_information/__init__.py

@@ -1,4 +1,4 @@
 # Copyright 2022 Scott K Logan
 # Licensed under the Apache License, Version 2.0
 
-__version__ = '0.0.0'
+__version__ = '0.0.1'

colcon_installed_package_information/package_augmentation/__init__.py

@@ -0,0 +1,39 @@
+# Copyright 2022 Open Source Robotics Foundation, Inc.
+# Licensed under the Apache License, Version 2.0
+
+from colcon_core.package_augmentation \
+    import augment_packages as augment_packages_impl
+from colcon_core.plugin_system import instantiate_extensions
+from colcon_core.plugin_system import order_extensions_by_priority
+
+
+def get_package_augmentation_extensions():
+    """
+    Get the available package augmentation extensions.
+
+    The extensions are ordered by their priority and entry point name.
+
+    :rtype: OrderedDict
+    """
+    extensions = instantiate_extensions(__name__)
+    for name, extension in extensions.items():
+        extension.PACKAGE_AUGMENTATION_NAME = name
+    return order_extensions_by_priority(extensions)
+
+
+def augment_packages(
+    descs, *, additional_argument_names=None, augmentation_extensions=None
+):
+    """
+    Augment package descriptors with additional information.
+
+    :param descs: the packages
+    :type descs: set of
+      :py:class:`colcon_core.package_descriptor.PackageDescriptor`
+    """
+    if augmentation_extensions is None:
+        augmentation_extensions = get_package_augmentation_extensions()
+
+    augment_packages_impl(
+        descs, additional_argument_names=additional_argument_names,
+        augmentation_extensions=augmentation_extensions)

colcon_installed_package_information/package_augmentation/colcon_index.py

@@ -0,0 +1,45 @@
+# Copyright 2022 Open Source Robotics Foundation, Inc.
+# Licensed under the Apache License, Version 2.0
+
+import os
+
+from colcon_core.dependency_descriptor import DependencyDescriptor
+from colcon_core.location import get_relative_package_index_path
+from colcon_core.package_augmentation \
+    import PackageAugmentationExtensionPoint
+from colcon_core.plugin_system import satisfies_version
+
+
+class ColconIndexPackageAugmentation(PackageAugmentationExtensionPoint):
+    """
+    Augment installed packages with information from a colcon index.
+
+    Only packages of the `installed` type are considered.
+    """
+
+    # This extension adds dependency information for installed packages more
+    # efficiently than the Python extension, so it should have a higher
+    # higher priority than it.
+    PRIORITY = 120
+
+    def __init__(self):  # noqa: D107
+        super().__init__()
+        satisfies_version(
+            PackageAugmentationExtensionPoint.EXTENSION_POINT_VERSION,
+            '^1.0')
+
+    def augment_package(  # noqa: D102
+        self, desc, *, additional_argument_names=None
+    ):
+        if desc.type != 'installed':
+            return
+
+        marker_file = desc.path / get_relative_package_index_path() / desc.name
+        if not marker_file.is_file():
+            return
+
+        with marker_file.open() as f:
+            raw_deps = f.read().split(os.pathsep)
+        desc.type = 'installed.colcon'
+        desc.dependencies['run'].update(
+            DependencyDescriptor(dep) for dep in raw_deps)

colcon_installed_package_information/package_augmentation/python.py

@@ -0,0 +1,58 @@
+# Copyright 2022 Open Source Robotics Foundation, Inc.
+# Licensed under the Apache License, Version 2.0
+
+import sysconfig
+
+from colcon_core.package_augmentation \
+    import PackageAugmentationExtensionPoint
+from colcon_core.package_augmentation.python \
+    import create_dependency_descriptor
+from colcon_core.plugin_system import satisfies_version
+from pkg_resources import Environment
+from pkg_resources import Requirement
+
+
+class InstalledPythonPackageAugmentation(PackageAugmentationExtensionPoint):
+    """
+    Augment installed packages with Python distribution information.
+
+    Only packages of the `installed` type are considered.
+    """
+
+    def __init__(self):  # noqa: D107
+        super().__init__()
+        satisfies_version(
+            PackageAugmentationExtensionPoint.EXTENSION_POINT_VERSION,
+            '^1.0')
+
+    def augment_packages(  # noqa: D102
+        self, descs, *, additional_argument_names=None
+    ):
+        descs = {d for d in descs if d.type == 'installed'}
+        if not descs:
+            return
+
+        environments = {}
+        for desc in descs:
+            key = Requirement.parse(desc.name).key
+            for lib_dir in _enumerate_python_dirs(str(desc.path)):
+                if lib_dir not in environments:
+                    environments[lib_dir] = Environment([lib_dir])
+                dist = next(iter(environments[lib_dir][key]), None)
+                if dist:
+                    break
+            else:
+                continue
+
+            if dist.version and not desc.metadata.get('version'):
+                desc.metadata['version'] = dist.version
+            desc.type = 'installed.python'
+            desc.dependencies['run'].update(
+                create_dependency_descriptor(str(req))
+                for req in dist.requires())
+
+
+def _enumerate_python_dirs(prefix):
+    get_path_vars = {'base': prefix, 'platbase': prefix}
+    yield sysconfig.get_path('purelib', vars=get_path_vars)
+    yield sysconfig.get_path('platlib', vars=get_path_vars)

colcon_installed_package_information/package_discovery/__init__.py

@@ -0,0 +1,71 @@
+# Copyright 2022 Open Source Robotics Foundation, Inc.
+# Licensed under the Apache License, Version 2.0
+
+from colcon_core.logging import colcon_logger
+from colcon_core.package_discovery import \
+    add_package_discovery_arguments as add_package_discovery_arguments_impl
+from colcon_core.package_discovery import \
+    discover_packages as discover_packages_impl
+from colcon_core.plugin_system import instantiate_extensions
+from colcon_core.plugin_system import order_extensions_by_priority
+
+logger = colcon_logger.getChild(__name__)
+
+
+def get_package_discovery_extensions():
+    """
+    Get the available package discovery extensions.
+
+    The extensions are ordered by their priority and entry point name.
+
+    :rtype: OrderedDict
+    """
+    extensions = instantiate_extensions(__name__)
+    for name, extension in extensions.items():
+        extension.PACKAGE_DISCOVERY_NAME = name
+    return order_extensions_by_priority(extensions)
+
+
+def add_package_discovery_arguments(parser, *, extensions=None):
+    """
+    Add the command line arguments for the package discovery extensions.
+
+    :param parser: The argument parser
+    :param extensions: The package discovery extensions to use, if `None` is
+      passed use the extensions provided by
+      :function:`get_package_discovery_extensions`
+    """
+    if extensions is None:
+        extensions = get_package_discovery_extensions()
+    add_package_discovery_arguments_impl(parser, extensions=extensions)
+
+
+def discover_packages(
+    args, identification_extensions, *, discovery_extensions=None
+):
+    """
+    Discover and identify packages.
+
+    All discovery extensions which report to have parameters are being used to
+    discover packages.
+    If none report to have parameters all discovery extensions are being used
+    but only the one with a default value should discover packages.
+    Each discovery extension uses the passed identification extensions to check
+    each potential location for the existence of a package.
+
+    :param args: The parsed command line arguments
+    :param identification_extensions: The package identification extensions to
+      pass to each invocation of
+      :function:`PackageDiscoveryExtensionPoint.discover`
+    :param discovery_extensions: The package discovery extensions to use, if
+      `None` is passed use the extensions provided by
+      :function:`get_package_discovery_extensions`
+    :returns: set of
+      :py:class:`colcon_core.package_descriptor.PackageDescriptor`
+    :rtype: set
+    """
+    if discovery_extensions is None:
+        discovery_extensions = get_package_discovery_extensions()
+    return discover_packages_impl(
+        args, identification_extensions,
+        discovery_extensions=discovery_extensions)

colcon_installed_package_information/package_discovery/prefix_path.py

@@ -0,0 +1,40 @@
+# Copyright 2022 Open Source Robotics Foundation, Inc.
+# Licensed under the Apache License, Version 2.0
+
+from pathlib import Path
+
+from colcon_core.package_descriptor import PackageDescriptor
+from colcon_core.package_discovery import PackageDiscoveryExtensionPoint
+from colcon_core.plugin_system import satisfies_version
+from colcon_core.prefix_path import get_chained_prefix_path
+from colcon_core.shell import find_installed_packages
+from colcon_installed_package_information.package_discovery import logger
+
+
+class PrefixPathPackageDiscovery(PackageDiscoveryExtensionPoint):
+    """Discover packages in chained prefix paths."""
+
+    def __init__(self):  # noqa: D107
+        super().__init__()
+        satisfies_version(
+            PackageDiscoveryExtensionPoint.EXTENSION_POINT_VERSION, '^1.0')
+
+    def has_parameters(self, *, args):  # noqa: D102
+        return False
+
+    def discover(self, *, args, identification_extensions):  # noqa: D102
+        descs = set()
+
+        for priority, prefix_path in enumerate(get_chained_prefix_path()):
+            packages = find_installed_packages(Path(prefix_path))
+            num_packages = len(packages)
+            logger.debug('Found {num_packages} installed packages in '
+                         '{prefix_path}'.format_map(locals()))
+            for pkg, path in (packages or {}).items():
+                desc = PackageDescriptor(path)
+                desc.name = pkg
+                desc.type = 'installed'
+                desc.metadata['override_priority'] = priority
+                descs.add(desc)
+
+        return descs

colcon_installed_package_information/package_identification/__init__.py

@@ -0,0 +1,20 @@
+# Copyright 2022 Open Source Robotics Foundation, Inc.
+# Licensed under the Apache License, Version 2.0
+
+from colcon_core.plugin_system import instantiate_extensions
+from colcon_core.plugin_system import order_extensions_grouped_by_priority
+
+
+def get_package_identification_extensions():
+    """
+    Get the available package identification extensions.
+
+    The extensions are grouped by their priority and each group is ordered by
+    the entry point name.
+
+    :rtype: OrderedDict
+    """
+    extensions = instantiate_extensions(__name__)
+    for name, extension in extensions.items():
+        extension.PACKAGE_IDENTIFICATION_NAME = name
+    return order_extensions_grouped_by_priority(extensions)

setup.cfg

@@ -56,6 +56,15 @@ filterwarnings =
 junit_suite_name = colcon-installed-package-information
 
 [options.entry_points]
+colcon_core.extension_point =
+    colcon_installed_package_information.package_augmentation = colcon_core.package_augmentation:PackageAugmentationExtensionPoint
+    colcon_installed_package_information.package_discovery = colcon_core.package_discovery:PackageDiscoveryExtensionPoint
+    colcon_installed_package_information.package_identification = colcon_core.package_identification:PackageIdentificationExtensionPoint
+colcon_installed_package_information.package_augmentation =
+    colcon_index = colcon_installed_package_information.package_augmentation.colcon_index:ColconIndexPackageAugmentation
+    python = colcon_installed_package_information.package_augmentation.python:InstalledPythonPackageAugmentation
+colcon_installed_package_information.package_discovery =
+    prefix_path = colcon_installed_package_information.package_discovery.prefix_path:PrefixPathPackageDiscovery
 
 [flake8]
 import-order-style = google

test/spell_check.words

@@ -1,9 +1,18 @@
 apache
 colcon
+deps
+descs
 iterdir
+noqa
 pathlib
+platbase
+platlib
+plugin
+purelib
 pytest
+rtype
 scott
 scspell
 setuptools
+sysconfig
 thomas