diff --git a/README.rst b/README.rst index ff86f54..4a7c1c9 100644 --- a/README.rst +++ b/README.rst @@ -2,3 +2,31 @@ colcon-installed-package-information ==================================== Extensions for `colcon-core `_ to inspect packages which have already been installed. + +Overview +-------- + +These colcon extensions provide a mechanism which can be used for getting information about packages outside of the workspace, which have already been built and installed prior to the current operation. +In general, it works similarly to and is based on the `PackageDiscoveryExtensionPoint `_ and `PackageAugmentationExtensionPoint `_ extensions provided by ``colcon-core``. + +Differences +----------- + +Installed packages don't generally have a single directory which stores the package content and metadata. +This set of extensions store the "prefix" under which the package resides rather than the package directory, meaning many packages will likely share the same ``path`` attribute value. + +Recursively crawling an entire system or even selective subdirectories to look for installed packages could be very slow, so this process also deviates from the Discover -> Identify -> Augment pipeline used in ``colcon-core``. +Rather than attempting identification on perspective package locations, the discovery phase generally loads a list of installed packages from a database of some kind. +In some cases, the database might already populate sufficient information on the descriptor to identify the package. +For others, only presence can be known, and augmentation extensions must add additional information to the descriptor by searching for specific files throughout the prefix directory. + +The ``type`` attribute of an installed package works similarly to workspace packages, but must always start with ``installed.`` followed a more specific package type. +If more information about a package cannot be determined and it is known only to exist under a certain prefix, the time should be set to ``installed``. + +Supported Package Types +----------------------- + +This package provides extensions which are able to discover packages using the ``PrefixPathExtensionPoint`` to enumerate install prefixes, and ``FindInstalledPackagesExtensionPoint`` to enumerate names of packages installed under those prefixes. +It can then use the colcon index in those prefixes as well as python eggs to determine dependency information and augment the packages appropriately. + +Support for more package databases for discovery and augmentation can be added by other packages by implementing and registering appropriate extensions in other packages. diff --git a/colcon_installed_package_information/__init__.py b/colcon_installed_package_information/__init__.py index efc72df..1826f2b 100644 --- a/colcon_installed_package_information/__init__.py +++ b/colcon_installed_package_information/__init__.py @@ -1,4 +1,4 @@ # Copyright 2022 Scott K Logan # Licensed under the Apache License, Version 2.0 -__version__ = '0.0.0' +__version__ = '0.0.1' diff --git a/colcon_installed_package_information/package_augmentation/__init__.py b/colcon_installed_package_information/package_augmentation/__init__.py new file mode 100644 index 0000000..a0ef566 --- /dev/null +++ b/colcon_installed_package_information/package_augmentation/__init__.py @@ -0,0 +1,39 @@ +# Copyright 2022 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +from colcon_core.package_augmentation \ + import augment_packages as augment_packages_impl +from colcon_core.plugin_system import instantiate_extensions +from colcon_core.plugin_system import order_extensions_by_priority + + +def get_package_augmentation_extensions(): + """ + Get the available package augmentation extensions. + + The extensions are ordered by their priority and entry point name. + + :rtype: OrderedDict + """ + extensions = instantiate_extensions(__name__) + for name, extension in extensions.items(): + extension.PACKAGE_AUGMENTATION_NAME = name + return order_extensions_by_priority(extensions) + + +def augment_packages( + descs, *, additional_argument_names=None, augmentation_extensions=None +): + """ + Augment package descriptors with additional information. + + :param descs: the packages + :type descs: set of + :py:class:`colcon_core.package_descriptor.PackageDescriptor` + """ + if augmentation_extensions is None: + augmentation_extensions = get_package_augmentation_extensions() + + augment_packages_impl( + descs, additional_argument_names=additional_argument_names, + augmentation_extensions=augmentation_extensions) diff --git a/colcon_installed_package_information/package_augmentation/colcon_index.py b/colcon_installed_package_information/package_augmentation/colcon_index.py new file mode 100644 index 0000000..5b29884 --- /dev/null +++ b/colcon_installed_package_information/package_augmentation/colcon_index.py @@ -0,0 +1,45 @@ +# Copyright 2022 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +import os + +from colcon_core.dependency_descriptor import DependencyDescriptor +from colcon_core.location import get_relative_package_index_path +from colcon_core.package_augmentation \ + import PackageAugmentationExtensionPoint +from colcon_core.plugin_system import satisfies_version + + +class ColconIndexPackageAugmentation(PackageAugmentationExtensionPoint): + """ + Augment installed packages with information from a colcon index. + + Only packages of the `installed` type are considered. + """ + + # This extension adds dependency information for installed packages more + # efficiently than the Python extension, so it should have a higher + # higher priority than it. + PRIORITY = 120 + + def __init__(self): # noqa: D107 + super().__init__() + satisfies_version( + PackageAugmentationExtensionPoint.EXTENSION_POINT_VERSION, + '^1.0') + + def augment_package( # noqa: D102 + self, desc, *, additional_argument_names=None + ): + if desc.type != 'installed': + return + + marker_file = desc.path / get_relative_package_index_path() / desc.name + if not marker_file.is_file(): + return + + with marker_file.open() as f: + raw_deps = f.read().split(os.pathsep) + desc.type = 'installed.colcon' + desc.dependencies['run'].update( + DependencyDescriptor(dep) for dep in raw_deps) diff --git a/colcon_installed_package_information/package_augmentation/python.py b/colcon_installed_package_information/package_augmentation/python.py new file mode 100644 index 0000000..b89599f --- /dev/null +++ b/colcon_installed_package_information/package_augmentation/python.py @@ -0,0 +1,58 @@ +# Copyright 2022 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +import sysconfig + +from colcon_core.package_augmentation \ + import PackageAugmentationExtensionPoint +from colcon_core.package_augmentation.python \ + import create_dependency_descriptor +from colcon_core.plugin_system import satisfies_version +from pkg_resources import Environment +from pkg_resources import Requirement + + +class InstalledPythonPackageAugmentation(PackageAugmentationExtensionPoint): + """ + Augment installed packages with Python distribution information. + + Only packages of the `installed` type are considered. + """ + + def __init__(self): # noqa: D107 + super().__init__() + satisfies_version( + PackageAugmentationExtensionPoint.EXTENSION_POINT_VERSION, + '^1.0') + + def augment_packages( # noqa: D102 + self, descs, *, additional_argument_names=None + ): + descs = {d for d in descs if d.type == 'installed'} + if not descs: + return + + environments = {} + for desc in descs: + key = Requirement.parse(desc.name).key + for lib_dir in _enumerate_python_dirs(str(desc.path)): + if lib_dir not in environments: + environments[lib_dir] = Environment([lib_dir]) + dist = next(iter(environments[lib_dir][key]), None) + if dist: + break + else: + continue + + if dist.version and not desc.metadata.get('version'): + desc.metadata['version'] = dist.version + desc.type = 'installed.python' + desc.dependencies['run'].update( + create_dependency_descriptor(str(req)) + for req in dist.requires()) + + +def _enumerate_python_dirs(prefix): + get_path_vars = {'base': prefix, 'platbase': prefix} + yield sysconfig.get_path('purelib', vars=get_path_vars) + yield sysconfig.get_path('platlib', vars=get_path_vars) diff --git a/colcon_installed_package_information/package_discovery/__init__.py b/colcon_installed_package_information/package_discovery/__init__.py new file mode 100644 index 0000000..d2deb11 --- /dev/null +++ b/colcon_installed_package_information/package_discovery/__init__.py @@ -0,0 +1,71 @@ +# Copyright 2022 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +from colcon_core.logging import colcon_logger +from colcon_core.package_discovery import \ + add_package_discovery_arguments as add_package_discovery_arguments_impl +from colcon_core.package_discovery import \ + discover_packages as discover_packages_impl +from colcon_core.plugin_system import instantiate_extensions +from colcon_core.plugin_system import order_extensions_by_priority + +logger = colcon_logger.getChild(__name__) + + +def get_package_discovery_extensions(): + """ + Get the available package discovery extensions. + + The extensions are ordered by their priority and entry point name. + + :rtype: OrderedDict + """ + extensions = instantiate_extensions(__name__) + for name, extension in extensions.items(): + extension.PACKAGE_DISCOVERY_NAME = name + return order_extensions_by_priority(extensions) + + +def add_package_discovery_arguments(parser, *, extensions=None): + """ + Add the command line arguments for the package discovery extensions. + + :param parser: The argument parser + :param extensions: The package discovery extensions to use, if `None` is + passed use the extensions provided by + :function:`get_package_discovery_extensions` + """ + if extensions is None: + extensions = get_package_discovery_extensions() + add_package_discovery_arguments_impl(parser, extensions=extensions) + + +def discover_packages( + args, identification_extensions, *, discovery_extensions=None +): + """ + Discover and identify packages. + + All discovery extensions which report to have parameters are being used to + discover packages. + If none report to have parameters all discovery extensions are being used + but only the one with a default value should discover packages. + Each discovery extension uses the passed identification extensions to check + each potential location for the existence of a package. + + :param args: The parsed command line arguments + :param identification_extensions: The package identification extensions to + pass to each invocation of + :function:`PackageDiscoveryExtensionPoint.discover` + :param discovery_extensions: The package discovery extensions to use, if + `None` is passed use the extensions provided by + :function:`get_package_discovery_extensions` + :returns: set of + :py:class:`colcon_core.package_descriptor.PackageDescriptor` + :rtype: set + """ + if discovery_extensions is None: + discovery_extensions = get_package_discovery_extensions() + return discover_packages_impl( + args, identification_extensions, + discovery_extensions=discovery_extensions) diff --git a/colcon_installed_package_information/package_discovery/prefix_path.py b/colcon_installed_package_information/package_discovery/prefix_path.py new file mode 100644 index 0000000..0cfad41 --- /dev/null +++ b/colcon_installed_package_information/package_discovery/prefix_path.py @@ -0,0 +1,40 @@ +# Copyright 2022 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +from pathlib import Path + +from colcon_core.package_descriptor import PackageDescriptor +from colcon_core.package_discovery import PackageDiscoveryExtensionPoint +from colcon_core.plugin_system import satisfies_version +from colcon_core.prefix_path import get_chained_prefix_path +from colcon_core.shell import find_installed_packages +from colcon_installed_package_information.package_discovery import logger + + +class PrefixPathPackageDiscovery(PackageDiscoveryExtensionPoint): + """Discover packages in chained prefix paths.""" + + def __init__(self): # noqa: D107 + super().__init__() + satisfies_version( + PackageDiscoveryExtensionPoint.EXTENSION_POINT_VERSION, '^1.0') + + def has_parameters(self, *, args): # noqa: D102 + return False + + def discover(self, *, args, identification_extensions): # noqa: D102 + descs = set() + + for priority, prefix_path in enumerate(get_chained_prefix_path()): + packages = find_installed_packages(Path(prefix_path)) + num_packages = len(packages) + logger.debug('Found {num_packages} installed packages in ' + '{prefix_path}'.format_map(locals())) + for pkg, path in (packages or {}).items(): + desc = PackageDescriptor(path) + desc.name = pkg + desc.type = 'installed' + desc.metadata['override_priority'] = priority + descs.add(desc) + + return descs diff --git a/colcon_installed_package_information/package_identification/__init__.py b/colcon_installed_package_information/package_identification/__init__.py new file mode 100644 index 0000000..8b07069 --- /dev/null +++ b/colcon_installed_package_information/package_identification/__init__.py @@ -0,0 +1,20 @@ +# Copyright 2022 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +from colcon_core.plugin_system import instantiate_extensions +from colcon_core.plugin_system import order_extensions_grouped_by_priority + + +def get_package_identification_extensions(): + """ + Get the available package identification extensions. + + The extensions are grouped by their priority and each group is ordered by + the entry point name. + + :rtype: OrderedDict + """ + extensions = instantiate_extensions(__name__) + for name, extension in extensions.items(): + extension.PACKAGE_IDENTIFICATION_NAME = name + return order_extensions_grouped_by_priority(extensions) diff --git a/setup.cfg b/setup.cfg index 1b8961e..b43a8f0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -56,6 +56,15 @@ filterwarnings = junit_suite_name = colcon-installed-package-information [options.entry_points] +colcon_core.extension_point = + colcon_installed_package_information.package_augmentation = colcon_core.package_augmentation:PackageAugmentationExtensionPoint + colcon_installed_package_information.package_discovery = colcon_core.package_discovery:PackageDiscoveryExtensionPoint + colcon_installed_package_information.package_identification = colcon_core.package_identification:PackageIdentificationExtensionPoint +colcon_installed_package_information.package_augmentation = + colcon_index = colcon_installed_package_information.package_augmentation.colcon_index:ColconIndexPackageAugmentation + python = colcon_installed_package_information.package_augmentation.python:InstalledPythonPackageAugmentation +colcon_installed_package_information.package_discovery = + prefix_path = colcon_installed_package_information.package_discovery.prefix_path:PrefixPathPackageDiscovery [flake8] import-order-style = google diff --git a/test/spell_check.words b/test/spell_check.words index d68f6b3..6f231d2 100644 --- a/test/spell_check.words +++ b/test/spell_check.words @@ -1,9 +1,18 @@ apache colcon +deps +descs iterdir +noqa pathlib +platbase +platlib +plugin +purelib pytest +rtype scott scspell setuptools +sysconfig thomas