Support ABOUT files for code that is patched or vendored #740 (#778)

Signed-off-by: Thomas Druez <[email protected]>

CHANGELOG.rst

@@ -34,6 +34,9 @@ v33.0.0 (unreleased)
   using the a new ``--notes`` option.
   https://github.com/nexB/scancode.io/issues/709
 
+- Add a mapper function to relate .ABOUT files during the d2d pipeline.
+  https://github.com/nexB/scancode.io/issues/740
+
 - Enhance the file viewer UI of the resource details view.
   A new search for the file content was added.
   Also, it is now possible to expand the file viewer in full screen mode.

scanpipe/filters.py

@@ -287,6 +287,7 @@ def __init__(self, *args, **kwargs):
             ("none", "No map"),
             ("any", "Any map"),
             ("many", "Many map"),
+            ("about_file", "about file"),
             ("java_to_class", "java to class"),
             ("jar_to_source", "jar to source"),
             ("js_compiled", "js compiled"),

scanpipe/pipelines/deploy_to_develop.py

@@ -47,6 +47,7 @@ def steps(cls):
             cls.collect_and_create_codebase_resources,
             cls.flag_empty_files,
             cls.flag_ignored_resources,
+            cls.map_about_files,
             cls.map_checksum,
             cls.find_java_packages,
             cls.map_java_to_class,
@@ -103,6 +104,10 @@ def collect_and_create_codebase_resources(self):
         """Collect and create codebase resources."""
         d2d.collect_and_create_codebase_resources(self.project)
 
+    def map_about_files(self):
+        """Map ``from/`` .ABOUT files to their related ``to/`` resources."""
+        d2d.map_about_files(project=self.project, logger=self.log)
+
     def map_checksum(self):
         """Map using SHA1 checksum."""
         d2d.map_checksum(project=self.project, checksum_field="sha1", logger=self.log)

scanpipe/pipes/d2d.py

@@ -33,6 +33,7 @@
 from scanpipe.pipes import jvm
 from scanpipe.pipes import pathmap
 from scanpipe.pipes import purldb
+from scanpipe.pipes import resolve
 from scanpipe.pipes import scancode
 
 FROM = "from/"
@@ -606,3 +607,52 @@ def _map_javascript_resource(
                 extra_data=extra_data,
             )
             resource.update(status=flag.MAPPED)
+
+
+def _map_about_file_resource(project, about_file_resource, to_resources):
+    about_file_location = str(about_file_resource.location_path)
+    package_data = resolve.resolve_about_package(about_file_location)
+    if not package_data:
+        return
+
+    filename = package_data.get("filename")
+    if not filename:
+        # Cannot map anything without the about_resource value.
+        return
+
+    # Fetch all resources that are covered by the .ABOUT file.
+    codebase_resources = to_resources.filter(path__contains=f"/{filename.lstrip('/')}")
+    if not codebase_resources:
+        # If there's nothing to map on the ``to/`` do not create the package.
+        return
+
+    # Create the Package using .ABOUT data and assigned related codebase_resources
+    pipes.update_or_create_package(project, package_data, codebase_resources)
+
+    # Map the .ABOUT file resource to all related resources in the ``to/`` side.
+    for to_resource in codebase_resources:
+        pipes.make_relation(
+            from_resource=about_file_resource,
+            to_resource=to_resource,
+            map_type="about_file",
+        )
+
+    codebase_resources.update(status=flag.MAPPED)
+    about_file_resource.update(status=flag.MAPPED)
+
+
+def map_about_files(project, logger=None):
+    """Map ``from/`` .ABOUT files to their related ``to/`` resources."""
+    project_resources = project.codebaseresources
+    from_files = project_resources.files().from_codebase()
+    from_about_files = from_files.filter(extension=".ABOUT")
+    to_resources = project_resources.to_codebase()
+
+    if logger:
+        logger(
+            f"Mapping {from_about_files.count():,d} .ABOUT files found in the from/ "
+            f"codebase."
+        )
+
+    for about_file_resource in from_about_files:
+        _map_about_file_resource(project, about_file_resource, to_resources)

scanpipe/pipes/resolve.py

@@ -74,8 +74,8 @@ def resolve_pypi_packages(input_location):
     return inspector_output.packages
 
 
-def resolve_about_packages(input_location):
-    """Resolve the packages from the `input_location` .ABOUT file."""
+def resolve_about_package(input_location):
+    """Resolve the package from the ``input_location`` .ABOUT file."""
     about = About(location=input_location)
     about_data = about.as_dict()
     package_data = about_data.copy()
@@ -92,12 +92,23 @@ def resolve_about_packages(input_location):
     if license_expression := about_data.get("license_expression"):
         package_data["declared_license_expression"] = license_expression
 
+    if notice_dict := about_data.get("notice_file"):
+        package_data["notice_text"] = list(notice_dict.values())[0]
+
     for field_name, value in about_data.items():
         if field_name.startswith("checksum_"):
             package_data[field_name.replace("checksum_", "")] = value
 
     package_data = DiscoveredPackage.clean_data(package_data)
-    return [package_data]
+    return package_data
+
+
+def resolve_about_packages(input_location):
+    """
+    Wrap ``resolve_about_package`` to return a list as expected by the
+    InspectManifest pipeline.
+    """
+    return [resolve_about_package(input_location)]
 
 
 def convert_spdx_expression(license_expression_spdx):