Base set of changes for the CodebaseResource migration #569

Signed-off-by: Thomas Druez <[email protected]>

scanpipe/api/serializers.py

@@ -251,8 +251,11 @@ class Meta:
             "is_archive",
             "is_media",
             "is_key_file",
-            "licenses",
-            "license_expressions",
+            "detected_license_expression",
+            "detected_license_expression_spdx",
+            "license_detections",
+            "license_clues",
+            "percentage_of_license_text",
             "compliance_alert",
             "copyrights",
             "holders",

scanpipe/filters.py

@@ -335,10 +335,11 @@ class ResourceFilterSet(FilterSetUtilsMixin, django_filters.FilterSet):
         label="License key",
         field_name="licenses",
     )
-    license_category = JSONContainsFilter(
-        label="License category",
-        field_name="licenses",
-    )
+    # TODO: We will have to compute that field as not available anymore
+    # license_category = JSONContainsFilter(
+    #     label="License category",
+    #     field_name="licenses",
+    # )
     compliance_alert = django_filters.ChoiceFilter(
         choices=CodebaseResource.Compliance.choices + [("EMPTY", "EMPTY")]
     )
@@ -373,9 +374,12 @@ class Meta:
             "copyrights",
             "holders",
             "authors",
-            "licenses",
-            "license_category",
-            "license_expressions",
+            # "license_category",
+            "detected_license_expression",
+            "detected_license_expression_spdx",
+            "license_detections",
+            "license_clues",
+            "percentage_of_license_text",
             "emails",
             "urls",
             "in_package",

scanpipe/migrations/0032_remove_codebaseresource_license_expressions_and_more.py

@@ -0,0 +1,49 @@
+# Generated by Django 4.2 on 2023-05-05 15:08
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("scanpipe", "0031_scancode_toolkit_v32_data_updates"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="codebaseresource",
+            name="license_expressions",
+        ),
+        migrations.RemoveField(
+            model_name="codebaseresource",
+            name="licenses",
+        ),
+        migrations.AddField(
+            model_name="codebaseresource",
+            name="detected_license_expression",
+            field=models.TextField(blank=True, help_text=""),
+        ),
+        migrations.AddField(
+            model_name="codebaseresource",
+            name="detected_license_expression_spdx",
+            field=models.TextField(blank=True, help_text=""),
+        ),
+        migrations.AddField(
+            model_name="codebaseresource",
+            name="license_clues",
+            field=models.JSONField(
+                blank=True, default=list, help_text="List of license clues."
+            ),
+        ),
+        migrations.AddField(
+            model_name="codebaseresource",
+            name="license_detections",
+            field=models.JSONField(
+                blank=True, default=list, help_text="List of license detection details."
+            ),
+        ),
+        migrations.AddField(
+            model_name="codebaseresource",
+            name="percentage_of_license_text",
+            field=models.FloatField(blank=True, help_text="", null=True),
+        ),
+    ]

scanpipe/migrations/0033_alter_codebaseresource_detected_license_expression_and_more.py

@@ -0,0 +1,27 @@
+# Generated by Django 4.2 on 2023-05-05 15:08
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("scanpipe", "0032_remove_codebaseresource_license_expressions_and_more"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="codebaseresource",
+            name="detected_license_expression",
+            field=models.TextField(blank=True, help_text=""),
+        ),
+        migrations.AlterField(
+            model_name="codebaseresource",
+            name="detected_license_expression_spdx",
+            field=models.TextField(blank=True, help_text=""),
+        ),
+        migrations.AlterField(
+            model_name="codebaseresource",
+            name="percentage_of_license_text",
+            field=models.FloatField(blank=True, help_text="", null=True),
+        ),
+    ]

scanpipe/models.py

@@ -926,8 +926,8 @@ def less_common(self, field_name, limit=most_common_limit):
         `field_name` starting at the `limit`.
         """
         json_fields_mapping = {
-            "license_key": ("licenses", "key"),
-            "license_category": ("licenses", "category"),
+            "license_key": ("license_detections", "key"),
+            "license_category": ("license_detections", "category"),
             "copyrights": ("copyrights", "copyright"),
             "holders": ("holders", "holder"),
         }
@@ -1438,9 +1438,33 @@ def has_value(self, field_name):
         return self.filter(~Q((f"{field_name}__in", EMPTY_VALUES)))
 
 
+# TODO: Add in UI, API, ...
 class ScanFieldsModelMixin(models.Model):
     """Fields returned by the ScanCode-toolkit scans."""
 
+    detected_license_expression = models.TextField(
+        blank=True,
+        help_text=_(""),
+    )
+    detected_license_expression_spdx = models.TextField(
+        blank=True,
+        help_text=_(""),
+    )
+    license_detections = models.JSONField(
+        blank=True,
+        default=list,
+        help_text=_("List of license detection details."),
+    )
+    license_clues = models.JSONField(
+        blank=True,
+        default=list,
+        help_text=_("List of license clues."),
+    )
+    percentage_of_license_text = models.FloatField(
+        blank=True,
+        null=True,
+        help_text=_(""),
+    )
     copyrights = models.JSONField(
         blank=True,
         default=list,
@@ -1460,16 +1484,6 @@ class ScanFieldsModelMixin(models.Model):
         default=list,
         help_text=_("List of detected authors (and related detection details)."),
     )
-    licenses = models.JSONField(
-        blank=True,
-        default=list,
-        help_text=_("List of license detection details."),
-    )
-    license_expressions = models.JSONField(
-        blank=True,
-        default=list,
-        help_text=_("List of detected license expressions."),
-    )
     emails = models.JSONField(
         blank=True,
         default=list,
@@ -1527,7 +1541,7 @@ class CodebaseResource(
     A project Codebase Resources are records of its code files and directories.
     Each record is identified by its path under the project workspace.
 
-    These model fields should be kept in line with `scancode.resource.Resource`.
+    These model fields should be kept in line with `commoncode.resource.Resource`.
     """
 
     path = models.CharField(
@@ -1688,15 +1702,17 @@ def save(self, codebase=None, *args, **kwargs):
         """
         if scanpipe_app.policies_enabled:
             loaded_licenses = getattr(self, "loaded_licenses", [])
-            if self.licenses != loaded_licenses:
+            # TODO: Use detected_license_expression instead
+            if self.license_detections != loaded_licenses:
                 self.inject_licenses_policy(scanpipe_app.license_policies_index)
                 self.compliance_alert = self.compute_compliance_alert()
 
         super().save(*args, **kwargs)
 
     def inject_licenses_policy(self, policies_index):
         """Inject license policies from the `policies_index` into the licenses field."""
-        for license_data in self.licenses:
+        for license_data in self.license_detections:
+            # TODO: key is not available anymore, license_expression only
             key = license_data.get("key")
             license_data["policy"] = policies_index.get(key, None)
 
@@ -1729,7 +1745,8 @@ def is_symlink(self):
 
     def compute_compliance_alert(self):
         """Compute and return the compliance_alert value from the licenses policies."""
-        if not self.licenses:
+        # TODO: Base this on self.detected_license_expression
+        if not self.license_detections:
             return ""
 
         ok = self.Compliance.OK
@@ -1738,7 +1755,7 @@ def compute_compliance_alert(self):
         missing = self.Compliance.MISSING
 
         alerts = []
-        for license_data in self.licenses:
+        for license_data in self.license_detections:
             policy = license_data.get("policy")
             if policy:
                 alerts.append(policy.get("compliance_alert") or ok)
@@ -1753,6 +1770,7 @@ def compute_compliance_alert(self):
             return missing
         return ok
 
+    # TODO: Remove this
     @property
     def unique_license_expressions(self):
         """Return the sorted set of unique license_expressions."""
@@ -1937,7 +1955,9 @@ def as_spdx(self):
         """Return this CodebaseResource as an SPDX Package entry."""
         from scanpipe.pipes import spdx
 
-        spdx_license_keys = [license["spdx_license_key"] for license in self.licenses]
+        spdx_license_keys = [
+            license["spdx_license_key"] for license in self.license_detections
+        ]
         copyrights = [copyright["copyright"] for copyright in self.copyrights]
         holders = [holder["holder"] for holder in self.holders]
         authors = [author["author"] for author in self.authors]

scanpipe/pipes/resolve.py

@@ -29,9 +29,6 @@
 from attributecode.model import About
 from packagedcode import APPLICATION_PACKAGE_DATAFILE_HANDLERS
 from packagedcode.licensing import get_license_detections_and_expression
-from packagedcode.licensing import (
-    get_license_detections_for_extracted_license_statement,
-)
 from packageurl import PackageURL
 from python_inspector.resolve_cli import resolver_api
 from scancode.api import get_package_data
@@ -108,14 +105,7 @@ def convert_spdx_expression(license_expression_spdx):
     Return an ScanCode license expression from a SPDX `license_expression_spdx`
     string.
     """
-    license_detections = get_license_detections_for_extracted_license_statement(
-        license_expression_spdx,
-        try_as_expression=True,
-    )
-    if license_detections:
-        return license_detections[0].license_expression
-
-    return ""
+    return get_license_detections_and_expression(license_expression_spdx)[1]
 
 
 def spdx_package_to_discovered_package_data(spdx_package):

scanpipe/templates/scanpipe/resource_list.html

@@ -55,13 +55,7 @@
               <a href="?tag={{ resource.tag }}" class="is-black-link">{{ resource.tag }}</a>
             </td>
             <td>
-               <ul>
-                {% for license_expression in resource.unique_license_expressions %}
-                  <li>
-                    <a href="?license_expressions={{ license_expression }}" class="is-black-link">{{ license_expression }}</a>
-                  </li>
-                {% endfor %}
-              </ul>
+               <a href="?license_expressions={{ resource.detected_license_expression }}" class="is-black-link">{{ resource.detected_license_expression }}</a>
             </td>
             {% if include_compliance_alert %}
               <td>

scanpipe/tests/pipes/test_resolve.py

@@ -78,7 +78,7 @@ def test_scanpipe_pipes_resolve_set_license_expression(self):
 
     def test_scanpipe_pipes_resolve_convert_spdx_expression(self):
         spdx = "MIT OR GPL-2.0-only WITH LicenseRef-scancode-generic-exception"
-        scancode_expression = "mit OR gpl-2.0 with generic-exception"
+        scancode_expression = "mit OR gpl-2.0 WITH generic-exception"
         self.assertEqual(scancode_expression, resolve.convert_spdx_expression(spdx))
 
     def test_scanpipe_pipes_resolve_resolve_packages(self):

scanpipe/views.py

@@ -619,7 +619,7 @@ def get_context_data(self, **kwargs):
             "mime_type",
             "holders",
             "copyrights",
-            "license_expressions",
+            "detected_license_expression",
         )
 
         packages = project.discoveredpackages.all().only(
@@ -861,7 +861,7 @@ class CodebaseResourceListView(
         "programming_language",
         "mime_type",
         "tag",
-        "license_expressions",
+        "detected_license_expression",
         {
             "field_name": "compliance_alert",
             "condition": scanpipe_app.policies_enabled,
@@ -1051,7 +1051,14 @@ class CodebaseResourceDetailsView(
         },
         "detection": {
             "fields": [
-                {"field_name": "license_expressions", "render_func": render_as_yaml},
+                "detected_license_expression",
+                {
+                    "field_name": "detected_license_expression_spdx",
+                    "label": "Detected license expression (SPDX)",
+                },
+                {"field_name": "license_detections", "render_func": render_as_yaml},
+                {"field_name": "license_clues", "render_func": render_as_yaml},
+                "percentage_of_license_text",
                 {"field_name": "copyrights", "render_func": render_as_yaml},
                 {"field_name": "holders", "render_func": render_as_yaml},
                 {"field_name": "authors", "render_func": render_as_yaml},
@@ -1135,7 +1142,7 @@ def get_context_data(self, **kwargs):
             messages.warning(self.request, message)
 
         context["detected_values"] = {
-            "licenses": self.get_annotations("licenses"),
+            "licenses": self.get_annotations("license_detections"),
             "copyrights": self.get_annotations("copyrights"),
             "holders": self.get_annotations("holders"),
             "authors": self.get_annotations("authors"),