Merge pull request #2250 from MaibornWolff/dev

chore: merge to main for release 1.22.4

.github/workflows/build_push_dev.yml

@@ -52,7 +52,7 @@ jobs:
             VERSION=dev
       - 
         name: Run SCA vulnerability scanners
-        uses: MaibornWolff/secobserve_actions_templates/actions/vulnerability_scanner@5606a843f40e25d002053dfd14cd525496c76560 # main
+        uses: MaibornWolff/secobserve_actions_templates/actions/vulnerability_scanner@d15c2401857055841b487d2f25d4a9eb6c4f4ef2 # main
         with:
           so_configuration: 'so_configuration_sca_dev.yml'
           SO_API_TOKEN: ${{ secrets.SO_API_TOKEN }}

.github/workflows/build_push_release.yml

@@ -64,13 +64,13 @@ jobs:
             VERSION=${{ github.event.inputs.release }}
       - 
         name: Run vulnerability scanners for images
-        uses: MaibornWolff/secobserve_actions_templates/actions/vulnerability_scanner@5606a843f40e25d002053dfd14cd525496c76560 # main
+        uses: MaibornWolff/secobserve_actions_templates/actions/vulnerability_scanner@d15c2401857055841b487d2f25d4a9eb6c4f4ef2 # main
         with:
           so_configuration: 'so_configuration_sca_current.yml'
           SO_API_TOKEN: ${{ secrets.SO_API_TOKEN }}
       - 
         name: Run vulnerability scanners for endpoints
-        uses: MaibornWolff/secobserve_actions_templates/actions/vulnerability_scanner@5606a843f40e25d002053dfd14cd525496c76560 # main
+        uses: MaibornWolff/secobserve_actions_templates/actions/vulnerability_scanner@d15c2401857055841b487d2f25d4a9eb6c4f4ef2 # main
         with:
           so_configuration: 'so_configuration_endpoints.yml'
           SO_API_TOKEN: ${{ secrets.SO_API_TOKEN }}

.github/workflows/check_licenses_dev.yml

@@ -37,7 +37,7 @@ jobs:
           cdxgen ./frontend --type npm --no-babel --required-only --profile license-compliance --no-auto-compositions --project-name secobserve --output sbom_frontend_application.json
       -
         name: Import backend SBOM
-        uses: MaibornWolff/secobserve_actions_templates/actions/importer@5606a843f40e25d002053dfd14cd525496c76560 # main
+        uses: MaibornWolff/secobserve_actions_templates/actions/importer@d15c2401857055841b487d2f25d4a9eb6c4f4ef2 # main
         with:
           so_product_name: 'SecObserve'
           so_file_name: 'sbom_backend_application.json'
@@ -47,7 +47,7 @@ jobs:
           so_api_token: ${{ secrets.SO_API_TOKEN }}
       -
         name: Import frontend SBOM
-        uses: MaibornWolff/secobserve_actions_templates/actions/importer@5606a843f40e25d002053dfd14cd525496c76560 # main
+        uses: MaibornWolff/secobserve_actions_templates/actions/importer@d15c2401857055841b487d2f25d4a9eb6c4f4ef2 # main
         with:
           so_product_name: 'SecObserve'
           so_file_name: 'sbom_frontend_application.json'

.github/workflows/check_vulnerabilities.yml

@@ -14,7 +14,7 @@ jobs:
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       -
         name: Run vulnerability scanners for code
-        uses: MaibornWolff/secobserve_actions_templates/actions/vulnerability_scanner@5606a843f40e25d002053dfd14cd525496c76560 # main
+        uses: MaibornWolff/secobserve_actions_templates/actions/vulnerability_scanner@d15c2401857055841b487d2f25d4a9eb6c4f4ef2 # main
         with:
           so_configuration: 'so_configuration_code.yml'
           SO_API_TOKEN: ${{ secrets.SO_API_TOKEN }}

.github/workflows/publish_docs.yml

@@ -4,7 +4,7 @@ on:
   push:
     branches:
       - main
-      - chore/doc_trivy_secrets
+      - chore/documentation_check_security_gate
 
 permissions: read-all
 

.github/workflows/scan_sca_current.yml

@@ -18,13 +18,13 @@ jobs:
           ref: 'v1.22.2'
       -
         name: Run SCA vulnerability scanners
-        uses: MaibornWolff/secobserve_actions_templates/actions/vulnerability_scanner@5606a843f40e25d002053dfd14cd525496c76560 # main
+        uses: MaibornWolff/secobserve_actions_templates/actions/vulnerability_scanner@d15c2401857055841b487d2f25d4a9eb6c4f4ef2 # main
         with:
           so_configuration: 'so_configuration_sca_current.yml'
           SO_API_TOKEN: ${{ secrets.SO_API_TOKEN }}
       - 
         name: Run endpoint vulnerability scanners
-        uses: MaibornWolff/secobserve_actions_templates/actions/vulnerability_scanner@5606a843f40e25d002053dfd14cd525496c76560 # main
+        uses: MaibornWolff/secobserve_actions_templates/actions/vulnerability_scanner@d15c2401857055841b487d2f25d4a9eb6c4f4ef2 # main
         with:
           so_configuration: 'so_configuration_endpoints.yml'
           SO_API_TOKEN: ${{ secrets.SO_API_TOKEN }}

.github/workflows/scorecard.yml

@@ -67,6 +67,6 @@ jobs:
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@ea9e4e37992a54ee68a9622e985e60c8e8f12d9f # v3.27.4
+        uses: github/codeql-action/upload-sarif@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v3.27.5
         with:
           sarif_file: results.sarif

backend/application/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "1.22.3"
+__version__ = "1.22.4"
 
 import pymysql
 

backend/application/import_observations/parsers/cyclone_dx/parser.py

@@ -141,11 +141,11 @@ def _get_component(self, component_data: dict[str, Any]) -> Optional[Component]:
         else:
             for my_license in licenses:
                 component_license = my_license.get("license", {}).get("id")
-                if component_license:
+                if component_license and component_license not in unknown_licenses:
                     unknown_licenses.append(component_license)
 
                 component_license = my_license.get("license", {}).get("name")
-                if component_license:
+                if component_license and component_license not in unknown_licenses:
                     unknown_licenses.append(component_license)
 
         return Component(

backend/application/licenses/api/filters.py

@@ -202,6 +202,22 @@ class Meta:
 
 class LicensePolicyFilter(FilterSet):
     name = CharFilter(field_name="name", lookup_expr="icontains")
+    license = NumberFilter(
+        field_name="license", method="get_license_policies_with_license"
+    )
+    license_group = NumberFilter(
+        field_name="license_group", method="get_license_policies_with_license_group"
+    )
+
+    def get_license_policies_with_license(
+        self, queryset, field_name, value  # pylint: disable=unused-argument
+    ) -> bool:
+        return queryset.filter(license_policy_items__license=value)
+
+    def get_license_policies_with_license_group(
+        self, queryset, field_name, value  # pylint: disable=unused-argument
+    ) -> bool:
+        return queryset.filter(license_policy_items__license_group=value)
 
     ordering = OrderingFilter(
         # tuple-mapping retains order

backend/application/licenses/api/views.py

@@ -86,7 +86,10 @@
     get_license_policy_member,
     get_license_policy_members,
 )
-from application.licenses.services.license_group import copy_license_group
+from application.licenses.services.license_group import (
+    copy_license_group,
+    import_scancode_licensedb,
+)
 from application.licenses.services.license_policy import (
     apply_license_policy,
     copy_license_policy,
@@ -259,6 +262,23 @@ def remove_license(self, request, pk):
 
         return Response(status=HTTP_204_NO_CONTENT)
 
+    @extend_schema(
+        methods=["POST"],
+        request=None,
+        responses={HTTP_204_NO_CONTENT: None},
+    )
+    @action(detail=False, methods=["post"])
+    def import_scancode_licensedb(self, request):
+        user = request.user
+        if not user.is_superuser:
+            raise PermissionDenied(
+                "User is not allowed to import license groups from ScanCode LicenseDB"
+            )
+
+        import_scancode_licensedb()
+
+        return Response(status=HTTP_204_NO_CONTENT)
+
 
 class LicenseGroupMemberViewSet(ModelViewSet):
     serializer_class = LicenseGroupMemberSerializer

backend/application/licenses/migrations/0008_alter_license_policy_item_license_group.py

@@ -0,0 +1,25 @@
+# Generated by Django 5.1.3 on 2024-11-21 21:00
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("licenses", "0007_license_component_evidence_and_more"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="license_policy_item",
+            name="license_group",
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.PROTECT,
+                related_name="license_policy_items",
+                to="licenses.license_group",
+            ),
+        ),
+    ]

backend/application/licenses/models.py

@@ -195,7 +195,7 @@ class License_Policy_Item(Model):
     license_group = ForeignKey(
         License_Group,
         related_name="license_policy_items",
-        on_delete=CASCADE,
+        on_delete=PROTECT,
         blank=True,
         null=True,
     )

backend/application/licenses/services/license_group.py

@@ -1,4 +1,8 @@
-from application.licenses.models import License_Group, License_Group_Member
+from json import loads
+
+import requests
+
+from application.licenses.models import License, License_Group, License_Group_Member
 
 
 def copy_license_group(source_license_group: License_Group, name: str) -> License_Group:
@@ -20,3 +24,44 @@ def copy_license_group(source_license_group: License_Group, name: str) -> Licens
         )
 
     return new_license_group
+
+
+def import_scancode_licensedb() -> None:
+    license_groups: dict[str, License_Group] = {}
+
+    response = requests.get(
+        "https://scancode-licensedb.aboutcode.org/index.json",
+        timeout=60,
+        stream=True,
+    )
+    response.raise_for_status()
+    data = loads(response.content)
+
+    for db_license in data:
+        category = db_license.get("category")
+        spdx_license_key = db_license.get("spdx_license_key")
+        other_spdx_license_keys = db_license.get("other_spdx_license_keys", [])
+
+        if category and spdx_license_key:
+            _add_license_to_group(license_groups, category, spdx_license_key)
+            for other_spdx_license_key in other_spdx_license_keys:
+                _add_license_to_group(license_groups, category, other_spdx_license_key)
+
+
+def _add_license_to_group(license_groups, category, spdx_license_key):
+    try:
+        spdx_license = License.objects.get(spdx_id=spdx_license_key)
+        license_group = license_groups.get(category)
+        if not license_group:
+            license_group, _ = License_Group.objects.get_or_create(
+                name=f"{category} (ScanCode LicenseDB)",
+                description="Do not edit! "
+                + "Imported from [ScanCode LicenseDB](https://scancode-licensedb.aboutcode.org/) "
+                + "under the CC-BY-4.0 license.",
+                is_public=True,
+            )
+            license_groups[category] = license_group
+            license_group.licenses.clear()
+        license_group.licenses.add(spdx_license)
+    except License.DoesNotExist:
+        pass

backend/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "SecObserve"
-version = "1.22.3"
+version = "1.22.4"
 description = "SecObserve is an open source vulnerability management system for software development and cloud environments."
 license = "BSD-3-Clause"
 authors = [

backend/unittests/access_control/api/test_authentication.py

@@ -328,6 +328,9 @@ def test_authentication(self, mock_user):
         self._check_authentication(["post"], "/api/license_groups/1/copy/")
         self._check_authentication(["post"], "/api/license_groups/1/add_license/")
         self._check_authentication(["post"], "/api/license_groups/1/remove_license/")
+        self._check_authentication(
+            ["post"], "/api/license_groups/import_scancode_licensedb/"
+        )
 
         self._check_authentication(["get", "post"], "/api/license_group_members/")
         self._check_authentication(

backend/unittests/access_control/api/test_authorization_license_group_authorization_group_members.py

@@ -1,4 +1,4 @@
-from application.licenses.models import License_Group
+from application.licenses.models import License_Group, License_Policy
 from unittests.access_control.api.test_authorization import (
     APITest,
     TestAuthorizationBase,
@@ -7,6 +7,7 @@
 
 class TestAuthorizationLicenseGroupAuthorizationGroupMembers(TestAuthorizationBase):
     def test_authorization_license_group_authorization_group_members(self):
+        License_Policy.objects.all().delete()
         License_Group.objects.filter(pk__lt=1000).delete()
 
         expected_data = "{'count': 3, 'next': None, 'previous': None, 'results': [{'id': 1000, 'license_group_data': {'id': 1003, 'is_manager': False, 'is_in_license_policy': False, 'has_licenses': False, 'has_users': False, 'has_authorization_groups': True, 'name': 'authorization_group_not_manager', 'description': '', 'is_public': False}, 'authorization_group_data': {'id': 2, 'name': 'oidc_group_2', 'oidc_group': 'oidc_2'}, 'is_manager': False, 'license_group': 1003, 'authorization_group': 2}, {'id': 1001, 'license_group_data': {'id': 1004, 'is_manager': False, 'is_in_license_policy': False, 'has_licenses': False, 'has_users': False, 'has_authorization_groups': True, 'name': 'authorization_group_manager', 'description': '', 'is_public': False}, 'authorization_group_data': {'id': 2, 'name': 'oidc_group_2', 'oidc_group': 'oidc_2'}, 'is_manager': True, 'license_group': 1004, 'authorization_group': 2}, {'id': 1002, 'license_group_data': {'id': 1003, 'is_manager': False, 'is_in_license_policy': False, 'has_licenses': False, 'has_users': False, 'has_authorization_groups': True, 'name': 'authorization_group_not_manager', 'description': '', 'is_public': False}, 'authorization_group_data': {'id': 3, 'name': 'non_oidc_group', 'oidc_group': ''}, 'is_manager': True, 'license_group': 1003, 'authorization_group': 3}]}"

backend/unittests/access_control/api/test_authorization_license_group_members.py

@@ -1,4 +1,4 @@
-from application.licenses.models import License_Group
+from application.licenses.models import License_Group, License_Policy
 from unittests.access_control.api.test_authorization import (
     APITest,
     TestAuthorizationBase,
@@ -7,6 +7,7 @@
 
 class TestAuthorizationLicenseGroupMembers(TestAuthorizationBase):
     def test_authorization_license_group_members(self):
+        License_Policy.objects.all().delete()
         License_Group.objects.filter(pk__lt=1000).delete()
 
         expected_data = "{'count': 3, 'next': None, 'previous': None, 'results': [{'id': 1001, 'license_group_data': {'id': 1001, 'is_manager': False, 'is_in_license_policy': False, 'has_licenses': True, 'has_users': True, 'has_authorization_groups': False, 'name': 'internal_read_not_manager', 'description': '', 'is_public': False}, 'user_data': {'id': 3, 'username': 'db_internal_read', 'first_name': '', 'last_name': '', 'full_name': 'db_internal_read', 'email': '', 'is_active': True, 'is_superuser': False, 'is_external': False, 'setting_theme': 'light', 'setting_list_size': 'medium', 'permissions': [<Permissions.Product_Create: 1104>, <Permissions.Product_Group_Create: 1004>], 'setting_list_properties': '', 'oidc_groups_hash': '', 'is_oidc_user': False, 'date_joined': '2022-12-07T20:25:06+01:00', 'has_password': False}, 'is_manager': False, 'license_group': 1001, 'user': 3}, {'id': 1002, 'license_group_data': {'id': 1002, 'is_manager': False, 'is_in_license_policy': False, 'has_licenses': True, 'has_users': True, 'has_authorization_groups': False, 'name': 'internal_write_manager', 'description': '', 'is_public': False}, 'user_data': {'id': 2, 'username': 'db_internal_write', 'first_name': '', 'last_name': '', 'full_name': 'db_internal_write', 'email': '', 'is_active': True, 'is_superuser': False, 'is_external': False, 'setting_theme': 'light', 'setting_list_size': 'medium', 'permissions': [<Permissions.Product_Create: 1104>, <Permissions.Product_Group_Create: 1004>], 'setting_list_properties': '', 'oidc_groups_hash': '', 'is_oidc_user': False, 'date_joined': '2022-12-07T20:24:53+01:00', 'has_password': False}, 'is_manager': True, 'license_group': 1002, 'user': 2}, {'id': 1003, 'license_group_data': {'id': 1001, 'is_manager': False, 'is_in_license_policy': False, 'has_licenses': True, 'has_users': True, 'has_authorization_groups': False, 'name': 'internal_read_not_manager', 'description': '', 'is_public': False}, 'user_data': {'id': 4, 'username': 'db_external', 'first_name': '', 'last_name': '', 'full_name': 'db_external', 'email': '', 'is_active': True, 'is_superuser': False, 'is_external': True, 'setting_theme': 'light', 'setting_list_size': 'medium', 'permissions': [], 'setting_list_properties': '', 'oidc_groups_hash': '', 'is_oidc_user': False, 'date_joined': '2022-12-12T19:48:08.514000+01:00', 'has_password': False}, 'is_manager': False, 'license_group': 1001, 'user': 4}]}"

backend/unittests/access_control/api/test_authorization_license_groups.py

@@ -1,4 +1,4 @@
-from application.licenses.models import License_Group
+from application.licenses.models import License_Group, License_Policy
 from unittests.access_control.api.test_authorization import (
     APITest,
     TestAuthorizationBase,
@@ -7,6 +7,7 @@
 
 class TestAuthorizationLicenseGroups(TestAuthorizationBase):
     def test_authorization_license_groups(self):
+        License_Policy.objects.all().delete()
         License_Group.objects.filter(pk__lt=1000).delete()
 
         expected_data = "{'count': 5, 'next': None, 'previous': None, 'results': [{'id': 1000, 'is_manager': False, 'is_in_license_policy': False, 'has_licenses': True, 'has_users': False, 'has_authorization_groups': False, 'name': 'public', 'description': '', 'is_public': True}, {'id': 1001, 'is_manager': False, 'is_in_license_policy': False, 'has_licenses': True, 'has_users': True, 'has_authorization_groups': False, 'name': 'internal_read_not_manager', 'description': '', 'is_public': False}, {'id': 1002, 'is_manager': False, 'is_in_license_policy': False, 'has_licenses': True, 'has_users': True, 'has_authorization_groups': False, 'name': 'internal_write_manager', 'description': '', 'is_public': False}, {'id': 1003, 'is_manager': False, 'is_in_license_policy': False, 'has_licenses': False, 'has_users': False, 'has_authorization_groups': True, 'name': 'authorization_group_not_manager', 'description': '', 'is_public': False}, {'id': 1004, 'is_manager': False, 'is_in_license_policy': False, 'has_licenses': False, 'has_users': False, 'has_authorization_groups': True, 'name': 'authorization_group_manager', 'description': '', 'is_public': False}]}"
@@ -353,3 +354,28 @@ def test_authorization_license_groups(self):
                 no_second_user=True,
             )
         )
+
+        expected_data = "{'message': 'User is not allowed to import license groups from ScanCode LicenseDB'}"
+        self._test_api(
+            APITest(
+                "db_internal_write",
+                "post",
+                "/api/license_groups/import_scancode_licensedb/",
+                post_data,
+                403,
+                expected_data,
+                no_second_user=True,
+            )
+        )
+
+        self._test_api(
+            APITest(
+                "db_admin",
+                "post",
+                "/api/license_groups/import_scancode_licensedb/",
+                post_data,
+                204,
+                None,
+                no_second_user=True,
+            )
+        )

backend/unittests/import_observations/parsers/cyclone_dx/files/changed/licenses_1.json

@@ -31,11 +31,11 @@
         ],
         "component": {
             "name": "SecObserve",
-            "version": "1.22.3",
+            "version": "1.22.4",
             "description": "SecObserve is an open source vulnerability management system for software development and cloud environments.",
             "type": "application",
-            "bom-ref": "pkg:pypi/[email protected].3",
-            "purl": "pkg:pypi/[email protected].3"
+            "bom-ref": "pkg:pypi/[email protected].4",
+            "purl": "pkg:pypi/[email protected].4"
         },
         "properties": [
             {
@@ -2792,7 +2792,7 @@
     ],
     "dependencies": [
         {
-            "ref": "pkg:pypi/[email protected].3",
+            "ref": "pkg:pypi/[email protected].4",
             "dependsOn": [
                 "pkg:pypi/[email protected]",
                 "pkg:pypi/[email protected]",

backend/unittests/import_observations/services/test_import_observations.py

@@ -542,7 +542,7 @@ def _file_upload_licenses(
             )
             self.assertEqual(license_components[1].purl_type, "pypi")
             self.assertEqual(license_components[1].cpe, "")
-            dependencies = """SecObserve:1.22.3 --> argon2-cffi:23.1.0
+            dependencies = """SecObserve:1.22.4 --> argon2-cffi:23.1.0
 argon2-cffi:23.1.0 --> argon2-cffi-bindings:21.2.0"""
             self.assertEqual(license_components[1].dependencies, dependencies)
             self.assertEqual(

backend/unittests/vex/api/files/csaf_given_vulnerability.json

@@ -18,7 +18,7 @@
             "generator": {
                 "engine": {
                     "name": "SecObserve",
-                    "version": "1.22.3"
+                    "version": "1.22.4"
                 }
             },
             "id": "CSAF_2024_0001_0001",