From d25f2eaa90f23e1828a17a505fd39d0d912bf010 Mon Sep 17 00:00:00 2001 From: Stefan Fleckenstein Date: Tue, 17 Dec 2024 18:41:54 +0000 Subject: [PATCH] feat: export license policy to JSON and YAML (#2356) * feat: export license policy to JSON and YAML * fix: typo in unittest --- .../application/commons/services/export.py | 30 ++++++ backend/application/licenses/api/views.py | 68 ++++++++++++ .../services/export_license_policy.py | 83 ++++++++++++++ backend/poetry.lock | 18 ++-- backend/pyproject.toml | 13 +-- .../access_control/api/test_authentication.py | 2 + .../test_authorization_license_policies.py | 71 ++++++++++++ .../unittests/licenses/services/__init__.py | 0 .../services/test_export_license_policy.py | 99 +++++++++++++++++ .../licenses/license_policies/ExportMenu.tsx | 102 ++++++++++++++++++ .../license_policies/LicensePolicyShow.tsx | 2 + 11 files changed, 473 insertions(+), 15 deletions(-) create mode 100644 backend/application/licenses/services/export_license_policy.py create mode 100644 backend/unittests/licenses/services/__init__.py create mode 100644 backend/unittests/licenses/services/test_export_license_policy.py create mode 100644 frontend/src/licenses/license_policies/ExportMenu.tsx diff --git a/backend/application/commons/services/export.py b/backend/application/commons/services/export.py index 31ae78eb2..0bd067c86 100644 --- a/backend/application/commons/services/export.py +++ b/backend/application/commons/services/export.py @@ -1,6 +1,8 @@ +import json from datetime import datetime from typing import Any +import jsonpickle from defusedcsv import csv from django.db.models.query import QuerySet from django.http import HttpResponse @@ -98,3 +100,31 @@ def export_csv( fields.append(value) writer.writerow(fields) + + +def object_to_json(object_to_encode: Any) -> str: + jsonpickle.set_encoder_options("json", ensure_ascii=False) + json_string = jsonpickle.encode(object_to_encode, unpicklable=False) + + json_dict = json.loads(json_string) + json_dict = _remove_empty_elements(json_dict) + + return json.dumps(json_dict, indent=4, sort_keys=True, ensure_ascii=False) + + +def _remove_empty_elements(d: dict) -> dict: + """recursively remove empty lists, empty dicts, or None elements from a dictionary""" + + def empty(x): + return x is None or x == {} or x == [] + + if not isinstance(d, (dict, list)): + return d + if isinstance(d, list): + return [v for v in (_remove_empty_elements(v) for v in d) if not empty(v)] + + return { + k: v + for k, v in ((k, _remove_empty_elements(v)) for k, v in d.items()) + if not empty(v) + } diff --git a/backend/application/licenses/api/views.py b/backend/application/licenses/api/views.py index 062a6a4ad..8e4be5f0b 100644 --- a/backend/application/licenses/api/views.py +++ b/backend/application/licenses/api/views.py @@ -2,6 +2,7 @@ from typing import Optional, Tuple from django.db.models.query import QuerySet +from django.http import HttpResponse from django_filters.rest_framework import DjangoFilterBackend from drf_spectacular.utils import OpenApiParameter, extend_schema from rest_framework.decorators import action @@ -15,6 +16,7 @@ from application.access_control.services.authorization import user_has_permission_or_403 from application.access_control.services.roles_permissions import Permissions +from application.commons.services.global_request import get_current_user from application.core.models import Branch, Product from application.core.queries.branch import get_branch_by_id from application.core.queries.product import get_product_by_id @@ -99,6 +101,10 @@ get_license_policy_member, get_license_policy_members, ) +from application.licenses.services.export_license_policy import ( + export_license_policy_json, + export_license_policy_yaml, +) from application.licenses.services.license_group import ( copy_license_group, import_scancode_licensedb, @@ -545,6 +551,68 @@ def apply_product(self, request): status=HTTP_204_NO_CONTENT, ) + @extend_schema( + methods=["GET"], + responses={200: None}, + ) + @action(detail=True, methods=["get"]) + def export_json(self, request, pk=None): + license_policy = self._get_license_policy(pk, False) + license_policy_export = export_license_policy_json(license_policy) + + response = HttpResponse( # pylint: disable=http-response-with-content-type-json + content=license_policy_export, + content_type="application/json", + ) + response["Content-Disposition"] = ( + f"attachment; filename=license_policy_{pk}.json" + ) + + return response + + @extend_schema( + methods=["GET"], + responses={200: None}, + ) + @action(detail=True, methods=["get"]) + def export_yaml(self, request, pk=None): + license_policy = self._get_license_policy(pk, False) + license_policy_export = export_license_policy_yaml(license_policy) + + response = HttpResponse( + content=license_policy_export, + content_type="application/yaml", + ) + response["Content-Disposition"] = ( + f"attachment; filename=license_policy_{pk}.yaml" + ) + + return response + + def _get_license_policy(self, pk: int, manager: bool) -> License_Policy: + license_policy = get_license_policy(pk) + if license_policy is None: + raise NotFound("License policy not found") + + if not manager and license_policy.is_public: + return license_policy + + user = get_current_user() + if not user: + raise PermissionDenied("No user found") + + if user.is_superuser: + return license_policy + + license_policy_member = get_license_policy_member(license_policy, user) + if not license_policy.is_public and not license_policy_member: + raise NotFound("License policy not found") + + if manager and license_policy_member and not license_policy_member.is_manager: + raise PermissionDenied("You are not a manager of this license policy") + + return license_policy + class LicensePolicyItemViewSet(ModelViewSet): serializer_class = LicensePolicyItemSerializer diff --git a/backend/application/licenses/services/export_license_policy.py b/backend/application/licenses/services/export_license_policy.py new file mode 100644 index 000000000..ffe381168 --- /dev/null +++ b/backend/application/licenses/services/export_license_policy.py @@ -0,0 +1,83 @@ +import json +from dataclasses import dataclass +from typing import Optional + +import yaml + +from application.commons.services.export import object_to_json +from application.licenses.models import License_Policy, License_Policy_Item +from application.licenses.services.license_policy import get_ignore_component_type_list + + +@dataclass +class License_Policy_Export_Item: + evaluation_result: str + spdx_license: Optional[str] = None + license_expression: Optional[str] = None + unknown_license: Optional[str] = None + license_group: Optional[str] = None + + +@dataclass +class License_Policy_Export_Ignore_Component_Type: + component_type: str + + +@dataclass +class License_Policy_Export: + name: str + description: str + items: list[License_Policy_Export_Item] + ignore_component_types: list[License_Policy_Export_Ignore_Component_Type] + + +def export_license_policy_yaml(license_policy: License_Policy) -> str: + return yaml.dump(json.loads(export_license_policy_json(license_policy))) + + +def export_license_policy_json(license_policy: License_Policy) -> str: + return object_to_json(_create_license_policy_export(license_policy)) + + +def _create_license_policy_export( + license_policy: License_Policy, +) -> License_Policy_Export: + license_policy_eport = License_Policy_Export( + name=license_policy.name, + description=license_policy.description, + items=[], + ignore_component_types=get_ignore_component_type_list( + license_policy.ignore_component_types + ), + ) + + license_policy_item: Optional[License_Policy_Item] = None + for license_policy_item in license_policy.license_policy_items.all(): + if license_policy_item.license_group: + for spdx_license in license_policy_item.license_group.licenses.all(): + license_policy_eport_item = License_Policy_Export_Item( + spdx_license=spdx_license.spdx_id, + license_group=license_policy_item.license_group.name, + evaluation_result=license_policy_item.evaluation_result, + ) + license_policy_eport.items.append(license_policy_eport_item) + elif license_policy_item.license: + license_policy_eport_item = License_Policy_Export_Item( + spdx_license=license_policy_item.license.spdx_id, + evaluation_result=license_policy_item.evaluation_result, + ) + license_policy_eport.items.append(license_policy_eport_item) + elif license_policy_item.license_expression: + license_policy_eport_item = License_Policy_Export_Item( + license_expression=license_policy_item.license_expression, + evaluation_result=license_policy_item.evaluation_result, + ) + license_policy_eport.items.append(license_policy_eport_item) + elif license_policy_item.unknown_license: + license_policy_eport_item = License_Policy_Export_Item( + unknown_license=license_policy_item.unknown_license, + evaluation_result=license_policy_item.evaluation_result, + ) + license_policy_eport.items.append(license_policy_eport_item) + + return license_policy_eport diff --git a/backend/poetry.lock b/backend/poetry.lock index 295d9c204..98d1aa4cd 100644 --- a/backend/poetry.lock +++ b/backend/poetry.lock @@ -105,19 +105,19 @@ test = ["astroid (>=2,<4)", "pytest", "pytest-cov", "pytest-xdist"] [[package]] name = "attrs" -version = "24.2.0" +version = "24.3.0" description = "Classes Without Boilerplate" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "attrs-24.2.0-py3-none-any.whl", hash = "sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2"}, - {file = "attrs-24.2.0.tar.gz", hash = "sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346"}, + {file = "attrs-24.3.0-py3-none-any.whl", hash = "sha256:ac96cd038792094f438ad1f6ff80837353805ac950cd2aa0e0625ef19850c308"}, + {file = "attrs-24.3.0.tar.gz", hash = "sha256:8f5c07333d543103541ba7be0e2ce16eeee8130cb0b3f9238ab904ce1e85baff"}, ] [package.extras] benchmark = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins", "pytest-xdist[psutil]"] cov = ["cloudpickle", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] -dev = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +dev = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pre-commit-uv", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier (<24.7)"] tests = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] tests-mypy = ["mypy (>=1.11.1)", "pytest-mypy-plugins"] @@ -181,13 +181,13 @@ files = [ [[package]] name = "certifi" -version = "2024.8.30" +version = "2024.12.14" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" files = [ - {file = "certifi-2024.8.30-py3-none-any.whl", hash = "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8"}, - {file = "certifi-2024.8.30.tar.gz", hash = "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9"}, + {file = "certifi-2024.12.14-py3-none-any.whl", hash = "sha256:1275f7a45be9464efc1173084eaa30f866fe2e47d389406136d332ed4967ec56"}, + {file = "certifi-2024.12.14.tar.gz", hash = "sha256:b650d30f370c2b724812bee08008be0c4163b163ddaec3f2546c1caf65f191db"}, ] [[package]] @@ -2572,4 +2572,4 @@ unittests = ["coverage", "django-coverage-plugin", "django-extensions"] [metadata] lock-version = "2.0" python-versions = ">= 3.10, < 3.13" -content-hash = "c99ff233f770a01fa4eb9ea3ccab8b91fd0596f2c0eef73196ae9eef9682a99d" +content-hash = "f45c89e9735ac316d4b9a124d9c1762226f3505615d982c17a8e5383c929a5ba" diff --git a/backend/pyproject.toml b/backend/pyproject.toml index a6bb6e334..bb7f8de5b 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -38,13 +38,18 @@ requests = "2.32.3" # https://github.com/psf/requests # ------------------------------------------------------------------------------ pymysql = "1.1.1" # https://github.com/PyMySQL/PyMySQL psycopg = { version = "3.2.3", extras = ["binary"] } # https://github.com/psycopg/psycopg -# Excel and CSV +# Dataformats # ------------------------------------------------------------------------------ defusedcsv = "2.0.0" # https://github.com/raphaelm/defusedcsv openpyxl = "3.1.5" # https://foss.heptapod.net/openpyxl/openpyxl -# Package URL +jsonpickle = "4.0.1" # https://github.com/jsonpickle/jsonpickle +pyyaml = "6.0.2" # https://github.com/yaml/pyyaml +# Field specifications # ------------------------------------------------------------------------------ packageurl-python = "0.16.0" # https://github.com/package-url/packageurl-python +cvss = "3.3" # https://github.com/RedHatProductSecurity/cvss +semver = "3.0.2" # https://github.com/python-semver/python-semver +license-expression = "30.4.0" # https://github.com/aboutcode-org/license-expression # Task queue # ------------------------------------------------------------------------------ huey = "2.5.2" # https://github.com/coleifer/huey @@ -55,11 +60,7 @@ jira = "3.8.0" # https://github.com/pycontribs/jira # ------------------------------------------------------------------------------ inflect = "7.4.0" # https://github.com/jaraco/inflect validators = "0.34.0" # https://github.com/python-validators/validators -cvss = "3.3" # https://github.com/RedHatProductSecurity/cvss -jsonpickle = "4.0.1" # https://github.com/jsonpickle/jsonpickle py-ocsf-models = "0.2.0" # https://github.com/prowler-cloud/py-ocsf-models -semver = "3.0.2" # https://github.com/python-semver/python-semver -license-expression = "30.4.0" # https://github.com/aboutcode-org/license-expression # Development dependencies # ------------------------------------------------------------------------------ diff --git a/backend/unittests/access_control/api/test_authentication.py b/backend/unittests/access_control/api/test_authentication.py index 17ee024ce..75d4999db 100644 --- a/backend/unittests/access_control/api/test_authentication.py +++ b/backend/unittests/access_control/api/test_authentication.py @@ -382,6 +382,8 @@ def test_authentication(self, mock_user): self._check_authentication(["post"], "/api/license_policies/1/copy/") self._check_authentication(["post"], "/api/license_policies/1/apply/") self._check_authentication(["post"], "/api/license_policies/apply_product/") + self._check_authentication(["get"], "/api/license_policies/1/export_json/") + self._check_authentication(["get"], "/api/license_policies/1/export_yaml/") self._check_authentication(["get", "post"], "/api/license_policy_items/") self._check_authentication( diff --git a/backend/unittests/access_control/api/test_authorization_license_policies.py b/backend/unittests/access_control/api/test_authorization_license_policies.py index e2161d243..dabcf179b 100644 --- a/backend/unittests/access_control/api/test_authorization_license_policies.py +++ b/backend/unittests/access_control/api/test_authorization_license_policies.py @@ -55,6 +55,29 @@ def test_authorization_license_policies(self): no_second_user=True, ) ) + self._test_api( + APITest( + "db_internal_write", + "get", + "/api/license_policies/1002/export_json/", + None, + 200, + None, + no_second_user=True, + ) + ) + self._test_api( + APITest( + "db_internal_write", + "get", + "/api/license_policies/1002/export_yaml/", + None, + 200, + None, + no_second_user=True, + ) + ) + expected_data = "{'message': 'No License_Policy matches the given query.'}" self._test_api( APITest( @@ -67,6 +90,31 @@ def test_authorization_license_policies(self): no_second_user=True, ) ) + expected_data = "{'message': 'License policy not found'}" + self._test_api( + APITest( + "db_internal_write", + "get", + "/api/license_policies/1001/export_json/", + None, + 404, + expected_data, + no_second_user=True, + ) + ) + self._test_api( + APITest( + "db_internal_write", + "get", + "/api/license_policies/1001/export_yaml/", + None, + 404, + expected_data, + no_second_user=True, + ) + ) + + expected_data = "{'message': 'No License_Policy matches the given query.'}" self._test_api( APITest( "db_internal_write", @@ -78,6 +126,29 @@ def test_authorization_license_policies(self): no_second_user=True, ) ) + expected_data = "{'message': 'License policy not found'}" + self._test_api( + APITest( + "db_internal_write", + "get", + "/api/license_policies/99999/export_json/", + None, + 404, + expected_data, + no_second_user=True, + ) + ) + self._test_api( + APITest( + "db_internal_write", + "get", + "/api/license_policies/99999/export_yaml/", + None, + 404, + expected_data, + no_second_user=True, + ) + ) post_data = {"name": "new_license_policy"} expected_data = "{'id': 1005, 'is_manager': True, 'has_products': False, 'has_product_groups': False, 'has_items': False, 'has_users': True, 'has_authorization_groups': False, 'name': 'new_license_policy', 'description': '', 'is_public': False, 'ignore_component_types': ''}" diff --git a/backend/unittests/licenses/services/__init__.py b/backend/unittests/licenses/services/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/unittests/licenses/services/test_export_license_policy.py b/backend/unittests/licenses/services/test_export_license_policy.py new file mode 100644 index 000000000..b3445c343 --- /dev/null +++ b/backend/unittests/licenses/services/test_export_license_policy.py @@ -0,0 +1,99 @@ +from django.core.management import call_command + +from application.licenses.models import License, License_Policy, License_Policy_Item +from application.licenses.services.export_license_policy import ( + export_license_policy_json, + export_license_policy_yaml, +) +from application.licenses.types import License_Policy_Evaluation_Result +from unittests.base_test_case import BaseTestCase + + +class TestLicenseGroupMemberSerializer(BaseTestCase): + @classmethod + def setUpClass(self): + call_command( + "loaddata", + [ + "application/licenses/fixtures/initial_data.json", + "unittests/fixtures/unittests_fixtures.json", + "unittests/fixtures/unittests_license_fixtures.json", + ], + ) + + license_policy = License_Policy.objects.get(pk=1000) + license_policy.description = "description_1000" + license_policy.ignore_component_types = "apk, oci" + license_policy.save() + License_Policy_Item( + license_policy=license_policy, + license=License.objects.get(pk=1), + evaluation_result=License_Policy_Evaluation_Result.RESULT_FORBIDDEN, + ).save() + License_Policy_Item( + license_policy=license_policy, + license_expression="MIT OR 3BSD", + evaluation_result=License_Policy_Evaluation_Result.RESULT_REVIEW_REQUIRED, + ).save() + License_Policy_Item( + license_policy=license_policy, + unknown_license="Unknown", + evaluation_result=License_Policy_Evaluation_Result.RESULT_FORBIDDEN, + ).save() + + super().setUpClass() + + def test_export_json(self): + license_policy = License_Policy.objects.get(pk=1000) + json_data = export_license_policy_json(license_policy) + + json_data_expected = """{ + "description": "description_1000", + "ignore_component_types": [ + "apk", + "oci" + ], + "items": [ + { + "evaluation_result": "Allowed", + "license_group": "Permissive Model (Blue Oak Council)", + "spdx_license": "BlueOak-1.0.0" + }, + { + "evaluation_result": "Forbidden", + "spdx_license": "0BSD" + }, + { + "evaluation_result": "Review required", + "license_expression": "MIT OR 3BSD" + }, + { + "evaluation_result": "Forbidden", + "unknown_license": "Unknown" + } + ], + "name": "public" +}""" + self.assertEqual(json_data_expected, json_data) + + def test_export_yaml(self): + license_policy = License_Policy.objects.get(pk=1000) + yaml_data = export_license_policy_yaml(license_policy) + + yaml_data_expected = """description: description_1000 +ignore_component_types: +- apk +- oci +items: +- evaluation_result: Allowed + license_group: Permissive Model (Blue Oak Council) + spdx_license: BlueOak-1.0.0 +- evaluation_result: Forbidden + spdx_license: 0BSD +- evaluation_result: Review required + license_expression: MIT OR 3BSD +- evaluation_result: Forbidden + unknown_license: Unknown +name: public +""" + self.assertEqual(yaml_data_expected, yaml_data) diff --git a/frontend/src/licenses/license_policies/ExportMenu.tsx b/frontend/src/licenses/license_policies/ExportMenu.tsx new file mode 100644 index 000000000..3651251e1 --- /dev/null +++ b/frontend/src/licenses/license_policies/ExportMenu.tsx @@ -0,0 +1,102 @@ +import DescriptionIcon from "@mui/icons-material/Description"; +import DownloadIcon from "@mui/icons-material/Download"; +import { ListItemIcon } from "@mui/material"; +import Button from "@mui/material/Button"; +import Menu from "@mui/material/Menu"; +import MenuItem from "@mui/material/MenuItem"; +import { Fragment, MouseEvent, useState } from "react"; +import { useNotify } from "react-admin"; + +import axios_instance from "../../access_control/auth_provider/axios_instance"; +import { getIconAndFontColor } from "../../commons/functions"; + +interface ExportMenuProps { + license_policy: any; +} + +const ExportMenu = ({ license_policy }: ExportMenuProps) => { + const notify = useNotify(); + const [anchorEl, setAnchorEl] = useState(null); + const open = Boolean(anchorEl); + const handleClick = (event: MouseEvent) => { + setAnchorEl(event.currentTarget); + }; + const handleClose = () => { + setAnchorEl(null); + }; + + const exportLicensePolicyJSON = async () => { + exportLicensePolicy("json"); + }; + + const exportLicensePolicyYAML = async () => { + exportLicensePolicy("yaml"); + }; + + const exportLicensePolicy = async (format: string) => { + axios_instance + .get("/license_policies/" + license_policy.id + "/export_" + format + "/") + .then(function (response) { + let blob = new Blob([response.data], { type: "application/" + format }); + if (format === "json") { + blob = new Blob([JSON.stringify(response.data, null, 4)], { type: "application/" + format }); + } + const url = window.URL.createObjectURL(blob); + const link = document.createElement("a"); + link.href = url; + link.download = "license_policy_" + license_policy.id + "." + format; + link.click(); + + notify("License Policy downloaded", { + type: "success", + }); + }) + .catch(function (error) { + notify(error.message, { + type: "warning", + }); + }); + handleClose(); + }; + + return ( + + + + + + + + JSON + + + + + + YAML + + + + ); +}; + +export default ExportMenu; diff --git a/frontend/src/licenses/license_policies/LicensePolicyShow.tsx b/frontend/src/licenses/license_policies/LicensePolicyShow.tsx index 2fd7bb434..6d5cdcea8 100644 --- a/frontend/src/licenses/license_policies/LicensePolicyShow.tsx +++ b/frontend/src/licenses/license_policies/LicensePolicyShow.tsx @@ -20,6 +20,7 @@ import ProductEmbeddedList from "../../core/products/ProductEmbeddedList"; import LicensePolicyAuthorizationGroupMemberEmbeddedList from "../license_policy_authorization_group_members/LicensePolicyAuthorizationGroupMemberEmbeddedList"; import LicensePolicyItemEmbeddedList from "../license_policy_items/LicensePolicyItemEmbeddedList"; import LicensePolicyMemberEmbeddedList from "../license_policy_members/LicensePolicyMemberEmbeddedList"; +import ExportMenu from "./ExportMenu"; import LicensePolicyApply from "./LicensePolicyApply"; import LicensePolicyCopy from "./LicensePolicyCopy"; @@ -34,6 +35,7 @@ const ShowActions = () => { filterDefaultValues={{ is_active: true }} storeKey="license_policies.embedded" /> + {license_policy && } {license_policy && (license_policy.is_manager || is_superuser()) && license_policy.has_products && ( )}