Merge branch 'main' into refactoring/#339-Remove-secret-ALTERNATIVE_G…

…ITHUB_TOKEN-from-GitHub-workflows

.github/workflows/report.yml

@@ -50,6 +50,7 @@ jobs:
         run: |
           echo -e "# Summary\n" >> $GITHUB_STEP_SUMMARY
           poetry run nox -s project:report -- -- --format markdown >> $GITHUB_STEP_SUMMARY
+          poetry run nox -s dependency:licenses >> $GITHUB_STEP_SUMMARY
           echo -e "\n\n# Coverage\n" >> $GITHUB_STEP_SUMMARY
           poetry run coverage report -- --format markdown >> $GITHUB_STEP_SUMMARY
           poetry run tbx lint pretty-print >> $GITHUB_STEP_SUMMARY

doc/changes/unreleased.md

@@ -3,6 +3,7 @@
 ## ✨ Added
 
 * added tbx task for markdown formating of .lint.json
+* Added a Nox task for dependencies packages and their licenses with Markdown output
 
 ## 🐞 Fixed
 * Fixed an issue in the CI workflow that caused it to be executed twice on the initial push of a PR if the PR branch was on the repo itself.

exasol/toolbox/nox/_dependencies.py

@@ -0,0 +1,221 @@
+from __future__ import annotations
+
+import subprocess
+import tempfile
+from collections import defaultdict
+from dataclasses import dataclass
+from inspect import cleandoc
+from json import loads
+from pathlib import Path
+
+import nox
+import tomlkit
+from nox import Session
+
+
+@dataclass(frozen=True)
+class Package:
+    name: str
+    package_link: str
+    version: str
+    license: str
+    license_link: str
+
+
+def _dependencies(toml_str: str) -> dict[str, list]:
+    toml = tomlkit.loads(toml_str)
+    poetry = toml.get("tool", {}).get("poetry", {})
+    dependencies: dict[str, list] = {}
+
+    packages = poetry.get("dependencies", {})
+    if packages:
+        dependencies["project"] = []
+    for package in packages:
+        dependencies["project"].append(package)
+
+    packages = poetry.get("dev", {}).get("dependencies", {})
+    if packages:
+        dependencies["dev"] = []
+    for package in packages:
+        dependencies["dev"].append(package)
+
+    groups = poetry.get("group", {})
+    for group in groups:
+        packages = groups.get(group, {}).get("dependencies")
+        if packages and not dependencies.get(group, {}):
+            dependencies[group] = []
+        for package in packages:
+            dependencies[group].append(package)
+    return dependencies
+
+
+def _normalize(_license: str) -> str:
+    def is_multi_license(l):
+        return ";" in l
+
+    def select_most_restrictive(licenses: list) -> str:
+        _max = 0
+        lic = "Unknown"
+        _mapping = {
+            "Unknown": -1,
+            "Unlicensed": 0,
+            "BSD": 1,
+            "MIT": 2,
+            "MPLv2": 3,
+            "LGPLv2": 4,
+            "GPLv2": 5,
+            "GPLv3": 6,
+        }
+        for l in licenses:
+            if l in _mapping:
+                if _mapping[l] > _mapping[lic]:
+                    lic = l
+            else:
+                return "<br>".join(licenses)
+        return lic
+
+    mapping = {
+        "BSD License": "BSD",
+        "MIT License": "MIT",
+        "The Unlicensed (Unlicensed)": "Unlicensed",
+        "Mozilla Public License 2.0 (MPL 2.0)": "MPLv2",
+        "GNU General Public License (GPL)": "GPL",
+        "GNU Lesser General Public License v2 (LGPLv2)": "LGPLv2",
+        "GNU General Public License v2 (GPLv2)": "GPLv2",
+        "GNU General Public License v2 or later (GPLv2+)": "GPLv2+",
+        "GNU General Public License v3 (GPLv3)": "GPLv3",
+        "Apache Software License": "Apache",
+    }
+
+    if is_multi_license(_license):
+        items = []
+        for item in _license.split(";"):
+            item = str(item).strip()
+            if item in mapping:
+                items.append(mapping[item])
+            else:
+                items.append(item)
+        return select_most_restrictive(items)
+
+    if _license not in mapping:
+        return _license
+
+    return mapping[_license]
+
+
+def _packages_from_json(json: str) -> list[Package]:
+    packages = loads(json)
+    packages_list = []
+    mapping = {
+        "GPLv1": "https://www.gnu.org/licenses/old-licenses/gpl-1.0.html",
+        "GPLv2": "https://www.gnu.org/licenses/old-licenses/gpl-2.0.html",
+        "LGPLv2": "https://www.gnu.org/licenses/old-licenses/lgpl-2.0.html",
+        "GPLv3": "https://www.gnu.org/licenses/gpl-3.0.html",
+        "LGPLv3": "https://www.gnu.org/licenses/lgpl-3.0.html",
+        "Apache": "https://www.apache.org/licenses/LICENSE-2.0",
+        "MIT": "https://mit-license.org/",
+        "BSD": "https://opensource.org/license/bsd-3-clause",
+    }
+    for package in packages:
+        package_license = _normalize(package["License"])
+        packages_list.append(
+            Package(
+                name=package["Name"],
+                package_link="" if package["URL"] == "UNKNOWN" else package["URL"],
+                version=package["Version"],
+                license=package_license,
+                license_link=(
+                    "" if package_license not in mapping else mapping[package_license]
+                ),
+            )
+        )
+    return packages_list
+
+
+def _licenses() -> list[Package]:
+    with tempfile.NamedTemporaryFile() as file:
+        subprocess.run(
+            [
+                "poetry",
+                "run",
+                "pip-licenses",
+                "--format=json",
+                "--output-file=" + file.name,
+                "--with-system",
+                "--with-urls",
+            ],
+            capture_output=True,
+        )
+        result = _packages_from_json(file.read().decode())
+    return result
+
+
+def _packages_to_markdown(
+    dependencies: dict[str, list], packages: list[Package]
+) -> str:
+    def heading():
+        text = "# Dependecies\n"
+        return text
+
+    def dependency(group: str, group_packages: list, packages: list[Package]) -> str:
+        def _header(_group: str):
+            _group = "".join([word.capitalize() for word in _group.strip().split()])
+            text = f"## {_group} Dependencies\n"
+            text += "|Package|version|Licence|\n"
+            text += "|---|---|---|\n"
+            return text
+
+        def _rows(_group_packages: list, _packages: list[Package]) -> str:
+            def _normalize_package_name(name: str) -> str:
+                _name = name.lower()
+                while "_" in _name:
+                    _name = _name.replace("_", "-")
+                return _name
+
+            text = ""
+            for package in _group_packages:
+                consistent = filter(
+                    lambda elem: (_normalize_package_name(elem.name) == package),
+                    _packages,
+                )
+                for content in consistent:
+                    if content.package_link:
+                        text += f"|[{content.name}]({content.package_link})"
+                    else:
+                        text += f"|{content.name}"
+                    text += f"|{content.version}"
+                    if content.license_link:
+                        text += f"|[{content.license}]({content.license_link})|\n"
+                    else:
+                        text += f"|{content.license}|\n"
+            text += "\n"
+            return text
+
+        _template = cleandoc(
+            """
+            {header}{rows}
+        """
+        )
+        return _template.format(
+            header=_header(group), rows=_rows(group_packages, packages)
+        )
+
+    template = cleandoc(
+        """
+        {heading}{rows}
+    """
+    )
+
+    rows = ""
+    for group in dependencies:
+        rows += dependency(group, dependencies[group], packages)
+    return template.format(heading=heading(), rows=rows)
+
+
+@nox.session(name="dependency:licenses", python=False)
+def dependency_licenses(session: Session) -> None:
+    """returns the packages and their licenses"""
+    toml = Path("pyproject.toml")
+    dependencies = _dependencies(toml.read_text())
+    package_infos = _licenses()
+    print(_packages_to_markdown(dependencies=dependencies, packages=package_infos))

exasol/toolbox/nox/tasks.py

@@ -78,5 +78,10 @@ def check(session: Session) -> None:
 from exasol.toolbox.nox._artifacts import (
     check_artifacts
 )
+
+from exasol.toolbox.nox._dependencies import (
+    dependency_licenses,
+)
+
 # isort: on
 # fmt: on

exasol/toolbox/templates/github/workflows/report.yml

@@ -50,6 +50,7 @@ jobs:
         run: |
           echo -e "# Summary\n" >> $GITHUB_STEP_SUMMARY
           poetry run nox -s project:report -- -- --format markdown >> $GITHUB_STEP_SUMMARY
+          poetry run nox -s dependency:licenses >> $GITHUB_STEP_SUMMARY
           echo -e "\n\n# Coverage\n" >> $GITHUB_STEP_SUMMARY
           poetry run coverage report -- --format markdown >> $GITHUB_STEP_SUMMARY
           poetry run tbx security pretty-print >> $GITHUB_STEP_SUMMARY