diff --git a/Pipfile b/Pipfile index 6f968f744..34faaf192 100644 --- a/Pipfile +++ b/Pipfile @@ -17,6 +17,7 @@ sqlalchemy-utils = "*" alembic = "*" thoth-ssdeep = "*" packaging = "*" +thoth-license-solver = ">=0.1.4" [dev-packages] pytest = "*" diff --git a/thoth/storages/data/alembic/versions/2a4632197512_added_license_solver.py b/thoth/storages/data/alembic/versions/2a4632197512_added_license_solver.py new file mode 100644 index 000000000..3730ee238 --- /dev/null +++ b/thoth/storages/data/alembic/versions/2a4632197512_added_license_solver.py @@ -0,0 +1,42 @@ +"""added license-solver + +Revision ID: 2a4632197512 +Revises: 2b787ddad4a4 +Create Date: 2022-06-09 07:26:05.576715+00:00 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "2a4632197512" +down_revision = "2b787ddad4a4" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "python_package_license", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("license_name", sa.Text(), nullable=False), + sa.Column("license_identifier", sa.Text(), nullable=False), + sa.Column("license_version", sa.Text(), nullable=False), + sa.PrimaryKeyConstraint("id"), + ) + + op.add_column("python_package_version", sa.Column("package_license", sa.Integer(), nullable=True)) + op.add_column("python_package_version", sa.Column("package_license_warning", sa.Boolean(), nullable=True)) + op.create_foreign_key("fk_license", "python_package_version", "python_package_license", ["package_license"], ["id"]) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint("fk_license", "python_package_version", type_="foreignkey") + op.drop_column("python_package_version", "package_license_warning") + op.drop_column("python_package_version", "package_license") + op.drop_table("python_package_license") + # ### end Alembic commands ### diff --git a/thoth/storages/graph/models.py b/thoth/storages/graph/models.py index 1bacbd68f..6f0920803 100644 --- a/thoth/storages/graph/models.py +++ b/thoth/storages/graph/models.py @@ -73,6 +73,9 @@ class PythonPackageVersion(Base, BaseExtension): ) is_missing = Column(Boolean, nullable=False, default=False) provides_source_distro = Column(Boolean, nullable=False, default=True) + package_license = Column(Integer, ForeignKey("python_package_license.id")) # nullable=False + package_license_warning = Column(Boolean) # nullable=False + # Relations dependencies = relationship("DependsOn", back_populates="version") solvers = relationship("Solved", back_populates="version") @@ -82,6 +85,7 @@ class PythonPackageVersion(Base, BaseExtension): si_aggregated = relationship("SIAggregated", back_populates="python_package_version") python_software_stacks = relationship("HasPythonRequirementsLock", back_populates="python_package_version") import_packages = relationship("FoundImportPackage", back_populates="python_package_version") + licenses = relationship("PythonPackageLicense", back_populates="python_package_version") __table_args__ = tuple( get_python_package_version_index_combinations() @@ -94,6 +98,21 @@ class PythonPackageVersion(Base, BaseExtension): ) +class PythonPackageLicense(Base, BaseExtension): + """Representation of a License.""" + + __tablename__ = "python_package_license" + + id = Column(Integer, primary_key=True, autoincrement=True, nullable=False) + + license_name = Column(Text, nullable=False) + license_identifier = Column(Text, nullable=False) + license_version = Column(Text, nullable=False) + + # Relations + python_package_version = relationship("PythonPackageVersion", back_populates="licenses") + + class HasArtifact(Base, BaseExtension): """The given package has the given artifact.""" @@ -1761,6 +1780,7 @@ class FoundImportPackage(Base, BaseExtension): PythonPackageRequirement, PythonPackageVersion, PythonPackageVersionEntity, + PythonPackageLicense, RPMPackageVersion, RPMRequirement, SecurityIndicatorAggregatedRun, diff --git a/thoth/storages/graph/postgres.py b/thoth/storages/graph/postgres.py index b8139823f..0e5cf1d39 100644 --- a/thoth/storages/graph/postgres.py +++ b/thoth/storages/graph/postgres.py @@ -110,6 +110,7 @@ from .models import SecurityIndicatorAggregatedRun from .models import SoftwareEnvironment from .models import VersionedSymbol +from .models import PythonPackageLicense from .models import Advised from .models import DebDepends @@ -4734,6 +4735,8 @@ def _create_python_package_version( python_version: Union[str, None], python_package_metadata_id: int = None, sync_only_entity: bool = False, + package_license: int = None, + package_license_warning: bool = None, ) -> Union[PythonPackageVersion, PythonPackageVersionEntity]: """Create a Python package version. @@ -4768,6 +4771,8 @@ def _create_python_package_version( os_version=os_version, python_version=python_version, entity_id=entity.id, + package_license=package_license, + package_license_warning=package_license_warning, ) # including this value in "get_or_create" will cause errors because it is not part of a unique entry @@ -5989,11 +5994,13 @@ def sync_solver_result(self, document: dict, *, force: bool = False) -> None: package_version = python_package_info["package_version_requested"] index_url = python_package_info["index_url"] importlib_metadata = python_package_info["importlib_metadata"]["metadata"] + package_license = python_package_info["package_license"] _LOGGER.info( - "Syncing solver result of package %r in version %r from %r solved by %r", + "Syncing solver result of package %r in version %r license %r from %r solved by %r", package_name, package_version, + package_license, index_url, solver_info, ) @@ -6032,6 +6039,14 @@ def sync_solver_result(self, document: dict, *, force: bool = False) -> None: f"No related columns for {list(importlib_metadata.keys())!r} " "found in PythonPackageMetadata table, the error is not fatal" ) + + license_metadata, _ = PythonPackageLicense.get_or_create( + session, + license_name=package_license["license"].get("full_name"), + license_identifier=package_license["license"].get("identifier_spdx"), + license_version=package_license["license_version"], + ) + try: python_package_version = self._create_python_package_version( session, @@ -6042,6 +6057,8 @@ def sync_solver_result(self, document: dict, *, force: bool = False) -> None: python_version=ecosystem_solver.python_version, index_url=index_url, python_package_metadata_id=package_metadata.id, + package_license=license_metadata.id, + package_license_warning=package_license["warning"], ) except NoResultFound: if not force: