diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 982def6ca..988cdf922 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -91,6 +91,12 @@ v31.0.0 (next) https://github.com/nexB/scancode.io/issues/444 +- ``CodebaseResource.for_packages`` now returns a list of + ``DiscoveredPackage.package_uid`` or ``DiscoveredPackage.package_url`` if + ``DiscoveredPackage.package_uid`` is not present. This is done to reflect the + how scancode-toolkit's JSON output returns ``package_uid``s in the + ``for_packages`` field for Resources. + v30.2.0 (2021-12-17) -------------------- diff --git a/scanpipe/models.py b/scanpipe/models.py index 448473c34..0c2519c55 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -1759,7 +1759,10 @@ def for_packages(self): """ Returns the list of all discovered packages associated to this resource. """ - return [str(package) for package in self.discovered_packages.all()] + return [ + package.package_uid if package.package_uid else str(package) + for package in self.discovered_packages.all() + ] class DiscoveredPackageQuerySet(PackageURLQuerySetMixin, ProjectRelatedQuerySet): diff --git a/scanpipe/tests/data/asgiref-3.3.0_load_inventory_expected.json b/scanpipe/tests/data/asgiref-3.3.0_load_inventory_expected.json index deb59b681..3547ac0a8 100644 --- a/scanpipe/tests/data/asgiref-3.3.0_load_inventory_expected.json +++ b/scanpipe/tests/data/asgiref-3.3.0_load_inventory_expected.json @@ -847,7 +847,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": [] }, { "for_packages": [], @@ -873,7 +874,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": [] }, { "for_packages": [], @@ -899,7 +901,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": [] }, { "for_packages": [], @@ -930,9 +933,9 @@ }, { "for_packages": [], - "path": "asgiref-3.3.0-py3-none-any.whl-extract/asgiref-3.3.0.dist-info/top_level.txt", - "sha1": "612390bd0d0227c009f9c99b479878adf7ac2f23", - "md5": "680e61db4d95c8d9501b7a49fa2bf0b2", + "path": "asgiref-3.3.0-py3-none-any.whl-extract/asgiref/compatibility.py", + "sha1": "9c74e64e9a71903bb227907ea1806eac77e52434", + "md5": "5231077fd0628314246fcba7817b561e", "extra_data": {}, "copyrights": [], "holders": [], @@ -944,10 +947,37 @@ "status": "", "tag": "", "type": "file", - "name": "top_level.txt", - "extension": ".txt", - "programming_language": "", - "mime_type": "text/plain", + "name": "compatibility.py", + "extension": ".py", + "programming_language": "Python", + "mime_type": "text/x-script.python", + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_key_file": false, + "is_media": false, + "package_data": [] + }, + { + "for_packages": [], + "path": "asgiref-3.3.0-py3-none-any.whl-extract/asgiref/current_thread_executor.py", + "sha1": "aacf7e5e2e5ba78ccfb67fa10e9e6b22c3935c9b", + "md5": "b4c45f37055d88dd11b15eb4de51b074", + "extra_data": {}, + "copyrights": [], + "holders": [], + "authors": [], + "licenses": [], + "license_expressions": [], + "emails": [], + "urls": [], + "status": "", + "tag": "", + "type": "file", + "name": "current_thread_executor.py", + "extension": ".py", + "programming_language": "Python", + "mime_type": "text/x-script.python", "is_binary": false, "is_text": true, "is_archive": false, diff --git a/scanpipe/tests/data/centos_scan_codebase.json b/scanpipe/tests/data/centos_scan_codebase.json index 378995c02..b27dee16c 100644 --- a/scanpipe/tests/data/centos_scan_codebase.json +++ b/scanpipe/tests/data/centos_scan_codebase.json @@ -186626,58 +186626,6 @@ "is_media": false, "package_data": [] }, - { - "for_packages": [], - "path": "centos.tar.gz-extract/a10cf747c363a52be048f884c084a25e03280d54a7ac02e17dbd8c5ad160e9bd/var/lib/rpm/.dbenv.lock", - "sha1": "", - "md5": "", - "extra_data": {}, - "copyrights": [], - "holders": [], - "authors": [], - "licenses": [], - "license_expressions": [], - "emails": [], - "urls": [], - "status": "ignored-empty-file", - "tag": "img-c967b7-layer-01-a10cf7", - "type": "file", - "name": ".dbenv.lock", - "extension": ".lock", - "programming_language": "", - "mime_type": "inode/x-empty", - "is_binary": false, - "is_text": true, - "is_archive": false, - "is_key_file": false, - "is_media": false - }, - { - "for_packages": [], - "path": "centos.tar.gz-extract/a10cf747c363a52be048f884c084a25e03280d54a7ac02e17dbd8c5ad160e9bd/var/lib/rpm/.rpm.lock", - "sha1": "", - "md5": "", - "extra_data": {}, - "copyrights": [], - "holders": [], - "authors": [], - "licenses": [], - "license_expressions": [], - "emails": [], - "urls": [], - "status": "ignored-empty-file", - "tag": "img-c967b7-layer-01-a10cf7", - "type": "file", - "name": ".rpm.lock", - "extension": ".lock", - "programming_language": "", - "mime_type": "inode/x-empty", - "is_binary": false, - "is_text": true, - "is_archive": false, - "is_key_file": false, - "is_media": false - }, { "for_packages": [ "pkg:rpm/rpm@4.14.2?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758" diff --git a/scanpipe/tests/data/gcr_io_distroless_base_scan_codebase.json b/scanpipe/tests/data/gcr_io_distroless_base_scan_codebase.json index 51c4abe2f..f90239cfb 100644 --- a/scanpipe/tests/data/gcr_io_distroless_base_scan_codebase.json +++ b/scanpipe/tests/data/gcr_io_distroless_base_scan_codebase.json @@ -8168,110 +8168,6 @@ "is_media": false, "package_data": [] }, - { - "for_packages": [], - "path": "gcr_io_distroless_base.tar.gz-extract/cb3279093e638ddfd56bff4d3d89c5a3ed6dd59dbcfbc2f3107045635996b822/usr/share/doc/libc6/NEWS.Debian.gz", - "sha1": "", - "md5": "", - "extra_data": {}, - "copyrights": [], - "holders": [], - "authors": [], - "licenses": [], - "license_expressions": [], - "emails": [], - "urls": [], - "status": "ignored-empty-file", - "tag": "img-f596dc-layer-02-cb3279", - "type": "file", - "name": "NEWS.Debian.gz", - "extension": ".gz", - "programming_language": "", - "mime_type": "inode/x-empty", - "is_binary": false, - "is_text": true, - "is_archive": false, - "is_key_file": false, - "is_media": false - }, - { - "for_packages": [], - "path": "gcr_io_distroless_base.tar.gz-extract/cb3279093e638ddfd56bff4d3d89c5a3ed6dd59dbcfbc2f3107045635996b822/usr/share/doc/libc6/NEWS.gz", - "sha1": "", - "md5": "", - "extra_data": {}, - "copyrights": [], - "holders": [], - "authors": [], - "licenses": [], - "license_expressions": [], - "emails": [], - "urls": [], - "status": "ignored-empty-file", - "tag": "img-f596dc-layer-02-cb3279", - "type": "file", - "name": "NEWS.gz", - "extension": ".gz", - "programming_language": "", - "mime_type": "inode/x-empty", - "is_binary": false, - "is_text": true, - "is_archive": false, - "is_key_file": false, - "is_media": false - }, - { - "for_packages": [], - "path": "gcr_io_distroless_base.tar.gz-extract/cb3279093e638ddfd56bff4d3d89c5a3ed6dd59dbcfbc2f3107045635996b822/usr/share/doc/libc6/README.Debian.gz", - "sha1": "", - "md5": "", - "extra_data": {}, - "copyrights": [], - "holders": [], - "authors": [], - "licenses": [], - "license_expressions": [], - "emails": [], - "urls": [], - "status": "ignored-empty-file", - "tag": "img-f596dc-layer-02-cb3279", - "type": "file", - "name": "README.Debian.gz", - "extension": ".gz", - "programming_language": "", - "mime_type": "inode/x-empty", - "is_binary": false, - "is_text": true, - "is_archive": false, - "is_key_file": false, - "is_media": false - }, - { - "for_packages": [], - "path": "gcr_io_distroless_base.tar.gz-extract/cb3279093e638ddfd56bff4d3d89c5a3ed6dd59dbcfbc2f3107045635996b822/usr/share/doc/libc6/README.hesiod.gz", - "sha1": "", - "md5": "", - "extra_data": {}, - "copyrights": [], - "holders": [], - "authors": [], - "licenses": [], - "license_expressions": [], - "emails": [], - "urls": [], - "status": "ignored-empty-file", - "tag": "img-f596dc-layer-02-cb3279", - "type": "file", - "name": "README.hesiod.gz", - "extension": ".gz", - "programming_language": "", - "mime_type": "inode/x-empty", - "is_binary": false, - "is_text": true, - "is_archive": false, - "is_key_file": false, - "is_media": false - }, { "for_packages": [], "path": "gcr_io_distroless_base.tar.gz-extract/cb3279093e638ddfd56bff4d3d89c5a3ed6dd59dbcfbc2f3107045635996b822/usr/share/doc/libc6/changelog.Debian.gz", @@ -8818,112 +8714,6 @@ "is_media": false, "package_data": [] }, - { - "for_packages": [], - "path": "gcr_io_distroless_base.tar.gz-extract/cb3279093e638ddfd56bff4d3d89c5a3ed6dd59dbcfbc2f3107045635996b822/usr/share/doc/openssl/changelog.Debian.gz", - "sha1": "", - "md5": "", - "extra_data": {}, - "copyrights": [], - "holders": [], - "authors": [], - "licenses": [], - "license_expressions": [], - "emails": [], - "urls": [], - "status": "ignored-empty-file", - "tag": "img-f596dc-layer-02-cb3279", - "type": "file", - "name": "changelog.Debian.gz", - "extension": ".gz", - "programming_language": "", - "mime_type": "inode/x-empty", - "is_binary": false, - "is_text": true, - "is_archive": false, - "is_key_file": false, - "is_media": false - }, - { - "for_packages": [], - "path": "gcr_io_distroless_base.tar.gz-extract/cb3279093e638ddfd56bff4d3d89c5a3ed6dd59dbcfbc2f3107045635996b822/usr/share/doc/openssl/changelog.gz", - "sha1": "", - "md5": "", - "extra_data": {}, - "copyrights": [], - "holders": [], - "authors": [], - "licenses": [], - "license_expressions": [], - "emails": [], - "urls": [], - "status": "ignored-empty-file", - "tag": "img-f596dc-layer-02-cb3279", - "type": "file", - "name": "changelog.gz", - "extension": ".gz", - "programming_language": "", - "mime_type": "inode/x-empty", - "is_binary": false, - "is_text": true, - "is_archive": false, - "is_key_file": false, - "is_media": false - }, - { - "for_packages": [ - "pkg:deb/distroless/openssl@1.1.1n-0%2Bdeb11u2?architecture=amd64&uuid=fixed-uid-done-for-testing-5642512d1758" - ], - "path": "gcr_io_distroless_base.tar.gz-extract/cb3279093e638ddfd56bff4d3d89c5a3ed6dd59dbcfbc2f3107045635996b822/usr/share/doc/openssl/copyright", - "sha1": "d2d7b4d2d035a7732fcded56e31300b65d471545", - "md5": "6ca25957fe201dc234ff25bc24f98e0a", - "extra_data": {}, - "copyrights": [], - "holders": [], - "authors": [], - "licenses": [], - "license_expressions": [], - "emails": [], - "urls": [], - "status": "system-package", - "tag": "img-f596dc-layer-02-cb3279", - "type": "file", - "name": "copyright", - "extension": "", - "programming_language": "", - "mime_type": "text/plain", - "is_binary": false, - "is_text": true, - "is_archive": false, - "is_key_file": false, - "is_media": false - }, - { - "for_packages": [], - "path": "gcr_io_distroless_base.tar.gz-extract/cb3279093e638ddfd56bff4d3d89c5a3ed6dd59dbcfbc2f3107045635996b822/usr/share/doc/openssl/fingerprints.txt", - "sha1": "", - "md5": "", - "extra_data": {}, - "copyrights": [], - "holders": [], - "authors": [], - "licenses": [], - "license_expressions": [], - "emails": [], - "urls": [], - "status": "ignored-empty-file", - "tag": "img-f596dc-layer-02-cb3279", - "type": "file", - "name": "fingerprints.txt", - "extension": ".txt", - "programming_language": "", - "mime_type": "inode/x-empty", - "is_binary": false, - "is_text": true, - "is_archive": false, - "is_key_file": false, - "is_media": false - }, { "for_packages": [], "path": "gcr_io_distroless_base.tar.gz-extract/cb3279093e638ddfd56bff4d3d89c5a3ed6dd59dbcfbc2f3107045635996b822/usr/share/lintian/overrides/libc6", @@ -12306,32 +12096,6 @@ "is_media": false, "package_data": [] }, - { - "for_packages": [], - "path": "gcr_io_distroless_base.tar.gz-extract/d92879194ba1c23b840306b007bce6568f71f0e954d63625d48504d533749e30/usr/share/doc/tzdata/README.Debian", - "sha1": "", - "md5": "", - "extra_data": {}, - "copyrights": [], - "holders": [], - "authors": [], - "licenses": [], - "license_expressions": [], - "emails": [], - "urls": [], - "status": "ignored-empty-file", - "tag": "img-f596dc-layer-01-d92879", - "type": "file", - "name": "README.Debian", - "extension": ".Debian", - "programming_language": "", - "mime_type": "inode/x-empty", - "is_binary": false, - "is_text": true, - "is_archive": false, - "is_key_file": false, - "is_media": false - }, { "for_packages": [], "path": "gcr_io_distroless_base.tar.gz-extract/d92879194ba1c23b840306b007bce6568f71f0e954d63625d48504d533749e30/usr/share/doc/tzdata/changelog.Debian.gz", @@ -14251,32 +14015,6 @@ "is_media": false, "package_data": [] }, - { - "for_packages": [], - "path": "gcr_io_distroless_base.tar.gz-extract/d92879194ba1c23b840306b007bce6568f71f0e954d63625d48504d533749e30/usr/share/zoneinfo/America/Fortaleza", - "sha1": "", - "md5": "", - "extra_data": {}, - "copyrights": [], - "holders": [], - "authors": [], - "licenses": [], - "license_expressions": [], - "emails": [], - "urls": [], - "status": "ignored-empty-file", - "tag": "img-f596dc-layer-01-d92879", - "type": "file", - "name": "Fortaleza", - "extension": "", - "programming_language": "", - "mime_type": "inode/x-empty", - "is_binary": false, - "is_text": true, - "is_archive": false, - "is_key_file": false, - "is_media": false - }, { "for_packages": [], "path": "gcr_io_distroless_base.tar.gz-extract/d92879194ba1c23b840306b007bce6568f71f0e954d63625d48504d533749e30/usr/share/zoneinfo/America/Glace_Bay", diff --git a/scanpipe/tests/test_api.py b/scanpipe/tests/test_api.py index 265105573..c46928761 100644 --- a/scanpipe/tests/test_api.py +++ b/scanpipe/tests/test_api.py @@ -315,7 +315,7 @@ def test_scanpipe_api_project_action_resources(self): self.assertEqual(1, len(response.data)) resource = response.data[0] self.assertEqual( - ["pkg:deb/debian/adduser@3.118?arch=all"], resource["for_packages"] + ["pkg:deb/debian/adduser@3.118?uuid=610bed29-ce39-40e7-92d6-fd8b"], resource["for_packages"] ) self.assertEqual("filename.ext", resource["path"]) diff --git a/scanpipe/tests/test_pipes.py b/scanpipe/tests/test_pipes.py index a0b2a1304..2b33cf9b9 100644 --- a/scanpipe/tests/test_pipes.py +++ b/scanpipe/tests/test_pipes.py @@ -23,6 +23,7 @@ import collections import json import os +import re import shutil import tempfile from pathlib import Path @@ -36,6 +37,7 @@ from django.test import override_settings from commoncode.archive import extract_tar +from scancode.cli_test_utils import purl_with_fake_uuid from scancode.interrupt import TimeoutError as InterruptTimeoutError from scanpipe.models import CodebaseResource @@ -129,7 +131,7 @@ def test_scanpipe_pipes_outputs_queryset_to_csv_file(self): expected = [ "for_packages,path\n", - "['pkg:deb/debian/adduser@3.118?arch=all'],filename.ext\n", + "['pkg:deb/debian/adduser@3.118?uuid=610bed29-ce39-40e7-92d6-fd8b'],filename.ext\n", ] with output_file_path.open() as f: self.assertEqual(expected, f.readlines()) @@ -172,7 +174,7 @@ def test_scanpipe_pipes_outputs_queryset_to_csv_stream(self): expected = [ "for_packages,path\n", - "['pkg:deb/debian/adduser@3.118?arch=all'],filename.ext\n", + f"['pkg:deb/debian/adduser@3.118?uuid=610bed29-ce39-40e7-92d6-fd8b'],filename.ext\n", ] with output_file.open() as f: self.assertEqual(expected, f.readlines()) @@ -1237,7 +1239,7 @@ def test_scanpipe_add_to_package(self): scancode.add_to_package(package1.package_uid, resource1, project1) self.assertEqual(len(resource1.for_packages), 1) - self.assertIn(package1.package_url, resource1.for_packages) + self.assertIn(package1.package_uid, resource1.for_packages) # Package will not be added twice since it is already associated with the # resource.