From 2622813ec7111ae44ee8c8939a2ae7a3e1856e07 Mon Sep 17 00:00:00 2001 From: Dustin Ingram Date: Wed, 21 Aug 2024 19:13:08 +0000 Subject: [PATCH] Revert "Store attestations for PEP740 (#16302)" This reverts commit 554588447ec96c1fefb20239b515f9ad56b28b97. --- requirements/main.in | 4 +- requirements/main.txt | 12 +- tests/common/db/attestation.py | 28 -- tests/common/db/packaging.py | 7 - tests/unit/api/test_simple.py | 25 -- tests/unit/attestations/__init__.py | 11 - tests/unit/attestations/test_services.py | 419 ----------------- tests/unit/forklift/test_legacy.py | 420 +++++++++++++++--- tests/unit/packaging/test_utils.py | 36 -- warehouse/attestations/__init__.py | 27 -- warehouse/attestations/errors.py | 19 - warehouse/attestations/interfaces.py | 54 --- warehouse/attestations/models.py | 55 --- warehouse/attestations/services.py | 231 ---------- warehouse/forklift/legacy.py | 124 ++++-- .../7f0c9f105f44_create_attestations_table.py | 47 -- warehouse/packaging/models.py | 8 - warehouse/packaging/utils.py | 8 +- warehouse/templates/api/simple/detail.html | 2 +- 19 files changed, 474 insertions(+), 1063 deletions(-) delete mode 100644 tests/common/db/attestation.py delete mode 100644 tests/unit/attestations/__init__.py delete mode 100644 tests/unit/attestations/test_services.py delete mode 100644 warehouse/attestations/__init__.py delete mode 100644 warehouse/attestations/errors.py delete mode 100644 warehouse/attestations/interfaces.py delete mode 100644 warehouse/attestations/models.py delete mode 100644 warehouse/attestations/services.py delete mode 100644 warehouse/migrations/versions/7f0c9f105f44_create_attestations_table.py diff --git a/requirements/main.in b/requirements/main.in index 6fa06b782f43..068b7ff21a4c 100644 --- a/requirements/main.in +++ b/requirements/main.in @@ -62,8 +62,8 @@ redis>=2.8.0,<6.0.0 rfc3986 sentry-sdk setuptools -sigstore~=3.2.0 -pypi-attestations==0.0.11 +sigstore~=3.0.0 +pypi-attestations==0.0.9 sqlalchemy[asyncio]>=2.0,<3.0 stdlib-list stripe diff --git a/requirements/main.txt b/requirements/main.txt index e1ff17e074af..922852b57c3e 100644 --- a/requirements/main.txt +++ b/requirements/main.txt @@ -1770,9 +1770,9 @@ pyparsing==3.1.2 \ --hash=sha256:a1bac0ce561155ecc3ed78ca94d3c9378656ad4c94c1270de543f621420f94ad \ --hash=sha256:f9db75911801ed778fe61bb643079ff86601aca99fcae6345aa67292038fb742 # via linehaul -pypi-attestations==0.0.11 \ - --hash=sha256:b730e6b23874d94da0f3817b1f9dd3ecb6a80d685f62a18ad96e5b0396149ded \ - --hash=sha256:e74329074f049568591e300373e12fcd46a35e21723110856546e33bf2949efa +pypi-attestations==0.0.9 \ + --hash=sha256:3bfc07f64a8db0d6e2646720e70df7c7cb01a2936056c764a2cc3268969332f2 \ + --hash=sha256:4b38cce5d221c8145cac255bfafe650ec0028d924d2b3572394df8ba8f07a609 # via -r requirements/main.in pyqrcode==1.2.1 \ --hash=sha256:1b2812775fa6ff5c527977c4cd2ccb07051ca7d0bc0aecf937a43864abe5eff6 \ @@ -2079,9 +2079,9 @@ sentry-sdk==2.13.0 \ --hash=sha256:6beede8fc2ab4043da7f69d95534e320944690680dd9a963178a49de71d726c6 \ --hash=sha256:8d4a576f7a98eb2fdb40e13106e41f330e5c79d72a68be1316e7852cf4995260 # via -r requirements/main.in -sigstore==3.2.0 \ - --hash=sha256:25c8a871a3a6adf959c0cde598ea8bef8794f1a29277d067111eb4ded4ba7f65 \ - --hash=sha256:d18508f34febb7775065855e92557fa1c2c16580df88f8e8903b9514438bad44 +sigstore==3.0.0 \ + --hash=sha256:6cc7dc92607c2fd481aada0f3c79e710e4c6086e3beab50b07daa9a50a79d109 \ + --hash=sha256:a6a9538a648e112a0c3d8092d3f73a351c7598164764f1e73a6b5ba406a3a0bd # via # -r requirements/main.in # pypi-attestations diff --git a/tests/common/db/attestation.py b/tests/common/db/attestation.py deleted file mode 100644 index 2080519a806f..000000000000 --- a/tests/common/db/attestation.py +++ /dev/null @@ -1,28 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import hashlib - -import factory - -from warehouse.attestations.models import Attestation - -from .base import WarehouseFactory - - -class AttestationFactory(WarehouseFactory): - class Meta: - model = Attestation - - file = factory.SubFactory("tests.common.db.packaging.FileFactory") - attestation_file_blake2_digest = factory.LazyAttribute( - lambda o: hashlib.blake2b(o.file.filename.encode("utf8")).hexdigest() - ) diff --git a/tests/common/db/packaging.py b/tests/common/db/packaging.py index 369dc9f092d0..2a12379da170 100644 --- a/tests/common/db/packaging.py +++ b/tests/common/db/packaging.py @@ -34,7 +34,6 @@ from warehouse.utils import readme from .accounts import UserFactory -from .attestation import AttestationFactory from .base import WarehouseFactory from .observations import ObserverFactory @@ -141,12 +140,6 @@ class Meta: ) ) - attestations = factory.RelatedFactoryList( - AttestationFactory, - factory_related_name="file", - size=1, - ) - class FileEventFactory(WarehouseFactory): class Meta: diff --git a/tests/unit/api/test_simple.py b/tests/unit/api/test_simple.py index 5f8003d50eb5..9937038dd145 100644 --- a/tests/unit/api/test_simple.py +++ b/tests/unit/api/test_simple.py @@ -18,7 +18,6 @@ from pyramid.testing import DummyRequest from warehouse.api import simple -from warehouse.attestations import IIntegrityService from warehouse.packaging.utils import API_VERSION from ...common.db.accounts import UserFactory @@ -88,16 +87,6 @@ def test_selects(self, header, expected): class TestSimpleIndex: - - @pytest.fixture - def db_request(self, db_request): - """Override db_request to add the Release Verification service""" - db_request.find_service = lambda svc, name=None, context=None: { - IIntegrityService: pretend.stub(), - }.get(svc) - - return db_request - @pytest.mark.parametrize( ("content_type", "renderer_override"), CONTENT_TYPE_PARAMS, @@ -196,17 +185,6 @@ def test_quarantined_project_omitted_from_index(self, db_request): class TestSimpleDetail: - @pytest.fixture - def db_request(self, db_request): - """Override db_request to add the Release Verification service""" - db_request.find_service = lambda svc, name=None, context=None: { - IIntegrityService: pretend.stub( - get_provenance_digest=lambda *args, **kwargs: None, - ), - }.get(svc) - - return db_request - def test_redirects(self, pyramid_request): project = pretend.stub(normalized_name="foo") @@ -308,7 +286,6 @@ def test_with_files_no_serial(self, db_request, content_type, renderer_override) "upload-time": f.upload_time.isoformat() + "Z", "data-dist-info-metadata": False, "core-metadata": False, - "provenance": None, } for f in files ], @@ -357,7 +334,6 @@ def test_with_files_with_serial(self, db_request, content_type, renderer_overrid "upload-time": f.upload_time.isoformat() + "Z", "data-dist-info-metadata": False, "core-metadata": False, - "provenance": None, } for f in files ], @@ -451,7 +427,6 @@ def test_with_files_with_version_multi_digit( if f.metadata_file_sha256_digest is not None else False ), - "provenance": None, } for f in files ], diff --git a/tests/unit/attestations/__init__.py b/tests/unit/attestations/__init__.py deleted file mode 100644 index 164f68b09175..000000000000 --- a/tests/unit/attestations/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/tests/unit/attestations/test_services.py b/tests/unit/attestations/test_services.py deleted file mode 100644 index a83a0c302b00..000000000000 --- a/tests/unit/attestations/test_services.py +++ /dev/null @@ -1,419 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import hashlib -import tempfile - -import pretend -import pytest - -from pydantic import TypeAdapter -from pypi_attestations import ( - Attestation, - AttestationBundle, - AttestationType, - Envelope, - GitHubPublisher, - GitLabPublisher, - Provenance, - VerificationError, - VerificationMaterial, -) -from sigstore.verify import Verifier -from zope.interface.verify import verifyClass - -from tests.common.db.oidc import GitHubPublisherFactory, GitLabPublisherFactory -from tests.common.db.packaging import FileEventFactory, FileFactory -from warehouse.attestations import ( - Attestation as DatabaseAttestation, - AttestationUploadError, - IIntegrityService, - IntegrityService, - UnsupportedPublisherError, - services, -) -from warehouse.events.tags import EventTag -from warehouse.metrics import IMetricsService -from warehouse.packaging import File, IFileStorage - -VALID_ATTESTATION = Attestation( - version=1, - verification_material=VerificationMaterial( - certificate="somebase64string", transparency_entries=[dict()] - ), - envelope=Envelope( - statement="somebase64string", - signature="somebase64string", - ), -) - - -class TestAttestationsService: - def test_interface_matches(self): - assert verifyClass(IIntegrityService, IntegrityService) - - def test_create_service(self): - request = pretend.stub( - find_service=pretend.call_recorder( - lambda svc, context=None, name=None: None - ), - ) - - assert IntegrityService.create_service(None, request) is not None - assert not set(request.find_service.calls) ^ { - pretend.call(IFileStorage), - pretend.call(IMetricsService), - } - - def test_persist_attestations(self, db_request, monkeypatch): - @pretend.call_recorder - def storage_service_store(path: str, file_path, *_args, **_kwargs): - expected = VALID_ATTESTATION.model_dump_json().encode("utf-8") - with open(file_path, "rb") as fp: - assert fp.read() == expected - - assert path.endswith(".attestation") - - integrity_service = IntegrityService( - storage=pretend.stub( - store=storage_service_store, - ), - metrics=pretend.stub(), - ) - - file = FileFactory.create(attestations=[]) - - integrity_service.persist_attestations([VALID_ATTESTATION], file) - - attestations_db = ( - db_request.db.query(DatabaseAttestation) - .join(DatabaseAttestation.file) - .filter(File.filename == file.filename) - .all() - ) - assert len(attestations_db) == 1 - assert len(file.attestations) == 1 - - def test_parse_no_publisher(self, db_request): - integrity_service = IntegrityService( - storage=pretend.stub(), - metrics=pretend.stub(), - ) - - db_request.oidc_publisher = None - with pytest.raises( - AttestationUploadError, - match="Attestations are only supported when using Trusted", - ): - integrity_service.parse_attestations(db_request, pretend.stub()) - - def test_parse_unsupported_publisher(self, db_request): - integrity_service = IntegrityService( - storage=pretend.stub(), - metrics=pretend.stub(), - ) - db_request.oidc_publisher = pretend.stub(publisher_name="not-existing") - with pytest.raises( - AttestationUploadError, - match="Attestations are only supported when using Trusted", - ): - integrity_service.parse_attestations(db_request, pretend.stub()) - - def test_parse_malformed_attestation(self, metrics, db_request): - integrity_service = IntegrityService( - storage=pretend.stub(), - metrics=metrics, - ) - - db_request.oidc_publisher = pretend.stub(publisher_name="GitHub") - db_request.POST["attestations"] = "{'malformed-attestation'}" - with pytest.raises( - AttestationUploadError, - match="Error while decoding the included attestation", - ): - integrity_service.parse_attestations(db_request, pretend.stub()) - - assert ( - pretend.call("warehouse.upload.attestations.malformed") - in metrics.increment.calls - ) - - def test_parse_multiple_attestations(self, metrics, db_request): - integrity_service = IntegrityService( - storage=pretend.stub(), - metrics=metrics, - ) - - db_request.oidc_publisher = pretend.stub(publisher_name="GitHub") - db_request.POST["attestations"] = TypeAdapter(list[Attestation]).dump_json( - [VALID_ATTESTATION, VALID_ATTESTATION] - ) - with pytest.raises( - AttestationUploadError, match="Only a single attestation per file" - ): - integrity_service.parse_attestations( - db_request, - pretend.stub(), - ) - - assert ( - pretend.call("warehouse.upload.attestations.failed_multiple_attestations") - in metrics.increment.calls - ) - - @pytest.mark.parametrize( - ("verify_exception", "expected_message"), - [ - ( - VerificationError, - "Could not verify the uploaded", - ), - ( - ValueError, - "Unknown error while", - ), - ], - ) - def test_parse_failed_verification( - self, metrics, monkeypatch, db_request, verify_exception, expected_message - ): - integrity_service = IntegrityService( - storage=pretend.stub(), - metrics=metrics, - ) - - db_request.oidc_publisher = pretend.stub( - publisher_name="GitHub", - publisher_verification_policy=pretend.call_recorder(lambda c: None), - ) - db_request.oidc_claims = {"sha": "somesha"} - db_request.POST["attestations"] = TypeAdapter(list[Attestation]).dump_json( - [VALID_ATTESTATION] - ) - - def failing_verify(_self, _verifier, _policy, _dist): - raise verify_exception("error") - - monkeypatch.setattr(Verifier, "production", lambda: pretend.stub()) - monkeypatch.setattr(Attestation, "verify", failing_verify) - - with pytest.raises(AttestationUploadError, match=expected_message): - integrity_service.parse_attestations( - db_request, - pretend.stub(), - ) - - def test_parse_wrong_predicate(self, metrics, monkeypatch, db_request): - integrity_service = IntegrityService( - storage=pretend.stub(), - metrics=metrics, - ) - db_request.oidc_publisher = pretend.stub( - publisher_name="GitHub", - publisher_verification_policy=pretend.call_recorder(lambda c: None), - ) - db_request.oidc_claims = {"sha": "somesha"} - db_request.POST["attestations"] = TypeAdapter(list[Attestation]).dump_json( - [VALID_ATTESTATION] - ) - - monkeypatch.setattr(Verifier, "production", lambda: pretend.stub()) - monkeypatch.setattr( - Attestation, "verify", lambda *args: ("wrong-predicate", {}) - ) - - with pytest.raises( - AttestationUploadError, match="Attestation with unsupported predicate" - ): - integrity_service.parse_attestations( - db_request, - pretend.stub(), - ) - - assert ( - pretend.call( - "warehouse.upload.attestations.failed_unsupported_predicate_type" - ) - in metrics.increment.calls - ) - - def test_parse_succeed(self, metrics, monkeypatch, db_request): - integrity_service = IntegrityService( - storage=pretend.stub(), - metrics=metrics, - ) - db_request.oidc_publisher = pretend.stub( - publisher_name="GitHub", - publisher_verification_policy=pretend.call_recorder(lambda c: None), - ) - db_request.oidc_claims = {"sha": "somesha"} - db_request.POST["attestations"] = TypeAdapter(list[Attestation]).dump_json( - [VALID_ATTESTATION] - ) - - monkeypatch.setattr(Verifier, "production", lambda: pretend.stub()) - monkeypatch.setattr( - Attestation, "verify", lambda *args: (AttestationType.PYPI_PUBLISH_V1, {}) - ) - - attestations = integrity_service.parse_attestations( - db_request, - pretend.stub(), - ) - assert attestations == [VALID_ATTESTATION] - - def test_generate_provenance_unsupported_publisher(self): - integrity_service = IntegrityService( - storage=pretend.stub(), - metrics=pretend.stub(), - ) - - oidc_publisher = pretend.stub(publisher_name="not-existing") - - assert ( - integrity_service.generate_provenance(oidc_publisher, pretend.stub()) - is None - ) - - @pytest.mark.parametrize( - "publisher_name", - [ - "github", - "gitlab", - ], - ) - def test_generate_provenance_succeeds(self, publisher_name: str, monkeypatch): - integrity_service = IntegrityService( - storage=pretend.stub(), - metrics=pretend.stub(), - ) - - if publisher_name == "github": - publisher = GitHubPublisher( - repository="fake-repository", - workflow="fake-workflow", - ) - else: - publisher = GitLabPublisher( - repository="fake-repository", - environment="fake-env", - ) - - monkeypatch.setattr( - services, - "_publisher_from_oidc_publisher", - lambda s: publisher, - ) - - provenance = integrity_service.generate_provenance( - pretend.stub(), - [VALID_ATTESTATION], - ) - - assert provenance == Provenance( - attestation_bundles=[ - AttestationBundle( - publisher=publisher, - attestations=[VALID_ATTESTATION], - ) - ] - ) - - def test_persist_provenance_succeeds(self, db_request): - provenance = Provenance( - attestation_bundles=[ - AttestationBundle( - publisher=GitHubPublisher( - repository="fake-repository", - workflow="fake-workflow", - ), - attestations=[VALID_ATTESTATION], - ) - ] - ) - - @pretend.call_recorder - def storage_service_store(path, file_path, *_args, **_kwargs): - expected = provenance.model_dump_json().encode("utf-8") - with open(file_path, "rb") as fp: - assert fp.read() == expected - - assert path.suffix == ".provenance" - - integrity_service = IntegrityService( - storage=pretend.stub(store=storage_service_store), - metrics=pretend.stub(), - ) - assert ( - integrity_service.persist_provenance(provenance, FileFactory.create()) - is None - ) - - def test_get_provenance_digest(self, db_request): - file = FileFactory.create() - FileEventFactory.create( - source=file, - tag=EventTag.File.FileAdd, - additional={"publisher_url": "fake-publisher-url"}, - ) - - with tempfile.NamedTemporaryFile() as f: - integrity_service = IntegrityService( - storage=pretend.stub(get=pretend.call_recorder(lambda p: f)), - metrics=pretend.stub(), - ) - - assert ( - integrity_service.get_provenance_digest(file) - == hashlib.file_digest(f, "sha256").hexdigest() - ) - - def test_get_provenance_digest_fails_no_attestations(self, db_request): - # If the attestations are missing, there is no provenance file - file = FileFactory.create() - file.attestations = [] - FileEventFactory.create( - source=file, - tag=EventTag.File.FileAdd, - additional={"publisher_url": "fake-publisher-url"}, - ) - integrity_service = IntegrityService( - storage=pretend.stub(), - metrics=pretend.stub(), - ) - - assert integrity_service.get_provenance_digest(file) is None - - -def test_publisher_from_oidc_publisher_github(db_request): - publisher = GitHubPublisherFactory.create() - - attestation_publisher = services._publisher_from_oidc_publisher(publisher) - assert isinstance(attestation_publisher, GitHubPublisher) - assert attestation_publisher.repository == publisher.repository - assert attestation_publisher.workflow == publisher.workflow_filename - assert attestation_publisher.environment == publisher.environment - - -def test_publisher_from_oidc_publisher_gitlab(db_request): - publisher = GitLabPublisherFactory.create() - - attestation_publisher = services._publisher_from_oidc_publisher(publisher) - assert isinstance(attestation_publisher, GitLabPublisher) - assert attestation_publisher.repository == publisher.project_path - assert attestation_publisher.environment == publisher.environment - - -def test_publisher_from_oidc_publisher_fails(): - publisher = pretend.stub(publisher_name="not-existing") - - with pytest.raises(UnsupportedPublisherError): - services._publisher_from_oidc_publisher(publisher) diff --git a/tests/unit/forklift/test_legacy.py b/tests/unit/forklift/test_legacy.py index 6e062ef1f0f2..e127299ca204 100644 --- a/tests/unit/forklift/test_legacy.py +++ b/tests/unit/forklift/test_legacy.py @@ -23,8 +23,15 @@ import pretend import pytest -from pypi_attestations import Attestation, Envelope, VerificationMaterial +from pypi_attestations import ( + Attestation, + Distribution, + Envelope, + VerificationError, + VerificationMaterial, +) from pyramid.httpexceptions import HTTPBadRequest, HTTPForbidden, HTTPTooManyRequests +from sigstore.verify import Verifier from sqlalchemy import and_, exists from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import joinedload @@ -35,11 +42,6 @@ from warehouse.accounts.utils import UserContext from warehouse.admin.flags import AdminFlag, AdminFlagValue -from warehouse.attestations import ( - Attestation as DatabaseAttestation, - AttestationUploadError, - IIntegrityService, -) from warehouse.classifiers.models import Classifier from warehouse.forklift import legacy, metadata from warehouse.macaroons import IMacaroonService, caveats, security_policy @@ -61,7 +63,6 @@ from warehouse.packaging.tasks import sync_file_to_cache, update_bigquery_release_files from ...common.db.accounts import EmailFactory, UserFactory -from ...common.db.attestation import AttestationFactory from ...common.db.classifiers import ClassifierFactory from ...common.db.oidc import GitHubPublisherFactory from ...common.db.packaging import ( @@ -2419,6 +2420,85 @@ def test_upload_fails_without_oidc_publisher_permission( "See /the/help/url/ for more information." ).format(project.name) + def test_upload_attestation_fails_without_oidc_publisher( + self, + monkeypatch, + pyramid_config, + db_request, + metrics, + project_service, + macaroon_service, + ): + project = ProjectFactory.create() + owner = UserFactory.create() + maintainer = UserFactory.create() + RoleFactory.create(user=owner, project=project, role_name="Owner") + RoleFactory.create(user=maintainer, project=project, role_name="Maintainer") + + EmailFactory.create(user=maintainer) + db_request.user = maintainer + raw_macaroon, macaroon = macaroon_service.create_macaroon( + "fake location", + "fake description", + [caveats.RequestUser(user_id=str(maintainer.id))], + user_id=maintainer.id, + ) + identity = UserContext(maintainer, macaroon) + + filename = "{}-{}.tar.gz".format(project.name, "1.0") + attestation = Attestation( + version=1, + verification_material=VerificationMaterial( + certificate="some_cert", transparency_entries=[dict()] + ), + envelope=Envelope( + statement="somebase64string", + signature="somebase64string", + ), + ) + + pyramid_config.testing_securitypolicy(identity=identity) + db_request.POST = MultiDict( + { + "metadata_version": "1.2", + "name": project.name, + "attestations": f"[{attestation.model_dump_json()}]", + "version": "1.0", + "filetype": "sdist", + "md5_digest": _TAR_GZ_PKG_MD5, + "content": pretend.stub( + filename=filename, + file=io.BytesIO(_TAR_GZ_PKG_TESTDATA), + type="application/tar", + ), + } + ) + + storage_service = pretend.stub(store=lambda path, filepath, meta: None) + extract_http_macaroon = pretend.call_recorder(lambda r, _: raw_macaroon) + monkeypatch.setattr( + security_policy, "_extract_http_macaroon", extract_http_macaroon + ) + + db_request.find_service = lambda svc, name=None, context=None: { + IFileStorage: storage_service, + IMacaroonService: macaroon_service, + IMetricsService: metrics, + IProjectService: project_service, + }.get(svc) + db_request.user_agent = "warehouse-tests/6.6.6" + + with pytest.raises(HTTPBadRequest) as excinfo: + legacy.file_upload(db_request) + + resp = excinfo.value + + assert resp.status_code == 400 + assert resp.status == ( + "400 Attestations are currently only supported when using Trusted " + "Publishing with GitHub Actions." + ) + @pytest.mark.parametrize( "plat", [ @@ -3328,10 +3408,8 @@ def test_upload_succeeds_creates_release( ), ] - @pytest.mark.parametrize("provenance_rv", [None, "fake-provenance-object"]) - def test_upload_succeeds_with_valid_attestation( + def test_upload_with_valid_attestation_succeeds( self, - provenance_rv, monkeypatch, pyramid_config, db_request, @@ -3386,68 +3464,296 @@ def test_upload_succeeds_with_valid_attestation( } ) - def persist_attestations(attestations, file): - file.attestations.append(AttestationFactory.create(file=file)) + storage_service = pretend.stub(store=lambda path, filepath, meta: None) + db_request.find_service = lambda svc, name=None, context=None: { + IFileStorage: storage_service, + IMetricsService: metrics, + }.get(svc) + + record_event = pretend.call_recorder( + lambda self, *, tag, request=None, additional: None + ) + monkeypatch.setattr(HasEvents, "record_event", record_event) + + verify = pretend.call_recorder( + lambda _self, _verifier, _policy, _dist: ( + "https://docs.pypi.org/attestations/publish/v1", + None, + ) + ) + monkeypatch.setattr(Attestation, "verify", verify) + monkeypatch.setattr(Verifier, "production", lambda: pretend.stub()) + + resp = legacy.file_upload(db_request) + + assert resp.status_code == 200 + + assert len(verify.calls) == 1 + verified_distribution = verify.calls[0].args[3] + assert verified_distribution == Distribution( + name=filename, digest=_TAR_GZ_PKG_SHA256 + ) - def persist_provenance(provenance_object, file): - assert provenance_object == provenance_rv + def test_upload_with_invalid_attestation_predicate_type_fails( + self, + monkeypatch, + pyramid_config, + db_request, + metrics, + ): + from warehouse.events.models import HasEvents - storage_service = pretend.stub(store=lambda path, filepath, *, meta=None: None) - integrity_service = pretend.stub( - parse_attestations=lambda *args, **kwargs: [attestation], - persist_attestations=persist_attestations, - generate_provenance=pretend.call_recorder( - lambda oidc_publisher, attestations: provenance_rv + project = ProjectFactory.create() + version = "1.0" + publisher = GitHubPublisherFactory.create(projects=[project]) + claims = { + "sha": "somesha", + "repository": f"{publisher.repository_owner}/{publisher.repository_name}", + "workflow": "workflow_name", + } + identity = PublisherTokenContext(publisher, SignedClaims(claims)) + db_request.oidc_publisher = identity.publisher + db_request.oidc_claims = identity.claims + + db_request.db.add(Classifier(classifier="Environment :: Other Environment")) + db_request.db.add(Classifier(classifier="Programming Language :: Python")) + + filename = "{}-{}.tar.gz".format(project.name, "1.0") + attestation = Attestation( + version=1, + verification_material=VerificationMaterial( + certificate="somebase64string", transparency_entries=[dict()] + ), + envelope=Envelope( + statement="somebase64string", + signature="somebase64string", ), - persist_provenance=persist_provenance, ) + + pyramid_config.testing_securitypolicy(identity=identity) + db_request.user = None + db_request.user_agent = "warehouse-tests/6.6.6" + db_request.POST = MultiDict( + { + "metadata_version": "1.2", + "name": project.name, + "attestations": f"[{attestation.model_dump_json()}]", + "version": version, + "summary": "This is my summary!", + "filetype": "sdist", + "md5_digest": _TAR_GZ_PKG_MD5, + "content": pretend.stub( + filename=filename, + file=io.BytesIO(_TAR_GZ_PKG_TESTDATA), + type="application/tar", + ), + } + ) + + storage_service = pretend.stub(store=lambda path, filepath, meta: None) db_request.find_service = lambda svc, name=None, context=None: { IFileStorage: storage_service, IMetricsService: metrics, - IIntegrityService: integrity_service, }.get(svc) record_event = pretend.call_recorder( lambda self, *, tag, request=None, additional: None ) monkeypatch.setattr(HasEvents, "record_event", record_event) - resp = legacy.file_upload(db_request) - assert resp.status_code == 200 + invalid_predicate_type = "Unsupported predicate type" + verify = pretend.call_recorder( + lambda _self, _verifier, _policy, _dist: (invalid_predicate_type, None) + ) + monkeypatch.setattr(Attestation, "verify", verify) + monkeypatch.setattr(Verifier, "production", lambda: pretend.stub()) - assert ( - pretend.call("warehouse.upload.attestations.ok") in metrics.increment.calls + with pytest.raises(HTTPBadRequest) as excinfo: + legacy.file_upload(db_request) + + resp = excinfo.value + + assert resp.status_code == 400 + assert resp.status.startswith( + f"400 Attestation with unsupported predicate type: {invalid_predicate_type}" ) - attestations_db = ( - db_request.db.query(DatabaseAttestation) - .join(DatabaseAttestation.file) - .filter(File.filename == filename) - .all() + + def test_upload_with_multiple_attestations_fails( + self, + monkeypatch, + pyramid_config, + db_request, + metrics, + ): + from warehouse.events.models import HasEvents + + project = ProjectFactory.create() + version = "1.0" + publisher = GitHubPublisherFactory.create(projects=[project]) + claims = { + "sha": "somesha", + "repository": f"{publisher.repository_owner}/{publisher.repository_name}", + "workflow": "workflow_name", + } + identity = PublisherTokenContext(publisher, SignedClaims(claims)) + db_request.oidc_publisher = identity.publisher + db_request.oidc_claims = identity.claims + + db_request.db.add(Classifier(classifier="Environment :: Other Environment")) + db_request.db.add(Classifier(classifier="Programming Language :: Python")) + + filename = "{}-{}.tar.gz".format(project.name, "1.0") + attestation = Attestation( + version=1, + verification_material=VerificationMaterial( + certificate="somebase64string", transparency_entries=[dict()] + ), + envelope=Envelope( + statement="somebase64string", + signature="somebase64string", + ), ) - assert len(attestations_db) == 1 - assert integrity_service.generate_provenance.calls == [ - pretend.call(db_request.oidc_publisher, [attestation]) - ] + pyramid_config.testing_securitypolicy(identity=identity) + db_request.user = None + db_request.user_agent = "warehouse-tests/6.6.6" + db_request.POST = MultiDict( + { + "metadata_version": "1.2", + "name": project.name, + "attestations": f"[{attestation.model_dump_json()}," + f" {attestation.model_dump_json()}]", + "version": version, + "summary": "This is my summary!", + "filetype": "sdist", + "md5_digest": _TAR_GZ_PKG_MD5, + "content": pretend.stub( + filename=filename, + file=io.BytesIO(_TAR_GZ_PKG_TESTDATA), + type="application/tar", + ), + } + ) + + storage_service = pretend.stub(store=lambda path, filepath, meta: None) + db_request.find_service = lambda svc, name=None, context=None: { + IFileStorage: storage_service, + IMetricsService: metrics, + }.get(svc) + + record_event = pretend.call_recorder( + lambda self, *, tag, request=None, additional: None + ) + monkeypatch.setattr(HasEvents, "record_event", record_event) + + verify = pretend.call_recorder( + lambda _self, _verifier, _policy, _dist: ( + "https://docs.pypi.org/attestations/publish/v1", + None, + ) + ) + monkeypatch.setattr(Attestation, "verify", verify) + monkeypatch.setattr(Verifier, "production", lambda: pretend.stub()) + + with pytest.raises(HTTPBadRequest) as excinfo: + legacy.file_upload(db_request) + + resp = excinfo.value + + assert resp.status_code == 400 + assert resp.status.startswith( + "400 Only a single attestation per-file is supported at the moment." + ) + + def test_upload_with_malformed_attestation_fails( + self, + monkeypatch, + pyramid_config, + db_request, + metrics, + ): + from warehouse.events.models import HasEvents + + project = ProjectFactory.create() + version = "1.0" + publisher = GitHubPublisherFactory.create(projects=[project]) + claims = { + "sha": "somesha", + "repository": f"{publisher.repository_owner}/{publisher.repository_name}", + "workflow": "workflow_name", + } + identity = PublisherTokenContext(publisher, SignedClaims(claims)) + db_request.oidc_publisher = identity.publisher + db_request.oidc_claims = identity.claims + + db_request.db.add(Classifier(classifier="Environment :: Other Environment")) + db_request.db.add(Classifier(classifier="Programming Language :: Python")) + + filename = "{}-{}.tar.gz".format(project.name, "1.0") + + pyramid_config.testing_securitypolicy(identity=identity) + db_request.user = None + db_request.user_agent = "warehouse-tests/6.6.6" + db_request.POST = MultiDict( + { + "metadata_version": "1.2", + "name": project.name, + "attestations": "[{'a_malformed_attestation': 3}]", + "version": version, + "summary": "This is my summary!", + "filetype": "sdist", + "md5_digest": _TAR_GZ_PKG_MD5, + "content": pretend.stub( + filename=filename, + file=io.BytesIO(_TAR_GZ_PKG_TESTDATA), + type="application/tar", + ), + } + ) + + storage_service = pretend.stub(store=lambda path, filepath, meta: None) + db_request.find_service = lambda svc, name=None, context=None: { + IFileStorage: storage_service, + IMetricsService: metrics, + }.get(svc) + + record_event = pretend.call_recorder( + lambda self, *, tag, request=None, additional: None + ) + monkeypatch.setattr(HasEvents, "record_event", record_event) + + with pytest.raises(HTTPBadRequest) as excinfo: + legacy.file_upload(db_request) + + resp = excinfo.value + + assert resp.status_code == 400 + assert resp.status.startswith( + "400 Error while decoding the included attestation:" + ) @pytest.mark.parametrize( - "expected_message", + ("verify_exception", "expected_msg"), [ - "Attestations are only supported when using", - "Error while decoding the included attestation", - "Only a single attestation", - "Could not verify the uploaded", - "Unknown error while trying", - "Attestation with unsupported predicate", + ( + VerificationError, + "400 Could not verify the uploaded artifact using the included " + "attestation", + ), + ( + ValueError, + "400 Unknown error while trying to verify included attestations", + ), ], ) - def test_upload_fails_attestation_error( + def test_upload_with_failing_attestation_verification( self, monkeypatch, pyramid_config, db_request, metrics, - expected_message, + verify_exception, + expected_msg, ): from warehouse.events.models import HasEvents @@ -3467,6 +3773,16 @@ def test_upload_fails_attestation_error( db_request.db.add(Classifier(classifier="Programming Language :: Python")) filename = "{}-{}.tar.gz".format(project.name, "1.0") + attestation = Attestation( + version=1, + verification_material=VerificationMaterial( + certificate="somebase64string", transparency_entries=[dict()] + ), + envelope=Envelope( + statement="somebase64string", + signature="somebase64string", + ), + ) pyramid_config.testing_securitypolicy(identity=identity) db_request.user = None @@ -3475,7 +3791,7 @@ def test_upload_fails_attestation_error( { "metadata_version": "1.2", "name": project.name, - "attestations": "", + "attestations": f"[{attestation.model_dump_json()}]", "version": version, "summary": "This is my summary!", "filetype": "sdist", @@ -3489,15 +3805,9 @@ def test_upload_fails_attestation_error( ) storage_service = pretend.stub(store=lambda path, filepath, meta: None) - - def stub_parse(*_args, **_kwargs): - raise AttestationUploadError(expected_message) - - integrity_service = pretend.stub(parse_attestations=stub_parse) db_request.find_service = lambda svc, name=None, context=None: { IFileStorage: storage_service, IMetricsService: metrics, - IIntegrityService: integrity_service, }.get(svc) record_event = pretend.call_recorder( @@ -3505,13 +3815,19 @@ def stub_parse(*_args, **_kwargs): ) monkeypatch.setattr(HasEvents, "record_event", record_event) + def failing_verify(_self, _verifier, _policy, _dist): + raise verify_exception("error") + + monkeypatch.setattr(Attestation, "verify", failing_verify) + monkeypatch.setattr(Verifier, "production", lambda: pretend.stub()) + with pytest.raises(HTTPBadRequest) as excinfo: legacy.file_upload(db_request) resp = excinfo.value assert resp.status_code == 400 - assert resp.status.startswith(f"400 {expected_message}") + assert resp.status.startswith(expected_msg) @pytest.mark.parametrize( ("url", "expected"), diff --git a/tests/unit/packaging/test_utils.py b/tests/unit/packaging/test_utils.py index 8065c8b9ad8c..afa7bd2056fe 100644 --- a/tests/unit/packaging/test_utils.py +++ b/tests/unit/packaging/test_utils.py @@ -15,7 +15,6 @@ import pretend -from warehouse.attestations import IIntegrityService from warehouse.packaging.interfaces import ISimpleStorage from warehouse.packaging.utils import _simple_detail, render_simple_detail @@ -28,37 +27,11 @@ def test_simple_detail_empty_string(db_request): FileFactory.create(release=release) db_request.route_url = lambda *a, **kw: "the-url" - db_request.find_service = lambda svc, name=None, context=None: { - IIntegrityService: pretend.stub( - get_provenance_digest=pretend.call_recorder(lambda f: None), - ), - }.get(svc) - expected_content = _simple_detail(project, db_request) assert expected_content["files"][0]["requires-python"] is None -def test_simple_detail_with_provenance(db_request): - project = ProjectFactory.create() - release = ReleaseFactory.create(project=project, version="1.0") - FileFactory.create(release=release) - - hash_digest = "deadbeefdeadbeefdeadbeefdeadbeef" - - db_request.route_url = lambda *a, **kw: "the-url" - db_request.find_service = pretend.call_recorder( - lambda svc, name=None, context=None: { - IIntegrityService: pretend.stub( - get_provenance_digest=pretend.call_recorder(lambda f: hash_digest), - ), - }.get(svc) - ) - - expected_content = _simple_detail(project, db_request) - assert expected_content["files"][0]["provenance"] == hash_digest - - def test_render_simple_detail(db_request, monkeypatch, jinja): project = ProjectFactory.create() release1 = ReleaseFactory.create(project=project, version="1.0") @@ -76,12 +49,6 @@ def test_render_simple_detail(db_request, monkeypatch, jinja): monkeypatch.setattr(hashlib, "blake2b", fakeblake2b) db_request.route_url = lambda *a, **kw: "the-url" - db_request.find_service = lambda svc, name=None, context=None: { - IIntegrityService: pretend.stub( - get_provenance_digest=pretend.call_recorder(lambda f: None), - ), - }.get(svc) - template = jinja.get_template("templates/api/simple/detail.html") expected_content = template.render( **_simple_detail(project, db_request), request=db_request @@ -111,9 +78,6 @@ def test_render_simple_detail_with_store(db_request, monkeypatch, jinja): db_request.find_service = pretend.call_recorder( lambda svc, name=None, context=None: { ISimpleStorage: storage_service, - IIntegrityService: pretend.stub( - get_provenance_digest=pretend.call_recorder(lambda f: None), - ), }.get(svc) ) diff --git a/warehouse/attestations/__init__.py b/warehouse/attestations/__init__.py deleted file mode 100644 index 76c816d7fef3..000000000000 --- a/warehouse/attestations/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from warehouse.attestations.errors import ( - AttestationUploadError, - UnsupportedPublisherError, -) -from warehouse.attestations.interfaces import IIntegrityService -from warehouse.attestations.models import Attestation -from warehouse.attestations.services import IntegrityService - -__all__ = [ - "Attestation", - "AttestationUploadError", - "IIntegrityService", - "IntegrityService", - "UnsupportedPublisherError", -] diff --git a/warehouse/attestations/errors.py b/warehouse/attestations/errors.py deleted file mode 100644 index 463a34a4da69..000000000000 --- a/warehouse/attestations/errors.py +++ /dev/null @@ -1,19 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -class UnsupportedPublisherError(Exception): - pass - - -class AttestationUploadError(Exception): - pass diff --git a/warehouse/attestations/interfaces.py b/warehouse/attestations/interfaces.py deleted file mode 100644 index 4055e92055a5..000000000000 --- a/warehouse/attestations/interfaces.py +++ /dev/null @@ -1,54 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from pypi_attestations import Attestation, Distribution, Provenance -from pyramid.request import Request -from zope.interface import Interface - - -class IIntegrityService(Interface): - - def create_service(context, request): - """ - Create the service, given the context and request for which it is being - created for. - """ - - def persist_attestations(attestations: list[Attestation], file): - """ - ̀¦Persist attestations in storage. - """ - pass - - def parse_attestations( - request: Request, distribution: Distribution - ) -> list[Attestation]: - """ - Process any attestations included in a file upload request - """ - - def generate_provenance( - oidc_publisher, attestations: list[Attestation] - ) -> Provenance | None: - """ - Generate a Provenance object from an OIDCPublisher and its attestations. - """ - - def persist_provenance(provenance: Provenance, file) -> None: - """ - Persist a Provenance object in storage. - """ - - def get_provenance_digest(file) -> str | None: - """ - Compute a provenance file digest for a `File` if it exists. - """ diff --git a/warehouse/attestations/models.py b/warehouse/attestations/models.py deleted file mode 100644 index 9b95bfb0d7ad..000000000000 --- a/warehouse/attestations/models.py +++ /dev/null @@ -1,55 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from __future__ import annotations - -import typing - -from pathlib import Path -from uuid import UUID - -from sqlalchemy import ForeignKey, orm -from sqlalchemy.dialects.postgresql import CITEXT -from sqlalchemy.ext.hybrid import hybrid_property -from sqlalchemy.orm import Mapped, mapped_column - -from warehouse import db - -if typing.TYPE_CHECKING: - from warehouse.packaging.models import File - - -class Attestation(db.Model): - """ - Table used to store Attestations. - - Attestations are stored on disk. We keep in database only the attestation hash. - """ - - __tablename__ = "attestation" - - file_id: Mapped[UUID] = mapped_column( - ForeignKey("release_files.id", onupdate="CASCADE", ondelete="CASCADE"), - ) - file: Mapped[File] = orm.relationship(back_populates="attestations") - - attestation_file_blake2_digest: Mapped[str] = mapped_column(CITEXT) - - @hybrid_property - def attestation_path(self): - return "/".join( - [ - self.attestation_file_blake2_digest[:2], - self.attestation_file_blake2_digest[2:4], - self.attestation_file_blake2_digest[4:], - f"{Path(self.file.path).name}.attestation", - ] - ) diff --git a/warehouse/attestations/services.py b/warehouse/attestations/services.py deleted file mode 100644 index 056c47c4894e..000000000000 --- a/warehouse/attestations/services.py +++ /dev/null @@ -1,231 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import hashlib -import tempfile -import typing - -from pathlib import Path - -import sentry_sdk - -from pydantic import TypeAdapter, ValidationError -from pypi_attestations import ( - Attestation, - AttestationBundle, - AttestationType, - Distribution, - GitHubPublisher, - GitLabPublisher, - Provenance, - Publisher, - VerificationError, -) -from pyramid.request import Request -from sigstore.verify import Verifier -from zope.interface import implementer - -from warehouse.attestations.errors import ( - AttestationUploadError, - UnsupportedPublisherError, -) -from warehouse.attestations.interfaces import IIntegrityService -from warehouse.attestations.models import Attestation as DatabaseAttestation -from warehouse.metrics.interfaces import IMetricsService -from warehouse.oidc.models import ( - GitHubPublisher as GitHubOIDCPublisher, - GitLabPublisher as GitLabOIDCPublisher, - OIDCPublisher, -) -from warehouse.packaging.interfaces import IFileStorage -from warehouse.packaging.models import File - - -def _publisher_from_oidc_publisher(publisher: OIDCPublisher) -> Publisher: - """ - Convert an OIDCPublisher object in a pypi-attestations Publisher. - """ - match publisher.publisher_name: - case "GitLab": - publisher = typing.cast(GitLabOIDCPublisher, publisher) - return GitLabPublisher( - repository=publisher.project_path, environment=publisher.environment - ) - case "GitHub": - publisher = typing.cast(GitHubOIDCPublisher, publisher) - return GitHubPublisher( - repository=publisher.repository, - workflow=publisher.workflow_filename, - environment=publisher.environment, - ) - case _: - raise UnsupportedPublisherError - - -@implementer(IIntegrityService) -class IntegrityService: - - def __init__( - self, - storage: IFileStorage, - metrics: IMetricsService, - ): - self.storage: IFileStorage = storage - self.metrics: IMetricsService = metrics - - @classmethod - def create_service(cls, _context, request: Request): - return cls( - storage=request.find_service(IFileStorage), - metrics=request.find_service(IMetricsService), - ) - - def persist_attestations(self, attestations: list[Attestation], file: File) -> None: - for attestation in attestations: - with tempfile.NamedTemporaryFile() as tmp_file: - tmp_file.write(attestation.model_dump_json().encode("utf-8")) - - attestation_digest = hashlib.file_digest( - tmp_file, "blake2b" - ).hexdigest() - database_attestation = DatabaseAttestation( - file=file, attestation_file_blake2_digest=attestation_digest - ) - - self.storage.store( - database_attestation.attestation_path, - tmp_file.name, - meta=None, - ) - - file.attestations.append(database_attestation) - - def parse_attestations( - self, request: Request, distribution: Distribution - ) -> list[Attestation]: - """ - Process any attestations included in a file upload request - - Attestations, if present, will be parsed and verified against the uploaded - artifact. Attestations are only allowed when uploading via a Trusted - Publisher, because a Trusted Publisher provides the identity that will be - used to verify the attestations. - Only GitHub Actions Trusted Publishers are supported. - """ - publisher: OIDCPublisher | None = request.oidc_publisher - if not publisher or not publisher.publisher_name == "GitHub": - raise AttestationUploadError( - "Attestations are only supported when using Trusted " - "Publishing with GitHub Actions.", - ) - - try: - attestations = TypeAdapter(list[Attestation]).validate_json( - request.POST["attestations"] - ) - except ValidationError as e: - # Log invalid (malformed) attestation upload - self.metrics.increment("warehouse.upload.attestations.malformed") - raise AttestationUploadError( - f"Error while decoding the included attestation: {e}", - ) - - if len(attestations) > 1: - self.metrics.increment( - "warehouse.upload.attestations.failed_multiple_attestations" - ) - - raise AttestationUploadError( - "Only a single attestation per file is supported.", - ) - - verification_policy = publisher.publisher_verification_policy( - request.oidc_claims - ) - for attestation_model in attestations: - try: - predicate_type, _ = attestation_model.verify( - Verifier.production(), - verification_policy, - distribution, - ) - except VerificationError as e: - # Log invalid (failed verification) attestation upload - self.metrics.increment("warehouse.upload.attestations.failed_verify") - raise AttestationUploadError( - f"Could not verify the uploaded artifact using the included " - f"attestation: {e}", - ) - except Exception as e: - with sentry_sdk.push_scope() as scope: - scope.fingerprint = [e] - sentry_sdk.capture_message( - f"Unexpected error while verifying attestation: {e}" - ) - - raise AttestationUploadError( - f"Unknown error while trying to verify included attestations: {e}", - ) - - if predicate_type != AttestationType.PYPI_PUBLISH_V1: - self.metrics.increment( - "warehouse.upload.attestations.failed_unsupported_predicate_type" - ) - raise AttestationUploadError( - f"Attestation with unsupported predicate type: {predicate_type}", - ) - - return attestations - - def generate_provenance( - self, oidc_publisher: OIDCPublisher, attestations: list[Attestation] - ) -> Provenance | None: - try: - publisher: Publisher = _publisher_from_oidc_publisher(oidc_publisher) - except UnsupportedPublisherError: - sentry_sdk.capture_message( - f"Unsupported OIDCPublisher found {oidc_publisher.publisher_name}" - ) - - return None - - attestation_bundle = AttestationBundle( - publisher=publisher, - attestations=attestations, - ) - - return Provenance(attestation_bundles=[attestation_bundle]) - - def persist_provenance( - self, - provenance: Provenance, - file: File, - ) -> None: - """ - Persist a Provenance object in storage. - """ - provenance_file_path = Path(f"{file.path}.provenance") - with tempfile.NamedTemporaryFile() as f: - f.write(provenance.model_dump_json().encode("utf-8")) - f.flush() - - self.storage.store( - provenance_file_path, - f.name, - ) - - def get_provenance_digest(self, file: File) -> str | None: - """Returns the sha256 digest of the provenance file for the release.""" - if not file.attestations: - return None - - provenance_file = self.storage.get(f"{file.path}.provenance") - return hashlib.file_digest(provenance_file, "sha256").hexdigest() diff --git a/warehouse/forklift/legacy.py b/warehouse/forklift/legacy.py index baae8da64e84..89010bbaaee0 100644 --- a/warehouse/forklift/legacy.py +++ b/warehouse/forklift/legacy.py @@ -30,7 +30,13 @@ import wtforms import wtforms.validators -from pypi_attestations import Attestation, Distribution +from pydantic import TypeAdapter, ValidationError +from pypi_attestations import ( + Attestation, + AttestationType, + Distribution, + VerificationError, +) from pyramid.httpexceptions import ( HTTPBadRequest, HTTPException, @@ -42,11 +48,11 @@ ) from pyramid.request import Request from pyramid.view import view_config +from sigstore.verify import Verifier from sqlalchemy import and_, exists, func, orm from sqlalchemy.exc import MultipleResultsFound, NoResultFound from warehouse.admin.flags import AdminFlagValue -from warehouse.attestations import AttestationUploadError, IIntegrityService from warehouse.authnz import Permissions from warehouse.classifiers.models import Classifier from warehouse.constants import MAX_FILESIZE, MAX_PROJECT_SIZE, ONE_GIB, ONE_MIB @@ -371,6 +377,88 @@ def _is_duplicate_file(db_session, filename, hashes): return None +def _process_attestations(request, distribution: Distribution): + """ + Process any attestations included in a file upload request + + Attestations, if present, will be parsed and verified against the uploaded + artifact. Attestations are only allowed when uploading via a Trusted + Publisher, because a Trusted Publisher provides the identity that will be + used to verify the attestations. + Currently, only GitHub Actions Trusted Publishers are supported, and + attestations are discarded after verification. + """ + + metrics = request.find_service(IMetricsService, context=None) + + publisher = request.oidc_publisher + if not publisher or not publisher.publisher_name == "GitHub": + raise _exc_with_message( + HTTPBadRequest, + "Attestations are currently only supported when using Trusted " + "Publishing with GitHub Actions.", + ) + try: + attestations = TypeAdapter(list[Attestation]).validate_json( + request.POST["attestations"] + ) + except ValidationError as e: + # Log invalid (malformed) attestation upload + metrics.increment("warehouse.upload.attestations.malformed") + raise _exc_with_message( + HTTPBadRequest, + f"Error while decoding the included attestation: {e}", + ) + + if len(attestations) > 1: + metrics.increment("warehouse.upload.attestations.failed_multiple_attestations") + raise _exc_with_message( + HTTPBadRequest, + "Only a single attestation per-file is supported at the moment.", + ) + + verification_policy = publisher.publisher_verification_policy(request.oidc_claims) + for attestation_model in attestations: + try: + # For now, attestations are not stored, just verified + predicate_type, _ = attestation_model.verify( + Verifier.production(), + verification_policy, + distribution, + ) + except VerificationError as e: + # Log invalid (failed verification) attestation upload + metrics.increment("warehouse.upload.attestations.failed_verify") + raise _exc_with_message( + HTTPBadRequest, + f"Could not verify the uploaded artifact using the included " + f"attestation: {e}", + ) + except Exception as e: + with sentry_sdk.push_scope() as scope: + scope.fingerprint = [e] + sentry_sdk.capture_message( + f"Unexpected error while verifying attestation: {e}" + ) + + raise _exc_with_message( + HTTPBadRequest, + f"Unknown error while trying to verify included attestations: {e}", + ) + + if predicate_type != AttestationType.PYPI_PUBLISH_V1: + metrics.increment( + "warehouse.upload.attestations.failed_unsupported_predicate_type" + ) + raise _exc_with_message( + HTTPBadRequest, + f"Attestation with unsupported predicate type: {predicate_type}", + ) + + # Log successful attestation upload + metrics.increment("warehouse.upload.attestations.ok") + + _pypi_project_urls = [ "https://pypi.org/project/", "https://pypi.org/p/", @@ -1181,6 +1269,12 @@ def file_upload(request): k: h.hexdigest().lower() for k, h in metadata_file_hashes.items() } + if "attestations" in request.POST: + _process_attestations( + request=request, + distribution=Distribution(name=filename, digest=file_hashes["sha256"]), + ) + # TODO: This should be handled by some sort of database trigger or a # SQLAlchemy hook or the like instead of doing it inline in this # view. @@ -1277,32 +1371,6 @@ def file_upload(request): }, ) - # If the user provided attestations, verify and store them - if "attestations" in request.POST: - integrity_service = request.find_service(IIntegrityService, context=None) - - try: - attestations: list[Attestation] = integrity_service.parse_attestations( - request, - Distribution(name=filename, digest=file_hashes["sha256"]), - ) - except AttestationUploadError as e: - raise _exc_with_message( - HTTPBadRequest, - str(e), - ) - - integrity_service.persist_attestations(attestations, file_) - - provenance = integrity_service.generate_provenance( - request.oidc_publisher, attestations - ) - if provenance: - integrity_service.persist_provenance(provenance, file_) - - # Log successful attestation upload - metrics.increment("warehouse.upload.attestations.ok") - # For existing releases, we check if any of the existing project URLs are unverified # and have been verified in the current upload. In that case, we mark them as # verified. diff --git a/warehouse/migrations/versions/7f0c9f105f44_create_attestations_table.py b/warehouse/migrations/versions/7f0c9f105f44_create_attestations_table.py deleted file mode 100644 index 2b15277127f6..000000000000 --- a/warehouse/migrations/versions/7f0c9f105f44_create_attestations_table.py +++ /dev/null @@ -1,47 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -create Attestations table - -Revision ID: 7f0c9f105f44 -Revises: 26455e3712a2 -Create Date: 2024-07-25 15:49:01.993869 -""" - -import sqlalchemy as sa - -from alembic import op -from sqlalchemy.dialects import postgresql - -revision = "7f0c9f105f44" -down_revision = "26455e3712a2" - - -def upgrade(): - op.create_table( - "attestation", - sa.Column("file_id", sa.UUID(), nullable=False), - sa.Column( - "attestation_file_blake2_digest", postgresql.CITEXT(), nullable=False - ), - sa.Column( - "id", sa.UUID(), server_default=sa.text("gen_random_uuid()"), nullable=False - ), - sa.ForeignKeyConstraint( - ["file_id"], ["release_files.id"], onupdate="CASCADE", ondelete="CASCADE" - ), - sa.PrimaryKeyConstraint("id"), - ) - - -def downgrade(): - op.drop_table("attestation") diff --git a/warehouse/packaging/models.py b/warehouse/packaging/models.py index 615739f3a1b8..344689b36d0b 100644 --- a/warehouse/packaging/models.py +++ b/warehouse/packaging/models.py @@ -81,7 +81,6 @@ from warehouse.utils.db.types import bool_false, datetime_now if typing.TYPE_CHECKING: - from warehouse.attestations.models import Attestation from warehouse.oidc.models import OIDCPublisher _MONOTONIC_SEQUENCE = 42 @@ -853,13 +852,6 @@ def __table_args__(cls): # noqa comment="If True, the metadata for the file cannot be backfilled.", ) - # PEP 740 attestations - attestations: Mapped[list[Attestation]] = orm.relationship( - cascade="all, delete-orphan", - lazy="joined", - passive_deletes=True, - ) - @property def uploaded_via_trusted_publisher(self) -> bool: """Return True if the file was uploaded via a trusted publisher.""" diff --git a/warehouse/packaging/utils.py b/warehouse/packaging/utils.py index 5fc67bc29761..30c85b3feb50 100644 --- a/warehouse/packaging/utils.py +++ b/warehouse/packaging/utils.py @@ -19,12 +19,10 @@ from pyramid_jinja2 import IJinja2Environment from sqlalchemy.orm import joinedload -from warehouse.attestations import IIntegrityService -from warehouse.attestations.models import Attestation from warehouse.packaging.interfaces import ISimpleStorage from warehouse.packaging.models import File, LifecycleStatus, Project, Release -API_VERSION = "1.2" +API_VERSION = "1.1" def _simple_index(request, serial): @@ -53,7 +51,6 @@ def _simple_detail(project, request): request.db.query(File) .options(joinedload(File.release)) .join(Release) - .join(Attestation) .filter(Release.project == project) # Exclude projects that are in the `quarantine-enter` lifecycle status. .join(Project) @@ -67,8 +64,6 @@ def _simple_detail(project, request): {f.release.version for f in files}, key=packaging_legacy.version.parse ) - integrity_service = request.find_service(IIntegrityService, context=None) - return { "meta": {"api-version": API_VERSION, "_last-serial": project.last_serial}, "name": project.normalized_name, @@ -102,7 +97,6 @@ def _simple_detail(project, request): if file.metadata_file_sha256_digest else False ), - "provenance": integrity_service.get_provenance_digest(file), } for file in files ], diff --git a/warehouse/templates/api/simple/detail.html b/warehouse/templates/api/simple/detail.html index 05e0221a5612..24b0042c5863 100644 --- a/warehouse/templates/api/simple/detail.html +++ b/warehouse/templates/api/simple/detail.html @@ -20,7 +20,7 @@

Links for {{ name }}

{% for file in files -%} - {{ file.filename }}
+ {{ file.filename }}
{% endfor -%}