From ddd32a9cdbdb4e64cd4a96e7356d9a440c622589 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 31 Oct 2024 11:25:50 +0000 Subject: [PATCH 1/7] fix: publish location claim to content claims service --- packages/upload-api/src/blob/accept.js | 26 ++++++++++ packages/upload-api/src/types.ts | 4 +- packages/upload-api/test/handlers/blob.js | 62 +++++++++++++++++++++++ 3 files changed, 90 insertions(+), 2 deletions(-) diff --git a/packages/upload-api/src/blob/accept.js b/packages/upload-api/src/blob/accept.js index 94bdd3f5f..12e1b2be5 100644 --- a/packages/upload-api/src/blob/accept.js +++ b/packages/upload-api/src/blob/accept.js @@ -54,6 +54,12 @@ export function blobAcceptProvider(context) { expiration: Infinity, }) + // Publish this claim to the content claims service + const pubClaim = await publishLocationClaim(context, { digest, location: createUrl.ok }) + if (pubClaim.error) { + return pubClaim + } + // Create result object /** @type {API.OkBuilder} */ const result = Server.ok({ @@ -137,3 +143,23 @@ export const poll = async (context, receipt) => { return { ok: {} } } + +/** + * @param {API.ClaimsClientContext} ctx + * @param {{ digest: API.MultihashDigest, location: API.URI }} params + */ +const publishLocationClaim = async (ctx, { digest, location }) => { + const { invocationConfig, connection } = ctx.claimsService + const { issuer, audience, with: resource, proofs } = invocationConfig + const res = await Assert.location + .invoke({ + issuer, + audience, + with: resource, + nb: { content: { digest: digest.bytes }, location: [location] }, + expiration: Infinity, + proofs, + }) + .execute(connection) + return res.out +} diff --git a/packages/upload-api/src/types.ts b/packages/upload-api/src/types.ts index c56d9a893..9e3fecb52 100644 --- a/packages/upload-api/src/types.ts +++ b/packages/upload-api/src/types.ts @@ -203,7 +203,7 @@ import { StorageGetError } from './types/storage.js' import { AllocationsStorage, BlobsStorage, BlobAddInput } from './types/blob.js' export type { AllocationsStorage, BlobsStorage, BlobAddInput } import { IPNIService, IndexServiceContext } from './types/index.js' -import { ClaimsClientConfig } from './types/content-claims.js' +import { ClaimsClientConfig, ClaimsClientContext } from './types/content-claims.js' import { Claim } from '@web3-storage/content-claims/client/api' export type { IndexServiceContext, @@ -378,7 +378,7 @@ export type BlobServiceContext = SpaceServiceContext & { getServiceConnection: () => ConnectionView } -export type W3ServiceContext = SpaceServiceContext & { +export type W3ServiceContext = SpaceServiceContext & ClaimsClientContext & { /** * Service signer */ diff --git a/packages/upload-api/test/handlers/blob.js b/packages/upload-api/test/handlers/blob.js index ea7f18e61..12576521f 100644 --- a/packages/upload-api/test/handlers/blob.js +++ b/packages/upload-api/test/handlers/blob.js @@ -9,6 +9,7 @@ import { alice, registerSpace } from '../util.js' import { BlobSizeOutsideOfSupportedRangeName } from '../../src/blob/lib.js' import { createConcludeInvocation } from '../../src/ucan/conclude.js' import { parseBlobAddReceiptNext } from '../helpers/blob.js' +import * as Result from '../helpers/result.js' /** * @type {API.Tests} @@ -429,6 +430,67 @@ export const test = { 'accept was not successful' ) }, + 'blob/accept publishes location claim to claims service': async ( + assert, + context + ) => { + const { proof, spaceDid } = await registerSpace(alice, context) + + const data = new Uint8Array([11, 22, 34, 44, 55]) + const digest = await sha256.digest(data) + const size = data.byteLength + + const service = createServer(context) + const connection = connect({ id: context.id, channel: service }) + + const blobAddInvocation = BlobCapabilities.add.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { blob: { digest: digest.bytes, size } }, + proofs: [proof], + }) + const receipt = await blobAddInvocation.execute(connection) + assert.ok(receipt.out.ok) + + const nextTasks = parseBlobAddReceiptNext(receipt) + const { address } = Result.unwrap(nextTasks.allocate.receipt.out) + assert.ok(address) + + if (address) { + const httpPut = await fetch(address.url, { + method: 'PUT', + mode: 'cors', + body: data, + headers: address.headers, + }) + assert.equal(httpPut.status, 200, await httpPut.text()) + } + + const keys = + /** @type {API.SignerArchive} */ + (nextTasks.put.task.facts[0]['keys']) + const blobProvider = ed25519.from(keys) + const httpPutReceipt = await Receipt.issue({ + issuer: blobProvider, + ran: nextTasks.put.task.link(), + result: { ok: {} }, + }) + const httpPutConcludeInvocation = createConcludeInvocation( + alice, + context.id, + httpPutReceipt + ) + const ucanConclude = await httpPutConcludeInvocation.execute(connection) + assert.ok(ucanConclude.out.ok) + + // ensure a location claim exists for the content root + const claims = Result.unwrap(await context.claimsService.read(digest)) + assert.ok( + claims.some(c => c.type === 'assert/location'), + 'did not find location claim' + ) + }, 'blob/add fails when a blob with size bigger than maximum size is added': async (assert, context) => { const { proof, spaceDid } = await registerSpace(alice, context) From 496bf6a030914aa4bb410e3ae505c7a1318a3479 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Fri, 1 Nov 2024 15:07:25 +0000 Subject: [PATCH 2/7] fix: apss space to location claim --- packages/upload-api/src/blob/accept.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/upload-api/src/blob/accept.js b/packages/upload-api/src/blob/accept.js index 12e1b2be5..3aa155e4c 100644 --- a/packages/upload-api/src/blob/accept.js +++ b/packages/upload-api/src/blob/accept.js @@ -55,7 +55,7 @@ export function blobAcceptProvider(context) { }) // Publish this claim to the content claims service - const pubClaim = await publishLocationClaim(context, { digest, location: createUrl.ok }) + const pubClaim = await publishLocationClaim(context, { space, digest, location: createUrl.ok }) if (pubClaim.error) { return pubClaim } @@ -146,7 +146,7 @@ export const poll = async (context, receipt) => { /** * @param {API.ClaimsClientContext} ctx - * @param {{ digest: API.MultihashDigest, location: API.URI }} params + * @param {{ space: API.SpaceDID, digest: API.MultihashDigest, location: API.URI }} params */ const publishLocationClaim = async (ctx, { digest, location }) => { const { invocationConfig, connection } = ctx.claimsService From 7d6f42319682db83033f83b8acdcf78f512b0876 Mon Sep 17 00:00:00 2001 From: ash Date: Thu, 28 Nov 2024 17:58:06 +0000 Subject: [PATCH 3/7] fix: update assertion message Co-authored-by: Petra Jaros --- packages/upload-api/test/handlers/blob.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/upload-api/test/handlers/blob.js b/packages/upload-api/test/handlers/blob.js index 12576521f..e788a751e 100644 --- a/packages/upload-api/test/handlers/blob.js +++ b/packages/upload-api/test/handlers/blob.js @@ -464,7 +464,7 @@ export const test = { body: data, headers: address.headers, }) - assert.equal(httpPut.status, 200, await httpPut.text()) + assert.equal(httpPut.status, 200, `PUT ${address.url} failed (${httpPut.status}): ${await httpPut.text()}`) } const keys = From 7d39ec53879017df2018802aae3895c0dd88cdf8 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 28 Nov 2024 18:03:05 +0000 Subject: [PATCH 4/7] fix: types --- packages/upload-api/src/types.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/upload-api/src/types.ts b/packages/upload-api/src/types.ts index 9e3fecb52..5f45f56f0 100644 --- a/packages/upload-api/src/types.ts +++ b/packages/upload-api/src/types.ts @@ -366,7 +366,7 @@ export interface W3sService { } } -export type BlobServiceContext = SpaceServiceContext & { +export type BlobServiceContext = SpaceServiceContext & ClaimsClientContext & { /** * Service signer */ From 56c9147dbfb8770e463eca2bbf798f7d8d1c3e93 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 28 Nov 2024 18:08:42 +0000 Subject: [PATCH 5/7] fix: include range in location claim --- packages/upload-api/src/blob/accept.js | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/upload-api/src/blob/accept.js b/packages/upload-api/src/blob/accept.js index 3aa155e4c..d5e36aebd 100644 --- a/packages/upload-api/src/blob/accept.js +++ b/packages/upload-api/src/blob/accept.js @@ -55,7 +55,7 @@ export function blobAcceptProvider(context) { }) // Publish this claim to the content claims service - const pubClaim = await publishLocationClaim(context, { space, digest, location: createUrl.ok }) + const pubClaim = await publishLocationClaim(context, { space, digest, size: blob.size, location: createUrl.ok }) if (pubClaim.error) { return pubClaim } @@ -146,9 +146,9 @@ export const poll = async (context, receipt) => { /** * @param {API.ClaimsClientContext} ctx - * @param {{ space: API.SpaceDID, digest: API.MultihashDigest, location: API.URI }} params + * @param {{ space: API.SpaceDID, digest: API.MultihashDigest, size: number, location: API.URI }} params */ -const publishLocationClaim = async (ctx, { digest, location }) => { +const publishLocationClaim = async (ctx, { digest, size, location }) => { const { invocationConfig, connection } = ctx.claimsService const { issuer, audience, with: resource, proofs } = invocationConfig const res = await Assert.location @@ -156,7 +156,11 @@ const publishLocationClaim = async (ctx, { digest, location }) => { issuer, audience, with: resource, - nb: { content: { digest: digest.bytes }, location: [location] }, + nb: { + content: { digest: digest.bytes }, + location: [location], + range: { offset: 0, length: size } + }, expiration: Infinity, proofs, }) From fdc96a674e4e6b837ac89461c3f2d03784768fc5 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 28 Nov 2024 18:15:00 +0000 Subject: [PATCH 6/7] fix: update dependencies --- packages/upload-api/package.json | 2 +- pnpm-lock.yaml | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/upload-api/package.json b/packages/upload-api/package.json index bcf57f99a..f4acf5f7f 100644 --- a/packages/upload-api/package.json +++ b/packages/upload-api/package.json @@ -202,7 +202,7 @@ "@web3-storage/access": "workspace:^", "@web3-storage/blob-index": "workspace:^", "@web3-storage/capabilities": "workspace:^", - "@web3-storage/content-claims": "^5.1.0", + "@web3-storage/content-claims": "^5.1.3", "@web3-storage/did-mailto": "workspace:^", "@web3-storage/filecoin-api": "workspace:^", "multiformats": "^12.1.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0b4f7bef5..e082fdaa8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -450,8 +450,8 @@ importers: specifier: workspace:^ version: link:../capabilities '@web3-storage/content-claims': - specifier: ^5.1.0 - version: 5.1.0 + specifier: ^5.1.3 + version: 5.1.3 '@web3-storage/did-mailto': specifier: workspace:^ version: link:../did-mailto @@ -2521,8 +2521,8 @@ packages: '@web3-storage/content-claims@5.0.0': resolution: {integrity: sha512-HJFRFsR0qHCe0cOERsb3AjAxxzohYMMoIWaGJgrShDycnl6yqXHrGcdua1BWUDu5pmvKzwD9D7VmI8aSfrCcRA==} - '@web3-storage/content-claims@5.1.0': - resolution: {integrity: sha512-3VStFKoeieRpRU7brFjKTsAuAffQzYDIZ8F3Gh0+niw+MgzBK72osW+fftdquT8neWir34Ndu3mBUKKJ3ck1RQ==} + '@web3-storage/content-claims@5.1.3': + resolution: {integrity: sha512-X+Cpm+EmGuEvFyM8oX1NqsBkuSje836B72yuvnVmgd80XPt+McpOhM6ko7rfs9Dx9UmpiZq+998jlvBhg2W5ZA==} '@web3-storage/data-segment@4.0.0': resolution: {integrity: sha512-AnNyJp3wHMa7LBzguQzm4rmXSi8vQBz4uFs+jiXnSNtLR5dAqHfhMvi9XdWonWPYvxNvT5ZhYCSF0mpDjymqKg==} @@ -9759,14 +9759,14 @@ snapshots: carstream: 2.1.0 multiformats: 13.1.0 - '@web3-storage/content-claims@5.1.0': + '@web3-storage/content-claims@5.1.3': dependencies: '@ucanto/client': 9.0.1 '@ucanto/interface': 10.0.1 '@ucanto/server': 10.0.0 '@ucanto/transport': 9.1.1 carstream: 2.1.0 - multiformats: 13.1.0 + multiformats: 13.3.0 '@web3-storage/data-segment@4.0.0': dependencies: From 9faebb8b099f05a2c13a650a9e517095024b5c2a Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 28 Nov 2024 18:28:43 +0000 Subject: [PATCH 7/7] fix: tests --- packages/w3up-client/test/capability/space.test.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/w3up-client/test/capability/space.test.js b/packages/w3up-client/test/capability/space.test.js index b288b8afc..d25f447db 100644 --- a/packages/w3up-client/test/capability/space.test.js +++ b/packages/w3up-client/test/capability/space.test.js @@ -142,7 +142,7 @@ export const SpaceClient = Test.withContext({ assert.equal(egressRecord.bytes, car.size, 'bytes should be the same') assert.equal( new Date(egressRecord.servedAt).getTime(), - Math.floor(new Date(egressData.servedAt).getTime() / 1000) * 1000, + Math.floor(new Date(egressData.servedAt).getTime() / 1000), 'servedAt should be the same' ) assert.ok(egressRecord.cause.toString(), 'cause should be a link') @@ -252,7 +252,7 @@ export const SpaceClient = Test.withContext({ assert.equal(egressRecord.bytes, car.size, 'bytes should be the same') assert.equal( new Date(egressRecord.servedAt).getTime(), - Math.floor(new Date(egressData.servedAt).getTime() / 1000) * 1000, + Math.floor(new Date(egressData.servedAt).getTime() / 1000), 'servedAt should be the same' ) assert.ok(egressRecord.cause.toString(), 'cause should be a link') @@ -364,7 +364,7 @@ export const SpaceClient = Test.withContext({ assert.equal(egressRecord.bytes, car.size, 'bytes should be the same') assert.equal( new Date(egressRecord.servedAt).getTime(), - Math.floor(new Date(egressData.servedAt).getTime() / 1000) * 1000, + Math.floor(new Date(egressData.servedAt).getTime() / 1000), 'servedAt should be the same' ) assert.ok(egressRecord.cause.toString(), 'cause should be a link') @@ -476,7 +476,7 @@ export const SpaceClient = Test.withContext({ assert.equal(egressRecord.bytes, car.size, 'bytes should be the same') assert.equal( new Date(egressRecord.servedAt).getTime(), - Math.floor(new Date(egressData.servedAt).getTime() / 1000) * 1000, + Math.floor(new Date(egressData.servedAt).getTime() / 1000), 'servedAt should be the same' ) assert.ok(egressRecord.cause.toString(), 'cause should be a link')