From bb5235ff841430036152989442770c298f6f9d63 Mon Sep 17 00:00:00 2001 From: DAG House <111574116+it-dag-house@users.noreply.github.com> Date: Wed, 13 Dec 2023 03:14:41 +0300 Subject: [PATCH 1/3] chore(main): release access 18.0.7 (#1237) :robot: I have created a release *beep* *boop* --- ## [18.0.7](https://github.com/web3-storage/w3up/compare/access-v18.0.6...access-v18.0.7) (2023-12-13) ### Fixes * support storing ArrayBuffers in conf ([#1236](https://github.com/web3-storage/w3up/issues/1236)) ([9b1aafb](https://github.com/web3-storage/w3up/commit/9b1aafbcf241d268e4f365ed99005458dda1a05a)) --- This PR was generated with [Release Please](https://github.com/googleapis/release-please). See [documentation](https://github.com/googleapis/release-please#release-please). --- .github/release-please-manifest.json | 2 +- packages/access-client/CHANGELOG.md | 7 +++++++ packages/access-client/package.json | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/release-please-manifest.json b/.github/release-please-manifest.json index 79b28a0bf..bc85069c5 100644 --- a/.github/release-please-manifest.json +++ b/.github/release-please-manifest.json @@ -1,5 +1,5 @@ { - "packages/access-client": "18.0.6", + "packages/access-client": "18.0.7", "packages/filecoin-api": "4.3.0", "packages/filecoin-client": "3.2.0", "packages/capabilities": "13.0.0", diff --git a/packages/access-client/CHANGELOG.md b/packages/access-client/CHANGELOG.md index 2d618d142..8786b3c34 100644 --- a/packages/access-client/CHANGELOG.md +++ b/packages/access-client/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [18.0.7](https://github.com/web3-storage/w3up/compare/access-v18.0.6...access-v18.0.7) (2023-12-13) + + +### Fixes + +* support storing ArrayBuffers in conf ([#1236](https://github.com/web3-storage/w3up/issues/1236)) ([9b1aafb](https://github.com/web3-storage/w3up/commit/9b1aafbcf241d268e4f365ed99005458dda1a05a)) + ## [18.0.6](https://github.com/web3-storage/w3up/compare/access-v18.0.5...access-v18.0.6) (2023-12-07) diff --git a/packages/access-client/package.json b/packages/access-client/package.json index 951563cca..bcacbcb4c 100644 --- a/packages/access-client/package.json +++ b/packages/access-client/package.json @@ -1,6 +1,6 @@ { "name": "@web3-storage/access", - "version": "18.0.6", + "version": "18.0.7", "description": "w3access client", "homepage": "https://web3.storage", "repository": { From 468bb79cfd6cbc7d513d0174da5b3b43a3f82cba Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Wed, 13 Dec 2023 10:09:05 -0800 Subject: [PATCH 2/3] fix: copy src into dist in w3up-client (#1239) I ran into an issue in a Next.js app that seems to be a result of having sourcecode and type declarations in separate directories: ``` ./node_modules/.pnpm/@web3-storage+w3up-client@11.2.1/node_modules/@web3-storage/w3up-client/src/types.ts:15:29 Type error: Could not find a declaration file for module './client.js'. '/Users/travis/dev/pl/fetch-test/node_modules/.pnpm/@web3-storage+w3up-client@11.2.1/node_modules/@web3-storage/w3up-client/src/client.js' implicitly has an 'any' type. 13 | Unit, 14 | } from '@ucanto/interface' > 15 | import { type Client } from './client.js' | ^ ``` Copying the sourcecode into `dist` and updating exports to point to it seems to fix this. --- packages/w3up-client/package.json | 29 +++++++++++++------------- packages/w3up-client/src/delegation.js | 2 ++ packages/w3up-client/tsconfig.json | 1 - 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/packages/w3up-client/package.json b/packages/w3up-client/package.json index 7c1a1f3bf..b1b4285c3 100644 --- a/packages/w3up-client/package.json +++ b/packages/w3up-client/package.json @@ -4,7 +4,7 @@ "description": "Client for the web3.storage w3up api", "license": "Apache-2.0 OR MIT", "type": "module", - "main": "src/index.js", + "main": "dist/src/index.js", "types": "dist/src/index.d.ts", "typesVersions": { "*": { @@ -25,56 +25,55 @@ "exports": { ".": { "types": "./dist/src/index.d.ts", - "node": "./src/index.node.js", - "import": "./src/index.js" + "node": "./dist/src/index.node.js", + "import": "./dist/src/index.js" }, "./account": { "types": "./dist/src/account.d.ts", - "import": "./src/account.js" + "import": "./dist/src/account.js" }, "./space": { "types": "./dist/src/space.d.ts", - "import": "./src/space.js" + "import": "./dist/src/space.js" }, "./result": { "types": "./dist/src/result.d.ts", - "import": "./src/result.js" + "import": "./dist/src/result.js" }, "./client": { "types": "./dist/src/client.d.ts", - "import": "./src/client.js" + "import": "./dist/src/client.js" }, "./capability/access": { "types": "./dist/src/capability/access.d.ts", - "import": "./src/capability/access.js" + "import": "./dist/src/capability/access.js" }, "./capability/space": { "types": "./dist/src/capability/space.d.ts", - "import": "./src/capability/space.js" + "import": "./dist/src/capability/space.js" }, "./capability/store": { "types": "./dist/src/capability/store.d.ts", - "import": "./src/capability/store.js" + "import": "./dist/src/capability/store.js" }, "./capability/subscription": { "types": "./dist/src/capability/subscription.d.ts", - "import": "./src/capability/subscription.js" + "import": "./dist/src/capability/subscription.js" }, "./capability/upload": { "types": "./dist/src/capability/upload.d.ts", - "import": "./src/capability/upload.js" + "import": "./dist/src/capability/upload.js" }, "./capability/usage": { "types": "./dist/src/capability/usage.d.ts", - "import": "./src/capability/usage.js" + "import": "./dist/src/capability/usage.js" }, - "./types": "./src/types.js" + "./types": "./dist/src/types.js" }, "publishConfig": { "access": "public" }, "files": [ - "src", "dist" ], "scripts": { diff --git a/packages/w3up-client/src/delegation.js b/packages/w3up-client/src/delegation.js index 358df2e66..3431cc90d 100644 --- a/packages/w3up-client/src/delegation.js +++ b/packages/w3up-client/src/delegation.js @@ -1,10 +1,12 @@ import { Delegation as CoreDelegation } from '@ucanto/core/delegation' +/* c8 ignore start */ /** * @template {import('./types.js').Capabilities} C * @extends {CoreDelegation} */ export class Delegation extends CoreDelegation { + /* c8 ignore stop */ /** @type {Record} */ #meta diff --git a/packages/w3up-client/tsconfig.json b/packages/w3up-client/tsconfig.json index 511812c4e..a4a804f40 100644 --- a/packages/w3up-client/tsconfig.json +++ b/packages/w3up-client/tsconfig.json @@ -2,7 +2,6 @@ "extends": "../../tsconfig.json", "compilerOptions": { "outDir": "dist", - "emitDeclarationOnly": true, "composite": true, "noUnusedParameters": false }, From 86aedbcb070ae67093f30b2b724766dee4c43f2c Mon Sep 17 00:00:00 2001 From: Benjamin Goering <171782+gobengo@users.noreply.github.com> Date: Wed, 13 Dec 2023 11:23:46 -0800 Subject: [PATCH 3/3] feat: dont use ipfs-utils fetch with upload progress by default (#1240) because importing it in bundled setups breaks, e.g. * https://github.com/web3-storage/w3up/issues/1183 * https://github.com/web3-storage/w3up/issues/828 I tested this change locally and confirmed that once it's made, upload-client can be relied on by a nextjs project and build successfully --- packages/upload-client/package.json | 1 + .../src/fetch-with-upload-progress.js | 8 +++++ packages/upload-client/src/store.js | 27 ++++++++++----- packages/upload-client/src/types.ts | 22 ++++++++++-- packages/upload-client/test/store.test.js | 34 +++++++++++++++++-- 5 files changed, 78 insertions(+), 14 deletions(-) create mode 100644 packages/upload-client/src/fetch-with-upload-progress.js diff --git a/packages/upload-client/package.json b/packages/upload-client/package.json index 21711085a..810fc2eed 100644 --- a/packages/upload-client/package.json +++ b/packages/upload-client/package.json @@ -31,6 +31,7 @@ "exports": { ".": "./dist/src/index.js", "./car": "./dist/src/car.js", + "./fetch-with-upload-progress": "./dist/src/fetch-with-upload-progress.js", "./sharding": "./dist/src/sharding.js", "./upload": "./dist/src/upload.js", "./store": "./dist/src/store.js", diff --git a/packages/upload-client/src/fetch-with-upload-progress.js b/packages/upload-client/src/fetch-with-upload-progress.js new file mode 100644 index 000000000..f33087c8f --- /dev/null +++ b/packages/upload-client/src/fetch-with-upload-progress.js @@ -0,0 +1,8 @@ +import ipfsUtilsFetch from 'ipfs-utils/src/http/fetch.js' + +/** + * @type {import('./types.js').FetchWithUploadProgress} + */ +export const fetchWithUploadProgress = (url, init) => { + return ipfsUtilsFetch.fetch(url, init) +} diff --git a/packages/upload-client/src/store.js b/packages/upload-client/src/store.js index d92b906a6..764c76238 100644 --- a/packages/upload-client/src/store.js +++ b/packages/upload-client/src/store.js @@ -4,8 +4,6 @@ import { SpaceDID } from '@web3-storage/capabilities/utils' import retry, { AbortError } from 'p-retry' import { servicePrincipal, connection } from './service.js' import { REQUEST_RETRIES } from './constants.js' -import fetchPkg from 'ipfs-utils/src/http/fetch.js' -const { fetch } = fetchPkg /** * @@ -90,10 +88,9 @@ export async function add( const responseAddUpload = result.out.ok const fetchWithUploadProgress = - /** @type {(url: string, init?: import('./types.js').FetchOptions) => Promise} */ ( - fetch - ) + options.fetchWithUploadProgress || options.fetch || globalThis.fetch + let fetchDidCallUploadProgressCb = false const res = await retry( async () => { try { @@ -103,12 +100,14 @@ export async function add( body: car, headers: responseAddUpload.headers, signal: options.signal, - onUploadProgress: options.onUploadProgress - ? createUploadProgressHandler( + onUploadProgress: (status) => { + fetchDidCallUploadProgressCb = true + if (options.onUploadProgress) + createUploadProgressHandler( responseAddUpload.url, options.onUploadProgress - ) - : undefined, + )(status) + }, // @ts-expect-error - this is needed by recent versions of node - see https://github.com/bluesky-social/atproto/pull/470 for more info duplex: 'half', }) @@ -128,6 +127,16 @@ export async function add( } ) + if (!fetchDidCallUploadProgressCb && options.onUploadProgress) { + // the fetch implementation didn't support onUploadProgress + const carBlob = new Blob([car]) + options.onUploadProgress({ + total: carBlob.size, + loaded: carBlob.size, + lengthComputable: false, + }) + } + if (!res.ok) { throw new Error(`upload failed: ${res.status}`) } diff --git a/packages/upload-client/src/types.ts b/packages/upload-client/src/types.ts index 65886f9c6..cc066729e 100644 --- a/packages/upload-client/src/types.ts +++ b/packages/upload-client/src/types.ts @@ -1,5 +1,5 @@ import type { - FetchOptions, + FetchOptions as IpfsUtilsFetchOptions, ProgressStatus as XHRProgressStatus, } from 'ipfs-utils/src/types.js' import { Link, UnknownLink, Version } from 'multiformats/link' @@ -45,6 +45,16 @@ import { UsageReportFailure, } from '@web3-storage/capabilities/types' +type Override = Omit & R + +type FetchOptions = Override< + IpfsUtilsFetchOptions, + { + // `fetch` is a browser API and browsers don't have `Readable` + body: Exclude + } +> + export type { FetchOptions, StoreAdd, @@ -186,7 +196,13 @@ export interface Connectable { connection?: ConnectionView } +export type FetchWithUploadProgress = ( + url: string, + init?: FetchOptions +) => Promise + export interface UploadProgressTrackable { + fetchWithUploadProgress?: FetchWithUploadProgress onUploadProgress?: ProgressFn } @@ -210,7 +226,9 @@ export interface RequestOptions extends Retryable, Abortable, Connectable, - UploadProgressTrackable {} + UploadProgressTrackable { + fetch?: typeof globalThis.fetch +} export interface ListRequestOptions extends RequestOptions, Pageable {} diff --git a/packages/upload-client/test/store.test.js b/packages/upload-client/test/store.test.js index 02deae8a5..90574ef8b 100644 --- a/packages/upload-client/test/store.test.js +++ b/packages/upload-client/test/store.test.js @@ -10,6 +10,7 @@ import { serviceSigner } from './fixtures.js' import { randomCAR } from './helpers/random.js' import { mockService } from './helpers/mocks.js' import { validateAuthorization } from './helpers/utils.js' +import { fetchWithUploadProgress } from '../src/fetch-with-upload-progress.js' describe('Store.add', () => { it('stores a DAG with the service', async () => { @@ -62,7 +63,8 @@ describe('Store.add', () => { channel: server, }) - let loaded = 0 + /** @type {import('../src/types.js').ProgressStatus[]} */ + const progress = [] const carCID = await Store.add( { issuer: agent, with: space.did(), proofs, audience: serviceSigner }, car, @@ -70,17 +72,43 @@ describe('Store.add', () => { connection, onUploadProgress: (status) => { assert(typeof status.loaded === 'number' && status.loaded > 0) - loaded = status.loaded + progress.push(status) }, + fetchWithUploadProgress, } ) assert(service.store.add.called) assert.equal(service.store.add.callCount, 1) - assert.equal(loaded, 225) + assert.equal( + progress.reduce((max, { loaded }) => Math.max(max, loaded), 0), + 225 + ) assert(carCID) assert.equal(carCID.toString(), car.cid.toString()) + + // make sure it can also work without fetchWithUploadProgress + /** @type {import('../src/types.js').ProgressStatus[]} */ + let progressWithoutUploadProgress = [] + const addedWithoutUploadProgress = await Store.add( + { issuer: agent, with: space.did(), proofs, audience: serviceSigner }, + car, + { + connection, + onUploadProgress: (status) => { + progressWithoutUploadProgress.push(status) + }, + } + ) + assert.equal(addedWithoutUploadProgress.toString(), car.cid.toString()) + assert.equal( + progressWithoutUploadProgress.reduce( + (max, { loaded }) => Math.max(max, loaded), + 0 + ), + 225 + ) }) it('throws for bucket URL client error 4xx', async () => {