feat: upload-client uploadDirectory, by default, sorts the provided files by file name to help the user call us in a way that is deterministic and minimizes cost

packages/upload-client/src/index.js

@@ -6,13 +6,17 @@ import * as Store from './store.js'
 import * as Upload from './upload.js'
 import * as UnixFS from './unixfs.js'
 import * as CAR from './car.js'
-import { ShardingStream } from './sharding.js'
+import { ShardingStream, requireSortedFiles } from './sharding.js'
 
 export { Store, Upload, UnixFS, CAR }
 export * from './sharding.js'
 
 const CONCURRENT_REQUESTS = 3
 
+/**
+ * @typedef {import('./types.js').FileLike} FileLike
+ */
+
 /**
  * Uploads a file to the service and returns the root data CID for the
  * generated DAG.
@@ -63,13 +67,16 @@ export async function uploadFile(conf, file, options = {}) {
  * has the capability to perform the action.
  *
  * The issuer needs the `store/add` and `upload/add` delegated capability.
- * @param {import('./types.js').FileLike[]} files File data.
+ * @param {Iterable<FileLike> & { sorted?: boolean }} files  Files that should be in the directory.
+ * To ensure determinism in the IPLD encoding, by default these files MUST be sorted by file.name or this function will return a rejected promise .
+ * To explicitly upload with in an indeterminate way, pass `files` with `files.sorted === false`.
  * @param {import('./types.js').UploadDirectoryOptions} [options]
  */
 export async function uploadDirectory(conf, files, options = {}) {
+  const entries = 'sorted' in files ? files : requireSortedFiles(files)
   return await uploadBlockStream(
     conf,
-    UnixFS.createDirectoryEncoderStream(files, options),
+    UnixFS.createDirectoryEncoderStream(entries, options),
     options
   )
 }

packages/upload-client/src/sharding.js

@@ -1,5 +1,9 @@
 import { blockEncodingLength, encode, headerEncodingLength } from './car.js'
 
+/**
+ * @typedef {import('./types.js').FileLike} FileLike
+ */
+
 // https://observablehq.com/@gozala/w3up-shard-size
 const SHARD_SIZE = 133_169_152
 
@@ -84,3 +88,78 @@ export class ShardingStream extends TransformStream {
     })
   }
 }
+
+/**
+ * @template T
+ * wrap an iterable in a new iterable that will iterate the same items,
+ * but will error if the iterated items aren't sorted according to a comparator
+ * @implements {Iterable<T>}
+ */
+class Sorted {
+  sorted = /** @type {const} */ (true)
+  /**
+   * @param {Iterable<T>} iterable
+   * @param {(a: T, b: T) => number} comparator
+   */
+  constructor(iterable, comparator) {
+    this.iterable = iterable
+    this.comparator = comparator
+  }
+  [Symbol.iterator]() {
+    const { comparator, iterable } = this
+    return function* () {
+      let prev = null
+      for (const cur of iterable) {
+        if (prev && comparator(prev, cur) === 1) {
+          throw Object.assign(
+            new Error(
+              `sequentially iterated items were not sorted as expected`
+            ),
+            { unsorted: [prev, cur], code: 'SORTED_EXPECTATION_UNMET' }
+          )
+        }
+        yield cur
+        prev = cur
+      }
+    }.bind(this)()
+  }
+}
+
+/**
+ * given an iterable of files, return another iterable that ensures
+ * that the files are iterated in a sorted order.
+ *
+ * @param {Iterable<import('./types.js').FileLike>} files
+ * @param {(file: import('./types.js').FileLike) => string} getSortKey - given a FileLike, return a value by which all the FileLikes should be sorted
+ * @returns
+ */
+export const requireSortedFiles = (files, getSortKey = (a) => a.name) => {
+  return new Sorted(files, (a, b) => defaultFileComparator(a, b, getSortKey))
+}
+
+/**
+ * Default comparator for FileLikes. Sorts by file name in ascending order.
+ *
+ * @param {FileLike} a
+ * @param {FileLike} b
+ * @param {(file: FileLike) => string} getSortKey
+ */
+export const defaultFileComparator = (a, b, getSortKey = (a) => a.name) => {
+  return ascending(a, b, getSortKey)
+}
+
+/**
+ * a comparator for sorting in ascending order. Use with Sorted or Array#sort.
+ *
+ * @template T
+ * @param {T} a
+ * @param {T} b
+ * @param {(i: T) => any} getSortKey - given an item being sorted, return the value by which it should be sorted
+ */
+function ascending(a, b, getSortKey) {
+  const ask = getSortKey(a)
+  const bsk = getSortKey(b)
+  if (ask === bsk) return 0
+  else if (ask < bsk) return -1
+  return 1
+}

packages/upload-client/test/index.test.js

@@ -6,7 +6,12 @@ import * as CAR from '@ucanto/transport/car'
 import * as Signer from '@ucanto/principal/ed25519'
 import * as StoreCapabilities from '@web3-storage/capabilities/store'
 import * as UploadCapabilities from '@web3-storage/capabilities/upload'
-import { uploadFile, uploadDirectory, uploadCAR } from '../src/index.js'
+import {
+  uploadFile,
+  uploadDirectory,
+  uploadCAR,
+  defaultFileComparator,
+} from '../src/index.js'
 import { serviceSigner } from './fixtures.js'
 import { randomBlock, randomBytes } from './helpers/random.js'
 import { toCAR } from './helpers/car.js'
@@ -368,6 +373,92 @@ describe('uploadDirectory', () => {
 
     assert.equal(carCIDs.length, 2)
   })
+
+  it('ensures files is sorted unless sorted property also provided', async () => {
+    const space = await Signer.generate()
+    const agent = await Signer.generate() // The "user" that will ask the service to accept the upload
+    const proofs = await Promise.all([
+      StoreCapabilities.add.delegate({
+        issuer: space,
+        audience: agent,
+        with: space.did(),
+        expiration: Infinity,
+      }),
+      UploadCapabilities.add.delegate({
+        issuer: space,
+        audience: agent,
+        with: space.did(),
+        expiration: Infinity,
+      }),
+    ])
+    const service = mockService({
+      store: {
+        add: provide(StoreCapabilities.add, ({ capability }) => ({
+          ok: {
+            status: 'upload',
+            headers: { 'x-test': 'true' },
+            url: 'http://localhost:9200',
+            with: space.did(),
+            link: /** @type {import('../src/types.js').CARLink} */ (
+              capability.nb.link
+            ),
+          },
+        })),
+      },
+      upload: {
+        add: provide(UploadCapabilities.add, ({ capability }) => {
+          if (!capability.nb) throw new Error('nb must be present')
+          return { ok: capability.nb }
+        }),
+      },
+    })
+
+    const server = Server.create({
+      id: serviceSigner,
+      service,
+      codec: CAR.inbound,
+      validateAuthorization,
+    })
+    const connection = Client.connect({
+      id: serviceSigner,
+      codec: CAR.outbound,
+      channel: server,
+    })
+    /**
+     * @param {Iterable<import('../src/types.js').FileLike>} files
+     */
+    const upload = (files) =>
+      uploadDirectory(
+        { issuer: agent, with: space.did(), proofs, audience: serviceSigner },
+        files,
+        {
+          connection,
+        }
+      )
+
+    const unsortedFiles = [
+      new File([await randomBytes(32)], '/b.txt'),
+      new File([await randomBytes(32)], '/b.txt'),
+      new File([await randomBytes(32)], 'c.txt'),
+      new File([await randomBytes(32)], 'a.txt'),
+    ]
+    assert.rejects(
+      upload(unsortedFiles),
+      'uploading unsorted files returns rejected promise'
+    )
+
+    // sorted files should work
+    const sortedFiles = [...unsortedFiles].sort(defaultFileComparator)
+    assert.doesNotReject(
+      upload(sortedFiles),
+      'uploading unsorted files returns rejected promise'
+    )
+
+    assert.doesNotReject(
+      upload(Object.assign([...unsortedFiles], { sorted: false })),
+      'can upload unsorted files if sorted property is provided'
+    )
+  })
 })
 
 describe('uploadCAR', () => {