From eef752578336af3155f7dff14e06cb0e8ad480b3 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Fri, 4 Feb 2022 16:28:18 +0000 Subject: [PATCH 1/4] fix: override hashing algorithm when importing files Looks like this got missed during the multiformats migration. Fixes #3952 --- packages/interface-ipfs-core/src/add-all.js | 17 +++++++++++++++++ packages/interface-ipfs-core/src/add.js | 17 +++++++++++++++++ .../ipfs-core/src/components/add-all/index.js | 13 ++++++++++++- packages/ipfs-core/src/components/index.js | 3 ++- packages/ipfs-core/src/components/root.js | 5 +++-- 5 files changed, 51 insertions(+), 4 deletions(-) diff --git a/packages/interface-ipfs-core/src/add-all.js b/packages/interface-ipfs-core/src/add-all.js index 62101f303d..9e879e9002 100644 --- a/packages/interface-ipfs-core/src/add-all.js +++ b/packages/interface-ipfs-core/src/add-all.js @@ -15,6 +15,7 @@ import bufferStream from 'it-buffer-stream' import * as raw from 'multiformats/codecs/raw' import * as dagPB from '@ipld/dag-pb' import resolve from 'aegir/utils/resolve.js' +import { sha256, sha512 } from 'multiformats/hashes/sha2' /** * @typedef {import('ipfsd-ctl').Factory} Factory @@ -479,6 +480,22 @@ export function testAddAll (factory, options) { .and.to.have.property('name').that.equals('TimeoutError') }) + it('should add with sha2-256 by default', async function () { + const content = String(Math.random() + Date.now()) + + const files = await all(ipfs.addAll(content)) + + expect(files).to.have.nested.property('[0].cid.multihash.code', sha256.code) + }) + + it('should add with a different hashing algorithm', async function () { + const content = String(Math.random() + Date.now()) + + const files = await all(ipfs.addAll(content)) + + expect(files).to.have.nested.property('[0].cid.multihash.code', sha512.code) + }) + it('should respect raw leaves when file is smaller than one block and no metadata is present', async () => { const files = await all(ipfs.addAll([Uint8Array.from([0, 1, 2])], { cidVersion: 1, diff --git a/packages/interface-ipfs-core/src/add.js b/packages/interface-ipfs-core/src/add.js index a0b582978b..9ee887b120 100644 --- a/packages/interface-ipfs-core/src/add.js +++ b/packages/interface-ipfs-core/src/add.js @@ -11,6 +11,7 @@ import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import last from 'it-last' import * as raw from 'multiformats/codecs/raw' import * as dagPB from '@ipld/dag-pb' +import { sha256, sha512 } from 'multiformats/hashes/sha2' const echoUrl = (/** @type {string} */ text) => `${process.env.ECHO_SERVER}/download?data=${encodeURIComponent(text)}` const redirectUrl = (/** @type {string} */ url) => `${process.env.ECHO_SERVER}/redirect?to=${encodeURI(url)}` @@ -274,6 +275,22 @@ export function testAdd (factory, options) { .and.to.have.property('name').that.equals('TimeoutError') }) + it('should add with sha2-256 by default', async function () { + const content = String(Math.random() + Date.now()) + + const file = await ipfs.add(content) + + expect(file).to.have.nested.property('cid.multihash.code', sha256.code) + }) + + it('should add with a different hashing algorithm', async function () { + const content = String(Math.random() + Date.now()) + + const file = await ipfs.add(content, { hashAlg: 'sha2-512' }) + + expect(file).to.have.nested.property('cid.multihash.code', sha512.code) + }) + it('should add with mode as string', async function () { // @ts-ignore this is mocha this.slow(10 * 1000) diff --git a/packages/ipfs-core/src/components/add-all/index.js b/packages/ipfs-core/src/components/add-all/index.js index 3b0e8ce78e..2cb5a15069 100644 --- a/packages/ipfs-core/src/components/add-all/index.js +++ b/packages/ipfs-core/src/components/add-all/index.js @@ -9,16 +9,19 @@ const mergeOptions = mergeOpts.bind({ ignoreUndefined: true }) /** * @typedef {import('multiformats/cid').CID} CID * @typedef {import('ipfs-unixfs-importer').ImportResult} ImportResult + * @typedef {import('multiformats/hashes/interface').MultihashHasher} MultihashHasher + * @typedef {import('ipfs-core-utils/multihashes').Multihashes} Multihashes */ /** * @typedef {Object} Context * @property {import('ipfs-repo').IPFSRepo} repo * @property {import('../../types').Preload} preload + * @property {Multihashes} hashers * @property {import('ipfs-core-types/src/root').ShardingOptions} [options] * @param {Context} context */ -export function createAddAll ({ repo, preload, options }) { +export function createAddAll ({ repo, preload, hashers, options }) { const isShardingEnabled = options && options.sharding /** @@ -81,6 +84,13 @@ export function createAddAll ({ repo, preload, options }) { } } + /** @type {MultihashHasher | undefined} */ + let hasher + + if (opts.hashAlg != null) { + hasher = await hashers.getHasher(opts.hashAlg) + } + const iterator = pipe( normaliseInput(source), /** @@ -88,6 +98,7 @@ export function createAddAll ({ repo, preload, options }) { */ source => importer(source, repo.blocks, { ...opts, + hasher, pin: false }), transformFile(opts), diff --git a/packages/ipfs-core/src/components/index.js b/packages/ipfs-core/src/components/index.js index f74b57f9d3..6eaafb829c 100644 --- a/packages/ipfs-core/src/components/index.js +++ b/packages/ipfs-core/src/components/index.js @@ -126,7 +126,8 @@ class IPFS { const { add, addAll, cat, get, ls } = new RootAPI({ preload, repo, - options: options.EXPERIMENTAL + options: options.EXPERIMENTAL, + hashers: this.hashers }) const files = createFiles({ diff --git a/packages/ipfs-core/src/components/root.js b/packages/ipfs-core/src/components/root.js index 1469dcf82d..f0da3f8458 100644 --- a/packages/ipfs-core/src/components/root.js +++ b/packages/ipfs-core/src/components/root.js @@ -15,11 +15,12 @@ export class RootAPI { /** * @param {Context} context */ - constructor ({ preload, repo, options }) { + constructor ({ preload, repo, hashers, options }) { const addAll = createAddAll({ preload, repo, - options + options, + hashers }) this.addAll = addAll From 527236fa415d65ecdb8f3b09ab947250b3153a96 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Fri, 4 Feb 2022 16:58:12 +0000 Subject: [PATCH 2/4] chore: pass multiples to add-all --- packages/interface-ipfs-core/src/add-all.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/interface-ipfs-core/src/add-all.js b/packages/interface-ipfs-core/src/add-all.js index 9e879e9002..a7701a8294 100644 --- a/packages/interface-ipfs-core/src/add-all.js +++ b/packages/interface-ipfs-core/src/add-all.js @@ -483,7 +483,7 @@ export function testAddAll (factory, options) { it('should add with sha2-256 by default', async function () { const content = String(Math.random() + Date.now()) - const files = await all(ipfs.addAll(content)) + const files = await all(ipfs.addAll([content])) expect(files).to.have.nested.property('[0].cid.multihash.code', sha256.code) }) @@ -491,7 +491,7 @@ export function testAddAll (factory, options) { it('should add with a different hashing algorithm', async function () { const content = String(Math.random() + Date.now()) - const files = await all(ipfs.addAll(content)) + const files = await all(ipfs.addAll([content])) expect(files).to.have.nested.property('[0].cid.multihash.code', sha512.code) }) From 4562ce8ec51a20bbe0529885ca2eb39eacd879d5 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Fri, 4 Feb 2022 18:35:01 +0000 Subject: [PATCH 3/4] chore: fix tests --- packages/interface-ipfs-core/src/add-all.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/interface-ipfs-core/src/add-all.js b/packages/interface-ipfs-core/src/add-all.js index a7701a8294..fdb9d5dbff 100644 --- a/packages/interface-ipfs-core/src/add-all.js +++ b/packages/interface-ipfs-core/src/add-all.js @@ -480,7 +480,7 @@ export function testAddAll (factory, options) { .and.to.have.property('name').that.equals('TimeoutError') }) - it('should add with sha2-256 by default', async function () { + it('should add all with sha2-256 by default', async function () { const content = String(Math.random() + Date.now()) const files = await all(ipfs.addAll([content])) @@ -488,10 +488,10 @@ export function testAddAll (factory, options) { expect(files).to.have.nested.property('[0].cid.multihash.code', sha256.code) }) - it('should add with a different hashing algorithm', async function () { + it('should add all with a different hashing algorithm', async function () { const content = String(Math.random() + Date.now()) - const files = await all(ipfs.addAll([content])) + const files = await all(ipfs.addAll([content], { hashAlg: 'sha2-512' })) expect(files).to.have.nested.property('[0].cid.multihash.code', sha512.code) }) From a934caece2a8f7ee7b2d72f1c1ddd985efbc180c Mon Sep 17 00:00:00 2001 From: achingbrain Date: Sat, 5 Feb 2022 11:55:33 +0000 Subject: [PATCH 4/4] chore: remove extra example dep --- .github/workflows/examples.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index 600cba9bdc..8f089b6419 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -120,7 +120,7 @@ jobs: deps: ipfs-core@$PWD/packages/ipfs-core/dist - name: types with typed js repo: https://github.com/ipfs-examples/js-ipfs-types-use-ipfs-from-typed-js.git - deps: ipfs-core@$PWD/packages/ipfs-core/dist,ipfs-core-types@$PWD/packages/ipfs-core-types/dist + deps: ipfs-core@$PWD/packages/ipfs-core/dist steps: - uses: actions/checkout@v2 - uses: actions/setup-node@v2