From 186b80af8e05c63bab0a4320968022e9933ea1e6 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 17 Jul 2018 15:12:12 +0100 Subject: [PATCH] feat: support --raw-leaves Also updates pinning to support CIDv1 --- package.json | 4 +-- src/cli/commands/files/add.js | 33 ++----------------- src/core/components/pin.js | 5 ++- src/core/utils.js | 13 +++++--- src/http/api/resources/files.js | 16 ++------- test/cli/files.js | 58 ++++++++++----------------------- 6 files changed, 35 insertions(+), 94 deletions(-) diff --git a/package.json b/package.json index 67d1819938..2cd7bcb477 100644 --- a/package.json +++ b/package.json @@ -108,11 +108,11 @@ "ipfs-block": "~0.7.1", "ipfs-block-service": "~0.14.0", "ipfs-http-response": "~0.1.2", - "ipfs-mfs": "~0.1.0", + "ipfs-mfs": "~0.2.2", "ipfs-multipart": "~0.1.0", "ipfs-repo": "~0.22.1", "ipfs-unixfs": "~0.1.15", - "ipfs-unixfs-engine": "~0.30.0", + "ipfs-unixfs-engine": "~0.31.3", "ipld": "~0.17.3", "ipld-dag-cbor": "~0.12.1", "ipld-dag-pb": "~0.14.5", diff --git a/src/cli/commands/files/add.js b/src/cli/commands/files/add.js index fe2a602a41..2547150e91 100644 --- a/src/cli/commands/files/add.js +++ b/src/cli/commands/files/add.js @@ -145,12 +145,13 @@ module.exports = { }, 'raw-leaves': { type: 'boolean', - default: undefined, + default: false, describe: 'Use raw blocks for leaf nodes. (experimental)' }, 'cid-version': { type: 'integer', - describe: 'Cid version. Non-zero value will change default of \'raw-leaves\' to true. (experimental)' + describe: 'CID version. Defaults to 0 unless an option that depends on CIDv1 is passed. (experimental)', + default: 0 }, hash: { type: 'string', @@ -197,34 +198,6 @@ module.exports = { pin: argv.pin } - // Temporary restriction on raw-leaves: - // When cid-version=1 then raw-leaves MUST be present and false. - // - // This is because raw-leaves is not yet implemented in js-ipfs, - // and go-ipfs changes the value of raw-leaves to true when - // cid-version > 0 unless explicitly set to false. - // - // This retains feature parity without having to implement raw-leaves. - if (options.cidVersion > 0 && options.rawLeaves !== false) { - throw new Error('Implied argument raw-leaves must be passed and set to false when cid-version is > 0') - } - - // Temporary restriction on raw-leaves: - // When hash != undefined then raw-leaves MUST be present and false. - // - // This is because raw-leaves is not yet implemented in js-ipfs, - // and go-ipfs changes the value of raw-leaves to true when - // hash != undefined unless explicitly set to false. - // - // This retains feature parity without having to implement raw-leaves. - if (options.hash && options.rawLeaves !== false) { - throw new Error('Implied argument raw-leaves must be passed and set to false when hash argument is specified') - } - - if (options.rawLeaves) { - throw new Error('Not implemented: raw-leaves') - } - if (options.enableShardingExperiment && utils.isDaemonOn()) { throw new Error('Error: Enabling the sharding experiment should be done on the daemon') } diff --git a/src/core/components/pin.js b/src/core/components/pin.js index c6f6cc9696..d2bd670e14 100644 --- a/src/core/components/pin.js +++ b/src/core/components/pin.js @@ -4,7 +4,6 @@ const promisify = require('promisify-es6') const { DAGNode, DAGLink } = require('ipld-dag-pb') const CID = require('cids') -const multihashes = require('multihashes') const async = require('async') const { Key } = require('interface-datastore') @@ -34,9 +33,9 @@ module.exports = (self) => { let recursivePins = new Set() const directKeys = () => - Array.from(directPins).map(key => multihashes.fromB58String(key)) + Array.from(directPins).map(key => new CID(key).buffer) const recursiveKeys = () => - Array.from(recursivePins).map(key => multihashes.fromB58String(key)) + Array.from(recursivePins).map(key => new CID(key).buffer) function getIndirectKeys (callback) { const indirectKeys = new Set() diff --git a/src/core/utils.js b/src/core/utils.js index a0d67e449a..55ac9be2a2 100644 --- a/src/core/utils.js +++ b/src/core/utils.js @@ -1,9 +1,9 @@ 'use strict' -const multihashes = require('multihashes') const promisify = require('promisify-es6') const map = require('async/map') const isIpfs = require('is-ipfs') +const CID = require('cids') exports.OFFLINE_ERROR = 'This command must be run in online mode. Try running \'ipfs daemon\' first.' @@ -61,12 +61,15 @@ const resolvePath = promisify(function (objectAPI, ipfsPaths, callback) { map(ipfsPaths, (path, cb) => { if (typeof path !== 'string') { + let cid + try { - multihashes.validate(path) + cid = new CID(path) } catch (err) { return cb(err) } - return cb(null, path) + + return cb(null, cid.buffer) } let parsedPath @@ -76,10 +79,10 @@ const resolvePath = promisify(function (objectAPI, ipfsPaths, callback) { return cb(err) } - const rootHash = multihashes.fromB58String(parsedPath.hash) + const rootHash = new CID(parsedPath.hash) const rootLinks = parsedPath.links if (!rootLinks.length) { - return cb(null, rootHash) + return cb(null, rootHash.buffer) } objectAPI.get(rootHash, follow.bind(null, rootLinks)) diff --git a/src/http/api/resources/files.js b/src/http/api/resources/files.js index a3b7e8eaed..0370c4b0c2 100644 --- a/src/http/api/resources/files.js +++ b/src/http/api/resources/files.js @@ -153,20 +153,8 @@ exports.add = { validate: { query: Joi.object() .keys({ - 'cid-version': Joi.number().integer().min(0).max(1), - // Temporary restriction on raw-leaves: - // When cid-version=1 then raw-leaves MUST be present and false. - // - // This is because raw-leaves is not yet implemented in js-ipfs, - // and go-ipfs changes the value of raw-leaves to true when - // cid-version > 0 unless explicitly set to false. - // - // This retains feature parity without having to implement raw-leaves. - 'raw-leaves': Joi.boolean().when('cid-version', { - is: 1, - then: Joi.boolean().valid(false).required(), - otherwise: Joi.boolean().valid(false) - }), + 'cid-version': Joi.number().integer().min(0).max(1).default(0), + 'raw-leaves': Joi.boolean(), 'only-hash': Joi.boolean(), pin: Joi.boolean().default(true), 'wrap-with-directory': Joi.boolean() diff --git a/test/cli/files.js b/test/cli/files.js index 2f749f1cfe..fda9c2de94 100644 --- a/test/cli/files.js +++ b/test/cli/files.js @@ -205,56 +205,34 @@ describe('files', () => runOnAndOff((thing) => { }) }) - // Temporarily expect to fail as raw-leaves not yet implemented. - // - // When cid-version=1 then raw-leaves MUST be present and false. - // - // This is because raw-leaves is not yet implemented in js-ipfs, - // and go-ipfs changes the value of raw-leaves to true when - // cid-version > 0 unless explicitly set to false. - // - // This retains feature parity without having to implement raw-leaves. it('add with cid-version=1', function () { this.timeout(30 * 1000) - return new Promise((resolve, reject) => { - ipfs('add src/init-files/init-docs/readme --cid-version=1') - .then(() => reject(new Error('Raw leaves not expected to be implemented'))) - .catch((err) => { - expect(err).to.exist() - resolve() - }) - }) + return ipfs('add src/init-files/init-docs/readme --cid-version=1') + .then((out) => { + expect(out) + .to.eql('added zdj7WWeQ43G6JJvLWQWZpyHuAMq6uYWRjkBXFad11vE2LHhQ7 readme\n') + }) }) - // TODO: this test is failing, @alanshaw? - it.skip('add with cid-version=1 and raw-leaves=false', () => { - return ipfs('add src/init-files/init-docs/readme --cid-version=1 --raw-leaves=false').then((out) => { - expect(out) - .to.eql('added zdj7WWeQ43G6JJvLWQWZpyHuAMq6uYWRjkBXFad11vE2LHhQ7 readme\n') - }) + it('add with cid-version=1 and raw-leaves=false', function () { + this.timeout(30 * 1000) + + return ipfs('add src/init-files/init-docs/readme --cid-version=1 --raw-leaves=false') + .then((out) => { + expect(out) + .to.eql('added zdj7WWeQ43G6JJvLWQWZpyHuAMq6uYWRjkBXFad11vE2LHhQ7 readme\n') + }) }) - // Temporarily expect to fail as raw-leaves not yet implemented - // - // When cid-version=1 then raw-leaves MUST be present and false. - // - // This is because raw-leaves is not yet implemented in js-ipfs, - // and go-ipfs changes the value of raw-leaves to true when - // cid-version > 0 unless explicitly set to false. - // - // This retains feature parity without having to implement raw-leaves. it('add with cid-version=1 and raw-leaves=true', function () { this.timeout(30 * 1000) - return new Promise((resolve, reject) => { - ipfs('add src/init-files/init-docs/readme --cid-version=1 --raw-leaves=true') - .then(() => reject(new Error('Raw leaves not expected to be implemented'))) - .catch((err) => { - expect(err).to.exist() - resolve() - }) - }) + return ipfs('add src/init-files/init-docs/readme --cid-version=1 --raw-leaves=true') + .then((out) => { + expect(out) + .to.eql('added zdj7WiLc855B1KPRgV7Fh8ivjuAhePE1tuJafmxH5HmmSjqaD readme\n') + }) }) it('add --quiet', function () {