From 63be9f51609b144c995a646a654ce09c20c712d9 Mon Sep 17 00:00:00 2001 From: Stephen Whitmore Date: Mon, 23 May 2016 12:16:57 -0700 Subject: [PATCH 1/4] Make ipfs.files.add return DAGNodes. --- package.json | 6 ++-- src/core/ipfs/files.js | 62 +++++++++++++++++++++++--------- test/core/both/test-files.js | 69 +++++++----------------------------- 3 files changed, 62 insertions(+), 75 deletions(-) diff --git a/package.json b/package.json index 781407f951..666a1f18ef 100644 --- a/package.json +++ b/package.json @@ -74,6 +74,7 @@ "ipfs-multipart": "^0.1.0", "ipfs-repo": "^0.8.0", "ipfs-unixfs-engine": "^0.8.0", + "isstream": "^0.1.2", "joi": "^8.0.5", "libp2p-ipfs": "^0.10.1", "libp2p-ipfs-browser": "^0.9.1", @@ -92,7 +93,8 @@ "run-parallel-limit": "^1.0.3", "run-series": "^1.1.4", "run-waterfall": "^1.1.3", - "temp": "^0.8.3" + "temp": "^0.8.3", + "through2": "^2.0.1" }, "contributors": [ "Andrew de Andrade ", @@ -111,4 +113,4 @@ "kumavis ", "nginnever " ] -} \ No newline at end of file +} diff --git a/src/core/ipfs/files.js b/src/core/ipfs/files.js index 4b4dcf8c34..29c2e3577f 100644 --- a/src/core/ipfs/files.js +++ b/src/core/ipfs/files.js @@ -3,38 +3,68 @@ const Importer = require('ipfs-unixfs-engine').importer const Exporter = require('ipfs-unixfs-engine').exporter const UnixFS = require('ipfs-unixfs') +const bs58 = require('bs58') +const through = require('through2') +const isStream = require('isstream') +const promisify = require('promisify-es6') module.exports = function files (self) { return { - add: (arr, callback) => { - if (typeof arr === 'function') { - callback = arr - arr = undefined + createAddStream: promisify((callback) => { + // TODO: wip + if (data === undefined) { + return new Importer(self._dagS) } - if (callback === undefined) { - callback = function noop () {} + }), + + add: promisify((data, callback) => { + // Buffer input + if (Buffer.isBuffer(data)) { + data = [{ + path: '', + content: data + }] } - if (arr === undefined) { - return new Importer(self._dagS) + // Readable stream input + if (isStream.isReadable(data)) { + data = [{ + path: '', + content: data + }] + } + if (!callback || typeof callback !== 'function') { + callback = function oop () {} + } + if (!Array.isArray(data)) { + return callback(new Error('"data" must be an array of { path: string, content: Buffer|Readable } or Buffer or Readable')) } const i = new Importer(self._dagS) const res = [] - i.on('data', (info) => { - res.push(info) - }) - - i.once('end', () => { + // Transform file info tuples to DAGNodes + i.pipe(through.obj(function transform (info, enc, next) { + const mh = bs58.encode(info.multihash).toString() + self._dagS.get(mh, (err, node) => { + if (err) return callback(err) + var obj = { + path: info.path || mh, + node: node + } + res.push(obj) + next() + }) + }, function end (done) { callback(null, res) - }) + })) - arr.forEach((tuple) => { + data.forEach((tuple) => { i.write(tuple) }) i.end() - }, + }), + cat: (hash, callback) => { self._dagS.get(hash, (err, fetchedNode) => { if (err) { diff --git a/test/core/both/test-files.js b/test/core/both/test-files.js index 353b75a273..18f7c88b34 100644 --- a/test/core/both/test-files.js +++ b/test/core/both/test-files.js @@ -1,64 +1,19 @@ /* eslint-env mocha */ 'use strict' -const bl = require('bl') -const expect = require('chai').expect -const Readable = require('stream').Readable -const bs58 = require('bs58') - +const test = require('interface-ipfs-core') const IPFS = require('../../../src/core') -describe('files', () => { - let ipfs - - before((done) => { - ipfs = new IPFS(require('../../utils/repo-path')) - ipfs.load(done) - }) - - it('add', (done) => { - const buffered = new Buffer('some data') - const rs = new Readable() - rs.push(buffered) - rs.push(null) - const arr = [] - const filePair = {path: 'data.txt', stream: rs} - arr.push(filePair) - ipfs.files.add(arr, (err, res) => { - expect(err).to.not.exist - expect(res[0].path).to.equal('data.txt') - expect(res[0].size).to.equal(17) - expect(bs58.encode(res[0].multihash).toString()).to.equal('QmVv4Wz46JaZJeH5PMV4LGbRiiMKEmszPYY3g6fjGnVXBS') - done() +const common = { + setup: function (cb) { + const ipfs = new IPFS(require('../../utils/repo-path')) + ipfs.load(() => { + cb(null, ipfs) }) - }) + }, + teardown: function (cb) { + cb() + } +} - it('cat', (done) => { - const hash = 'QmT78zSuBmuS4z925WZfrqQ1qHaJ56DQaTfyMUF7F8ff5o' - ipfs.files.cat(hash, (err, res) => { - expect(err).to.not.exist - res.on('data', (data) => { - data.stream.pipe(bl((err, bldata) => { - expect(err).to.not.exist - expect(bldata.toString()).to.equal('hello world\n') - done() - })) - }) - }) - }) - - it('get', (done) => { - // TODO create non-trival get test - const hash = 'QmT78zSuBmuS4z925WZfrqQ1qHaJ56DQaTfyMUF7F8ff5o' - ipfs.files.get(hash, (err, res) => { - expect(err).to.not.exist - res.on('data', (data) => { - data.stream.pipe(bl((err, bldata) => { - expect(err).to.not.exist - expect(bldata.toString()).to.equal('hello world\n') - done() - })) - }) - }) - }) -}) +test.files(common) From 5603e52b081b468a997574a28c264c978984043b Mon Sep 17 00:00:00 2001 From: Stephen Whitmore Date: Thu, 26 May 2016 21:32:56 -0700 Subject: [PATCH 2/4] Rename unixfs output from 'stream' to 'content'. --- src/cli/commands/files/add.js | 4 ++-- src/cli/commands/files/cat.js | 2 +- src/cli/commands/files/get.js | 4 ++-- src/core/ipfs/init.js | 2 +- src/http-api/resources/files.js | 2 +- test/core/both/test-bitswap.js | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/cli/commands/files/add.js b/src/cli/commands/files/add.js index f69a8b700f..09466b319e 100644 --- a/src/cli/commands/files/add.js +++ b/src/cli/commands/files/add.js @@ -73,7 +73,7 @@ module.exports = Command.extend({ if (!fs.statSync(element).isDirectory()) { i.write({ path: element.substring(index + 1, element.length), - stream: fs.createReadStream(element) + content: fs.createReadStream(element) }) } callback() @@ -86,7 +86,7 @@ module.exports = Command.extend({ } else { rs = fs.createReadStream(inPath) inPath = inPath.substring(inPath.lastIndexOf('/') + 1, inPath.length) - filePair = {path: inPath, stream: rs} + filePair = {path: inPath, content: rs} i.write(filePair) i.end() } diff --git a/src/cli/commands/files/cat.js b/src/cli/commands/files/cat.js index c517b8ad3e..dfb8c175f0 100644 --- a/src/cli/commands/files/cat.js +++ b/src/cli/commands/files/cat.js @@ -36,7 +36,7 @@ module.exports = Command.extend({ throw (err) } res.on('data', (data) => { - data.stream.pipe(process.stdout) + data.content.pipe(process.stdout) }) }) }) diff --git a/src/cli/commands/files/get.js b/src/cli/commands/files/get.js index 3640cf1617..e85542dbe2 100644 --- a/src/cli/commands/files/get.js +++ b/src/cli/commands/files/get.js @@ -48,7 +48,7 @@ function fileHandler (result, dir) { const dirPath = path.join(dir, file.path) // Check to see if the result is a directory if (file.dir === false) { - file.stream.pipe(fs.createWriteStream(dirPath)) + file.content.pipe(fs.createWriteStream(dirPath)) } else { ensureDir(dirPath, (err) => { if (err) { @@ -64,7 +64,7 @@ function fileHandler (result, dir) { throw err } - file.stream.pipe(fs.createWriteStream(dirPath)) + file.content.pipe(fs.createWriteStream(dirPath)) }) } } diff --git a/src/core/ipfs/init.js b/src/core/ipfs/init.js index 71ef24a6f1..a9116dab9a 100644 --- a/src/core/ipfs/init.js +++ b/src/core/ipfs/init.js @@ -91,7 +91,7 @@ module.exports = function init (self) { const rs = new Readable() rs.push(fs.readFileSync(element)) rs.push(null) - const filePair = {path: addPath, stream: rs} + const filePair = {path: addPath, content: rs} i.write(filePair) } callback() diff --git a/src/http-api/resources/files.js b/src/http-api/resources/files.js index 8647a8e314..3ebe82f5b1 100644 --- a/src/http-api/resources/files.js +++ b/src/http-api/resources/files.js @@ -43,7 +43,7 @@ exports.cat = { }).code(500) } stream.on('data', (data) => { - return reply(data.stream) + return reply(data.content) }) }) } diff --git a/test/core/both/test-bitswap.js b/test/core/both/test-bitswap.js index efdcc44349..98cfab56b1 100644 --- a/test/core/both/test-bitswap.js +++ b/test/core/both/test-bitswap.js @@ -196,7 +196,7 @@ describe('bitswap', () => { ipfs.files.cat(hash, (err, res) => { expect(err).to.not.exist res.on('file', (data) => { - data.stream.pipe(bl((err, bldata) => { + data.content.pipe(bl((err, bldata) => { expect(err).to.not.exist expect(bldata.toString()).to.equal('I love IPFS <3') cb() From 98667adc33f6b3fce9c5fb0bed4e97bf06aa2ac4 Mon Sep 17 00:00:00 2001 From: Stephen Whitmore Date: Thu, 26 May 2016 21:34:22 -0700 Subject: [PATCH 3/4] Mitigate test timing issues. Though one of these is a fix (using 'before' properly), the rest are timing issues that are more noticeable on slower computers. --- test/cli/test-bitswap.js | 2 +- test/core/both/test-bitswap.js | 3 ++- test/core/node-only/test-swarm.js | 2 +- test/http-api/test-files.js | 3 ++- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/test/cli/test-bitswap.js b/test/cli/test-bitswap.js index ee59f63a90..4fbacda1f0 100644 --- a/test/cli/test-bitswap.js +++ b/test/cli/test-bitswap.js @@ -12,7 +12,7 @@ const createTempNode = require('../utils/temp-node') const repoPath = require('./index').repoPath describe('bitswap', function () { - this.timeout(20000) + this.timeout(40000) const env = _.clone(process.env) env.IPFS_PATH = repoPath diff --git a/test/core/both/test-bitswap.js b/test/core/both/test-bitswap.js index 98cfab56b1..4d6befa706 100644 --- a/test/core/both/test-bitswap.js +++ b/test/core/both/test-bitswap.js @@ -101,7 +101,8 @@ describe('bitswap', () => { }) afterEach((done) => { - setTimeout(() => ipfs.goOffline(done), 500) + // ipfs.goOffline(done) + setTimeout(() => ipfs.goOffline(done), 1500) }) it('2 peers', (done) => { diff --git a/test/core/node-only/test-swarm.js b/test/core/node-only/test-swarm.js index e6c6238bf1..2d3284bc2a 100644 --- a/test/core/node-only/test-swarm.js +++ b/test/core/node-only/test-swarm.js @@ -7,7 +7,7 @@ const parallel = require('run-parallel') const createTempNode = require('../../utils/temp-node') describe('swarm', function () { - this.timeout(20 * 1000) + this.timeout(40 * 1000) let nodeA let nodeB diff --git a/test/http-api/test-files.js b/test/http-api/test-files.js index c41dda2862..ef6d1a78ff 100644 --- a/test/http-api/test-files.js +++ b/test/http-api/test-files.js @@ -9,8 +9,9 @@ module.exports = (httpAPI) => { describe('api', () => { let api - it('api', () => { + before((done) => { api = httpAPI.server.select('API') + done() }) describe('/files/cat', () => { From 7b698b3529be4cf51321a52670bae1ee0498c934 Mon Sep 17 00:00:00 2001 From: Stephen Whitmore Date: Tue, 31 May 2016 13:12:59 -0700 Subject: [PATCH 4/4] Update ipfs-unixfs-engine to 0.9.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 666a1f18ef..74d5f8f032 100644 --- a/package.json +++ b/package.json @@ -73,7 +73,7 @@ "ipfs-merkle-dag": "^0.6.0", "ipfs-multipart": "^0.1.0", "ipfs-repo": "^0.8.0", - "ipfs-unixfs-engine": "^0.8.0", + "ipfs-unixfs-engine": "^0.9.0", "isstream": "^0.1.2", "joi": "^8.0.5", "libp2p-ipfs": "^0.10.1",