From f4bfc2b4703fab1c96a3e4e26a8c9a2d7b7eaa96 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Fri, 9 Nov 2018 10:32:58 +0000 Subject: [PATCH 1/5] fix: dht validate if receiving stream --- package.json | 3 +- src/dht/findpeer.js | 48 +++++++++++++++++-- src/dht/findprovs.js | 30 ++++++++++-- src/dht/query.js | 14 +++++- src/utils/stream-to-value-with-transformer.js | 18 +++++++ 5 files changed, 104 insertions(+), 9 deletions(-) create mode 100644 src/utils/stream-to-value-with-transformer.js diff --git a/package.json b/package.json index 224d39882..911fafc8f 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "debug": "^4.1.0", "detect-node": "^2.0.4", "end-of-stream": "^1.4.1", + "err-code": "^1.1.2", "flatmap": "0.0.3", "glob": "^7.1.3", "ipfs-block": "~0.8.0", @@ -85,7 +86,7 @@ "eslint-plugin-react": "^7.11.1", "go-ipfs-dep": "~0.4.18", "gulp": "^3.9.1", - "interface-ipfs-core": "~0.90.0", + "interface-ipfs-core": "ipfs/interface-ipfs-core#fix/update-dht-responses", "ipfsd-ctl": "~0.40.0", "nock": "^10.0.2", "pull-stream": "^3.6.9", diff --git a/src/dht/findpeer.js b/src/dht/findpeer.js index 4b19cafbb..9ae9d996d 100644 --- a/src/dht/findpeer.js +++ b/src/dht/findpeer.js @@ -1,7 +1,9 @@ 'use strict' const promisify = require('promisify-es6') -const streamToValue = require('../utils/stream-to-value') +const streamToValueWithTransformer = require('../utils/stream-to-value-with-transformer') + +const errcode = require('err-code') module.exports = (send) => { return promisify((peerId, opts, callback) => { @@ -17,10 +19,50 @@ module.exports = (send) => { opts = {} } - send.andTransform({ + const handleResult = (res, callback) => { + // Inconsistent return values in the browser + if (Array.isArray(res)) { + res = res[0] + } + + // Type 2 keys + if (res.Type !== 2) { + const errMsg = `key was not found (type 2)` + + return callback(errcode(new Error(errMsg), 'ERR_KEY_TYPE_2_NOT_FOUND')) + } + + const id = res.Responses[0].ID + const addresses = res.Responses[0].Addrs.map((addr) => { + // inconsistencies js / go - go does not add `/ipfs/{id}` to the address + if (addr.split('/ipfs/') > -1) { + return addr + } else { + return `${addr}/ipfs/${id}` + } + }) + + const response = { + ...res, + Responses: [{ + ID: id, + Addrs: addresses + }] + } + + callback(null, response) + } + + send({ path: 'dht/findpeer', args: peerId, qs: opts - }, streamToValue, callback) + }, (err, result) => { + if (err) { + return callback(err) + } + + streamToValueWithTransformer(result, handleResult, callback) + }) }) } diff --git a/src/dht/findprovs.js b/src/dht/findprovs.js index 23aec0127..1ffd53310 100644 --- a/src/dht/findprovs.js +++ b/src/dht/findprovs.js @@ -1,7 +1,9 @@ 'use strict' const promisify = require('promisify-es6') -const streamToValue = require('../utils/stream-to-value') +const streamToValueWithTransformer = require('../utils/stream-to-value-with-transformer') + +const errcode = require('err-code') module.exports = (send) => { return promisify((cid, opts, callback) => { @@ -17,10 +19,32 @@ module.exports = (send) => { opts = {} } - send.andTransform({ + const handleResult = (res, callback) => { + // Inconsistent return values in the browser vs node + if (Array.isArray(res)) { + res = res[0] + } + + // Type 4 keys + if (res.Type !== 4) { + const errMsg = `key was not found (type 4)` + + return callback(errcode(new Error(errMsg), 'ERR_KEY_TYPE_4_NOT_FOUND')) + } + + callback(null, res) + } + + send({ path: 'dht/findprovs', args: cid, qs: opts - }, streamToValue, callback) + }, (err, result) => { + if (err) { + return callback(err) + } + + streamToValueWithTransformer(result, handleResult, callback) + }) }) } diff --git a/src/dht/query.js b/src/dht/query.js index 7de751466..f1cdb32d0 100644 --- a/src/dht/query.js +++ b/src/dht/query.js @@ -17,10 +17,20 @@ module.exports = (send) => { opts = {} } - send.andTransform({ + send({ path: 'dht/query', args: peerId, qs: opts - }, streamToValue, callback) + }, (err, result) => { + if (err) { + return callback(err) + } + + if (typeof result.pipe === 'function') { + streamToValue(result, callback) + } else { + callback(null, result) + } + }) }) } diff --git a/src/utils/stream-to-value-with-transformer.js b/src/utils/stream-to-value-with-transformer.js new file mode 100644 index 000000000..402148667 --- /dev/null +++ b/src/utils/stream-to-value-with-transformer.js @@ -0,0 +1,18 @@ +'use strict' + +const streamToValue = require('./stream-to-value') + +function streamToValueWithTransformer (response, transformer, callback) { + if (typeof response.pipe === 'function') { + streamToValue(response, (err, res) => { + if (err) { + return callback(err) + } + transformer(res, callback) + }) + } else { + transformer(response, callback) + } +} + +module.exports = streamToValueWithTransformer From f703c17ce141d830205cf883842714536e14967d Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Tue, 13 Nov 2018 13:29:19 +0000 Subject: [PATCH 2/5] chore: update responses --- src/dht/findpeer.js | 34 +++++++++++++++++++++------------- src/dht/findprovs.js | 15 ++++++++++++--- 2 files changed, 33 insertions(+), 16 deletions(-) diff --git a/src/dht/findpeer.js b/src/dht/findpeer.js index 9ae9d996d..fa34a7a92 100644 --- a/src/dht/findpeer.js +++ b/src/dht/findpeer.js @@ -25,16 +25,27 @@ module.exports = (send) => { res = res[0] } - // Type 2 keys - if (res.Type !== 2) { + // Type 2 keys (inconsistencies between go core and js core) + if (res.Type !== 2 && res.type !== 2) { const errMsg = `key was not found (type 2)` return callback(errcode(new Error(errMsg), 'ERR_KEY_TYPE_2_NOT_FOUND')) } - const id = res.Responses[0].ID - const addresses = res.Responses[0].Addrs.map((addr) => { - // inconsistencies js / go - go does not add `/ipfs/{id}` to the address + // inconsistencies between go core and js core + let id + let addrs + + if (res.Responses) { + id = res.Responses[0].ID + addrs = res.Responses[0].Addrs + } else { + id = res.responses[0].id + addrs = res.responses[0].addrs + } + + // inconsistencies js / go - go does not add `/ipfs/{id}` to the address + addrs = addrs.map((addr) => { if (addr.split('/ipfs/') > -1) { return addr } else { @@ -42,15 +53,12 @@ module.exports = (send) => { } }) - const response = { - ...res, - Responses: [{ - ID: id, - Addrs: addresses + callback(null, { + responses: [{ + id, + addrs }] - } - - callback(null, response) + }) } send({ diff --git a/src/dht/findprovs.js b/src/dht/findprovs.js index 1ffd53310..24c946c64 100644 --- a/src/dht/findprovs.js +++ b/src/dht/findprovs.js @@ -25,14 +25,23 @@ module.exports = (send) => { res = res[0] } - // Type 4 keys - if (res.Type !== 4) { + // Type 4 keys (inconsistencies between go core and js core) + if (res.Type !== 4 && res.type !== 4) { const errMsg = `key was not found (type 4)` return callback(errcode(new Error(errMsg), 'ERR_KEY_TYPE_4_NOT_FOUND')) } - callback(null, res) + // inconsistencies between go core and js core + const recResponses = res.Responses || res.responses + + // providers array (handling inconsistencies) + const responses = recResponses.map((r) => ({ + id: r.ID || r.id, + addrs: r.Addrs || r.addrs + })) + + callback(null, { responses }) } send({ From c9dabf2f0a50b0d858b666487d24ff6f9ed1026e Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Sun, 9 Dec 2018 22:10:11 +0000 Subject: [PATCH 3/5] fix: code review --- README.md | 4 ++-- package.json | 2 +- src/dht/findpeer.js | 36 +++++++++++------------------------- src/dht/findprovs.js | 26 ++++++++++++++++---------- src/dht/index.js | 4 ++-- src/dht/query.js | 17 +++++++++++------ test/sub-modules.spec.js | 4 ++-- 7 files changed, 45 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index ea7159850..c9e0df980 100644 --- a/README.md +++ b/README.md @@ -259,8 +259,8 @@ const ipfs = ipfsClient({ - [`ipfs.bitswap.unwant(cid)`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/BITSWAP.md#bitswapunwant) - [dht](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/DHT.md) - - [`ipfs.dht.findpeer(peerId, [callback])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/DHT.md#dhtfindpeer) - - [`ipfs.dht.findprovs(hash, [callback])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/DHT.md#dhtfindprovs) + - [`ipfs.dht.findPeer(peerId, [callback])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/DHT.md#dhtfindpeer) + - [`ipfs.dht.findProvs(hash, [callback])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/DHT.md#dhtfindprovs) - [`ipfs.dht.get(key, [callback])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/DHT.md#dhtget) - [`ipfs.dht.provide(cid, [callback])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/DHT.md#dhtprovide) - [`ipfs.dht.put(key, value, [callback])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/DHT.md#dhtput) diff --git a/package.json b/package.json index 911fafc8f..72374523e 100644 --- a/package.json +++ b/package.json @@ -86,7 +86,7 @@ "eslint-plugin-react": "^7.11.1", "go-ipfs-dep": "~0.4.18", "gulp": "^3.9.1", - "interface-ipfs-core": "ipfs/interface-ipfs-core#fix/update-dht-responses", + "interface-ipfs-core": "~0.91.0", "ipfsd-ctl": "~0.40.0", "nock": "^10.0.2", "pull-stream": "^3.6.9", diff --git a/src/dht/findpeer.js b/src/dht/findpeer.js index fa34a7a92..98611468c 100644 --- a/src/dht/findpeer.js +++ b/src/dht/findpeer.js @@ -3,6 +3,9 @@ const promisify = require('promisify-es6') const streamToValueWithTransformer = require('../utils/stream-to-value-with-transformer') +const multiaddr = require('multiaddr') +const PeerId = require('peer-id') +const PeerInfo = require('peer-info') const errcode = require('err-code') module.exports = (send) => { @@ -25,40 +28,23 @@ module.exports = (send) => { res = res[0] } - // Type 2 keys (inconsistencies between go core and js core) - if (res.Type !== 2 && res.type !== 2) { + // Type 2 keys + if (res.Type !== 2) { const errMsg = `key was not found (type 2)` return callback(errcode(new Error(errMsg), 'ERR_KEY_TYPE_2_NOT_FOUND')) } - // inconsistencies between go core and js core - let id - let addrs + const responseReceived = res.Responses[0] + const peerInfo = new PeerInfo(PeerId.createFromB58String(responseReceived.ID)) - if (res.Responses) { - id = res.Responses[0].ID - addrs = res.Responses[0].Addrs - } else { - id = res.responses[0].id - addrs = res.responses[0].addrs - } + responseReceived.Addrs.forEach((addr) => { + const ma = multiaddr(addr) - // inconsistencies js / go - go does not add `/ipfs/{id}` to the address - addrs = addrs.map((addr) => { - if (addr.split('/ipfs/') > -1) { - return addr - } else { - return `${addr}/ipfs/${id}` - } + peerInfo.multiaddrs.add(ma) }) - callback(null, { - responses: [{ - id, - addrs - }] - }) + callback(null, peerInfo) } send({ diff --git a/src/dht/findprovs.js b/src/dht/findprovs.js index 24c946c64..8585b054c 100644 --- a/src/dht/findprovs.js +++ b/src/dht/findprovs.js @@ -3,6 +3,9 @@ const promisify = require('promisify-es6') const streamToValueWithTransformer = require('../utils/stream-to-value-with-transformer') +const multiaddr = require('multiaddr') +const PeerId = require('peer-id') +const PeerInfo = require('peer-info') const errcode = require('err-code') module.exports = (send) => { @@ -25,23 +28,26 @@ module.exports = (send) => { res = res[0] } - // Type 4 keys (inconsistencies between go core and js core) - if (res.Type !== 4 && res.type !== 4) { + // Type 4 keys + if (res.Type !== 4) { const errMsg = `key was not found (type 4)` return callback(errcode(new Error(errMsg), 'ERR_KEY_TYPE_4_NOT_FOUND')) } - // inconsistencies between go core and js core - const recResponses = res.Responses || res.responses + const responses = res.Responses.map((r) => { + const peerInfo = new PeerInfo(PeerId.createFromB58String(r.ID)) - // providers array (handling inconsistencies) - const responses = recResponses.map((r) => ({ - id: r.ID || r.id, - addrs: r.Addrs || r.addrs - })) + r.Addrs.forEach((addr) => { + const ma = multiaddr(addr) - callback(null, { responses }) + peerInfo.multiaddrs.add(ma) + }) + + return peerInfo + }) + + callback(null, responses) } send({ diff --git a/src/dht/index.js b/src/dht/index.js index d3677c49f..b4ab8c640 100644 --- a/src/dht/index.js +++ b/src/dht/index.js @@ -8,8 +8,8 @@ module.exports = (arg) => { return { get: require('./get')(send), put: require('./put')(send), - findprovs: require('./findprovs')(send), - findpeer: require('./findpeer')(send), + findProvs: require('./findprovs')(send), + findPeer: require('./findpeer')(send), provide: require('./provide')(send), // find closest peerId to given peerId query: require('./query')(send) diff --git a/src/dht/query.js b/src/dht/query.js index f1cdb32d0..2dbc62a47 100644 --- a/src/dht/query.js +++ b/src/dht/query.js @@ -1,7 +1,10 @@ 'use strict' const promisify = require('promisify-es6') -const streamToValue = require('../utils/stream-to-value') +const streamToValueWithTransformer = require('../utils/stream-to-value-with-transformer') + +const PeerId = require('peer-id') +const PeerInfo = require('peer-info') module.exports = (send) => { return promisify((peerId, opts, callback) => { @@ -17,6 +20,12 @@ module.exports = (send) => { opts = {} } + const handleResult = (res, callback) => { + const peerIds = res.map((r) => (new PeerInfo(PeerId.createFromB58String(r.ID)))) + + callback(null, peerIds) + } + send({ path: 'dht/query', args: peerId, @@ -26,11 +35,7 @@ module.exports = (send) => { return callback(err) } - if (typeof result.pipe === 'function') { - streamToValue(result, callback) - } else { - callback(null, result) - } + streamToValueWithTransformer(result, handleResult, callback) }) }) } diff --git a/test/sub-modules.spec.js b/test/sub-modules.spec.js index 0401eccfe..1700a1097 100644 --- a/test/sub-modules.spec.js +++ b/test/sub-modules.spec.js @@ -49,8 +49,8 @@ describe('submodules', () => { expect(dht.get).to.be.a('function') expect(dht.put).to.be.a('function') - expect(dht.findprovs).to.be.a('function') - expect(dht.findpeer).to.be.a('function') + expect(dht.findProvs).to.be.a('function') + expect(dht.findPeer).to.be.a('function') expect(dht.provide).to.be.a('function') expect(dht.query).to.be.a('function') }) From a750197960c50221a2171ebab1b2d666f114ea8a Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Tue, 11 Dec 2018 06:48:55 +0000 Subject: [PATCH 4/5] chore: update dependencies License: MIT Signed-off-by: Alan Shaw --- package.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index 224d39882..2a4297a1d 100644 --- a/package.json +++ b/package.json @@ -47,14 +47,14 @@ "is-stream": "^1.1.0", "libp2p-crypto": "~0.14.0", "lodash": "^4.17.11", - "lru-cache": "^4.1.3", - "multiaddr": "^5.0.2", + "lru-cache": "^5.1.1", + "multiaddr": "^6.0.0", "multibase": "~0.6.0", "multihashes": "~0.4.14", "ndjson": "^1.5.0", "once": "^1.4.0", "peer-id": "~0.12.0", - "peer-info": "~0.14.1", + "peer-info": "~0.15.0", "promisify-es6": "^1.0.3", "pull-defer": "~0.2.3", "pull-pushable": "^2.2.0", @@ -66,7 +66,7 @@ "stream-to-pull-stream": "^1.7.2", "streamifier": "~0.1.1", "tar-stream": "^1.6.2", - "through2": "^2.0.3" + "through2": "^3.0.0" }, "engines": { "node": ">=8.0.0", From ca234e5d6de375cc5160016dda21b52322311d5f Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Tue, 11 Dec 2018 07:34:27 +0000 Subject: [PATCH 5/5] chore: fix LRU cache usage License: MIT Signed-off-by: Alan Shaw --- src/object/data.js | 2 +- src/object/get.js | 2 +- src/object/links.js | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/object/data.js b/src/object/data.js index ace3cc992..bbc43ebd6 100644 --- a/src/object/data.js +++ b/src/object/data.js @@ -8,7 +8,7 @@ const lruOptions = { max: 128 } -const cache = LRU(lruOptions) +const cache = new LRU(lruOptions) module.exports = (send) => { return promisify((cid, options, callback) => { diff --git a/src/object/get.js b/src/object/get.js index 28b208003..068d92237 100644 --- a/src/object/get.js +++ b/src/object/get.js @@ -11,7 +11,7 @@ const lruOptions = { max: 128 } -const cache = LRU(lruOptions) +const cache = new LRU(lruOptions) module.exports = (send) => { return promisify((cid, options, callback) => { diff --git a/src/object/links.js b/src/object/links.js index 943509e67..30196c3ec 100644 --- a/src/object/links.js +++ b/src/object/links.js @@ -9,7 +9,7 @@ const lruOptions = { max: 128 } -const cache = LRU(lruOptions) +const cache = new LRU(lruOptions) module.exports = (send) => { return promisify((multihash, options, callback) => {