API update (#80)

* chore: update deps

* feat: return CID on .put

* feat: treeStream

* fix: pick right resolver for the job

* fix: path -> remainderPath

* test: update remaining tests

* chore: update deps

* refactor: .treeStream non recursive

* refactor: tree recursive as pully pull

* revert unneeded change

* tests: fix tests, not assume sorting

README.md

@@ -52,10 +52,12 @@ const Resolver = new Resolver(blockService)
 
 > Store the given node of a recognized IPLD Format.
 
-Options is an object that must contain one of the following combinations:
+`options` is an object that must contain one of the following combinations:
 - `cid` - the CID of the node
 - `hashAlg` and `format` - the hashAlg and the format that should be used to create the CID of the node
 
+`callback` is a function that should have the signature as following: `function (err, cid) {}`, where `err` is an Error object in case of error and `cid` is the cid of the stored object.
+
 ### `.get(cid [, path] [, options], callback)`
 
 > Retrieve a node by the given `cid` or `cid + path`
@@ -73,6 +75,12 @@ Options is an object that must contain one of the following combinations:
 
 > Same as get, but returns a source pull-stream that is used to pass the fetched node.
 
+### `.treeStream(cid [, path] [, options])`
+
+> Returns all the paths under a cid + path through a pull-stream. Accepts the following options:
+
+- `recursive` - bool - traverse through links to complete the graph.
+
 ### `.remove(cid, callback)`
 
 > Remove a node by the given `cid`

package.json

@@ -42,18 +42,18 @@
     "lodash": "^4.17.4",
     "ncp": "^2.0.0",
     "pre-commit": "^1.2.2",
-    "rimraf": "^2.5.4",
+    "rimraf": "^2.6.1",
     "rlp": "^2.0.0"
   },
   "dependencies": {
-    "async": "^2.1.4",
+    "async": "^2.1.5",
     "cids": "~0.4.1",
     "interface-pull-blob-store": "~0.6.0",
-    "ipfs-block": "~0.5.4",
-    "ipfs-block-service": "~0.8.1",
-    "ipfs-repo": "~0.11.2",
-    "ipld-dag-cbor": "~0.9.1",
-    "ipld-dag-pb": "~0.9.5",
+    "ipfs-block": "~0.5.5",
+    "ipfs-block-service": "~0.8.3",
+    "ipfs-repo": "~0.11.3",
+    "ipld-dag-cbor": "~0.10.0",
+    "ipld-dag-pb": "~0.10.0",
     "ipld-eth-block": "^2.2.1",
     "ipld-eth-block-list": "^1.0.3",
     "ipld-eth-state-trie": "^1.0.2",
@@ -62,7 +62,8 @@
     "is-ipfs": "~0.3.0",
     "lodash.flatten": "^4.4.0",
     "lodash.includes": "^4.3.0",
-    "multihashes": "~0.3.3",
+    "multihashes": "~0.4.3",
+    "pull-sort": "^1.0.0",
     "pull-stream": "^3.5.0",
     "pull-traverse": "^1.0.3"
   },
@@ -75,4 +76,4 @@
     "kumavis <[email protected]>",
     "wanderer <[email protected]>"
   ]
-}
+}

src/index.js

@@ -9,6 +9,9 @@ const MemoryStore = require('interface-pull-blob-store')
 const BlockService = require('ipfs-block-service')
 const joinPath = require('path').join
 const pullDeferSource = require('pull-defer').source
+const pullTraverse = require('pull-traverse')
+const map = require('async/map')
+const waterfall = require('async/waterfall')
 
 const dagPB = require('ipld-dag-pb')
 const dagCBOR = require('ipld-dag-cbor')
@@ -18,7 +21,9 @@ const ipldEthTxTrie = require('ipld-eth-tx-trie')
 const ipldEthStateTrie = require('ipld-eth-state-trie')
 const ipldEthStorageTrie = require('ipld-eth-storage-trie')
 
-module.exports = class IPLDResolver {
+function noop () {}
+
+class IPLDResolver {
   constructor (blockService) {
     // nicola will love this!
     if (!blockService) {
@@ -103,7 +108,7 @@ module.exports = class IPLDResolver {
         }
         callback(null, {
           value: node,
-          path: ''
+          remainderPath: ''
         })
       })
     }
@@ -150,7 +155,7 @@ module.exports = class IPLDResolver {
         }
         return callback(null, {
           value: value,
-          path: path
+          remainderPath: path
         })
       }
     )
@@ -213,16 +218,126 @@ module.exports = class IPLDResolver {
 
       pull(
         pull.values([nodeAndCID]),
-        this._putStream(callback)
+        this._putStream((err) => {
+          if (err) {
+            return callback(err)
+          }
+          callback(null, nodeAndCID.cid)
+        })
       )
     }
   }
 
+  treeStream (cid, path, options) {
+    if (typeof path === 'object') {
+      options = path
+      path = undefined
+    }
+
+    options = options || {}
+
+    let p
+
+    if (!options.recursive) {
+      p = pullDeferSource()
+      const r = this.resolvers[cid.codec]
+
+      waterfall([
+        (cb) => this.bs.get(cid, cb),
+        (block, cb) => r.resolver.tree(block, cb)
+      ], (err, paths) => {
+        if (err) {
+          return p.abort(err)
+        }
+        p.resolve(pull.values(paths))
+      })
+    }
+
+    // recursive
+    if (options.recursive) {
+      p = pull(
+        pullTraverse.widthFirst({
+          basePath: null,
+          cid: cid
+        }, (el) => {
+          // pass the paths through the pushable pull stream
+          // continue traversing the graph by returning
+          // the next cids with deferred
+
+          if (typeof el === 'string') {
+            return pull.empty()
+          }
+
+          const deferred = pullDeferSource()
+          const r = this.resolvers[el.cid.codec]
+
+          waterfall([
+            (cb) => this.bs.get(el.cid, cb),
+            (block, cb) => r.resolver.tree(block, (err, paths) => {
+              if (err) {
+                return cb(err)
+              }
+              map(paths, (p, cb) => {
+                r.resolver.isLink(block, p, (err, link) => {
+                  if (err) {
+                    return cb(err)
+                  }
+                  cb(null, {path: p, link: link})
+                })
+              }, cb)
+            })
+          ], (err, paths) => {
+            if (err) {
+              return deferred.abort(err)
+            }
+
+            deferred.resolve(pull.values(paths.map((p) => {
+              const base = el.basePath ? el.basePath + '/' + p.path : p.path
+              if (p.link) {
+                return {
+                  basePath: base,
+                  cid: new CID(p.link['/'])
+                }
+              }
+              return base
+            })))
+          })
+          return deferred
+        }),
+        pull.map((e) => {
+          if (typeof e === 'string') {
+            return e
+          }
+          return e.basePath
+        }),
+        pull.filter(Boolean)
+      )
+    }
+
+    // filter out by path
+    if (path) {
+      return pull(
+        p,
+        pull.map((el) => {
+          if (el.indexOf(path) === 0) {
+            el = el.slice(path.length + 1)
+            return el
+          }
+        }),
+        pull.filter(Boolean)
+      )
+    }
+
+    return p
+  }
+
   remove (cids, callback) {
     this.bs.delete(cids, callback)
   }
 
+  /*           */
   /* internals */
+  /*           */
 
   _get (cid, callback) {
     pull(
@@ -279,4 +394,4 @@ module.exports = class IPLDResolver {
   }
 }
 
-function noop () {}
+module.exports = IPLDResolver

test/ipld-all.js

@@ -68,7 +68,7 @@ describe('IPLD Resolver for dag-cbor + dag-pb', () => {
   })
 
   it('resolve through different formats', (done) => {
-    resolver.get(cidCbor, 'pb/data', (err, result) => {
+    resolver.get(cidCbor, 'pb/Data', (err, result) => {
       expect(err).to.not.exist
       expect(result.value).to.eql(new Buffer('I am inside a Protobuf'))
       done()

test/ipld-dag-cbor.js

@@ -235,10 +235,83 @@ module.exports = (repo) => {
         resolver.get(cid3, 'two/one/someData', (err, result) => {
           expect(err).to.not.exist
           expect(result.value).to.eql('I am 1')
+          expect(result.remainderPath).to.eql('')
+
           done()
         })
       })
 
+      it('resolver.tree', (done) => {
+        pull(
+          resolver.treeStream(cid3),
+          pull.collect((err, values) => {
+            expect(err).to.not.exist
+            expect(values).to.eql([
+              'one',
+              'two',
+              'someData'
+            ])
+            done()
+          })
+        )
+      })
+
+      it('resolver.tree with existent path', (done) => {
+        pull(
+          resolver.treeStream(cid3, 'one'),
+          pull.collect((err, values) => {
+            expect(err).to.not.exist
+            expect(values).to.eql([])
+            done()
+          })
+        )
+      })
+
+      it('resolver.tree with non existent path', (done) => {
+        pull(
+          resolver.treeStream(cid3, 'bananas'),
+          pull.collect((err, values) => {
+            expect(err).to.not.exist
+            expect(values).to.eql([])
+            done()
+          })
+        )
+      })
+
+      it('resolver.tree recursive', (done) => {
+        pull(
+          resolver.treeStream(cid3, { recursive: true }),
+          pull.collect((err, values) => {
+            expect(err).to.not.exist
+            expect(values).to.eql([
+              'one',
+              'two',
+              'someData',
+              'one/someData',
+              'two/one',
+              'two/someData',
+              'two/one/someData'
+            ])
+            done()
+          })
+        )
+      })
+
+      it('resolver.tree with existent path recursive', (done) => {
+        pull(
+          resolver.treeStream(cid3, 'two', { recursive: true }),
+          pull.collect((err, values) => {
+            expect(err).to.not.exist
+            expect(values).to.eql([
+              'one',
+              'someData',
+              'one/someData'
+            ])
+            done()
+          })
+        )
+      })
+
       it('resolver.remove', (done) => {
         resolver.put(node1, { cid: cid1 }, (err) => {
           expect(err).to.not.exist

test/ipld-dag-pb.js

@@ -201,33 +201,33 @@ module.exports = (repo) => {
       })
 
       it('resolver.get value within 1st node scope', (done) => {
-        resolver.get(cid1, 'data', (err, result) => {
+        resolver.get(cid1, 'Data', (err, result) => {
           expect(err).to.not.exist
           expect(result.value).to.eql(new Buffer('I am 1'))
           done()
         })
       })
 
       it('resolver.get value within nested scope (1 level)', (done) => {
-        resolver.get(cid2, 'links/0/data', (err, result) => {
+        resolver.get(cid2, 'Links/0/Hash/Data', (err, result) => {
           expect(err).to.not.exist
           expect(result.value).to.eql(new Buffer('I am 1'))
           done()
         })
       })
 
       it('resolver.get value within nested scope (2 levels)', (done) => {
-        resolver.get(cid3, 'links/1/links/0/data', (err, result) => {
+        resolver.get(cid3, 'Links/1/Hash/Links/0/Hash/Data', (err, result) => {
           expect(err).to.not.exist
           expect(result.value).to.eql(new Buffer('I am 1'))
           done()
         })
       })
 
       it('resolver.get with option localResolve: true', (done) => {
-        resolver.get(cid3, 'links/1/links/0/data', { localResolve: true }, (err, result) => {
+        resolver.get(cid3, 'Links/1/Hash/Links/0/Hash/Data', { localResolve: true }, (err, result) => {
           expect(err).to.not.exist
-          expect(result.path).to.equal('links/0/data')
+          expect(result.remainderPath).to.equal('Links/0/Hash/Data')
           expect(result.value).to.eql({
             '/': 'QmS149H7EbyMuZ2wtEF1sAd7gPwjj4rKAorweAjKMkxr8D'
           })

test/ipld-eth-star.js

@@ -85,7 +85,7 @@ module.exports = (repo) => {
       it('block-to-block', (done) => {
         resolver.get(ethObjs.child.cid, '/parent', (err, result) => {
           expect(err).to.not.exist
-          expect(result.path).to.equal('')
+          expect(result.remainderPath).to.equal('')
           expect(result.value.number.toString('hex')).to.equal('302516')
           done()
         })
@@ -95,7 +95,7 @@ module.exports = (repo) => {
         resolver.get(ethObjs.child.cid, '/parent/state/0/0/0/0/1/7/nonce', (err, result) => {
           expect(err).to.not.exist
           expect(result.value.toString('hex'), '03')
-          expect(result.path).to.equal('')
+          expect(result.remainderPath).to.equal('')
           done()
         })
       })