API update

README.md

@@ -52,10 +52,12 @@ const Resolver = new Resolver(blockService)
 
 > Store the given node of a recognized IPLD Format.
 
-Options is an object that must contain one of the following combinations:
+`options` is an object that must contain one of the following combinations:
 - `cid` - the CID of the node
 - `hashAlg` and `format` - the hashAlg and the format that should be used to create the CID of the node
 
+`callback` is a function that should have the signature as following: `function (err, cid) {}`, where `err` is an Error object in case of error and `cid` is the cid of the stored object.
+
 ### `.get(cid [, path] [, options], callback)`
 
 > Retrieve a node by the given `cid` or `cid + path`
@@ -73,6 +75,12 @@ Options is an object that must contain one of the following combinations:
 
 > Same as get, but returns a source pull-stream that is used to pass the fetched node.
 
+### `.treeStream(cid [, path] [, options])`
+
+> Returns all the paths under a cid + path through a pull-stream. Accepts the following options:
+
+- `recursive` - bool - traverse through links to complete the graph.
+
 ### `.remove(cid, callback)`
 
 > Remove a node by the given `cid`

package.json

@@ -42,16 +42,16 @@
     "lodash": "^4.17.4",
     "ncp": "^2.0.0",
     "pre-commit": "^1.2.2",
-    "rimraf": "^2.5.4",
+    "rimraf": "^2.6.1",
     "rlp": "^2.0.0"
   },
   "dependencies": {
-    "async": "^2.1.4",
+    "async": "^2.1.5",
     "cids": "~0.4.1",
     "interface-pull-blob-store": "~0.6.0",
-    "ipfs-block": "~0.5.4",
-    "ipfs-block-service": "~0.8.1",
-    "ipfs-repo": "~0.11.2",
+    "ipfs-block": "~0.5.5",
+    "ipfs-block-service": "~0.8.3",
+    "ipfs-repo": "~0.11.3",
     "ipld-dag-cbor": "~0.9.1",
     "ipld-dag-pb": "~0.9.5",
     "ipld-eth-block": "^2.2.1",
@@ -62,7 +62,8 @@
     "is-ipfs": "~0.3.0",
     "lodash.flatten": "^4.4.0",
     "lodash.includes": "^4.3.0",
-    "multihashes": "~0.3.3",
+    "multihashes": "~0.4.3",
+    "pull-sort": "^1.0.0",
     "pull-stream": "^3.5.0",
     "pull-traverse": "^1.0.3"
   },
@@ -75,4 +76,4 @@
     "kumavis <[email protected]>",
     "wanderer <[email protected]>"
   ]
-}
+}

src/index.js

@@ -2,13 +2,17 @@
 
 const Block = require('ipfs-block')
 const pull = require('pull-stream')
+const pullPushable = require('pull-pushable')
 const CID = require('cids')
 const doUntil = require('async/doUntil')
 const IPFSRepo = require('ipfs-repo')
 const MemoryStore = require('interface-pull-blob-store')
 const BlockService = require('ipfs-block-service')
 const joinPath = require('path').join
 const pullDeferSource = require('pull-defer').source
+const pullTraverse = require('pull-traverse')
+const asyncEach = require('async/each')
+const pullSort = require('pull-sort')
 
 const dagPB = require('ipld-dag-pb')
 const dagCBOR = require('ipld-dag-cbor')
@@ -18,7 +22,9 @@ const ipldEthTxTrie = require('ipld-eth-tx-trie')
 const ipldEthStateTrie = require('ipld-eth-state-trie')
 const ipldEthStorageTrie = require('ipld-eth-storage-trie')
 
-module.exports = class IPLDResolver {
+function noop () {}
+
+class IPLDResolver {
   constructor (blockService) {
     // nicola will love this!
     if (!blockService) {
@@ -213,16 +219,137 @@ module.exports = class IPLDResolver {
 
       pull(
         pull.values([nodeAndCID]),
-        this._putStream(callback)
+        this._putStream((err) => {
+          if (err) {
+            return callback(err)
+          }
+          callback(null, nodeAndCID.cid)
+        })
+      )
+    }
+  }
+
+  treeStream (cid, path, options) {
+    if (typeof path === 'object') {
+      options = path
+      path = undefined
+    }
+
+    options = options || {}
+
+    // non recursive
+    const p = pullPushable()
+
+    if (!options.recursive) {
+      const r = this.resolvers[cid.codec]
+
+      this.bs.get(cid, (err, block) => {
+        if (err) {
+          return p(err)
+        }
+
+        r.resolver.tree(block, (err, paths) => {
+          if (err) {
+            return p(err)
+          }
+          paths.forEach((path) => p.push(path))
+          p.end()
+        })
+      })
+    }
+
+    // recursive
+    if (options.recursive) {
+      pull(
+        pullTraverse.widthFirst({ basePath: null, cid: cid }, (el) => {
+          // pass the paths through the pushable pull stream
+          // continue traversing the graph by returning
+          // the next cids with deferred
+
+          const deferred = pullDeferSource()
+
+          this.bs.get(el.cid, (err, block) => {
+            if (err) {
+              return p(err)
+            }
+
+            const r = this.resolvers[el.cid.codec]
+
+            r.resolver.tree(block, (err, paths) => {
+              if (err) {
+                p(err)
+                return deferred.resolve(pull.empty())
+              }
+
+              const next = []
+
+              asyncEach(paths, (path, cb) => {
+                r.resolver.isLink(block, path, (err, link) => {
+                  if (err) {
+                    return cb(err)
+                  }
+
+                  p.push(el.basePath
+                    ? el.basePath + '/' + path
+                    : path
+                  )
+
+                  // if it is a link, continue traversing
+                  if (link) {
+                    next.push({
+                      basePath: el.basePath
+                        ? el.basePath + '/' + path
+                        : path,
+                      cid: new CID(link['/'])
+                    })
+                  }
+                  cb()
+                })
+              }, (err) => {
+                if (err) {
+                  p(err)
+                  return deferred.resolve(pull.empty())
+                }
+
+                deferred.resolve(pull.values(next))
+              })
+            })
+          })
+
+          return deferred
+        }),
+        pull.onEnd(() => p.end())
+      )
+    }
+
+    // filter out by path
+    if (path) {
+      return pull(
+        p,
+        pull.map((el) => {
+          if (el.indexOf(path) === 0) {
+            el = el.slice(path.length + 1)
+            return el
+          }
+        }),
+        pull.filter((el) => el && el.length > 0),
+        pullSort((a, b) => a.localeCompare(b))
       )
     }
+
+    return pull(
+      p,
+      pullSort((a, b) => a.localeCompare(b))
+    )
   }
 
   remove (cids, callback) {
     this.bs.delete(cids, callback)
   }
 
+  /*           */
   /* internals */
+  /*           */
 
   _get (cid, callback) {
     pull(
@@ -279,4 +406,4 @@ module.exports = class IPLDResolver {
   }
 }
 
-function noop () {}
+module.exports = IPLDResolver

test/ipld-dag-cbor.js

@@ -113,7 +113,7 @@ module.exports = (repo) => {
       })
     })
 
-    describe('public api', () => {
+    describe.only('public api', () => {
       it('resolver.put with CID', (done) => {
         resolver.put(node1, { cid: cid1 }, done)
       })
@@ -239,6 +239,77 @@ module.exports = (repo) => {
         })
       })
 
+      it('resolver.tree', (done) => {
+        pull(
+          resolver.treeStream(cid3),
+          pull.collect((err, values) => {
+            expect(err).to.not.exist
+            expect(values).to.eql([
+              'one',
+              'someData',
+              'two'
+            ])
+            done()
+          })
+        )
+      })
+
+      it('resolver.tree with existent path', (done) => {
+        pull(
+          resolver.treeStream(cid3, 'one'),
+          pull.collect((err, values) => {
+            expect(err).to.not.exist
+            expect(values).to.eql([])
+            done()
+          })
+        )
+      })
+
+      it('resolver.tree with non existent path', (done) => {
+        pull(
+          resolver.treeStream(cid3, 'bananas'),
+          pull.collect((err, values) => {
+            expect(err).to.not.exist
+            expect(values).to.eql([])
+            done()
+          })
+        )
+      })
+
+      it('resolver.tree recursive', (done) => {
+        pull(
+          resolver.treeStream(cid3, { recursive: true }),
+          pull.collect((err, values) => {
+            expect(err).to.not.exist
+            expect(values).to.eql([
+              'one',
+              'one/someData',
+              'someData',
+              'two',
+              'two/one',
+              'two/one/someData',
+              'two/someData'
+            ])
+            done()
+          })
+        )
+      })
+
+      it('resolver.tree with existent path recursive', (done) => {
+        pull(
+          resolver.treeStream(cid3, 'two', { recursive: true }),
+          pull.collect((err, values) => {
+            expect(err).to.not.exist
+            expect(values).to.eql([
+              'one',
+              'one/someData',
+              'someData'
+            ])
+            done()
+          })
+        )
+      })
+
       it('resolver.remove', (done) => {
         resolver.put(node1, { cid: cid1 }, (err) => {
           expect(err).to.not.exist