From 9ab49eebdc7725498415c90659edf74d54dec191 Mon Sep 17 00:00:00 2001 From: AdamStone Date: Wed, 30 Mar 2016 20:12:10 -0700 Subject: [PATCH] Pinning core, cli and http --- package.json | 1 + src/cli/commands/pin.js | 15 ++ src/cli/commands/pin/add.js | 76 ++++++++ src/cli/commands/pin/ls.js | 133 +++++++++++++ src/cli/commands/pin/rm.js | 81 ++++++++ src/core/index.js | 2 + src/core/ipfs/pinner-utils.js | 268 +++++++++++++++++++++++++ src/core/ipfs/pinner.js | 324 +++++++++++++++++++++++++++++++ src/http-api/resources/index.js | 1 + src/http-api/resources/pin.js | 206 ++++++++++++++++++++ src/http-api/routes/index.js | 1 + src/http-api/routes/pin.js | 37 ++++ test/cli/test-commands.js | 2 +- test/cli/test-pin.js | 153 +++++++++++++++ test/core/both/test-pin.js | 267 +++++++++++++++++++++++++ test/http-api/test-pin.js | 160 +++++++++++++++ test/test-data/tree/branch.json | 1 + test/test-data/tree/leaf.json | 1 + test/test-data/tree/root.json | 1 + test/test-data/tree/subLeaf.json | 1 + 20 files changed, 1730 insertions(+), 1 deletion(-) create mode 100644 src/cli/commands/pin.js create mode 100644 src/cli/commands/pin/add.js create mode 100644 src/cli/commands/pin/ls.js create mode 100644 src/cli/commands/pin/rm.js create mode 100644 src/core/ipfs/pinner-utils.js create mode 100644 src/core/ipfs/pinner.js create mode 100644 src/http-api/resources/pin.js create mode 100644 src/http-api/routes/pin.js create mode 100644 test/cli/test-pin.js create mode 100644 test/core/both/test-pin.js create mode 100644 test/http-api/test-pin.js create mode 100644 test/test-data/tree/branch.json create mode 100644 test/test-data/tree/leaf.json create mode 100644 test/test-data/tree/root.json create mode 100644 test/test-data/tree/subLeaf.json diff --git a/package.json b/package.json index c110eb5020..6a79f585c9 100644 --- a/package.json +++ b/package.json @@ -64,6 +64,7 @@ "bs58": "^3.0.0", "debug": "^2.2.0", "detect-node": "^2.0.3", + "fnv1a": "^1.0.1", "fs-blob-store": "^5.2.1", "glob": "^7.0.5", "hapi": "^14.0.0", diff --git a/src/cli/commands/pin.js b/src/cli/commands/pin.js new file mode 100644 index 0000000000..d7a68fb023 --- /dev/null +++ b/src/cli/commands/pin.js @@ -0,0 +1,15 @@ +'use strict' + +module.exports = { + command: 'pin', + + description: 'Pin and unpin objects to local storage.', + + builder (yargs) { + return yargs + .commandDir('pin') + }, + + handler (argv) { + } +} diff --git a/src/cli/commands/pin/add.js b/src/cli/commands/pin/add.js new file mode 100644 index 0000000000..01dfe1d423 --- /dev/null +++ b/src/cli/commands/pin/add.js @@ -0,0 +1,76 @@ +'use strict' + +const utils = require('../../utils') +const bs58 = require('bs58') +const debug = require('debug') +const log = debug('cli:pin') +log.error = debug('cli:pin:error') + +const onError = (err) => { + if (err) { + console.error(err) + throw err + } +} + +module.exports = { + command: 'add ', + + describe: 'Pins objects to local storage.', + + builder: { + recursive: { + type: 'boolean', + alias: 'r', + default: true, + describe: 'Recursively pin the object linked to by the specified object(s).' + } + }, + + handler (argv) { + const path = argv['ipfs-path'] + const recursive = argv.recursive + utils.getIPFS((err, ipfs) => { + onError(err) + // load persistent pin set from datastore + ipfs.pinner.load(() => { + const matched = path.match(/^(?:\/ipfs\/)?([^\/]+(?:\/[^\/]+)*)\/?$/) + if (!matched) { + onError(new Error('invalid ipfs ref path')) + } + const split = matched[1].split('/') + const rootHash = split[0] + const key = new Buffer(bs58.decode(rootHash)) + const links = split.slice(1, split.length) + const pathFn = (err, obj) => { + onError(err) + if (links.length) { + const linkName = links.shift() + const nextLink = obj.links.filter((link) => { + return (link.name === linkName) + }) + if (!nextLink.length) { + onError(new Error( + 'pin: no link named ' + linkName + + ' under ' + obj.toJSON().Hash + )) + } + const nextHash = nextLink[0].hash + ipfs.object.get(nextHash, pathFn) + } else { + ipfs.pinner.pin(obj, recursive, (err) => { + onError(err) + // save modified pin state to datastore + ipfs.pinner.flush((err, root) => { + onError(err) + const mode = recursive ? ' recursively' : ' directly' + console.log('pinned ' + obj.toJSON().Hash + mode) + }) + }) + } + } + ipfs.object.get(key, pathFn) + }) + }) + } +} diff --git a/src/cli/commands/pin/ls.js b/src/cli/commands/pin/ls.js new file mode 100644 index 0000000000..1c9a0475f7 --- /dev/null +++ b/src/cli/commands/pin/ls.js @@ -0,0 +1,133 @@ +'use strict' + +const utils = require('../../utils') +const bs58 = require('bs58') +const debug = require('debug') +const log = debug('cli:pin') +log.error = debug('cli:pin:error') + +const onError = (err) => { + if (err) { + console.error(err) + throw err + } +} + +module.exports = { + command: 'ls', + + describe: 'List objects pinned to local storage.', + + builder: { + path: { + type: 'string', + describe: 'List pinned state of specific .' + }, + type: { + type: 'string', + alias: 't', + default: 'all', + describe: ('The type of pinned keys to list. ' + + 'Can be "direct", "indirect", "recursive", or "all".') + }, + quiet: { + type: 'boolean', + alias: 'q', + default: false, + describe: 'Write just hashes of objects.' + } + }, + + handler: (argv) => { + const path = argv.path + const type = argv.type + const quiet = argv.quiet + utils.getIPFS((err, ipfs) => { + onError(err) + const types = ipfs.pinner.types + // load persistent pin set from datastore + ipfs.pinner.load(() => { + if (path) { + const matched = path.match(/^(?:\/ipfs\/)?([^\/]+(?:\/[^\/]+)*)\/?$/) + if (!matched) { + onError(new Error('invalid ipfs ref path')) + } + const split = matched[1].split('/') + const rootHash = split[0] + const key = new Buffer(bs58.decode(rootHash)) + const links = split.slice(1, split.length) + const pathFn = (err, obj) => { + onError(err) + if (links.length) { + const linkName = links.shift() + const nextLink = obj.links.filter((link) => { + return (link.name === linkName) + }) + if (!nextLink.length) { + onError(new Error( + 'pin: no link named ' + linkName + + ' under ' + obj.toJSON().Hash + )) + } + const nextHash = nextLink[0].hash + ipfs.object.get(nextHash, pathFn) + } else { + ipfs.pinner.isPinnedWithType(obj.multihash(), type, (err, pinned, reason) => { + onError(err) + if (!pinned) { + onError(new Error('Path ' + path + ' is not pinned')) + } + if (reason !== types.direct && + reason !== types.recursive) { + reason = 'indirect through ' + reason + } + console.log(obj.toJSON().Hash + (quiet ? '' : ' ' + reason)) + }) + } + } + ipfs.object.get(key, pathFn) + } else { + const printDirect = () => { + ipfs.pinner.directKeyStrings().forEach((key) => { + console.log(key + (quiet ? '' : ' direct')) + }) + } + const printRecursive = () => { + ipfs.pinner.recursiveKeyStrings().forEach((key) => { + console.log(key + (quiet ? '' : ' recursive')) + }) + } + const printIndirect = () => { + ipfs.pinner.getIndirectKeys((err, keys) => { + onError(err) + keys.forEach((key) => { + console.log(key + (quiet ? '' : ' indirect')) + }) + }) + } + switch (type) { + case types.direct: + printDirect() + break + case types.recursive: + printRecursive() + break + case types.indirect: + printIndirect() + break + case types.all: + printDirect() + printRecursive() + printIndirect() + break + default: + onError(new Error( + "Invalid type '" + type + "', " + + 'must be one of {direct, indirect, recursive, all}' + )) + } + } + }) + }) + } +} diff --git a/src/cli/commands/pin/rm.js b/src/cli/commands/pin/rm.js new file mode 100644 index 0000000000..0a66b842e3 --- /dev/null +++ b/src/cli/commands/pin/rm.js @@ -0,0 +1,81 @@ +'use strict' + +const utils = require('../../utils') +const bs58 = require('bs58') +const debug = require('debug') +const log = debug('cli:pin') +log.error = debug('cli:pin:error') + +const onError = (err) => { + if (err) { + console.error(err) + throw err + } +} + +module.exports = { + command: 'rm ', + + describe: 'Removes the pinned object from local storage.', + + builder: { + recursive: { + type: 'boolean', + alias: 'r', + default: true, + describe: 'Recursively unpin the objects linked to by the specified object(s).' + } + }, + + handler: (argv) => { + const path = argv['ipfs-path'] + const recursive = argv.recursive + utils.getIPFS((err, ipfs) => { + onError(err) + // load persistent pin set from datastore + ipfs.pinner.load(() => { + const matched = path.match(/^(?:\/ipfs\/)?([^\/]+(?:\/[^\/]+)*)\/?$/) + if (!matched) { + onError(new Error('invalid ipfs ref path')) + } + const split = matched[1].split('/') + const rootHash = split[0] + const key = new Buffer(bs58.decode(rootHash)) + const links = split.slice(1, split.length) + const pathFn = (err, obj) => { + onError(err) + if (links.length) { + const linkName = links.shift() + const nextLink = obj.links.filter((link) => { + return (link.name === linkName) + }) + if (!nextLink.length) { + onError(new Error( + 'pin: no link named ' + linkName + + ' under ' + obj.toJSON().Hash + )) + } + const nextHash = nextLink[0].hash + ipfs.object.get(nextHash, pathFn) + } else { + ipfs.pinner.isPinned(obj.multihash(), (err, pinned, reason) => { + onError(err) + if (!pinned) { + onError(new Error('not pinned')) + } + ipfs.pinner.unpin(obj.multihash(), recursive, (err) => { + onError(err) + // save modified pin state to datastore + ipfs.pinner.flush((err, root) => { + onError(err) + console.log('unpinned ' + obj.toJSON().Hash) + }) + }) + }) + } + } + ipfs.object.get(key, pathFn) + }) + }) + } +} diff --git a/src/core/index.js b/src/core/index.js index 4ac3b388d0..1f932c088f 100644 --- a/src/core/index.js +++ b/src/core/index.js @@ -17,6 +17,7 @@ const repo = require('./ipfs/repo') const init = require('./ipfs/init') const bootstrap = require('./ipfs/bootstrap') const config = require('./ipfs/config') +const pinner = require('./ipfs/pinner') const block = require('./ipfs/block') const object = require('./ipfs/object') const libp2p = require('./ipfs/libp2p') @@ -53,6 +54,7 @@ function IPFS (repoInstance) { this.init = init(this) this.bootstrap = bootstrap(this) this.config = config(this) + this.pinner = pinner(this) this.block = block(this) this.object = object(this) this.libp2p = libp2p(this) diff --git a/src/core/ipfs/pinner-utils.js b/src/core/ipfs/pinner-utils.js new file mode 100644 index 0000000000..ef33922a7e --- /dev/null +++ b/src/core/ipfs/pinner-utils.js @@ -0,0 +1,268 @@ +'use strict' + +const bs58 = require('bs58') +const protobuf = require('protocol-buffers') +const crypto = require('crypto') +const fnv1a = require('fnv1a') +const mDAG = require('ipfs-merkle-dag') +const DAGNode = mDAG.DAGNode +const DAGLink = mDAG.DAGLink +const varint = require('varint') + +const emptyKeyHash = 'QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n' +const emptyKey = new Buffer(bs58.decode(emptyKeyHash)) +const defaultFanout = 256 +const maxItems = 8192 + +// Protobuf interface +const pbSchema = ( + // from go-ipfs/pin/internal/pb/header.proto + 'message Set { ' + + // 1 for now + 'optional uint32 version = 1; ' + + // how many of the links are subtrees + 'optional uint32 fanout = 2; ' + + // hash seed for subtree selection, a random number + 'optional fixed32 seed = 3; ' + + '}' +) +const pb = protobuf(pbSchema) +function readHeader (rootNode) { + // rootNode.data should be a buffer of the format: + // < varint(headerLength) | header | itemData... > + const rootData = rootNode.data + const hdrLength = varint.decode(rootData) + const vBytes = varint.decode.bytes + if (vBytes <= 0) { + return { err: 'Invalid Set header length' } + } + if (vBytes + hdrLength > rootData.length) { + return { err: 'Impossibly large set header length' } + } + const hdrSlice = rootData.slice(vBytes, hdrLength + vBytes) + const header = pb.Set.decode(hdrSlice) + if (header.version !== 1) { + return { err: 'Unsupported Set version: ' + header.version } + } + if (header.fanout > rootNode.links.length) { + return { err: 'Impossibly large fanout' } + } + return { + header: header, + data: rootData.slice(hdrLength + vBytes) + } +} + +exports = module.exports = function (dagS) { + const pinnerUtils = { + // should this be part of `object` rather than `pinner`? + hasChild: (root, childhash, callback, _links, _checked, _seen) => { + // callback (err, has) + if (callback.fired) { return } + if (typeof childhash === 'object') { + childhash = bs58.encode(childhash).toString() + } + _links = _links || root.links.length + _checked = _checked || 0 + _seen = _seen || {} + + if (!root.links.length && _links === _checked) { + // all nodes have been checked + return callback(null, false) + } + root.links.forEach((link) => { + const bs58link = bs58.encode(link.hash).toString() + if (bs58link === childhash) { + callback.fired = true + return callback(null, true) + } + dagS.get(link.hash, (err, obj) => { + if (err) { + callback.fired = true + return callback(err) + } + // don't check the same links twice + if (bs58link in _seen) { return } + _seen[bs58link] = true + + _checked++ + _links += obj.links.length + pinnerUtils.hasChild(obj, childhash, callback, _links, _checked, _seen) + }) + }) + }, + + storeSet: (keys, logInternalKey, callback) => { + // callback (err, rootNode) + const items = keys.map((key) => { + return { + key: key, + data: null + } + }) + pinnerUtils.storeItems(items, logInternalKey, (err, rootNode) => { + if (err) { return callback(err) } + dagS.add(rootNode, (err) => { + if (err) { return callback(err) } + logInternalKey(rootNode.multihash()) + callback(null, rootNode) + }) + }) + }, + + storeItems: (items, logInternalKey, callback, _subcalls, _done) => { + // callback (err, rootNode) + const seed = crypto.randomBytes(4).readUInt32LE(0, true) + const pbHeader = pb.Set.encode({ + version: 1, + fanout: defaultFanout, + seed: seed + }) + let rootData = Buffer.concat([ + new Buffer(varint.encode(pbHeader.length)), pbHeader + ]) + let rootLinks = [] + for (let i = 0; i < defaultFanout; i++) { + rootLinks.push(new DAGLink('', 0, emptyKey)) + } + logInternalKey(emptyKey) + + if (items.length <= maxItems) { + // the items will fit in a single root node + const itemLinks = [] + const itemData = [] + const indices = [] + for (let i = 0; i < items.length; i++) { + itemLinks.push(new DAGLink('', 0, items[i].key)) + itemData.push(items[i].data || new Buffer([])) + indices.push(i) + } + indices.sort((a, b) => { + const x = Buffer.compare(itemLinks[a].hash, itemLinks[b].hash) + if (x) { return x } + return (a < b ? -1 : 1) + }) + const sortedLinks = indices.map((i) => { return itemLinks[i] }) + const sortedData = indices.map((i) => { return itemData[i] }) + rootLinks = rootLinks.concat(sortedLinks) + rootData = Buffer.concat([rootData].concat(sortedData)) + readHeader(new DAGNode(rootData, rootLinks)) + return callback(null, new DAGNode(rootData, rootLinks)) + } else { + // need to split up the items into multiple root nodes + // (using go-ipfs "wasteful but simple" approach for consistency) + _subcalls = _subcalls || 0 + _done = _done || 0 + const hashed = {} + const hashFn = (seed, key) => { + const buf = new Buffer(4) + buf.writeUInt32LE(seed, 0) + const data = Buffer.concat([ + buf, new Buffer(bs58.encode(key).toString()) + ]) + return fnv1a(data.toString('binary')) + } + // items will be distributed among `defaultFanout` bins + for (let i = 0; i < items.length; i++) { + let h = hashFn(seed, items[i].key) % defaultFanout + hashed[h] = hashed[h] || [] + hashed[h].push(items[i]) + } + const storeItemsCb = (err, child) => { + if (callback.fired) { return } + if (err) { + callback.fired = true + return callback(err) + } + dagS.add(child, (err) => { + if (callback.fired) { return } + if (err) { + callback.fired = true + return callback(err) + } + logInternalKey(child.multihash()) + rootLinks[this.h] = new DAGLink( + '', child.size(), child.multihash() + ) + _done++ + if (_done === _subcalls) { + // all finished + return callback(null, new DAGNode(rootData, rootLinks)) + } + }) + } + _subcalls += Object.keys(hashed).length + for (let h in hashed) { + if (hashed.hasOwnProperty(h)) { + pinnerUtils.storeItems( + hashed[h], + logInternalKey, + storeItemsCb.bind({h: h}), + _subcalls, + _done + ) + } + } + } + }, + + loadSet: (rootNode, name, logInternalKey, callback) => { + // callback (err, keys) + const link = rootNode.links.filter((link) => { + return link.name === name + }).pop() + if (!link) { return callback('No link found with name ' + name) } + logInternalKey(link.hash) + dagS.get(link.hash, (err, obj) => { + if (err) { return callback(err) } + const keys = [] + const walkerFn = (link) => { + keys.push(link.hash) + } + pinnerUtils.walkItems(obj, walkerFn, logInternalKey, (err) => { + if (err) { return callback(err) } + return callback(null, keys) + }) + }) + }, + + walkItems: (node, walkerFn, logInternalKey, callback) => { + // callback (err) + const h = readHeader(node) + if (h.err) { return callback(h.err) } + const fanout = h.header.fanout + let subwalkCount = 0 + let finishedCount = 0 + + const walkCb = (err) => { + if (err) { return callback(err) } + finishedCount++ + if (subwalkCount === finishedCount) { + return callback() + } + } + + for (let i = 0; i < node.links.length; i++) { + const link = node.links[i] + if (i >= fanout) { + // item link + walkerFn(link, i, h.data) + } else { + // fanout link + logInternalKey(link.hash) + if (!emptyKey.equals(link.hash)) { + subwalkCount++ + dagS.get(link.hash, (err, obj) => { + if (err) { return callback(err) } + pinnerUtils.walkItems(obj, walkerFn, logInternalKey, walkCb) + }) + } + } + } + if (!subwalkCount) { + return callback() + } + } + } + return pinnerUtils +} diff --git a/src/core/ipfs/pinner.js b/src/core/ipfs/pinner.js new file mode 100644 index 0000000000..4b261bc95e --- /dev/null +++ b/src/core/ipfs/pinner.js @@ -0,0 +1,324 @@ +'use strict' + +const bs58 = require('bs58') +const mDAG = require('ipfs-merkle-dag') +const DAGNode = mDAG.DAGNode +const pinnerUtils = require('./pinner-utils') + +function keyString (key) { + return bs58.encode(key).toString() +} + +function KeySet (keys) { + // Buffers with identical data are still different objects, so + // they need to be cast to strings to prevent duplicates in Sets + this.keys = {} + this.add = (key) => { + this.keys[keyString(key)] = key + } + this.delete = (key) => { + delete this.keys[keyString(key)] + } + this.clear = () => { + this.keys = {} + } + this.has = (key) => { + return (keyString(key) in this.keys) + } + this.toArray = () => { + return Object.keys(this.keys).map((hash) => { + return this.keys[hash] + }) + } + this.toStringArray = () => { + return Object.keys(this.keys) + } + keys = keys || [] + keys.forEach(this.add) +} + +module.exports = function (self) { + let directPins = new KeySet() + let recursivePins = new KeySet() + let internalPins = new KeySet() + + // repo.datastore makes a subfolder using first 8 chars of key, so + // pin data will be saved under /blocks/internal/internal_pins.data + const pinDataStoreKey = 'internal_pins' + + const repo = self._repo + const dagS = self._dagS + + const pinner = { + types: { + direct: 'direct', + recursive: 'recursive', + indirect: 'indirect', + internal: 'internal', + all: 'all' + }, + + clear: () => { + directPins.clear() + recursivePins.clear() + internalPins.clear() + }, + + utils: pinnerUtils(dagS), + + pin: (obj, recursive, callback) => { + // callback (err) + if (typeof recursive === 'function') { + callback = recursive + recursive = true + } + const multihash = obj.multihash() + if (recursive) { + if (recursivePins.has(multihash)) { + return callback(null) + } + directPins.delete(multihash) + dagS.getRecursive(multihash, (err, objs) => { + if (err) { + return callback(err) + } + recursivePins.add(multihash) + return callback(null) + }) + } else { + dagS.get(multihash, (err, obj) => { + if (err) { + return callback(err) + } + if (recursivePins.has(multihash)) { + return callback(keyString(multihash) + ' already pinned recursively') + } + directPins.add(multihash) + return callback(null) + }) + } + }, + + unpin: (multihash, recursive, callback) => { + // callback (err) + if (typeof recursive === 'function') { + callback = recursive + recursive = true + } + pinner.isPinnedWithType(multihash, pinner.types.all, (err, pinned, reason) => { + if (err) { return callback(err) } + if (!pinned) { return callback(new Error('not pinned')) } + switch (reason) { + case (pinner.types.recursive): + if (recursive) { + recursivePins.delete(multihash) + return callback(null) + } + return callback(new Error(keyString(multihash) + ' is pinned recursively')) + case (pinner.types.direct): + directPins.delete(multihash) + return callback(null) + default: + return callback(new Error(keyString(multihash) + + ' is pinned indirectly under ' + reason)) + } + }) + }, + + isPinned: (multihash, callback) => { + // callback (err, pinned, reason) + pinner.isPinnedWithType(multihash, pinner.types.all, callback) + }, + + isPinnedWithType: (multihash, pinType, callback) => { + // callback (err, pinned, reason) + + // recursive + if ((pinType === pinner.types.recursive || pinType === pinner.types.all) && + recursivePins.has(multihash)) { + return callback(null, true, pinner.types.recursive) + } + if ((pinType === pinner.types.recursive)) { + return callback(null, false) + } + // direct + if ((pinType === pinner.types.direct || pinType === pinner.types.all) && + directPins.has(multihash)) { + return callback(null, true, pinner.types.direct) + } + if ((pinType === pinner.types.direct)) { + return callback(null, false) + } + if ((pinType === pinner.types.internal || pinType === pinner.types.all) && + internalPins.has(multihash)) { + return callback(null, true, pinner.types.internal) + } + if ((pinType === pinner.types.internal)) { + return callback(null, false) + } + + // indirect (default) + let checkedCount = 0 + const cbs = [callback] + const done = () => { + // flag pending callbacks to break early when result found + cbs.forEach((cb) => { + cb.fired = true + }) + } + const rkeys = pinner.recursiveKeys() + if (!rkeys.length) { + return callback(null, false) + } + rkeys.forEach((rkey) => { + dagS.get(rkey, (err, obj) => { + if (callback.fired) { return } + if (err) { + done() + return callback(err) + } + const thisCb = (err, has) => { + if (err) { + done() + return callback(err) + } + if (has) { + done() + return callback( + null, true, keyString(obj.multihash()) + ) + } else { + checkedCount++ + if (checkedCount === rkeys.length) { + done() + return callback(null, false) + } + } + } + cbs.push(thisCb) + pinner.utils.hasChild(obj, multihash, thisCb) + }) + }) + }, + + directKeys: () => { + return directPins.toArray() + }, + + directKeyStrings: () => { + return directPins.toStringArray() + }, + + recursiveKeys: () => { + return recursivePins.toArray() + }, + + recursiveKeyStrings: () => { + return recursivePins.toStringArray() + }, + + getIndirectKeys: (callback) => { + // callback (err, keys) + const indirectKeys = new KeySet() + const rKeys = pinner.recursiveKeys() + if (!rKeys.length) { return callback(null, []) } + let doneCount = 0 + rKeys.forEach((multihash) => { + dagS.getRecursive(multihash, (err, objs) => { + if (callback.fired) { return } + if (err) { + callback.fired = true + return callback(err) + } + objs.forEach((obj) => { + const mh = obj.multihash() + if (!directPins.has(mh) && !recursivePins.has(mh)) { + // not already pinned recursively or directly + indirectKeys.add(mh) + } + }) + if (doneCount === rKeys.length - 1) { + return callback(null, indirectKeys.toStringArray()) + } + doneCount++ + }) + }) + }, + + internalKeys: () => { + return internalPins.toArray() + }, + + internalKeyStrings: () => { + return internalPins.toStringArray() + }, + + // encodes and writes pinner key sets to the datastore + flush: (callback) => { + // callback (err, root) + const newInternalPins = new KeySet() + const logInternalKey = (multihash) => { + newInternalPins.add(multihash) + } + // each key set will be stored as a DAG node, and root will link to both + const root = new DAGNode() + pinner.utils.storeSet(pinner.directKeys(), logInternalKey, (err, dRoot) => { + if (err) { return callback(err) } + root.addNodeLink(pinner.types.direct, dRoot) + pinner.utils.storeSet(pinner.recursiveKeys(), logInternalKey, (err, rRoot) => { + if (err) { return callback(err) } + root.addNodeLink(pinner.types.recursive, rRoot) + // the set nodes link to an empty node, so make sure it's added + dagS.add(new DAGNode(), (err) => { + if (err) { return callback(err) } + // then add the root node to dagS + dagS.add(root, (err) => { + if (err) { return callback(err) } + // update pinner's internal pin set + logInternalKey(root.multihash()) + internalPins = newInternalPins + // save a reference to root hash under a consistent key + const pseudoblock = { + data: root.marshal(), + key: pinDataStoreKey, + extension: null + } + repo.datastore.put(pseudoblock, (err, metadata) => { + if (err) { return callback(err) } + return callback(null, root) + }) + }) + }) + }) + }) + }, + + load: (callback) => { + // callback (err) + repo.datastore.get(pinDataStoreKey, (err, pseudoblock) => { + if (err) { return callback(err) } + const rootBytes = pseudoblock.data + const root = (new DAGNode()).unMarshal(rootBytes) + const newInternalPins = new KeySet([root.multihash()]) + const logInternalKey = (multihash) => { + newInternalPins.add(multihash) + } + pinner.utils.loadSet( + root, pinner.types.recursive, logInternalKey, (err, keys) => { + if (err) { return callback(err) } + recursivePins = new KeySet(keys) + pinner.utils.loadSet( + root, pinner.types.direct, logInternalKey, (err, keys) => { + if (err) { return callback(err) } + directPins = new KeySet(keys) + internalPins = newInternalPins + return callback() + } + ) + } + ) + }) + } + } + return pinner +} diff --git a/src/http-api/resources/index.js b/src/http-api/resources/index.js index b90a9d912c..315491d018 100644 --- a/src/http-api/resources/index.js +++ b/src/http-api/resources/index.js @@ -5,6 +5,7 @@ exports.id = require('./id') exports.bootstrap = require('./bootstrap') exports.repo = require('./repo') exports.object = require('./object') +exports.pin = require('./pin') exports.config = require('./config') exports.block = require('./block') exports.swarm = require('./swarm') diff --git a/src/http-api/resources/pin.js b/src/http-api/resources/pin.js new file mode 100644 index 0000000000..ec6ec2edcd --- /dev/null +++ b/src/http-api/resources/pin.js @@ -0,0 +1,206 @@ + +'use strict' + +const bs58 = require('bs58') +const debug = require('debug') +const log = debug('http-api:pin') +log.error = debug('http-api:pin:error') + +exports = module.exports + +function parsePath (path) { + if (!path) { + return null + } + const matched = path.match(/^(?:\/ipfs\/)?([^\/]+(?:\/[^\/]+)*)\/?$/) + if (!matched) { + return null + } + const split = matched[1].split('/') + return { + rootHash: split[0], + links: split.slice(1, split.length) + } +} + +function followLinks (ipfs, links, onError, callback) { + const recursor = (err, obj) => { + if (err) { return onError(err) } + if (links.length) { + const linkName = links.shift() + const nextLink = obj.links.filter((link) => { + return (link.name === linkName) + }) + if (!nextLink.length) { + return onError(new Error( + 'pin: no link named ' + linkName + + ' under ' + obj.toJSON().Hash + )) + } + const nextHash = nextLink[0].hash + ipfs.object.get(nextHash, recursor) + } else { + return callback(obj) + } + } + return recursor +} + +// common pre request handler that parses the args and +// returns `rootHash` and `links` assigned to `request.pre.args` +exports.parseArgs = (request, reply) => { + if (!request.query.arg) { + return reply("Argument 'ipfs-path' is required").code(400).takeover() + } + + try { + const parsed = parsePath(request.query.arg) + if (!parsed) { + throw new Error('invalid ipfs ref path') + } + return reply(parsed) + } catch (err) { + log.error(err) + return reply({ + Message: 'invalid ipfs ref path', + Code: 0 + }).code(500).takeover() + } +} + +exports.ls = (request, reply) => { + const ipfs = request.server.app.ipfs + const types = ipfs.pinner.types + const path = request.query.arg + const type = request.query.type || types.all + const onError = (err) => { + log.error(err) + return reply({ + Message: `Failed to list pins: ${err.message}`, + Code: 0 + }).code(500) + } + + // load persistent pin set from datastore + ipfs.pinner.load(() => { + const parsed = parsePath(path) + if (parsed) { + const rootKey = new Buffer(bs58.decode(parsed.rootHash)) + ipfs.object.get(rootKey, followLinks( + ipfs, parsed.links, onError, (obj) => { + ipfs.pinner.isPinnedWithType( + obj.multihash(), type, (err, pinned, reason) => { + if (err) { return onError(err) } + if (!pinned) { + return onError(new Error('Path ' + path + ' is not pinned')) + } + if (reason !== types.direct && + reason !== types.recursive) { + reason = 'indirect through ' + reason + } + const result = {} + result[obj.toJSON().Hash] = reason + return reply(result) + } + ) + }) + ) + } else { + switch (type) { + case types.direct: + return reply({ direct: ipfs.pinner.directKeyStrings() }) + case types.recursive: + return reply({ recursive: ipfs.pinner.recursiveKeyStrings() }) + case types.indirect: + ipfs.pinner.getIndirectKeys((err, keys) => { + if (err) { return onError(err) } + return reply({ indirect: keys }) + }) + break + case types.all: + ipfs.pinner.getIndirectKeys((err, keys) => { + if (err) { return onError(err) } + return reply({ + indirect: keys, + direct: ipfs.pinner.directKeyStrings(), + recursive: ipfs.pinner.recursiveKeyStrings() + }) + }) + break + default: + return onError(new Error( + "Invalid type '" + type + "', " + + 'must be one of {direct, indirect, recursive, all}' + )) + } + } + }) +} + +exports.add = { + // uses common parsePath method that returns a `path` + parseArgs: exports.parseArgs, + + // main route handler which is called after `parseArgs`, + // but only if the args were valid + handler: (request, reply) => { + const ipfs = request.server.app.ipfs + let recursive = true + if (request.query.recursive === 'false') { + recursive = false + } + const onError = (err) => { + log.error(err) + return reply({ + Message: `Failed to add pin: ${err.message}`, + Code: 0 + }).code(500) + } + const rootKey = new Buffer(bs58.decode(request.pre.args.rootHash)) + ipfs.object.get(rootKey, followLinks( + ipfs, request.pre.args.links, onError, (obj) => { + ipfs.pinner.pin(obj, recursive, (err) => { + if (err) { return onError(err) } + ipfs.pinner.flush((err) => { + if (err) { return onError(err) } + return reply('pinned ' + obj.toJSON().Hash + (recursive ? ' recursively' : ' directly')) + }) + }) + }) + ) + } +} + +exports.rm = { + // uses common parsePath method that returns a `path` + parseArgs: exports.parseArgs, + + // main route handler which is called after `parseArgs`, + // but only if the args were valid + handler: (request, reply) => { + const ipfs = request.server.app.ipfs + let recursive = true + if (request.query.recursive === 'false') { + recursive = false + } + const onError = (err) => { + log.error(err) + return reply({ + Message: `Failed to remove pin: ${err.message}`, + Code: 0 + }).code(500) + } + const rootKey = new Buffer(bs58.decode(request.pre.args.rootHash)) + ipfs.object.get(rootKey, followLinks( + ipfs, request.pre.args.links, onError, (obj) => { + ipfs.pinner.unpin(obj.multihash(), recursive, (err) => { + if (err) { return onError(err) } + ipfs.pinner.flush((err) => { + if (err) { return onError(err) } + return reply('unpinned ' + obj.toJSON().Hash) + }) + }) + }) + ) + } +} diff --git a/src/http-api/routes/index.js b/src/http-api/routes/index.js index 587f25de77..3e302b66e9 100644 --- a/src/http-api/routes/index.js +++ b/src/http-api/routes/index.js @@ -6,6 +6,7 @@ module.exports = (server) => { require('./bootstrap')(server) require('./block')(server) require('./object')(server) + require('./pin')(server) // require('./repo')(server) require('./config')(server) require('./swarm')(server) diff --git a/src/http-api/routes/pin.js b/src/http-api/routes/pin.js new file mode 100644 index 0000000000..c0d73e93e2 --- /dev/null +++ b/src/http-api/routes/pin.js @@ -0,0 +1,37 @@ +'use strict' + +const resources = require('./../resources') + +module.exports = (server) => { + const api = server.select('API') + + api.route({ + method: '*', + path: '/api/v0/pin/add', + config: { + pre: [ + { method: resources.pin.add.parseArgs, assign: 'args' } + ], + handler: resources.pin.add.handler + } + }) + + api.route({ + method: '*', + path: '/api/v0/pin/rm', + config: { + pre: [ + { method: resources.pin.add.parseArgs, assign: 'args' } + ], + handler: resources.pin.rm.handler + } + }) + + api.route({ + method: '*', + path: '/api/v0/pin/ls', + config: { + handler: resources.pin.ls + } + }) +} diff --git a/test/cli/test-commands.js b/test/cli/test-commands.js index ca3492e00a..1b9e4b2be7 100644 --- a/test/cli/test-commands.js +++ b/test/cli/test-commands.js @@ -10,7 +10,7 @@ describe('commands', () => { .run((err, stdout, exitcode) => { expect(err).to.not.exist expect(exitcode).to.equal(0) - expect(stdout.length).to.equal(56) + expect(stdout.length).to.equal(60) done() }) }) diff --git a/test/cli/test-pin.js b/test/cli/test-pin.js new file mode 100644 index 0000000000..56b458bc47 --- /dev/null +++ b/test/cli/test-pin.js @@ -0,0 +1,153 @@ +/* eslint-env mocha */ +'use strict' + +const expect = require('chai').expect +const nexpect = require('nexpect') +const repoPath = require('./index').repoPath +const _ = require('lodash') + +// use a tree of ipfs objects for recursive tests: +// root +// |`leaf +// `branch +// `subLeaf + +const filenames = [ + 'root', 'leaf', 'branch', 'subLeaf' +] +const keys = { + root: 'QmWQwS2Xh1SFGMPzUVYQ52b7RC7fTfiaPHm3ZyTRZuHmer', + leaf: 'QmaZoTQ6wFe7EtvaePBUeXavfeRqCAq3RUMomFxBpZLrLA', + branch: 'QmNxjjP7dtx6pzxWGBRCrgmjX3JqKL7uF2Kjx7ExiZDbSB', + subLeaf: 'QmUzzznkyQL7FjjBztG3D1tTjBuxeArLceDZnuSowUggXL' +} + +describe('pin', function () { + this.timeout(30000) + const env = _.clone(process.env) + env.IPFS_PATH = repoPath + const opts = {env: env, stream: 'all'} + const bin = process.cwd() + '/src/cli/bin.js' + const filesDir = process.cwd() + '/test/test-data/tree/' + + before((done) => { + let doneCount = 0 + filenames.forEach((filename) => { + const hash = keys[filename] + nexpect.spawn('node', [bin, 'object', 'put', + filesDir + filename + '.json'], opts) + .run((err, stdout, exitcode) => { + expect(err).to.not.exist + expect(stdout[0]).to.equal('added ' + hash) + doneCount === filenames.length - 1 ? done() : doneCount++ + }) + }) + }) + + describe('api offline', () => { + it('add (recursively by default)', (done) => { + nexpect.spawn('node', [bin, 'pin', 'add', keys.root], opts) + .run((err, stdout, exitcode) => { + expect(err).to.not.exist + expect(exitcode).to.equal(0) + expect(stdout[0]).to.equal('pinned ' + keys.root + ' recursively') + done() + }) + }) + it('add (direct)', (done) => { + nexpect.spawn('node', [bin, 'pin', 'add', + '--recursive=false', keys.leaf], opts) + .run((err, stdout, exitcode) => { + expect(err).to.not.exist + expect(exitcode).to.equal(0) + expect(stdout[0]).to.equal('pinned ' + keys.leaf + ' directly') + done() + }) + }) + + it('ls (recursive)', (done) => { + nexpect.spawn('node', [bin, 'pin', 'ls', '--path=' + keys.root], opts) + .run((err, stdout, exitcode) => { + expect(err).to.not.exist + expect(exitcode).to.equal(0) + expect(stdout[0]).to.equal(keys.root + ' recursive') + done() + }) + }) + it('ls (direct)', (done) => { + nexpect.spawn('node', [bin, 'pin', 'ls', '--path=' + keys.leaf], opts) + .run((err, stdout, exitcode) => { + expect(err).to.not.exist + expect(exitcode).to.equal(0) + expect(stdout[0]).to.equal(keys.leaf + ' direct') + done() + }) + }) + it('ls (indirect)', (done) => { + nexpect.spawn('node', [bin, 'pin', 'ls', '--path=' + keys.subLeaf], opts) + .run((err, stdout, exitcode) => { + expect(err).to.not.exist + expect(exitcode).to.equal(0) + expect(stdout[0]).to.equal( + keys.subLeaf + ' indirect through ' + keys.root + ) + done() + }) + }) + it('ls (all)', (done) => { + nexpect.spawn('node', [bin, 'pin', 'ls'], opts) + .run((err, stdout, exitcode) => { + expect(err).to.not.exist + expect(exitcode).to.equal(0) + expect(stdout.length).to.equal(4) + expect(stdout.indexOf(keys.root + ' recursive') >= 0).to.be.true + expect(stdout.indexOf(keys.leaf + ' direct') >= 0).to.be.true + expect(stdout.indexOf(keys.branch + ' indirect') >= 0).to.be.true + expect(stdout.indexOf(keys.subLeaf + ' indirect') >= 0).to.be.true + done() + }) + }) + it('ls (quiet)', (done) => { + nexpect.spawn('node', [bin, 'pin', 'ls', '--quiet=true'], opts) + .run((err, stdout, exitcode) => { + expect(err).to.not.exist + expect(exitcode).to.equal(0) + expect(stdout.length).to.equal(4) + filenames.forEach((filename) => { + expect(stdout.indexOf(keys[filename]) >= 0).to.be.true + }) + done() + }) + }) + + it('rm (recursively by default)', (done) => { + nexpect.spawn('node', [bin, 'pin', 'rm', keys.root], opts) + .run((err, stdout, exitcode) => { + expect(err).to.not.exist + expect(exitcode).to.equal(0) + expect(stdout[0]).to.equal('unpinned ' + keys.root) + done() + }) + }) + it('rm (direct)', (done) => { + nexpect.spawn('node', [bin, 'pin', 'rm', + '--recursive=false', keys.leaf], opts) + .run((err, stdout, exitcode) => { + expect(err).to.not.exist + expect(exitcode).to.equal(0) + expect(stdout[0]).to.equal('unpinned ' + keys.leaf) + done() + }) + }) + + it('confirm removal', (done) => { + nexpect.spawn('node', [bin, 'pin', 'ls'], opts) + .run((err, stdout, exitcode) => { + expect(err).to.not.exist + expect(exitcode).to.equal(0) + expect(stdout.length).to.equal(0) + done() + }) + }) + }) +}) diff --git a/test/core/both/test-pin.js b/test/core/both/test-pin.js new file mode 100644 index 0000000000..51c9c108c9 --- /dev/null +++ b/test/core/both/test-pin.js @@ -0,0 +1,267 @@ +/* eslint-env mocha */ +'use strict' + +const expect = require('chai').expect +const IPFS = require('../../../src/core') +const bs58 = require('bs58') +const mDAG = require('ipfs-merkle-dag') +const DAGNode = mDAG.DAGNode +const createTempRepo = require('../../utils/temp-repo') + +const emptyKeyHash = 'QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n' + +describe('pinner', function () { + this.timeout(30000) + let ipfs + let repo + let pinner + const Obj = {} + + function encode (key) { + return bs58.encode(key).toString() + } + + before((done) => { + repo = createTempRepo() + ipfs = new IPFS(repo) + pinner = ipfs.pinner + ipfs.init({ emptyRepo: true }, (err) => { + expect(err).to.not.exist + ipfs.load(() => { + // Use this tree for multiple tests + // + // B E + // / \ + // A C + // \ + // D + + const labels = ['A', 'B', 'C', 'D', 'E'] + labels.forEach((label) => { + Obj[label] = new DAGNode(new Buffer('Node ' + label)) + }) + // make links from bottom up to avoid mutating the hash after linking + Obj.C.addNodeLink('Child D', Obj.D) + Obj.B.addNodeLink('Child A', Obj.A) + Obj.B.addNodeLink('Child C', Obj.C) + let count = 0 + labels.forEach((label) => { + ipfs.object.put(Obj[label], (err) => { + expect(err).to.not.exist + count++ + if (count === labels.length) { + done() + } + }) + }) + }) + }) + }) + + beforeEach((done) => { + pinner.clear() + done() + }) + + describe('pin', () => { + it('pins object directly', (done) => { + pinner.pin(Obj.A, false, (err) => { + expect(err).to.not.exist + pinner.isPinned(Obj.A.multihash(), (err, pinned, reason) => { + expect(err).to.not.exist + expect(pinned).to.be.true + expect(reason).to.equal(pinner.types.direct) + done() + }) + }) + }) + + it('pins recursively by default', (done) => { + // direct pin A which is child of B + pinner.pin(Obj.A, false, (err) => { + expect(err).to.not.exist + // recursive pin B which has children A and C + pinner.pin(Obj.B, (err) => { + expect(err).to.not.exist + // B should be 'recursive' pin + pinner.isPinned(Obj.B.multihash(), (err, pinned, reason) => { + expect(err).to.not.exist + expect(pinned).to.be.true + expect(reason).to.equal(pinner.types.recursive) + // A should still be 'direct' pin + pinner.isPinned(Obj.A.multihash(), (err, pinned, reason) => { + expect(err).to.not.exist + expect(pinned).to.be.true + expect(reason).to.equal(pinner.types.direct) + // C should be 'indirect' pin + pinner.isPinned(Obj.C.multihash(), (err, pinned, reason) => { + expect(err).to.not.exist + expect(pinned).to.be.true + // indirect pin 'reason' is the b58 hash of recursive root pin + expect(reason).to.equal(encode(Obj.B.multihash())) + done() + }) + }) + }) + }) + }) + }) + + it('rejects direct pin if already recursively pinned', (done) => { + // recursive pin B which has children A and C + pinner.pin(Obj.B, (err) => { + expect(err).to.not.exist + // direct pin B should fail + pinner.pin(Obj.B, false, (err) => { + expect(err).to.equal(encode(Obj.B.multihash()) + + ' already pinned recursively') + // pinning recursively again should succeed + pinner.pin(Obj.B, (err) => { + expect(err).to.not.exist + done() + }) + }) + }) + }) + + it('rejects recursive pin if child object is not stored', (done) => { + Obj.Y = new DAGNode(new Buffer('Node Y')) + Obj.Z = new DAGNode(new Buffer('Node Z')) + Obj.Y.addNodeLink('Child Z', Obj.Z) + ipfs.object.put(Obj.Y, (err) => { + expect(err).to.not.exist + // this should fail because Z is not stored + pinner.pin(Obj.Y, (err) => { + expect(err).to.exist + ipfs.object.put(Obj.Z, (err) => { + expect(err).to.not.exist + // now it should succeed + pinner.pin(Obj.Y, (err) => { + expect(err).to.not.exist + done() + }) + }) + }) + }) + }) + }) + + describe('unpin', () => { + it('unpins directly pinned object', (done) => { + pinner.pin(Obj.A, false, (err) => { + expect(err).to.not.exist + pinner.unpin(Obj.A.multihash(), false, (err) => { + expect(err).to.not.exist + pinner.isPinned(Obj.A.multihash(), (err, pinned) => { + expect(err).to.not.exist + expect(pinned).to.be.false + done() + }) + }) + }) + }) + + it('unpins recursively by default', (done) => { + const bs58A = encode(Obj.A.multihash()) + const bs58B = encode(Obj.B.multihash()) + // recursive pin B which has children A and C + pinner.pin(Obj.B, (err) => { + expect(err).to.not.exist + // indirect pin A should not be unpinnable while B is pinned + pinner.unpin(Obj.A.multihash(), (err) => { + expect(err.message).to.equal( + bs58A + ' is pinned indirectly under ' + bs58B + ) + // unpinning B should also unpin A + pinner.unpin(Obj.B.multihash(), (err) => { + expect(err).to.not.exist + pinner.isPinned(Obj.B.multihash(), (err, pinned) => { + expect(err).to.not.exist + expect(pinned).to.be.false + pinner.isPinned(Obj.A.multihash(), (err, pinned) => { + expect(err).to.not.exist + expect(pinned).to.be.false + done() + }) + }) + }) + }) + }) + }) + }) + + describe('flush and load (roundtrip)', () => { + it('writes pinned keys to datastore and reads them back', (done) => { + const checkInternal = (root) => { + let internal = pinner.internalKeyStrings() + expect(internal.length).to.equal(4) + internal = new Set(internal) + expect(internal.has(emptyKeyHash)).to.be.true + expect(internal.has(encode(root.multihash()))).to.be.true + expect(root.links.length).to.equal(2) + root.links.forEach((link) => { + expect(internal.has(encode(link.hash))).to.be.true + }) + } + const checkClear = () => { + expect(pinner.directKeys().length).to.equal(0) + expect(pinner.recursiveKeys().length).to.equal(0) + expect(pinner.internalKeys().length).to.equal(0) + } + checkClear() + // recursive pin + pinner.pin(Obj.B, (err) => { + expect(err).to.not.exist + // direct pin + pinner.pin(Obj.E, false, (err) => { + expect(err).to.not.exist + // save to datastore + pinner.flush((err, root) => { + // internalPins should have a recursive root node, a direct root + // node, a root header node with links to both, and an empty node + expect(err).to.not.exist + checkInternal(root) + // clear from memory + pinner.clear() + checkClear() + // load from datastore + pinner.load((err) => { + expect(err).to.not.exist + // Obj.E should be restored as a direct pin + const direct = pinner.directKeyStrings() + expect(direct.length).to.equal(1) + expect(direct[0]).to.equal(encode(Obj.E.multihash())) + // Obj.B should be restored as a recursive pin + const recursive = pinner.recursiveKeyStrings() + expect(recursive.length).to.equal(1) + expect(recursive[0]).to.equal(encode(Obj.B.multihash())) + // Internal should be the same as before + checkInternal(root) + done() + }) + }) + }) + }) + }) + }) + + describe('utils', () => { + describe('hasChild', () => { + it('finds if child hash is somewhere in object tree', (done) => { + pinner.utils.hasChild(Obj.B, Obj.D.multihash(), (err, has) => { + expect(err).to.not.exist + expect(has).to.be.true + pinner.utils.hasChild(Obj.B, Obj.E.multihash(), (err, has) => { + expect(err).to.not.exist + expect(has).to.be.false + done() + }) + }) + }) + }) + }) + + after((done) => { + repo.teardown(done) + }) +}) diff --git a/test/http-api/test-pin.js b/test/http-api/test-pin.js new file mode 100644 index 0000000000..26adf4b74d --- /dev/null +++ b/test/http-api/test-pin.js @@ -0,0 +1,160 @@ +/* eslint-env mocha */ +'use strict' + +const expect = require('chai').expect +const fs = require('fs') +const FormData = require('form-data') +const streamToPromise = require('stream-to-promise') + +// use a tree of ipfs objects for recursive tests: +// root +// |`leaf +// `branch +// `subLeaf + +const filenames = [ + 'root', 'leaf', 'branch', 'subLeaf' +] +const keys = { + root: 'QmWQwS2Xh1SFGMPzUVYQ52b7RC7fTfiaPHm3ZyTRZuHmer', + leaf: 'QmaZoTQ6wFe7EtvaePBUeXavfeRqCAq3RUMomFxBpZLrLA', + branch: 'QmNxjjP7dtx6pzxWGBRCrgmjX3JqKL7uF2Kjx7ExiZDbSB', + subLeaf: 'QmUzzznkyQL7FjjBztG3D1tTjBuxeArLceDZnuSowUggXL' +} + +module.exports = (httpAPI) => { + describe('pin', () => { + describe('api', () => { + let api + + before((done) => { + // add test tree to repo + api = httpAPI.server.select('API') + let doneCount = 0 + filenames.forEach((filename) => { + const filePath = 'test/test-data/tree/' + filename + '.json' + const form = new FormData() + form.append('file', fs.createReadStream(filePath)) + const headers = form.getHeaders() + streamToPromise(form).then((payload) => { + api.inject({ + method: 'POST', + url: '/api/v0/object/put', + headers: headers, + payload: payload + }, (res) => { + expect(res.statusCode).to.equal(200) + doneCount++ + if (doneCount === filenames.length) { + done() + } + }) + }) + }) + }) + + describe('/pin/add', () => { + it('pins object recursively by default', (done) => { + api.inject({ + method: 'POST', + url: ('/api/v0/pin/add?arg=' + keys.root) + }, (res) => { + expect(res.statusCode).to.equal(200) + expect(res.result) + .to.equal('pinned ' + keys.root + ' recursively') + done() + }) + }) + }) + + describe('/pin/add (direct)', () => { + it('pins object directly if specified', (done) => { + api.inject({ + method: 'POST', + url: ('/api/v0/pin/add?arg=' + keys.leaf + + '&recursive=false') + }, (res) => { + expect(res.statusCode).to.equal(200) + expect(res.result) + .to.equal('pinned ' + keys.leaf + ' directly') + done() + }) + }) + }) + + describe('/pin/ls (with path)', () => { + it('finds specified pinned object', (done) => { + api.inject({ + method: 'GET', + url: ('/api/v0/pin/ls?arg=/ipfs/' + + keys.root + '/branch/subLeaf') + }, (res) => { + expect(res.statusCode).to.equal(200) + expect(res.result[keys.subLeaf]) + .to.equal('indirect through ' + keys.root) + done() + }) + }) + }) + + describe('/pin/ls (without path or type)', () => { + it('finds all pinned objects', (done) => { + api.inject({ + method: 'GET', + url: ('/api/v0/pin/ls') + }, (res) => { + expect(res.statusCode).to.equal(200) + expect(res.result.direct.length).to.equal(1) + expect(res.result.direct).to.contain(keys.leaf) + expect(res.result.recursive.length).to.equal(1) + expect(res.result.recursive).to.contain(keys.root) + expect(res.result.indirect.length).to.equal(2) + expect(res.result.indirect).to.contain(keys.branch) + expect(res.result.indirect).to.contain(keys.subLeaf) + done() + }) + }) + }) + + describe('/pin/rm (direct)', () => { + it('unpins only directly pinned objects if specified', (done) => { + api.inject({ + method: 'POST', + url: ('/api/v0/pin/rm?arg=' + keys.leaf + + '&recursive=false') + }, (res) => { + expect(res.statusCode).to.equal(200) + expect(res.result) + .to.equal('unpinned ' + keys.leaf) + + api.inject({ + method: 'POST', + url: ('/api/v0/pin/rm?arg=' + keys.root + + '&recursive=false') + }, (res) => { + expect(res.statusCode).to.equal(500) + expect(res.result.Message) + .to.equal('Failed to remove pin: ' + + 'QmWQwS2Xh1SFGMPzUVYQ52b7RC7fTfiaPHm3ZyTRZuHmer is pinned recursively') + done() + }) + }) + }) + }) + + describe('/pin/rm', () => { + it('unpins recursively by default', (done) => { + api.inject({ + method: 'POST', + url: ('/api/v0/pin/rm?arg=' + keys.root) + }, (res) => { + expect(res.statusCode).to.equal(200) + expect(res.result) + .to.equal('unpinned ' + keys.root) + done() + }) + }) + }) + }) + }) +} diff --git a/test/test-data/tree/branch.json b/test/test-data/tree/branch.json new file mode 100644 index 0000000000..459498c85c --- /dev/null +++ b/test/test-data/tree/branch.json @@ -0,0 +1 @@ +{"Links":[{"Name":"subLeaf","Hash":"QmUzzznkyQL7FjjBztG3D1tTjBuxeArLceDZnuSowUggXL","Size":15}],"Data":"\u0008\u0001"} diff --git a/test/test-data/tree/leaf.json b/test/test-data/tree/leaf.json new file mode 100644 index 0000000000..547be2cd24 --- /dev/null +++ b/test/test-data/tree/leaf.json @@ -0,0 +1 @@ +{"Links":[],"Data":"\u0008\u0002\u0012\u0004leaf\u0018\u0004"} diff --git a/test/test-data/tree/root.json b/test/test-data/tree/root.json new file mode 100644 index 0000000000..22f1229788 --- /dev/null +++ b/test/test-data/tree/root.json @@ -0,0 +1 @@ +{"Links":[{"Name":"leaf","Hash":"QmaZoTQ6wFe7EtvaePBUeXavfeRqCAq3RUMomFxBpZLrLA","Size":12},{"Name":"branch","Hash":"QmNxjjP7dtx6pzxWGBRCrgmjX3JqKL7uF2Kjx7ExiZDbSB","Size":68}],"Data":"\u0008\u0001"} diff --git a/test/test-data/tree/subLeaf.json b/test/test-data/tree/subLeaf.json new file mode 100644 index 0000000000..d77789e6f0 --- /dev/null +++ b/test/test-data/tree/subLeaf.json @@ -0,0 +1 @@ +{"Links":[],"Data":"\u0008\u0002\u0012\u0007subLeaf\u0018\u0007"}