diff --git a/lib/config.js b/lib/config.js index e1598813..33ac0ecc 100644 --- a/lib/config.js +++ b/lib/config.js @@ -56,6 +56,14 @@ const DEFAULTS = { from: 'robot@storj.io' }, application: { + activateSIP6: false, + powOpts: { + retargetPeriod: 10000, // milliseconds + retargetCount: 10, // per retargetPeriod + }, + publishBenchThreshold: 4000, // milliseconds + publishTotal: 36, // number of farmers to publish in active pool + publishBenchTotal: 9, // number of farmers to publish in bench pool shardsPerMinute: 1000, farmerTimeoutIgnore: '10m', freeTier: { diff --git a/lib/engine.js b/lib/engine.js index 5a46463f..931a0ce9 100644 --- a/lib/engine.js +++ b/lib/engine.js @@ -12,6 +12,7 @@ const Config = require('./config'); const Storage = require('storj-service-storage-models'); const middleware = require('storj-service-middleware'); const Server = require('./server'); +const pow = require('./server/middleware/pow'); const Mailer = require('storj-service-mailer'); const log = require('./logger'); const ComplexClient = require('storj-complex').createClient; @@ -62,6 +63,11 @@ Engine.prototype.start = function(callback) { this.redis = require('redis').createClient(this._config.redis); this.redis.on('ready', () => { log.info('connected to redis'); + pow.checkInitTarget(this.redis, (err) => { + if (err) { + log.error('unable to initialize pow settings', err); + } + }); }); this.redis.on('error', (err) => { log.error('error connecting to redis', err); diff --git a/lib/server/middleware/farmer-auth.js b/lib/server/middleware/farmer-auth.js new file mode 100644 index 00000000..ebddd743 --- /dev/null +++ b/lib/server/middleware/farmer-auth.js @@ -0,0 +1,109 @@ +'use strict'; + +const errors = require('storj-service-error-types'); +const crypto = require('crypto'); +const secp256k1 = require('secp256k1'); + +const THRESHOLD = 300000; + +function isHexString(a) { + if (typeof a !== 'string') { + return false; + } + return /^([0-9a-fA-F]{2})+$/.test(a); +} + +function getSigHash(req) { + const hasher = crypto.createHash('sha256'); + const timestamp = req.headers['x-node-timestamp']; + const url = req.protocol + '://' + req.get('host') + req.originalUrl; + hasher.update(req.method); + hasher.update(url); + hasher.update(timestamp); + hasher.update(req.rawbody); + return hasher.digest(); +} + +function checkSig(req) { + const sighash = getSigHash(req); + let sigstr = req.headers['x-node-signature']; + if (!isHexString(sigstr)) { + return false; + } + const buf = Buffer.from(req.headers['x-node-signature'], 'hex'); + const sig = secp256k1.signatureImport(buf); + const pubkey = Buffer.from(req.headers['x-node-pubkey'], 'hex'); + return secp256k1.verify(sighash, sig, pubkey); +} + +function checkPubkey(pubkey) { + if (!isHexString(pubkey)) { + return false; + } + const buf = Buffer.from(pubkey, 'hex'); + return secp256k1.publicKeyVerify(buf); +} + +function checkTimestamp(ts) { + const timestamp = parseInt(ts); + if (!Number.isSafeInteger(timestamp)) { + return false; + } + const now = Date.now(); + if (timestamp < now - THRESHOLD || timestamp > now + THRESHOLD) { + return false; + } + return true; +} + +function checkNodeID(nodeID, pubkey) { + if (!nodeID || nodeID.length !== 40 || !isHexString(nodeID)) { + return false; + } + const sha256 = crypto.createHash('sha256'); + const rmd160 = crypto.createHash('rmd160'); + sha256.update(Buffer.from(pubkey, 'hex')); + rmd160.update(sha256.digest()); + if (rmd160.digest('hex') !== nodeID) { + return false; + } + return true; +} + +function authFarmer(req, res, next) { + const nodeID = req.headers['x-node-id']; + const timestamp = req.headers['x-node-timestamp']; + const pubkey = req.headers['x-node-pubkey']; + + if (!module.exports.checkTimestamp(timestamp)) { + return next(new errors.BadRequestError('Invalid timestamp header')); + } + + if (!module.exports.checkPubkey(pubkey)) { + return next(new errors.BadRequestError('Invalid pubkey header')); + } + + if (!module.exports.checkNodeID(nodeID, pubkey)) { + return next(new errors.BadRequestError('Invalid nodeID header')); + } + + if (!req.rawbody || !Buffer.isBuffer(req.rawbody)) { + return next(new errors.BadRequestError('Invalid body')); + } + + if (!module.exports.checkSig(req)) { + return next(new errors.BadRequestError('Invalid signature header')); + } + + next(); +} + +module.exports = { + authFarmer: authFarmer, + getSigHash: getSigHash, + isHexString: isHexString, + checkTimestamp: checkTimestamp, + checkNodeID: checkNodeID, + checkPubkey: checkPubkey, + checkSig: checkSig +}; diff --git a/lib/server/middleware/pow.js b/lib/server/middleware/pow.js new file mode 100644 index 00000000..0c441a93 --- /dev/null +++ b/lib/server/middleware/pow.js @@ -0,0 +1,190 @@ +'use strict'; + +const async = require('async'); +const assert = require('assert'); +const scrypt = require('scrypt'); +const crypto = require('crypto'); +const BN = require('bn.js'); + +const CHALLENGE_TTL_SECONDS = 3600; +const MAX = 'ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff'; +const MAXBN = new BN(MAX, 16); + +function getPOWMiddleware(db) { + + return function(req, res, next) { + + const challenge = req.headers['x-challenge']; + const nonce = req.headers['x-challenge-nonce']; + + let salt = Buffer.alloc(8, 0); + salt.writeDoubleBE(parseInt(nonce)); + + const key = 'contact-' + challenge; + const scryptOpts = { N: Math.pow(2, 10), r: 1, p: 1 }; + + db.get(key, function(err, target) { + if (err) { + return next(err); + } + + if (!target) { + return next(new Error('Challenge not found')); + } + + scrypt.hash(challenge, scryptOpts, 32, salt, function(err, result) { + if (err) { + return next(err); + } + + // Check the proof of work + if (result.toString('hex') > target) { + return next(new Error('Invalid proof of work')); + } + + // Increase the count and remove the challenge + db.hincrby('contact-stats', 'count', 1); + db.del(key); + + return next(); + }); + }); + }; +} + +function checkInitTarget(db, callback) { + db.hgetall('contact-stats', function(err, stats) { + if (err) { + callback(err); + } else if (!stats) { + module.exports.initTarget(db, callback); + } else { + callback(); + } + }); +} + +function initTarget(db, callback) { + const initialTarget = 'fffffffffffffffffffffffffffffff' + + 'fffffffffffffffffffffffffffffffff'; + async.series([ + (next) => { + db.hset('contact-stats', 'target', initialTarget, next); + }, + (next) => { + db.hset('contact-stats', 'timestamp', 0, next); + }, + (next) => { + db.hset('contact-stats', 'count', 0, next); + }, + ], callback); +} + +function getTarget(db, opts, callback) { + + const precision = 100000000; + const precisionBN = new BN(precision); + const retargetPeriod = opts.retargetPeriod; + const count = opts.retargetCount; + + assert(Number.isSafeInteger(count), + 'retargetCount is expected to be a safe integer'); + assert(Number.isSafeInteger(retargetPeriod), + 'retargetPeriod is expected to be a safe integer'); + + /* jshint maxstatements: 100 */ + db.hgetall('contact-stats', function(err, stats) { + if (err) { + return callback(err); + } + + if (!stats) { + return callback(new Error('Unknown pow settings')); + } + + if (!Number.isSafeInteger(parseInt(stats.count)) || + !Number.isSafeInteger(parseInt(stats.timestamp))) { + return callback(new Error('Invalid pow settings')); + } + + const now = Date.now(); + const actual = parseInt(stats.count) || 1; + const timestamp = parseInt(stats.timestamp); + + if (now > timestamp + retargetPeriod) { + const timeDelta = now - timestamp; + const expectedRatio = retargetPeriod / count; + const actualRatio = timeDelta / actual; + const adjustmentAmount = actualRatio / expectedRatio; + + let adjustmentNum = adjustmentAmount * precision; + let adjustment = null; + if (adjustmentNum > Number.MAX_SAFE_INTEGER) { + adjustment = new BN(adjustmentAmount); + adjustment.mul(new BN(precision)); + } else { + adjustment = new BN(adjustmentNum); + } + + const target = new BN(stats.target, 16); + const newTargetBN = target.mul(adjustment).div(precisionBN); + let newTarget = null; + if (newTargetBN.cmp(MAXBN) > 0) { + newTarget = MAXBN.toString(16, 32); + } else { + newTarget = newTargetBN.toString(16, 32); + } + + async.series([ + (next) => { + db.hset('contact-stats', 'target', newTarget, next); + }, + (next) => { + db.hset('contact-stats', 'timestamp', now, next); + }, + (next) => { + db.hset('contact-stats', 'count', 0, next); + }, + ], (err) => { + if (err) { + return callback(err); + } + callback(null, newTarget); + }); + + } else { + callback(null, stats.target); + } + + }); + +} + +function getChallenge(db, opts, callback) { + module.exports.getTarget(db, opts, function(err, target) { + if (err) { + return callback(err); + } + + const challenge = crypto.randomBytes(32).toString('hex'); + const key = 'contact-' + challenge; + + db.set(key, target, 'EX', CHALLENGE_TTL_SECONDS, function(err) { + if (err) { + return callback(err); + } + callback(null, { + challenge: challenge, + target: target + }); + }); + }); +} + +module.exports = { + getChallenge: getChallenge, + getTarget: getTarget, + checkInitTarget: checkInitTarget, + initTarget: initTarget, + getPOWMiddleware: getPOWMiddleware +}; diff --git a/lib/server/middleware/raw-body.js b/lib/server/middleware/raw-body.js new file mode 100644 index 00000000..6d72e920 --- /dev/null +++ b/lib/server/middleware/raw-body.js @@ -0,0 +1,22 @@ +'use strict'; + +const concat = require('concat-stream'); + +module.exports = function rawbody(req, res, next) { + // Do not buffer the request body for file uploads + if (req.get('Content-Type') === 'multipart/form-data') { + return next(); + } + + req.pipe(concat(function(data) { + req.rawbody = data; + + try { + req.body = JSON.parse(req.rawbody.toString()); + } catch (err) { + req.body = {}; + } + + next(); + })); +}; diff --git a/lib/server/routes/contacts.js b/lib/server/routes/contacts.js index 33ee66e6..e5a4f518 100644 --- a/lib/server/routes/contacts.js +++ b/lib/server/routes/contacts.js @@ -5,6 +5,9 @@ const errors = require('storj-service-error-types'); const inherits = require('util').inherits; const middleware = require('storj-service-middleware'); const limiter = require('../limiter').DEFAULTS; +const rawBody = require('../middleware/raw-body'); +const {getPOWMiddleware, getChallenge} = require('../middleware/pow'); +const {authFarmer} = require('../middleware/farmer-auth'); /** * Handles endpoints for all contact related endpoints @@ -18,6 +21,8 @@ function ContactsRouter(options) { Router.apply(this, arguments); + this.redis = options.redis; + this.checkPOW = getPOWMiddleware(options.redis); this.getLimiter = middleware.rateLimiter(options.redis); } @@ -41,6 +46,76 @@ ContactsRouter.prototype._getSkipLimitFromPage = function(page) { }; }; +ContactsRouter.prototype.createChallenge = function(req, res, next) { + let powOpts = this.config.application.powOpts; + getChallenge(this.redis, powOpts, function(err, data) { + if (err) { + return next(new errors.InternalError(err.message)); + } + res.status(201).send(data); + }); +}; + +ContactsRouter.prototype.createContact = function(req, res, next) { + const Contact = this.storage.models.Contact; + Contact.record({ + nodeID: req.headers['x-node-id'], + address: req.body.address, + port: req.body.port, + lastSeen: Date.now(), + spaceAvailable: req.body.spaceAvailable + }, function(err, contact) { + if (err) { + return next(new errors.InternalError(err.message)); + } + // TODO Send 201 status when created, and 200 when it already + // exists. Multiple calls to record should behave the same, + // as is current. + res.status(200).send({ + nodeID: contact.nodeID, + address: contact.address, + port: contact.port + }); + }); +}; + +ContactsRouter.prototype.patchContactByNodeID = function(req, res, next) { + const Contact = this.storage.models.Contact; + const nodeID = req.headers['x-node-id']; + + const data = {}; + if (req.body.address) { + data.address = req.body.address; + } + + if (req.body.port) { + data.port = req.body.port; + } + + if (req.body.spaceAvailable === false || + req.body.spaceAvailable === true) { + data.spaceAvailable = req.body.spaceAvailable; + } + + Contact.findOneAndUpdate({ _id: nodeID }, data, { + upsert: false, + new: false + }, function(err, contact) { + if (err) { + return next(new errors.InternalError(err.message)); + } + if (!contact) { + return next(new errors.NotFoundError('Contact not found')); + } + + res.status(201).send({ + nodeID: contact.nodeID, + address: contact.address, + port: contact.port + }); + }); +}; + /** * Lists the contacts according the the supplied query * @param {http.IncomingMessage} req @@ -107,8 +182,11 @@ ContactsRouter.prototype.getContactByNodeID = function(req, res, next) { */ ContactsRouter.prototype._definitions = function() { return [ - ['GET', '/contacts', this.getLimiter(limiter(1000)), this.getContactList], - ['GET', '/contacts/:nodeID', this.getLimiter(limiter(1000)), this.getContactByNodeID] + ['GET', '/contacts', this.getLimiter(limiter(200)), this.getContactList], + ['GET', '/contacts/:nodeID', this.getLimiter(limiter(200)), this.getContactByNodeID], + ['PATCH', '/contacts/:nodeID', this.getLimiter(limiter(200)), rawBody, authFarmer, this.patchContactByNodeID], + ['POST', '/contacts', this.getLimiter(limiter(200)), this.checkPOW, rawBody, authFarmer, this.createContact], + ['POST', '/contacts/challenges', this.getLimiter(limiter(200)), rawBody, authFarmer, this.createChallenge] ]; }; diff --git a/lib/server/routes/frames.js b/lib/server/routes/frames.js index 03e02c7c..2bbd3852 100644 --- a/lib/server/routes/frames.js +++ b/lib/server/routes/frames.js @@ -73,6 +73,7 @@ FramesRouter.prototype.createFrame = function(req, res, next) { }); }; + /** * Negotiates a contract and updates persistence for the given contract data * @private @@ -118,12 +119,100 @@ FramesRouter.prototype._getContractForShard = function(contr, audit, bl, res, do }); }; +/** + * Negotiates a contract and updates persistence for the given contract data + * @private + * @param {storj.Contract} contract - The contract object to publish + * @param {storj.AuditStream} audit - The audit object to add to persistence + * @param {Array} blacklist - Do not accept offers from these nodeIDs + * @param {Object} res - The associated response + * @param {Function} callback - Called with error or (farmer, contract) + */ +FramesRouter.prototype._getContractForShardSIP6 = function(contr, audit, bl, res, done) { + this._selectFarmers(bl, (err, farmers) => { + if (err) { + return done(new errors.InternalError(err.message)); + } + + if (!farmers || !farmers.length) { + return done(new errors.InternalError('Could not locate farmers')); + } + + this._publishContract(farmers, contr, audit, (err, farmerContact, farmerContract, token) => { + if (err) { + return done(new errors.InternalError(err.message)); + } + + done(null, farmerContact, farmerContract, token); + }); + }); +}; + FramesRouter._sortByResponseTime = function(a, b) { const aTime = a.contact.responseTime || Infinity; const bTime = b.contact.responseTime || Infinity; return (aTime === bTime) ? 0 : (aTime > bTime) ? 1 : -1; }; +FramesRouter.prototype._selectFarmers = function(excluded, callback) { + async.parallel([ + (next) => { + this.storage.models.Contact.find({ + responseTime: { $lte: this._defaults.publishBenchThreshold }, + spaceAvailable: true, + _id: { $nin: excluded } + }).sort({ lastContractSent: 1 }) + .limit(this._defaults.publishTotal) + .exec(next); + }, + (next) => { + this.storage.models.Contact.find({ + responseTime: { $gt: this._defaults.publishBenchThreshold }, + spaceAvailable: true, + _id: { $nin: excluded } + }).sort({ lastContractSent: 1 }) + .limit(this._defaults.publishBenchTotal) + .exec(next); + } + ], (err, results) => { + if (err) { + return callback(err); + } + const combined = results[0].concat(results[1]); + callback(null, combined); + }); +}; + +FramesRouter.prototype._publishContract = function(nodes, contract, audit, callback) { + const hash = contract.get('data_hash'); + + this.contracts.load(hash, (err, item) => { + if (err) { + item = new storj.StorageItem({ hash: hash }); + } + + this.network.publishContract(nodes, contract, (err, data) => { + if (err) { + return callback(err); + } + + const farmerContact = storj.Contact(data.contact); + const farmerContract = storj.Contract(data.contract); + + item.addContract(farmerContact, farmerContract); + item.addAuditRecords(farmerContact, audit); + + this.contracts.save(item, (err) => { + if (err) { + return callback(new errors.InternalError(err.message)); + } + + callback(null, farmerContact, farmerContract, data.token); + }); + }); + }); +}; + /** * Negotiates a storage contract and adds the shard to the frame * @param {http.IncomingMessage} req @@ -244,15 +333,30 @@ FramesRouter.prototype.addShardToFrame = function(req, res, next) { log.debug('Requesting contract for frame: %s, shard hash: %s and size: %s', req.params.frame, req.body.hash, req.body.size); - self._getContractForShard(contr, audit, bl, res, function(err, _farmer, _contract) { + // Check if SIP6 is activated, otherwise we'll continue to use the + // existing implementation. Once this has fully been deployed, this + // switch can be removed and SIP6 used exclusively. + let getContractForShard = self._defaults.activateSIP6 ? + self._getContractForShardSIP6.bind(self) : + self._getContractForShard.bind(self); + + getContractForShard(contr, audit, bl, res, function(err, _contact, _contract, _token) { if (err) { log.warn('Could not get contract for frame: %s and ' + 'shard hash: %s, reason: %s', req.params.frame, req.body.hash, err.message); done(new errors.ServiceUnavailableError(err.message)); } else { - farmer = _farmer; + farmer = _contact; contract = _contract; + + // Only set the token if SIP6 is activated, this value will be + // undefined without it. Once SIP6 is fully activated, this check + // can be removed. + if (self._defaults.activateSIP6) { + token = _token; + } + done(); } }); @@ -283,6 +387,12 @@ FramesRouter.prototype.addShardToFrame = function(req, res, next) { }); }, function getToken(done) { + if (self._defaults.activateSIP6) { + // There is no need to get the token seperately with SIP6, + // we can skip this step. Once SIP6 is fully activated, this + // step can be completely removed. + return done(); + } self.network.getConsignmentPointer( farmer, contract, audit, function(err, dcPointer) { diff --git a/package.json b/package.json index 5b7a4d68..5db4b558 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "storj-bridge", - "version": "5.24.1", + "version": "6.0.0", "description": "Access the Storj network using a simple REST API.", "main": "index.js", "directories": { @@ -59,7 +59,9 @@ "dependencies": { "analytics-node": "^2.4.0", "async": "^2.1.4", + "bn.js": "^4.11.8", "commander": "^2.9.0", + "concat-stream": "^1.6.0", "cors": "^2.7.1", "csv-write-stream": "^2.0.0", "elliptic": "^6.0.2", @@ -75,14 +77,16 @@ "rc": "^1.1.6", "readable-stream": "^2.0.5", "redis": "^2.7.1", + "scrypt": "^6.0.3", + "secp256k1": "^3.3.0", "storj-analytics": "^1.0.0", - "storj-complex": "^5.6.0", - "storj-lib": "^6.2.0", - "storj-mongodb-adapter": "^7.0.1", + "storj-complex": "^6.0.0", + "storj-lib": "^7.0.0", + "storj-mongodb-adapter": "^8.0.0", "storj-service-error-types": "^1.2.0", "storj-service-mailer": "^1.0.0", "storj-service-middleware": "^1.3.1", - "storj-service-storage-models": "^8.23.1", + "storj-service-storage-models": "^9.0.0", "through": "^2.3.8" } } diff --git a/test/server/middleware/farmer-auth.unit.js b/test/server/middleware/farmer-auth.unit.js new file mode 100644 index 00000000..91c68c9d --- /dev/null +++ b/test/server/middleware/farmer-auth.unit.js @@ -0,0 +1,236 @@ +'use strict'; + +const chai = require('chai'); +const expect = chai.expect; +const sinon = require('sinon'); +const errors = require('storj-service-error-types'); +const secp256k1 = require('secp256k1'); +const auth = require('../../../lib/server/middleware/farmer-auth'); + +describe('Farmer Authentication Middleware', function() { + const sandbox = sinon.sandbox.create(); + afterEach(() => sandbox.restore()); + + describe('#authFarmer', function() { + const nodeID = 'e6a498de631c6f3eba57da0e416881f9d4a6fca1'; + const pubkey = '03f716a870a72aaa61a75f5b06381ea1771f49c3a9866636007affc4ac06ef54b8'; + const timestamp = '1502390208007'; + const signature = 'signature'; + const req = { + headers: { + 'x-node-id': nodeID, + 'x-node-pubkey': pubkey, + 'x-node-timestamp': timestamp, + 'x-node-signature': signature + }, + rawbody: Buffer.from('ffff', 'hex') + }; + const res = {}; + it('will give error for invalid timestamp', function(done) { + sandbox.stub(auth, 'checkTimestamp').returns(false); + auth.authFarmer(req, res, function(err) { + expect(err).to.be.instanceOf(errors.BadRequestError); + done(); + }); + }); + it('will give error for invalid pubkey', function(done) { + sandbox.stub(auth, 'checkTimestamp').returns(true); + sandbox.stub(auth, 'checkPubkey').returns(false); + auth.authFarmer(req, res, function(err) { + expect(err).to.be.instanceOf(errors.BadRequestError); + done(); + }); + }); + it('will give error for invalid nodeid', function(done) { + sandbox.stub(auth, 'checkTimestamp').returns(true); + sandbox.stub(auth, 'checkPubkey').returns(true); + sandbox.stub(auth, 'checkNodeID').returns(false); + auth.authFarmer(req, res, function(err) { + expect(err).to.be.instanceOf(errors.BadRequestError); + done(); + }); + }); + it('will give error if missing body', function(done) { + const reqNoBody = { + headers: { + 'x-node-id': nodeID, + 'x-node-pubkey': pubkey, + 'x-node-timestamp': timestamp, + 'x-node-signature': signature + }, + rawbody: null + }; + sandbox.stub(auth, 'checkTimestamp').returns(true); + sandbox.stub(auth, 'checkPubkey').returns(true); + sandbox.stub(auth, 'checkNodeID').returns(true); + auth.authFarmer(reqNoBody, res, function(err) { + expect(err).to.be.instanceOf(errors.BadRequestError); + done(); + }); + }); + it('will give error for invalid signature', function(done) { + sandbox.stub(auth, 'checkTimestamp').returns(true); + sandbox.stub(auth, 'checkPubkey').returns(true); + sandbox.stub(auth, 'checkNodeID').returns(true); + sandbox.stub(auth, 'checkSig').returns(false); + auth.authFarmer(req, res, function(err) { + expect(err).to.be.instanceOf(errors.BadRequestError); + done(); + }); + }); + it('will continue without error', function(done) { + sandbox.stub(auth, 'checkTimestamp').returns(true); + sandbox.stub(auth, 'checkPubkey').returns(true); + sandbox.stub(auth, 'checkNodeID').returns(true); + sandbox.stub(auth, 'checkSig').returns(true); + auth.authFarmer(req, res, done); + }); + }); + + + describe('#checkTimestamp', function() { + it('return false with timestamp below threshold', function() { + const clock = sandbox.useFakeTimers(); + clock.tick(1502390208007 + 300000); + let timestamp = (1502390208007 - 300000 - 1).toString(); + expect(auth.checkTimestamp(timestamp)).to.equal(false); + }); + it('return false with timestamp above threshold', function() { + const clock = sandbox.useFakeTimers(); + clock.tick(1502390208007 + 300000); + let timestamp = (1502390208007 + 600000 + 1).toString(); + expect(auth.checkTimestamp(timestamp)).to.equal(false); + }); + it('return true with timestamp within threshold', function() { + const clock = sandbox.useFakeTimers(); + clock.tick(1502390208007 + 300000); + let timestamp = (1502390208007 + 300000 + 1).toString(); + expect(auth.checkTimestamp(timestamp)).to.equal(true); + }); + }); + + describe('#checkNodeID', function() { + it('return false for invalid nodeID (nonhex)', function() { + const nodeID = 'somegarbage'; + const pubkey = '038cdc0b987405176647449b7f727444d263101f74e2a593d76ecedf11230706dd'; + expect(auth.checkNodeID(nodeID, pubkey)).to.equal(false); + }); + it('return false for invalid nodeID (does not match pubkey)', function() { + const nodeID = 'e6a498de631c6f3eba57da0e416881f9d4a6fca1'; + const pubkey = '038cdc0b987405176647449b7f727444d263101f74e2a593d76ecedf11230706dd'; + expect(auth.checkNodeID(nodeID, pubkey)).to.equal(false); + }); + it('return true for valid nodeID ', function() { + const nodeID = 'e6a498de631c6f3eba57da0e416881f9d4a6fca1'; + const pubkey = '03f716a870a72aaa61a75f5b06381ea1771f49c3a9866636007affc4ac06ef54b8'; + expect(auth.checkNodeID(nodeID, pubkey)).to.equal(true); + }); + }); + + describe('#checkPubkey', function() { + it('will fail if pubkey is an invalid format (nonhex doubles)', function() { + const pubkey = '38cdc0b987405176647449b7f727444d263101f74e2a593d76ecedf11230706dd'; + expect(auth.checkPubkey(pubkey)).to.equal(false); + }); + it('will fail if pubkey is an invalid format (nonhex)', function() { + const pubkey = 'z38cdc0b987405176647449b7f727444d263101f74e2a593d76ecedf11230706dd'; + expect(auth.checkPubkey(pubkey)).to.equal(false); + }); + it('return false if invalid pubkey (serialization)', function() { + const pubkey = '098cdc0b987405176647449b7f727444d263101f74e2a593d76ecedf11230706dd'; + expect(auth.checkPubkey(pubkey)).to.equal(false); + }); + it('return true for valid pubkey', function() { + const pubkey = '038cdc0b987405176647449b7f727444d263101f74e2a593d76ecedf11230706dd'; + expect(auth.checkPubkey(pubkey)).to.equal(true); + }); + }); + + describe('#checkSig', function() { + it('will verify that signature is correct', function() { + let privkey = '8e812246e61ea983efdd4d1c86e246832667a4e4b8fc2d9ff01c534c8a6d7681'; + let pubkey = '03ea58aff546b28bb748d560ad05bb78c0e1b9f5de8edc5021494833c73c224284'; + let req = { + headers: { + 'x-node-timestamp': '1502390208007', + 'x-node-pubkey': pubkey + }, + method: 'POST', + protocol: 'https', + originalUrl: '/contacts?someQueryArgument=value', + get: function(key) { + if (key === 'host') { + return 'api.storj.io'; + } + }, + rawbody: Buffer.from('{"key": "value"}', 'utf8') + }; + const sighash = auth.getSigHash(req); + const sigObj = secp256k1.sign(sighash, Buffer.from(privkey, 'hex')); + let sig = secp256k1.signatureExport(sigObj.signature).toString('hex'); + req.headers['x-node-signature'] = sig; + expect(auth.checkSig(req)).to.equal(true); + }); + it('will verify that signature is incorrect', function() { + let privkey = '8e812246e61ea983efdd4d1c86e246832667a4e4b8fc2d9ff01c534c8a6d7681'; + let pubkey = '03ea58aff546b28bb748d560ad05bb78c0e1b9f5de8edc5021494833c73c224284'; + let timestamp = '1502390208007'; + let sig = null; + let req = { + headers: { + 'x-node-timestamp': timestamp, + 'x-node-pubkey': pubkey, + 'x-node-signature': sig + }, + method: 'POST', + protocol: 'https', + originalUrl: '/contacts?someQueryArgument=value', + get: function(key) { + if (key === 'host') { + return 'api.storj.io'; + } + }, + rawbody: Buffer.from('{"key": "value"}', 'utf8') + }; + const sighash = auth.getSigHash(req); + const sigObj = secp256k1.sign(sighash, Buffer.from(privkey, 'hex')); + sig = secp256k1.signatureExport(sigObj.signature).toString('hex'); + // change the data so the signature fails + timestamp = '1502390208009'; + expect(auth.checkSig(req)).to.equal(false); + }); + }); + + describe('#isHexaString', function() { + it('return false for nonhex string', function() { + expect(auth.isHexString('zz')).to.equal(false); + }); + it('return false for nonhex string (incorrect bytes)', function() { + expect(auth.isHexString('aaa')).to.equal(false); + }); + it('return true for hex string', function() { + expect(auth.isHexString('038c')).to.equal(true); + }); + }); + + describe('#getSigHash', function() { + it('will get the expected hash from the request', function() { + let req = { + headers: { + 'x-node-timestamp': '1502390208007' + }, + method: 'POST', + protocol: 'https', + originalUrl: '/contacts?someQueryArgument=value', + get: function(key) { + if (key === 'host') { + return 'api.storj.io'; + } + }, + rawbody: Buffer.from('{"key": "value"}', 'utf8') + }; + const hash = auth.getSigHash(req); + expect(hash.toString('hex')).to.equal('59146f00725c9c052ef5ec6acd63f3842728c9d191ac146668204de6ed4a648b'); + }); + }); +}); diff --git a/test/server/middleware/pow.unit.js b/test/server/middleware/pow.unit.js new file mode 100644 index 00000000..0989eedf --- /dev/null +++ b/test/server/middleware/pow.unit.js @@ -0,0 +1,307 @@ +'use strict'; + +const chai = require('chai'); +const expect = chai.expect; +const sinon = require('sinon'); +const redis = require('redis').createClient(); +const pow = require('../../../lib/server/middleware/pow'); + +const MAX = 'ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff'; + +describe('POW Middleware', function() { + + after(function(done) { + redis.flushdb(done); + }); + + describe('#getPOWMiddleware', function() { + let challenge = '2db77b11eab714c46febb51a78d56d9b34b306d6fc46aa6e6e25a92b48eff4bf'; + let target = '00000fffffffffffffffffffffffffffffffffffffffffffffffffffffffffff'; + + let challenge2 = '4fccbb094116bf90e8dcea7e2b531b9a52574737a6cab9e77e2e5599fd35eb5b'; + let target2 = 'ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff'; + + let unknownChallenge = '328bfdaa0d2bf6c3c6495f06ffc2087e0b092fa534f1dea699b88f11b0082ab2'; + + before(function() { + redis.hset('contact-stats', 'count', 0); + redis.set('contact-' + challenge, target, 'EX', 3600); + redis.set('contact-' + challenge2, target2, 'EX', 3600); + }); + + it('will get invalid pow error', function(done) { + let middleware = pow.getPOWMiddleware(redis); + + let req = { + headers: { + 'x-challenge': challenge, + 'x-challenge-nonce': '0xdd170bf2' + } + }; + let res = {}; + + middleware(req, res, function(err) { + expect(err).to.be.instanceOf(Error); + expect(err.message).to.equal('Invalid proof of work'); + done(); + }); + + }); + + it('will get unknown challenge error', function(done) { + let middleware = pow.getPOWMiddleware(redis); + + let req = { + headers: { + 'x-challenge': unknownChallenge, + 'x-challenge-nonce': '0xdd170bf2' + } + }; + let res = {}; + + middleware(req, res, function(err) { + expect(err).to.be.instanceOf(Error); + expect(err.message).to.equal('Challenge not found'); + done(); + }); + + }); + + it('will increment count by one and remove challenge', function(done) { + let middleware = pow.getPOWMiddleware(redis); + + let req = { + headers: { + 'x-challenge': challenge2, + 'x-challenge-nonce': '0xdd170bf2' + } + }; + let res = {}; + + middleware(req, res, function(err) { + if (err) { + return done(err); + } + + redis.hgetall('contact-stats', function(err, stats) { + if (err) { + return done(err); + } + expect(stats.count).to.equal('1'); + + redis.get('contact-' + challenge2, function(err, target) { + if (err) { + return done(err); + } + expect(target).to.equal(null); + done(); + }); + }); + }); + + }); + }); + + describe('#checkInitTarget', function() { + const sandbox = sinon.sandbox.create(); + beforeEach(function() { + redis.del('contact-stats'); + }); + afterEach(() => { + sandbox.restore(); + }); + + it('will init target if not set', function(done) { + sandbox.stub(pow, 'initTarget').callsArg(1); + pow.checkInitTarget(redis, (err) => { + if (err) { + return done(err); + } + expect(pow.initTarget.callCount).to.equal(1); + done(); + }); + }); + }); + + describe('#initTarget', function() { + it('will set target to initial values', function(done) { + const initialTarget = 'fffffffffffffffffffffffffffffff' + + 'fffffffffffffffffffffffffffffffff'; + pow.initTarget(redis, (err) => { + if (err) { + return done(err); + } + redis.hgetall('contact-stats', (err, stats) => { + if (err) { + return done(err); + } + expect(stats.count).to.equal('0'); + expect(stats.timestamp).to.equal('0'); + expect(stats.target).to.equal(initialTarget); + done(); + }); + }); + }); + }); + + describe('#getTarget', function() { + let beginTime = 0; + let clock = null; + const count = 1000; + const startTarget = '0000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff'; + const moreTarget = '00008020c470b2c58f96b6655747ba7ec17c0bbf25f0299e34081a55c1a88168'; + const lessTarget = '000200831243a46f8b7e22f09add3b84e6593ad9c5aea066e841726623f65458'; + + const sandbox = sinon.sandbox.create(); + + beforeEach(function() { + clock = sandbox.useFakeTimers(); + beginTime = Date.now(); + redis.hset('contact-stats', 'timestamp', beginTime); + redis.hset('contact-stats', 'count', count); + redis.hset('contact-stats', 'target', startTarget); + }); + + afterEach(() => { + sandbox.restore(); + clock.restore(); + }); + + it('it will adjust the difficulty (less)', function(done) { + const opts = { + retargetPeriod: 1000, + retargetCount: 2000 + }; + clock.tick(1001); + pow.getTarget(redis, opts, function(err, target) { + if (err) { + return done(err); + } + expect(target).to.equal(lessTarget); + redis.hgetall('contact-stats', (err, stats) => { + expect(parseInt(parseInt(stats.timestamp))).to.equal(Date.now()); + expect(parseInt(stats.count)).to.equal(0); + done(); + }); + }); + }); + + it('it will adjust the difficulty (more)', function(done) { + const opts = { + retargetPeriod: 1000, + retargetCount: 500 + }; + clock.tick(1001); + pow.getTarget(redis, opts, function(err, target) { + if (err) { + return done(err); + } + expect(target).to.equal(moreTarget); + redis.hgetall('contact-stats', (err, stats) => { + expect(parseInt(parseInt(stats.timestamp))).to.equal(Date.now()); + expect(parseInt(stats.count)).to.equal(0); + done(); + }); + }); + }); + + it('will not adjust the difficulty', function(done) { + const opts = { + retargetPeriod: 1000, + retargetCount: 500 + }; + clock.tick(999); + pow.getTarget(redis, opts, function(err, target) { + if (err) { + return done(err); + } + expect(target).to.equal(startTarget); + redis.hgetall('contact-stats', (err, stats) => { + expect(parseInt(stats.timestamp)).to.equal(beginTime); + expect(parseInt(stats.count)).to.equal(count); + done(); + }); + }); + }); + + it('will adjust from init stats', function(done) { + redis.hset('contact-stats', 'timestamp', 0); + redis.hset('contact-stats', 'count', 0); + redis.hset('contact-stats', 'target', MAX); + const opts = { + retargetPeriod: 1000, + retargetCount: 500 + }; + clock.tick(1504182357109); + pow.getTarget(redis, opts, function(err, target) { + if (err) { + return done(err); + } + expect(target).to.equal('ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff'); + redis.hgetall('contact-stats', (err, stats) => { + expect(stats.timestamp).to.equal('1504182357109'); + expect(stats.count).to.equal('0'); + done(); + }); + }); + + }); + + }); + + describe('#getChallenge', function() { + let beginTime = 0; + let clock = null; + const count = 1000; + const startTarget = '0000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff'; + const sandbox = sinon.sandbox.create(); + const opts = { + retargetPeriod: 1000, + retargetCount: 500 + }; + + beforeEach(function() { + clock = sandbox.useFakeTimers(); + beginTime = Date.now(); + redis.hset('contact-stats', 'timestamp', beginTime); + redis.hset('contact-stats', 'count', count); + redis.hset('contact-stats', 'target', startTarget); + }); + + afterEach(() => { + sandbox.restore(); + clock.restore(); + }); + + it('will create a new challenge', function(done) { + pow.getChallenge(redis, opts, function(err, data) { + if (err) { + return done(err); + } + expect(data.challenge.length).to.equal(32 * 2); + expect(data.target).to.equal(startTarget); + done(); + }); + }); + + it('will handle error from getTarget', function(done) { + sandbox.stub(pow, 'getTarget').callsArgWith(2, new Error('test')); + pow.getChallenge(redis, opts, function(err) { + expect(err).to.be.instanceOf(Error); + expect(err.message).to.equal('test'); + done(); + }); + }); + + it('will handle error from db', function(done) { + sandbox.stub(redis, 'set').callsArgWith(4, new Error('test')); + pow.getChallenge(redis, opts, function(err) { + expect(err).to.be.instanceOf(Error); + expect(err.message).to.equal('test'); + done(); + }); + }); + + }); + +}); diff --git a/test/server/middleware/raw-body.unit.js b/test/server/middleware/raw-body.unit.js new file mode 100644 index 00000000..b622f00f --- /dev/null +++ b/test/server/middleware/raw-body.unit.js @@ -0,0 +1,62 @@ +'use strict'; + +const util = require('util'); +const ReadableStream = require('stream').Readable; +const expect = require('chai').expect; +const sinon = require('sinon'); + +const rawbody = require('../../../lib/server/middleware/raw-body'); + +describe('Raw Body Middleware', function() { + it('will not try to parse multipart', function(done) { + var req = { + get: sinon.stub().returns('multipart/form-data'), + pipe: sinon.stub() + }; + var res = {}; + rawbody(req, res, function() { + expect(req.pipe.callCount).to.equal(0); + expect(req.body).to.equal(undefined); + expect(req.rawbody).to.equal(undefined); + done(); + }); + }); + it('will set rawbody and body', function(done) { + var res = {}; + function Stream(options) { + ReadableStream.call(this, options); + } + util.inherits(Stream, ReadableStream); + var data = '{"hello": "world"}'; + Stream.prototype._read = function() { + this.push(data); + data = null; + }; + var req = new Stream(); + req.get = sinon.stub(); + rawbody(req, res, function() { + expect(req.rawbody.toString()).to.equal('{"hello": "world"}'); + expect(req.body).to.deep.equal({hello: 'world'}); + done(); + }); + }); + it('will set rawbody and body with JSON parse error', function(done) { + var res = {}; + function Stream(options) { + ReadableStream.call(this, options); + } + util.inherits(Stream, ReadableStream); + var data = '{"hello":'; + Stream.prototype._read = function() { + this.push(data); + data = null; + }; + var req = new Stream(); + req.get = sinon.stub(); + rawbody(req, res, function() { + expect(req.rawbody.toString()).to.equal('{"hello":'); + expect(req.body).to.deep.equal({}); + done(); + }); + }); +}); diff --git a/test/server/routes/frames.unit.js b/test/server/routes/frames.unit.js index 1ad5b2db..01483619 100644 --- a/test/server/routes/frames.unit.js +++ b/test/server/routes/frames.unit.js @@ -132,6 +132,197 @@ describe('FramesRouter', function() { }); }); + describe('#_selectFarmers', function() { + const sandbox = sinon.sandbox.create(); + afterEach(() => sandbox.restore()); + + it('will handle error from contact query', function(done) { + sandbox.stub( + framesRouter.storage.models.Contact, + 'find' + ).returns({ + sort: sandbox.stub().returns({ + limit: sandbox.stub().returns({ + exec: sandbox.stub().callsArgWith(0, new Error('test')) + }) + }) + }); + + let excluded = []; + framesRouter._selectFarmers(excluded, (err) => { + expect(err).to.be.instanceOf(Error); + expect(err.message).to.equal('test'); + done(); + }); + }); + + it('combine results from two queries', function(done) { + sandbox.stub( + framesRouter.storage.models.Contact, + 'find' + ).returns({ + sort: sandbox.stub().returns({ + limit: sandbox.stub().returns({ + exec: sandbox.stub().callsArgWith(0, null, ['a']) + }) + }) + }); + + + let excluded = []; + framesRouter._selectFarmers(excluded, (err, results) => { + if (err) { + return done(err); + } + expect(results).to.eql(['a', 'a']); + done(); + }); + }); + + }); + + describe('#_publishContract', function() { + const sandbox = sinon.sandbox.create(); + afterEach(() => sandbox.restore()); + + it('will create item on error (e.g. no contract)', function(done) { + let item = { + addContract: sandbox.stub(), + addAuditRecords: sandbox.stub() + }; + let StorageItem = sandbox.stub(storj, 'StorageItem').returns(item); + + let data = { + contact: { + address: '127.0.0.1', + port: 1001 + }, + contract: {} + }; + sandbox.stub(framesRouter.network, 'publishContract').callsArgWith(2, null, data); + sandbox.stub(framesRouter.contracts, 'load').callsArgWith(1, new Error('Not found')); + sandbox.stub(framesRouter.contracts, 'save').callsArg(1); + + let nodes = []; + let contract = { + get: function(key) { + if (key === 'data_hash') { + return 'data_hash'; + } + } + }; + let audit = {}; + framesRouter._publishContract(nodes, contract, audit, (err) => { + if (err) { + return done(err); + } + expect(StorageItem.callCount).to.equal(1); + expect(item.addContract.callCount).to.equal(1); + expect(item.addAuditRecords.callCount).to.equal(1); + done(); + }); + }); + + it('will handle error when publishing contract', function(done) { + sandbox.stub(framesRouter.network, 'publishContract').callsArgWith(2, new Error('test')); + + let item = { + addContract: sandbox.stub(), + addAuditRecords: sandbox.stub() + }; + sandbox.stub(framesRouter.contracts, 'load').callsArgWith(1, null, item); + + let nodes = []; + let contract = { + get: function(key) { + if (key === 'data_hash') { + return 'data_hash'; + } + } + }; + let audit = {}; + framesRouter._publishContract(nodes, contract, audit, (err) => { + expect(err).to.be.instanceOf(Error); + expect(err.message).to.equal('test'); + done(); + }); + }); + + it('will handle error when saving contract', function(done) { + let data = { + contact: { + address: '127.0.0.1', + port: 1001 + }, + contract: {} + }; + sandbox.stub(framesRouter.network, 'publishContract').callsArgWith(2, null, data); + + let item = { + addContract: sandbox.stub(), + addAuditRecords: sandbox.stub() + }; + sandbox.stub(framesRouter.contracts, 'load').callsArgWith(1, null, item); + sandbox.stub(framesRouter.contracts, 'save').callsArgWith(1, new Error('test')); + + let nodes = []; + let contract = { + get: function(key) { + if (key === 'data_hash') { + return 'data_hash'; + } + } + }; + let audit = {}; + framesRouter._publishContract(nodes, contract, audit, (err) => { + expect(err).to.be.instanceOf(Error); + expect(err.message).to.equal('test'); + done(); + }); + }); + + it('save contract and return completed contract and farmer', function(done) { + let data = { + contact: { + address: '127.0.0.1', + port: 1001 + }, + contract: {}, + token: 'token' + }; + sandbox.stub(framesRouter.network, 'publishContract').callsArgWith(2, null, data); + + let item = { + addContract: sandbox.stub(), + addAuditRecords: sandbox.stub() + }; + sandbox.stub(framesRouter.contracts, 'load').callsArgWith(1, null, item); + sandbox.stub(framesRouter.contracts, 'save').callsArg(1); + + let nodes = []; + let contract = { + get: function(key) { + if (key === 'data_hash') { + return 'data_hash'; + } + } + }; + let audit = {}; + framesRouter._publishContract(nodes, contract, audit, (err, f, c, t) => { + if (err) { + return done(err); + } + expect(f).to.be.instanceOf(storj.Contact); + expect(f.port).to.equal(1001); + expect(f.address).to.equal('127.0.0.1'); + expect(c).to.be.instanceOf(storj.Contract); + expect(t).to.equal('token'); + done(); + }); + }); + + }); + describe('#_getContractForShard', function() { const sandbox = sinon.sandbox.create(); afterEach(() => sandbox.restore());