Merge pull request #464 from braydonf/offers

Implement SIP6

lib/config.js

@@ -56,6 +56,14 @@ const DEFAULTS = {
     from: '[email protected]'
   },
   application: {
+    activateSIP6: false,
+    powOpts: {
+      retargetPeriod: 10000, // milliseconds
+      retargetCount: 10, // per retargetPeriod
+    },
+    publishBenchThreshold: 4000, // milliseconds
+    publishTotal: 36, // number of farmers to publish in active pool
+    publishBenchTotal: 9, // number of farmers to publish in bench pool
     shardsPerMinute: 1000,
     farmerTimeoutIgnore: '10m',
     freeTier: {

lib/engine.js

@@ -12,6 +12,7 @@ const Config = require('./config');
 const Storage = require('storj-service-storage-models');
 const middleware = require('storj-service-middleware');
 const Server = require('./server');
+const pow = require('./server/middleware/pow');
 const Mailer = require('storj-service-mailer');
 const log = require('./logger');
 const ComplexClient = require('storj-complex').createClient;
@@ -62,6 +63,11 @@ Engine.prototype.start = function(callback) {
   this.redis = require('redis').createClient(this._config.redis);
   this.redis.on('ready', () => {
     log.info('connected to redis');
+    pow.checkInitTarget(this.redis, (err) => {
+      if (err) {
+        log.error('unable to initialize pow settings', err);
+      }
+    });
   });
   this.redis.on('error', (err) => {
     log.error('error connecting to redis', err);

lib/server/middleware/farmer-auth.js

@@ -0,0 +1,109 @@
+'use strict';
+
+const errors = require('storj-service-error-types');
+const crypto = require('crypto');
+const secp256k1 = require('secp256k1');
+
+const THRESHOLD = 300000;
+
+function isHexString(a) {
+  if (typeof a !== 'string') {
+    return false;
+  }
+  return /^([0-9a-fA-F]{2})+$/.test(a);
+}
+
+function getSigHash(req) {
+  const hasher = crypto.createHash('sha256');
+  const timestamp = req.headers['x-node-timestamp'];
+  const url = req.protocol + '://' + req.get('host') + req.originalUrl;
+  hasher.update(req.method);
+  hasher.update(url);
+  hasher.update(timestamp);
+  hasher.update(req.rawbody);
+  return hasher.digest();
+}
+
+function checkSig(req) {
+  const sighash = getSigHash(req);
+  let sigstr = req.headers['x-node-signature'];
+  if (!isHexString(sigstr)) {
+    return false;
+  }
+  const buf = Buffer.from(req.headers['x-node-signature'], 'hex');
+  const sig = secp256k1.signatureImport(buf);
+  const pubkey = Buffer.from(req.headers['x-node-pubkey'], 'hex');
+  return secp256k1.verify(sighash, sig, pubkey);
+}
+
+function checkPubkey(pubkey) {
+  if (!isHexString(pubkey)) {
+    return false;
+  }
+  const buf = Buffer.from(pubkey, 'hex');
+  return secp256k1.publicKeyVerify(buf);
+}
+
+function checkTimestamp(ts) {
+  const timestamp = parseInt(ts);
+  if (!Number.isSafeInteger(timestamp)) {
+    return false;
+  }
+  const now = Date.now();
+  if (timestamp < now - THRESHOLD || timestamp > now + THRESHOLD) {
+    return false;
+  }
+  return true;
+}
+
+function checkNodeID(nodeID, pubkey) {
+  if (!nodeID || nodeID.length !== 40 || !isHexString(nodeID)) {
+    return false;
+  }
+  const sha256 = crypto.createHash('sha256');
+  const rmd160 = crypto.createHash('rmd160');
+  sha256.update(Buffer.from(pubkey, 'hex'));
+  rmd160.update(sha256.digest());
+  if (rmd160.digest('hex') !== nodeID) {
+    return false;
+  }
+  return true;
+}
+
+function authFarmer(req, res, next) {
+  const nodeID = req.headers['x-node-id'];
+  const timestamp = req.headers['x-node-timestamp'];
+  const pubkey = req.headers['x-node-pubkey'];
+
+  if (!module.exports.checkTimestamp(timestamp)) {
+    return next(new errors.BadRequestError('Invalid timestamp header'));
+  }
+
+  if (!module.exports.checkPubkey(pubkey)) {
+    return next(new errors.BadRequestError('Invalid pubkey header'));
+  }
+
+  if (!module.exports.checkNodeID(nodeID, pubkey)) {
+    return next(new errors.BadRequestError('Invalid nodeID header'));
+  }
+
+  if (!req.rawbody || !Buffer.isBuffer(req.rawbody)) {
+    return next(new errors.BadRequestError('Invalid body'));
+  }
+
+  if (!module.exports.checkSig(req)) {
+    return next(new errors.BadRequestError('Invalid signature header'));
+  }
+
+  next();
+}
+
+module.exports = {
+  authFarmer: authFarmer,
+  getSigHash: getSigHash,
+  isHexString: isHexString,
+  checkTimestamp: checkTimestamp,
+  checkNodeID: checkNodeID,
+  checkPubkey: checkPubkey,
+  checkSig: checkSig
+};

lib/server/middleware/pow.js

@@ -0,0 +1,190 @@
+'use strict';
+
+const async = require('async');
+const assert = require('assert');
+const scrypt = require('scrypt');
+const crypto = require('crypto');
+const BN = require('bn.js');
+
+const CHALLENGE_TTL_SECONDS = 3600;
+const MAX = 'ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff';
+const MAXBN = new BN(MAX, 16);
+
+function getPOWMiddleware(db) {
+
+  return function(req, res, next) {
+
+    const challenge = req.headers['x-challenge'];
+    const nonce = req.headers['x-challenge-nonce'];
+
+    let salt = Buffer.alloc(8, 0);
+    salt.writeDoubleBE(parseInt(nonce));
+
+    const key = 'contact-' + challenge;
+    const scryptOpts = { N: Math.pow(2, 10), r: 1, p: 1 };
+
+    db.get(key, function(err, target) {
+      if (err) {
+        return next(err);
+      }
+
+      if (!target) {
+        return next(new Error('Challenge not found'));
+      }
+
+      scrypt.hash(challenge, scryptOpts, 32, salt, function(err, result) {
+        if (err) {
+          return next(err);
+        }
+
+        // Check the proof of work
+        if (result.toString('hex') > target) {
+          return next(new Error('Invalid proof of work'));
+        }
+
+        // Increase the count and remove the challenge
+        db.hincrby('contact-stats', 'count', 1);
+        db.del(key);
+
+        return next();
+      });
+    });
+  };
+}
+
+function checkInitTarget(db, callback) {
+  db.hgetall('contact-stats', function(err, stats) {
+    if (err) {
+      callback(err);
+    } else if (!stats) {
+      module.exports.initTarget(db, callback);
+    } else {
+      callback();
+    }
+  });
+}
+
+function initTarget(db, callback) {
+  const initialTarget = 'fffffffffffffffffffffffffffffff' +
+        'fffffffffffffffffffffffffffffffff';
+  async.series([
+    (next) => {
+      db.hset('contact-stats', 'target', initialTarget, next);
+    },
+    (next) => {
+      db.hset('contact-stats', 'timestamp', 0, next);
+    },
+    (next) => {
+      db.hset('contact-stats', 'count', 0, next);
+    },
+  ], callback);
+}
+
+function getTarget(db, opts, callback) {
+
+  const precision = 100000000;
+  const precisionBN = new BN(precision);
+  const retargetPeriod = opts.retargetPeriod;
+  const count = opts.retargetCount;
+
+  assert(Number.isSafeInteger(count),
+         'retargetCount is expected to be a safe integer');
+  assert(Number.isSafeInteger(retargetPeriod),
+         'retargetPeriod is expected to be a safe integer');
+
+  /* jshint maxstatements: 100 */
+  db.hgetall('contact-stats', function(err, stats) {
+    if (err) {
+      return callback(err);
+    }
+
+    if (!stats) {
+      return callback(new Error('Unknown pow settings'));
+    }
+
+    if (!Number.isSafeInteger(parseInt(stats.count)) ||
+        !Number.isSafeInteger(parseInt(stats.timestamp))) {
+      return callback(new Error('Invalid pow settings'));
+    }
+
+    const now = Date.now();
+    const actual = parseInt(stats.count) || 1;
+    const timestamp = parseInt(stats.timestamp);
+
+    if (now > timestamp + retargetPeriod) {
+      const timeDelta = now - timestamp;
+      const expectedRatio = retargetPeriod / count;
+      const actualRatio = timeDelta / actual;
+      const adjustmentAmount = actualRatio / expectedRatio;
+
+      let adjustmentNum = adjustmentAmount * precision;
+      let adjustment = null;
+      if (adjustmentNum > Number.MAX_SAFE_INTEGER) {
+        adjustment = new BN(adjustmentAmount);
+        adjustment.mul(new BN(precision));
+      } else {
+        adjustment = new BN(adjustmentNum);
+      }
+
+      const target = new BN(stats.target, 16);
+      const newTargetBN = target.mul(adjustment).div(precisionBN);
+      let newTarget = null;
+      if (newTargetBN.cmp(MAXBN) > 0) {
+        newTarget = MAXBN.toString(16, 32);
+      } else {
+        newTarget = newTargetBN.toString(16, 32);
+      }
+
+      async.series([
+        (next) => {
+          db.hset('contact-stats', 'target', newTarget, next);
+        },
+        (next) => {
+          db.hset('contact-stats', 'timestamp', now, next);
+        },
+        (next) => {
+          db.hset('contact-stats', 'count', 0, next);
+        },
+      ], (err) => {
+        if (err) {
+          return callback(err);
+        }
+        callback(null, newTarget);
+      });
+
+    } else {
+      callback(null, stats.target);
+    }
+
+  });
+
+}
+
+function getChallenge(db, opts, callback) {
+  module.exports.getTarget(db, opts, function(err, target) {
+    if (err) {
+      return callback(err);
+    }
+
+    const challenge = crypto.randomBytes(32).toString('hex');
+    const key = 'contact-' + challenge;
+
+    db.set(key, target, 'EX', CHALLENGE_TTL_SECONDS, function(err) {
+      if (err) {
+        return callback(err);
+      }
+      callback(null, {
+        challenge: challenge,
+        target: target
+      });
+    });
+  });
+}
+
+module.exports = {
+  getChallenge: getChallenge,
+  getTarget: getTarget,
+  checkInitTarget: checkInitTarget,
+  initTarget: initTarget,
+  getPOWMiddleware: getPOWMiddleware
+};

lib/server/middleware/raw-body.js

@@ -0,0 +1,22 @@
+'use strict';
+
+const concat = require('concat-stream');
+
+module.exports = function rawbody(req, res, next) {
+  // Do not buffer the request body for file uploads
+  if (req.get('Content-Type') === 'multipart/form-data') {
+    return next();
+  }
+
+  req.pipe(concat(function(data) {
+    req.rawbody = data;
+
+    try {
+      req.body = JSON.parse(req.rawbody.toString());
+    } catch (err) {
+      req.body = {};
+    }
+
+    next();
+  }));
+};