Skip to content

Commit

Permalink
feat: support args.checkAddress
Browse files Browse the repository at this point in the history
  • Loading branch information
@dead-horse
dead-horse committed Mar 24, 2018
1 parent 691ef84 commit cf00d1f
Show file tree
Hide file tree
Showing 6 changed files with 249 additions and 103 deletions.
2 changes: 2 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,8 @@ httpclient.request('http://nodejs.org', function (err, body) {
- ***timing*** Boolean - Enable timing or not, default is `false`.
- ***enableProxy*** Boolean - Enable proxy request, default is `false`.
- ***proxy*** String | Object - proxy agent uri or options, default is `null`.
- ***lookup*** Function - Custom DNS lookup function, default is `dns.lookup`. Require node >= 4.0.0(for http protocol) and node >=8(for https protocol)
- ***checkAddress*** Function: optional, check request address to protect from SSRF and similar attacks. It relays on `lookup` and have require same node version.
- ***callback(err, data, res)*** Function - Optional callback.
- **err** Error - Would be `null` if no error accured.
- **data** Buffer | Object - The data responsed. Would be a Buffer if `dataType` is set to `text` or an JSON parsed into Object if it's set to `json`.
Expand Down
240 changes: 139 additions & 101 deletions lib/urllib.js
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
'use strict';

var debug = require('debug')('urllib');
var dns = require('dns');
var http = require('http');
var https = require('https');
var urlutil = require('url');
var util = require('util');
var qs = require('qs');
var ip = require('ip');
var querystring = require('querystring');
var zlib = require('zlib');
var ua = require('default-user-agent');
Expand Down Expand Up @@ -119,6 +121,7 @@ var KEEP_ALIVE_RE = /^timeout=(\d+)/i;
* Require node >= 4.0.0 and only work on `http` protocol.
* - {Boolean} [enableProxy]: optional, enable proxy request. Default is `false`.
* - {String|Object} [proxy]: optional proxy agent uri or options. Default is `null`.
* - {Function} checkAddress: optional, check request address to protect from SSRF and similar attacks.
* @param {Function} [callback]: callback(error, data, res). If missing callback, will return a promise object.
* @return {HttpRequest} req object.
* @api public
Expand Down Expand Up @@ -243,6 +246,23 @@ exports.requestWithCallback = function requestWithCallback(url, args, callback)
agent = proxyTunnelAgent;
}

var _lookup = args.lookup || dns.lookup;
var lookup = _lookup;

// check address to protect from SSRF and similar attacks
if (args.checkAddress) {
lookup = function(host, dnsopts, callback) {
_lookup(host, dnsopts, function emitLookup(err, ip, addressType) {
// add check address logic in custom dns lookup
if (!err && !args.checkAddress(ip)) {
err = new Error('illegal address');
err.name = 'IllegalAddressError';
}
callback(err, ip, addressType);
});
};
}

var options = {
host: parsedUrl.hostname || parsedUrl.host || 'localhost',
path: parsedUrl.path || '/',
Expand All @@ -252,9 +272,9 @@ exports.requestWithCallback = function requestWithCallback(url, args, callback)
headers: {},
// default is dns.lookup
// https://github.com/nodejs/node/blob/master/lib/net.js#L986
// custom dnslookup require node >= 4.0.0
// custom dnslookup require node >= 4.0.0 (for http), node >=8 (for https)
// https://github.com/nodejs/node/blob/archived-io.js-v0.12/lib/net.js#L952
lookup: args.lookup,
lookup: lookup,
};
if (args.headers) {
for (var k in args.headers) {
Expand Down Expand Up @@ -382,6 +402,8 @@ exports.requestWithCallback = function requestWithCallback(url, args, callback)
}
}

var req;

function done(err, data, res) {
cancelResponseTimer();
if (!callback) {
Expand Down Expand Up @@ -774,18 +796,100 @@ exports.requestWithCallback = function requestWithCallback(url, args, callback)
}, responseTimeout);
}

var req;
// request headers checker will throw error
try {
req = httplib.request(options, onResponse);
} catch (err) {
return done(err);
}
function handleRequestEvents(req) {
if (timing) {
// request sent
req.on('finish', function() {
timing.requestSent = Date.now() - requestStartTime;
});
}

req.once('socket', function (socket) {
if (timing) {
// socket queuing time
timing.queuing = Date.now() - requestStartTime;
}

// https://github.com/nodejs/node/blob/master/lib/net.js#L377
// https://github.com/nodejs/node/blob/v0.10.40-release/lib/net.js#L352
// should use socket.socket on 0.10.x
if (isNode010 && socket.socket) {
socket = socket.socket;
}

var readyState = socket.readyState;
if (readyState === 'opening') {
socket.once('lookup', function(err, ip, addressType) {
debug('Request#%d %s lookup: %s, %s, %s', reqId, url, err, ip, addressType);
if (timing) {
timing.dnslookup = Date.now() - requestStartTime;
}
if (ip) {
remoteAddress = ip;
}
});
socket.once('connect', function() {
if (timing) {
// socket connected
timing.connected = Date.now() - requestStartTime;
}

// cancel socket timer at first and start tick for TTFB
cancelConnectTimer();
startResposneTimer();

debug('Request#%d %s new socket connected', reqId, url);
connected = true;
if (!remoteAddress) {
remoteAddress = socket.remoteAddress;
}
remotePort = socket.remotePort;
});
return;
}

debug('Request#%d %s reuse socket connected, readyState: %s', reqId, url, readyState);
connected = true;
keepAliveSocket = true;
if (!remoteAddress) {
remoteAddress = socket.remoteAddress;
}
remotePort = socket.remotePort;

// reuse socket, timer should be canceled.
cancelConnectTimer();
startResposneTimer();
});

req.on('error', function (err) {
if (err.name === 'Error') {
err.name = connected ? 'ResponseError' : 'RequestError';
}
err.message += ' (req "error")';
debug('Request#%d %s `req error` event emit, %s: %s', reqId, url, err.name, err.message);
done(__err || err);
});

if (writeStream) {
writeStream.once('error', function (err) {
err.message += ' (writeStream "error")';
__err = err;
debug('Request#%d %s `writeStream error` event emit, %s: %s', reqId, url, err.name, err.message);
abortRequest();
});
}

// environment detection: browser or nodejs
if (typeof(window) === 'undefined') {
// start connect timer just after `request` return, and just in nodejs environment
startConnectTimer();
if (args.stream) {
args.stream.pipe(req);
args.stream.once('error', function (err) {
err.message += ' (stream "error")';
__err = err;
debug('Request#%d %s `readStream error` event emit, %s: %s', reqId, url, err.name, err.message);
abortRequest();
});
} else {
req.end(body);
}
}

function abortRequest() {
Expand All @@ -798,101 +902,35 @@ exports.requestWithCallback = function requestWithCallback(url, args, callback)
req.abort();
}

if (timing) {
// request sent
req.on('finish', function() {
timing.requestSent = Date.now() - requestStartTime;
});
}

req.once('socket', function (socket) {
if (timing) {
// socket queuing time
timing.queuing = Date.now() - requestStartTime;
}

// https://github.com/nodejs/node/blob/master/lib/net.js#L377
// https://github.com/nodejs/node/blob/v0.10.40-release/lib/net.js#L352
// should use socket.socket on 0.10.x
if (isNode010 && socket.socket) {
socket = socket.socket;
if (args.checkAddress) {
// if request hostname is ip, custom lookup wont excute
if (ip.isV4Format(parsedUrl.hostname) || ip.isV6Format(parsedUrl.hostname)) {
if (!args.checkAddress(parsedUrl.hostname)) {
var err = new Error('illegal address');
err.name = 'IllegalAddressError';
return done(err);
}
}
}

var readyState = socket.readyState;
if (readyState === 'opening') {
socket.once('lookup', function(err, ip, addressType) {
debug('Request#%d %s lookup: %s, %s, %s', reqId, url, err, ip, addressType);
if (timing) {
timing.dnslookup = Date.now() - requestStartTime;
}
if (ip) {
remoteAddress = ip;
}
});
socket.once('connect', function() {
if (timing) {
// socket connected
timing.connected = Date.now() - requestStartTime;
}

// cancel socket timer at first and start tick for TTFB
cancelConnectTimer();
startResposneTimer();

debug('Request#%d %s new socket connected', reqId, url);
connected = true;
if (!remoteAddress) {
remoteAddress = socket.remoteAddress;
}
remotePort = socket.remotePort;
});
function triggerRequest() {
// request headers checker will throw error
try {
req = httplib.request(options, onResponse);
req.requestId = reqId;
handleRequestEvents(req);
} catch (err) {
done(err);
return;
}

debug('Request#%d %s reuse socket connected, readyState: %s', reqId, url, readyState);
connected = true;
keepAliveSocket = true;
if (!remoteAddress) {
remoteAddress = socket.remoteAddress;
}
remotePort = socket.remotePort;

// reuse socket, timer should be canceled.
cancelConnectTimer();
startResposneTimer();
});

req.on('error', function (err) {
if (err.name === 'Error') {
err.name = connected ? 'ResponseError' : 'RequestError';
// environment detection: browser or nodejs
if (typeof(window) === 'undefined') {
// start connect timer just after `request` return, and just in nodejs environment
startConnectTimer();
}
err.message += ' (req "error")';
debug('Request#%d %s `req error` event emit, %s: %s', reqId, url, err.name, err.message);
done(__err || err);
});

if (writeStream) {
writeStream.once('error', function (err) {
err.message += ' (writeStream "error")';
__err = err;
debug('Request#%d %s `writeStream error` event emit, %s: %s', reqId, url, err.name, err.message);
abortRequest();
});
}

if (args.stream) {
args.stream.pipe(req);
args.stream.once('error', function (err) {
err.message += ' (stream "error")';
__err = err;
debug('Request#%d %s `readStream error` event emit, %s: %s', reqId, url, err.name, err.message);
abortRequest();
});
} else {
req.end(body);
}

req.requestId = reqId;
triggerRequest();
return req;
};

Expand Down
3 changes: 2 additions & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
"ee-first": "~1.1.1",
"humanize-ms": "^1.2.0",
"iconv-lite": "^0.4.15",
"ip": "^1.1.5",
"proxy-agent": "^2.1.0",
"qs": "^6.4.0",
"statuses": "^1.3.1",
Expand All @@ -56,7 +57,7 @@
"istanbul": "*",
"jshint": "*",
"mocha": "3",
"muk": "^0.4.0",
"muk": "^0.5.3",
"pedding": "^1.1.0",
"power-assert": "^1.4.2",
"semver": "5",
Expand Down
3 changes: 2 additions & 1 deletion test/config.js
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
'use strict';

module.exports = process.env.CI ? {
npmWeb: 'https://www.npmjs.com',
// npmjs.com do not support gzip now
npmWeb: 'https://cnpmjs.org',
npmRegistry: 'https://registry.npmjs.com',
npmHttpRegistry: 'http://registry.npmjs.com',
} : {
Expand Down
8 changes: 8 additions & 0 deletions test/fixtures/server.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,14 @@ var server = http.createServer(function (req, res) {
} else if (req.url === '/redirect_no_location') {
res.statusCode = 302;
return res.end('I am 302 body');
} else if (req.url === '/redirect_to_ip') {
res.statusCode = 302;
res.setHeader('Location', 'http://10.10.10.10/');
return res.end('Redirect to http://10.10.10.10/');
} else if (req.url === '/redirect_to_domain') {
res.statusCode = 302;
res.setHeader('Location', 'https://www.google.com/');
return res.end('Redirect to https://www.google.com/');
} else if (req.url === '/204') {
res.statusCode = 204;
return res.end();
Expand Down
Loading

0 comments on commit cf00d1f

Please sign in to comment.