From a6ce20bfa7585b63a9e51719b00c0f27040ed31a Mon Sep 17 00:00:00 2001 From: Matias Lopez Date: Wed, 27 Nov 2019 19:40:00 -0500 Subject: [PATCH 1/7] Delegate proxy support to `request` --- lib/install.js | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/lib/install.js b/lib/install.js index f68cd7fd6d..c76672f1fb 100644 --- a/lib/install.js +++ b/lib/install.js @@ -409,18 +409,31 @@ function download (gyp, env, url) { requestOpts.ca = readCAFile(cafile) } + /** + * Proxy support is delegated to the `request` module. `request` already + * supports proxies using the ones provided in the environment. For + * compatibility those provided using the CLI or `npm config` will be + * attached to the environment, keeping previous precedence, for + * `request` to use. + * + * See: https://github.com/request/request/blob/master/lib/getProxyFromURI.js + */ + // basic support for a proxy server - var proxyUrl = gyp.opts.proxy || + env.http_proxy = env.HTTP_PROXY = gyp.opts.proxy || env.http_proxy || env.HTTP_PROXY || env.npm_config_proxy - if (proxyUrl) { - if (/^https?:\/\//i.test(proxyUrl)) { - log.verbose('download', 'using proxy url: "%s"', proxyUrl) - requestOpts.proxy = proxyUrl - } else { - log.warn('download', 'ignoring invalid "proxy" config setting: "%s"', proxyUrl) - } + if (env.http_proxy) { + log.verbose('download', 'using proxy url: "%s"', env.http_proxy) + } + + // basic support for proxy overrides + env.no_proxy = env.NO_PROXY = env.no_proxy || + env.NO_PROXY || + env.npm_config_noproxy + if (env.no_proxy) { + log.verbose('download', 'using proxy url: "%s"', env.no_proxy) } var req = request(requestOpts) From df628aeb3fb1d459343f6d5a7ae915a921194f6d Mon Sep 17 00:00:00 2001 From: Matias Lopez Date: Wed, 27 Nov 2019 19:40:12 -0500 Subject: [PATCH 2/7] Add CLI support --- lib/install.js | 3 ++- lib/node-gyp.js | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/install.js b/lib/install.js index c76672f1fb..974fe495c7 100644 --- a/lib/install.js +++ b/lib/install.js @@ -429,7 +429,8 @@ function download (gyp, env, url) { } // basic support for proxy overrides - env.no_proxy = env.NO_PROXY = env.no_proxy || + env.no_proxy = env.NO_PROXY = gyp.opts.noproxy || + env.no_proxy || env.NO_PROXY || env.npm_config_noproxy if (env.no_proxy) { diff --git a/lib/node-gyp.js b/lib/node-gyp.js index 9d24103900..81fc590919 100644 --- a/lib/node-gyp.js +++ b/lib/node-gyp.js @@ -67,6 +67,7 @@ proto.configDefs = { ensure: Boolean, // 'install' solution: String, // 'build' (windows only) proxy: String, // 'install' + noproxy: String, // 'install' devdir: String, // everywhere nodedir: String, // 'configure' loglevel: String, // everywhere From 5f038fb1c42ad2082a48ac509be418278e6050f6 Mon Sep 17 00:00:00 2001 From: Matias Lopez Date: Wed, 27 Nov 2019 19:41:16 -0500 Subject: [PATCH 3/7] Add flag documentation --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f6dd0e9561..9099caa914 100644 --- a/README.md +++ b/README.md @@ -193,7 +193,8 @@ Some additional resources for Node.js native addons and writing `gyp` configurat | `--devdir=$path` | SDK download directory (default is OS cache directory) | `--ensure` | Don't reinstall headers if already present | `--dist-url=$url` | Download header tarball from custom URL -| `--proxy=$url` | Set HTTP proxy for downloading header tarball +| `--proxy=$url` | Set HTTP(S) proxy for downloading header tarball +| `--noproxy=$urls` | Set urls to ignore proxies when downloading header tarball | `--cafile=$cafile` | Override default CA chain (to download tarball) | `--nodedir=$path` | Set the path to the node source code | `--python=$path` | Set path to the Python binary From 481918d6c5d34ac04c5dc7ea060d1733cc64b7e6 Mon Sep 17 00:00:00 2001 From: Matias Lopez Date: Wed, 27 Nov 2019 19:55:49 -0500 Subject: [PATCH 4/7] mirror behavior with https_proxy --- lib/install.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/install.js b/lib/install.js index 974fe495c7..b4a7a5e34a 100644 --- a/lib/install.js +++ b/lib/install.js @@ -420,6 +420,10 @@ function download (gyp, env, url) { */ // basic support for a proxy server + env.https_proxy = env.HTTPS_PROXY = gyp.opts.proxy || + env.https_proxy || + env.HTTPS_PROXY || + env.npm_config_proxy env.http_proxy = env.HTTP_PROXY = gyp.opts.proxy || env.http_proxy || env.HTTP_PROXY || @@ -430,9 +434,9 @@ function download (gyp, env, url) { // basic support for proxy overrides env.no_proxy = env.NO_PROXY = gyp.opts.noproxy || - env.no_proxy || - env.NO_PROXY || - env.npm_config_noproxy + env.no_proxy || + env.NO_PROXY || + env.npm_config_noproxy if (env.no_proxy) { log.verbose('download', 'using proxy url: "%s"', env.no_proxy) } From 4065cfadf8c40ef71367eec3a5318f09816a96c0 Mon Sep 17 00:00:00 2001 From: Matias Lopez Date: Wed, 27 Nov 2019 20:04:28 -0500 Subject: [PATCH 5/7] Fix verbose noproxy log message --- lib/install.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/install.js b/lib/install.js index b4a7a5e34a..81f392b239 100644 --- a/lib/install.js +++ b/lib/install.js @@ -438,7 +438,7 @@ function download (gyp, env, url) { env.NO_PROXY || env.npm_config_noproxy if (env.no_proxy) { - log.verbose('download', 'using proxy url: "%s"', env.no_proxy) + log.verbose('download', 'ignoring proxy for: "%s"', env.no_proxy) } var req = request(requestOpts) From f2e8dd7a46b74c1446e9ea3c7521b2a9a49e1d09 Mon Sep 17 00:00:00 2001 From: Matias Lopez Date: Thu, 28 Nov 2019 03:23:18 -0500 Subject: [PATCH 6/7] Add tests --- test/test-download.js | 95 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/test/test-download.js b/test/test-download.js index 738a43f276..c88f0c612c 100644 --- a/test/test-download.js +++ b/test/test-download.js @@ -90,6 +90,101 @@ test('download over https with custom ca', function (t) { }) }) +test('download over http with proxy', function (t) { + t.plan(2) + + var server = http.createServer(function (req, res) { + t.strictEqual(req.headers['user-agent'], + 'node-gyp v42 (node ' + process.version + ')') + res.end('ok') + pserver.close(function () { + server.close() + }) + }) + + var pserver = http.createServer(function (req, res) { + t.strictEqual(req.headers['user-agent'], + 'node-gyp v42 (node ' + process.version + ')') + res.end('proxy ok') + server.close(function () { + pserver.close() + }) + }) + + var host = 'localhost' + server.listen(0, host, function () { + var port = this.address().port + pserver.listen(port + 1, host, function () { + var gyp = { + opts: { + proxy: 'http://' + host + ':' + (port + 1) + }, + version: '42' + } + var url = 'http://' + host + ':' + port + var req = install.test.download(gyp, {}, url) + req.on('response', function (res) { + var body = '' + res.setEncoding('utf8') + res.on('data', function (data) { + body += data + }) + res.on('end', function () { + t.strictEqual(body, 'proxy ok') + }) + }) + }) + }) +}) + +test('download over http with noproxy', function (t) { + t.plan(2) + + var server = http.createServer(function (req, res) { + t.strictEqual(req.headers['user-agent'], + 'node-gyp v42 (node ' + process.version + ')') + res.end('ok') + pserver.close(function () { + server.close() + }) + }) + + var pserver = http.createServer(function (req, res) { + t.strictEqual(req.headers['user-agent'], + 'node-gyp v42 (node ' + process.version + ')') + res.end('proxy ok') + server.close(function () { + pserver.close() + }) + }) + + var host = 'localhost' + server.listen(0, host, function () { + var port = this.address().port + pserver.listen(port + 1, host, function () { + var gyp = { + opts: { + proxy: 'http://' + host + ':' + (port + 1), + noproxy: 'localhost' + }, + version: '42' + } + var url = 'http://' + host + ':' + port + var req = install.test.download(gyp, {}, url) + req.on('response', function (res) { + var body = '' + res.setEncoding('utf8') + res.on('data', function (data) { + body += data + }) + res.on('end', function () { + t.strictEqual(body, 'ok') + }) + }) + }) + }) +}) + test('download with missing cafile', function (t) { t.plan(1) var gyp = { From 38df5d8edb36a6c21b47c621e589355ae4dd83c2 Mon Sep 17 00:00:00 2001 From: Matias Lopez Date: Thu, 28 Nov 2019 03:23:43 -0500 Subject: [PATCH 7/7] Remove side effects, handle no proxy. --- lib/install.js | 38 +++++---------------- lib/proxy.js | 92 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+), 29 deletions(-) create mode 100644 lib/proxy.js diff --git a/lib/install.js b/lib/install.js index 81f392b239..c919c10588 100644 --- a/lib/install.js +++ b/lib/install.js @@ -11,6 +11,7 @@ const request = require('request') const mkdir = require('mkdirp') const processRelease = require('./process-release') const win = process.platform === 'win32' +const getProxyFromURI = require('./proxy') function install (fs, gyp, argv, callback) { var release = processRelease(argv, gyp, process.version, process.release) @@ -409,36 +410,15 @@ function download (gyp, env, url) { requestOpts.ca = readCAFile(cafile) } - /** - * Proxy support is delegated to the `request` module. `request` already - * supports proxies using the ones provided in the environment. For - * compatibility those provided using the CLI or `npm config` will be - * attached to the environment, keeping previous precedence, for - * `request` to use. - * - * See: https://github.com/request/request/blob/master/lib/getProxyFromURI.js - */ - // basic support for a proxy server - env.https_proxy = env.HTTPS_PROXY = gyp.opts.proxy || - env.https_proxy || - env.HTTPS_PROXY || - env.npm_config_proxy - env.http_proxy = env.HTTP_PROXY = gyp.opts.proxy || - env.http_proxy || - env.HTTP_PROXY || - env.npm_config_proxy - if (env.http_proxy) { - log.verbose('download', 'using proxy url: "%s"', env.http_proxy) - } - - // basic support for proxy overrides - env.no_proxy = env.NO_PROXY = gyp.opts.noproxy || - env.no_proxy || - env.NO_PROXY || - env.npm_config_noproxy - if (env.no_proxy) { - log.verbose('download', 'ignoring proxy for: "%s"', env.no_proxy) + var proxyUrl = getProxyFromURI(gyp, env, url) + if (proxyUrl) { + if (/^https?:\/\//i.test(proxyUrl)) { + log.verbose('download', 'using proxy url: "%s"', proxyUrl) + requestOpts.proxy = proxyUrl + } else { + log.warn('download', 'ignoring invalid "proxy" config setting: "%s"', proxyUrl) + } } var req = request(requestOpts) diff --git a/lib/proxy.js b/lib/proxy.js new file mode 100644 index 0000000000..92d9ed2f7f --- /dev/null +++ b/lib/proxy.js @@ -0,0 +1,92 @@ +'use strict' +// Taken from https://github.com/request/request/blob/212570b/lib/getProxyFromURI.js + +const url = require('url') + +function formatHostname (hostname) { + // canonicalize the hostname, so that 'oogle.com' won't match 'google.com' + return hostname.replace(/^\.*/, '.').toLowerCase() +} + +function parseNoProxyZone (zone) { + zone = zone.trim().toLowerCase() + + var zoneParts = zone.split(':', 2) + var zoneHost = formatHostname(zoneParts[0]) + var zonePort = zoneParts[1] + var hasPort = zone.indexOf(':') > -1 + + return { hostname: zoneHost, port: zonePort, hasPort: hasPort } +} + +function uriInNoProxy (uri, noProxy) { + var port = uri.port || (uri.protocol === 'https:' ? '443' : '80') + var hostname = formatHostname(uri.hostname) + var noProxyList = noProxy.split(',') + + // iterate through the noProxyList until it finds a match. + return noProxyList.map(parseNoProxyZone).some(function (noProxyZone) { + var isMatchedAt = hostname.indexOf(noProxyZone.hostname) + var hostnameMatched = ( + isMatchedAt > -1 && + (isMatchedAt === hostname.length - noProxyZone.hostname.length) + ) + + if (noProxyZone.hasPort) { + return (port === noProxyZone.port) && hostnameMatched + } + + return hostnameMatched + }) +} + +function getProxyFromURI (gyp, env, uri) { + // If a string URI/URL was given, parse it into a URL object + if (typeof uri === 'string') { + // eslint-disable-next-line + uri = url.parse(uri) + } + + // Decide the proper request proxy to use based on the request URI object and the + // environmental variables (NO_PROXY, HTTP_PROXY, etc.) + // respect NO_PROXY environment variables (see: https://lynx.invisible-island.net/lynx2.8.7/breakout/lynx_help/keystrokes/environments.html) + + var noProxy = gyp.opts.noproxy || env.NO_PROXY || env.no_proxy || env.npm_config_noproxy || '' + + // if the noProxy is a wildcard then return null + + if (noProxy === '*') { + return null + } + + // if the noProxy is not empty and the uri is found return null + + if (noProxy !== '' && uriInNoProxy(uri, noProxy)) { + return null + } + + // Check for HTTP or HTTPS Proxy in environment Else default to null + + if (uri.protocol === 'http:') { + return gyp.opts.proxy || + env.HTTP_PROXY || + env.http_proxy || + env.npm_config_proxy || null + } + + if (uri.protocol === 'https:') { + return gyp.opts.proxy || + env.HTTPS_PROXY || + env.https_proxy || + env.HTTP_PROXY || + env.http_proxy || + env.npm_config_proxy || null + } + + // if none of that works, return null + // (What uri protocol are you using then?) + + return null +} + +module.exports = getProxyFromURI