diff --git a/README.md b/README.md index 5124fa02..6b2a9826 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ Available options: -t/--timeout NUM The number of seconds before timing out and resetting a connection. default: 10 -T/--title TITLE - The title to place in the results for identifcation. + The title to place in the results for identification. -b/--body BODY The body of the request. -i/--input FILE @@ -92,6 +92,8 @@ Available options: Print all the latency data. default: false. -j/--json Print the output as newline delimited json. This will cause the progress bar and results not to be rendered. default: false. + -f/--forever + Run the benchmark forever. Efficiently restarts the benchmark on completion. default: false. -v/--version Print the version number. -h/--help @@ -138,7 +140,8 @@ Start autocannon against the given target. * `overallRate`: A `Number` stating the rate of requests to make per second from all connections. `conenctionRate` takes precedence if both are set. No rate limiting by default. _OPTIONAL_ * `reconnectRate`: A `Number` which makes the individual connections disconnect and reconnect to the server whenever it has sent that number of requests. _OPTIONAL_ * `requests`: An `Array` of `Object`s which represents the sequence of requests to make while benchmarking. Can be used in conjunction with the `body`, `headers` and `method` params above. The `Object`s in this array can have `body`, `headers`, `method`, or `path` attributes, which overwrite those that are passed in this `opts` object. Therefore, the ones in this (`opts`) object take precedence and should be viewed as defaults. Check the samples folder for an example of how this might be used. _OPTIONAL_. -* `cb`: The callback which is called on completion of the benchmark. Takes the following params. _OPTIONAL_. + * `forever`: A `Boolean` which allows you to setup an instance of autocannon that restarts indefinatly after emiting results with the `done` event. Useful for efficiently restarting your instance. To stop running forever, you must cause a `SIGINT` or call the `.stop()` function on your instance. _OPTIONAL_ default: `false` +* `cb`: The callback which is called on completion of a benchmark. Takes the following params. _OPTIONAL_. * `err`: If there was an error encountered with the run. * `results`: The results of the run. @@ -182,6 +185,7 @@ Checkout [this example](./samples/track-run.js) to see it in use, as well. Because an autocannon instance is an `EventEmitter`, it emits several events. these are below: +* `start`: Emitted once everything has been setup in your autocannon instance and it has started. Useful for if running the instance forever. * `tick`: Emitted every second this autocannon is running a benchmark. Useful for displaying stats, etc. Used by the `track` function. * `done`: Emitted when the autocannon finishes a benchmark. passes the `results` as an argument to the callback. * `response`: Emitted when the autocannons http-client gets a http response from the server. This passes the following arguments to the callback: @@ -189,6 +193,8 @@ Because an autocannon instance is an `EventEmitter`, it emits several events. th * `statusCode`: The http status code of the response. * `resBytes`: The response byte length. * `responseTime`: The time taken to get a response for the initiating the request. +* `reqError`: Emitted in the case of a request error e.g. a timeout. +* `error`: Emitted if there is an error during the setup phase of autocannon. ### `Client` API diff --git a/autocannon.js b/autocannon.js index f2424326..dae2aca6 100755 --- a/autocannon.js +++ b/autocannon.js @@ -14,7 +14,7 @@ module.exports.track = track function start () { const argv = minimist(process.argv.slice(2), { - boolean: ['json', 'n', 'help', 'renderLatencyTable', 'renderProgressBar'], + boolean: ['json', 'n', 'help', 'renderLatencyTable', 'renderProgressBar', 'forever'], alias: { connections: 'c', pipelining: 'p', @@ -36,6 +36,7 @@ function start () { renderProgressBar: 'progress', title: 'T', version: 'v', + forever: 'f', help: 'h' }, default: { @@ -47,6 +48,7 @@ function start () { renderLatencyTable: false, renderProgressBar: true, json: false, + forever: false, method: 'GET' } }) @@ -86,16 +88,20 @@ function start () { }, {}) } - const tracker = run(argv, (err, result) => { - if (err) { - throw err - } + const tracker = run(argv) + tracker.on('done', (result) => { if (argv.json) { console.log(JSON.stringify(result)) } }) + tracker.on('error', (err) => { + if (err) { + throw err + } + }) + // if not rendering json, or if std isn't a tty, track progress if (!argv.json || !process.stdout.isTTY) track(tracker, argv) diff --git a/help.txt b/help.txt index cde0c3c4..14426153 100644 --- a/help.txt +++ b/help.txt @@ -46,6 +46,8 @@ Available options: Print all the latency data. default: false. -j/--json Print the output as newline delimited json. This will cause the progress bar and results not to be rendered. default: false. + -f/--forever + Run the benchmark forever. Efficiently restarts the benchmark on completion. default: false. -v/--version Print the version number. -h/--help diff --git a/lib/progressTracker.js b/lib/progressTracker.js index 2f4d8535..9a957dac 100644 --- a/lib/progressTracker.js +++ b/lib/progressTracker.js @@ -24,26 +24,47 @@ function track (instance, opts) { opts = xtend(defaults, opts) const chalk = new Chalk.constructor({ enabled: testColorSupport({ stream: opts.outputStream }) }) - // this default needs to be set after chalk is setup, because chalk is now local to this func opts.progressBarString = opts.progressBarString || `${chalk.green('running')} [:bar] :percent` const iOpts = instance.opts - if (opts.renderProgressBar) { - let msg = `${iOpts.connections} connections` + let durationProgressBar + let amountProgressBar - if (iOpts.pipelining > 1) { - msg += ` with ${iOpts.pipelining} pipelining factor` - } + instance.on('start', () => { + if (opts.renderProgressBar) { + let msg = `${iOpts.connections} connections` + + if (iOpts.pipelining > 1) { + msg += ` with ${iOpts.pipelining} pipelining factor` + } - if (!iOpts.amount) { - logToStream(`Running ${iOpts.duration}s test @ ${iOpts.url}\n${msg}\n`) + if (!iOpts.amount) { + logToStream(`Running ${iOpts.duration}s test @ ${iOpts.url}\n${msg}\n`) - trackDuration(instance, opts, iOpts) - } else { - logToStream(`Running ${iOpts.amount} requests test @ ${iOpts.url}\n${msg}\n`) + durationProgressBar = trackDuration(instance, opts, iOpts) + } else { + logToStream(`Running ${iOpts.amount} requests test @ ${iOpts.url}\n${msg}\n`) + + amountProgressBar = trackAmount(instance, opts, iOpts) + } + } + }) - trackAmount(instance, opts, iOpts) + // add listeners for progress bar to instance here so they aren't + // added on restarting, causing listener leaks + // note: Attempted to curry the functions below, but that breaks the functionality + // as they use the scope/closure of the progress bar variables to allow them to be reset + if (opts.renderProgressBar && opts.outputStream.isTTY) { + if (!iOpts.amount) { // duration progress bar + instance.on('tick', () => { durationProgressBar.tick() }) + instance.on('done', () => { durationProgressBar.tick(iOpts.duration - 1) }) + process.once('SIGINT', () => { durationProgressBar.tick(iOpts.duration - 1) }) + } else { // amount progress bar + instance.on('response', () => { amountProgressBar.tick() }) + instance.on('reqError', () => { amountProgressBar.tick() }) + instance.on('done', () => { amountProgressBar.tick(iOpts.amount - 1) }) + process.once('SIGINT', () => { amountProgressBar.tick(iOpts.amount - 1) }) } } @@ -117,11 +138,7 @@ function trackDuration (instance, opts, iOpts) { }) progressBar.tick(0) - instance.on('done', () => progressBar.tick(iOpts.duration - 1)) - instance.on('tick', () => progressBar.tick()) - process.once('SIGINT', () => { - progressBar.tick(iOpts.duration - 1) - }) + return progressBar } function trackAmount (instance, opts, iOpts) { @@ -138,18 +155,7 @@ function trackAmount (instance, opts, iOpts) { }) progressBar.tick(0) - - instance.on('done', () => progressBar.tick(iOpts.amount - 1)) - instance.on('response', () => { - progressBar.tick() - }) - instance.on('reqError', () => { - progressBar.tick() - }) - - process.once('SIGINT', () => { - progressBar.tick(iOpts.amount - 1) - }) + return progressBar } function asRow (name, stat) { diff --git a/lib/run.js b/lib/run.js index aff133fe..dcde2895 100644 --- a/lib/run.js +++ b/lib/run.js @@ -7,6 +7,7 @@ const timestring = require('timestring') const Client = require('./httpClient') const xtend = require('xtend') const histUtil = require('hdr-histogram-percentiles-obj') +const reInterval = require('reinterval') const histAsObj = histUtil.histAsObj const addPercentiles = histUtil.addPercentiles @@ -24,10 +25,13 @@ const defaultOptions = { overallRate: 0, amount: 0, reconnectRate: 0, + forever: false, requests: [{}] } function run (opts, cb) { + const cbPassedIn = (typeof cb === 'function') + cb = cb || noop const tracker = new EE() @@ -64,6 +68,7 @@ function run (opts, cb) { let totalCompletedRequests = 0 let amount = opts.amount let stop = false + let restart = true let numRunning = opts.connections let startTime = Date.now() @@ -81,62 +86,7 @@ function run (opts, cb) { url.rate = opts.connectionRate || opts.overallRate let clients = [] - for (let i = 0; i < opts.connections; i++) { - if (!amount && !opts.maxConnectionRequests && opts.maxOverallRequests) { - url.responseMax = distributeNums(opts.maxOverallRequests, i) - } - if (amount) { - url.responseMax = distributeNums(amount, i) - } - if (!opts.connectionRate && opts.overallRate) { - url.rate = distributeNums(opts.overallRate, i) - } - - let client = new Client(url) - client.on('response', onResponse) - client.on('connError', onError) - client.on('timeout', onTimeout) - client.on('request', () => { totalRequests++ }) - client.on('done', onDone) - clients.push(client) - - // we will miss the initial request emits because the client emits request on construction - totalRequests += url.pipelining < url.rate ? url.rate : url.pipelining - } - - function distributeNums (x, i) { - return (Math.floor(x / opts.connections) + (((i + 1) <= (x % opts.connections)) ? 1 : 0)) - } - - function onResponse (statusCode, resBytes, responseTime) { - tracker.emit('response', this, statusCode, resBytes, responseTime) - const codeIndex = Math.floor(parseInt(statusCode) / 100) - 1 - statusCodes[codeIndex] += 1 - // only record 2xx latencies - if (codeIndex === 1) latencies.record(responseTime) - bytes += resBytes - counter++ - } - - function onError () { - for (let i = 0; i < opts.pipelining; i++) tracker.emit('reqError') - errors++ - if (opts.bailout && errors >= opts.bailout) tracker.stop() - } - - // treat a timeout as a special type of error - function onTimeout () { - for (let i = 0; i < opts.pipelining; i++) tracker.emit('reqError') - errors++ - timeouts++ - if (opts.bailout && errors >= opts.bailout) tracker.stop() - } - - function onDone () { - if (!--numRunning) { - tracker.stop() - } - } + initialiseClients(clients) if (!amount) { var stopTimer = setTimeout(() => { @@ -145,11 +95,16 @@ function run (opts, cb) { } tracker.stop = () => { - if (stopTimer) clearTimeout(stopTimer) stop = true + restart = false } - const interval = setInterval(() => { + const interval = reInterval(tickInterval, 1000) + + // put the start emit in a setImmediate so trackers can be added, etc. + setImmediate(() => { tracker.emit('start') }) + + function tickInterval () { totalBytes += bytes totalCompletedRequests += counter requests.record(counter) @@ -159,7 +114,8 @@ function run (opts, cb) { tracker.emit('tick') if (stop) { - clearInterval(interval) + if (stopTimer) clearTimeout(stopTimer) + interval.clear() clients.forEach((client) => client.destroy()) let result = { title: opts.title, @@ -178,16 +134,100 @@ function run (opts, cb) { } result.requests.sent = totalRequests statusCodes.forEach((code, index) => { result[(index + 1) + 'xx'] = code }) + tracker.emit('done', result) + if (!opts.forever) cb(null, result) + + // the restart function + setImmediate(() => { + if (opts.forever && restart) { + stop = false + stopTimer = setTimeout(() => { + stop = true + }, opts.duration * 1000) + errors = 0 + timeouts = 0 + totalBytes = 0 + totalRequests = 0 + totalCompletedRequests = 0 + statusCodes.fill(0) + requests.reset() + latencies.reset() + throughput.reset() + startTime = Date.now() + + // reinitialise clients + clients = [] + initialiseClients(clients) + + interval.reschedule(1000) + tracker.emit('start') + } + }) + } + } - cb(null, result) + function initialiseClients (clients) { + for (let i = 0; i < opts.connections; i++) { + if (!amount && !opts.maxConnectionRequests && opts.maxOverallRequests) { + url.responseMax = distributeNums(opts.maxOverallRequests, i) + } + if (amount) { + url.responseMax = distributeNums(amount, i) + } + if (!opts.connectionRate && opts.overallRate) { + url.rate = distributeNums(opts.overallRate, i) + } + + let client = new Client(url) + client.on('response', onResponse) + client.on('connError', onError) + client.on('timeout', onTimeout) + client.on('request', () => { totalRequests++ }) + client.on('done', onDone) + clients.push(client) + + // we will miss the initial request emits because the client emits request on construction + totalRequests += url.pipelining < url.rate ? url.rate : url.pipelining + } + + function distributeNums (x, i) { + return (Math.floor(x / opts.connections) + (((i + 1) <= (x % opts.connections)) ? 1 : 0)) + } + + function onResponse (statusCode, resBytes, responseTime) { + tracker.emit('response', this, statusCode, resBytes, responseTime) + const codeIndex = Math.floor(parseInt(statusCode) / 100) - 1 + statusCodes[codeIndex] += 1 + // only record 2xx latencies + if (codeIndex === 1) latencies.record(responseTime) + bytes += resBytes + counter++ + } + + function onError () { + for (let i = 0; i < opts.pipelining; i++) tracker.emit('reqError') + errors++ + if (opts.bailout && errors >= opts.bailout) stop = true } - }, 1000) + + // treat a timeout as a special type of error + function onTimeout () { + for (let i = 0; i < opts.pipelining; i++) tracker.emit('reqError') + errors++ + timeouts++ + if (opts.bailout && errors >= opts.bailout) stop = true + } + + function onDone () { + if (!--numRunning) stop = true + } + } // will return true if error with opts entered function checkOptsForErrors () { if (!opts.url) { - cb(new Error('url option required')) + errorCb(new Error('url option required')) return true } @@ -199,7 +239,7 @@ function run (opts, cb) { if (typeof opts.duration === 'number') { if (lessThanZeroError(opts.duration, 'duration')) return true } else { - cb(new Error('duration entered was in an invalid format')) + errorCb(new Error('duration entered was in an invalid format')) return true } @@ -213,9 +253,14 @@ function run (opts, cb) { if (opts.maxConnectionRequests && lessThanOneError(opts.maxConnectionRequests, 'maxConnectionRequests')) return true if (opts.maxOverallRequests && lessThanOneError(opts.maxOverallRequests, 'maxOverallRequests')) return true + if (opts.forever && cbPassedIn) { + errorCb(new Error('should not use the callback parameter when the `forever` option is set to true. Use the `done` event on this event emitter')) + return true + } + function lessThanZeroError (x, label) { if (x < 0) { - cb(new Error(`${label} can not be less than 0`)) + errorCb(new Error(`${label} can not be less than 0`)) return true } return false @@ -223,7 +268,7 @@ function run (opts, cb) { function lessThanOneError (x, label) { if (x < 1) { - cb(new Error(`${label} can not be less than 1`)) + errorCb(new Error(`${label} can not be less than 1`)) return true } return false @@ -231,12 +276,22 @@ function run (opts, cb) { function greaterThanZeroError (x, label) { if (x <= 0) { - cb(new Error(`${label} must be greater than 0`)) + errorCb(new Error(`${label} must be greater than 0`)) return true } return false } + function errorCb (error) { + if (cbPassedIn) { + cb(error) + } else { + // wrapped in setImmediate so any error event handlers that are added to + // the tracker can be added before being emitted + setImmediate(() => { tracker.emit('error', error) }) + } + } + return false } // checkOptsForErrors diff --git a/package.json b/package.json index 16bbed90..3dfcc8fd 100644 --- a/package.json +++ b/package.json @@ -48,9 +48,10 @@ "hdr-histogram-percentiles-obj": "^1.1.0", "http-parser-js": "^0.4.2", "minimist": "^1.2.0", - "native-hdr-histogram": "^0.3.0", + "native-hdr-histogram": "^0.4.0", "pretty-bytes": "^3.0.1", "progress": "^1.1.8", + "reinterval": "^1.1.0", "retimer": "^1.0.1", "table": "^3.7.8", "timestring": "^3.0.1", diff --git a/test/forever.test.js b/test/forever.test.js new file mode 100644 index 00000000..ef306f6a --- /dev/null +++ b/test/forever.test.js @@ -0,0 +1,41 @@ +'use strict' + +const test = require('tap').test +const run = require('../lib/run') +const helper = require('./helper') +const server = helper.startServer() + +test('running with forever set to true and passing in a callback should cause an error to be returned in the callback', (t) => { + t.plan(2) + + run({ + url: `http://localhost:${server.address().port}`, + forever: true + }, (err, res) => { + t.ok(err, 'should be error when callback passed to run') + t.notOk(res, 'should not exist') + t.end() + }) +}) + +test('run forever should run until .stop() is called', (t) => { + t.plan(3) + let numRuns = 0 + + let instance = run({ + url: `http://localhost:${server.address().port}`, + duration: 0.5, + forever: true + }) + + instance.on('done', (results) => { + t.ok(results, 'should have gotten results') + if (++numRuns === 2) { + instance.stop() + setTimeout(() => { + t.ok(true, 'should have reached here without the callback being called again') + t.end() + }, 1000) + } + }) +})