From e357fc853eb6d509075b5708b7f65123e1288ae5 Mon Sep 17 00:00:00 2001 From: uzlopak Date: Wed, 1 Dec 2021 14:59:03 +0100 Subject: [PATCH 1/6] add bytesRead to FileStream --- README.md | 1 + lib/main.d.ts | 19 +++++++++++++++---- lib/types/multipart.js | 8 ++++++-- test/types-multipart.spec.js | 2 ++ test/types/main.test-d.ts | 17 ++++++++--------- 5 files changed, 32 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index e5db971..ee14d32 100644 --- a/README.md +++ b/README.md @@ -194,6 +194,7 @@ Busboy (special) events * **file**(< _string_ >fieldname, < _ReadableStream_ >stream, < _string_ >filename, < _string_ >transferEncoding, < _string_ >mimeType) - Emitted for each new file form field found. `transferEncoding` contains the 'Content-Transfer-Encoding' value for the file stream. `mimeType` contains the 'Content-Type' value for the file stream. * Note: if you listen for this event, you should always handle the `stream` no matter if you care about the file contents or not (e.g. you can simply just do `stream.resume();` if you want to discard the contents), otherwise the 'finish' event will never fire on the Busboy instance. However, if you don't care about **any** incoming files, you can simply not listen for the 'file' event at all and any/all files will be automatically and safely discarded (these discarded files do still count towards `files` and `parts` limits). * If a configured file size limit was reached, `stream` will both have a boolean property `truncated` (best checked at the end of the stream) and emit a 'limit' event to notify you when this happens. + * The property `bytesRead` informs about the number of bytes that have been read so far. * **field**(< _string_ >fieldname, < _string_ >value, < _boolean_ >fieldnameTruncated, < _boolean_ >valueTruncated, < _string_ >transferEncoding, < _string_ >mimeType) - Emitted for each new non-file field found. diff --git a/lib/main.d.ts b/lib/main.d.ts index da6fb3c..acfeffc 100644 --- a/lib/main.d.ts +++ b/lib/main.d.ts @@ -5,7 +5,7 @@ /// import * as http from 'http'; -import {Readable, Writable} from 'stream'; +import { Readable, Writable } from 'stream'; declare const busboy: BusboyConstructor; export default busboy @@ -39,7 +39,7 @@ export interface BusboyConfig { * Various limits on incoming data. */ limits?: - | { + | { /** * Max field name size (in bytes) * @default 100 bytes @@ -76,11 +76,22 @@ export interface BusboyConfig { */ headerPairs?: number | undefined; } - | undefined; + | undefined; } export type BusboyHeaders = { 'content-type': string } & http.IncomingHttpHeaders; +export interface BusboyFileStream extends + Readable { + + truncated: boolean; + + /** + * The number of bytes that have been read so far. + */ + bytesRead: number; +} + export interface Busboy extends Writable { addListener(event: Event, listener: BusboyEvents[Event]): this; @@ -128,7 +139,7 @@ export interface BusboyEvents { */ file: ( fieldname: string, - stream: Readable, + stream: BusboyFileStream, filename: string, transferEncoding: string, mimeType: string, diff --git a/lib/types/multipart.js b/lib/types/multipart.js index 1fa2eb0..f433422 100644 --- a/lib/types/multipart.js +++ b/lib/types/multipart.js @@ -38,7 +38,7 @@ function Multipart (boy, cfg) { for (i = 0, len = parsedConType.length; i < len; ++i) { if (Array.isArray(parsedConType[i]) && - RE_BOUNDARY.test(parsedConType[i][0])) { + RE_BOUNDARY.test(parsedConType[i][0])) { boundary = parsedConType[i][1] break } @@ -199,6 +199,8 @@ function Multipart (boy, cfg) { file.truncated = true part.removeAllListeners('data') } else if (!file.push(data)) { self._pause = true } + + file.bytesRead = nsize > fileSizeLimit ? fileSizeLimit : nsize } onEnd = function () { @@ -285,10 +287,12 @@ function FileStream (opts) { if (!(this instanceof FileStream)) { return new FileStream(opts) } ReadableStream.call(this, opts) + this.bytesRead = 0 + this.truncated = false } inherits(FileStream, ReadableStream) -FileStream.prototype._read = function (n) {} +FileStream.prototype._read = function (n) { } module.exports = Multipart diff --git a/test/types-multipart.spec.js b/test/types-multipart.spec.js index 0324e4d..90bba3e 100644 --- a/test/types-multipart.spec.js +++ b/test/types-multipart.spec.js @@ -320,6 +320,8 @@ describe('types-multipart', () => { ++info[3] }).on('end', function () { info[2] = nb + assert(typeof (stream.bytesRead) === 'number', 'file.bytesRead is missing') + assert(stream.bytesRead === nb, 'file.bytesRead is not equal to filesize') if (stream.truncated) { ++info[3] } }) }) diff --git a/test/types/main.test-d.ts b/test/types/main.test-d.ts index e33eaf2..5fae8b1 100644 --- a/test/types/main.test-d.ts +++ b/test/types/main.test-d.ts @@ -1,6 +1,5 @@ -import BusboyDefault, { BusboyConstructor, BusboyConfig, BusboyHeaders, Busboy, BusboyEvents } from '../..'; +import BusboyDefault, { BusboyConstructor, BusboyConfig, BusboyHeaders, Busboy, BusboyEvents, BusboyFileStream } from '../..'; import {expectError, expectType} from "tsd"; -import {Readable} from "stream"; // test type exports type Constructor = BusboyConstructor; @@ -26,7 +25,7 @@ new BusboyDefault({ headers: { 'content-type': 'foo' }, limits: { headerPairs: 2 busboy.addListener('file', (fieldname, file, filename, encoding, mimetype) => { expectType (fieldname) - expectType(file); + expectType(file); expectType(filename); expectType(encoding); expectType(mimetype); @@ -56,7 +55,7 @@ busboy.on(Symbol('foo'), foo => { busboy.on('file', (fieldname, file, filename, encoding, mimetype) => { expectType (fieldname); - expectType (file); + expectType (file); expectType (filename); expectType (encoding); expectType (mimetype); @@ -86,7 +85,7 @@ busboy.on(Symbol('foo'), foo => { busboy.once('file', (fieldname, file, filename, encoding, mimetype) => { expectType (fieldname); - expectType (file); + expectType (file); expectType (filename); expectType (encoding); expectType (mimetype); @@ -116,7 +115,7 @@ busboy.once(Symbol('foo'), foo => { busboy.removeListener('file', (fieldname, file, filename, encoding, mimetype) => { expectType (fieldname); - expectType (file); + expectType (file); expectType (filename); expectType (encoding); expectType (mimetype); @@ -146,7 +145,7 @@ busboy.removeListener(Symbol('foo'), foo => { busboy.off('file', (fieldname, file, filename, encoding, mimetype) => { expectType (fieldname); - expectType (file); + expectType (file); expectType (filename); expectType (encoding); expectType (mimetype); @@ -176,7 +175,7 @@ busboy.off(Symbol('foo'), foo => { busboy.prependListener('file', (fieldname, file, filename, encoding, mimetype) => { expectType (fieldname); - expectType (file); + expectType (file); expectType (filename); expectType (encoding); expectType (mimetype); @@ -206,7 +205,7 @@ busboy.prependListener(Symbol('foo'), foo => { busboy.prependOnceListener('file', (fieldname, file, filename, encoding, mimetype) => { expectType (fieldname); - expectType (file); + expectType (file); expectType (filename); expectType (encoding); expectType (mimetype); From a35170af3b12f64fd3ddfce05145606bf8ad4dd2 Mon Sep 17 00:00:00 2001 From: uzlopak Date: Wed, 1 Dec 2021 15:00:23 +0100 Subject: [PATCH 2/6] add info to changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 692392a..e0f6669 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,3 +9,4 @@ Major changes since the last busboy release (0.31): * Error on non-number limit rather than ignoring (#7) * Dicer is now part of the busboy itself and not an external dependency (#14) * Tests were converted to Mocha (#11, #12, #22, #23) +* FileStreams also provide the property `bytesRead` From 9fed525c708c73c9e1cd96c9b6c8dcbbcf8fc2c4 Mon Sep 17 00:00:00 2001 From: Igor Savin Date: Thu, 2 Dec 2021 19:20:05 +0200 Subject: [PATCH 3/6] Run on medium preset by default for increased accuracy --- benchmarks/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/package.json b/benchmarks/package.json index 55ed418..9dd5153 100644 --- a/benchmarks/package.json +++ b/benchmarks/package.json @@ -11,8 +11,8 @@ }, "scripts": { "install-node": "nvm install 17.2.0 && nvm install 16.13.1 && nvm install 14.18.2 && nvm install 12.22.7", - "benchmark-busboy": "node busboy/executioner.js -c 0", - "benchmark-fastify": "node busboy/executioner.js -c 1", + "benchmark-busboy": "node busboy/executioner.js -c 0 -p medium", + "benchmark-fastify": "node busboy/executioner.js -c 1 -p medium", "benchmark-all": "npm run benchmark-busboy -- -p high && npm run benchmark-fastify -- -p high", "benchmark-all-medium": "npm run benchmark-busboy -- -p medium && npm run benchmark-fastify -- -p medium", "benchmark-all-low": "npm run benchmark-busboy -- -p low && npm run benchmark-fastify -- -p low", From 4c6033d0f5dbdb15170f5329c1c8f027c59bb8b4 Mon Sep 17 00:00:00 2001 From: uzlopak Date: Thu, 2 Dec 2021 19:32:41 +0100 Subject: [PATCH 4/6] remove ternerary operator --- lib/types/multipart.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/types/multipart.js b/lib/types/multipart.js index f433422..d9865ae 100644 --- a/lib/types/multipart.js +++ b/lib/types/multipart.js @@ -193,14 +193,15 @@ function Multipart (boy, cfg) { onData = function (data) { if ((nsize += data.length) > fileSizeLimit) { - const extralen = (fileSizeLimit - (nsize - data.length)) + const extralen = nsize - fileSizeLimit if (extralen > 0) { file.push(data.slice(0, extralen)) } - file.emit('limit') file.truncated = true + nsize -= extralen + file.emit('limit') part.removeAllListeners('data') } else if (!file.push(data)) { self._pause = true } - file.bytesRead = nsize > fileSizeLimit ? fileSizeLimit : nsize + file.bytesRead = nsize } onEnd = function () { From 8868ea9ab17e99091be0ea199db585e0620ef12c Mon Sep 17 00:00:00 2001 From: uzlopak Date: Thu, 2 Dec 2021 23:47:19 +0100 Subject: [PATCH 5/6] avoid potential race conditions --- lib/types/multipart.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/types/multipart.js b/lib/types/multipart.js index 27b2863..62f3a58 100644 --- a/lib/types/multipart.js +++ b/lib/types/multipart.js @@ -199,8 +199,10 @@ function Multipart (boy, cfg) { if (extralen > 0) { file.push(data.slice(0, extralen)) } file.truncated = true nsize -= extralen - file.emit('limit') + file.bytesRead = nsize part.removeAllListeners('data') + file.emit('limit') + return } else if (!file.push(data)) { self._pause = true } file.bytesRead = nsize From 1c4e8372d664de04a66bca9077726f7edd17a6dd Mon Sep 17 00:00:00 2001 From: uzlopak Date: Fri, 3 Dec 2021 01:02:23 +0100 Subject: [PATCH 6/6] fix quick math --- lib/types/multipart.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/types/multipart.js b/lib/types/multipart.js index 62f3a58..bbbc53a 100644 --- a/lib/types/multipart.js +++ b/lib/types/multipart.js @@ -195,11 +195,10 @@ function Multipart (boy, cfg) { onData = function (data) { if ((nsize += data.length) > fileSizeLimit) { - const extralen = nsize - fileSizeLimit + const extralen = fileSizeLimit - nsize + data.length if (extralen > 0) { file.push(data.slice(0, extralen)) } file.truncated = true - nsize -= extralen - file.bytesRead = nsize + file.bytesRead = fileSizeLimit part.removeAllListeners('data') file.emit('limit') return