From 585c9c1055c08e1cf9b1ee24dd2db4dc572bb7c4 Mon Sep 17 00:00:00 2001 From: delvedor Date: Tue, 14 Sep 2021 10:59:36 +0200 Subject: [PATCH 1/3] Add support for maxResponseSize and maxCompressedResponseSize --- index.d.ts | 2 ++ index.js | 17 +++++++++++++++-- lib/Transport.d.ts | 4 ++++ lib/Transport.js | 14 ++++++++++---- 4 files changed, 31 insertions(+), 6 deletions(-) diff --git a/index.d.ts b/index.d.ts index 8e923d6c1..1d7ccae41 100644 --- a/index.d.ts +++ b/index.d.ts @@ -119,6 +119,8 @@ interface ClientOptions { }; disablePrototypePoisoningProtection?: boolean | 'proto' | 'constructor'; caFingerprint?: string; + maxResponseSize?: number; + maxCompressedResponseSize?: number; } declare class Client { diff --git a/index.js b/index.js index 6c8212d9a..d90aa1a1b 100644 --- a/index.js +++ b/index.js @@ -21,6 +21,7 @@ const { EventEmitter } = require('events') const { URL } = require('url') +const buffer = require('buffer') const debug = require('debug')('elasticsearch') const Transport = require('./lib/Transport') const Connection = require('./lib/Connection') @@ -114,9 +115,19 @@ class Client extends ESAPI { context: null, proxy: null, enableMetaHeader: true, - disablePrototypePoisoningProtection: false + disablePrototypePoisoningProtection: false, + maxResponseSize: null, + maxCompressedResponseSize: null }, opts) + if (options.maxResponseSize !== null && options.maxResponseSize > buffer.constants.MAX_STRING_LENGTH) { + throw new ConfigurationError(`The maxResponseSize cannot be bigger than ${buffer.constants.MAX_STRING_LENGTH}`) + } + + if (options.maxCompressedResponseSize !== null && options.maxCompressedResponseSize > buffer.constants.MAX_LENGTH) { + throw new ConfigurationError(`The maxCompressedResponseSize cannot be bigger than ${buffer.constants.MAX_LENGTH}`) + } + if (options.caFingerprint !== null && isHttpConnection(opts.node || opts.nodes)) { throw new ConfigurationError('You can\'t configure the caFingerprint with a http connection') } @@ -178,7 +189,9 @@ class Client extends ESAPI { generateRequestId: options.generateRequestId, name: options.name, opaqueIdPrefix: options.opaqueIdPrefix, - context: options.context + context: options.context, + maxResponseSize: options.maxResponseSize, + maxCompressedResponseSize: options.maxCompressedResponseSize }) this.helpers = new Helpers({ diff --git a/lib/Transport.d.ts b/lib/Transport.d.ts index 2313bf23f..25b770fdb 100644 --- a/lib/Transport.d.ts +++ b/lib/Transport.d.ts @@ -61,6 +61,8 @@ interface TransportOptions { generateRequestId?: generateRequestIdFn; name?: string; opaqueIdPrefix?: string; + maxResponseSize?: number; + maxCompressedResponseSize?: number; } export interface RequestEvent, TContext = Context> { @@ -113,6 +115,8 @@ export interface TransportRequestOptions { context?: Context; warnings?: string[]; opaqueId?: string; + maxResponseSize?: number; + maxCompressedResponseSize?: number; } export interface TransportRequestCallback { diff --git a/lib/Transport.js b/lib/Transport.js index ca9477ae6..83a218a64 100644 --- a/lib/Transport.js +++ b/lib/Transport.js @@ -43,6 +43,8 @@ const MAX_STRING_LENGTH = buffer.constants.MAX_STRING_LENGTH const kProductCheck = Symbol('product check') const kApiVersioning = Symbol('api versioning') const kEventEmitter = Symbol('event emitter') +const kMaxResponseSize = Symbol('max response size') +const kMaxCompressedResponseSize = Symbol('max compressed response size') class Transport { constructor (opts) { @@ -72,6 +74,8 @@ class Transport { this[kProductCheck] = 0 // 0 = to be checked, 1 = checking, 2 = checked-ok, 3 checked-notok, 4 checked-nodefault this[kApiVersioning] = process.env.ELASTIC_CLIENT_APIVERSIONING === 'true' this[kEventEmitter] = new EventEmitter() + this[kMaxResponseSize] = opts.maxResponseSize || MAX_STRING_LENGTH + this[kMaxCompressedResponseSize] = opts.maxCompressedResponseSize || MAX_BUFFER_LENGTH this.nodeFilter = opts.nodeFilter || defaultNodeFilter if (typeof opts.nodeSelector === 'function') { @@ -162,6 +166,8 @@ class Transport { ? 0 : (typeof options.maxRetries === 'number' ? options.maxRetries : this.maxRetries) const compression = options.compression !== undefined ? options.compression : this.compression + const maxResponseSize = options.maxResponseSize || this[kMaxResponseSize] + const maxCompressedResponseSize = options.maxCompressedResponseSize || this[kMaxCompressedResponseSize] let request = { abort: noop } const transportReturn = { then (onFulfilled, onRejected) { @@ -244,15 +250,15 @@ class Transport { /* istanbul ignore else */ if (result.headers['content-length'] !== undefined) { const contentLength = Number(result.headers['content-length']) - if (isCompressed && contentLength > MAX_BUFFER_LENGTH) { + if (isCompressed && contentLength > maxCompressedResponseSize) { response.destroy() return onConnectionError( - new RequestAbortedError(`The content length (${contentLength}) is bigger than the maximum allowed buffer (${MAX_BUFFER_LENGTH})`, result) + new RequestAbortedError(`The content length (${contentLength}) is bigger than the maximum allowed buffer (${maxCompressedResponseSize})`, result) ) - } else if (contentLength > MAX_STRING_LENGTH) { + } else if (contentLength > maxResponseSize) { response.destroy() return onConnectionError( - new RequestAbortedError(`The content length (${contentLength}) is bigger than the maximum allowed string (${MAX_STRING_LENGTH})`, result) + new RequestAbortedError(`The content length (${contentLength}) is bigger than the maximum allowed string (${maxResponseSize})`, result) ) } } From bcfb14e05e59c48e1c72bbbbd1994063acd1d2fa Mon Sep 17 00:00:00 2001 From: delvedor Date: Tue, 14 Sep 2021 10:59:42 +0200 Subject: [PATCH 2/3] Updated test --- test/unit/client.test.js | 217 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 217 insertions(+) diff --git a/test/unit/client.test.js b/test/unit/client.test.js index 20c1ce568..3503e17a7 100644 --- a/test/unit/client.test.js +++ b/test/unit/client.test.js @@ -1308,6 +1308,223 @@ test('Content length too big (string)', t => { }) }) +test('Content length too big custom (buffer)', t => { + t.plan(4) + + class MockConnection extends Connection { + request (params, callback) { + const stream = intoStream(JSON.stringify({ hello: 'world' })) + stream.statusCode = 200 + stream.headers = { + 'content-type': 'application/json;utf=8', + 'content-encoding': 'gzip', + 'content-length': 1100, + connection: 'keep-alive', + date: new Date().toISOString() + } + stream.on('close', () => t.pass('Stream destroyed')) + process.nextTick(callback, null, stream) + return { abort () {} } + } + } + + const client = new Client({ + node: 'http://localhost:9200', + Connection: MockConnection, + maxCompressedResponseSize: 1000 + }) + client.info((err, result) => { + t.ok(err instanceof errors.RequestAbortedError) + t.equal(err.message, 'The content length (1100) is bigger than the maximum allowed buffer (1000)') + t.equal(result.meta.attempts, 0) + }) +}) + +test('Content length too big custom (string)', t => { + t.plan(4) + + class MockConnection extends Connection { + request (params, callback) { + const stream = intoStream(JSON.stringify({ hello: 'world' })) + stream.statusCode = 200 + stream.headers = { + 'content-type': 'application/json;utf=8', + 'content-length': 1100, + connection: 'keep-alive', + date: new Date().toISOString() + } + stream.on('close', () => t.pass('Stream destroyed')) + process.nextTick(callback, null, stream) + return { abort () {} } + } + } + + const client = new Client({ + node: 'http://localhost:9200', + Connection: MockConnection, + maxResponseSize: 1000 + }) + client.info((err, result) => { + t.ok(err instanceof errors.RequestAbortedError) + t.equal(err.message, 'The content length (1100) is bigger than the maximum allowed string (1000)') + t.equal(result.meta.attempts, 0) + }) +}) + +test('Content length too big custom option (buffer)', t => { + t.plan(4) + + class MockConnection extends Connection { + request (params, callback) { + const stream = intoStream(JSON.stringify({ hello: 'world' })) + stream.statusCode = 200 + stream.headers = { + 'content-type': 'application/json;utf=8', + 'content-encoding': 'gzip', + 'content-length': 1100, + connection: 'keep-alive', + date: new Date().toISOString() + } + stream.on('close', () => t.pass('Stream destroyed')) + process.nextTick(callback, null, stream) + return { abort () {} } + } + } + + const client = new Client({ + node: 'http://localhost:9200', + Connection: MockConnection + }) + client.info({}, { maxCompressedResponseSize: 1000 }, (err, result) => { + t.ok(err instanceof errors.RequestAbortedError) + t.equal(err.message, 'The content length (1100) is bigger than the maximum allowed buffer (1000)') + t.equal(result.meta.attempts, 0) + }) +}) + +test('Content length too big custom option (string)', t => { + t.plan(4) + + class MockConnection extends Connection { + request (params, callback) { + const stream = intoStream(JSON.stringify({ hello: 'world' })) + stream.statusCode = 200 + stream.headers = { + 'content-type': 'application/json;utf=8', + 'content-length': 1100, + connection: 'keep-alive', + date: new Date().toISOString() + } + stream.on('close', () => t.pass('Stream destroyed')) + process.nextTick(callback, null, stream) + return { abort () {} } + } + } + + const client = new Client({ + node: 'http://localhost:9200', + Connection: MockConnection + }) + client.info({}, { maxResponseSize: 1000 }, (err, result) => { + t.ok(err instanceof errors.RequestAbortedError) + t.equal(err.message, 'The content length (1100) is bigger than the maximum allowed string (1000)') + t.equal(result.meta.attempts, 0) + }) +}) + +test('Content length too big custom option override (buffer)', t => { + t.plan(4) + + class MockConnection extends Connection { + request (params, callback) { + const stream = intoStream(JSON.stringify({ hello: 'world' })) + stream.statusCode = 200 + stream.headers = { + 'content-type': 'application/json;utf=8', + 'content-encoding': 'gzip', + 'content-length': 1100, + connection: 'keep-alive', + date: new Date().toISOString() + } + stream.on('close', () => t.pass('Stream destroyed')) + process.nextTick(callback, null, stream) + return { abort () {} } + } + } + + const client = new Client({ + node: 'http://localhost:9200', + Connection: MockConnection, + maxCompressedResponseSize: 2000 + }) + client.info({}, { maxCompressedResponseSize: 1000 }, (err, result) => { + t.ok(err instanceof errors.RequestAbortedError) + t.equal(err.message, 'The content length (1100) is bigger than the maximum allowed buffer (1000)') + t.equal(result.meta.attempts, 0) + }) +}) + +test('Content length too big custom option override (string)', t => { + t.plan(4) + + class MockConnection extends Connection { + request (params, callback) { + const stream = intoStream(JSON.stringify({ hello: 'world' })) + stream.statusCode = 200 + stream.headers = { + 'content-type': 'application/json;utf=8', + 'content-length': 1100, + connection: 'keep-alive', + date: new Date().toISOString() + } + stream.on('close', () => t.pass('Stream destroyed')) + process.nextTick(callback, null, stream) + return { abort () {} } + } + } + + const client = new Client({ + node: 'http://localhost:9200', + Connection: MockConnection, + maxResponseSize: 2000 + }) + client.info({}, { maxResponseSize: 1000 }, (err, result) => { + t.ok(err instanceof errors.RequestAbortedError) + t.equal(err.message, 'The content length (1100) is bigger than the maximum allowed string (1000)') + t.equal(result.meta.attempts, 0) + }) +}) + +test('maxResponseSize cannot be bigger than buffer.constants.MAX_STRING_LENGTH', t => { + t.plan(2) + + try { + new Client({ // eslint-disable-line + node: 'http://localhost:9200', + maxResponseSize: buffer.constants.MAX_STRING_LENGTH + 10 + }) + t.fail('should throw') + } catch (err) { + t.ok(err instanceof errors.ConfigurationError) + t.equal(err.message, `The maxResponseSize cannot be bigger than ${buffer.constants.MAX_STRING_LENGTH}`) + } +}) + +test('maxCompressedResponseSize cannot be bigger than buffer.constants.MAX_STRING_LENGTH', t => { + t.plan(2) + + try { + new Client({ // eslint-disable-line + node: 'http://localhost:9200', + maxCompressedResponseSize: buffer.constants.MAX_LENGTH + 10 + }) + t.fail('should throw') + } catch (err) { + t.ok(err instanceof errors.ConfigurationError) + t.equal(err.message, `The maxCompressedResponseSize cannot be bigger than ${buffer.constants.MAX_LENGTH}`) + } +}) + test('Meta header enabled', t => { t.plan(2) From bcb95c383f4f5e788f0bde71e95d9ca18a57acb2 Mon Sep 17 00:00:00 2001 From: delvedor Date: Tue, 14 Sep 2021 10:59:49 +0200 Subject: [PATCH 3/3] Updated docs --- docs/basic-config.asciidoc | 8 ++++++++ docs/connecting.asciidoc | 9 +++++++++ 2 files changed, 17 insertions(+) diff --git a/docs/basic-config.asciidoc b/docs/basic-config.asciidoc index c800b38c0..c9b4660dd 100644 --- a/docs/basic-config.asciidoc +++ b/docs/basic-config.asciidoc @@ -259,6 +259,14 @@ _Default:_ `false` |`string` - If configured, verify that the fingerprint of the CA certificate that has signed the certificate of the server matches the supplied fingerprint. Only accepts SHA256 digest fingerprints. + _Default:_ `null` +|`maxResponseSize` +|`number` - When configured, it verifies that the uncompressed response size is lower than the configured number, if it's higher it will abort the request. It cannot be higher than buffer.constants.MAX_STRING_LENTGH + +_Default:_ `null` + +|`maxCompressedResponseSize` +|`number` - When configured, it verifies that the compressed response size is lower than the configured number, if it's higher it will abort the request. It cannot be higher than buffer.constants.MAX_LENTGH + +_Default:_ `null` + |=== [discrete] diff --git a/docs/connecting.asciidoc b/docs/connecting.asciidoc index c76bf340b..acfbe120f 100644 --- a/docs/connecting.asciidoc +++ b/docs/connecting.asciidoc @@ -418,6 +418,15 @@ _Default:_ `null` |`context` |`any` - Custom object per request. _(you can use it to pass data to the clients events)_ + _Default:_ `null` + +|`maxResponseSize` +|`number` - When configured, it verifies that the uncompressed response size is lower than the configured number, if it's higher it will abort the request. It cannot be higher than buffer.constants.MAX_STRING_LENTGH + +_Default:_ `null` + +|`maxCompressedResponseSize` +|`number` - When configured, it verifies that the compressed response size is lower than the configured number, if it's higher it will abort the request. It cannot be higher than buffer.constants.MAX_LENTGH + +_Default:_ `null` + |=== [discrete]