From 009646418ded49381707622b7ce400429d4810d8 Mon Sep 17 00:00:00 2001 From: streamich Date: Sat, 23 Dec 2023 10:50:05 +0100 Subject: [PATCH 1/4] =?UTF-8?q?perf(json-pack):=20=E2=9A=A1=EF=B8=8F=20spe?= =?UTF-8?q?ed=20up=20length=20encoding?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/json-pack/resp/RespEncoder.ts | 38 ++++++++++++------------------- 1 file changed, 15 insertions(+), 23 deletions(-) diff --git a/src/json-pack/resp/RespEncoder.ts b/src/json-pack/resp/RespEncoder.ts index ffca7b42dc..8e48963f79 100644 --- a/src/json-pack/resp/RespEncoder.ts +++ b/src/json-pack/resp/RespEncoder.ts @@ -52,31 +52,23 @@ export class RespEncoder= pow) { - digits++; - pow *= 10; + const writer = this.writer; + if (length < 100) { + if (length < 10) { + writer.u8(length + 48); + return; } + const octet1 = length % 10; + const octet2 = (length - octet1) / 10; + writer.u16(((octet2 + 48) << 8) + octet1 + 48); + return; + } + let digits = 1; + let pow = 10; + while (length >= pow) { + digits++; + pow *= 10; } - const writer = this.writer; writer.ensureCapacity(digits); const uint8 = writer.uint8; const x = writer.x; From 655614e57ebf8d98d96aa6d938501f227c9c5de9 Mon Sep 17 00:00:00 2001 From: streamich Date: Sat, 23 Dec 2023 10:58:41 +0100 Subject: [PATCH 2/4] =?UTF-8?q?feat(json-pack):=20=F0=9F=8E=B8=20add=20ver?= =?UTF-8?q?batim=20string=20extension?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/json-pack/resp/RespEncoder.ts | 10 +++++++--- src/json-pack/resp/__tests__/RespEncoder.spec.ts | 7 +++++++ src/json-pack/resp/extensions.ts | 6 ++++++ 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/src/json-pack/resp/RespEncoder.ts b/src/json-pack/resp/RespEncoder.ts index 8e48963f79..3f8eb8d937 100644 --- a/src/json-pack/resp/RespEncoder.ts +++ b/src/json-pack/resp/RespEncoder.ts @@ -1,7 +1,8 @@ import {Writer} from '../../util/buffers/Writer'; import {RESP} from './constants'; import {utf8Size} from '../../util/strings/utf8'; -import {RespAttributes, RespPush} from './extensions'; +import {RespAttributes, RespPush, RespVerbatimString} from './extensions'; +import {JsonPackExtension} from '../JsonPackExtension'; import type {IWriter, IWriterGrowable} from '../../util/buffers'; import type {BinaryJsonEncoder, StreamingBinaryJsonEncoder, TlvBinaryJsonEncoder} from '../types'; import type {Slice} from '../../util/buffers/Slice'; @@ -38,8 +39,11 @@ export class RespEncoder); } case 'undefined': diff --git a/src/json-pack/resp/__tests__/RespEncoder.spec.ts b/src/json-pack/resp/__tests__/RespEncoder.spec.ts index d5654433db..9a1b1b306c 100644 --- a/src/json-pack/resp/__tests__/RespEncoder.spec.ts +++ b/src/json-pack/resp/__tests__/RespEncoder.spec.ts @@ -1,5 +1,6 @@ import {bufferToUint8Array} from '../../../util/buffers/bufferToUint8Array'; import {RespEncoder} from '../RespEncoder'; +import {RespVerbatimString} from '../extensions'; const Parser = require('redis-parser'); const parse = (uint8: Uint8Array): unknown => { @@ -76,6 +77,12 @@ describe('strings', () => { const encoded = encoder.writer.flush(); expect(toStr(encoded)).toBe('=8\r\ntxt:asdf\r\n'); }); + + test('can encode verbatim string using RespVerbatimString', () => { + const encoder = new RespEncoder(); + const encoded = encoder.encode(new RespVerbatimString('asdf')); + expect(toStr(encoded)).toBe('=8\r\ntxt:asdf\r\n'); + }); }); }); diff --git a/src/json-pack/resp/extensions.ts b/src/json-pack/resp/extensions.ts index c680a49ea1..c179e4e359 100644 --- a/src/json-pack/resp/extensions.ts +++ b/src/json-pack/resp/extensions.ts @@ -11,3 +11,9 @@ export class RespAttributes extends JsonPackExtension> { super(2, val); } } + +export class RespVerbatimString extends JsonPackExtension { + constructor(public readonly val: string) { + super(3, val); + } +} From d03bb0351172cfc17ae47d9dba88d6125752d511 Mon Sep 17 00:00:00 2001 From: streamich Date: Sat, 23 Dec 2023 11:22:00 +0100 Subject: [PATCH 3/4] =?UTF-8?q?feat(json-pack):=20=F0=9F=8E=B8=20add=20RES?= =?UTF-8?q?P=20v2=20encoding=20ability?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/json-pack/resp/RespEncoder.ts | 10 ++ src/json-pack/resp/RespEncoderLegacy.ts | 96 +++++++++++++++++++ .../resp/__tests__/RespEncoderLegacy.spec.ts | 52 ++++++++++ src/json-pack/resp/index.ts | 2 + 4 files changed, 160 insertions(+) create mode 100644 src/json-pack/resp/RespEncoderLegacy.ts create mode 100644 src/json-pack/resp/__tests__/RespEncoderLegacy.spec.ts diff --git a/src/json-pack/resp/RespEncoder.ts b/src/json-pack/resp/RespEncoder.ts index 3f8eb8d937..b534c2ac29 100644 --- a/src/json-pack/resp/RespEncoder.ts +++ b/src/json-pack/resp/RespEncoder.ts @@ -10,6 +10,9 @@ import type {Slice} from '../../util/buffers/Slice'; const REG_RN = /[\r\n]/; const isSafeInteger = Number.isSafeInteger; +/** + * Implements RESP3 encoding. + */ export class RespEncoder implements BinaryJsonEncoder, StreamingBinaryJsonEncoder, TlvBinaryJsonEncoder { @@ -244,6 +247,13 @@ export class RespEncoder extends RespEncoder { + public writeAny(value: unknown): void { + switch (typeof value) { + case 'number': + return this.writeNumber(value as number); + case 'string': + return this.writeStr(value); + case 'boolean': + return this.writeSimpleStr(value ? 'TRUE' : 'FALSE'); + case 'object': { + if (!value) return this.writeNull(); + if (value instanceof Array) return this.writeArr(value); + if (value instanceof Uint8Array) return this.writeBin(value); + if (value instanceof Error) return this.writeErr(value.message); + if (value instanceof Set) return this.writeSet(value); + if (value instanceof JsonPackExtension) { + if (value instanceof RespPush) return this.writeArr(value.val); + if (value instanceof RespVerbatimString) return this.writeStr(value.val); + if (value instanceof RespAttributes) return this.writeObj(value.val); + } + return this.writeObj(value as Record); + } + case 'undefined': + return this.writeUndef(); + case 'bigint': + return this.writeSimpleStrAscii(value + ''); + default: + return this.writeUnknown(value); + } + } + + public writeNumber(num: number): void { + if (isSafeInteger(num)) this.writeInteger(num); + else this.writeSimpleStrAscii(num + ''); + } + + public writeStr(str: string): void { + const length = str.length; + if (length < 64 && !REG_RN.test(str)) this.writeSimpleStr(str); + else this.writeBulkStr(str); + } + + public writeNull(): void { + this.writeNullArr(); + } + + public writeErr(str: string): void { + if (str.length < 64 && !REG_RN.test(str)) this.writeSimpleErr(str); + else this.writeBulkStr(str); + } + + public writeSet(set: Set): void { + this.writeArr([...set]); + } + + public writeArr(arr: unknown[]): void { + const writer = this.writer; + const length = arr.length; + writer.u8(RESP.ARR); // * + this.writeLength(length); + writer.u16(RESP.RN); // \r\n + for (let i = 0; i < length; i++) { + const val = arr[i]; + if (val === null) this.writeNullStr(); + else this.writeAny(val); + }; + } + + public writeObj(obj: Record): void { + const writer = this.writer; + const keys = Object.keys(obj); + const length = keys.length; + writer.u8(RESP.ARR); // % + this.writeLength(length << 1); + writer.u16(RESP.RN); // \r\n + for (let i = 0; i < length; i++) { + const key = keys[i]; + this.writeStr(key); + const val = obj[key]; + if (val === null) this.writeNullStr(); + else this.writeAny(val); + } + } +} diff --git a/src/json-pack/resp/__tests__/RespEncoderLegacy.spec.ts b/src/json-pack/resp/__tests__/RespEncoderLegacy.spec.ts new file mode 100644 index 0000000000..a7d67763e2 --- /dev/null +++ b/src/json-pack/resp/__tests__/RespEncoderLegacy.spec.ts @@ -0,0 +1,52 @@ +import {RespEncoderLegacy} from '../RespEncoderLegacy'; + +const encode = (value: unknown): string => { + const encoder = new RespEncoderLegacy(); + const encoded = encoder.encode(value); + return Buffer.from(encoded).toString(); +}; + +test('can encode simple strings', () => { + expect(encode('')).toBe('+\r\n'); + expect(encode('asdf')).toBe('+asdf\r\n'); +}); + +test('can encode simple errors', () => { + expect(encode(new Error('asdf'))).toBe('-asdf\r\n'); +}); + +test('can encode integers', () => { + expect(encode(0)).toBe(':0\r\n'); + expect(encode(123)).toBe(':123\r\n'); + expect(encode(-422469777)).toBe(':-422469777\r\n'); +}); + +test('can encode bulk strings', () => { + expect(encode('ab\nc')).toBe('$4\r\nab\nc\r\n'); + expect(encode(new Uint8Array([65]))).toBe('$1\r\nA\r\n'); +}); + +test('can encode arrays', () => { + expect(encode(['a', 1])).toBe('*2\r\n+a\r\n:1\r\n'); +}); + +test('encodes null as nullable array', () => { + expect(encode(null)).toBe('*-1\r\n'); +}); + +test('encodes null in nested structure as nullable string', () => { + expect(encode(['a', 'b', null])).toBe('*3\r\n+a\r\n+b\r\n$-1\r\n'); +}); + +test('encodes booleans as strings', () => { + expect(encode(true)).toBe('+TRUE\r\n'); + expect(encode(false)).toBe('+FALSE\r\n'); +}); + +test('encodes floats as strings', () => { + expect(encode(1.23)).toBe('+1.23\r\n'); +}); + +test('encodes objects as 2-tuple arrays', () => { + expect(encode({foo: 'bar'})).toBe('*2\r\n+foo\r\n+bar\r\n'); +}); diff --git a/src/json-pack/resp/index.ts b/src/json-pack/resp/index.ts index 65147cc44b..82c1e20632 100644 --- a/src/json-pack/resp/index.ts +++ b/src/json-pack/resp/index.ts @@ -1,4 +1,6 @@ export * from './constants'; export * from './extensions'; export * from './RespEncoder'; +export * from './RespEncoderLegacy'; export * from './RespDecoder'; +export * from './RespStreamingDecoder'; From 5a60649b200712764997ae861e2f7fd7617b6b82 Mon Sep 17 00:00:00 2001 From: streamich Date: Sat, 23 Dec 2023 11:22:33 +0100 Subject: [PATCH 4/4] =?UTF-8?q?style(json-pack):=20=F0=9F=92=84=20run=20Pr?= =?UTF-8?q?ettier?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/json-pack/resp/RespEncoderLegacy.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/json-pack/resp/RespEncoderLegacy.ts b/src/json-pack/resp/RespEncoderLegacy.ts index 474e44f1f5..81e848dbef 100644 --- a/src/json-pack/resp/RespEncoderLegacy.ts +++ b/src/json-pack/resp/RespEncoderLegacy.ts @@ -75,7 +75,7 @@ export class RespEncoderLegacy): void {