diff --git a/js/bin/print-buffer-alignment.js b/js/bin/print-buffer-alignment.js index 639c0af5d80ea..5df83eb83f0fa 100755 --- a/js/bin/print-buffer-alignment.js +++ b/js/bin/print-buffer-alignment.js @@ -22,50 +22,74 @@ const fs = require('fs'); const path = require('path'); const extension = process.env.ARROW_JS_DEBUG === 'src' ? '.ts' : '.cjs'; -const { RecordBatch, AsyncMessageReader } = require(`../index${extension}`); const { VectorLoader } = require(`../targets/apache-arrow/visitor/vectorloader`); +const { RecordBatch, AsyncMessageReader, makeData, Struct, Schema, Field } = require(`../index${extension}`); (async () => { const readable = process.argv.length < 3 ? process.stdin : fs.createReadStream(path.resolve(process.argv[2])); const reader = new AsyncMessageReader(readable); - let schema, recordBatchIndex = 0, dictionaryBatchIndex = 0; + let schema, metadataLength, message; + let byteOffset = 0; + let recordBatchCount = 0; + let dictionaryBatchCount = 0; - for await (const message of reader) { - - let bufferRegions = []; + while (1) { + if ((metadataLength = (await reader.readMetadataLength())).done) { break; } + if (metadataLength.value === -1) { + if ((metadataLength = (await reader.readMetadataLength())).done) { break; } + } + if ((message = (await reader.readMetadata(metadataLength.value))).done) { break; } - if (message.isSchema()) { - schema = message.header(); - continue; - } else if (message.isRecordBatch()) { - const header = message.header(); - bufferRegions = header.buffers; - const body = await reader.readMessageBody(message.bodyLength); + if (message.value.isSchema()) { + console.log( + `Schema:`, + { + byteOffset, + metadataLength: metadataLength.value, + }); + schema = message.value.header(); + byteOffset += metadataLength.value; + } else if (message.value.isRecordBatch()) { + const header = message.value.header(); + const bufferRegions = header.buffers; + const body = await reader.readMessageBody(message.value.bodyLength); const recordBatch = loadRecordBatch(schema, header, body); - console.log(`record batch ${++recordBatchIndex}: ${JSON.stringify({ - offset: body.byteOffset, - length: body.byteLength, - numRows: recordBatch.length, - })}`); - } else if (message.isDictionaryBatch()) { - const header = message.header(); - bufferRegions = header.data.buffers; + console.log( + `RecordBatch ${++recordBatchCount}:`, + { + numRows: recordBatch.numRows, + byteOffset, + metadataLength: metadataLength.value, + bodyByteLength: body.byteLength, + }); + byteOffset += metadataLength.value; + bufferRegions.forEach(({ offset, length: byteLength }, i) => { + console.log(`\tbuffer ${i + 1}:`, { byteOffset: byteOffset + offset, byteLength }); + }); + byteOffset += body.byteLength; + } else if (message.value.isDictionaryBatch()) { + const header = message.value.header(); + const bufferRegions = header.data.buffers; const type = schema.dictionaries.get(header.id); - const body = await reader.readMessageBody(message.bodyLength); + const body = await reader.readMessageBody(message.value.bodyLength); const recordBatch = loadDictionaryBatch(header.data, body, type); - console.log(`dictionary batch ${++dictionaryBatchIndex}: ${JSON.stringify({ - offset: body.byteOffset, - length: body.byteLength, - numRows: recordBatch.length, - dictionaryId: header.id, - })}`); + console.log( + `DictionaryBatch ${++dictionaryBatchCount}:`, + { + id: header.id, + numRows: recordBatch.numRows, + byteOffset, + metadataLength: metadataLength.value, + bodyByteLength: body.byteLength, + }); + byteOffset += metadataLength.value; + bufferRegions.forEach(({ offset, length: byteLength }, i) => { + console.log(`\tbuffer ${i + 1}:`, { byteOffset: byteOffset + offset, byteLength }); + }); + byteOffset += body.byteLength; } - - bufferRegions.forEach(({ offset, length }, i) => { - console.log(`\tbuffer ${i + 1}: { offset: ${offset}, length: ${length} }`); - }); } await reader.return(); @@ -73,9 +97,26 @@ const { VectorLoader } = require(`../targets/apache-arrow/visitor/vectorloader`) })().catch((e) => { console.error(e); process.exit(1); }); function loadRecordBatch(schema, header, body) { - return new RecordBatch(schema, header.length, new VectorLoader(body, header.nodes, header.buffers, new Map()).visitMany(schema.fields)); + const children = new VectorLoader(body, header.nodes, header.buffers, new Map()).visitMany(schema.fields); + return new RecordBatch( + schema, + makeData({ + type: new Struct(schema.fields), + length: header.length, + children: children + }) + ); } function loadDictionaryBatch(header, body, dictionaryType) { - return RecordBatch.new(new VectorLoader(body, header.nodes, header.buffers, new Map()).visitMany([dictionaryType])); + const schema = new Schema([new Field('', dictionaryType)]); + const children = new VectorLoader(body, header.nodes, header.buffers, new Map()).visitMany([dictionaryType]); + return new RecordBatch( + schema, + makeData({ + type: new Struct(schema.fields), + length: header.length, + children: children + }) + ); } diff --git a/js/jest.config.js b/js/jest.config.js index d30090a00ee45..8aaf60a0ff4a2 100644 --- a/js/jest.config.js +++ b/js/jest.config.js @@ -16,38 +16,46 @@ // under the License. export default { - verbose: false, - testEnvironment: "node", - globals: { - "ts-jest": { - diagnostics: false, - tsconfig: "test/tsconfig.json", - useESM: true, + verbose: false, + testEnvironment: "node", + rootDir: ".", + roots: [ + "/test/", + ], + extensionsToTreatAsEsm: [".ts"], + moduleFileExtensions: ["js", "mjs", "ts"], + coverageReporters: ["lcov", "json",], + coveragePathIgnorePatterns: [ + "fb\\/.*\\.(js|ts)$", + "test\\/.*\\.(ts|js)$", + "/node_modules/", + ], + moduleNameMapper: { + "^apache-arrow$": "/src/Arrow.node", + "^apache-arrow(.*)": "/src$1", + "^(\\.{1,2}/.*)\\.js$": "$1", }, - }, - rootDir: ".", - roots: ["/test/"], - preset: "ts-jest/presets/default-esm", - moduleFileExtensions: ["mjs", "js", "ts"], - coverageReporters: ["lcov", "json"], - coveragePathIgnorePatterns: [ - "fb\\/.*\\.(js|ts)$", - "test\\/.*\\.(ts|js)$", - "/node_modules/", - ], - transform: { - "^.+\\.js$": "ts-jest", - "^.+\\.ts$": "ts-jest", - }, - transformIgnorePatterns: [ - "/targets/(es5|es2015|esnext|apache-arrow)/", - "/node_modules/(?!@openpgp/web-stream-tools)/", - ], - testRegex: "(.*(-|\\.)(test|spec)s?)\\.(ts|js)$", - testMatch: null, - moduleNameMapper: { - "^apache-arrow$": "/src/Arrow.node", - "^apache-arrow(.*)": "/src$1", - "^(\\.{1,2}/.*)\\.js$": "$1", - }, + testRegex: "(.*(-|\\.)(test|spec)s?)\\.(ts|js)$", + transform: { + "^.+\\.js$": [ + "ts-jest", + { + diagnostics: false, + tsconfig: "test/tsconfig.json", + useESM: true, + }, + ], + "^.+\\.ts$": [ + "ts-jest", + { + diagnostics: false, + tsconfig: "test/tsconfig.json", + useESM: true, + }, + ], + }, + transformIgnorePatterns: [ + "/targets/(es5|es2015|esnext|apache-arrow)/", + "/node_modules/(?!@openpgp/web-stream-tools)/", + ], }; diff --git a/js/jestconfigs/jest.apache-arrow.config.js b/js/jestconfigs/jest.apache-arrow.config.js index e945d944e2b81..8fc4118f50215 100644 --- a/js/jestconfigs/jest.apache-arrow.config.js +++ b/js/jestconfigs/jest.apache-arrow.config.js @@ -20,16 +20,28 @@ import config from "../jest.config.js"; export default { ...config, rootDir: "../", - preset: "ts-jest", moduleFileExtensions: ["js", "ts"], - globals: { - "ts-jest": { - diagnostics: false, - tsconfig: "/test/tsconfig/tsconfig.apache-arrow.json", - }, - }, moduleNameMapper: { "^apache-arrow(.*)": "/targets/apache-arrow$1", "^(\\.{1,2}/.*)\\.js$": "$1", }, + transform: { + ...config.transform, + "^.+\\.js$": [ + "ts-jest", + { + diagnostics: false, + tsconfig: "/test/tsconfig/tsconfig.apache-arrow.json", + useESM: true, + }, + ], + "^.+\\.ts$": [ + "ts-jest", + { + diagnostics: false, + tsconfig: "/test/tsconfig/tsconfig.apache-arrow.json", + useESM: true, + }, + ], + }, }; diff --git a/js/jestconfigs/jest.coverage.config.js b/js/jestconfigs/jest.coverage.config.js index 3e7444b736add..3b53d109cdb02 100644 --- a/js/jestconfigs/jest.coverage.config.js +++ b/js/jestconfigs/jest.coverage.config.js @@ -22,11 +22,23 @@ export default { rootDir: "../", collectCoverage: true, reporters: undefined, - globals: { - "ts-jest": { - diagnostics: false, - tsconfig: "/test/tsconfig/tsconfig.coverage.json", - useESM: true, - }, + transform: { + ...config.transform, + "^.+\\.js$": [ + "ts-jest", + { + diagnostics: false, + tsconfig: "/test/tsconfig/tsconfig.coverage.json", + useESM: true, + }, + ], + "^.+\\.ts$": [ + "ts-jest", + { + diagnostics: false, + tsconfig: "/test/tsconfig/tsconfig.coverage.json", + useESM: true, + }, + ], }, }; diff --git a/js/jestconfigs/jest.es2015.cjs.config.js b/js/jestconfigs/jest.es2015.cjs.config.js index 260aa39b9ae73..ec80b5436654a 100644 --- a/js/jestconfigs/jest.es2015.cjs.config.js +++ b/js/jestconfigs/jest.es2015.cjs.config.js @@ -20,16 +20,26 @@ import config from "../jest.config.js"; export default { ...config, rootDir: "../", - preset: "ts-jest", moduleFileExtensions: ["js", "ts"], - globals: { - "ts-jest": { - diagnostics: false, - tsconfig: "/test/tsconfig/tsconfig.es2015.cjs.json", - }, - }, moduleNameMapper: { "^apache-arrow(.*)": "/targets/es2015/cjs$1", "^(\\.{1,2}/.*)\\.js$": "$1", }, + transform: { + ...config.transform, + "^.+\\.js$": [ + "ts-jest", + { + diagnostics: false, + tsconfig: "/test/tsconfig/tsconfig.es2015.cjs.json", + }, + ], + "^.+\\.ts$": [ + "ts-jest", + { + diagnostics: false, + tsconfig: "/test/tsconfig/tsconfig.es2015.cjs.json", + }, + ], + }, }; diff --git a/js/jestconfigs/jest.es2015.esm.config.js b/js/jestconfigs/jest.es2015.esm.config.js index b5978f6a1d3f6..7e69123d72a6f 100644 --- a/js/jestconfigs/jest.es2015.esm.config.js +++ b/js/jestconfigs/jest.es2015.esm.config.js @@ -20,16 +20,28 @@ import config from "../jest.config.js"; export default { ...config, rootDir: "../", - globals: { - "ts-jest": { - diagnostics: false, - tsconfig: "/test/tsconfig/tsconfig.es2015.esm.json", - useESM: true, - }, - }, moduleNameMapper: { "^apache-arrow(.*)": "/targets/es2015/esm$1", tslib: "tslib/tslib.es6.js", "^(\\.{1,2}/.*)\\.js$": "$1", }, + transform: { + ...config.transform, + "^.+\\.js$": [ + "ts-jest", + { + diagnostics: false, + tsconfig: "/test/tsconfig/tsconfig.es2015.esm.json", + useESM: true, + }, + ], + "^.+\\.ts$": [ + "ts-jest", + { + diagnostics: false, + tsconfig: "/test/tsconfig/tsconfig.es2015.esm.json", + useESM: true, + }, + ], + }, }; diff --git a/js/jestconfigs/jest.es2015.umd.config.js b/js/jestconfigs/jest.es2015.umd.config.js index 5c67846e0efcb..fb0c9c510407f 100644 --- a/js/jestconfigs/jest.es2015.umd.config.js +++ b/js/jestconfigs/jest.es2015.umd.config.js @@ -20,16 +20,26 @@ import config from "../jest.config.js"; export default { ...config, rootDir: "../", - preset: "ts-jest", moduleFileExtensions: ["js", "ts"], - globals: { - "ts-jest": { - diagnostics: false, - tsconfig: "/test/tsconfig/tsconfig.es2015.umd.json", - }, - }, moduleNameMapper: { "^apache-arrow": "/targets/es2015/umd/Arrow.js", "^(\\.{1,2}/.*)\\.js$": "$1", }, + transform: { + ...config.transform, + "^.+\\.js$": [ + "ts-jest", + { + diagnostics: false, + tsconfig: "/test/tsconfig/tsconfig.es2015.umd.json", + }, + ], + "^.+\\.ts$": [ + "ts-jest", + { + diagnostics: false, + tsconfig: "/test/tsconfig/tsconfig.es2015.umd.json", + }, + ], + }, }; diff --git a/js/jestconfigs/jest.es5.cjs.config.js b/js/jestconfigs/jest.es5.cjs.config.js index 61ff88f3d9a94..bfc6bce0ea9c4 100644 --- a/js/jestconfigs/jest.es5.cjs.config.js +++ b/js/jestconfigs/jest.es5.cjs.config.js @@ -20,16 +20,26 @@ import config from "../jest.config.js"; export default { ...config, rootDir: "../", - preset: "ts-jest", moduleFileExtensions: ["js", "ts"], - globals: { - "ts-jest": { - diagnostics: false, - tsconfig: "/test/tsconfig/tsconfig.es5.cjs.json", - }, - }, moduleNameMapper: { "^apache-arrow(.*)": "/targets/es5/cjs$1", "^(\\.{1,2}/.*)\\.js$": "$1", }, + transform: { + ...config.transform, + "^.+\\.js$": [ + "ts-jest", + { + diagnostics: false, + tsconfig: "/test/tsconfig/tsconfig.es5.cjs.json", + }, + ], + "^.+\\.ts$": [ + "ts-jest", + { + diagnostics: false, + tsconfig: "/test/tsconfig/tsconfig.es5.cjs.json", + }, + ], + }, }; diff --git a/js/jestconfigs/jest.es5.esm.config.js b/js/jestconfigs/jest.es5.esm.config.js index aaa73b212714e..55aaffa3ebd09 100644 --- a/js/jestconfigs/jest.es5.esm.config.js +++ b/js/jestconfigs/jest.es5.esm.config.js @@ -20,16 +20,28 @@ import config from "../jest.config.js"; export default { ...config, rootDir: "../", - globals: { - "ts-jest": { - diagnostics: false, - tsconfig: "/test/tsconfig/tsconfig.es5.esm.json", - useESM: true, - }, - }, moduleNameMapper: { "^apache-arrow(.*)": "/targets/es5/esm$1", tslib: "tslib/tslib.es6.js", "^(\\.{1,2}/.*)\\.js$": "$1", }, + transform: { + ...config.transform, + "^.+\\.js$": [ + "ts-jest", + { + diagnostics: false, + tsconfig: "/test/tsconfig/tsconfig.es5.esm.json", + useESM: true, + }, + ], + "^.+\\.ts$": [ + "ts-jest", + { + diagnostics: false, + tsconfig: "/test/tsconfig/tsconfig.es5.esm.json", + useESM: true, + }, + ], + }, }; diff --git a/js/jestconfigs/jest.es5.umd.config.js b/js/jestconfigs/jest.es5.umd.config.js index 280575eac1c2b..d125b0b780099 100644 --- a/js/jestconfigs/jest.es5.umd.config.js +++ b/js/jestconfigs/jest.es5.umd.config.js @@ -20,16 +20,26 @@ import config from "../jest.config.js"; export default { ...config, rootDir: "../", - preset: "ts-jest", moduleFileExtensions: ["js", "ts"], - globals: { - "ts-jest": { - diagnostics: false, - tsconfig: "/test/tsconfig/tsconfig.es5.umd.json", - }, - }, moduleNameMapper: { "^apache-arrow": "/targets/es5/umd/Arrow.js", "^(\\.{1,2}/.*)\\.js$": "$1", }, + transform: { + ...config.transform, + "^.+\\.js$": [ + "ts-jest", + { + diagnostics: false, + tsconfig: "/test/tsconfig/tsconfig.es5.umd.json", + }, + ], + "^.+\\.ts$": [ + "ts-jest", + { + diagnostics: false, + tsconfig: "/test/tsconfig/tsconfig.es5.umd.json", + }, + ], + }, }; diff --git a/js/jestconfigs/jest.esnext.cjs.config.js b/js/jestconfigs/jest.esnext.cjs.config.js index febfe6de9634e..8eddd96e0e783 100644 --- a/js/jestconfigs/jest.esnext.cjs.config.js +++ b/js/jestconfigs/jest.esnext.cjs.config.js @@ -20,16 +20,26 @@ import config from "../jest.config.js"; export default { ...config, rootDir: "../", - preset: "ts-jest", moduleFileExtensions: ["js", "ts"], - globals: { - "ts-jest": { - diagnostics: false, - tsconfig: "/test/tsconfig/tsconfig.esnext.cjs.json", - }, - }, moduleNameMapper: { "^apache-arrow(.*)": "/targets/esnext/cjs$1", "^(\\.{1,2}/.*)\\.js$": "$1", }, + transform: { + ...config.transform, + "^.+\\.js$": [ + "ts-jest", + { + diagnostics: false, + tsconfig: "/test/tsconfig/tsconfig.esnext.cjs.json", + }, + ], + "^.+\\.ts$": [ + "ts-jest", + { + diagnostics: false, + tsconfig: "/test/tsconfig/tsconfig.esnext.cjs.json", + }, + ], + }, }; diff --git a/js/jestconfigs/jest.esnext.esm.config.js b/js/jestconfigs/jest.esnext.esm.config.js index 6513637881bf3..4a344813df76f 100644 --- a/js/jestconfigs/jest.esnext.esm.config.js +++ b/js/jestconfigs/jest.esnext.esm.config.js @@ -20,16 +20,28 @@ import config from "../jest.config.js"; export default { ...config, rootDir: "../", - globals: { - "ts-jest": { - diagnostics: false, - tsconfig: "/test/tsconfig/tsconfig.esnext.esm.json", - useESM: true, - }, - }, moduleNameMapper: { "^apache-arrow(.*)": "/targets/esnext/esm$1", tslib: "tslib/tslib.es6.js", "^(\\.{1,2}/.*)\\.js$": "$1", }, + transform: { + ...config.transform, + "^.+\\.js$": [ + "ts-jest", + { + diagnostics: false, + tsconfig: "/test/tsconfig/tsconfig.esnext.esm.json", + useESM: true, + }, + ], + "^.+\\.ts$": [ + "ts-jest", + { + diagnostics: false, + tsconfig: "/test/tsconfig/tsconfig.esnext.esm.json", + useESM: true, + }, + ], + }, }; diff --git a/js/jestconfigs/jest.esnext.umd.config.js b/js/jestconfigs/jest.esnext.umd.config.js index a0f4f13335d61..fd520104a6111 100644 --- a/js/jestconfigs/jest.esnext.umd.config.js +++ b/js/jestconfigs/jest.esnext.umd.config.js @@ -20,16 +20,26 @@ import config from "../jest.config.js"; export default { ...config, rootDir: "../", - preset: "ts-jest", moduleFileExtensions: ["js", "ts"], - globals: { - "ts-jest": { - diagnostics: false, - tsconfig: "/test/tsconfig/tsconfig.esnext.umd.json", - }, - }, moduleNameMapper: { "^apache-arrow": "/targets/esnext/umd/Arrow.js", "^(\\.{1,2}/.*)\\.js$": "$1", }, + transform: { + ...config.transform, + "^.+\\.js$": [ + "ts-jest", + { + diagnostics: false, + tsconfig: "/test/tsconfig/tsconfig.esnext.umd.json", + }, + ], + "^.+\\.ts$": [ + "ts-jest", + { + diagnostics: false, + tsconfig: "/test/tsconfig/tsconfig.esnext.umd.json", + }, + ], + }, }; diff --git a/js/jestconfigs/jest.src.config.js b/js/jestconfigs/jest.src.config.js index 18766713e38df..c30155b8c6f24 100644 --- a/js/jestconfigs/jest.src.config.js +++ b/js/jestconfigs/jest.src.config.js @@ -20,11 +20,23 @@ import config from "../jest.config.js"; export default { ...config, rootDir: "../", - globals: { - "ts-jest": { - diagnostics: false, - tsconfig: "/test/tsconfig/tsconfig.src.json", - useESM: true, - }, + transform: { + ...config.transform, + "^.+\\.js$": [ + "ts-jest", + { + diagnostics: false, + tsconfig: "/test/tsconfig/tsconfig.src.json", + useESM: true, + }, + ], + "^.+\\.ts$": [ + "ts-jest", + { + diagnostics: false, + tsconfig: "/test/tsconfig/tsconfig.src.json", + useESM: true, + }, + ], }, }; diff --git a/js/jestconfigs/jest.ts.config.js b/js/jestconfigs/jest.ts.config.js index 3d4eccd9ee76d..08f5514f3b708 100644 --- a/js/jestconfigs/jest.ts.config.js +++ b/js/jestconfigs/jest.ts.config.js @@ -20,16 +20,28 @@ import config from "../jest.config.js"; export default { ...config, rootDir: "../", - globals: { - "ts-jest": { - diagnostics: false, - tsconfig: "/test/tsconfig/tsconfig.ts.json", - useESM: true, - }, - }, moduleNameMapper: { "^apache-arrow$": "/targets/ts/Arrow.node", "^apache-arrow(.*)": "/targets/ts$1", "^(\\.{1,2}/.*)\\.js$": "$1", }, + transform: { + ...config.transform, + "^.+\\.js$": [ + "ts-jest", + { + diagnostics: false, + tsconfig: "/test/tsconfig/tsconfig.ts.json", + useESM: true, + }, + ], + "^.+\\.ts$": [ + "ts-jest", + { + diagnostics: false, + tsconfig: "/test/tsconfig/tsconfig.ts.json", + useESM: true, + }, + ], + }, }; diff --git a/js/src/bin/arrow2csv.ts b/js/src/bin/arrow2csv.ts index 41b928fc4d70b..eae7f5805c41c 100644 --- a/js/src/bin/arrow2csv.ts +++ b/js/src/bin/arrow2csv.ts @@ -178,10 +178,10 @@ function batchesToString(state: ToStringState, schema: Schema) { maxColWidths = state.maxColWidths; for (const row of batch) { if (state.closed) { break; } else if (!row) { continue; } - if (rowId++ % 350 === 0) { + if (rowId % 350 === 0) { this.push(`${formatRow(header, maxColWidths, sep)}\n`); } - this.push(`${formatRow([rowId, ...row.toArray()].map(v => valueToString(v)), maxColWidths, sep)}\n`); + this.push(`${formatRow([rowId++, ...row.toArray()].map(v => valueToString(v)), maxColWidths, sep)}\n`); } } cb(); diff --git a/js/src/builder/buffer.ts b/js/src/builder/buffer.ts index e65cdc430e821..03d4f33349a7a 100644 --- a/js/src/builder/buffer.ts +++ b/js/src/builder/buffer.ts @@ -29,7 +29,7 @@ import { any; /** @ignore */ -const roundLengthUpToNearest64Bytes = (len: number, BPE: number) => ((((len * BPE) + 63) & ~63) || 64) / BPE; +const roundLengthUpToNearest64Bytes = (len: number, BPE: number) => ((((Math.ceil(len) * BPE) + 63) & ~63) || 64) / BPE; /** @ignore */ const sliceOrExtendArray = (arr: T, len = 0) => ( arr.length >= len ? arr.subarray(0, len) : memcpy(new (arr.constructor as any)(len), arr, 0) @@ -48,7 +48,7 @@ export class BufferBuilder; - this._resize(this.length = Math.trunc(buffer.length / stride)); + this._resize(this.length = Math.ceil(buffer.length / stride)); } public buffer: T; @@ -57,7 +57,9 @@ export class BufferBuilder; public readonly BYTES_PER_ELEMENT: number; - public get byteLength() { return Math.trunc(this.length * this.stride * this.BYTES_PER_ELEMENT); } + public get byteLength() { + return Math.ceil(this.length * this.stride) * this.BYTES_PER_ELEMENT; + } public get reservedLength() { return this.buffer.length / this.stride; } public get reservedByteLength() { return this.buffer.byteLength; } @@ -153,29 +155,3 @@ export class OffsetsBufferBuilder extends DataBufferBuilder { return super.flush(length + 1); } } - -// /** @ignore */ -// export class WideBufferBuilder extends BufferBuilder> { -// public buffer64!: R; -// protected _ArrayType64!: BigIntArrayConstructor; -// public get ArrayType64() { -// return this._ArrayType64 || (this._ArrayType64 = > (this.buffer instanceof Int32Array ? BigInt64Array : BigUint64Array)); -// } -// public set(index: number, value: DataValue) { -// this.reserve(index - this.length + 1); -// switch (typeof value) { -// case 'bigint': this.buffer64[index] = value; break; -// case 'number': this.buffer[index * this.stride] = value; break; -// default: this.buffer.set(value as TypedArray, index * this.stride); -// } -// return this; -// } -// protected _resize(newLength: number) { -// const data = super._resize(newLength); -// const length = data.byteLength / (this.BYTES_PER_ELEMENT * this.stride); -// if (BigIntAvailable) { -// this.buffer64 = new this.ArrayType64(data.buffer, data.byteOffset, length); -// } -// return data; -// } -// } diff --git a/js/test/generate-test-data.ts b/js/test/generate-test-data.ts index 8174a3d0046cb..a84344e1a0395 100644 --- a/js/test/generate-test-data.ts +++ b/js/test/generate-test-data.ts @@ -638,7 +638,7 @@ function iterateBitmap(length: number, bitmap: Uint8Array, fn: (index: number, v function createBitmap(length: number, nullCount: number) { const nulls = Object.create(null) as { [key: number]: boolean }; - const bytes = new Uint8Array((((length >> 3) + 7) & ~7) || 8).fill(255); + const bytes = new Uint8Array((Math.ceil(length / 8) + 63) & ~63).fill(255); for (let i, j = -1; ++j < nullCount;) { // eslint-disable-next-line unicorn/prefer-math-trunc while (nulls[i = (rand() * length) | 0]); @@ -655,7 +655,7 @@ function createVariableWidthOffsets(length: number, nullBitmap: Uint8Array, max offsets[i + 1] = offsets[i]; } else { do { - offsets[i + 1] = Math.min(max, offsets[i] + (Math.trunc(rand() * stride))); + offsets[i + 1] = Math.min(max, offsets[i] + Math.max(10, Math.trunc(rand() * stride))); } while (!allowEmpty && offsets[i + 1] === offsets[i]); } }); diff --git a/js/test/unit/builders/builder-tests.ts b/js/test/unit/builders/builder-tests.ts index 46b2d71b9708a..a73183a7a5d47 100644 --- a/js/test/unit/builders/builder-tests.ts +++ b/js/test/unit/builders/builder-tests.ts @@ -75,6 +75,7 @@ function validateBuilder(generate: (length?: number, nullCount?: number, ...args for (let i = -1; ++i < 1;) { validateBuilderWithNullValues(`no nulls`, [], generate(100, 0)); validateBuilderWithNullValues(`with nulls`, [null], generate(100)); + validateBuilderWithNullValues(`with nulls (length=518)`, [null], generate(518)); if (DataType.isUtf8(type)) { validateBuilderWithNullValues(`with \\0`, ['\0'], generate(100)); validateBuilderWithNullValues(`with n/a`, ['n/a'], generate(100));