From a8c88b227f3d574077ffcacd44087ffe677078e7 Mon Sep 17 00:00:00 2001 From: Kitson Kelly Date: Wed, 28 Apr 2021 11:04:48 +1000 Subject: [PATCH] feat(io): add readRange, readRangeSync --- io/buffer.ts | 8 +-- io/buffer_test.ts | 6 +- io/util.ts | 84 ++++++++++++++++++++++- io/util_test.ts | 169 +++++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 253 insertions(+), 14 deletions(-) diff --git a/io/buffer.ts b/io/buffer.ts index 6ce7e58f90f3..c28ea1404047 100644 --- a/io/buffer.ts +++ b/io/buffer.ts @@ -27,12 +27,8 @@ export class Buffer { #buf: Uint8Array; // contents are the bytes buf[off : len(buf)] #off = 0; // read at buf[off], write at buf[buf.byteLength] - constructor(ab?: ArrayBuffer) { - if (ab === undefined) { - this.#buf = new Uint8Array(0); - return; - } - this.#buf = new Uint8Array(ab); + constructor(ab?: ArrayBufferLike | ArrayLike) { + this.#buf = ab === undefined ? new Uint8Array(0) : new Uint8Array(ab); } /** Returns a slice holding the unread portion of the buffer. diff --git a/io/buffer_test.ts b/io/buffer_test.ts index 3d1c7c2bbce5..484a536b5910 100644 --- a/io/buffer_test.ts +++ b/io/buffer_test.ts @@ -90,7 +90,7 @@ Deno.test("bufferNewBuffer", () => { init(); assert(testBytes); assert(testString); - const buf = new Buffer(testBytes.buffer as ArrayBuffer); + const buf = new Buffer(testBytes.buffer); check(buf, testString); }); @@ -158,7 +158,7 @@ Deno.test("bufferTooLargeByteWrites", async () => { const tmp = new Uint8Array(72); const growLen = Number.MAX_VALUE; const xBytes = repeat("x", 0); - const buf = new Buffer(xBytes.buffer as ArrayBuffer); + const buf = new Buffer(xBytes.buffer); await buf.read(tmp); assertThrows( @@ -338,7 +338,7 @@ Deno.test("bufferTestGrow", async () => { for (const startLen of [0, 100, 1000, 10000]) { const xBytes = repeat("x", startLen); for (const growLen of [0, 100, 1000, 10000]) { - const buf = new Buffer(xBytes.buffer as ArrayBuffer); + const buf = new Buffer(xBytes.buffer); // If we read, this affects buf.off, which is good to test. const nread = (await buf.read(tmp)) ?? 0; buf.grow(growLen); diff --git a/io/util.ts b/io/util.ts index 58461488a221..528d24d3e917 100644 --- a/io/util.ts +++ b/io/util.ts @@ -1,4 +1,6 @@ import { Buffer } from "./buffer.ts"; +import { copy } from "../bytes/mod.ts"; +import { assert } from "../testing/asserts.ts"; const DEFAULT_BUFFER_SIZE = 32 * 1024; @@ -18,7 +20,7 @@ const DEFAULT_BUFFER_SIZE = 32 * 1024; * // Example from buffer * const myData = new Uint8Array(100); * // ... fill myData array with data - * const reader = new Buffer(myData.buffer as ArrayBuffer); + * const reader = new Buffer(myData.buffer); * const bufferContent = await readAll(reader); * ``` */ @@ -43,7 +45,7 @@ export async function readAll(r: Deno.Reader): Promise { * // Example from buffer * const myData = new Uint8Array(100); * // ... fill myData array with data - * const reader = new Buffer(myData.buffer as ArrayBuffer); + * const reader = new Buffer(myData.buffer); * const bufferContent = readAllSync(reader); * ``` */ @@ -53,6 +55,84 @@ export function readAllSync(r: Deno.ReaderSync): Uint8Array { return buf.bytes(); } +export interface ByteRange { + /** The 0 based index of the start byte for a range. */ + start: number; + + /** The 0 based index of the end byte for a range, which is inclusive. */ + end: number; +} + +/** + * Read a range of bytes from a file or other resource that is readable and + * seekable. The range start and end are inclusive of the bytes within that + * range. + * + * ```ts + * // Read the first 10 bytes of a file + * const file = await Deno.open("example.txt", { read: true }); + * const bytes = await readRange(file, { start: 0, end: 9 }); + * assert(bytes.length, 10); + * ``` + */ +export async function readRange( + r: Deno.Reader & Deno.Seeker, + range: ByteRange, +): Promise { + // byte ranges are inclusive, so we have to add one to the end + let length = range.end - range.start + 1; + assert(length > 0, "Invalid byte range was passed."); + await r.seek(range.start, Deno.SeekMode.Start); + const result = new Uint8Array(length); + let off = 0; + while (length) { + const p = new Uint8Array(Math.min(length, DEFAULT_BUFFER_SIZE)); + const nread = await r.read(p); + assert(nread !== null, "Unexpected EOF reach while reading a range."); + assert(nread > 0, "Unexpected read of 0 bytes while reading a range."); + copy(p, result, off); + off += nread; + length -= nread; + assert(length >= 0, "Unexpected length remaining after reading range."); + } + return result; +} + +/** + * Read a range of bytes synchronously from a file or other resource that is + * readable and seekable. The range start and end are inclusive of the bytes + * within that range. + * + * ```ts + * // Read the first 10 bytes of a file + * const file = Deno.openSync("example.txt", { read: true }); + * const bytes = readRangeSync(file, { start: 0, end: 9 }); + * assert(bytes.length, 10); + * ``` + */ +export function readRangeSync( + r: Deno.ReaderSync & Deno.SeekerSync, + range: ByteRange, +): Uint8Array { + // byte ranges are inclusive, so we have to add one to the end + let length = range.end - range.start + 1; + assert(length > 0, "Invalid byte range was passed."); + r.seekSync(range.start, Deno.SeekMode.Start); + const result = new Uint8Array(length); + let off = 0; + while (length) { + const p = new Uint8Array(Math.min(length, DEFAULT_BUFFER_SIZE)); + const nread = r.readSync(p); + assert(nread !== null, "Unexpected EOF reach while reading a range."); + assert(nread > 0, "Unexpected read of 0 bytes while reading a range."); + copy(p, result, off); + off += nread; + length -= nread; + assert(length >= 0, "Unexpected length remaining after reading range."); + } + return result; +} + /** Write all the content of the array buffer (`arr`) to the writer (`w`). * * ```ts diff --git a/io/util_test.ts b/io/util_test.ts index 0d3784f99e49..d931965ab9e0 100644 --- a/io/util_test.ts +++ b/io/util_test.ts @@ -3,13 +3,22 @@ // This code has been ported almost directly from Go's src/bytes/buffer_test.go // Copyright 2009 The Go Authors. All rights reserved. BSD license. // https://github.com/golang/go/blob/master/LICENSE -import { assert, assertEquals } from "../testing/asserts.ts"; + +import { copy } from "../bytes/mod.ts"; +import { + assert, + assertEquals, + assertThrows, + assertThrowsAsync, +} from "../testing/asserts.ts"; import { Buffer } from "./buffer.ts"; import { iter, iterSync, readAll, readAllSync, + readRange, + readRangeSync, writeAll, writeAllSync, } from "./util.ts"; @@ -30,7 +39,7 @@ export function init(): void { Deno.test("testReadAll", async () => { init(); assert(testBytes); - const reader = new Buffer(testBytes.buffer as ArrayBuffer); + const reader = new Buffer(testBytes.buffer); const actualBytes = await readAll(reader); assertEquals(testBytes.byteLength, actualBytes.byteLength); for (let i = 0; i < testBytes.length; ++i) { @@ -41,7 +50,7 @@ Deno.test("testReadAll", async () => { Deno.test("testReadAllSync", () => { init(); assert(testBytes); - const reader = new Buffer(testBytes.buffer as ArrayBuffer); + const reader = new Buffer(testBytes.buffer); const actualBytes = readAllSync(reader); assertEquals(testBytes.byteLength, actualBytes.byteLength); for (let i = 0; i < testBytes.length; ++i) { @@ -49,6 +58,160 @@ Deno.test("testReadAllSync", () => { } }); +class MockFile + implements + Deno.Seeker, + Deno.SeekerSync, + Deno.Reader, + Deno.ReaderSync, + Deno.Closer { + #buf: Uint8Array; + #closed = false; + #offset = 0; + + get closed() { + return this.#closed; + } + + constructor(buf: Uint8Array) { + this.#buf = buf; + } + + close() { + this.#closed = true; + } + + read(p: Uint8Array): Promise { + if (this.#offset >= this.#buf.length) { + return Promise.resolve(null); + } + const nread = Math.min(p.length, 16_384, this.#buf.length - this.#offset); + if (nread === 0) { + return Promise.resolve(0); + } + copy(this.#buf.subarray(this.#offset, this.#offset + nread), p); + this.#offset += nread; + return Promise.resolve(nread); + } + + readSync(p: Uint8Array): number | null { + if (this.#offset >= this.#buf.length) { + return null; + } + const nread = Math.min(p.length, 16_384, this.#buf.length - this.#offset); + if (nread === 0) { + return 0; + } + copy(this.#buf.subarray(this.#offset, this.#offset + nread), p); + this.#offset += nread; + return nread; + } + + seek(offset: number, whence: Deno.SeekMode): Promise { + assert(whence === Deno.SeekMode.Start); + if (offset >= this.#buf.length) { + return Promise.reject(new RangeError("seeked pass end")); + } + this.#offset = offset; + return Promise.resolve(this.#offset); + } + + seekSync(offset: number, whence: Deno.SeekMode): number { + assert(whence === Deno.SeekMode.Start); + if (offset >= this.#buf.length) { + throw new RangeError("seeked pass end"); + } + this.#offset = offset; + return this.#offset; + } +} + +Deno.test({ + name: "readRange", + async fn() { + init(); + assert(testBytes); + const file = new MockFile(testBytes); + const actual = await readRange(file, { start: 0, end: 9 }); + assertEquals(actual, testBytes.subarray(0, 10)); + }, +}); + +Deno.test({ + name: "readRange - invalid range", + async fn() { + init(); + assert(testBytes); + const file = new MockFile(testBytes); + await assertThrowsAsync( + async () => { + await readRange(file, { start: 100, end: 0 }); + }, + Error, + "Invalid byte range was passed.", + ); + }, +}); + +Deno.test({ + name: "readRange - read past EOF", + async fn() { + init(); + assert(testBytes); + const file = new MockFile(testBytes); + await assertThrowsAsync( + async () => { + await readRange(file, { start: 99, end: 100 }); + }, + Error, + "Unexpected EOF reach while reading a range.", + ); + }, +}); + +Deno.test({ + name: "readRangeSync", + fn() { + init(); + assert(testBytes); + const file = new MockFile(testBytes); + const actual = readRangeSync(file, { start: 0, end: 9 }); + assertEquals(actual, testBytes.subarray(0, 10)); + }, +}); + +Deno.test({ + name: "readRangeSync - invalid range", + fn() { + init(); + assert(testBytes); + const file = new MockFile(testBytes); + assertThrows( + () => { + readRangeSync(file, { start: 100, end: 0 }); + }, + Error, + "Invalid byte range was passed.", + ); + }, +}); + +Deno.test({ + name: "readRangeSync - read past EOF", + fn() { + init(); + assert(testBytes); + const file = new MockFile(testBytes); + assertThrows( + () => { + readRangeSync(file, { start: 99, end: 100 }); + }, + Error, + "Unexpected EOF reach while reading a range.", + ); + }, +}); + Deno.test("testwriteAll", async () => { init(); assert(testBytes);