From 4e8a6fc752b712737c6c8eb6792ee21fdc439cc4 Mon Sep 17 00:00:00 2001 From: Hugo Dias Date: Sun, 10 Jan 2021 18:00:32 +0000 Subject: [PATCH 01/10] fix: remove files functions these were copied to ipfs-core-utils --- src/files/format-mode.js | 66 ------- src/files/format-mtime.js | 21 -- src/files/normalise-input.js | 298 ----------------------------- test/files/format-mode.spec.js | 54 ------ test/files/format-mtime.spec.js | 15 -- test/files/normalise-input.spec.js | 146 -------------- 6 files changed, 600 deletions(-) delete mode 100644 src/files/format-mode.js delete mode 100644 src/files/format-mtime.js delete mode 100644 src/files/normalise-input.js delete mode 100644 test/files/format-mode.spec.js delete mode 100644 test/files/format-mtime.spec.js delete mode 100644 test/files/normalise-input.spec.js diff --git a/src/files/format-mode.js b/src/files/format-mode.js deleted file mode 100644 index f25ae5c..0000000 --- a/src/files/format-mode.js +++ /dev/null @@ -1,66 +0,0 @@ -'use strict' - -const S_ISUID = parseInt('4000', 8) // set UID bit -const S_ISGID = parseInt('2000', 8) // set-group-ID bit (see below) -const S_ISVTX = parseInt('1000', 8) // sticky bit (see below) -// const S_IRWXU = parseInt('700', 8) // mask for file owner permissions -const S_IRUSR = parseInt('400', 8) // owner has read permission -const S_IWUSR = parseInt('200', 8) // owner has write permission -const S_IXUSR = parseInt('100', 8) // owner has execute permission -// const S_IRWXG = parseInt('70', 8) // mask for group permissions -const S_IRGRP = parseInt('40', 8) // group has read permission -const S_IWGRP = parseInt('20', 8) // group has write permission -const S_IXGRP = parseInt('10', 8) // group has execute permission -// const S_IRWXO = parseInt('7', 8) // mask for permissions for others (not in group) -const S_IROTH = parseInt('4', 8) // others have read permission -const S_IWOTH = parseInt('2', 8) // others have write permission -const S_IXOTH = parseInt('1', 8) // others have execute permission - -function checkPermission (mode, perm, type, output) { - if ((mode & perm) === perm) { - output.push(type) - } else { - output.push('-') - } -} - -function formatMode (mode, isDirectory) { - const output = [] - - if (isDirectory) { - output.push('d') - } else { - output.push('-') - } - - checkPermission(mode, S_IRUSR, 'r', output) - checkPermission(mode, S_IWUSR, 'w', output) - - if ((mode & S_ISUID) === S_ISUID) { - output.push('s') - } else { - checkPermission(mode, S_IXUSR, 'x', output) - } - - checkPermission(mode, S_IRGRP, 'r', output) - checkPermission(mode, S_IWGRP, 'w', output) - - if ((mode & S_ISGID) === S_ISGID) { - output.push('s') - } else { - checkPermission(mode, S_IXGRP, 'x', output) - } - - checkPermission(mode, S_IROTH, 'r', output) - checkPermission(mode, S_IWOTH, 'w', output) - - if ((mode & S_ISVTX) === S_ISVTX) { - output.push('t') - } else { - checkPermission(mode, S_IXOTH, 'x', output) - } - - return output.join('') -} - -module.exports = formatMode diff --git a/src/files/format-mtime.js b/src/files/format-mtime.js deleted file mode 100644 index 486cecb..0000000 --- a/src/files/format-mtime.js +++ /dev/null @@ -1,21 +0,0 @@ -'use strict' - -function formatMtime (mtime) { - if (mtime == null) { - return '-' - } - - const date = new Date((mtime.secs * 1000) + Math.round(mtime.nsecs / 1000)) - - return date.toLocaleDateString(Intl.DateTimeFormat().resolvedOptions().locale, { - year: 'numeric', - month: 'short', - day: 'numeric', - hour: '2-digit', - minute: '2-digit', - second: '2-digit', - timeZoneName: 'short' - }) -} - -module.exports = formatMtime diff --git a/src/files/normalise-input.js b/src/files/normalise-input.js deleted file mode 100644 index 32c9c11..0000000 --- a/src/files/normalise-input.js +++ /dev/null @@ -1,298 +0,0 @@ -'use strict' - -const errCode = require('err-code') -const { Buffer } = require('buffer') -const globalThis = require('../globalthis') - -/* - * Transform one of: - * - * ``` - * Bytes (Buffer|ArrayBuffer|TypedArray) [single file] - * Bloby (Blob|File) [single file] - * String [single file] - * { path, content: Bytes } [single file] - * { path, content: Bloby } [single file] - * { path, content: String } [single file] - * { path, content: Iterable } [single file] - * { path, content: Iterable } [single file] - * { path, content: AsyncIterable } [single file] - * Iterable [single file] - * Iterable [single file] - * Iterable [multiple files] - * Iterable [multiple files] - * Iterable<{ path, content: Bytes }> [multiple files] - * Iterable<{ path, content: Bloby }> [multiple files] - * Iterable<{ path, content: String }> [multiple files] - * Iterable<{ path, content: Iterable }> [multiple files] - * Iterable<{ path, content: Iterable }> [multiple files] - * Iterable<{ path, content: AsyncIterable }> [multiple files] - * AsyncIterable [single file] - * AsyncIterable [multiple files] - * AsyncIterable [multiple files] - * AsyncIterable<{ path, content: Bytes }> [multiple files] - * AsyncIterable<{ path, content: Bloby }> [multiple files] - * AsyncIterable<{ path, content: String }> [multiple files] - * AsyncIterable<{ path, content: Iterable }> [multiple files] - * AsyncIterable<{ path, content: Iterable }> [multiple files] - * AsyncIterable<{ path, content: AsyncIterable }> [multiple files] - * ``` - * Into: - * - * ``` - * AsyncIterable<{ path, content: AsyncIterable }> - * ``` - * - * @param {Object} input - * @return AsyncInterable<{ path, content: AsyncIterable }> - */ -module.exports = function normaliseInput (input) { - // must give us something - if (input === null || input === undefined) { - throw errCode(new Error(`Unexpected input: ${input}`), 'ERR_UNEXPECTED_INPUT') - } - - // String - if (typeof input === 'string' || input instanceof String) { - return (async function * () { // eslint-disable-line require-await - yield toFileObject(input) - })() - } - - // Buffer|ArrayBuffer|TypedArray - // Blob|File - if (isBytes(input) || isBloby(input)) { - return (async function * () { // eslint-disable-line require-await - yield toFileObject(input) - })() - } - - // Iterable - if (input[Symbol.iterator]) { - return (async function * () { // eslint-disable-line require-await - const iterator = input[Symbol.iterator]() - const first = iterator.next() - if (first.done) return iterator - - // Iterable - // Iterable - if (Number.isInteger(first.value) || isBytes(first.value)) { - yield toFileObject((function * () { - yield first.value - yield * iterator - })()) - return - } - - // Iterable - // Iterable - // Iterable<{ path, content }> - if (isFileObject(first.value) || isBloby(first.value) || typeof first.value === 'string') { - yield toFileObject(first.value) - for (const obj of iterator) { - yield toFileObject(obj) - } - return - } - - throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') - })() - } - - // window.ReadableStream - if (typeof input.getReader === 'function') { - return (async function * () { - for await (const obj of browserStreamToIt(input)) { - yield toFileObject(obj) - } - })() - } - - // AsyncIterable - if (input[Symbol.asyncIterator]) { - return (async function * () { - const iterator = input[Symbol.asyncIterator]() - const first = await iterator.next() - if (first.done) return iterator - - // AsyncIterable - if (isBytes(first.value)) { - yield toFileObject((async function * () { // eslint-disable-line require-await - yield first.value - yield * iterator - })()) - return - } - - // AsyncIterable - // AsyncIterable - // AsyncIterable<{ path, content }> - if (isFileObject(first.value) || isBloby(first.value) || typeof first.value === 'string') { - yield toFileObject(first.value) - for await (const obj of iterator) { - yield toFileObject(obj) - } - return - } - - throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') - })() - } - - // { path, content: ? } - // Note: Detected _after_ AsyncIterable because Node.js streams have a - // `path` property that passes this check. - if (isFileObject(input)) { - return (async function * () { // eslint-disable-line require-await - yield toFileObject(input) - })() - } - - throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') -} - -function toFileObject (input) { - const obj = { - path: input.path || '', - mode: input.mode, - mtime: input.mtime - } - - if (input.content) { - obj.content = toAsyncIterable(input.content) - } else if (!input.path) { // Not already a file object with path or content prop - obj.content = toAsyncIterable(input) - } - - return obj -} - -function toAsyncIterable (input) { - // Bytes | String - if (isBytes(input) || typeof input === 'string') { - return (async function * () { // eslint-disable-line require-await - yield toBuffer(input) - })() - } - - // Bloby - if (isBloby(input)) { - return blobToAsyncGenerator(input) - } - - // Browser stream - if (typeof input.getReader === 'function') { - return browserStreamToIt(input) - } - - // Iterator - if (input[Symbol.iterator]) { - return (async function * () { // eslint-disable-line require-await - const iterator = input[Symbol.iterator]() - const first = iterator.next() - if (first.done) return iterator - - // Iterable - if (Number.isInteger(first.value)) { - yield toBuffer(Array.from((function * () { - yield first.value - yield * iterator - })())) - return - } - - // Iterable - if (isBytes(first.value)) { - yield toBuffer(first.value) - for (const chunk of iterator) { - yield toBuffer(chunk) - } - return - } - - throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') - })() - } - - // AsyncIterable - if (input[Symbol.asyncIterator]) { - return (async function * () { - for await (const chunk of input) { - yield toBuffer(chunk) - } - })() - } - - throw errCode(new Error(`Unexpected input: ${input}`), 'ERR_UNEXPECTED_INPUT') -} - -function toBuffer (chunk) { - return isBytes(chunk) ? chunk : Buffer.from(chunk) -} - -function isBytes (obj) { - return Buffer.isBuffer(obj) || ArrayBuffer.isView(obj) || obj instanceof ArrayBuffer -} - -function isBloby (obj) { - return typeof globalThis.Blob !== 'undefined' && obj instanceof globalThis.Blob -} - -// An object with a path or content property -function isFileObject (obj) { - return typeof obj === 'object' && (obj.path || obj.content) -} - -function blobToAsyncGenerator (blob) { - if (typeof blob.stream === 'function') { - // firefox < 69 does not support blob.stream() - return browserStreamToIt(blob.stream()) - } - - return readBlob(blob) -} - -async function * browserStreamToIt (stream) { - const reader = stream.getReader() - - while (true) { - const result = await reader.read() - - if (result.done) { - return - } - - yield result.value - } -} - -async function * readBlob (blob, options) { - options = options || {} - - const reader = new globalThis.FileReader() - const chunkSize = options.chunkSize || 1024 * 1024 - let offset = options.offset || 0 - - const getNextChunk = () => new Promise((resolve, reject) => { - reader.onloadend = e => { - const data = e.target.result - resolve(data.byteLength === 0 ? null : data) - } - reader.onerror = reject - - const end = offset + chunkSize - const slice = blob.slice(offset, end) - reader.readAsArrayBuffer(slice) - offset = end - }) - - while (true) { - const data = await getNextChunk() - - if (data == null) { - return - } - - yield Buffer.from(data) - } -} diff --git a/test/files/format-mode.spec.js b/test/files/format-mode.spec.js deleted file mode 100644 index 21a03e1..0000000 --- a/test/files/format-mode.spec.js +++ /dev/null @@ -1,54 +0,0 @@ -'use strict' - -/* eslint-env mocha */ -const { expect } = require('aegir/utils/chai') -const formatMode = require('../../src/files/format-mode') - -describe('format-mode', function () { - it('formats mode for directories', function () { - expect(formatMode(parseInt('0777', 8), true)).to.equal('drwxrwxrwx') - }) - - it('formats mode for files', function () { - expect(formatMode(parseInt('0777', 8), false)).to.equal('-rwxrwxrwx') - }) - - it('setgid, setuid and stick bit', function () { - expect(formatMode(parseInt('1777', 8), false)).to.equal('-rwxrwxrwt') - expect(formatMode(parseInt('2777', 8), false)).to.equal('-rwxrwsrwx') - expect(formatMode(parseInt('4777', 8), false)).to.equal('-rwsrwxrwx') - expect(formatMode(parseInt('5777', 8), false)).to.equal('-rwsrwxrwt') - expect(formatMode(parseInt('6777', 8), false)).to.equal('-rwsrwsrwx') - expect(formatMode(parseInt('7777', 8), false)).to.equal('-rwsrwsrwt') - }) - - it('formats user', function () { - expect(formatMode(parseInt('0100', 8), false)).to.equal('---x------') - expect(formatMode(parseInt('0200', 8), false)).to.equal('--w-------') - expect(formatMode(parseInt('0300', 8), false)).to.equal('--wx------') - expect(formatMode(parseInt('0400', 8), false)).to.equal('-r--------') - expect(formatMode(parseInt('0500', 8), false)).to.equal('-r-x------') - expect(formatMode(parseInt('0600', 8), false)).to.equal('-rw-------') - expect(formatMode(parseInt('0700', 8), false)).to.equal('-rwx------') - }) - - it('formats group', function () { - expect(formatMode(parseInt('0010', 8), false)).to.equal('------x---') - expect(formatMode(parseInt('0020', 8), false)).to.equal('-----w----') - expect(formatMode(parseInt('0030', 8), false)).to.equal('-----wx---') - expect(formatMode(parseInt('0040', 8), false)).to.equal('----r-----') - expect(formatMode(parseInt('0050', 8), false)).to.equal('----r-x---') - expect(formatMode(parseInt('0060', 8), false)).to.equal('----rw----') - expect(formatMode(parseInt('0070', 8), false)).to.equal('----rwx---') - }) - - it('formats other', function () { - expect(formatMode(parseInt('0001', 8), false)).to.equal('---------x') - expect(formatMode(parseInt('0002', 8), false)).to.equal('--------w-') - expect(formatMode(parseInt('0003', 8), false)).to.equal('--------wx') - expect(formatMode(parseInt('0004', 8), false)).to.equal('-------r--') - expect(formatMode(parseInt('0005', 8), false)).to.equal('-------r-x') - expect(formatMode(parseInt('0006', 8), false)).to.equal('-------rw-') - expect(formatMode(parseInt('0007', 8), false)).to.equal('-------rwx') - }) -}) diff --git a/test/files/format-mtime.spec.js b/test/files/format-mtime.spec.js deleted file mode 100644 index fdb4b63..0000000 --- a/test/files/format-mtime.spec.js +++ /dev/null @@ -1,15 +0,0 @@ -'use strict' - -/* eslint-env mocha */ -const { expect } = require('aegir/utils/chai') -const formatMtime = require('../../src/files/format-mtime') - -describe('format-mtime', function () { - it('formats mtime', function () { - expect(formatMtime({ secs: 15768000, nsecs: 0 })).to.include('1970') - }) - - it('formats empty mtime', function () { - expect(formatMtime()).to.equal('-') - }) -}) diff --git a/test/files/normalise-input.spec.js b/test/files/normalise-input.spec.js deleted file mode 100644 index 6b4d694..0000000 --- a/test/files/normalise-input.spec.js +++ /dev/null @@ -1,146 +0,0 @@ -'use strict' - -/* eslint-env mocha */ -const { expect, assert } = require('aegir/utils/chai') -const normalise = require('../../src/files/normalise-input') -const { supportsFileReader } = require('../../src/supports') -const { Buffer } = require('buffer') -const all = require('it-all') -const globalThis = require('../../src/globalthis') - -const STRING = () => 'hello world' -const BUFFER = () => Buffer.from(STRING()) -const ARRAY = () => Array.from(BUFFER()) -const TYPEDARRAY = () => Uint8Array.from(ARRAY()) -let BLOB -let WINDOW_READABLE_STREAM - -if (supportsFileReader) { - BLOB = () => new globalThis.Blob([ - STRING() - ]) - - WINDOW_READABLE_STREAM = () => new globalThis.ReadableStream({ - start (controller) { - controller.enqueue(BUFFER()) - controller.close() - } - }) -} - -async function verifyNormalisation (input) { - expect(input.length).to.equal(1) - - if (!input[0].content[Symbol.asyncIterator] && !input[0].content[Symbol.iterator]) { - assert.fail('Content should have been an iterable or an async iterable') - } - - expect(await all(input[0].content)).to.deep.equal([BUFFER()]) - expect(input[0].path).to.equal('') -} - -async function testContent (input) { - const result = await all(normalise(input)) - - await verifyNormalisation(result) -} - -function iterableOf (thing) { - return [thing] -} - -function asyncIterableOf (thing) { - return (async function * () { // eslint-disable-line require-await - yield thing - }()) -} - -describe('normalise-input', function () { - function testInputType (content, name, isBytes) { - it(name, async function () { - await testContent(content()) - }) - - if (isBytes) { - it(`Iterable<${name}>`, async function () { - await testContent(iterableOf(content())) - }) - - it(`AsyncIterable<${name}>`, async function () { - await testContent(asyncIterableOf(content())) - }) - } - - it(`{ path: '', content: ${name} }`, async function () { - await testContent({ path: '', content: content() }) - }) - - if (isBytes) { - it(`{ path: '', content: Iterable<${name}> }`, async function () { - await testContent({ path: '', content: iterableOf(content()) }) - }) - - it(`{ path: '', content: AsyncIterable<${name}> }`, async function () { - await testContent({ path: '', content: asyncIterableOf(content()) }) - }) - } - - it(`Iterable<{ path: '', content: ${name} }`, async function () { - await testContent(iterableOf({ path: '', content: content() })) - }) - - it(`AsyncIterable<{ path: '', content: ${name} }`, async function () { - await testContent(asyncIterableOf({ path: '', content: content() })) - }) - - if (isBytes) { - it(`Iterable<{ path: '', content: Iterable<${name}> }>`, async function () { - await testContent(iterableOf({ path: '', content: iterableOf(content()) })) - }) - - it(`Iterable<{ path: '', content: AsyncIterable<${name}> }>`, async function () { - await testContent(iterableOf({ path: '', content: asyncIterableOf(content()) })) - }) - - it(`AsyncIterable<{ path: '', content: Iterable<${name}> }>`, async function () { - await testContent(asyncIterableOf({ path: '', content: iterableOf(content()) })) - }) - - it(`AsyncIterable<{ path: '', content: AsyncIterable<${name}> }>`, async function () { - await testContent(asyncIterableOf({ path: '', content: asyncIterableOf(content()) })) - }) - } - } - - describe('String', () => { - testInputType(STRING, 'String', false) - }) - - describe('Buffer', () => { - testInputType(BUFFER, 'Buffer', true) - }) - - describe('Blob', () => { - if (!supportsFileReader) { - return - } - - testInputType(BLOB, 'Blob', false) - }) - - describe('window.ReadableStream', () => { - if (!supportsFileReader) { - return - } - - testInputType(WINDOW_READABLE_STREAM, 'window.ReadableStream', false) - }) - - describe('Iterable', () => { - testInputType(ARRAY, 'Iterable', false) - }) - - describe('TypedArray', () => { - testInputType(TYPEDARRAY, 'TypedArray', true) - }) -}) From 89c2cdea212e9607b002cd6ad514c12a03d71144 Mon Sep 17 00:00:00 2001 From: Hugo Dias Date: Sun, 10 Jan 2021 18:00:50 +0000 Subject: [PATCH 02/10] fix: add github actions --- .github/workflows/main.yml | 67 ++++++++++++++++++++++++++++++++++++++ .travis.yml | 52 ----------------------------- 2 files changed, 67 insertions(+), 52 deletions(-) create mode 100644 .github/workflows/main.yml delete mode 100644 .travis.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..9e81b21 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,67 @@ +name: ci +on: + push: + branches: + - master + pull_request: + branches: + - master + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - run: npm install + - run: npx aegir lint + - uses: gozala/typescript-error-reporter-action@v1.0.8 + - run: npx aegir build + - run: npx aegir dep-check + - uses: ipfs/aegir/actions/bundle-size@master + name: size + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + test-node: + needs: check + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [windows-latest, ubuntu-latest, macos-latest] + node: [12, 14] + fail-fast: true + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node }} + - run: npm install + - run: npx nyc --reporter=lcov aegir test -t node -- --bail + - uses: codecov/codecov-action@v1 + test-chrome: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - run: npm install + - run: npx aegir test -t browser -t webworker --bail + test-firefox: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - run: npm install + - run: npx aegir test -t browser -t webworker --bail -- --browsers FirefoxHeadless + test-electron-main: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - run: npm install + - run: npx xvfb-maybe aegir test -t electron-main --bail + test-electron-renderer: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - run: npm install + - run: npx xvfb-maybe aegir test -t electron-renderer --bail \ No newline at end of file diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index bd3dcaf..0000000 --- a/.travis.yml +++ /dev/null @@ -1,52 +0,0 @@ -language: node_js -cache: npm -stages: - - check - - test - - cov - -node_js: - - '10' - - '12' - -os: - - linux - - osx - -script: npx nyc -s npm run test:node -- --bail -after_success: npx nyc report --reporter=text-lcov > coverage.lcov && npx codecov - -jobs: - include: - - os: windows - filter_secrets: false - cache: false - - - stage: check - script: - - npx aegir commitlint --travis - - npx aegir dep-check - - npm run lint - - - stage: test - name: chrome - addons: - chrome: stable - script: npx aegir test -t browser -t webworker - - - stage: test - name: firefox - addons: - firefox: latest - script: npx aegir test -t browser -t webworker -- --browsers FirefoxHeadless - - - stage: test - name: electron - services: - - xvfb - script: - - npm run test:electron - - npm run test:electron-renderer - -notifications: - email: false From c8eec700377673bc89d52cd51432158bec230277 Mon Sep 17 00:00:00 2001 From: Hugo Dias Date: Sun, 10 Jan 2021 18:01:28 +0000 Subject: [PATCH 03/10] feat: types for files methods --- src/files/glob-source.js | 13 ++++++++++--- src/files/url-source.js | 17 +++++++++++++---- test/files/glob-source.spec.js | 9 +++++++++ test/files/url-source.spec.js | 1 + 4 files changed, 33 insertions(+), 7 deletions(-) diff --git a/src/files/glob-source.js b/src/files/glob-source.js index 29a6eb5..6a62e41 100644 --- a/src/files/glob-source.js +++ b/src/files/glob-source.js @@ -3,12 +3,13 @@ const fs = require('fs-extra') const glob = require('it-glob') const Path = require('path') +// @ts-ignore const errCode = require('err-code') /** * Create an async iterator that yields paths that match requested file paths. * - * @param {Iterable | AsyncIterable | string} paths - File system path(s) to glob from + * @param {Iterable | AsyncIterable | string} paths - File system path(s) to glob from * @param {Object} [options] - Optional options * @param {boolean} [options.recursive] - Recursively glob all paths in directories * @param {boolean} [options.hidden] - Include .dot files in matched paths @@ -16,8 +17,8 @@ const errCode = require('err-code') * @param {boolean} [options.followSymlinks] - follow symlinks * @param {boolean} [options.preserveMode] - preserve mode * @param {boolean} [options.preserveMtime] - preserve mtime - * @param {boolean} [options.mode] - mode to use - if preserveMode is true this will be ignored - * @param {boolean} [options.mtime] - mtime to use - if preserveMtime is true this will be ignored + * @param {boolean | number} [options.mode] - mode to use - if preserveMode is true this will be ignored + * @param {boolean | Date} [options.mtime] - mtime to use - if preserveMtime is true this will be ignored * @yields {Object} File objects in the form `{ path: String, content: AsyncIterator }` */ module.exports = async function * globSource (paths, options) { @@ -53,12 +54,14 @@ module.exports = async function * globSource (paths, options) { let mode = options.mode if (options.preserveMode) { + // @ts-ignore mode = stat.mode } let mtime = options.mtime if (options.preserveMtime) { + // @ts-ignore mtime = stat.mtime } @@ -82,6 +85,7 @@ module.exports = async function * globSource (paths, options) { } } +// @ts-ignore async function * toGlobSource ({ path, type, prefix, mode, mtime, preserveMode, preserveMtime }, options) { options = options || {} @@ -135,4 +139,7 @@ async function * toGlobSource ({ path, type, prefix, mode, mtime, preserveMode, } } +/** + * @param {string} path + */ const toPosix = path => path.replace(/\\/g, '/') diff --git a/src/files/url-source.js b/src/files/url-source.js index fd2e0dc..a779aeb 100644 --- a/src/files/url-source.js +++ b/src/files/url-source.js @@ -1,9 +1,16 @@ 'use strict' -const Http = require('../http') - -module.exports = async function * urlSource (url, options) { - const http = new Http() +const HTTP = require('../http') +/** + * @param {string} url + * @param {import("../types").HTTPOptions} [options] + * @returns {AsyncIterable<{ + path: string; + content?: AsyncIterable; +}>} + */ +async function * urlSource (url, options) { + const http = new HTTP() const response = await http.get(url, options) yield { @@ -11,3 +18,5 @@ module.exports = async function * urlSource (url, options) { content: response.iterator() } } + +module.exports = urlSource diff --git a/test/files/glob-source.spec.js b/test/files/glob-source.spec.js index 460fb2e..4e56e56 100644 --- a/test/files/glob-source.spec.js +++ b/test/files/glob-source.spec.js @@ -10,14 +10,23 @@ const { } = require('../../src/env') const fs = require('fs-extra') +/** + * @param {string} file + */ function fixture (file) { return path.resolve(path.join(__dirname, '..', 'fixtures', file)) } +/** + * @param {string} file + */ function findMode (file) { return fs.statSync(fixture(file)).mode } +/** + * @param {string} file + */ function findMtime (file) { return fs.statSync(fixture(file)).mtime } diff --git a/test/files/url-source.spec.js b/test/files/url-source.spec.js index 7d12fae..2a02e9f 100644 --- a/test/files/url-source.spec.js +++ b/test/files/url-source.spec.js @@ -13,6 +13,7 @@ describe('url-source', function () { const content = 'foo' const file = await last(urlSource(`${process.env.ECHO_SERVER}/download?data=${content}`)) + // @ts-ignore await expect(all(file.content)).to.eventually.deep.equal([Buffer.from(content)]) }) }) From 362a735c496b04a6997a4c499b3d6dd4891c53c3 Mon Sep 17 00:00:00 2001 From: Hugo Dias Date: Sun, 10 Jan 2021 18:03:26 +0000 Subject: [PATCH 04/10] feat: use web-encoding for encode/decode --- package.json | 7 +------ src/text-decoder.browser.js | 3 --- src/text-decoder.js | 3 ++- src/text-encoder.browser.js | 3 --- src/text-encoder.js | 5 ++++- 5 files changed, 7 insertions(+), 14 deletions(-) delete mode 100644 src/text-decoder.browser.js delete mode 100644 src/text-encoder.browser.js diff --git a/package.json b/package.json index 81bdfd9..f49e51d 100644 --- a/package.json +++ b/package.json @@ -13,8 +13,6 @@ ], "browser": { "./src/http/fetch.js": "./src/http/fetch.browser.js", - "./src/text-encoder.js": "./src/text-encoder.browser.js", - "./src/text-decoder.js": "./src/text-decoder.browser.js", "./src/temp-dir.js": "./src/temp-dir.browser.js", "./src/path-join.js": "./src/path-join.browser.js", "./test/files/glob-source.spec.js": false, @@ -47,10 +45,7 @@ "merge-options": "^2.0.0", "nanoid": "^3.1.3", "native-abort-controller": "0.0.3", - "native-fetch": "^2.0.0", - "node-fetch": "^2.6.0", - "stream-to-it": "^0.2.0", - "it-to-stream": "^0.1.2" + "web-encoding": "^1.0.6" }, "devDependencies": { "aegir": "^28.1.0", diff --git a/src/text-decoder.browser.js b/src/text-decoder.browser.js deleted file mode 100644 index de02262..0000000 --- a/src/text-decoder.browser.js +++ /dev/null @@ -1,3 +0,0 @@ -'use strict' - -module.exports = require('./globalthis').TextDecoder diff --git a/src/text-decoder.js b/src/text-decoder.js index 44fc233..9d3eb12 100644 --- a/src/text-decoder.js +++ b/src/text-decoder.js @@ -1,2 +1,3 @@ 'use strict' -module.exports = require('util').TextDecoder + +module.exports = require('web-encoding').TextDecoder diff --git a/src/text-encoder.browser.js b/src/text-encoder.browser.js deleted file mode 100644 index 8f27641..0000000 --- a/src/text-encoder.browser.js +++ /dev/null @@ -1,3 +0,0 @@ -'use strict' - -module.exports = require('./globalthis').TextEncoder diff --git a/src/text-encoder.js b/src/text-encoder.js index 25a82b1..6960639 100644 --- a/src/text-encoder.js +++ b/src/text-encoder.js @@ -1,2 +1,5 @@ 'use strict' -module.exports = require('util').TextEncoder + +const { TextEncoder } = require('web-encoding') + +module.exports = TextEncoder From 38e6daff835e8b7a0cf442a04945732fc0edb8ea Mon Sep 17 00:00:00 2001 From: Hugo Dias Date: Sun, 10 Jan 2021 18:04:05 +0000 Subject: [PATCH 05/10] fix: types for env, path, supports and temp-dir --- src/env.js | 2 +- src/index.js | 2 ++ src/path-join.browser.js | 3 +++ src/path-join.js | 4 ++-- src/supports.js | 2 -- src/temp-dir.browser.js | 2 +- src/temp-dir.js | 2 +- 7 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/env.js b/src/env.js index fbd8383..eb1e826 100644 --- a/src/env.js +++ b/src/env.js @@ -7,7 +7,7 @@ const IS_BROWSER = IS_ENV_WITH_DOM && !IS_ELECTRON const IS_ELECTRON_MAIN = IS_ELECTRON && !IS_ENV_WITH_DOM const IS_ELECTRON_RENDERER = IS_ELECTRON && IS_ENV_WITH_DOM const IS_NODE = typeof require === 'function' && typeof process !== 'undefined' && typeof process.release !== 'undefined' && process.release.name === 'node' && !IS_ELECTRON -// eslint-disable-next-line no-undef +// @ts-ignore - we either ignore worker scope or dom scope const IS_WEBWORKER = typeof importScripts === 'function' && typeof self !== 'undefined' && typeof WorkerGlobalScope !== 'undefined' && self instanceof WorkerGlobalScope const IS_TEST = typeof process !== 'undefined' && typeof process.env !== 'undefined' && process.env.NODE_ENV === 'test' diff --git a/src/index.js b/src/index.js index e69de29..f177540 100644 --- a/src/index.js +++ b/src/index.js @@ -0,0 +1,2 @@ +'use strict' +module.exports = {} diff --git a/src/path-join.browser.js b/src/path-join.browser.js index b922a7b..e425a56 100644 --- a/src/path-join.browser.js +++ b/src/path-join.browser.js @@ -1,5 +1,8 @@ 'use strict' +/** + * @param {any[]} args + */ function join (...args) { if (args.length === 0) { return '.' diff --git a/src/path-join.js b/src/path-join.js index a67792b..aa8409e 100644 --- a/src/path-join.js +++ b/src/path-join.js @@ -1,3 +1,3 @@ 'use strict' - -module.exports = require('path').join +const { join } = require('path') +module.exports = join diff --git a/src/supports.js b/src/supports.js index 3affb98..f66f9bd 100644 --- a/src/supports.js +++ b/src/supports.js @@ -1,7 +1,5 @@ 'use strict' -const globalThis = require('./globalthis') - module.exports = { supportsFileReader: typeof self !== 'undefined' && 'FileReader' in self, supportsWebRTC: 'RTCPeerConnection' in globalThis && diff --git a/src/temp-dir.browser.js b/src/temp-dir.browser.js index 6b4ba68..b10a5ae 100644 --- a/src/temp-dir.browser.js +++ b/src/temp-dir.browser.js @@ -5,7 +5,7 @@ const { nanoid } = require('nanoid') /** * Temporary folder * - * @param {function(string): string} transform - Transform function to add prefixes or sufixes to the unique id + * @param {(uuid: string) => string} transform - Transform function to add prefixes or sufixes to the unique id * @returns {string} - Full real path to a temporary folder */ const tempdir = (transform = d => d) => { diff --git a/src/temp-dir.js b/src/temp-dir.js index bc94b19..86f35af 100644 --- a/src/temp-dir.js +++ b/src/temp-dir.js @@ -8,7 +8,7 @@ const { nanoid } = require('nanoid') /** * Temporary folder * - * @param {function(string): string} transform - Transform function to add prefixes or sufixes to the unique id + * @param {(uuid: string) => string} [transform=(p) => p] - Transform function to add prefixes or sufixes to the unique id * @returns {string} - Full real path to a temporary folder */ const tempdir = (transform = d => d) => { From c41612e346cd10976e00b1365d60c34f8e4d94c8 Mon Sep 17 00:00:00 2001 From: Hugo Dias Date: Sun, 10 Jan 2021 18:06:29 +0000 Subject: [PATCH 06/10] feat: remove globalthis not needed anymore --- src/globalthis.js | 20 -------------------- 1 file changed, 20 deletions(-) delete mode 100644 src/globalthis.js diff --git a/src/globalthis.js b/src/globalthis.js deleted file mode 100644 index 6e56a77..0000000 --- a/src/globalthis.js +++ /dev/null @@ -1,20 +0,0 @@ -/* eslint-disable no-undef */ -/* eslint-disable no-extend-native */ -/* eslint-disable strict */ - -// polyfill for globalThis -// https://v8.dev/features/globalthis -// https://mathiasbynens.be/notes/globalthis -(function () { - if (typeof globalThis === 'object') return - Object.defineProperty(Object.prototype, '__magic__', { - get: function () { - return this - }, - configurable: true - }) - __magic__.globalThis = __magic__ - delete Object.prototype.__magic__ -}()) - -module.exports = globalThis From b0bc5e0a9b85937815cf1ca4fac0deac631bc826 Mon Sep 17 00:00:00 2001 From: Hugo Dias Date: Sun, 10 Jan 2021 18:08:12 +0000 Subject: [PATCH 07/10] feat: http types --- package.json | 40 +++- src/fetch.js | 2 +- src/http.js | 267 ++++++++++++----------- src/http/error.js | 3 + src/http/fetch.browser.js | 51 +++-- src/http/fetch.node.js | 87 +++----- src/types.ts | 53 +++++ test/http.spec.js | 10 +- tsconfig.json | 11 + types/electron-fetch/index.d.ts | 117 ++++++++++ types/is-electron/index.d.ts | 2 + types/iso-url/index.d.ts | 1 + types/native-abort-controller/index.d.ts | 4 + types/native-fetch/index.d.ts | 117 ++++++++++ 14 files changed, 539 insertions(+), 226 deletions(-) create mode 100644 src/types.ts create mode 100644 tsconfig.json create mode 100644 types/electron-fetch/index.d.ts create mode 100644 types/is-electron/index.d.ts create mode 100644 types/iso-url/index.d.ts create mode 100644 types/native-abort-controller/index.d.ts create mode 100644 types/native-fetch/index.d.ts diff --git a/package.json b/package.json index f49e51d..b46b3a0 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,15 @@ "./test/files/glob-source.spec.js": false, "electron-fetch": false }, + "types": "dist/src/index.d.ts", + "typesVersions": { + "*": { + "src/*": [ + "dist/src/*", + "dist/src/*/index" + ] + } + }, "repository": "github:ipfs/js-ipfs-utils", "scripts": { "test": "aegir test", @@ -33,27 +42,38 @@ }, "license": "MIT", "dependencies": { - "electron-fetch": "^1.7.2", "abort-controller": "^3.0.0", "any-signal": "^2.1.0", "buffer": "^6.0.1", - "err-code": "^2.0.0", + "electron-fetch": "^1.7.2", + "err-code": "^2.0.3", "fs-extra": "^9.0.1", "is-electron": "^2.2.0", "iso-url": "^1.0.0", "it-glob": "0.0.10", - "merge-options": "^2.0.0", - "nanoid": "^3.1.3", + "it-to-stream": "^0.1.2", + "merge-options": "^3.0.4", + "nanoid": "^3.1.20", "native-abort-controller": "0.0.3", + "native-fetch": "2.0.1", + "node-fetch": "^2.6.1", + "stream-to-it": "^0.2.2", "web-encoding": "^1.0.6" }, "devDependencies": { - "aegir": "^28.1.0", - "delay": "^4.3.0", - "it-all": "^1.0.2", - "it-drain": "^1.0.1", - "it-last": "^1.0.2", - "uint8arrays": "^1.1.0" + "@types/fs-extra": "^9.0.5", + "aegir": "^30.3.0", + "delay": "^4.4.0", + "it-all": "^1.0.4", + "it-drain": "^1.0.3", + "it-last": "^1.0.4", + "uint8arrays": "^2.0.5" + }, + "eslintConfig": { + "extends": "ipfs", + "env": { + "worker": true + } }, "contributors": [ "Hugo Dias ", diff --git a/src/fetch.js b/src/fetch.js index a5b48cc..d3ee792 100644 --- a/src/fetch.js +++ b/src/fetch.js @@ -5,6 +5,6 @@ const { isElectronMain } = require('./env') if (isElectronMain) { module.exports = require('electron-fetch') } else { - // use window.fetch if it is available, fall back to node-fetch if not +// use window.fetch if it is available, fall back to node-fetch if not module.exports = require('native-fetch') } diff --git a/src/http.js b/src/http.js index 7240c1c..fd05696 100644 --- a/src/http.js +++ b/src/http.js @@ -9,6 +9,19 @@ const TextDecoder = require('./text-decoder') const AbortController = require('native-abort-controller') const anySignal = require('any-signal') +/** + * @typedef {import('electron-fetch').Response} Response + * @typedef {import('stream').Readable} NodeReadableStream + * @typedef {import('./types').HTTPOptions} HTTPOptions + */ + +/** + * @template TResponse + * @param {Promise} promise + * @param {number | undefined} ms + * @param {AbortController} abortController + * @returns {Promise} + */ const timeout = (promise, ms, abortController) => { if (ms === undefined) { return promise @@ -30,8 +43,14 @@ const timeout = (promise, ms, abortController) => { } }, ms) + /** + * @param {(value: any) => void } next + */ const after = (next) => { - return (res) => { + /** + * @param {any} res + */ + const fn = (res) => { clearTimeout(timeoutID) if (timedOut()) { @@ -41,6 +60,7 @@ const timeout = (promise, ms, abortController) => { next(res) } + return fn } promise @@ -49,91 +69,77 @@ const timeout = (promise, ms, abortController) => { } const defaults = { - headers: {}, throwHttpErrors: true, - credentials: 'same-origin', - transformSearchParams: p => p + credentials: 'same-origin' } -/** - * @typedef {Object} APIOptions - creates a new type named 'SpecialType' - * @property {any} [body] - Request body - * @property {Object} [json] - JSON shortcut - * @property {string} [method] - GET, POST, PUT, DELETE, etc. - * @property {string} [base] - The base URL to use in case url is a relative URL - * @property {Headers|Record} [headers] - Request header. - * @property {number} [timeout] - Amount of time until request should timeout in ms. - * @property {AbortSignal} [signal] - Signal to abort the request. - * @property {URLSearchParams|Object} [searchParams] - URL search param. - * @property {string} [credentials] - * @property {boolean} [throwHttpErrors] - * @property {function(URLSearchParams): URLSearchParams } [transformSearchParams] - * @property {function(any): any} [transform] - When iterating the response body, transform each chunk with this function. - * @property {function(Response): Promise} [handleError] - Handle errors - * @property {function({total:number, loaded:number, lengthComputable:boolean}):void} [onUploadProgress] - Can be passed to track upload progress. - * Note that if this option in passed underlying request will be performed using `XMLHttpRequest` and response will not be streamed. - */ - class HTTP { /** * - * @param {APIOptions} options + * @param {HTTPOptions} options */ constructor (options = {}) { - /** @type {APIOptions} */ + /** @type {HTTPOptions} */ this.opts = merge(defaults, options) } /** * Fetch * - * @param {string | URL | Request} resource - * @param {APIOptions} options + * @param {string | Request} resource + * @param {HTTPOptions} options * @returns {Promise} */ async fetch (resource, options = {}) { - /** @type {APIOptions} */ + /** @type {HTTPOptions} */ const opts = merge(this.opts, options) - opts.headers = new Headers(opts.headers) + const headers = new Headers(opts.headers) // validate resource type if (typeof resource !== 'string' && !(resource instanceof URL || resource instanceof Request)) { throw new TypeError('`resource` must be a string, URL, or Request') } - // validate resource format and normalize with prefixUrl - if (opts.base && typeof opts.base === 'string' && typeof resource === 'string') { - if (resource.startsWith('/')) { - throw new Error('`resource` must not begin with a slash when using `base`') + const url = new URL(resource.toString(), opts.base) + + const { + searchParams, + transformSearchParams, + json + } = opts + + if (searchParams) { + if (typeof transformSearchParams === 'function') { + // @ts-ignore + url.search = transformSearchParams(new URLSearchParams(opts.searchParams)) + } else { + // @ts-ignore + url.search = new URLSearchParams(opts.searchParams) } - - if (!opts.base.endsWith('/')) { - opts.base += '/' - } - - resource = opts.base + resource - } - - // TODO: try to remove the logic above or fix URL instance input without trailing '/' - const url = new URL(resource, opts.base) - - if (opts.searchParams) { - url.search = opts.transformSearchParams(new URLSearchParams(opts.searchParams)) } - if (opts.json !== undefined) { + if (json) { opts.body = JSON.stringify(opts.json) - opts.headers.set('content-type', 'application/json') + headers.set('content-type', 'application/json') } const abortController = new AbortController() + // @ts-ignore const signal = anySignal([abortController.signal, opts.signal]) - const response = await timeout(fetch(url.toString(), { - ...opts, - signal, - timeout: undefined - }), opts.timeout, abortController) + const response = await timeout( + fetch( + url.toString(), + { + ...opts, + signal, + timeout: undefined, + headers + } + ), + opts.timeout, + abortController + ) if (!response.ok && opts.throwHttpErrors) { if (opts.handleError) { @@ -143,13 +149,7 @@ class HTTP { } response.iterator = function () { - const it = streamToAsyncIterator(response.body) - - if (!isAsyncIterator(it)) { - throw new Error('Can\'t convert fetch body into a Async Iterator:') - } - - return it + return fromStream(response.body) } response.ndjson = async function * () { @@ -166,71 +166,56 @@ class HTTP { } /** - * @param {string | URL | Request} resource - * @param {APIOptions} options + * @param {string | Request} resource + * @param {HTTPOptions} options * @returns {Promise} */ post (resource, options = {}) { - return this.fetch(resource, { - ...options, - method: 'POST' - }) + return this.fetch(resource, { ...options, method: 'POST' }) } /** - * @param {string | URL | Request} resource - * @param {APIOptions} options + * @param {string | Request} resource + * @param {HTTPOptions} options * @returns {Promise} */ get (resource, options = {}) { - return this.fetch(resource, { - ...options, - method: 'GET' - }) + return this.fetch(resource, { ...options, method: 'GET' }) } /** - * @param {string | URL | Request} resource - * @param {APIOptions} options + * @param {string | Request} resource + * @param {HTTPOptions} options * @returns {Promise} */ put (resource, options = {}) { - return this.fetch(resource, { - ...options, - method: 'PUT' - }) + return this.fetch(resource, { ...options, method: 'PUT' }) } /** - * @param {string | URL | Request} resource - * @param {APIOptions} options + * @param {string | Request} resource + * @param {HTTPOptions} options * @returns {Promise} */ delete (resource, options = {}) { - return this.fetch(resource, { - ...options, - method: 'DELETE' - }) + return this.fetch(resource, { ...options, method: 'DELETE' }) } /** - * @param {string | URL | Request} resource - * @param {APIOptions} options + * @param {string | Request} resource + * @param {HTTPOptions} options * @returns {Promise} */ options (resource, options = {}) { - return this.fetch(resource, { - ...options, - method: 'OPTIONS' - }) + return this.fetch(resource, { ...options, method: 'OPTIONS' }) } } /** * Parses NDJSON chunks from an iterator * - * @param {AsyncGenerator} source - * @returns {AsyncGenerator} + * @param {AsyncIterable} source + * @returns {AsyncIterable} */ const ndjson = async function * (source) { const decoder = new TextDecoder() @@ -255,47 +240,67 @@ const ndjson = async function * (source) { } } -const streamToAsyncIterator = function (source) { - if (isAsyncIterator(source)) { +/** + * Stream to AsyncIterable + * + * @template TChunk + * @param {ReadableStream | NodeReadableStream | null} source + * @returns {AsyncIterable} + */ +const fromStream = (source) => { + if (isAsyncIterable(source)) { // Workaround for https://github.com/node-fetch/node-fetch/issues/766 if (Object.prototype.hasOwnProperty.call(source, 'readable') && Object.prototype.hasOwnProperty.call(source, 'writable')) { - const iter = source[Symbol.asyncIterator]() - - const wrapper = { + const nodeSource = /** @type {NodeReadableStream} */(source) + const iter = nodeSource[Symbol.asyncIterator]() + return { + // @ts-ignore next: iter.next.bind(iter), - return: () => { - source.destroy() - - return iter.return() + return () { + nodeSource.destroy() + if (iter.return) { + return iter.return() + } + return {} }, - [Symbol.asyncIterator]: () => { - return wrapper + [Symbol.asyncIterator] () { + return this } } - - return wrapper } - return source + return /** @type {AsyncIterable} */(source) } - const reader = source.getReader() - - return { - next () { - return reader.read() - }, - return () { - reader.releaseLock() - return {} - }, - [Symbol.asyncIterator] () { - return this - } + if (source && typeof /** @type {ReadableStream} */(source).getReader === 'function') { + const reader = /** @type {ReadableStream} */(source).getReader() + return (async function * () { + try { + while (true) { + // Read from the stream + const { done, value } = await reader.read() + // Exit if we're done + if (done) return + // Else yield the chunk + if (value) { + yield value + } + } + } finally { + reader.releaseLock() + } + })() } + + throw new TypeError('Body can\'t be converted to AsyncIterable') } -const isAsyncIterator = (obj) => { +/** + * Check if it's an AsyncIterable + * + * @param {any} obj + */ +const isAsyncIterable = (obj) => { return typeof obj === 'object' && obj !== null && // typeof obj.next === 'function' && @@ -304,39 +309,39 @@ const isAsyncIterator = (obj) => { HTTP.HTTPError = HTTPError HTTP.TimeoutError = TimeoutError -HTTP.streamToAsyncIterator = streamToAsyncIterator +HTTP.streamToAsyncIterator = fromStream /** - * @param {string | URL | Request} resource - * @param {APIOptions} options + * @param {string | Request} resource + * @param {HTTPOptions} [options] * @returns {Promise} */ HTTP.post = (resource, options) => new HTTP(options).post(resource, options) /** - * @param {string | URL | Request} resource - * @param {APIOptions} options + * @param {string | Request} resource + * @param {HTTPOptions} [options] * @returns {Promise} */ HTTP.get = (resource, options) => new HTTP(options).get(resource, options) /** - * @param {string | URL | Request} resource - * @param {APIOptions} options + * @param {string | Request} resource + * @param {HTTPOptions} [options] * @returns {Promise} */ HTTP.put = (resource, options) => new HTTP(options).put(resource, options) /** - * @param {string | URL | Request} resource - * @param {APIOptions} options + * @param {string | Request} resource + * @param {HTTPOptions} [options] * @returns {Promise} */ HTTP.delete = (resource, options) => new HTTP(options).delete(resource, options) /** - * @param {string | URL | Request} resource - * @param {APIOptions} options + * @param {string | Request} resource + * @param {HTTPOptions} [options] * @returns {Promise} */ HTTP.options = (resource, options) => new HTTP(options).options(resource, options) diff --git a/src/http/error.js b/src/http/error.js index 057d2c5..0c61dc9 100644 --- a/src/http/error.js +++ b/src/http/error.js @@ -17,6 +17,9 @@ class AbortError extends Error { exports.AbortError = AbortError class HTTPError extends Error { + /** + * @param {import('electron-fetch').Response} response + */ constructor (response) { super(response.statusText) this.name = 'HTTPError' diff --git a/src/http/fetch.browser.js b/src/http/fetch.browser.js index 275b912..78ff7e4 100644 --- a/src/http/fetch.browser.js +++ b/src/http/fetch.browser.js @@ -1,39 +1,37 @@ 'use strict' -/* eslint-env browser */ const { TimeoutError, AbortError } = require('./error') -const { Request, Response, Headers } = require('../fetch') +const { Response, Request, Headers, default: fetch } = require('../fetch') /** - * @typedef {RequestInit & ExtraFetchOptions} FetchOptions - * @typedef {Object} ExtraFetchOptions - * @property {number} [timeout] - * @property {URLSearchParams} [searchParams] - * @property {function({total:number, loaded:number, lengthComputable:boolean}):void} [onUploadProgress] - * @property {string} [overrideMimeType] - * @returns {Promise} + * @typedef {import('../types').FetchOptions} FetchOptions + * @typedef {import('../types').ProgressFn} ProgressFn */ /** - * @param {string|URL} url + * Fetch with progress + * + * @param {string | Request} url * @param {FetchOptions} [options] - * @returns {Promise} + * @returns {Promise} */ const fetchWithProgress = (url, options = {}) => { const request = new XMLHttpRequest() request.open(options.method || 'GET', url.toString(), true) - const { timeout } = options - if (timeout > 0 && timeout < Infinity) { - request.timeout = options.timeout + const { timeout, headers } = options + + if (timeout && timeout > 0 && timeout < Infinity) { + request.timeout = timeout } if (options.overrideMimeType != null) { request.overrideMimeType(options.overrideMimeType) } - if (options.headers) { - for (const [name, value] of options.headers.entries()) { + if (headers) { + // @ts-ignore TS does not know that headers is an iterator + for (const [name, value] of headers) { request.setRequestHeader(name, value) } } @@ -91,22 +89,24 @@ const fetchWithProgress = (url, options = {}) => { request.ontimeout = handleEvent request.onabort = handleEvent - request.send(options.body) + request.send(/** @type {BodyInit} */(options.body)) }) } const fetchWithStreaming = fetch +/** + * @param {string | Request} url + * @param {FetchOptions} options + */ const fetchWith = (url, options = {}) => (options.onUploadProgress != null) ? fetchWithProgress(url, options) : fetchWithStreaming(url, options) -exports.fetch = fetchWith -exports.Request = Request -exports.Headers = Headers - /** + * Parse Headers from a XMLHttpRequest + * * @param {string} input * @returns {Headers} */ @@ -125,7 +125,7 @@ const parseHeaders = (input) => { class ResponseWithURL extends Response { /** * @param {string} url - * @param {string|Blob|ArrayBufferView|ArrayBuffer|FormData|ReadableStream} body + * @param {BodyInit} body * @param {ResponseInit} options */ constructor (url, body, options) { @@ -133,3 +133,10 @@ class ResponseWithURL extends Response { Object.defineProperty(this, 'url', { value: url }) } } + +module.exports = { + fetch: fetchWith, + Request, + Headers, + ResponseWithURL +} diff --git a/src/http/fetch.node.js b/src/http/fetch.node.js index bf3cf97..049b241 100644 --- a/src/http/fetch.node.js +++ b/src/http/fetch.node.js @@ -1,39 +1,25 @@ -// @ts-check 'use strict' -const { Request, Response, Headers, default: nodeFetch } = require('../fetch') +const { Request, Response, Headers, default: nativeFetch } = require('../fetch') +// @ts-ignore const toStream = require('it-to-stream') const { Buffer } = require('buffer') - /** - * @typedef {RequestInit & ExtraFetchOptions} FetchOptions + * @typedef {import('electron-fetch').BodyInit} BodyInit + * @typedef {import('stream').Readable} NodeReadableStream * - * @typedef {import('stream').Readable} Readable - * @typedef {Object} LoadProgress - * @property {number} total - * @property {number} loaded - * @property {boolean} lengthComputable - * @typedef {Object} ExtraFetchOptions - * @property {number} [timeout] - * @property {URLSearchParams} [searchParams] - * @property {function(LoadProgress):void} [onUploadProgress] - * @property {function(LoadProgress):void} [onDownloadProgress] - * @property {string} [overrideMimeType] - * @returns {Promise} + * @typedef {import('../types').FetchOptions} FetchOptions + * @typedef {import('../types').ProgressFn} ProgressFn */ /** - * @param {string|URL} url + * @param {string|Request} url * @param {FetchOptions} [options] * @returns {Promise} */ const fetch = (url, options = {}) => // @ts-ignore - nodeFetch(url, withUploadProgress(options)) - -exports.fetch = fetch -exports.Request = Request -exports.Headers = Headers + nativeFetch(url, withUploadProgress(options)) /** * Takes fetch options and wraps request body to track upload progress if @@ -43,12 +29,17 @@ exports.Headers = Headers * @returns {FetchOptions} */ const withUploadProgress = (options) => { - const { onUploadProgress } = options - if (onUploadProgress) { + const { onUploadProgress, body } = options + if (onUploadProgress && body) { + // This works around the fact that electron-fetch serializes `Uint8Array`s + // and `ArrayBuffer`s to strings. + const content = normalizeBody(body) + + const rsp = new Response(content) + const source = iterateBodyWithProgress(/** @type {NodeReadableStream} */(rsp.body), onUploadProgress) return { ...options, - // @ts-ignore - body: bodyWithUploadProgress(options, onUploadProgress) + body: toStream.readable(source) } } else { return options @@ -56,43 +47,18 @@ const withUploadProgress = (options) => { } /** - * Takes request `body` and `onUploadProgress` handler and returns wrapped body - * that as consumed will report progress to supplied `onUploadProgress` handler. - * - * @param {FetchOptions} init - * @param {function(LoadProgress):void} onUploadProgress - * @returns {Readable} + * @param {BodyInit} input + * @returns {Blob | FormData | URLSearchParams | ReadableStream | string | NodeReadableStream | Buffer} */ -const bodyWithUploadProgress = (init, onUploadProgress) => { - // This works around the fact that electron-fetch serializes `Uint8Array`s - // and `ArrayBuffer`s to strings. - const content = normalizeBody(init.body) - - // @ts-ignore - Response does not accept node `Readable` streams. - const { body } = new Response(content, init) - // @ts-ignore - Unlike standard Response, node-fetch `body` has a differnt - // type see: see https://github.com/node-fetch/node-fetch/blob/master/src/body.js - const source = iterateBodyWithProgress(body, onUploadProgress) - return toStream.readable(source) -} - -/** - * @param {BodyInit} [input] - * @returns {Buffer|Readable|Blob|null} - */ -const normalizeBody = (input = null) => { +const normalizeBody = (input) => { if (input instanceof ArrayBuffer) { return Buffer.from(input) } else if (ArrayBuffer.isView(input)) { return Buffer.from(input.buffer, input.byteOffset, input.byteLength) } else if (typeof input === 'string') { return Buffer.from(input) - } else { - // @ts-ignore - Could be FormData|URLSearchParams|ReadableStream - // however electron-fetch does not support either of those types and - // node-fetch normalizes those to node streams. - return input } + return input } /** @@ -100,12 +66,11 @@ const normalizeBody = (input = null) => { * and returns async iterable that emits body chunks and emits * `onUploadProgress`. * - * @param {Buffer|null|Readable} body - * @param {function(LoadProgress):void} onUploadProgress + * @param {NodeReadableStream | null} body + * @param {ProgressFn} onUploadProgress * @returns {AsyncIterable} */ const iterateBodyWithProgress = async function * (body, onUploadProgress) { - /** @type {Buffer|null|Readable} */ if (body == null) { onUploadProgress({ total: 0, loaded: 0, lengthComputable: true }) } else if (Buffer.isBuffer(body)) { @@ -124,3 +89,9 @@ const iterateBodyWithProgress = async function * (body, onUploadProgress) { } } } + +module.exports = { + fetch, + Request, + Headers +} diff --git a/src/types.ts b/src/types.ts new file mode 100644 index 0000000..322550e --- /dev/null +++ b/src/types.ts @@ -0,0 +1,53 @@ +import type { RequestInit, Response } from 'electron-fetch' +interface ProgressStatus { + total: number + loaded: number + lengthComputable: boolean +} + +export interface ProgressFn { (status: ProgressStatus): void } + +export interface FetchOptions extends RequestInit { + /** + * Amount of time until request should timeout in ms. + */ + timeout?: number + /** + * URL search param. + */ + searchParams?: URLSearchParams + /** + * Can be passed to track upload progress. + * Note that if this option in passed underlying request will be performed using `XMLHttpRequest` and response will not be streamed. + */ + onUploadProgress?: ProgressFn + /** + * Can be passed to track download progress. + */ + onDownloadProgress?: ProgressFn + overrideMimeType?: string +} + +export interface HTTPOptions extends FetchOptions { + json?: any + /** + * The base URL to use in case url is a relative URL + */ + base? : string + /** + * Throw not ok responses as Errors + */ + throwHttpErrors?: boolean + /** + * Transform search params + */ + transformSearchParams?: (params: URLSearchParams) => URLSearchParams + /** + * When iterating the response body, transform each chunk with this function. + */ + transform?: (chunk: any) => any + /** + * Handle errors + */ + handleError?: (rsp: Response) => Promise +} diff --git a/test/http.spec.js b/test/http.spec.js index e5dc3f5..65d3b8b 100644 --- a/test/http.spec.js +++ b/test/http.spec.js @@ -3,6 +3,7 @@ /* eslint-env mocha */ const { expect } = require('aegir/utils/chai') const HTTP = require('../src/http') +// @ts-ignore const toStream = require('it-to-stream') const delay = require('delay') const AbortController = require('native-abort-controller') @@ -78,7 +79,7 @@ describe('http', function () { it('allow async aborting', async function () { const controller = new AbortController() - + // @ts-ignore its never undefined const res = HTTP.get(process.env.ECHO_SERVER, { signal: controller.signal }) @@ -123,11 +124,12 @@ describe('http', function () { throw err }()) + // @ts-ignore its never undefined const res = await HTTP.post(process.env.ECHO_SERVER, { body: toStream.readable(body) }) - await expect(drain(HTTP.ndjson(res.body))).to.eventually.be.rejectedWith(/aborted/) + await expect(drain(res.ndjson())).to.eventually.be.rejectedWith(/aborted/) }) it.skip('should handle errors in streaming bodies when a signal is passed', async function () { @@ -145,13 +147,13 @@ describe('http', function () { throw err }()) - + // @ts-ignore its never undefined const res = await HTTP.post(process.env.ECHO_SERVER, { body: toStream.readable(body), signal: controller.signal }) - await expect(drain(HTTP.ndjson(res.body))).to.eventually.be.rejectedWith(/aborted/) + await expect(drain(res.ndjson())).to.eventually.be.rejectedWith(/aborted/) }) it('progress events', async () => { diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..b008382 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "aegir/src/config/tsconfig.aegir.json", + "compilerOptions": { + "outDir": "dist", + "baseUrl": "./", + "paths": { + "*": ["./types/*"] + } + }, + "include": ["types", "test", "src"] +} diff --git a/types/electron-fetch/index.d.ts b/types/electron-fetch/index.d.ts new file mode 100644 index 0000000..596d04a --- /dev/null +++ b/types/electron-fetch/index.d.ts @@ -0,0 +1,117 @@ +import { Readable as NodeReadableStream } from 'stream' + +export default function fetch (input: RequestInfo, init?: RequestInit): Promise + +export interface Body { + readonly body: NodeReadableStream | ReadableStream | null + readonly bodyUsed: boolean + arrayBuffer: () => Promise + blob: () => Promise + formData: () => Promise + json: () => Promise + text: () => Promise + buffer: () => Promise +} +export class Headers extends globalThis.Headers {} + +/** This Fetch API interface represents the response to a request. */ +export class Response extends globalThis.Response { + constructor (body?: BodyInit | null, init?: ResponseInit) + readonly headers: Headers + readonly ok: boolean + readonly redirected: boolean + readonly status: number + readonly statusText: string + readonly trailer: Promise + readonly type: ResponseType + readonly url: string + clone: () => Response + + // Body interface + readonly body: NodeReadableStream | ReadableStream | null + readonly bodyUsed: boolean + arrayBuffer: () => Promise + blob: () => Promise + formData: () => Promise + json: () => Promise + text: () => Promise + buffer: () => Promise + iterator: () => AsyncIterable + ndjson: () => AsyncIterable + + static error (): Response + static redirect (url: string, status?: number): Response +} + +export class Request extends globalThis.Request { + constructor (input: RequestInfo, init?: RequestInit) + + // Body interface + readonly body: NodeReadableStream | ReadableStream | null + readonly bodyUsed: boolean + arrayBuffer: () => Promise + blob: () => Promise + formData: () => Promise + json: () => Promise + text: () => Promise + buffer: () => Promise +} + +export type RequestInfo = Request | string + +export type BodyInit = Blob | BufferSource | FormData | URLSearchParams | ReadableStream | string | Buffer | NodeReadableStream + +export interface RequestInit { + /** + * A BodyInit object or null to set request's body. + */ + body?: BodyInit | null + /** + * A string indicating how the request will interact with the browser's cache to set request's cache. + */ + cache?: RequestCache + /** + * A string indicating whether credentials will be sent with the request always, never, or only when sent to a same-origin URL. Sets request's credentials. + */ + credentials?: RequestCredentials + /** + * A Headers object, an object literal, or an array of two-item arrays to set request's headers. + */ + headers?: HeadersInit + /** + * A cryptographic hash of the resource to be fetched by request. Sets request's integrity. + */ + integrity?: string + /** + * A boolean to set request's keepalive. + */ + keepalive?: boolean + /** + * A string to set request's method. + */ + method?: string + /** + * A string to indicate whether the request will use CORS, or will be restricted to same-origin URLs. Sets request's mode. + */ + mode?: RequestMode + /** + * A string indicating whether request follows redirects, results in an error upon encountering a redirect, or returns the redirect (in an opaque fashion). Sets request's redirect. + */ + redirect?: RequestRedirect + /** + * A string whose value is a same-origin URL, "about:client", or the empty string, to set request's referrer. + */ + referrer?: string + /** + * A referrer policy to set request's referrerPolicy. + */ + referrerPolicy?: ReferrerPolicy + /** + * An AbortSignal to set request's signal. + */ + signal?: AbortSignal | null + /** + * Can only be null. Used to disassociate request from any Window. + */ + window?: any +} diff --git a/types/is-electron/index.d.ts b/types/is-electron/index.d.ts new file mode 100644 index 0000000..0f33e26 --- /dev/null +++ b/types/is-electron/index.d.ts @@ -0,0 +1,2 @@ +function isElectron (): boolean +export = isElectron diff --git a/types/iso-url/index.d.ts b/types/iso-url/index.d.ts new file mode 100644 index 0000000..eac8a7e --- /dev/null +++ b/types/iso-url/index.d.ts @@ -0,0 +1 @@ +export { URL, URLSearchParams } diff --git a/types/native-abort-controller/index.d.ts b/types/native-abort-controller/index.d.ts new file mode 100644 index 0000000..a448ef7 --- /dev/null +++ b/types/native-abort-controller/index.d.ts @@ -0,0 +1,4 @@ +export type { AbortController, AbortSignal } + +export default AbortController +export = AbortController diff --git a/types/native-fetch/index.d.ts b/types/native-fetch/index.d.ts new file mode 100644 index 0000000..596d04a --- /dev/null +++ b/types/native-fetch/index.d.ts @@ -0,0 +1,117 @@ +import { Readable as NodeReadableStream } from 'stream' + +export default function fetch (input: RequestInfo, init?: RequestInit): Promise + +export interface Body { + readonly body: NodeReadableStream | ReadableStream | null + readonly bodyUsed: boolean + arrayBuffer: () => Promise + blob: () => Promise + formData: () => Promise + json: () => Promise + text: () => Promise + buffer: () => Promise +} +export class Headers extends globalThis.Headers {} + +/** This Fetch API interface represents the response to a request. */ +export class Response extends globalThis.Response { + constructor (body?: BodyInit | null, init?: ResponseInit) + readonly headers: Headers + readonly ok: boolean + readonly redirected: boolean + readonly status: number + readonly statusText: string + readonly trailer: Promise + readonly type: ResponseType + readonly url: string + clone: () => Response + + // Body interface + readonly body: NodeReadableStream | ReadableStream | null + readonly bodyUsed: boolean + arrayBuffer: () => Promise + blob: () => Promise + formData: () => Promise + json: () => Promise + text: () => Promise + buffer: () => Promise + iterator: () => AsyncIterable + ndjson: () => AsyncIterable + + static error (): Response + static redirect (url: string, status?: number): Response +} + +export class Request extends globalThis.Request { + constructor (input: RequestInfo, init?: RequestInit) + + // Body interface + readonly body: NodeReadableStream | ReadableStream | null + readonly bodyUsed: boolean + arrayBuffer: () => Promise + blob: () => Promise + formData: () => Promise + json: () => Promise + text: () => Promise + buffer: () => Promise +} + +export type RequestInfo = Request | string + +export type BodyInit = Blob | BufferSource | FormData | URLSearchParams | ReadableStream | string | Buffer | NodeReadableStream + +export interface RequestInit { + /** + * A BodyInit object or null to set request's body. + */ + body?: BodyInit | null + /** + * A string indicating how the request will interact with the browser's cache to set request's cache. + */ + cache?: RequestCache + /** + * A string indicating whether credentials will be sent with the request always, never, or only when sent to a same-origin URL. Sets request's credentials. + */ + credentials?: RequestCredentials + /** + * A Headers object, an object literal, or an array of two-item arrays to set request's headers. + */ + headers?: HeadersInit + /** + * A cryptographic hash of the resource to be fetched by request. Sets request's integrity. + */ + integrity?: string + /** + * A boolean to set request's keepalive. + */ + keepalive?: boolean + /** + * A string to set request's method. + */ + method?: string + /** + * A string to indicate whether the request will use CORS, or will be restricted to same-origin URLs. Sets request's mode. + */ + mode?: RequestMode + /** + * A string indicating whether request follows redirects, results in an error upon encountering a redirect, or returns the redirect (in an opaque fashion). Sets request's redirect. + */ + redirect?: RequestRedirect + /** + * A string whose value is a same-origin URL, "about:client", or the empty string, to set request's referrer. + */ + referrer?: string + /** + * A referrer policy to set request's referrerPolicy. + */ + referrerPolicy?: ReferrerPolicy + /** + * An AbortSignal to set request's signal. + */ + signal?: AbortSignal | null + /** + * Can only be null. Used to disassociate request from any Window. + */ + window?: any +} From 730edd1b4367e5934bd2b8f5234c3a65afdb9cf0 Mon Sep 17 00:00:00 2001 From: Hugo Dias Date: Mon, 11 Jan 2021 16:06:53 +0000 Subject: [PATCH 08/10] fix: add prepare --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index b46b3a0..beb06d3 100644 --- a/package.json +++ b/package.json @@ -22,13 +22,13 @@ "typesVersions": { "*": { "src/*": [ - "dist/src/*", - "dist/src/*/index" + "dist/src/*" ] } }, "repository": "github:ipfs/js-ipfs-utils", "scripts": { + "prepare": "aegir build --no-bundle", "test": "aegir test", "test:browser": "aegir test -t browser", "test:node": "aegir test -t node", From 9d5addbfb89dcc7ecc8b241777b3a078e3074f7e Mon Sep 17 00:00:00 2001 From: Hugo Dias Date: Wed, 13 Jan 2021 15:36:23 +0000 Subject: [PATCH 09/10] fix: feedback --- package.json | 1 + src/files/glob-source.js | 5 ++--- src/http/fetch.browser.js | 3 +-- src/index.js | 1 + src/path-join.browser.js | 2 +- test/files/url-source.spec.js | 7 +++++-- test/http.spec.js | 33 ++++++++++++++++----------------- 7 files changed, 27 insertions(+), 25 deletions(-) diff --git a/package.json b/package.json index beb06d3..7581f64 100644 --- a/package.json +++ b/package.json @@ -61,6 +61,7 @@ "web-encoding": "^1.0.6" }, "devDependencies": { + "@types/err-code": "^2.0.0", "@types/fs-extra": "^9.0.5", "aegir": "^30.3.0", "delay": "^4.4.0", diff --git a/src/files/glob-source.js b/src/files/glob-source.js index 6a62e41..8bdedcd 100644 --- a/src/files/glob-source.js +++ b/src/files/glob-source.js @@ -3,7 +3,6 @@ const fs = require('fs-extra') const glob = require('it-glob') const Path = require('path') -// @ts-ignore const errCode = require('err-code') /** @@ -17,8 +16,8 @@ const errCode = require('err-code') * @param {boolean} [options.followSymlinks] - follow symlinks * @param {boolean} [options.preserveMode] - preserve mode * @param {boolean} [options.preserveMtime] - preserve mtime - * @param {boolean | number} [options.mode] - mode to use - if preserveMode is true this will be ignored - * @param {boolean | Date} [options.mtime] - mtime to use - if preserveMtime is true this will be ignored + * @param {number} [options.mode] - mode to use - if preserveMode is true this will be ignored + * @param {Date} [options.mtime] - mtime to use - if preserveMtime is true this will be ignored * @yields {Object} File objects in the form `{ path: String, content: AsyncIterator }` */ module.exports = async function * globSource (paths, options) { diff --git a/src/http/fetch.browser.js b/src/http/fetch.browser.js index 78ff7e4..4c70938 100644 --- a/src/http/fetch.browser.js +++ b/src/http/fetch.browser.js @@ -30,8 +30,7 @@ const fetchWithProgress = (url, options = {}) => { } if (headers) { - // @ts-ignore TS does not know that headers is an iterator - for (const [name, value] of headers) { + for (const [name, value] of new Headers(headers)) { request.setRequestHeader(name, value) } } diff --git a/src/index.js b/src/index.js index f177540..b4d1ea2 100644 --- a/src/index.js +++ b/src/index.js @@ -1,2 +1,3 @@ 'use strict' +// just a empty entry point to avoid errors from aegir module.exports = {} diff --git a/src/path-join.browser.js b/src/path-join.browser.js index e425a56..a274ca4 100644 --- a/src/path-join.browser.js +++ b/src/path-join.browser.js @@ -1,7 +1,7 @@ 'use strict' /** - * @param {any[]} args + * @param {string[]} args */ function join (...args) { if (args.length === 0) { diff --git a/test/files/url-source.spec.js b/test/files/url-source.spec.js index 2a02e9f..e20aea9 100644 --- a/test/files/url-source.spec.js +++ b/test/files/url-source.spec.js @@ -13,7 +13,10 @@ describe('url-source', function () { const content = 'foo' const file = await last(urlSource(`${process.env.ECHO_SERVER}/download?data=${content}`)) - // @ts-ignore - await expect(all(file.content)).to.eventually.deep.equal([Buffer.from(content)]) + if (file && file.content) { + await expect(all(file.content)).to.eventually.deep.equal([Buffer.from(content)]) + } else { + throw new Error('empty response') + } }) }) diff --git a/test/http.spec.js b/test/http.spec.js index 65d3b8b..f4819b9 100644 --- a/test/http.spec.js +++ b/test/http.spec.js @@ -15,27 +15,29 @@ const uint8ArrayFromString = require('uint8arrays/from-string') const uint8ArrayEquals = require('uint8arrays/equals') const uint8ArrayConcat = require('uint8arrays/concat') +const ECHO_SERVER = process.env.ECHO_SERVER || '' + describe('http', function () { it('makes a GET request', async function () { - const req = await HTTP.get(`${process.env.ECHO_SERVER}/echo/query?test=one`) + const req = await HTTP.get(`${ECHO_SERVER}/echo/query?test=one`) const rsp = await req.json() expect(rsp).to.be.deep.eq({ test: 'one' }) }) it('makes a GET request with redirect', async function () { - const req = await HTTP.get(`${process.env.ECHO_SERVER}/redirect?to=${encodeURI(`${process.env.ECHO_SERVER}/echo/query?test=one`)}`) + const req = await HTTP.get(`${ECHO_SERVER}/redirect?to=${encodeURI(`${ECHO_SERVER}/echo/query?test=one`)}`) const rsp = await req.json() expect(rsp).to.be.deep.eq({ test: 'one' }) }) it('makes a GET request with a really short timeout', function () { - return expect(HTTP.get(`${process.env.ECHO_SERVER}/redirect?to=${encodeURI(`${process.env.ECHO_SERVER}/echo/query?test=one`)}`, { + return expect(HTTP.get(`${ECHO_SERVER}/redirect?to=${encodeURI(`${ECHO_SERVER}/echo/query?test=one`)}`, { timeout: 1 })).to.eventually.be.rejectedWith().instanceOf(HTTP.TimeoutError) }) it('respects headers', async function () { - const req = await HTTP.post(`${process.env.ECHO_SERVER}/echo/headers`, { + const req = await HTTP.post(`${ECHO_SERVER}/echo/headers`, { headers: { foo: 'bar' } @@ -50,13 +52,13 @@ describe('http', function () { bar: 'baz' } }) - const req = await http.post(`${process.env.ECHO_SERVER}/echo/headers`) + const req = await http.post(`${ECHO_SERVER}/echo/headers`) const rsp = await req.json() expect(rsp).to.have.property('bar', 'baz') }) it('makes a JSON request', async () => { - const req = await HTTP.post(`${process.env.ECHO_SERVER}/echo`, { + const req = await HTTP.post(`${ECHO_SERVER}/echo`, { json: { test: 2 } @@ -67,7 +69,7 @@ describe('http', function () { }) it('makes a DELETE request', async () => { - const req = await HTTP.delete(`${process.env.ECHO_SERVER}/echo`, { + const req = await HTTP.delete(`${ECHO_SERVER}/echo`, { json: { test: 2 } @@ -79,8 +81,7 @@ describe('http', function () { it('allow async aborting', async function () { const controller = new AbortController() - // @ts-ignore its never undefined - const res = HTTP.get(process.env.ECHO_SERVER, { + const res = HTTP.get(ECHO_SERVER, { signal: controller.signal }) controller.abort() @@ -89,7 +90,7 @@ describe('http', function () { }) it('parses the response as ndjson', async function () { - const res = await HTTP.post(`${process.env.ECHO_SERVER}/echo`, { + const res = await HTTP.post(`${ECHO_SERVER}/echo`, { body: '{}\n{}' }) @@ -100,7 +101,7 @@ describe('http', function () { it('parses the response as an async iterable', async function () { const res = await HTTP.post('echo', { - base: process.env.ECHO_SERVER, + base: ECHO_SERVER, body: 'hello world' }) @@ -124,8 +125,7 @@ describe('http', function () { throw err }()) - // @ts-ignore its never undefined - const res = await HTTP.post(process.env.ECHO_SERVER, { + const res = await HTTP.post(ECHO_SERVER, { body: toStream.readable(body) }) @@ -147,8 +147,7 @@ describe('http', function () { throw err }()) - // @ts-ignore its never undefined - const res = await HTTP.post(process.env.ECHO_SERVER, { + const res = await HTTP.post(ECHO_SERVER, { body: toStream.readable(body), signal: controller.signal }) @@ -159,7 +158,7 @@ describe('http', function () { it('progress events', async () => { let upload = 0 const body = new Uint8Array(1000000 / 2) - const request = await HTTP.post(`${process.env.ECHO_SERVER}/echo`, { + const request = await HTTP.post(`${ECHO_SERVER}/echo`, { body, onUploadProgress: (progress) => { expect(progress).to.have.property('lengthComputable').to.be.a('boolean') @@ -179,7 +178,7 @@ describe('http', function () { const buf = uint8ArrayFromString('a163666f6f6c6461672d63626f722d626172', 'base16') const params = Array.from(buf).map(val => `data=${val.toString()}`).join('&') - const req = await HTTP.get(`${process.env.ECHO_SERVER}/download?${params}`) + const req = await HTTP.get(`${ECHO_SERVER}/download?${params}`) const rsp = await req.arrayBuffer() expect(uint8ArrayEquals(new Uint8Array(rsp), buf)).to.be.true() }) From 42e0b1e0780cda0d8ae074edb177b12a88a5b972 Mon Sep 17 00:00:00 2001 From: Hugo Dias Date: Thu, 14 Jan 2021 12:47:03 +0000 Subject: [PATCH 10/10] fix: more feedback --- src/http.js | 74 ++++++++++++++++++++++++++++++++++------------------- 1 file changed, 48 insertions(+), 26 deletions(-) diff --git a/src/http.js b/src/http.js index fd05696..e3182e8 100644 --- a/src/http.js +++ b/src/http.js @@ -12,6 +12,7 @@ const anySignal = require('any-signal') /** * @typedef {import('electron-fetch').Response} Response * @typedef {import('stream').Readable} NodeReadableStream + * @typedef {import('stream').Duplex} NodeDuplexStream * @typedef {import('./types').HTTPOptions} HTTPOptions */ @@ -248,32 +249,27 @@ const ndjson = async function * (source) { * @returns {AsyncIterable} */ const fromStream = (source) => { - if (isAsyncIterable(source)) { - // Workaround for https://github.com/node-fetch/node-fetch/issues/766 - if (Object.prototype.hasOwnProperty.call(source, 'readable') && Object.prototype.hasOwnProperty.call(source, 'writable')) { - const nodeSource = /** @type {NodeReadableStream} */(source) - const iter = nodeSource[Symbol.asyncIterator]() - return { - // @ts-ignore - next: iter.next.bind(iter), - return () { - nodeSource.destroy() - if (iter.return) { - return iter.return() + // Workaround for https://github.com/node-fetch/node-fetch/issues/766 + if (isNodeReadableStream(source)) { + const iter = source[Symbol.asyncIterator]() + return { + [Symbol.asyncIterator] () { + return { + next: iter.next.bind(iter), + return (value) { + source.destroy() + if (typeof iter.return === 'function') { + return iter.return() + } + return Promise.resolve({ done: true, value }) } - return {} - }, - [Symbol.asyncIterator] () { - return this } } } - - return /** @type {AsyncIterable} */(source) } - if (source && typeof /** @type {ReadableStream} */(source).getReader === 'function') { - const reader = /** @type {ReadableStream} */(source).getReader() + if (isWebReadableStream(source)) { + const reader = source.getReader() return (async function * () { try { while (true) { @@ -292,21 +288,47 @@ const fromStream = (source) => { })() } + if (isAsyncIterable(source)) { + return source + } + throw new TypeError('Body can\'t be converted to AsyncIterable') } /** * Check if it's an AsyncIterable * - * @param {any} obj + * @template {unknown} TChunk + * @template {any} Other + * @param {Other|AsyncIterable} value + * @returns {value is AsyncIterable} + */ +const isAsyncIterable = (value) => { + return typeof value === 'object' && + value !== null && + typeof /** @type {any} */(value)[Symbol.asyncIterator] === 'function' +} + +/** + * Check for web readable stream + * + * @template {unknown} TChunk + * @template {any} Other + * @param {Other|ReadableStream} value + * @returns {value is ReadableStream} */ -const isAsyncIterable = (obj) => { - return typeof obj === 'object' && - obj !== null && - // typeof obj.next === 'function' && - typeof obj[Symbol.asyncIterator] === 'function' +const isWebReadableStream = (value) => { + return value && typeof /** @type {any} */(value).getReader === 'function' } +/** + * @param {any} value + * @returns {value is NodeReadableStream} + */ +const isNodeReadableStream = (value) => + Object.prototype.hasOwnProperty.call(value, 'readable') && + Object.prototype.hasOwnProperty.call(value, 'writable') + HTTP.HTTPError = HTTPError HTTP.TimeoutError = TimeoutError HTTP.streamToAsyncIterator = fromStream