diff --git a/README.md b/README.md index 227e3dd6..6fd9f126 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ -# ipfs-interfaces +# stores [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) -[![codecov](https://img.shields.io/codecov/c/github/ipfs/js-ipfs-interfaces.svg?style=flat-square)](https://codecov.io/gh/ipfs/js-ipfs-interfaces) -[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/js-ipfs-interfaces/js-test-and-release.yml?branch=master\&style=flat-square)](https://github.com/ipfs/js-ipfs-interfaces/actions/workflows/js-test-and-release.yml?query=branch%3Amaster) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/js-stores.svg?style=flat-square)](https://codecov.io/gh/ipfs/js-stores) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/js-stores/js-test-and-release.yml?branch=master\&style=flat-square)](https://github.com/ipfs/js-stores/actions/workflows/js-test-and-release.yml?query=branch%3Amaster) -> TypeScript interfaces used by IPFS internals +> Blockstores and datastores used by IP-JS internals ## Table of contents @@ -17,6 +17,16 @@ ## Structure +- [`/packages/blockstore-core`](./packages/blockstore-core) Contains various implementations of the API contract described in interface-blockstore +- [`/packages/blockstore-fs`](./packages/blockstore-fs) Blockstore implementation with file system backend +- [`/packages/blockstore-idb`](./packages/blockstore-idb) Blockstore implementation with IndexedDB backend +- [`/packages/blockstore-level`](./packages/blockstore-level) Blockstore implementation with level(up|down) backend +- [`/packages/blockstore-s3`](./packages/blockstore-s3) IPFS blockstore implementation backed by s3 +- [`/packages/datastore-core`](./packages/datastore-core) Wrapper implementation for interface-datastore +- [`/packages/datastore-fs`](./packages/datastore-fs) Datastore implementation with file system backend +- [`/packages/datastore-idb`](./packages/datastore-idb) Datastore implementation with IndexedDB backend. +- [`/packages/datastore-level`](./packages/datastore-level) Datastore implementation with level(up|down) backend +- [`/packages/datastore-s3`](./packages/datastore-s3) IPFS datastore implementation backed by s3 - [`/packages/interface-blockstore`](./packages/interface-blockstore) An interface for storing and retrieving blocks - [`/packages/interface-blockstore-tests`](./packages/interface-blockstore-tests) Compliance tests for the blockstore interface - [`/packages/interface-datastore`](./packages/interface-datastore) datastore interface @@ -29,7 +39,7 @@ See the [./packages](./packages) directory for the various interfaces. ## API Docs -- +- ## License @@ -40,7 +50,7 @@ Licensed under either of ## Contribute -Contributions welcome! Please check out [the issues](https://github.com/ipfs/js-ipfs-interfaces/issues). +Contributions welcome! Please check out [the issues](https://github.com/ipfs/js-stores/issues). Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. diff --git a/package.json b/package.json index 66266863..37138928 100644 --- a/package.json +++ b/package.json @@ -1,15 +1,15 @@ { - "name": "ipfs-interfaces", + "name": "stores", "version": "1.0.0", - "description": "TypeScript interfaces used by IPFS internals", + "description": "Blockstores and datastores used by IP-JS internals", "license": "Apache-2.0 OR MIT", - "homepage": "https://github.com/ipfs/js-ipfs-interfaces#readme", + "homepage": "https://github.com/ipfs/js-stores#readme", "repository": { "type": "git", - "url": "git+https://github.com/ipfs/js-ipfs-interfaces.git" + "url": "git+https://github.com/ipfs/js-stores.git" }, "bugs": { - "url": "https://github.com/ipfs/js-ipfs-interfaces/issues" + "url": "https://github.com/ipfs/js-stores/issues" }, "engines": { "node": ">=16.0.0", diff --git a/packages/blockstore-core/CHANGELOG.md b/packages/blockstore-core/CHANGELOG.md new file mode 100644 index 00000000..cfd6b315 --- /dev/null +++ b/packages/blockstore-core/CHANGELOG.md @@ -0,0 +1,97 @@ +## [4.0.2](https://github.com/ipfs/js-blockstore-core/compare/v4.0.1...v4.0.2) (2023-03-23) + + +### Dependencies + +* update interface-store to 5.x.x ([#74](https://github.com/ipfs/js-blockstore-core/issues/74)) ([32e0e52](https://github.com/ipfs/js-blockstore-core/commit/32e0e52e87c1ec9c245edeec20b3df369d479034)) + +## [4.0.1](https://github.com/ipfs/js-blockstore-core/compare/v4.0.0...v4.0.1) (2023-03-14) + + +### Bug Fixes + +* add errors for get and has failed ([#71](https://github.com/ipfs/js-blockstore-core/issues/71)) ([cd990dd](https://github.com/ipfs/js-blockstore-core/commit/cd990dd6ebd4cb0d399b225e501365a3b8653f67)) + +## [4.0.0](https://github.com/ipfs/js-blockstore-core/compare/v3.0.0...v4.0.0) (2023-03-13) + + +### ⚠ BREAKING CHANGES + +* `interface-blockstore@5` removes query/batch methods and adds getAll + +### Features + +* update to latest blockstore interface ([#70](https://github.com/ipfs/js-blockstore-core/issues/70)) ([273397d](https://github.com/ipfs/js-blockstore-core/commit/273397d7fca96db8cf95afc07ed0ea1a7d4239f3)) + +## [3.0.0](https://github.com/ipfs/js-blockstore-core/compare/v2.0.2...v3.0.0) (2023-01-06) + + +### ⚠ BREAKING CHANGES + +* update blockstore deps (#59) + +### Dependencies + +* update blockstore deps ([#59](https://github.com/ipfs/js-blockstore-core/issues/59)) ([0299bf5](https://github.com/ipfs/js-blockstore-core/commit/0299bf558bc7f2ff3d63ce69a4dee55775f4389a)) + +## [2.0.2](https://github.com/ipfs/js-blockstore-core/compare/v2.0.1...v2.0.2) (2022-10-14) + + +### Dependencies + +* bump multiformats from 9.9.0 to 10.0.0 ([#49](https://github.com/ipfs/js-blockstore-core/issues/49)) ([65b9c3f](https://github.com/ipfs/js-blockstore-core/commit/65b9c3ffeb0d1db1e13ee31104cf693aece9fc28)) + +## [2.0.1](https://github.com/ipfs/js-blockstore-core/compare/v2.0.0...v2.0.1) (2022-08-13) + + +### Dependencies + +* bump interface-blockstore from 2.0.3 to 3.0.0 ([#41](https://github.com/ipfs/js-blockstore-core/issues/41)) ([be1323a](https://github.com/ipfs/js-blockstore-core/commit/be1323a7c4f6deb9d19163c3f6d3ecd57296a25c)) +* bump interface-store from 2.0.2 to 3.0.0 ([#42](https://github.com/ipfs/js-blockstore-core/issues/42)) ([54caf74](https://github.com/ipfs/js-blockstore-core/commit/54caf748cf4277abb7f82a1eefb91bd48141c307)) +* **dev:** bump interface-blockstore-tests from 2.0.4 to 3.0.0 ([#44](https://github.com/ipfs/js-blockstore-core/issues/44)) ([232033d](https://github.com/ipfs/js-blockstore-core/commit/232033db6a8204e2852d4ad29e6d4ff8d1c9a685)) + +## [2.0.0](https://github.com/ipfs/js-blockstore-core/compare/v1.0.5...v2.0.0) (2022-08-13) + + +### ⚠ BREAKING CHANGES + +* this module is now ESM-only + +### Features + +* publish as ESM-only ([#45](https://github.com/ipfs/js-blockstore-core/issues/45)) ([01009f6](https://github.com/ipfs/js-blockstore-core/commit/01009f683bbc8b50c202919c448dc0d4cbb86249)) + + +### Trivial Changes + +* Update .github/workflows/stale.yml [skip ci] ([9e7d0d6](https://github.com/ipfs/js-blockstore-core/commit/9e7d0d6a2e2900b7739a956c371a3a0dcb623f92)) + +### [1.0.5](https://github.com/ipfs/js-blockstore-core/compare/v1.0.4...v1.0.5) (2022-01-07) + + +### Bug Fixes + +* add semantic release config ([#13](https://github.com/ipfs/js-blockstore-core/issues/13)) ([de1e758](https://github.com/ipfs/js-blockstore-core/commit/de1e758decc2af2ec5e85e99c9b05d2110ade86f)) + +## [1.0.2](https://github.com/ipfs/js-blockstore-core/compare/v1.0.1...v1.0.2) (2021-09-09) + + + +## [1.0.1](https://github.com/ipfs/js-blockstore-core/compare/v1.0.0...v1.0.1) (2021-09-09) + + + +# [1.0.0](https://github.com/ipfs/js-blockstore-core/compare/v0.0.2...v1.0.0) (2021-09-09) + + + +## [0.0.2](https://github.com/ipfs/js-blockstore-core/compare/v0.0.1...v0.0.2) (2021-09-09) + + +### Bug Fixes + +* add types versions ([#1](https://github.com/ipfs/js-blockstore-core/issues/1)) ([fa8ce28](https://github.com/ipfs/js-blockstore-core/commit/fa8ce287da9e2528f7581151e6fa3ac86fcd4196)) + + + +## 0.0.1 (2021-09-09) diff --git a/packages/blockstore-core/LICENSE b/packages/blockstore-core/LICENSE new file mode 100644 index 00000000..20ce483c --- /dev/null +++ b/packages/blockstore-core/LICENSE @@ -0,0 +1,4 @@ +This project is dual licensed under MIT and Apache-2.0. + +MIT: https://www.opensource.org/licenses/mit +Apache-2.0: https://www.apache.org/licenses/license-2.0 diff --git a/packages/blockstore-core/LICENSE-APACHE b/packages/blockstore-core/LICENSE-APACHE new file mode 100644 index 00000000..14478a3b --- /dev/null +++ b/packages/blockstore-core/LICENSE-APACHE @@ -0,0 +1,5 @@ +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/packages/blockstore-core/LICENSE-MIT b/packages/blockstore-core/LICENSE-MIT new file mode 100644 index 00000000..72dc60d8 --- /dev/null +++ b/packages/blockstore-core/LICENSE-MIT @@ -0,0 +1,19 @@ +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/packages/blockstore-core/README.md b/packages/blockstore-core/README.md new file mode 100644 index 00000000..68acc5c3 --- /dev/null +++ b/packages/blockstore-core/README.md @@ -0,0 +1,94 @@ +# blockstore-core + +[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) +[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/js-stores.svg?style=flat-square)](https://codecov.io/gh/ipfs/js-stores) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/js-stores/js-test-and-release.yml?branch=master\&style=flat-square)](https://github.com/ipfs/js-stores/actions/workflows/js-test-and-release.yml?query=branch%3Amaster) + +> Contains various implementations of the API contract described in interface-blockstore + +## Table of contents + +- [Install](#install) + - [Browser ` +``` + +## Implementations + +- Base: [`src/base`](src/base.js) +- Memory: [`src/memory`](src/memory.js) + +## Usage + +### BaseBlockstore + +Provides a complete implementation of the Blockstore interface. You must implement `.get`, `.put`, etc. + +```js +import { BaseBlockstore } from 'blockstore-core/base' + +class MyCustomBlockstore extends BaseBlockstore { + put (key, val, options) { + // store a block + } + + get (key, options) { + // retrieve a block + } + + // ...etc +} +``` + +### MemoryBlockstore + +A simple Blockstore that stores blocks in memory. + +```js +import { MemoryBlockstore } from 'blockstore-core/memory' + +const store = new MemoryBlockstore() +``` + +## API Docs + +- + +## License + +Licensed under either of + +- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) +- MIT ([LICENSE-MIT](LICENSE-MIT) / ) + +## Contribute + +Contributions welcome! Please check out [the issues](https://github.com/ipfs/js-stores/issues). + +Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. + +Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) diff --git a/packages/blockstore-core/package.json b/packages/blockstore-core/package.json new file mode 100644 index 00000000..fe61fd61 --- /dev/null +++ b/packages/blockstore-core/package.json @@ -0,0 +1,186 @@ +{ + "name": "blockstore-core", + "version": "4.0.2", + "description": "Contains various implementations of the API contract described in interface-blockstore", + "author": "Alex Potsides ", + "license": "Apache-2.0 OR MIT", + "homepage": "https://github.com/ipfs/js-stores/tree/master/packages/blockstore-core#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/ipfs/js-stores.git" + }, + "bugs": { + "url": "https://github.com/ipfs/js-stores/issues" + }, + "keywords": [ + "datastore", + "interface", + "ipfs", + "key-value" + ], + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + }, + "type": "module", + "types": "./dist/src/index.d.ts", + "typesVersions": { + "*": { + "*": [ + "*", + "dist/*", + "dist/src/*", + "dist/src/*/index" + ], + "src/*": [ + "*", + "dist/*", + "dist/src/*", + "dist/src/*/index" + ] + } + }, + "files": [ + "src", + "dist", + "!dist/test", + "!**/*.tsbuildinfo" + ], + "exports": { + ".": { + "types": "./dist/src/index.d.ts", + "import": "./dist/src/index.js" + }, + "./base": { + "types": "./dist/src/base.d.ts", + "import": "./dist/src/base.js" + }, + "./errors": { + "types": "./dist/src/errors.d.ts", + "import": "./dist/src/errors.js" + }, + "./memory": { + "types": "./dist/src/memory.d.ts", + "import": "./dist/src/memory.js" + } + }, + "eslintConfig": { + "extends": "ipfs", + "parserOptions": { + "sourceType": "module" + } + }, + "release": { + "branches": [ + "master" + ], + "plugins": [ + [ + "@semantic-release/commit-analyzer", + { + "preset": "conventionalcommits", + "releaseRules": [ + { + "breaking": true, + "release": "major" + }, + { + "revert": true, + "release": "patch" + }, + { + "type": "feat", + "release": "minor" + }, + { + "type": "fix", + "release": "patch" + }, + { + "type": "docs", + "release": "patch" + }, + { + "type": "test", + "release": "patch" + }, + { + "type": "deps", + "release": "patch" + }, + { + "scope": "no-release", + "release": false + } + ] + } + ], + [ + "@semantic-release/release-notes-generator", + { + "preset": "conventionalcommits", + "presetConfig": { + "types": [ + { + "type": "feat", + "section": "Features" + }, + { + "type": "fix", + "section": "Bug Fixes" + }, + { + "type": "chore", + "section": "Trivial Changes" + }, + { + "type": "docs", + "section": "Documentation" + }, + { + "type": "deps", + "section": "Dependencies" + }, + { + "type": "test", + "section": "Tests" + } + ] + } + } + ], + "@semantic-release/changelog", + "@semantic-release/npm", + "@semantic-release/github", + "@semantic-release/git" + ] + }, + "scripts": { + "clean": "aegir clean", + "lint": "aegir lint", + "build": "aegir build", + "release": "aegir release", + "test": "aegir test -t node -t browser -t webworker -t electron-main", + "test:node": "aegir test -t node --cov", + "test:chrome": "aegir test -t browser --cov", + "test:chrome-webworker": "aegir test -t webworker", + "test:firefox": "aegir test -t browser -- --browser firefox", + "test:firefox-webworker": "aegir test -t webworker -- --browser firefox", + "test:electron-main": "aegir test -t electron-main", + "dep-check": "aegir dep-check", + "docs": "aegir docs" + }, + "dependencies": { + "err-code": "^3.0.1", + "interface-blockstore": "^5.0.0", + "interface-store": "^5.0.0", + "multiformats": "^11.0.2" + }, + "devDependencies": { + "aegir": "^38.1.7", + "interface-blockstore-tests": "^6.0.0" + }, + "typedoc": { + "entryPoint": "./src/index.ts" + } +} diff --git a/packages/blockstore-core/src/base.ts b/packages/blockstore-core/src/base.ts new file mode 100644 index 00000000..1abcf540 --- /dev/null +++ b/packages/blockstore-core/src/base.ts @@ -0,0 +1,51 @@ +import type { Blockstore, Pair } from 'interface-blockstore' +import type { AbortOptions, Await, AwaitIterable } from 'interface-store' +import type { CID } from 'multiformats/cid' + +export class BaseBlockstore implements Blockstore { + has (key: CID, options?: AbortOptions): Await { + return Promise.reject(new Error('.has is not implemented')) + } + + put (key: CID, val: Uint8Array, options?: AbortOptions): Await { + return Promise.reject(new Error('.put is not implemented')) + } + + async * putMany (source: AwaitIterable, options?: AbortOptions): AwaitIterable { + for await (const { cid, block } of source) { + await this.put(cid, block, options) + yield cid + } + } + + get (key: CID, options?: AbortOptions): Await { + return Promise.reject(new Error('.get is not implemented')) + } + + async * getMany (source: AwaitIterable, options?: AbortOptions): AwaitIterable { + for await (const key of source) { + yield { + cid: key, + block: await this.get(key, options) + } + } + } + + async delete (key: CID, options?: AbortOptions): Promise { + await Promise.reject(new Error('.delete is not implemented')) + } + + async * deleteMany (source: AwaitIterable, options?: AbortOptions): AwaitIterable { + for await (const key of source) { + await this.delete(key, options) + yield key + } + } + + /** + * Extending classes should override `query` or implement this method + */ + async * getAll (options?: AbortOptions): AwaitIterable { // eslint-disable-line require-yield + throw new Error('.getAll is not implemented') + } +} diff --git a/packages/blockstore-core/src/errors.ts b/packages/blockstore-core/src/errors.ts new file mode 100644 index 00000000..d4533c91 --- /dev/null +++ b/packages/blockstore-core/src/errors.ts @@ -0,0 +1,41 @@ +import errCode from 'err-code' + +export function openFailedError (err?: Error): Error { + err = err ?? new Error('Open failed') + return errCode(err, 'ERR_OPEN_FAILED') +} + +export function closeFailedError (err?: Error): Error { + err = err ?? new Error('Close failed') + return errCode(err, 'ERR_CLOSE_FAILED') +} + +export function putFailedError (err?: Error): Error { + err = err ?? new Error('Put failed') + return errCode(err, 'ERR_PUT_FAILED') +} + +export function getFailedError (err?: Error): Error { + err = err ?? new Error('Get failed') + return errCode(err, 'ERR_GET_FAILED') +} + +export function deleteFailedError (err?: Error): Error { + err = err ?? new Error('Delete failed') + return errCode(err, 'ERR_DELETE_FAILED') +} + +export function hasFailedError (err?: Error): Error { + err = err ?? new Error('Has failed') + return errCode(err, 'ERR_HAS_FAILED') +} + +export function notFoundError (err?: Error): Error { + err = err ?? new Error('Not Found') + return errCode(err, 'ERR_NOT_FOUND') +} + +export function abortedError (err?: Error): Error { + err = err ?? new Error('Aborted') + return errCode(err, 'ERR_ABORTED') +} diff --git a/packages/blockstore-core/src/index.ts b/packages/blockstore-core/src/index.ts new file mode 100644 index 00000000..c8961594 --- /dev/null +++ b/packages/blockstore-core/src/index.ts @@ -0,0 +1,8 @@ +import * as ErrorsImport from './errors.js' + +export { BaseBlockstore } from './base.js' +export { MemoryBlockstore } from './memory.js' + +export const Errors = { + ...ErrorsImport +} diff --git a/packages/blockstore-core/src/memory.ts b/packages/blockstore-core/src/memory.ts new file mode 100644 index 00000000..68a0372b --- /dev/null +++ b/packages/blockstore-core/src/memory.ts @@ -0,0 +1,51 @@ +import { BaseBlockstore } from './base.js' +import { base32 } from 'multiformats/bases/base32' +import * as raw from 'multiformats/codecs/raw' +import { CID } from 'multiformats/cid' +import * as Digest from 'multiformats/hashes/digest' +import * as Errors from './errors.js' +import type { Await, AwaitIterable } from 'interface-store' +import type { Pair } from 'interface-blockstore' + +export class MemoryBlockstore extends BaseBlockstore { + private readonly data: Map + + constructor () { + super() + + this.data = new Map() + } + + put (key: CID, val: Uint8Array): Await { // eslint-disable-line require-await + this.data.set(base32.encode(key.multihash.bytes), val) + + return key + } + + get (key: CID): Await { + const buf = this.data.get(base32.encode(key.multihash.bytes)) + + if (buf == null) { + throw Errors.notFoundError() + } + + return buf + } + + has (key: CID): Await { + return this.data.has(base32.encode(key.multihash.bytes)) + } + + async delete (key: CID): Promise { + this.data.delete(base32.encode(key.multihash.bytes)) + } + + async * getAll (): AwaitIterable { + for (const [key, value] of this.data.entries()) { + yield { + cid: CID.createV1(raw.code, Digest.decode(base32.decode(key))), + block: value + } + } + } +} diff --git a/packages/blockstore-core/test/memory.spec.ts b/packages/blockstore-core/test/memory.spec.ts new file mode 100644 index 00000000..d42c548b --- /dev/null +++ b/packages/blockstore-core/test/memory.spec.ts @@ -0,0 +1,15 @@ +/* eslint-env mocha */ + +import { interfaceBlockstoreTests } from 'interface-blockstore-tests' +import { MemoryBlockstore } from '../src/memory.js' + +describe('memory', () => { + describe('interface-datastore', () => { + interfaceBlockstoreTests({ + setup () { + return new MemoryBlockstore() + }, + teardown () { } + }) + }) +}) diff --git a/packages/blockstore-core/tsconfig.json b/packages/blockstore-core/tsconfig.json new file mode 100644 index 00000000..73395b59 --- /dev/null +++ b/packages/blockstore-core/tsconfig.json @@ -0,0 +1,21 @@ +{ + "extends": "aegir/src/config/tsconfig.aegir.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": [ + "src", + "test" + ], + "references": [ + { + "path": "../interface-blockstore" + }, + { + "path": "../interface-blockstore-tests" + }, + { + "path": "../interface-store" + } + ] +} diff --git a/packages/blockstore-fs/CHANGELOG.md b/packages/blockstore-fs/CHANGELOG.md new file mode 100644 index 00000000..a85e2a6f --- /dev/null +++ b/packages/blockstore-fs/CHANGELOG.md @@ -0,0 +1,19 @@ +## [1.0.1](https://github.com/ipfs/js-blockstore-fs/compare/v1.0.0...v1.0.1) (2023-03-23) + + +### Dependencies + +* update interface-store to 5.x.x ([#1](https://github.com/ipfs/js-blockstore-fs/issues/1)) ([88fa91c](https://github.com/ipfs/js-blockstore-fs/commit/88fa91cb1405ed66f053ed265c1690ac0ad22214)) + +## 1.0.0 (2023-03-13) + + +### Bug Fixes + +* update ci build files ([bdc1881](https://github.com/ipfs/js-blockstore-fs/commit/bdc18810e6d63ffdbf6fc6617757aaa96b0ba82c)) + + +### Trivial Changes + +* add missing dep ([f5ba415](https://github.com/ipfs/js-blockstore-fs/commit/f5ba41536816f08eb1b97f298091c7221c6c9360)) +* initial import ([7a576cb](https://github.com/ipfs/js-blockstore-fs/commit/7a576cbad5696ad396c0cd2d557edf71d624a860)) diff --git a/packages/blockstore-fs/LICENSE b/packages/blockstore-fs/LICENSE new file mode 100644 index 00000000..20ce483c --- /dev/null +++ b/packages/blockstore-fs/LICENSE @@ -0,0 +1,4 @@ +This project is dual licensed under MIT and Apache-2.0. + +MIT: https://www.opensource.org/licenses/mit +Apache-2.0: https://www.apache.org/licenses/license-2.0 diff --git a/packages/blockstore-fs/LICENSE-APACHE b/packages/blockstore-fs/LICENSE-APACHE new file mode 100644 index 00000000..14478a3b --- /dev/null +++ b/packages/blockstore-fs/LICENSE-APACHE @@ -0,0 +1,5 @@ +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/packages/blockstore-fs/LICENSE-MIT b/packages/blockstore-fs/LICENSE-MIT new file mode 100644 index 00000000..72dc60d8 --- /dev/null +++ b/packages/blockstore-fs/LICENSE-MIT @@ -0,0 +1,19 @@ +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/packages/blockstore-fs/README.md b/packages/blockstore-fs/README.md new file mode 100644 index 00000000..62195de2 --- /dev/null +++ b/packages/blockstore-fs/README.md @@ -0,0 +1,53 @@ +# blockstore-fs + +[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) +[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/js-stores.svg?style=flat-square)](https://codecov.io/gh/ipfs/js-stores) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/js-stores/js-test-and-release.yml?branch=master\&style=flat-square)](https://github.com/ipfs/js-stores/actions/workflows/js-test-and-release.yml?query=branch%3Amaster) + +> Blockstore implementation with file system backend + +## Table of contents + +- [Install](#install) +- [Usage](#usage) +- [API Docs](#api-docs) +- [License](#license) +- [Contribute](#contribute) + +## Install + +```console +$ npm i blockstore-fs +``` + +## Usage + +```js +import { FSBlockstore } from 'blockstore-fs' + +const store = new FSBlockstore('path/to/store') +``` + +## API Docs + +- + +## License + +Licensed under either of + +- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) +- MIT ([LICENSE-MIT](LICENSE-MIT) / ) + +## Contribute + +Contributions welcome! Please check out [the issues](https://github.com/ipfs/js-stores/issues). + +Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. + +Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) diff --git a/packages/blockstore-fs/benchmarks/encoding/package.json b/packages/blockstore-fs/benchmarks/encoding/package.json new file mode 100644 index 00000000..00104d4f --- /dev/null +++ b/packages/blockstore-fs/benchmarks/encoding/package.json @@ -0,0 +1,18 @@ +{ + "name": "benchmarks-encoding", + "version": "1.0.0", + "main": "index.js", + "private": true, + "type": "module", + "scripts": { + "clean": "aegir clean", + "build": "aegir build --bundle false", + "lint": "aegir lint", + "dep-check": "aegir dep-check", + "start": "npm run build && node dist/src/index.js" + }, + "devDependencies": { + "multiformats": "^11.0.1", + "tinybench": "^2.4.0" + } +} diff --git a/packages/blockstore-fs/benchmarks/encoding/src/README.md b/packages/blockstore-fs/benchmarks/encoding/src/README.md new file mode 100644 index 00000000..3caf1e9e --- /dev/null +++ b/packages/blockstore-fs/benchmarks/encoding/src/README.md @@ -0,0 +1,31 @@ +# Encoding Benchmark + +Multiformats ships a number of base encoding algorithms. This module has no strong opinion +on which is best, as long as it is case insensitive, so benchmark them to choose the fastest. + +At the time of writing `base8` is the fastest, followed other alorithms using `rfc4648` encoding +internally in `multiformats` (e.g. `base16`, `base32`), and finally anything using `baseX` encoding. + +We choose base32upper which uses `rfc4648` because it has a longer alphabet so will shard better. + +## Usage + +```console +$ npm i +$ npm start + +> benchmarks-gc@1.0.0 start +> npm run build && node dist/src/index.js + + +> benchmarks-gc@1.0.0 build +> aegir build --bundle false + +[14:51:28] tsc [started] +[14:51:33] tsc [completed] +generating Ed25519 keypair... +┌─────────┬────────────────┬─────────┬───────────┬──────┐ +│ (index) │ Implementation │ ops/s │ ms/op │ runs │ +├─────────┼────────────────┼─────────┼───────────┼──────┤ +//... results here +``` diff --git a/packages/blockstore-fs/benchmarks/encoding/src/index.ts b/packages/blockstore-fs/benchmarks/encoding/src/index.ts new file mode 100644 index 00000000..6bb13797 --- /dev/null +++ b/packages/blockstore-fs/benchmarks/encoding/src/index.ts @@ -0,0 +1,73 @@ +import { Bench } from 'tinybench' +import { CID } from 'multiformats/cid' +import { base8 } from 'multiformats/bases/base8' +import { base10 } from 'multiformats/bases/base10' +import { base16upper } from 'multiformats/bases/base16' +import { base32, base32upper, base32hexupper, base32z } from 'multiformats/bases/base32' +import { base36, base36upper } from 'multiformats/bases/base36' +import { base256emoji } from 'multiformats/bases/base256emoji' + +const RESULT_PRECISION = 2 + +const cid = CID.parse('QmeimKZyjcBnuXmAD9zMnSjM9JodTbgGT3gutofkTqz9rE') + +async function main (): Promise { + const suite = new Bench() + suite.add('base8', () => { + base8.encode(cid.bytes) + }) + suite.add('base10', () => { + base10.encode(cid.bytes) + }) + suite.add('base16upper', () => { + base16upper.encode(cid.bytes) + }) + suite.add('base32', () => { + base32.encode(cid.bytes) + }) + suite.add('base32upper', () => { + base32upper.encode(cid.bytes) + }) + suite.add('base32hexupper', () => { + base32hexupper.encode(cid.bytes) + }) + suite.add('base32z', () => { + base32z.encode(cid.bytes) + }) + suite.add('base36', () => { + base36.encode(cid.bytes) + }) + suite.add('base36upper', () => { + base36upper.encode(cid.bytes) + }) + suite.add('base256emoji', () => { + base256emoji.encode(cid.bytes) + }) + + await suite.run() + + console.table(suite.tasks.sort((a, b) => { // eslint-disable-line no-console + const resultA = a.result?.hz ?? 0 + const resultB = b.result?.hz ?? 0 + + if (resultA === resultB) { + return 0 + } + + if (resultA < resultB) { + return 1 + } + + return -1 + }).map(({ name, result }) => ({ + Implementation: name, + 'ops/s': parseFloat(result?.hz.toFixed(RESULT_PRECISION) ?? '0'), + 'ms/op': parseFloat(result?.period.toFixed(RESULT_PRECISION) ?? '0'), + runs: result?.samples.length + }))) +} + +main().catch(err => { + console.error(err) // eslint-disable-line no-console + process.exit(1) +}) diff --git a/packages/blockstore-fs/benchmarks/encoding/tsconfig.json b/packages/blockstore-fs/benchmarks/encoding/tsconfig.json new file mode 100644 index 00000000..fee64009 --- /dev/null +++ b/packages/blockstore-fs/benchmarks/encoding/tsconfig.json @@ -0,0 +1,13 @@ +{ + "extends": "aegir/src/config/tsconfig.aegir.json", + "compilerOptions": { + "outDir": "dist", + "target": "ES2022", + "module": "ES2022", + "lib": ["ES2022", "DOM", "DOM.Iterable"] + }, + "include": [ + "src", + "test" + ] +} diff --git a/packages/blockstore-fs/package.json b/packages/blockstore-fs/package.json new file mode 100644 index 00000000..bc607104 --- /dev/null +++ b/packages/blockstore-fs/package.json @@ -0,0 +1,182 @@ +{ + "name": "blockstore-fs", + "version": "1.0.1", + "description": "Blockstore implementation with file system backend", + "license": "Apache-2.0 OR MIT", + "homepage": "https://github.com/ipfs/js-stores/tree/master/packages/blockstore-fs#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/ipfs/js-stores.git" + }, + "bugs": { + "url": "https://github.com/ipfs/js-stores/issues" + }, + "keywords": [ + "datastore", + "fs", + "interface", + "ipfs", + "key-value" + ], + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + }, + "type": "module", + "types": "./dist/src/index.d.ts", + "typesVersions": { + "*": { + "*": [ + "*", + "dist/*", + "dist/src/*", + "dist/src/*/index" + ], + "src/*": [ + "*", + "dist/*", + "dist/src/*", + "dist/src/*/index" + ] + } + }, + "files": [ + "src", + "dist", + "!dist/test", + "!**/*.tsbuildinfo" + ], + "exports": { + ".": { + "types": "./dist/src/index.d.ts", + "import": "./dist/src/index.js" + }, + "./sharding": { + "types": "./dist/src/sharding.d.ts", + "import": "./dist/src/sharding.js" + } + }, + "eslintConfig": { + "extends": "ipfs", + "parserOptions": { + "sourceType": "module", + "project": [ + "tsconfig.json", + "benchmarks/encoding/tsconfig.json" + ] + } + }, + "release": { + "branches": [ + "master" + ], + "plugins": [ + [ + "@semantic-release/commit-analyzer", + { + "preset": "conventionalcommits", + "releaseRules": [ + { + "breaking": true, + "release": "major" + }, + { + "revert": true, + "release": "patch" + }, + { + "type": "feat", + "release": "minor" + }, + { + "type": "fix", + "release": "patch" + }, + { + "type": "docs", + "release": "patch" + }, + { + "type": "test", + "release": "patch" + }, + { + "type": "deps", + "release": "patch" + }, + { + "scope": "no-release", + "release": false + } + ] + } + ], + [ + "@semantic-release/release-notes-generator", + { + "preset": "conventionalcommits", + "presetConfig": { + "types": [ + { + "type": "feat", + "section": "Features" + }, + { + "type": "fix", + "section": "Bug Fixes" + }, + { + "type": "chore", + "section": "Trivial Changes" + }, + { + "type": "docs", + "section": "Documentation" + }, + { + "type": "deps", + "section": "Dependencies" + }, + { + "type": "test", + "section": "Tests" + } + ] + } + } + ], + "@semantic-release/changelog", + "@semantic-release/npm", + "@semantic-release/github", + "@semantic-release/git" + ] + }, + "scripts": { + "clean": "aegir clean", + "lint": "aegir lint", + "build": "aegir build --bundle false", + "release": "aegir release", + "test": "aegir test -t node -t electron-main", + "test:node": "aegir test -t node", + "test:electron-main": "aegir test -t electron-main", + "dep-check": "aegir dep-check", + "docs": "aegir docs" + }, + "dependencies": { + "blockstore-core": "^4.0.0", + "fast-write-atomic": "^0.2.0", + "interface-blockstore": "^5.0.0", + "interface-store": "^5.0.0", + "it-glob": "^2.0.1", + "it-map": "^2.0.1", + "it-parallel-batch": "^2.0.1", + "multiformats": "^11.0.2" + }, + "devDependencies": { + "aegir": "^38.1.7", + "interface-blockstore-tests": "^6.0.0" + }, + "typedoc": { + "entryPoint": "./src/index.ts" + } +} diff --git a/packages/blockstore-fs/src/index.ts b/packages/blockstore-fs/src/index.ts new file mode 100644 index 00000000..ff02d92e --- /dev/null +++ b/packages/blockstore-fs/src/index.ts @@ -0,0 +1,256 @@ +import fs from 'node:fs/promises' +import glob from 'it-glob' +import path from 'node:path' +import { promisify } from 'node:util' +import { + Errors +} from 'blockstore-core' +import map from 'it-map' +import parallelBatch from 'it-parallel-batch' +// @ts-expect-error no types +import fwa from 'fast-write-atomic' +import type { CID } from 'multiformats/cid' +import type { Blockstore, Pair } from 'interface-blockstore' +import type { AwaitIterable } from 'interface-store' +import { NextToLast, ShardingStrategy } from './sharding.js' + +const writeAtomic = promisify(fwa) + +/** + * Write a file atomically + */ +async function writeFile (file: string, contents: Uint8Array): Promise { + try { + await writeAtomic(file, contents) + } catch (err: any) { + if (err.code === 'EPERM' && err.syscall === 'rename') { + // fast-write-atomic writes a file to a temp location before renaming it. + // On Windows, if the final file already exists this error is thrown. + // No such error is thrown on Linux/Mac + // Make sure we can read & write to this file + await fs.access(file, fs.constants.F_OK | fs.constants.W_OK) + + // The file was created by another context - this means there were + // attempts to write the same block by two different function calls + return + } + + throw err + } +} + +export interface FsBlockstoreInit { + /** + * If true and the passed blockstore location does not exist, create + * it on startup. default: true + */ + createIfMissing?: boolean + + /** + * If true and the passed blockstore location exists on startup, throw + * an error. default: false + */ + errorIfExists?: boolean + + /** + * The file extension to use when storing blocks. default: '.data' + */ + extension?: string + + /** + * How many blocks to put in parallel when `.putMany` is called. + * default: 50 + */ + putManyConcurrency?: number + + /** + * How many blocks to read in parallel when `.getMany` is called. + * default: 50 + */ + getManyConcurrency?: number + + /** + * How many blocks to delete in parallel when `.deleteMany` is called. + * default: 50 + */ + deleteManyConcurrency?: number + + /** + * Control how CIDs map to paths and back + */ + shardingStrategy?: ShardingStrategy +} + +/** + * A blockstore backed by the file system + */ +export class FsBlockstore implements Blockstore { + public path: string + private readonly createIfMissing: boolean + private readonly errorIfExists: boolean + private readonly putManyConcurrency: number + private readonly getManyConcurrency: number + private readonly deleteManyConcurrency: number + private readonly shardingStrategy: ShardingStrategy + + constructor (location: string, init: FsBlockstoreInit = {}) { + this.path = path.resolve(location) + this.createIfMissing = init.createIfMissing ?? true + this.errorIfExists = init.errorIfExists ?? false + this.deleteManyConcurrency = init.deleteManyConcurrency ?? 50 + this.getManyConcurrency = init.getManyConcurrency ?? 50 + this.putManyConcurrency = init.putManyConcurrency ?? 50 + this.shardingStrategy = init.shardingStrategy ?? new NextToLast() + } + + async open (): Promise { + try { + await fs.access(this.path, fs.constants.F_OK | fs.constants.W_OK) + + if (this.errorIfExists) { + throw Errors.openFailedError(new Error(`Blockstore directory: ${this.path} already exists`)) + } + + return + } catch (err: any) { + if (err.code === 'ENOENT') { + if (this.createIfMissing) { + await fs.mkdir(this.path, { recursive: true }) + return + } else { + throw Errors.openFailedError(new Error(`Blockstore directory: ${this.path} does not exist`)) + } + } + + throw err + } + } + + async close (): Promise { + await Promise.resolve() + } + + async put (key: CID, val: Uint8Array): Promise { + const { dir, file } = this.shardingStrategy.encode(key) + + try { + if (dir != null && dir !== '') { + await fs.mkdir(path.join(this.path, dir), { + recursive: true + }) + } + + await writeFile(path.join(this.path, dir, file), val) + + return key + } catch (err: any) { + throw Errors.putFailedError(err) + } + } + + async * putMany (source: AwaitIterable): AsyncIterable { + yield * parallelBatch( + map(source, ({ cid, block }) => { + return async () => { + await this.put(cid, block) + + return cid + } + }), + this.putManyConcurrency + ) + } + + async get (key: CID): Promise { + const { dir, file } = this.shardingStrategy.encode(key) + + try { + return await fs.readFile(path.join(this.path, dir, file)) + } catch (err: any) { + throw Errors.notFoundError(err) + } + } + + async * getMany (source: AwaitIterable): AsyncIterable { + yield * parallelBatch( + map(source, key => { + return async () => { + return { + cid: key, + block: await this.get(key) + } + } + }), + this.getManyConcurrency + ) + } + + async delete (key: CID): Promise { + const { dir, file } = this.shardingStrategy.encode(key) + + try { + await fs.unlink(path.join(this.path, dir, file)) + } catch (err: any) { + if (err.code === 'ENOENT') { + return + } + + throw Errors.deleteFailedError(err) + } + } + + async * deleteMany (source: AwaitIterable): AsyncIterable { + yield * parallelBatch( + map(source, key => { + return async () => { + await this.delete(key) + + return key + } + }), + this.deleteManyConcurrency + ) + } + + /** + * Check for the existence of the given key + */ + async has (key: CID): Promise { + const { dir, file } = this.shardingStrategy.encode(key) + + try { + await fs.access(path.join(this.path, dir, file)) + } catch (err: any) { + return false + } + return true + } + + async * getAll (): AsyncIterable { + const pattern = `**/*${this.shardingStrategy.extension}` + .split(path.sep) + .join('/') + const files = glob(this.path, pattern, { + absolute: true + }) + + for await (const file of files) { + try { + const buf = await fs.readFile(file) + + const pair: Pair = { + cid: this.shardingStrategy.decode(file), + block: buf + } + + yield pair + } catch (err: any) { + // if keys are removed from the datastore while the query is + // running, we may encounter missing files. + if (err.code !== 'ENOENT') { + throw err + } + } + } + } +} diff --git a/packages/blockstore-fs/src/sharding.ts b/packages/blockstore-fs/src/sharding.ts new file mode 100644 index 00000000..5742390f --- /dev/null +++ b/packages/blockstore-fs/src/sharding.ts @@ -0,0 +1,117 @@ +import path from 'node:path' +import { CID } from 'multiformats/cid' +import { base32upper } from 'multiformats/bases/base32' +import type { MultibaseCodec } from 'multiformats/bases/interface' + +export interface ShardingStrategy { + extension: string + encode: (cid: CID) => { dir: string, file: string } + decode: (path: string) => CID +} + +export interface NextToLastInit { + /** + * The file extension to use. default: '.data' + */ + extension?: string + + /** + * How many characters to take from the end of the CID. default: 2 + */ + prefixLength?: number + + /** + * The multibase codec to use - nb. should be case insensitive. + * default: base32upper + */ + base?: MultibaseCodec +} + +/** + * A sharding strategy that takes the last few characters of a multibase encoded + * CID and uses them as the directory to store the block in. This prevents + * storing all blocks in a single directory which would overwhelm most + * filesystems. + */ +export class NextToLast implements ShardingStrategy { + public extension: string + private readonly prefixLength: number + private readonly base: MultibaseCodec + + constructor (init: NextToLastInit = {}) { + this.extension = init.extension ?? '.data' + this.prefixLength = init.prefixLength ?? 2 + this.base = init.base ?? base32upper + } + + encode (cid: CID): { dir: string, file: string } { + const str = this.base.encoder.encode(cid.multihash.bytes) + const prefix = str.substring(str.length - this.prefixLength) + + return { + dir: prefix, + file: `${str}${this.extension}` + } + } + + decode (str: string): CID { + let fileName = path.basename(str) + + if (fileName.endsWith(this.extension)) { + fileName = fileName.substring(0, fileName.length - this.extension.length) + } + + return CID.decode(this.base.decoder.decode(fileName)) + } +} + +export interface FlatDirectoryInit { + /** + * The file extension to use. default: '.data' + */ + extension?: string + + /** + * How many characters to take from the end of the CID. default: 2 + */ + prefixLength?: number + + /** + * The multibase codec to use - nb. should be case insensitive. + * default: base32padupper + */ + base?: MultibaseCodec +} + +/** + * A sharding strategy that does not do any sharding and stores all files + * in one directory. Only for testing, do not use in production. + */ +export class FlatDirectory implements ShardingStrategy { + public extension: string + private readonly base: MultibaseCodec + + constructor (init: NextToLastInit = {}) { + this.extension = init.extension ?? '.data' + this.base = init.base ?? base32upper + } + + encode (cid: CID): { dir: string, file: string } { + const str = this.base.encoder.encode(cid.multihash.bytes) + + return { + dir: '', + file: `${str}${this.extension}` + } + } + + decode (str: string): CID { + let fileName = path.basename(str) + + if (fileName.endsWith(this.extension)) { + fileName = fileName.substring(0, fileName.length - this.extension.length) + } + + return CID.decode(this.base.decoder.decode(fileName)) + } +} diff --git a/packages/blockstore-fs/test/index.spec.ts b/packages/blockstore-fs/test/index.spec.ts new file mode 100644 index 00000000..d2dfe45d --- /dev/null +++ b/packages/blockstore-fs/test/index.spec.ts @@ -0,0 +1,161 @@ +/* eslint-env mocha */ +import { expect } from 'aegir/chai' +import path from 'node:path' +import os from 'node:os' +import fs from 'node:fs/promises' +import { CID } from 'multiformats/cid' +import { interfaceBlockstoreTests } from 'interface-blockstore-tests' +import { FsBlockstore } from '../src/index.js' +import { FlatDirectory, NextToLast } from '../src/sharding.js' +import { base256emoji } from 'multiformats/bases/base256emoji' + +const utf8Encoder = new TextEncoder() + +describe('FsBlockstore', () => { + describe('construction', () => { + it('defaults - folder missing', async () => { + const dir = path.join(os.tmpdir(), `test-${Math.random()}`) + await expect( + (async () => { + const fs = new FsBlockstore(dir) + await fs.open() + await fs.close() + })() + ).to.eventually.be.undefined() + }) + + it('defaults - folder exists', async () => { + const dir = path.join(os.tmpdir(), `test-${Math.random()}`) + await fs.mkdir(dir, { + recursive: true + }) + await expect( + (async () => { + const fs = new FsBlockstore(dir) + await fs.open() + await fs.close() + })() + ).to.eventually.be.undefined() + }) + }) + + describe('open', () => { + it('createIfMissing: false - folder missing', async () => { + const dir = path.join(os.tmpdir(), `test-${Math.random()}`) + const store = new FsBlockstore(dir, { createIfMissing: false }) + await expect(store.open()).to.eventually.be.rejected + .with.property('code', 'ERR_OPEN_FAILED') + }) + + it('errorIfExists: true - folder exists', async () => { + const dir = path.join(os.tmpdir(), `test-${Math.random()}`) + await fs.mkdir(dir, { + recursive: true + }) + const store = new FsBlockstore(dir, { errorIfExists: true }) + await expect(store.open()).to.eventually.be.rejected + .with.property('code', 'ERR_OPEN_FAILED') + }) + }) + + it('deleting files', async () => { + const dir = path.join(os.tmpdir(), `test-${Math.random()}`) + const fs = new FsBlockstore(dir) + await fs.open() + + const key = CID.parse('QmeimKZyjcBnuXmAD9zMnSjM9JodTbgGT3gutofkTqz9rE') + await fs.put(key, Uint8Array.from([0, 1, 2, 3])) + await fs.delete(key) + + await expect(fs.get(key)).to.eventually.be.rejected + .with.property('code', 'ERR_NOT_FOUND') + }) + + it('deleting non-existent files', async () => { + const dir = path.join(os.tmpdir(), `test-${Math.random()}`) + const fs = new FsBlockstore(dir) + await fs.open() + + const key = CID.parse('QmeimKZyjcBnuXmAD9zMnSjM9JodTbgGT3gutofkTqz9rE') + + await fs.delete(key) + + await expect(fs.get(key)).to.eventually.be.rejected + .with.property('code', 'ERR_NOT_FOUND') + }) + + describe('interface-datastore (flat directory)', () => { + interfaceBlockstoreTests({ + setup: async () => { + const store = new FsBlockstore(path.join(os.tmpdir(), `test-${Math.random()}`), { + shardingStrategy: new FlatDirectory() + }) + await store.open() + + return store + }, + teardown: async (store) => { + await store.close() + await fs.rm(store.path, { + recursive: true + }) + } + }) + }) + + describe('interface-datastore (default sharding)', () => { + interfaceBlockstoreTests({ + setup: async () => { + const store = new FsBlockstore(path.join(os.tmpdir(), `test-${Math.random()}`)) + await store.open() + + return store + }, + teardown: async (store) => { + await store.close() + await fs.rm(store.path, { + recursive: true + }) + } + }) + }) + + describe('interface-datastore (custom encoding)', () => { + interfaceBlockstoreTests({ + setup: async () => { + const store = new FsBlockstore(path.join(os.tmpdir(), `test-${Math.random()}`), { + shardingStrategy: new NextToLast({ + base: base256emoji + }) + }) + + await store.open() + + return store + }, + teardown: async (store) => { + await store.close() + await fs.rm(store.path, { + recursive: true + }) + } + }) + }) + + it('can survive concurrent writes', async () => { + const dir = path.join(os.tmpdir(), `test-${Math.random()}`) + const fs = new FsBlockstore(dir) + await fs.open() + + const key = CID.parse('QmeimKZyjcBnuXmAD9zMnSjM9JodTbgGT3gutofkTqz9rE') + const value = utf8Encoder.encode('Hello world') + + await Promise.all( + new Array(100).fill(0).map(async () => { await fs.put(key, value) }) + ) + + const res = await fs.get(key) + + expect(res).to.deep.equal(value) + }) +}) diff --git a/packages/blockstore-fs/test/sharding.spec.ts b/packages/blockstore-fs/test/sharding.spec.ts new file mode 100644 index 00000000..99253b3b --- /dev/null +++ b/packages/blockstore-fs/test/sharding.spec.ts @@ -0,0 +1,109 @@ +/* eslint-env mocha */ +import { expect } from 'aegir/chai' +import { CID } from 'multiformats/cid' +import { FlatDirectory, NextToLast } from '../src/sharding.js' +import { base32upper } from 'multiformats/bases/base32' + +describe('flat', () => { + it('should encode', () => { + const cid = CID.parse('QmeimKZyjcBnuXmAD9zMnSjM9JodTbgGT3gutofkTqz9rE') + const strategy = new FlatDirectory() + const { dir, file } = strategy.encode(cid) + + expect(dir).to.equal('') + expect(file).to.equal(`${base32upper.encode(cid.multihash.bytes)}.data`) + }) + + it('should encode with extension', () => { + const cid = CID.parse('QmeimKZyjcBnuXmAD9zMnSjM9JodTbgGT3gutofkTqz9rE') + const strategy = new FlatDirectory({ + extension: '.file' + }) + const { dir, file } = strategy.encode(cid) + + expect(dir).to.equal('') + expect(file).to.equal(`${base32upper.encode(cid.multihash.bytes)}.file`) + }) + + it('should decode', () => { + const mh = 'BCIQPGZJ6QLZOFG3OP45NLMSJUWGJCO72QQKHLDTB6FXIB6BDSLRQYLY' + const strategy = new FlatDirectory() + const cid = strategy.decode(`${mh}.data`) + + expect(cid).to.eql(CID.decode(base32upper.decode(mh))) + }) + + it('should decode with extension', () => { + const mh = 'BCIQPGZJ6QLZOFG3OP45NLMSJUWGJCO72QQKHLDTB6FXIB6BDSLRQYLY' + const strategy = new FlatDirectory({ + extension: '.file' + }) + const cid = strategy.decode(`${mh}.file`) + + expect(cid).to.eql(CID.decode(base32upper.decode(mh))) + }) +}) + +describe('next to last', () => { + it('should encode', () => { + const mh = 'BCIQPGZJ6QLZOFG3OP45NLMSJUWGJCO72QQKHLDTB6FXIB6BDSLRQYLY' + const cid = CID.decode(base32upper.decode(mh)) + const strategy = new NextToLast() + const { dir, file } = strategy.encode(cid) + + expect(dir).to.equal('LY') + expect(file).to.equal(`${mh}.data`) + }) + + it('should encode with prefix length', () => { + const mh = 'BCIQPGZJ6QLZOFG3OP45NLMSJUWGJCO72QQKHLDTB6FXIB6BDSLRQYLY' + const cid = CID.decode(base32upper.decode(mh)) + const strategy = new NextToLast({ + prefixLength: 4 + }) + const { dir, file } = strategy.encode(cid) + + expect(dir).to.equal('QYLY') + expect(file).to.equal(`${mh}.data`) + }) + + it('should encode with extension', () => { + const mh = 'BCIQPGZJ6QLZOFG3OP45NLMSJUWGJCO72QQKHLDTB6FXIB6BDSLRQYLY' + const cid = CID.decode(base32upper.decode(mh)) + const strategy = new NextToLast({ + extension: '.file' + }) + const { dir, file } = strategy.encode(cid) + + expect(dir).to.equal('LY') + expect(file).to.equal(`${mh}.file`) + }) + + it('should decode', () => { + const mh = 'BCIQPGZJ6QLZOFG3OP45NLMSJUWGJCO72QQKHLDTB6FXIB6BDSLRQYLY' + const strategy = new NextToLast() + const cid = strategy.decode(`LY/${mh}.data`) + + expect(cid).to.eql(CID.decode(base32upper.decode(mh))) + }) + + it('should decode with prefix length', () => { + const mh = 'BCIQPGZJ6QLZOFG3OP45NLMSJUWGJCO72QQKHLDTB6FXIB6BDSLRQYLY' + const strategy = new NextToLast({ + prefixLength: 4 + }) + const cid = strategy.decode(`QYLY/${mh}.data`) + + expect(cid).to.eql(CID.decode(base32upper.decode(mh))) + }) + + it('should decode with extension', () => { + const mh = 'BCIQPGZJ6QLZOFG3OP45NLMSJUWGJCO72QQKHLDTB6FXIB6BDSLRQYLY' + const strategy = new NextToLast({ + extension: '.file' + }) + const cid = strategy.decode(`LY/${mh}.file`) + + expect(cid).to.eql(CID.decode(base32upper.decode(mh))) + }) +}) diff --git a/packages/blockstore-fs/tsconfig.json b/packages/blockstore-fs/tsconfig.json new file mode 100644 index 00000000..e27c7fa6 --- /dev/null +++ b/packages/blockstore-fs/tsconfig.json @@ -0,0 +1,24 @@ +{ + "extends": "aegir/src/config/tsconfig.aegir.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": [ + "src", + "test" + ], + "references": [ + { + "path": "../blockstore-core" + }, + { + "path": "../interface-blockstore" + }, + { + "path": "../interface-blockstore-tests" + }, + { + "path": "../interface-store" + } + ] +} diff --git a/packages/blockstore-idb/CHANGELOG.md b/packages/blockstore-idb/CHANGELOG.md new file mode 100644 index 00000000..11b85b8f --- /dev/null +++ b/packages/blockstore-idb/CHANGELOG.md @@ -0,0 +1,20 @@ +## [1.0.2](https://github.com/ipfs/js-blockstore-idb/compare/v1.0.1...v1.0.2) (2023-03-23) + + +### Dependencies + +* update interface-store to 5.x.x ([#2](https://github.com/ipfs/js-blockstore-idb/issues/2)) ([0bc838f](https://github.com/ipfs/js-blockstore-idb/commit/0bc838f082483861bc9ebe4bd0272fc746724ccf)) + +## [1.0.1](https://github.com/ipfs/js-blockstore-idb/compare/v1.0.0...v1.0.1) (2023-03-14) + + +### Bug Fixes + +* use getKey for has ([#1](https://github.com/ipfs/js-blockstore-idb/issues/1)) ([a0b7563](https://github.com/ipfs/js-blockstore-idb/commit/a0b75638655a55270cea7eff6df43e06d086b538)) + +## 1.0.0 (2023-03-14) + + +### Features + +* initial import ([d837104](https://github.com/ipfs/js-blockstore-idb/commit/d837104d2213a5914ab3dc3e8e4c022a69f7003f)) diff --git a/packages/blockstore-idb/LICENSE b/packages/blockstore-idb/LICENSE new file mode 100644 index 00000000..20ce483c --- /dev/null +++ b/packages/blockstore-idb/LICENSE @@ -0,0 +1,4 @@ +This project is dual licensed under MIT and Apache-2.0. + +MIT: https://www.opensource.org/licenses/mit +Apache-2.0: https://www.apache.org/licenses/license-2.0 diff --git a/packages/blockstore-idb/LICENSE-APACHE b/packages/blockstore-idb/LICENSE-APACHE new file mode 100644 index 00000000..14478a3b --- /dev/null +++ b/packages/blockstore-idb/LICENSE-APACHE @@ -0,0 +1,5 @@ +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/packages/blockstore-idb/LICENSE-MIT b/packages/blockstore-idb/LICENSE-MIT new file mode 100644 index 00000000..72dc60d8 --- /dev/null +++ b/packages/blockstore-idb/LICENSE-MIT @@ -0,0 +1,19 @@ +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/packages/blockstore-idb/README.md b/packages/blockstore-idb/README.md new file mode 100644 index 00000000..784072c8 --- /dev/null +++ b/packages/blockstore-idb/README.md @@ -0,0 +1,53 @@ +# blockstore-idb + +[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) +[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/js-stores.svg?style=flat-square)](https://codecov.io/gh/ipfs/js-stores) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/js-stores/js-test-and-release.yml?branch=master\&style=flat-square)](https://github.com/ipfs/js-stores/actions/workflows/js-test-and-release.yml?query=branch%3Amaster) + +> Blockstore implementation with IndexedDB backend + +## Table of contents + +- [Install](#install) + - [Browser ` +``` + +## API Docs + +- + +## License + +Licensed under either of + +- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) +- MIT ([LICENSE-MIT](LICENSE-MIT) / ) + +## Contribute + +Contributions welcome! Please check out [the issues](https://github.com/ipfs/js-stores/issues). + +Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. + +Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) diff --git a/packages/blockstore-idb/package.json b/packages/blockstore-idb/package.json new file mode 100644 index 00000000..4189f418 --- /dev/null +++ b/packages/blockstore-idb/package.json @@ -0,0 +1,158 @@ +{ + "name": "blockstore-idb", + "version": "1.0.2", + "description": "Blockstore implementation with IndexedDB backend", + "license": "Apache-2.0 OR MIT", + "homepage": "https://github.com/ipfs/js-stores/tree/master/packages/blockstore-idb#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/ipfs/js-stores.git" + }, + "bugs": { + "url": "https://github.com/ipfs/js-stores/issues" + }, + "keywords": [ + "browser", + "datastore", + "idb", + "interface", + "ipfs", + "key-value" + ], + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + }, + "type": "module", + "types": "./dist/src/index.d.ts", + "files": [ + "src", + "dist", + "!dist/test", + "!**/*.tsbuildinfo" + ], + "exports": { + ".": { + "types": "./dist/src/index.d.ts", + "import": "./dist/src/index.js" + } + }, + "eslintConfig": { + "extends": "ipfs", + "parserOptions": { + "sourceType": "module" + } + }, + "release": { + "branches": [ + "master" + ], + "plugins": [ + [ + "@semantic-release/commit-analyzer", + { + "preset": "conventionalcommits", + "releaseRules": [ + { + "breaking": true, + "release": "major" + }, + { + "revert": true, + "release": "patch" + }, + { + "type": "feat", + "release": "minor" + }, + { + "type": "fix", + "release": "patch" + }, + { + "type": "docs", + "release": "patch" + }, + { + "type": "test", + "release": "patch" + }, + { + "type": "deps", + "release": "patch" + }, + { + "scope": "no-release", + "release": false + } + ] + } + ], + [ + "@semantic-release/release-notes-generator", + { + "preset": "conventionalcommits", + "presetConfig": { + "types": [ + { + "type": "feat", + "section": "Features" + }, + { + "type": "fix", + "section": "Bug Fixes" + }, + { + "type": "chore", + "section": "Trivial Changes" + }, + { + "type": "docs", + "section": "Documentation" + }, + { + "type": "deps", + "section": "Dependencies" + }, + { + "type": "test", + "section": "Tests" + } + ] + } + } + ], + "@semantic-release/changelog", + "@semantic-release/npm", + "@semantic-release/github", + "@semantic-release/git" + ] + }, + "scripts": { + "clean": "aegir clean", + "lint": "aegir lint", + "build": "aegir build", + "release": "aegir release", + "test": "aegir test -t browser", + "test:chrome": "aegir test -t browser", + "test:chrome-webworker": "aegir test -t webworker", + "test:firefox": "aegir test -t browser -- --browser firefox", + "test:firefox-webworker": "aegir test -t webworker -- --browser firefox", + "dep-check": "aegir dep-check", + "docs": "aegir docs" + }, + "dependencies": { + "blockstore-core": "^4.0.0", + "idb": "^7.1.1", + "interface-blockstore": "^5.0.0", + "interface-store": "^5.0.0", + "multiformats": "^11.0.2" + }, + "devDependencies": { + "aegir": "^38.1.7", + "interface-blockstore-tests": "^6.0.0" + }, + "typedoc": { + "entryPoint": "./src/index.ts" + } +} diff --git a/packages/blockstore-idb/src/index.ts b/packages/blockstore-idb/src/index.ts new file mode 100644 index 00000000..82316624 --- /dev/null +++ b/packages/blockstore-idb/src/index.ts @@ -0,0 +1,148 @@ +import { openDB, IDBPDatabase, deleteDB } from 'idb' +import { + BaseBlockstore, + Errors +} from 'blockstore-core' +import { CID } from 'multiformats/cid' +import type { MultibaseCodec } from 'multiformats/bases/interface' +import { base32upper } from 'multiformats/bases/base32' +import * as raw from 'multiformats/codecs/raw' +import * as Digest from 'multiformats/hashes/digest' +import type { Pair } from 'interface-blockstore' +import type { AbortOptions, AwaitIterable } from 'interface-store' + +export interface IDBDatastoreInit { + /** + * A prefix to use for all database keys (default: '') + */ + prefix?: string + + /** + * The database version (default: 1) + */ + version?: number + + /** + * The multibase codec to use - nb. should be case insensitive. + * (default: base32upper) + */ + base?: MultibaseCodec +} + +export class IDBBlockstore extends BaseBlockstore { + private readonly location: string + private readonly version: number + private db?: IDBPDatabase + private readonly base: MultibaseCodec + + constructor (location: string, init: IDBDatastoreInit = {}) { + super() + + this.location = `${init.prefix ?? ''}${location}` + this.version = init.version ?? 1 + + // this.transactionQueue = new PQueue({ concurrency: 1 }) + this.base = init.base ?? base32upper + } + + #encode (cid: CID): string { + return `/${this.base.encoder.encode(cid.multihash.bytes)}` + } + + #decode (key: string): CID { + return CID.createV1(raw.code, Digest.decode(this.base.decoder.decode(key.substring(1)))) + } + + async open (): Promise { + try { + const location = this.location + + this.db = await openDB(location, this.version, { + upgrade (db) { + db.createObjectStore(location) + } + }) + } catch (err: any) { + throw Errors.openFailedError(err) + } + } + + async close (): Promise { + this.db?.close() + } + + async put (key: CID, val: Uint8Array): Promise { + if (this.db == null) { + throw new Error('Blockstore needs to be opened.') + } + + try { + await this.db.put(this.location, val, this.#encode(key)) + + return key + } catch (err: any) { + throw Errors.putFailedError(err) + } + } + + async get (key: CID): Promise { + if (this.db == null) { + throw new Error('Blockstore needs to be opened.') + } + + let val: Uint8Array | undefined + + try { + val = await this.db.get(this.location, this.#encode(key)) + } catch (err: any) { + throw Errors.putFailedError(err) + } + + if (val === undefined) { + throw Errors.notFoundError() + } + + return val + } + + async delete (key: CID): Promise { + if (this.db == null) { + throw new Error('Blockstore needs to be opened.') + } + + try { + await this.db.delete(this.location, this.#encode(key)) + } catch (err: any) { + throw Errors.putFailedError(err) + } + } + + async has (key: CID): Promise { + if (this.db == null) { + throw new Error('Blockstore needs to be opened.') + } + + try { + return Boolean(await this.db.getKey(this.location, this.#encode(key))) + } catch (err: any) { + throw Errors.putFailedError(err) + } + } + + async * getAll (options?: AbortOptions): AwaitIterable { + if (this.db == null) { + throw new Error('Blockstore needs to be opened.') + } + + for (const key of await this.db.getAllKeys(this.location)) { + const cid = this.#decode(key.toString()) // eslint-disable-line @typescript-eslint/no-base-to-string + const block = await this.get(cid) + + yield { cid, block } + } + } + + async destroy (): Promise { + await deleteDB(this.location) + } +} diff --git a/packages/blockstore-idb/test/index.spec.ts b/packages/blockstore-idb/test/index.spec.ts new file mode 100644 index 00000000..51697205 --- /dev/null +++ b/packages/blockstore-idb/test/index.spec.ts @@ -0,0 +1,89 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import { IDBBlockstore } from '../src/index.js' +import { CID } from 'multiformats/cid' +import { interfaceBlockstoreTests } from 'interface-blockstore-tests' + +describe('IndexedDB Blockstore', function () { + describe('interface-blockstore (idb)', () => { + interfaceBlockstoreTests({ + async setup () { + const store = new IDBBlockstore(`hello-${Math.random()}`) + await store.open() + return store + }, + async teardown (store) { + await store.close() + await store.destroy() + } + }) + }) + + describe('concurrency', () => { + let store: IDBBlockstore + + before(async () => { + store = new IDBBlockstore('hello') + await store.open() + }) + + it('should not explode under unreasonable load', function (done) { + this.timeout(10000) + + const updater = setInterval(async () => { // eslint-disable-line @typescript-eslint/no-misused-promises + try { + const key = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') + + await store.put(key, Uint8Array.from([0, 1, 2, 3])) + await store.has(key) + await store.get(key) + } catch (err) { + clearInterval(updater) + clearInterval(mutatorQuery) + clearInterval(readOnlyQuery) + done(err) + } + }, 0) + + const mutatorQuery = setInterval(async () => { // eslint-disable-line @typescript-eslint/no-misused-promises + try { + for await (const { cid } of store.getAll()) { + await store.get(cid) + + const otherKey = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') + const otherValue = Uint8Array.from([0, 1, 2, 3]) + await store.put(otherKey, otherValue) + const res = await store.get(otherKey) + expect(res).to.deep.equal(otherValue) + } + } catch (err) { + clearInterval(updater) + clearInterval(mutatorQuery) + clearInterval(readOnlyQuery) + done(err) + } + }, 0) + + const readOnlyQuery = setInterval(async () => { // eslint-disable-line @typescript-eslint/no-misused-promises + try { + for await (const { cid } of store.getAll()) { + await store.has(cid) + } + } catch (err) { + clearInterval(updater) + clearInterval(mutatorQuery) + clearInterval(readOnlyQuery) + done(err) + } + }, 0) + + setTimeout(() => { + clearInterval(updater) + clearInterval(mutatorQuery) + clearInterval(readOnlyQuery) + done() + }, 5000) + }) + }) +}) diff --git a/packages/blockstore-idb/tsconfig.json b/packages/blockstore-idb/tsconfig.json new file mode 100644 index 00000000..e27c7fa6 --- /dev/null +++ b/packages/blockstore-idb/tsconfig.json @@ -0,0 +1,24 @@ +{ + "extends": "aegir/src/config/tsconfig.aegir.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": [ + "src", + "test" + ], + "references": [ + { + "path": "../blockstore-core" + }, + { + "path": "../interface-blockstore" + }, + { + "path": "../interface-blockstore-tests" + }, + { + "path": "../interface-store" + } + ] +} diff --git a/packages/blockstore-level/.aegir.js b/packages/blockstore-level/.aegir.js new file mode 100644 index 00000000..b8e889ea --- /dev/null +++ b/packages/blockstore-level/.aegir.js @@ -0,0 +1,6 @@ +/** @type {import('aegir').PartialOptions} */ +export default { + build: { + bundlesizeMax: '27KB' + } +} diff --git a/packages/blockstore-level/CHANGELOG.md b/packages/blockstore-level/CHANGELOG.md new file mode 100644 index 00000000..aff46d8d --- /dev/null +++ b/packages/blockstore-level/CHANGELOG.md @@ -0,0 +1,19 @@ +## [1.0.1](https://github.com/ipfs/js-blockstore-level/compare/v1.0.0...v1.0.1) (2023-03-23) + + +### Dependencies + +* update interface-store to 5.x.x ([#1](https://github.com/ipfs/js-blockstore-level/issues/1)) ([f4b7fb6](https://github.com/ipfs/js-blockstore-level/commit/f4b7fb697262f32c5997a4d2026ac383fde38db4)) + +## 1.0.0 (2023-03-14) + + +### Features + +* initial import ([2f262c8](https://github.com/ipfs/js-blockstore-level/commit/2f262c8809d6b04184b6e8990bd77966fa458b87)) + + +### Bug Fixes + +* generate docs ([60002c4](https://github.com/ipfs/js-blockstore-level/commit/60002c48a0fd2de5898bebf97939aa841b67650b)) +* update deps ([c6d8d39](https://github.com/ipfs/js-blockstore-level/commit/c6d8d39e587a7eaa404751ccef17b3cfdc1eb7e3)) diff --git a/packages/blockstore-level/LICENSE b/packages/blockstore-level/LICENSE new file mode 100644 index 00000000..20ce483c --- /dev/null +++ b/packages/blockstore-level/LICENSE @@ -0,0 +1,4 @@ +This project is dual licensed under MIT and Apache-2.0. + +MIT: https://www.opensource.org/licenses/mit +Apache-2.0: https://www.apache.org/licenses/license-2.0 diff --git a/packages/blockstore-level/LICENSE-APACHE b/packages/blockstore-level/LICENSE-APACHE new file mode 100644 index 00000000..14478a3b --- /dev/null +++ b/packages/blockstore-level/LICENSE-APACHE @@ -0,0 +1,5 @@ +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/packages/blockstore-level/LICENSE-MIT b/packages/blockstore-level/LICENSE-MIT new file mode 100644 index 00000000..72dc60d8 --- /dev/null +++ b/packages/blockstore-level/LICENSE-MIT @@ -0,0 +1,19 @@ +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/packages/blockstore-level/README.md b/packages/blockstore-level/README.md new file mode 100644 index 00000000..ff399ca5 --- /dev/null +++ b/packages/blockstore-level/README.md @@ -0,0 +1,53 @@ +# blockstore-level + +[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) +[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/js-stores.svg?style=flat-square)](https://codecov.io/gh/ipfs/js-stores) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/js-stores/js-test-and-release.yml?branch=master\&style=flat-square)](https://github.com/ipfs/js-stores/actions/workflows/js-test-and-release.yml?query=branch%3Amaster) + +> Blockstore implementation with level(up|down) backend + +## Table of contents + +- [Install](#install) + - [Browser ` +``` + +## API Docs + +- + +## License + +Licensed under either of + +- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) +- MIT ([LICENSE-MIT](LICENSE-MIT) / ) + +## Contribute + +Contributions welcome! Please check out [the issues](https://github.com/ipfs/js-stores/issues). + +Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. + +Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) diff --git a/packages/blockstore-level/package.json b/packages/blockstore-level/package.json new file mode 100644 index 00000000..06c7f2ac --- /dev/null +++ b/packages/blockstore-level/package.json @@ -0,0 +1,163 @@ +{ + "name": "blockstore-level", + "version": "1.0.1", + "description": "Blockstore implementation with level(up|down) backend", + "license": "Apache-2.0 OR MIT", + "homepage": "https://github.com/ipfs/js-stores/tree/master/packages/blockstore-level#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/ipfs/js-stores.git" + }, + "bugs": { + "url": "https://github.com/ipfs/js-stores/issues" + }, + "keywords": [ + "blockstore", + "interface", + "ipfs", + "key-value", + "leveldb", + "leveldown", + "levelup" + ], + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + }, + "type": "module", + "types": "./dist/src/index.d.ts", + "files": [ + "src", + "dist", + "!dist/test", + "!**/*.tsbuildinfo" + ], + "exports": { + ".": { + "types": "./dist/src/index.d.ts", + "import": "./dist/src/index.js" + } + }, + "eslintConfig": { + "extends": "ipfs", + "parserOptions": { + "sourceType": "module" + } + }, + "release": { + "branches": [ + "master" + ], + "plugins": [ + [ + "@semantic-release/commit-analyzer", + { + "preset": "conventionalcommits", + "releaseRules": [ + { + "breaking": true, + "release": "major" + }, + { + "revert": true, + "release": "patch" + }, + { + "type": "feat", + "release": "minor" + }, + { + "type": "fix", + "release": "patch" + }, + { + "type": "docs", + "release": "patch" + }, + { + "type": "test", + "release": "patch" + }, + { + "type": "deps", + "release": "patch" + }, + { + "scope": "no-release", + "release": false + } + ] + } + ], + [ + "@semantic-release/release-notes-generator", + { + "preset": "conventionalcommits", + "presetConfig": { + "types": [ + { + "type": "feat", + "section": "Features" + }, + { + "type": "fix", + "section": "Bug Fixes" + }, + { + "type": "chore", + "section": "Trivial Changes" + }, + { + "type": "docs", + "section": "Documentation" + }, + { + "type": "deps", + "section": "Dependencies" + }, + { + "type": "test", + "section": "Tests" + } + ] + } + } + ], + "@semantic-release/changelog", + "@semantic-release/npm", + "@semantic-release/github", + "@semantic-release/git" + ] + }, + "scripts": { + "clean": "aegir clean", + "lint": "aegir lint", + "build": "aegir build", + "release": "aegir release", + "test": "aegir test", + "test:node": "aegir test -t node --cov", + "test:chrome": "aegir test -t browser --cov", + "test:chrome-webworker": "aegir test -t webworker", + "test:firefox": "aegir test -t browser -- --browser firefox", + "test:firefox-webworker": "aegir test -t webworker -- --browser firefox", + "test:electron-main": "aegir test -t electron-main", + "dep-check": "aegir dep-check", + "docs": "aegir docs" + }, + "dependencies": { + "blockstore-core": "^4.0.0", + "interface-blockstore": "^5.0.0", + "interface-store": "^5.0.0", + "level": "^8.0.0", + "multiformats": "^11.0.2" + }, + "devDependencies": { + "aegir": "^38.1.7", + "interface-blockstore-tests": "^6.0.0", + "ipfs-utils": "^9.0.4", + "memory-level": "^1.0.0" + }, + "typedoc": { + "entryPoint": "./src/index.ts" + } +} diff --git a/packages/blockstore-level/src/index.ts b/packages/blockstore-level/src/index.ts new file mode 100644 index 00000000..747f9574 --- /dev/null +++ b/packages/blockstore-level/src/index.ts @@ -0,0 +1,146 @@ +import type { Pair } from 'interface-blockstore' +import { BaseBlockstore, Errors } from 'blockstore-core' +import { Level } from 'level' +import { CID } from 'multiformats/cid' +import type { DatabaseOptions, OpenOptions, IteratorOptions } from 'level' +import type { MultibaseCodec } from 'multiformats/bases/interface' +import { base32upper } from 'multiformats/bases/base32' +import * as raw from 'multiformats/codecs/raw' +import * as Digest from 'multiformats/hashes/digest' +import type { AbortOptions, AwaitIterable } from 'interface-store' + +export interface LevelBlockstoreInit extends DatabaseOptions, OpenOptions { + /** + * The multibase codec to use - nb. should be case insensitive. + * default: base32upper + */ + base?: MultibaseCodec +} + +/** + * A datastore backed by leveldb + */ +export class LevelBlockstore extends BaseBlockstore { + public db: Level + private readonly opts: OpenOptions + private readonly base: MultibaseCodec + + constructor (path: string | Level, init: LevelBlockstoreInit = {}) { + super() + + this.db = typeof path === 'string' + ? new Level(path, { + ...init, + keyEncoding: 'utf8', + valueEncoding: 'view' + }) + : path + + this.opts = { + createIfMissing: true, + compression: false, // same default as go + ...init + } + + this.base = init.base ?? base32upper + } + + #encode (cid: CID): string { + return `/${this.base.encoder.encode(cid.multihash.bytes)}` + } + + #decode (key: string): CID { + return CID.createV1(raw.code, Digest.decode(this.base.decoder.decode(key.substring(1)))) + } + + async open (): Promise { + try { + await this.db.open(this.opts) + } catch (err: any) { + throw Errors.openFailedError(err) + } + } + + async put (key: CID, value: Uint8Array): Promise { + try { + await this.db.put(this.#encode(key), value) + + return key + } catch (err: any) { + throw Errors.putFailedError(err) + } + } + + async get (key: CID): Promise { + let data + try { + data = await this.db.get(this.#encode(key)) + } catch (err: any) { + if (err.notFound != null) { + throw Errors.notFoundError(err) + } + + throw Errors.getFailedError(err) + } + return data + } + + async has (key: CID): Promise { + try { + await this.db.get(this.#encode(key)) + } catch (err: any) { + if (err.notFound != null) { + return false + } + + throw err + } + return true + } + + async delete (key: CID): Promise { + try { + await this.db.del(this.#encode(key)) + } catch (err: any) { + throw Errors.deleteFailedError(err) + } + } + + async close (): Promise { + await this.db.close() + } + + async * getAll (options?: AbortOptions | undefined): AwaitIterable { + for await (const { key, value } of this.#query({ values: true })) { + yield { cid: this.#decode(key), block: value } + } + } + + async * #query (opts: { values: boolean, prefix?: string }): AsyncIterable<{ key: string, value: Uint8Array }> { + const iteratorOpts: IteratorOptions = { + keys: true, + keyEncoding: 'buffer', + values: opts.values + } + + // Let the db do the prefix matching + if (opts.prefix != null) { + const prefix = opts.prefix.toString() + // Match keys greater than or equal to `prefix` and + iteratorOpts.gte = prefix + // less than `prefix` + \xFF (hex escape sequence) + iteratorOpts.lt = prefix + '\xFF' + } + + const li = this.db.iterator(iteratorOpts) + + try { + for await (const [key, value] of li) { + // @ts-expect-error key is a buffer because keyEncoding is "buffer" + yield { key: new TextDecoder().decode(key), value } + } + } finally { + await li.close() + } + } +} diff --git a/packages/blockstore-level/test/fixtures/test-level-iterator-destroy.ts b/packages/blockstore-level/test/fixtures/test-level-iterator-destroy.ts new file mode 100644 index 00000000..4183fbf0 --- /dev/null +++ b/packages/blockstore-level/test/fixtures/test-level-iterator-destroy.ts @@ -0,0 +1,17 @@ +import { LevelBlockstore } from '../../src/index.js' +import tempdir from 'ipfs-utils/src/temp-dir.js' +import { CID } from 'multiformats/cid' + +async function testLevelIteratorDestroy (): Promise { + const store = new LevelBlockstore(tempdir()) + await store.open() + await store.put(CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F'), new TextEncoder().encode(`TESTDATA${Date.now()}`)) + for await (const d of store.getAll()) { + console.log(d) // eslint-disable-line no-console + } +} + +// Will exit with: +// > Assertion failed: (ended_), function ~Iterator, file ../binding.cc, line 546. +// If iterator gets destroyed (in c++ land) and .end() was not called on it. +void testLevelIteratorDestroy() diff --git a/packages/blockstore-level/test/index.spec.ts b/packages/blockstore-level/test/index.spec.ts new file mode 100644 index 00000000..877c6811 --- /dev/null +++ b/packages/blockstore-level/test/index.spec.ts @@ -0,0 +1,63 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import { MemoryLevel } from 'memory-level' +import { Level } from 'level' +import { LevelBlockstore } from '../src/index.js' +import tempdir from 'ipfs-utils/src/temp-dir.js' +import { interfaceBlockstoreTests } from 'interface-blockstore-tests' + +describe('LevelBlockstore', () => { + describe('initialization', () => { + it('should default to a leveldown database', async () => { + const levelStore = new LevelBlockstore(`${tempdir()}/init-default-${Date.now()}`) + await levelStore.open() + + expect(levelStore.db).to.be.an.instanceOf(Level) + }) + + it('should be able to override the database', async () => { + const levelStore = new LevelBlockstore( + // @ts-expect-error MemoryLevel does not implement the same interface as Level + new MemoryLevel({ + keyEncoding: 'utf8', + valueEncoding: 'view' + }) + ) + + await levelStore.open() + + expect(levelStore.db).to.be.an.instanceOf(MemoryLevel) + }) + }) + + describe('interface-blockstore MemoryLevel', () => { + interfaceBlockstoreTests({ + async setup () { + const store = new LevelBlockstore( + // @ts-expect-error MemoryLevel does not implement the same interface as Level + new MemoryLevel({ + keyEncoding: 'utf8', + valueEncoding: 'view' + }) + ) + await store.open() + + return store + }, + teardown () {} + }) + }) + + describe('interface-blockstore Level', () => { + interfaceBlockstoreTests({ + async setup () { + const store = new LevelBlockstore(tempdir()) + await store.open() + + return store + }, + teardown () {} + }) + }) +}) diff --git a/packages/blockstore-level/test/node.ts b/packages/blockstore-level/test/node.ts new file mode 100644 index 00000000..230ba594 --- /dev/null +++ b/packages/blockstore-level/test/node.ts @@ -0,0 +1,52 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import path from 'path' +import childProcess from 'child_process' +import { interfaceBlockstoreTests } from 'interface-blockstore-tests' +import { LevelBlockstore } from '../src/index.js' +import tempdir from 'ipfs-utils/src/temp-dir.js' + +describe('LevelDatastore', () => { + describe('interface-blockstore (leveldown)', () => { + interfaceBlockstoreTests({ + async setup () { + const store = new LevelBlockstore(tempdir()) + await store.open() + + return store + }, + teardown () {} + }) + }) + + // The `.end()` method MUST be called on LevelDB iterators or they remain open, + // leaking memory. + // + // This test exposes this problem by causing an error to be thrown on process + // exit when an iterator is open AND leveldb is not closed. + // + // Normally when leveldb is closed it'll automatically clean up open iterators + // but if you don't close the store this error will occur: + // + // > Assertion failed: (ended_), function ~Iterator, file ../binding.cc, line 546. + // + // This is thrown by a destructor function for iterator objects that asserts + // the iterator has ended before cleanup. + // + // https://github.com/Level/leveldown/blob/d3453fbde4d2a8aa04d9091101c25c999649069b/binding.cc#L545 + it('should not leave iterators open and leak memory', (done) => { + const cp = childProcess.fork(path.join(process.cwd(), '/dist/test/fixtures/test-level-iterator-destroy'), { stdio: 'pipe' }) + + let out = '' + const { stdout, stderr } = cp + stdout?.on('data', d => { out = `${out}${d}` }) + stderr?.on('data', d => { out = `${out}${d}` }) + + cp.on('exit', code => { + expect(code).to.equal(0) + expect(out).to.not.include('Assertion failed: (ended_)') + done() + }) + }) +}) diff --git a/packages/blockstore-level/tsconfig.json b/packages/blockstore-level/tsconfig.json new file mode 100644 index 00000000..9c42667d --- /dev/null +++ b/packages/blockstore-level/tsconfig.json @@ -0,0 +1,24 @@ +{ + "extends": "aegir/src/config/tsconfig.aegir.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": [ + "test", + "src" + ], + "references": [ + { + "path": "../blockstore-core" + }, + { + "path": "../interface-blockstore" + }, + { + "path": "../interface-blockstore-tests" + }, + { + "path": "../interface-store" + } + ] +} diff --git a/packages/blockstore-s3/CHANGELOG.md b/packages/blockstore-s3/CHANGELOG.md new file mode 100644 index 00000000..8508e37b --- /dev/null +++ b/packages/blockstore-s3/CHANGELOG.md @@ -0,0 +1,222 @@ +## [11.0.0](https://github.com/ipfs/js-datastore-s3/compare/v10.0.1...v11.0.0) (2023-03-23) + + +### ⚠ BREAKING CHANGES + +* this module is now ESM-only and uses the v3 @aws-sdk/s3-client + +### Features + +* convert to typescript and publish as ESM only ([#106](https://github.com/ipfs/js-datastore-s3/issues/106)) ([0f372e1](https://github.com/ipfs/js-datastore-s3/commit/0f372e180ae22d0c53eaeedcf3975007e1f12466)) + +## [10.0.1](https://github.com/ipfs/js-datastore-s3/compare/v10.0.0...v10.0.1) (2022-10-18) + + +### Dependencies + +* bump it-filter from 1.0.3 to 2.0.0 ([#79](https://github.com/ipfs/js-datastore-s3/issues/79)) ([edb6264](https://github.com/ipfs/js-datastore-s3/commit/edb6264e61c0bcde9e10afb66f80077dca3ad769)) +* bump it-to-buffer from 2.0.2 to 3.0.0 ([#80](https://github.com/ipfs/js-datastore-s3/issues/80)) ([fa9bf96](https://github.com/ipfs/js-datastore-s3/commit/fa9bf963ab0a14bb43d729fd9aecf67e7c9cc437)) +* bump uint8arrays from 3.1.1 to 4.0.2 ([#78](https://github.com/ipfs/js-datastore-s3/issues/78)) ([53af2c6](https://github.com/ipfs/js-datastore-s3/commit/53af2c61cfa72e7e3aa823e6fd41da0c91d99027)) +* bump uint8arrays to 4.0.3 ([40a1444](https://github.com/ipfs/js-datastore-s3/commit/40a1444c37d4f71c73f4bfc8dd5a131691ac4391)) + +## [10.0.0](https://github.com/ipfs/js-datastore-s3/compare/v9.0.0...v10.0.0) (2022-08-12) + + +### ⚠ BREAKING CHANGES + +* this module used to be published as ESM/CJS now it is just ESM + +### Features + +* publish as ESM only ([#75](https://github.com/ipfs/js-datastore-s3/issues/75)) ([dca5704](https://github.com/ipfs/js-datastore-s3/commit/dca57045fa52498245c6e85c2c03cf9a6a9ff177)) + + +### Trivial Changes + +* Update .github/workflows/stale.yml [skip ci] ([b6cbc38](https://github.com/ipfs/js-datastore-s3/commit/b6cbc38b2d8d9dfad58bcaefd21621cd48345263)) + +## [9.0.0](https://github.com/ipfs/js-datastore-s3/compare/v8.0.0...v9.0.0) (2022-01-19) + + +### ⚠ BREAKING CHANGES + +* updates config to use unified ci + +### Trivial Changes + +* fix IPFS-101 example URL ([#40](https://github.com/ipfs/js-datastore-s3/issues/40)) ([0044df3](https://github.com/ipfs/js-datastore-s3/commit/0044df35dbc6fd2a2b01eaaf9e9432796fd9525e)) +* switch to unified ci ([#41](https://github.com/ipfs/js-datastore-s3/issues/41)) ([a969b40](https://github.com/ipfs/js-datastore-s3/commit/a969b404293ca6122ce8d26c000e36724e8186fd)) + +# [8.0.0](https://github.com/ipfs/js-datastore-s3/compare/v7.0.0...v8.0.0) (2021-09-09) + + +### chore + +* switch to esm ([#37](https://github.com/ipfs/js-datastore-s3/issues/37)) ([25f4dce](https://github.com/ipfs/js-datastore-s3/commit/25f4dceaf3e6678756b4c93b56a082c0282cc9f6)) + + +### BREAKING CHANGES + +* only uses named exports + + + +# [7.0.0](https://github.com/ipfs/js-datastore-s3/compare/v6.0.0...v7.0.0) (2021-08-20) + + + +# [6.0.0](https://github.com/ipfs/js-datastore-s3/compare/v4.0.0...v6.0.0) (2021-07-06) + + +### chore + +* update deps ([be4fec6](https://github.com/ipfs/js-datastore-s3/commit/be4fec68ba1854acab5d2eec24f2719f685546fd)) + + +### Features + +* split .query into .query and .queryKeys ([#34](https://github.com/ipfs/js-datastore-s3/issues/34)) ([29423d1](https://github.com/ipfs/js-datastore-s3/commit/29423d13acc4ca137f9708578fe50764b33e0970)) + + +### BREAKING CHANGES + +* uses new interface-datastore types + + + +# [5.0.0](https://github.com/ipfs/js-datastore-s3/compare/v4.0.0...v5.0.0) (2021-04-15) + + +### Features + +* split .query into .query and .queryKeys ([#34](https://github.com/ipfs/js-datastore-s3/issues/34)) ([29423d1](https://github.com/ipfs/js-datastore-s3/commit/29423d13acc4ca137f9708578fe50764b33e0970)) + + + +# [4.0.0](https://github.com/ipfs/js-datastore-s3/compare/v3.0.0...v4.0.0) (2021-04-12) + + + + +# [3.0.0](https://github.com/ipfs/js-datastore-s3/compare/v2.0.0...v3.0.0) (2020-09-22) + + +### Bug Fixes + +* convert input to buffers before passing to aws-sdk ([#30](https://github.com/ipfs/js-datastore-s3/issues/30)) ([b844c63](https://github.com/ipfs/js-datastore-s3/commit/b844c63)) + + +### BREAKING CHANGES + +* - Returns Uint8Arrays only where before it was node Buffers + +* chore: configure pin store + +* docs: update example + +Co-authored-by: Jacob Heun + + + + +# [2.0.0](https://github.com/ipfs/js-datastore-s3/compare/v1.0.0...v2.0.0) (2020-06-19) + + + + +# [1.0.0](https://github.com/ipfs/js-datastore-s3/compare/v0.3.0...v1.0.0) (2020-05-08) + + +### Bug Fixes + +* **ci:** add empty commit to fix lint checks on master ([4251456](https://github.com/ipfs/js-datastore-s3/commit/4251456)) + + +### Features + +* adds interface-datastore streaming api ([6c74394](https://github.com/ipfs/js-datastore-s3/commit/6c74394)) + + + + +# [0.3.0](https://github.com/ipfs/js-datastore-s3/compare/v0.2.4...v0.3.0) (2019-08-15) + + +### Code Refactoring + +* callbacks -> async / await ([#17](https://github.com/ipfs/js-datastore-s3/issues/17)) ([629dba7](https://github.com/ipfs/js-datastore-s3/commit/629dba7)) + + +### BREAKING CHANGES + +* All places in the API that used callbacks are now replaced with async/await + + + + +## [0.2.4](https://github.com/ipfs/js-datastore-s3/compare/v0.2.3...v0.2.4) (2019-03-27) + + +### Bug Fixes + +* **create-repo:** pass sub paths in repo to each store ([1113c61](https://github.com/ipfs/js-datastore-s3/commit/1113c61)) + + + + +## [0.2.3](https://github.com/ipfs/js-datastore-s3/compare/v0.2.2...v0.2.3) (2019-02-14) + + +### Bug Fixes + +* aws-sdk should be a peer dependency ([836355c](https://github.com/ipfs/js-datastore-s3/commit/836355c)) + + + + +## [0.2.2](https://github.com/ipfs/js-datastore-s3/compare/v0.2.1...v0.2.2) (2019-02-14) + + +### Features + +* add createRepo utility ([0f5021c](https://github.com/ipfs/js-datastore-s3/commit/0f5021c)) + + + + +## [0.2.1](https://github.com/ipfs/js-datastore-s3/compare/v0.2.0...v0.2.1) (2019-02-07) + + +### Bug Fixes + +* use once to prevent multiple callback calls ([db99ae8](https://github.com/ipfs/js-datastore-s3/commit/db99ae8)) + + +### Features + +* have the s3 lock cleanup gracefully ([7f6b2c8](https://github.com/ipfs/js-datastore-s3/commit/7f6b2c8)) + + + + +# 0.2.0 (2018-10-01) + + +### Bug Fixes + +* **flow:** make flow pass and fix query abort call ([46e8e5e](https://github.com/ipfs/js-datastore-s3/commit/46e8e5e)) +* add windows support ([feaed0d](https://github.com/ipfs/js-datastore-s3/commit/feaed0d)) +* linting ([e00974f](https://github.com/ipfs/js-datastore-s3/commit/e00974f)) +* resolve an issue where a new repo wouldnt init properly ([104d6e9](https://github.com/ipfs/js-datastore-s3/commit/104d6e9)) + + +### Features + +* add basic error codes and update test ([#8](https://github.com/ipfs/js-datastore-s3/issues/8)) ([31ba28a](https://github.com/ipfs/js-datastore-s3/commit/31ba28a)) +* add querying and make all tests pass ([0c89c78](https://github.com/ipfs/js-datastore-s3/commit/0c89c78)) +* initial featureset aside from querying ([b710421](https://github.com/ipfs/js-datastore-s3/commit/b710421)) + + + + +# 0.1.0 (2018-05-07) diff --git a/packages/blockstore-s3/LICENSE b/packages/blockstore-s3/LICENSE new file mode 100644 index 00000000..20ce483c --- /dev/null +++ b/packages/blockstore-s3/LICENSE @@ -0,0 +1,4 @@ +This project is dual licensed under MIT and Apache-2.0. + +MIT: https://www.opensource.org/licenses/mit +Apache-2.0: https://www.apache.org/licenses/license-2.0 diff --git a/packages/blockstore-s3/LICENSE-APACHE b/packages/blockstore-s3/LICENSE-APACHE new file mode 100644 index 00000000..14478a3b --- /dev/null +++ b/packages/blockstore-s3/LICENSE-APACHE @@ -0,0 +1,5 @@ +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/packages/blockstore-s3/LICENSE-MIT b/packages/blockstore-s3/LICENSE-MIT new file mode 100644 index 00000000..72dc60d8 --- /dev/null +++ b/packages/blockstore-s3/LICENSE-MIT @@ -0,0 +1,19 @@ +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/packages/blockstore-s3/README.md b/packages/blockstore-s3/README.md new file mode 100644 index 00000000..674557f4 --- /dev/null +++ b/packages/blockstore-s3/README.md @@ -0,0 +1,79 @@ +# blockstore-s3 + +[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) +[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/js-stores.svg?style=flat-square)](https://codecov.io/gh/ipfs/js-stores) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/js-stores/js-test-and-release.yml?branch=master\&style=flat-square)](https://github.com/ipfs/js-stores/actions/workflows/js-test-and-release.yml?query=branch%3Amaster) + +> IPFS blockstore implementation backed by s3 + +## Table of contents + +- [Install](#install) + - [Browser ` +``` + +## Usage + +If the flag `createIfMissing` is not set or is false, then the bucket must be created prior to using datastore-s3. Please see the AWS docs for information on how to configure the S3 instance. A bucket name is required to be set at the s3 instance level, see the below example. + +```js +import S3 from 'aws-sdk/clients/s3.js' +import { S3Datastore } from 'datastore-s3' + +const s3Instance = new S3({ params: { Bucket: 'my-ipfs-bucket' } }) +const store = new S3Datastore('.ipfs/datastore', { + s3: s3Instance + createIfMissing: false +}) +``` + +### Create a Repo + +See [examples/full-s3-repo](./examples/full-s3-repo) for how to quickly create an S3 backed repo using the `createRepo` convenience function. + +### Examples + +You can see examples of S3 backed ipfs in the [examples folder](examples/) + +## API Docs + +- + +## License + +Licensed under either of + +- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) +- MIT ([LICENSE-MIT](LICENSE-MIT) / ) + +## Contribute + +Contributions welcome! Please check out [the issues](https://github.com/ipfs/js-stores/issues). + +Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. + +Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) diff --git a/packages/blockstore-s3/examples/helia/README.md b/packages/blockstore-s3/examples/helia/README.md new file mode 100644 index 00000000..b9bd8027 --- /dev/null +++ b/packages/blockstore-s3/examples/helia/README.md @@ -0,0 +1,25 @@ +Use with Helia +====== + +This example uses a Datastore S3 instance to serve as the entire backend for Helia. + +## Running +The S3 parameters must be updated with an existing Bucket and credentials with access to it: +```js +// Configure S3 as normal +const s3 = new S3({ + region: 'region', + credentials: { + accessKeyId: 'myaccesskey', + secretAccessKey: 'mysecretkey' + } +}) + +const datastore = new DatastoreS3(s3, 'my-bucket') +``` + +Once the S3 instance has its needed data, you can run the example: +``` +npm install +node index.js +``` diff --git a/packages/blockstore-s3/examples/helia/index.js b/packages/blockstore-s3/examples/helia/index.js new file mode 100644 index 00000000..0de17895 --- /dev/null +++ b/packages/blockstore-s3/examples/helia/index.js @@ -0,0 +1,55 @@ +import { createHelia } from 'helia' +import { unixfs } from '@helia/unixfs' +import toBuffer from 'it-to-buffer' +import { S3 } from '@aws-sdk/client-s3' +import { DatastoreS3 } from 'datastore-s3' + +async function main () { + // Configure S3 as normal + const s3 = new S3({ + region: 'region', + credentials: { + accessKeyId: 'myaccesskey', + secretAccessKey: 'mysecretkey' + } + }) + + const datastore = new DatastoreS3(s3, 'my-bucket') + + // Create a new Helia node with our S3 backed Repo + console.log('Start Helia') + const node = await createHelia({ + datastore + }) + + // Test out the repo by sending and fetching some data + console.log('Helia is ready') + + try { + const fs = unixfs(helia) + + // Let's add a file to Helia + const cid = await fs.addBytes(Uint8Array.from([0, 1, 2, 3])) + + console.log('\nAdded file:', cid) + + // Log out the added files metadata and cat the file from IPFS + const data = await toBuffer(fs.cat(cid)) + + // Print out the files contents to console + console.log(`\nFetched file content containing ${data.byteLength} bytes`) + } catch (err) { + // Log out the error + console.log('File Processing Error:', err) + } + // After everything is done, shut the node down + // We don't need to worry about catching errors here + console.log('\n\nStopping the node') + await node.stop() +} + +main() + .catch(err => { + console.error(err) + process.exit(1) + }) diff --git a/packages/blockstore-s3/examples/helia/package.json b/packages/blockstore-s3/examples/helia/package.json new file mode 100644 index 00000000..0d887034 --- /dev/null +++ b/packages/blockstore-s3/examples/helia/package.json @@ -0,0 +1,19 @@ +{ + "name": "helia", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "author": "", + "license": "ISC", + "dependencies": { + "@aws-sdk/client-s3": "^3.297.0", + "@helia/unixfs": "^1.2.0", + "datastore-s3": "../../", + "helia": "^1.0.0", + "it-to-buffer": "^3.0.1" + } +} diff --git a/packages/blockstore-s3/package.json b/packages/blockstore-s3/package.json new file mode 100644 index 00000000..f57bc8ab --- /dev/null +++ b/packages/blockstore-s3/package.json @@ -0,0 +1,162 @@ +{ + "name": "blockstore-s3", + "version": "0.0.0", + "description": "IPFS blockstore implementation backed by s3", + "license": "Apache-2.0 OR MIT", + "homepage": "https://github.com/ipfs/js-stores/tree/master/packages/blockstore-s3#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/ipfs/js-stores.git" + }, + "bugs": { + "url": "https://github.com/ipfs/js-stores/issues" + }, + "keywords": [ + "datastore", + "interface", + "ipfs", + "key-value", + "s3" + ], + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + }, + "type": "module", + "types": "./dist/src/index.d.ts", + "files": [ + "src", + "dist", + "!dist/test", + "!**/*.tsbuildinfo" + ], + "exports": { + ".": { + "types": "./dist/src/index.d.ts", + "import": "./src/index.js" + } + }, + "eslintConfig": { + "extends": "ipfs", + "parserOptions": { + "sourceType": "module" + } + }, + "release": { + "branches": [ + "master" + ], + "plugins": [ + [ + "@semantic-release/commit-analyzer", + { + "preset": "conventionalcommits", + "releaseRules": [ + { + "breaking": true, + "release": "major" + }, + { + "revert": true, + "release": "patch" + }, + { + "type": "feat", + "release": "minor" + }, + { + "type": "fix", + "release": "patch" + }, + { + "type": "docs", + "release": "patch" + }, + { + "type": "test", + "release": "patch" + }, + { + "type": "deps", + "release": "patch" + }, + { + "scope": "no-release", + "release": false + } + ] + } + ], + [ + "@semantic-release/release-notes-generator", + { + "preset": "conventionalcommits", + "presetConfig": { + "types": [ + { + "type": "feat", + "section": "Features" + }, + { + "type": "fix", + "section": "Bug Fixes" + }, + { + "type": "chore", + "section": "Trivial Changes" + }, + { + "type": "docs", + "section": "Documentation" + }, + { + "type": "deps", + "section": "Dependencies" + }, + { + "type": "test", + "section": "Tests" + } + ] + } + } + ], + "@semantic-release/changelog", + "@semantic-release/npm", + "@semantic-release/github", + "@semantic-release/git" + ] + }, + "scripts": { + "clean": "aegir clean", + "lint": "aegir lint", + "build": "aegir build", + "release": "aegir release", + "test": "aegir test", + "test:node": "aegir test -t node --cov", + "test:chrome": "aegir test -t browser --cov", + "test:chrome-webworker": "aegir test -t webworker", + "test:firefox": "aegir test -t browser -- --browser firefox", + "test:firefox-webworker": "aegir test -t webworker -- --browser firefox", + "test:electron-main": "aegir test -t electron-main", + "dep-check": "aegir dep-check" + }, + "dependencies": { + "@aws-sdk/client-s3": "^3.297.0", + "blockstore-core": "^4.0.0", + "interface-blockstore": "^5.0.0", + "interface-store": "^5.0.0", + "it-to-buffer": "^3.0.0", + "multiformats": "^11.0.2", + "uint8arrays": "^4.0.2" + }, + "devDependencies": { + "aegir": "^38.1.7", + "interface-blockstore-tests": "^6.0.0", + "p-defer": "^4.0.0", + "sinon": "^15.0.2" + }, + "typedoc": { + "entryPoint": "./src/index.ts" + } +} diff --git a/packages/blockstore-s3/src/index.ts b/packages/blockstore-s3/src/index.ts new file mode 100644 index 00000000..99daac8b --- /dev/null +++ b/packages/blockstore-s3/src/index.ts @@ -0,0 +1,271 @@ +import type { Pair } from 'interface-blockstore' +import { BaseBlockstore } from 'blockstore-core/base' +import * as Errors from 'blockstore-core/errors' +import { fromString as unint8arrayFromString } from 'uint8arrays' +import toBuffer from 'it-to-buffer' +import type { S3 } from '@aws-sdk/client-s3' +import type { AbortOptions } from 'interface-store' +import { + PutObjectCommand, + CreateBucketCommand, + GetObjectCommand, + HeadObjectCommand, + DeleteObjectCommand, + ListObjectsV2Command +} from '@aws-sdk/client-s3' +import { CID } from 'multiformats/cid' + +export interface S3DatastoreInit { + /** + * An optional path to use within the bucket for all files - this setting can + * affect S3 performance as it does internal sharding based on 'prefixes' - + * these can be delimited by '/' so it's often better to wrap this datastore in + * a sharding datastore which will generate prefixed datastore keys for you. + * + * See - https://docs.aws.amazon.com/AmazonS3/latest/userguide/optimizing-performance.html + * and https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-prefixes.html + */ + path?: string + + /** + * Whether to try to create the bucket if it is missing when `.open` is called + */ + createIfMissing?: boolean +} + +/** + * A blockstore backed by AWS S3 + */ +export class S3Blockstore extends BaseBlockstore { + public path?: string + public createIfMissing: boolean + private readonly s3: S3 + private readonly bucket: string + + constructor (s3: S3, bucket: string, init?: S3DatastoreInit) { + super() + + if (s3 == null) { + throw new Error('An S3 instance must be supplied. See the datastore-s3 README for examples.') + } + + if (bucket == null) { + throw new Error('An bucket must be supplied. See the datastore-s3 README for examples.') + } + + this.path = init?.path + this.s3 = s3 + this.bucket = bucket + this.createIfMissing = init?.createIfMissing ?? false + } + + /** + * Returns the full key which includes the path to the ipfs store + */ + _getFullKey (key: CID): string { + // Avoid absolute paths with s3 + return [this.path, key.toString()].filter(Boolean).join('/').replace(/\/\/+/g, '/') + } + + /** + * Store the given value under the key. + */ + async put (key: CID, val: Uint8Array, options?: AbortOptions): Promise { + try { + // @ts-expect-error the AWS AbortSignal types are different to the @types/node version + await this.s3.send( + new PutObjectCommand({ + Bucket: this.bucket, + Key: this._getFullKey(key), + Body: val + }), { + abortSignal: options?.signal + } + ) + + return key + } catch (err: any) { + throw Errors.putFailedError(err) + } + } + + /** + * Read from s3 + */ + async get (key: CID, options?: AbortOptions): Promise { + try { + // @ts-expect-error the AWS AbortSignal types are different to the @types/node version + const data = await this.s3.send( + new GetObjectCommand({ + Bucket: this.bucket, + Key: this._getFullKey(key) + }), { + abortSignal: options?.signal + } + ) + + if (data.Body == null) { + throw new Error('Response had no body') + } + + // If a body was returned, ensure it's a Uint8Array + if (data.Body instanceof Uint8Array) { + return data.Body + } + + if (typeof data.Body === 'string') { + return unint8arrayFromString(data.Body) + } + + if (data.Body instanceof Blob) { + const buf = await data.Body.arrayBuffer() + + return new Uint8Array(buf, 0, buf.byteLength) + } + + // @ts-expect-error s3 types define their own Blob as an empty interface + return await toBuffer(data.Body) + } catch (err: any) { + if (err.statusCode === 404) { + throw Errors.notFoundError(err) + } + throw err + } + } + + /** + * Check for the existence of the given key + */ + async has (key: CID, options?: AbortOptions): Promise { + try { + // @ts-expect-error the AWS AbortSignal types are different to the @types/node version + await this.s3.send( + new HeadObjectCommand({ + Bucket: this.bucket, + Key: this._getFullKey(key) + }), { + abortSignal: options?.signal + } + ) + + return true + } catch (err: any) { + // doesn't exist and permission policy includes s3:ListBucket + if (err.$metadata?.httpStatusCode === 404) { + return false + } + + // doesn't exist, permission policy does not include s3:ListBucket + if (err.$metadata?.httpStatusCode === 403) { + return false + } + + throw err + } + } + + /** + * Delete the record under the given key + */ + async delete (key: CID, options?: AbortOptions): Promise { + try { + // @ts-expect-error the AWS AbortSignal types are different to the @types/node version + await this.s3.send( + new DeleteObjectCommand({ + Bucket: this.bucket, + Key: this._getFullKey(key) + }), { + abortSignal: options?.signal + } + ) + } catch (err: any) { + throw Errors.deleteFailedError(err) + } + } + + async * getAll (options?: AbortOptions): AsyncIterable { + const params: Record = {} + + try { + while (true) { + // @ts-expect-error the AWS AbortSignal types are different to the @types/node version + const data = await this.s3.send( + new ListObjectsV2Command({ + Bucket: this.bucket, + ...params + }), { + abortSignal: options?.signal + } + ) + + if (options?.signal?.aborted === true) { + return + } + + if (data == null || data.Contents == null) { + throw new Error('Not found') + } + + for (const d of data.Contents) { + if (d.Key == null) { + throw new Error('Not found') + } + + // Remove the path from the key + const cid = CID.parse(d.Key.slice((this.path ?? '').length)) + + yield { + cid, + block: await this.get(cid, options) + } + } + + // If we didn't get all records, recursively query + if (data.IsTruncated === true) { + // If NextMarker is absent, use the key from the last result + params.StartAfter = data.Contents[data.Contents.length - 1].Key + + // recursively fetch keys + continue + } + + break + } + } catch (err: any) { + throw new Error(err.code) + } + } + + /** + * This will check the s3 bucket to ensure access and existence + */ + async open (options?: AbortOptions): Promise { + try { + // @ts-expect-error the AWS AbortSignal types are different to the @types/node version + await this.s3.send( + new HeadObjectCommand({ + Bucket: this.bucket, + Key: this.path ?? '' + }), { + abortSignal: options?.signal + } + ) + } catch (err: any) { + if (err.statusCode !== 404) { + if (this.createIfMissing) { + // @ts-expect-error the AWS AbortSignal types are different to the @types/node version + await this.s3.send( + new CreateBucketCommand({ + Bucket: this.bucket + }), { + abortSignal: options?.signal + } + ) + return + } + + throw Errors.openFailedError(err) + } + } + } +} diff --git a/packages/blockstore-s3/test/index.spec.ts b/packages/blockstore-s3/test/index.spec.ts new file mode 100644 index 00000000..00b40096 --- /dev/null +++ b/packages/blockstore-s3/test/index.spec.ts @@ -0,0 +1,229 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import sinon from 'sinon' +import { CreateBucketCommand, PutObjectCommand, HeadObjectCommand, S3, GetObjectCommand } from '@aws-sdk/client-s3' +import defer from 'p-defer' +import { interfaceBlockstoreTests } from 'interface-blockstore-tests' +import { CID } from 'multiformats/cid' + +import { s3Resolve, s3Reject, S3Error, s3Mock } from './utils/s3-mock.js' +import { S3Blockstore } from '../src/index.js' + +const cid = CID.parse('QmeimKZyjcBnuXmAD9zMnSjM9JodTbgGT3gutofkTqz9rE') + +describe('S3Blockstore', () => { + describe('construction', () => { + it('requires an s3', () => { + expect( + // @ts-expect-error missing params + () => new S3Blockstore() + ).to.throw() + }) + + it('requires a bucket', () => { + const s3 = new S3({ region: 'REGION' }) + expect( + // @ts-expect-error missing params + () => new S3Blockstore(s3) + ).to.throw() + }) + + it('createIfMissing defaults to false', () => { + const s3 = new S3({ region: 'REGION' }) + const store = new S3Blockstore(s3, 'test') + expect(store.createIfMissing).to.equal(false) + }) + + it('createIfMissing can be set to true', () => { + const s3 = new S3({ region: 'REGION' }) + const store = new S3Blockstore(s3, 'test', { createIfMissing: true }) + expect(store.createIfMissing).to.equal(true) + }) + }) + + describe('put', () => { + it('should include the path in the key', async () => { + const s3 = new S3({ region: 'REGION' }) + const store = new S3Blockstore(s3, 'test', { + path: '.ipfs/datastore' + }) + + const deferred = defer() + + sinon.replace(s3, 'send', (command: PutObjectCommand) => { + deferred.resolve(command) + return s3Resolve(null) + }) + + await store.put(cid, new TextEncoder().encode('test data')) + + const command = await deferred.promise + expect(command).to.have.nested.property('input.Key', '.ipfs/datastore/QmeimKZyjcBnuXmAD9zMnSjM9JodTbgGT3gutofkTqz9rE') + }) + + it('should return a standard error when the put fails', async () => { + const s3 = new S3({ region: 'REGION' }) + const store = new S3Blockstore(s3, 'test') + + sinon.replace(s3, 'send', (command: any) => { + if (command.constructor.name === 'PutObjectCommand') { + return s3Reject(new Error('bad things happened')) + } + + return s3Reject(new S3Error('UnknownCommand')) + }) + + await expect(store.put(cid, new TextEncoder().encode('test data'))).to.eventually.rejected + .with.property('code', 'ERR_PUT_FAILED') + }) + }) + + describe('get', () => { + it('should include the path in the fetch key', async () => { + const s3 = new S3({ region: 'REGION' }) + const store = new S3Blockstore(s3, 'test', { + path: '.ipfs/datastore' + }) + const buf = new TextEncoder().encode('test') + + const deferred = defer() + + sinon.replace(s3, 'send', (command: any) => { + if (command.constructor.name === 'GetObjectCommand') { + deferred.resolve(command) + return s3Resolve({ Body: buf }) + } + + return s3Reject(new S3Error('UnknownCommand')) + }) + + const value = await store.get(cid) + + expect(value).to.equalBytes(buf) + + const getObjectCommand = await deferred.promise + expect(getObjectCommand).to.have.nested.property('input.Key', '.ipfs/datastore/QmeimKZyjcBnuXmAD9zMnSjM9JodTbgGT3gutofkTqz9rE') + }) + + it('should return a standard not found error code if the key isn\'t found', async () => { + const s3 = new S3({ region: 'REGION' }) + const store = new S3Blockstore(s3, 'test') + + sinon.replace(s3, 'send', (command: any) => { + if (command.constructor.name === 'GetObjectCommand') { + return s3Reject(new S3Error('NotFound', 404)) + } + + return s3Reject(new S3Error('UnknownCommand')) + }) + + await expect(store.get(cid)).to.eventually.rejected + .with.property('code', 'ERR_NOT_FOUND') + }) + }) + + describe('delete', () => { + it('should return a standard delete error if deletion fails', async () => { + const s3 = new S3({ region: 'REGION' }) + const store = new S3Blockstore(s3, 'test') + + sinon.replace(s3, 'send', (command: any) => { + if (command.constructor.name === 'DeleteObjectCommand') { + return s3Reject(new Error('bad things')) + } + + return s3Reject(new S3Error('UnknownCommand')) + }) + + await expect(store.delete(cid)).to.eventually.rejected + .with.property('code', 'ERR_DELETE_FAILED') + }) + }) + + describe('open', () => { + it('should create the bucket when missing if createIfMissing is true', async () => { + const s3 = new S3({ region: 'REGION' }) + const store = new S3Blockstore(s3, 'test', { createIfMissing: true }) + + // 1. On the first call upload will fail with a NoSuchBucket error + // 2. This should result in the `createBucket` standin being called + // 3. upload is then called a second time and it passes + + const bucketTested = defer() + const bucketCreated = defer() + + sinon.replace(s3, 'send', (command: any) => { + if (command.constructor.name === 'HeadObjectCommand') { + bucketTested.resolve(command) + return s3Reject(new S3Error('NoSuchBucket')) + } + + if (command.constructor.name === 'CreateBucketCommand') { + bucketCreated.resolve(command) + return s3Resolve(null) + } + + return s3Reject(new S3Error('UnknownCommand')) + }) + + await store.open() + + const headObjectCommand = await bucketTested.promise + expect(headObjectCommand).to.have.nested.property('input.Bucket', 'test') + + const createBucketCommand = await bucketCreated.promise + expect(createBucketCommand).to.have.nested.property('input.Bucket', 'test') + }) + + it('should not create the bucket when missing if createIfMissing is false', async () => { + const s3 = new S3({ region: 'REGION' }) + const store = new S3Blockstore(s3, 'test') + + const bucketTested = defer() + + sinon.replace(s3, 'send', (command: any) => { + if (command.constructor.name === 'HeadObjectCommand') { + bucketTested.resolve(command) + return s3Reject(new S3Error('NoSuchBucket')) + } + + return s3Reject(new S3Error('UnknownCommand')) + }) + + await expect(store.open()).to.eventually.rejected + .with.property('code', 'ERR_OPEN_FAILED') + + const headObjectCommand = await bucketTested.promise + expect(headObjectCommand).to.have.nested.property('input.Bucket', 'test') + }) + + it('should return a standard open error if the head request fails with an unknown error', async () => { + const s3 = new S3({ region: 'REGION' }) + const store = new S3Blockstore(s3, 'test') + + sinon.replace(s3, 'send', (command: any) => { + if (command.constructor.name === 'HeadObjectCommand') { + return s3Reject(new Error('bad things')) + } + + return s3Reject(new S3Error('UnknownCommand')) + }) + + await expect(store.open()).to.eventually.rejected + .with.property('code', 'ERR_OPEN_FAILED') + }) + }) + + describe('interface-datastore', () => { + interfaceBlockstoreTests({ + setup () { + const s3 = s3Mock(new S3({ region: 'REGION' })) + + return new S3Blockstore(s3, 'test') + }, + teardown () { + } + }) + }) +}) diff --git a/packages/blockstore-s3/test/utils/s3-mock.ts b/packages/blockstore-s3/test/utils/s3-mock.ts new file mode 100644 index 00000000..6c4bf01f --- /dev/null +++ b/packages/blockstore-s3/test/utils/s3-mock.ts @@ -0,0 +1,87 @@ +import sinon from 'sinon' +import type { S3 } from '@aws-sdk/client-s3' + +export class S3Error extends Error { + public code: string + public statusCode?: number + public $metadata?: { httpStatusCode: number } + + constructor (message: string, code?: number) { + super(message) + this.code = message + this.statusCode = code + + this.$metadata = { + httpStatusCode: code ?? 200 + } + } +} + +export const s3Resolve = (res?: any): any => { + return Promise.resolve(res) +} + +export const s3Reject = (err: T): any => { + return Promise.reject(err) +} + +/** + * Mocks out the s3 calls made by datastore-s3 + */ +export function s3Mock (s3: S3): S3 { + const mocks: any = {} + const storage: Map = new Map() + + mocks.send = sinon.replace(s3, 'send', (command) => { + const commandName = command.constructor.name + const input: any = command.input + + if (commandName === 'PutObjectCommand') { + storage.set(input.Key, input.Body) + return s3Resolve({}) + } + + if (commandName === 'HeadObjectCommand') { + if (storage.has(input.Key)) { + return s3Resolve({}) + } + + return s3Reject(new S3Error('NotFound', 404)) + } + + if (commandName === 'GetObjectCommand') { + if (!storage.has(input.Key)) { + return s3Reject(new S3Error('NotFound', 404)) + } + + return s3Resolve({ + Body: storage.get(input.Key) + }) + } + + if (commandName === 'DeleteObjectCommand') { + storage.delete(input.Key) + return s3Resolve({}) + } + + if (commandName === 'ListObjectsV2Command') { + const results: { Contents: Array<{ Key: string }> } = { + Contents: [] + } + + for (const k of storage.keys()) { + if (k.startsWith(`${input.Prefix ?? ''}`)) { + results.Contents.push({ + Key: k + }) + } + } + + return s3Resolve(results) + } + + return s3Reject(new S3Error('UnknownCommand', 400)) + }) + + return s3 +} diff --git a/packages/blockstore-s3/tsconfig.json b/packages/blockstore-s3/tsconfig.json new file mode 100644 index 00000000..e27c7fa6 --- /dev/null +++ b/packages/blockstore-s3/tsconfig.json @@ -0,0 +1,24 @@ +{ + "extends": "aegir/src/config/tsconfig.aegir.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": [ + "src", + "test" + ], + "references": [ + { + "path": "../blockstore-core" + }, + { + "path": "../interface-blockstore" + }, + { + "path": "../interface-blockstore-tests" + }, + { + "path": "../interface-store" + } + ] +} diff --git a/packages/datastore-core/CHANGELOG.md b/packages/datastore-core/CHANGELOG.md new file mode 100644 index 00000000..22200d60 --- /dev/null +++ b/packages/datastore-core/CHANGELOG.md @@ -0,0 +1,324 @@ +## [9.0.4](https://github.com/ipfs/js-datastore-core/compare/v9.0.3...v9.0.4) (2023-03-23) + + +### Dependencies + +* update interface-store to 5.x.x ([#149](https://github.com/ipfs/js-datastore-core/issues/149)) ([2820b0d](https://github.com/ipfs/js-datastore-core/commit/2820b0ddce94aefe85cb9ba09b387cc59494e130)) + +## [9.0.3](https://github.com/ipfs/js-datastore-core/compare/v9.0.2...v9.0.3) (2023-03-14) + + +### Bug Fixes + +* add error for db read failures ([#146](https://github.com/ipfs/js-datastore-core/issues/146)) ([afab18a](https://github.com/ipfs/js-datastore-core/commit/afab18a9b6000ebe4670abee5bd401e024d0ae0f)) + +## [9.0.2](https://github.com/ipfs/js-datastore-core/compare/v9.0.1...v9.0.2) (2023-03-13) + + +### Bug Fixes + +* update shard creation ([59081a7](https://github.com/ipfs/js-datastore-core/commit/59081a7a3b8e596705fd5834b35bfb9ac73c3f39)) + +## [9.0.1](https://github.com/ipfs/js-datastore-core/compare/v9.0.0...v9.0.1) (2023-03-13) + + +### Bug Fixes + +* update project config ([#145](https://github.com/ipfs/js-datastore-core/issues/145)) ([5bb5601](https://github.com/ipfs/js-datastore-core/commit/5bb5601735c59f6aed7dd52112194d233e86267b)) + +## [9.0.0](https://github.com/ipfs/js-datastore-core/compare/v8.0.4...v9.0.0) (2023-03-13) + + +### ⚠ BREAKING CHANGES + +* update to latest datastore interface + +### Features + +* update to latest datastore interface ([#144](https://github.com/ipfs/js-datastore-core/issues/144)) ([46059f1](https://github.com/ipfs/js-datastore-core/commit/46059f184bb49414fb494135ca544ca6a2d0ee66)) + +## [8.0.4](https://github.com/ipfs/js-datastore-core/compare/v8.0.3...v8.0.4) (2023-01-11) + + +### Bug Fixes + +* throw error with message from error in tiered datastore ([#133](https://github.com/ipfs/js-datastore-core/issues/133)) ([2fd4be4](https://github.com/ipfs/js-datastore-core/commit/2fd4be4b72ff203b4bc3643ca469aab7a17abbc3)) + +## [8.0.3](https://github.com/ipfs/js-datastore-core/compare/v8.0.2...v8.0.3) (2022-12-23) + + +### Dependencies + +* bump it-* deps ([#132](https://github.com/ipfs/js-datastore-core/issues/132)) ([4b7fd55](https://github.com/ipfs/js-datastore-core/commit/4b7fd559e21bfb5645037a177b32c973e7f05b4a)) + +## [8.0.2](https://github.com/ipfs/js-datastore-core/compare/v8.0.1...v8.0.2) (2022-10-12) + + +### Dependencies + +* update uint8arrays from 3.x.x to 4.x.x ([#121](https://github.com/ipfs/js-datastore-core/issues/121)) ([16d99b7](https://github.com/ipfs/js-datastore-core/commit/16d99b72454df33453f68d644a1daa6656d9dcf2)) + +## [8.0.1](https://github.com/ipfs/js-datastore-core/compare/v8.0.0...v8.0.1) (2022-08-12) + + +### Dependencies + +* update interface-datastore and interface-datastore-tests ([#118](https://github.com/ipfs/js-datastore-core/issues/118)) ([e5f47e2](https://github.com/ipfs/js-datastore-core/commit/e5f47e200059e056c34da1fcf4b795a3272f1459)) + +## [8.0.0](https://github.com/ipfs/js-datastore-core/compare/v7.0.3...v8.0.0) (2022-08-12) + + +### ⚠ BREAKING CHANGES + +* this module is now ESM-only + +### Features + +* switch to ESM-only ([#109](https://github.com/ipfs/js-datastore-core/issues/109)) ([cbaef20](https://github.com/ipfs/js-datastore-core/commit/cbaef2000a78bcdaa750e932b3a681d8e41cd727)) + +### [7.0.3](https://github.com/ipfs/js-datastore-core/compare/v7.0.2...v7.0.3) (2022-07-25) + + +### Bug Fixes + +* errors export ([#111](https://github.com/ipfs/js-datastore-core/issues/111)) ([be15e74](https://github.com/ipfs/js-datastore-core/commit/be15e74bb2a505aeee81d06da2dfdc07fc919096)), closes [#109](https://github.com/ipfs/js-datastore-core/issues/109) + +### [7.0.2](https://github.com/ipfs/js-datastore-core/compare/v7.0.1...v7.0.2) (2022-07-21) + + +### Bug Fixes + +* pass options to tiered put ([#108](https://github.com/ipfs/js-datastore-core/issues/108)) ([7a8f9ff](https://github.com/ipfs/js-datastore-core/commit/7a8f9ffb226c4940145637a6c881b835ce4886cb)) + + +### Trivial Changes + +* Update .github/workflows/stale.yml [skip ci] ([7e56180](https://github.com/ipfs/js-datastore-core/commit/7e56180b4d1da793a2ab3102135a4665b80bd462)) +* update package.json version ([54b543d](https://github.com/ipfs/js-datastore-core/commit/54b543d08aef9c75e6bd3cb015d2a3d000ef26c6)) + +### [7.0.1](https://github.com/ipfs/js-datastore-core/compare/v7.0.0...v7.0.1) (2022-01-28) + + +### Bug Fixes + +* fix generated types to missing import ([#86](https://github.com/ipfs/js-datastore-core/issues/86)) ([8805e71](https://github.com/ipfs/js-datastore-core/commit/8805e71f5840dac2dea4d3220b4f3975b39d5161)), closes [#85](https://github.com/ipfs/js-datastore-core/issues/85) + +## [7.0.0](https://github.com/ipfs/js-datastore-core/compare/v6.0.7...v7.0.0) (2022-01-19) + + +### ⚠ BREAKING CHANGES + +* key prefixes are no longer stripped by MountDatastore + +### Bug Fixes + +* do not strip prefixes for MountDatastores ([#82](https://github.com/ipfs/js-datastore-core/issues/82)) ([73bae74](https://github.com/ipfs/js-datastore-core/commit/73bae74ac794bf7585d432a3337e40fe85cb68f4)) + + +### Trivial Changes + +* switch to unified ci ([#83](https://github.com/ipfs/js-datastore-core/issues/83)) ([b3f79e6](https://github.com/ipfs/js-datastore-core/commit/b3f79e6d07440e37aabc83d1ebabe2b5c0e2bd30)) + +## [6.0.7](https://github.com/ipfs/js-datastore-core/compare/v6.0.6...v6.0.7) (2021-09-09) + + + +## [6.0.6](https://github.com/ipfs/js-datastore-core/compare/v6.0.5...v6.0.6) (2021-09-09) + + + +## [6.0.5](https://github.com/ipfs/js-datastore-core/compare/v6.0.4...v6.0.5) (2021-09-08) + + + +## [6.0.4](https://github.com/ipfs/js-datastore-core/compare/v6.0.3...v6.0.4) (2021-09-08) + + + +## [6.0.3](https://github.com/ipfs/js-datastore-core/compare/v6.0.2...v6.0.3) (2021-09-08) + + + +## [6.0.2](https://github.com/ipfs/js-datastore-core/compare/v6.0.1...v6.0.2) (2021-09-08) + + + +## [6.0.1](https://github.com/ipfs/js-datastore-core/compare/v6.0.0...v6.0.1) (2021-09-08) + + + +# [6.0.0](https://github.com/ipfs/js-datastore-core/compare/v5.0.2...v6.0.0) (2021-09-08) + + +### chore + +* update to esm ([#71](https://github.com/ipfs/js-datastore-core/issues/71)) ([ace9a06](https://github.com/ipfs/js-datastore-core/commit/ace9a06e879ff3fef9493754a7bbc7981595d9a4)) + + +### BREAKING CHANGES + +* deep requires/imports are no longer possible + + + +## [5.0.2](https://github.com/ipfs/js-datastore-core/compare/v5.0.1...v5.0.2) (2021-08-23) + + + +## [5.0.1](https://github.com/ipfs/js-datastore-core/compare/v5.0.0...v5.0.1) (2021-07-23) + + +### Bug Fixes + +* use streaming methods where possible ([#65](https://github.com/ipfs/js-datastore-core/issues/65)) ([ec90398](https://github.com/ipfs/js-datastore-core/commit/ec9039827829dcd0e6e5c652ecb8707e80c48010)) + + + +# [5.0.0](https://github.com/ipfs/js-datastore-core/compare/v4.0.0...v5.0.0) (2021-07-06) + + +### chore + +* update deps ([e154404](https://github.com/ipfs/js-datastore-core/commit/e154404317838f147fbb9ab11417a8d671aba9a4)) + + +### BREAKING CHANGES + +* implements new interface-datastore types, will cause duplicates in the dep tree + + + +# [4.0.0](https://github.com/ipfs/js-datastore-core/compare/v3.0.0...v4.0.0) (2021-04-15) + + +### Bug Fixes + +* fix async sharding tests ([#49](https://github.com/ipfs/js-datastore-core/issues/49)) ([a546afb](https://github.com/ipfs/js-datastore-core/commit/a546afb0cef751025aac73c5f3bcfa3bedd1dff4)) + + +### Features + +* split .query into .query and .queryKeys ([#59](https://github.com/ipfs/js-datastore-core/issues/59)) ([6f829db](https://github.com/ipfs/js-datastore-core/commit/6f829db2e8d1aed3b9c36a7bb95625a15c077e02)) + + + +# [3.0.0](https://github.com/ipfs/js-datastore-core/compare/v2.0.1...v3.0.0) (2021-01-22) + + +### Bug Fixes + +* fix ShardingDatastore creation ([#48](https://github.com/ipfs/js-datastore-core/issues/48)) ([21b48e7](https://github.com/ipfs/js-datastore-core/commit/21b48e7ccc965a1422f8a0f7eadbd83715b5cac6)) + + +### Features + +* ts types, github ci and clean up ([#39](https://github.com/ipfs/js-datastore-core/issues/39)) ([bee45ae](https://github.com/ipfs/js-datastore-core/commit/bee45ae0b778171d0919e135ea8affcf2d6de635)) + + + +## [2.0.1](https://github.com/ipfs/js-datastore-core/compare/v2.0.0...v2.0.1) (2020-11-09) + + + + +# [2.0.0](https://github.com/ipfs/js-datastore-core/compare/v1.1.0...v2.0.0) (2020-07-29) + + +### Bug Fixes + +* remove node buffers ([#27](https://github.com/ipfs/js-datastore-core/issues/27)) ([a9786b9](https://github.com/ipfs/js-datastore-core/commit/a9786b9)) + + +### BREAKING CHANGES + +* no longer uses node Buffers, only Uint8Arrays + + + + +# [1.1.0](https://github.com/ipfs/js-datastore-core/compare/v1.0.0...v1.1.0) (2020-05-07) + + +### Bug Fixes + +* **ci:** add empty commit to fix lint checks on master ([a19da65](https://github.com/ipfs/js-datastore-core/commit/a19da65)) + + +### Features + +* add streaming/cancellable API ([#23](https://github.com/ipfs/js-datastore-core/issues/23)) ([5e7858e](https://github.com/ipfs/js-datastore-core/commit/5e7858e)) + + + + +# [1.0.0](https://github.com/ipfs/js-datastore-core/compare/v0.7.0...v1.0.0) (2020-04-06) + + +### Bug Fixes + +* add buffer and cleanup ([#22](https://github.com/ipfs/js-datastore-core/issues/22)) ([f0f64a9](https://github.com/ipfs/js-datastore-core/commit/f0f64a9)) + + + + +# [0.7.0](https://github.com/ipfs/js-datastore-core/compare/v0.6.1...v0.7.0) (2019-05-29) + + + + +## [0.6.1](https://github.com/ipfs/js-datastore-core/compare/v0.6.0...v0.6.1) (2019-05-23) + + +### Bug Fixes + +* remove leftpad and cleanup ([8558fcd](https://github.com/ipfs/js-datastore-core/commit/8558fcd)) + + + + +# [0.6.0](https://github.com/ipfs/js-datastore-core/compare/v0.5.0...v0.6.0) (2018-10-24) + + + + +# [0.5.0](https://github.com/ipfs/js-datastore-core/compare/v0.4.0...v0.5.0) (2018-09-19) + + +### Bug Fixes + +* **ci:** build on appveyor ([#9](https://github.com/ipfs/js-datastore-core/issues/9)) ([687314b](https://github.com/ipfs/js-datastore-core/commit/687314b)) + + +### Features + +* add basic error codes ([bf79768](https://github.com/ipfs/js-datastore-core/commit/bf79768)) + + + + +# [0.4.0](https://github.com/ipfs/js-datastore-core/compare/v0.3.0...v0.4.0) (2017-11-04) + + +### Bug Fixes + +* sharding and query for windows interop ([#6](https://github.com/ipfs/js-datastore-core/issues/6)) ([845316d](https://github.com/ipfs/js-datastore-core/commit/845316d)) + + + + +# [0.3.0](https://github.com/ipfs/js-datastore-core/compare/v0.2.0...v0.3.0) (2017-07-23) + + + + +# [0.2.0](https://github.com/ipfs/js-datastore-core/compare/v0.1.0...v0.2.0) (2017-03-23) + + +### Features + +* add open method ([1462fcc](https://github.com/ipfs/js-datastore-core/commit/1462fcc)) + + + + +# 0.1.0 (2017-03-15) diff --git a/packages/datastore-core/LICENSE b/packages/datastore-core/LICENSE new file mode 100644 index 00000000..20ce483c --- /dev/null +++ b/packages/datastore-core/LICENSE @@ -0,0 +1,4 @@ +This project is dual licensed under MIT and Apache-2.0. + +MIT: https://www.opensource.org/licenses/mit +Apache-2.0: https://www.apache.org/licenses/license-2.0 diff --git a/packages/datastore-core/LICENSE-APACHE b/packages/datastore-core/LICENSE-APACHE new file mode 100644 index 00000000..14478a3b --- /dev/null +++ b/packages/datastore-core/LICENSE-APACHE @@ -0,0 +1,5 @@ +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/packages/datastore-core/LICENSE-MIT b/packages/datastore-core/LICENSE-MIT new file mode 100644 index 00000000..72dc60d8 --- /dev/null +++ b/packages/datastore-core/LICENSE-MIT @@ -0,0 +1,19 @@ +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/packages/datastore-core/README.md b/packages/datastore-core/README.md new file mode 100644 index 00000000..4cafec3c --- /dev/null +++ b/packages/datastore-core/README.md @@ -0,0 +1,115 @@ +# datastore-core + +[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) +[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/js-stores.svg?style=flat-square)](https://codecov.io/gh/ipfs/js-stores) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/js-stores/js-test-and-release.yml?branch=master\&style=flat-square)](https://github.com/ipfs/js-stores/actions/workflows/js-test-and-release.yml?query=branch%3Amaster) + +> Wrapper implementation for interface-datastore + +## Table of contents + +- [Install](#install) + - [Browser ` +``` + +## Implementations + +- Wrapper Implementations + - Mount: [`src/mount`](src/mount.js) + - Keytransform: [`src/keytransform`](src/keytransform.js) + - Sharding: [`src/sharding`](src/sharding.js) + - Tiered: [`src/tiered`](src/tirered.js) + - Namespace: [`src/namespace`](src/namespace.js) + +## Usage + +### BaseDatastore + +An base store is made available to make implementing your own datastore easier: + +```javascript +import { BaseDatastore } from 'datastore-core' + +class MyDatastore extends BaseDatastore { + constructor () { + super() + } + + async put (key, val) { + // your implementation here + } + + async get (key) { + // your implementation here + } + + // etc... +} +``` + +See the [MemoryDatastore](./src/memory.js) for an example of how it is used. + +### Wrapping Stores + +```js +import { Key } from 'interface-datastore' +import { + MemoryStore, + MountStore +} from 'datastore-core' + +const store = new MountStore({prefix: new Key('/a'), datastore: new MemoryStore()}) +``` + +## Contribute + +Feel free to join in. All welcome. Open an [issue](https://github.com/ipfs/js-ipfs-unixfs-importer/issues)! + +This repository falls under the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) + +## API Docs + +- + +## License + +Licensed under either of + +- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) +- MIT ([LICENSE-MIT](LICENSE-MIT) / ) + +## Contribute + +Contributions welcome! Please check out [the issues](https://github.com/ipfs/js-stores/issues). + +Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. + +Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) diff --git a/packages/datastore-core/package.json b/packages/datastore-core/package.json new file mode 100644 index 00000000..958d6ca2 --- /dev/null +++ b/packages/datastore-core/package.json @@ -0,0 +1,219 @@ +{ + "name": "datastore-core", + "version": "9.0.4", + "description": "Wrapper implementation for interface-datastore", + "author": "Friedel Ziegelmayer ", + "license": "Apache-2.0 OR MIT", + "homepage": "https://github.com/ipfs/js-stores/tree/master/packages/datastore-core#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/ipfs/js-stores.git" + }, + "bugs": { + "url": "https://github.com/ipfs/js-stores/issues" + }, + "keywords": [ + "datastore", + "interface", + "ipfs", + "key-value" + ], + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + }, + "type": "module", + "types": "./dist/src/index.d.ts", + "typesVersions": { + "*": { + "*": [ + "*", + "dist/*", + "dist/src/*", + "dist/src/*/index" + ], + "src/*": [ + "*", + "dist/*", + "dist/src/*", + "dist/src/*/index" + ] + } + }, + "files": [ + "src", + "dist", + "!dist/test", + "!**/*.tsbuildinfo" + ], + "exports": { + ".": { + "types": "./dist/src/index.d.ts", + "import": "./dist/src/index.js" + }, + "./base": { + "types": "./dist/src/base.d.ts", + "import": "./dist/src/base.js" + }, + "./errors": { + "types": "./dist/src/errors.d.ts", + "import": "./dist/src/errors.js" + }, + "./keytransform": { + "types": "./dist/src/keytransform.d.ts", + "import": "./dist/src/keytransform.js" + }, + "./memory": { + "types": "./dist/src/memory.d.ts", + "import": "./dist/src/memory.js" + }, + "./mount": { + "types": "./dist/src/mount.d.ts", + "import": "./dist/src/mount.js" + }, + "./namespace": { + "types": "./dist/src/namespace.d.ts", + "import": "./dist/src/namespace.js" + }, + "./shard": { + "types": "./dist/src/shard.d.ts", + "import": "./dist/src/shard.js" + }, + "./sharding": { + "types": "./dist/src/sharding.d.ts", + "import": "./dist/src/sharding.js" + }, + "./tiered": { + "types": "./dist/src/tiered.d.ts", + "import": "./dist/src/tiered.js" + } + }, + "eslintConfig": { + "extends": "ipfs", + "parserOptions": { + "sourceType": "module" + } + }, + "release": { + "branches": [ + "master" + ], + "plugins": [ + [ + "@semantic-release/commit-analyzer", + { + "preset": "conventionalcommits", + "releaseRules": [ + { + "breaking": true, + "release": "major" + }, + { + "revert": true, + "release": "patch" + }, + { + "type": "feat", + "release": "minor" + }, + { + "type": "fix", + "release": "patch" + }, + { + "type": "docs", + "release": "patch" + }, + { + "type": "test", + "release": "patch" + }, + { + "type": "deps", + "release": "patch" + }, + { + "scope": "no-release", + "release": false + } + ] + } + ], + [ + "@semantic-release/release-notes-generator", + { + "preset": "conventionalcommits", + "presetConfig": { + "types": [ + { + "type": "feat", + "section": "Features" + }, + { + "type": "fix", + "section": "Bug Fixes" + }, + { + "type": "chore", + "section": "Trivial Changes" + }, + { + "type": "docs", + "section": "Documentation" + }, + { + "type": "deps", + "section": "Dependencies" + }, + { + "type": "test", + "section": "Tests" + } + ] + } + } + ], + "@semantic-release/changelog", + "@semantic-release/npm", + "@semantic-release/github", + "@semantic-release/git" + ] + }, + "scripts": { + "clean": "aegir clean", + "lint": "aegir lint", + "build": "aegir build", + "release": "aegir release", + "test": "aegir test", + "test:node": "aegir test -t node --cov", + "test:chrome": "aegir test -t browser --cov", + "test:chrome-webworker": "aegir test -t webworker", + "test:firefox": "aegir test -t browser -- --browser firefox", + "test:firefox-webworker": "aegir test -t webworker -- --browser firefox", + "test:electron-main": "aegir test -t electron-main", + "dep-check": "aegir dep-check" + }, + "dependencies": { + "@libp2p/logger": "^2.0.0", + "err-code": "^3.0.1", + "interface-store": "^5.0.0", + "it-all": "^2.0.0", + "it-drain": "^2.0.0", + "it-filter": "^2.0.1", + "it-map": "^2.0.1", + "it-merge": "^2.0.0", + "it-pipe": "^2.0.3", + "it-pushable": "^3.0.0", + "it-sort": "^2.0.1", + "it-take": "^2.0.0", + "uint8arrays": "^4.0.2" + }, + "devDependencies": { + "aegir": "^38.1.7", + "interface-datastore": "^8.0.0", + "interface-datastore-tests": "^5.0.0" + }, + "typedoc": { + "entryPoint": "./src/index.ts" + } +} diff --git a/packages/datastore-core/src/base.ts b/packages/datastore-core/src/base.ts new file mode 100644 index 00000000..821112eb --- /dev/null +++ b/packages/datastore-core/src/base.ts @@ -0,0 +1,144 @@ +import sort from 'it-sort' +import drain from 'it-drain' +import filter from 'it-filter' +import take from 'it-take' +import type { Batch, Datastore, Key, KeyQuery, Pair, Query } from 'interface-datastore' +import type { AbortOptions, Await, AwaitIterable } from 'interface-store' + +export class BaseDatastore implements Datastore { + put (key: Key, val: Uint8Array, options?: AbortOptions): Await { + return Promise.reject(new Error('.put is not implemented')) + } + + get (key: Key, options?: AbortOptions): Await { + return Promise.reject(new Error('.get is not implemented')) + } + + has (key: Key, options?: AbortOptions): Await { + return Promise.reject(new Error('.has is not implemented')) + } + + delete (key: Key, options?: AbortOptions): Await { + return Promise.reject(new Error('.delete is not implemented')) + } + + async * putMany (source: AwaitIterable, options: AbortOptions = {}): AwaitIterable { + for await (const { key, value } of source) { + await this.put(key, value, options) + yield key + } + } + + async * getMany (source: AwaitIterable, options: AbortOptions = {}): AwaitIterable { + for await (const key of source) { + yield { + key, + value: await this.get(key, options) + } + } + } + + async * deleteMany (source: AwaitIterable, options: AbortOptions = {}): AwaitIterable { + for await (const key of source) { + await this.delete(key, options) + yield key + } + } + + batch (): Batch { + let puts: Pair[] = [] + let dels: Key[] = [] + + return { + put (key, value) { + puts.push({ key, value }) + }, + + delete (key) { + dels.push(key) + }, + commit: async (options) => { + await drain(this.putMany(puts, options)) + puts = [] + await drain(this.deleteMany(dels, options)) + dels = [] + } + } + } + + /** + * Extending classes should override `query` or implement this method + */ + // eslint-disable-next-line require-yield + async * _all (q: Query, options?: AbortOptions): AwaitIterable { + throw new Error('._all is not implemented') + } + + /** + * Extending classes should override `queryKeys` or implement this method + */ + // eslint-disable-next-line require-yield + async * _allKeys (q: KeyQuery, options?: AbortOptions): AwaitIterable { + throw new Error('._allKeys is not implemented') + } + + query (q: Query, options?: AbortOptions): AwaitIterable { + let it = this._all(q, options) + + if (q.prefix != null) { + const prefix = q.prefix + it = filter(it, (e) => e.key.toString().startsWith(prefix)) + } + + if (Array.isArray(q.filters)) { + it = q.filters.reduce((it, f) => filter(it, f), it) + } + + if (Array.isArray(q.orders)) { + it = q.orders.reduce((it, f) => sort(it, f), it) + } + + if (q.offset != null) { + let i = 0 + const offset = q.offset + it = filter(it, () => i++ >= offset) + } + + if (q.limit != null) { + it = take(it, q.limit) + } + + return it + } + + queryKeys (q: KeyQuery, options?: AbortOptions): AwaitIterable { + let it = this._allKeys(q, options) + + if (q.prefix != null) { + const prefix = q.prefix + it = filter(it, (key) => + key.toString().startsWith(prefix) + ) + } + + if (Array.isArray(q.filters)) { + it = q.filters.reduce((it, f) => filter(it, f), it) + } + + if (Array.isArray(q.orders)) { + it = q.orders.reduce((it, f) => sort(it, f), it) + } + + if (q.offset != null) { + const offset = q.offset + let i = 0 + it = filter(it, () => i++ >= offset) + } + + if (q.limit != null) { + it = take(it, q.limit) + } + + return it + } +} diff --git a/packages/datastore-core/src/errors.ts b/packages/datastore-core/src/errors.ts new file mode 100644 index 00000000..8009aff7 --- /dev/null +++ b/packages/datastore-core/src/errors.ts @@ -0,0 +1,31 @@ +import errCode from 'err-code' + +export function dbOpenFailedError (err?: Error): Error { + err = err ?? new Error('Cannot open database') + return errCode(err, 'ERR_DB_OPEN_FAILED') +} + +export function dbDeleteFailedError (err?: Error): Error { + err = err ?? new Error('Delete failed') + return errCode(err, 'ERR_DB_DELETE_FAILED') +} + +export function dbWriteFailedError (err?: Error): Error { + err = err ?? new Error('Write failed') + return errCode(err, 'ERR_DB_WRITE_FAILED') +} + +export function dbReadFailedError (err?: Error): Error { + err = err ?? new Error('Read failed') + return errCode(err, 'ERR_DB_READ_FAILED') +} + +export function notFoundError (err?: Error): Error { + err = err ?? new Error('Not Found') + return errCode(err, 'ERR_NOT_FOUND') +} + +export function abortedError (err?: Error): Error { + err = err ?? new Error('Aborted') + return errCode(err, 'ERR_ABORTED') +} diff --git a/packages/datastore-core/src/index.ts b/packages/datastore-core/src/index.ts new file mode 100644 index 00000000..04f205c6 --- /dev/null +++ b/packages/datastore-core/src/index.ts @@ -0,0 +1,28 @@ +import type { Key } from 'interface-datastore' + +import * as Errors from './errors.js' +import * as shard from './shard.js' + +export { BaseDatastore } from './base.js' +export { MemoryDatastore } from './memory.js' +export { KeyTransformDatastore } from './keytransform.js' +export { ShardingDatastore } from './sharding.js' +export { MountDatastore } from './mount.js' +export { TieredDatastore } from './tiered.js' +export { NamespaceDatastore } from './namespace.js' + +export { Errors } +export { shard } + +export interface Shard { + name: string + param: number + readonly _padding: string + fun: (s: string) => string + toString: () => string +} + +export interface KeyTransform { + convert: (key: Key) => Key + invert: (key: Key) => Key +} diff --git a/packages/datastore-core/src/keytransform.ts b/packages/datastore-core/src/keytransform.ts new file mode 100644 index 00000000..a9d2c9f6 --- /dev/null +++ b/packages/datastore-core/src/keytransform.ts @@ -0,0 +1,181 @@ +import { BaseDatastore } from './base.js' +import map from 'it-map' +import { pipe } from 'it-pipe' +import type { KeyTransform } from './index.js' +import type { Batch, Datastore, Key, KeyQuery, Pair, Query } from 'interface-datastore' +import type { AbortOptions, AwaitIterable } from 'interface-store' + +/** + * A datastore shim, that wraps around a given datastore, changing + * the way keys look to the user, for example namespacing + * keys, reversing them, etc. + */ +export class KeyTransformDatastore extends BaseDatastore { + private readonly child: Datastore + public transform: KeyTransform + + constructor (child: Datastore, transform: KeyTransform) { + super() + + this.child = child + this.transform = transform + } + + async put (key: Key, val: Uint8Array, options?: AbortOptions): Promise { + await this.child.put(this.transform.convert(key), val, options) + + return key + } + + async get (key: Key, options?: AbortOptions): Promise { + return await this.child.get(this.transform.convert(key), options) + } + + async has (key: Key, options?: AbortOptions): Promise { + return await this.child.has(this.transform.convert(key), options) + } + + async delete (key: Key, options?: AbortOptions): Promise { + await this.child.delete(this.transform.convert(key), options) + } + + async * putMany (source: AwaitIterable, options: AbortOptions = {}): AsyncIterable { + const transform = this.transform + const child = this.child + + yield * pipe( + source, + async function * (source) { + yield * map(source, ({ key, value }) => ({ + key: transform.convert(key), + value + })) + }, + async function * (source) { + yield * child.putMany(source, options) + }, + async function * (source) { + yield * map(source, key => transform.invert(key)) + } + ) + } + + async * getMany (source: AwaitIterable, options: AbortOptions = {}): AsyncIterable { + const transform = this.transform + const child = this.child + + yield * pipe( + source, + async function * (source) { + yield * map(source, key => transform.convert(key)) + }, + async function * (source) { + yield * child.getMany(source, options) + }, + async function * (source) { + yield * map(source, ({ key, value }) => ({ + key: transform.invert(key), + value + })) + } + ) + } + + async * deleteMany (source: AwaitIterable, options: AbortOptions = {}): AsyncIterable { + const transform = this.transform + const child = this.child + + yield * pipe( + source, + async function * (source) { + yield * map(source, key => transform.convert(key)) + }, + async function * (source) { + yield * child.deleteMany(source, options) + }, + async function * (source) { + yield * map(source, key => transform.invert(key)) + } + ) + } + + batch (): Batch { + const b = this.child.batch() + return { + put: (key, value) => { + b.put(this.transform.convert(key), value) + }, + delete: (key) => { + b.delete(this.transform.convert(key)) + }, + commit: async (options) => { + await b.commit(options) + } + } + } + + query (q: Query, options?: AbortOptions): AsyncIterable { + const query: Query = { + ...q + } + + query.filters = (query.filters ?? []).map(filter => { + return ({ key, value }) => filter({ key: this.transform.convert(key), value }) + }) + + const { prefix } = q + if (prefix != null && prefix !== '/') { + delete query.prefix + query.filters.push(({ key }) => { + return this.transform.invert(key).toString().startsWith(prefix) + }) + } + + if (query.orders != null) { + query.orders = query.orders.map(order => { + return (a, b) => order( + { key: this.transform.invert(a.key), value: a.value }, + { key: this.transform.invert(b.key), value: b.value } + ) + }) + } + + return map(this.child.query(query, options), ({ key, value }) => { + return { + key: this.transform.invert(key), + value + } + }) + } + + queryKeys (q: KeyQuery, options?: AbortOptions): AsyncIterable { + const query = { + ...q + } + + query.filters = (query.filters ?? []).map(filter => { + return (key) => filter(this.transform.convert(key)) + }) + + const { prefix } = q + if (prefix != null && prefix !== '/') { + delete query.prefix + query.filters.push((key) => { + return this.transform.invert(key).toString().startsWith(prefix) + }) + } + + if (query.orders != null) { + query.orders = query.orders.map(order => { + return (a, b) => order( + this.transform.invert(a), + this.transform.invert(b) + ) + }) + } + + return map(this.child.queryKeys(query, options), key => { + return this.transform.invert(key) + }) + } +} diff --git a/packages/datastore-core/src/memory.ts b/packages/datastore-core/src/memory.ts new file mode 100644 index 00000000..ad5cce4b --- /dev/null +++ b/packages/datastore-core/src/memory.ts @@ -0,0 +1,51 @@ +import { BaseDatastore } from './base.js' +import { Key } from 'interface-datastore/key' +import * as Errors from './errors.js' +import type { Pair } from 'interface-datastore' +import type { Await, AwaitIterable } from 'interface-store' + +export class MemoryDatastore extends BaseDatastore { + private readonly data: Map + + constructor () { + super() + + this.data = new Map() + } + + put (key: Key, val: Uint8Array): Await { // eslint-disable-line require-await + this.data.set(key.toString(), val) + + return key + } + + get (key: Key): Await { + const result = this.data.get(key.toString()) + + if (result == null) { + throw Errors.notFoundError() + } + + return result + } + + has (key: Key): Await { // eslint-disable-line require-await + return this.data.has(key.toString()) + } + + delete (key: Key): Await { // eslint-disable-line require-await + this.data.delete(key.toString()) + } + + * _all (): AwaitIterable { + for (const [key, value] of this.data.entries()) { + yield { key: new Key(key), value } + } + } + + * _allKeys (): AwaitIterable { + for (const key of this.data.keys()) { + yield new Key(key) + } + } +} diff --git a/packages/datastore-core/src/mount.ts b/packages/datastore-core/src/mount.ts new file mode 100644 index 00000000..80e7b8c6 --- /dev/null +++ b/packages/datastore-core/src/mount.ts @@ -0,0 +1,153 @@ +import filter from 'it-filter' +import take from 'it-take' +import merge from 'it-merge' +import { BaseDatastore } from './base.js' +import * as Errors from './errors.js' +import sort from 'it-sort' +import type { Batch, Datastore, Key, KeyQuery, Pair, Query } from 'interface-datastore' +import type { AbortOptions } from 'interface-store' + +/** + * A datastore that can combine multiple stores inside various + * key prefixes + */ +export class MountDatastore extends BaseDatastore { + private readonly mounts: Array<{ prefix: Key, datastore: Datastore }> + + constructor (mounts: Array<{ prefix: Key, datastore: Datastore }>) { + super() + + this.mounts = mounts.slice() + } + + /** + * Lookup the matching datastore for the given key + */ + private _lookup (key: Key): { datastore: Datastore, mountpoint: Key } | undefined { + for (const mount of this.mounts) { + if (mount.prefix.toString() === key.toString() || mount.prefix.isAncestorOf(key)) { + return { + datastore: mount.datastore, + mountpoint: mount.prefix + } + } + } + } + + async put (key: Key, value: Uint8Array, options?: AbortOptions): Promise { + const match = this._lookup(key) + if (match == null) { + throw Errors.dbWriteFailedError(new Error('No datastore mounted for this key')) + } + + await match.datastore.put(key, value, options) + + return key + } + + /** + * @param {Key} key + * @param {Options} [options] + */ + async get (key: Key, options: AbortOptions = {}): Promise { + const match = this._lookup(key) + if (match == null) { + throw Errors.notFoundError(new Error('No datastore mounted for this key')) + } + return await match.datastore.get(key, options) + } + + async has (key: Key, options?: AbortOptions): Promise { + const match = this._lookup(key) + if (match == null) { + return await Promise.resolve(false) + } + return await match.datastore.has(key, options) + } + + async delete (key: Key, options?: AbortOptions): Promise { + const match = this._lookup(key) + if (match == null) { + throw Errors.dbDeleteFailedError(new Error('No datastore mounted for this key')) + } + + await match.datastore.delete(key, options) + } + + batch (): Batch { + const batchMounts: Record = {} + + const lookup = (key: Key): { batch: Batch } => { + const match = this._lookup(key) + if (match == null) { + throw new Error('No datastore mounted for this key') + } + + const m = match.mountpoint.toString() + if (batchMounts[m] == null) { + batchMounts[m] = match.datastore.batch() + } + + return { + batch: batchMounts[m] + } + } + + return { + put: (key, value) => { + const match = lookup(key) + match.batch.put(key, value) + }, + delete: (key) => { + const match = lookup(key) + match.batch.delete(key) + }, + commit: async (options) => { + await Promise.all(Object.keys(batchMounts).map(async p => { await batchMounts[p].commit(options) })) + } + } + } + + query (q: Query, options?: AbortOptions): AsyncIterable { + const qs = this.mounts.map(m => { + return m.datastore.query({ + prefix: q.prefix, + filters: q.filters + }, options) + }) + + let it = merge(...qs) + if (q.filters != null) q.filters.forEach(f => { it = filter(it, f) }) + if (q.orders != null) q.orders.forEach(o => { it = sort(it, o) }) + if (q.offset != null) { + let i = 0 + const offset = q.offset + it = filter(it, () => i++ >= offset) + } + if (q.limit != null) it = take(it, q.limit) + + return it + } + + queryKeys (q: KeyQuery, options?: AbortOptions): AsyncIterable { + const qs = this.mounts.map(m => { + return m.datastore.queryKeys({ + prefix: q.prefix, + filters: q.filters + }, options) + }) + + /** @type AsyncIterable */ + let it = merge(...qs) + if (q.filters != null) q.filters.forEach(f => { it = filter(it, f) }) + if (q.orders != null) q.orders.forEach(o => { it = sort(it, o) }) + if (q.offset != null) { + let i = 0 + const offset = q.offset + it = filter(it, () => i++ >= offset) + } + if (q.limit != null) it = take(it, q.limit) + + return it + } +} diff --git a/packages/datastore-core/src/namespace.ts b/packages/datastore-core/src/namespace.ts new file mode 100644 index 00000000..08f0dcd6 --- /dev/null +++ b/packages/datastore-core/src/namespace.ts @@ -0,0 +1,32 @@ +import { Key } from 'interface-datastore' +import type { Datastore } from 'interface-datastore' +import { KeyTransformDatastore } from './keytransform.js' + +/** + * Wraps a given datastore into a keytransform which + * makes a given prefix transparent. + * + * For example, if the prefix is `new Key(/hello)` a call + * to `store.put(new Key('/world'), mydata)` would store the data under + * `/hello/world`. + */ +export class NamespaceDatastore extends KeyTransformDatastore { + constructor (child: Datastore, prefix: Key) { + super(child, { + convert (key) { + return prefix.child(key) + }, + invert (key) { + if (prefix.toString() === '/') { + return key + } + + if (!prefix.isAncestorOf(key)) { + throw new Error(`Expected prefix: (${prefix.toString()}) in key: ${key.toString()}`) + } + + return new Key(key.toString().slice(prefix.toString().length), false) + } + }) + } +} diff --git a/packages/datastore-core/src/shard.ts b/packages/datastore-core/src/shard.ts new file mode 100644 index 00000000..4005f1c5 --- /dev/null +++ b/packages/datastore-core/src/shard.ts @@ -0,0 +1,115 @@ +import type { Datastore } from 'interface-datastore' +import { Key } from 'interface-datastore/key' +import type { Shard } from './index.js' + +export const PREFIX = '/repo/flatfs/shard/' +export const SHARDING_FN = 'SHARDING' + +export class ShardBase implements Shard { + public param: number + public name: string + public _padding: string + + constructor (param: number) { + this.param = param + this.name = 'base' + this._padding = '' + } + + fun (s: string): string { + return 'implement me' + } + + toString (): string { + return `${PREFIX}v1/${this.name}/${this.param}` + } +} + +export class Prefix extends ShardBase { + constructor (prefixLen: number) { + super(prefixLen) + this._padding = ''.padStart(prefixLen, '_') + this.name = 'prefix' + } + + fun (noslash: string): string { + return (noslash + this._padding).slice(0, this.param) + } +} + +export class Suffix extends ShardBase { + constructor (suffixLen: number) { + super(suffixLen) + + this._padding = ''.padStart(suffixLen, '_') + this.name = 'suffix' + } + + fun (noslash: string): string { + const s = this._padding + noslash + return s.slice(s.length - this.param) + } +} + +export class NextToLast extends ShardBase { + constructor (suffixLen: number) { + super(suffixLen) + this._padding = ''.padStart(suffixLen + 1, '_') + this.name = 'next-to-last' + } + + fun (noslash: string): string { + const s = this._padding + noslash + const offset = s.length - this.param - 1 + return s.slice(offset, offset + this.param) + } +} + +/** + * Convert a given string to the matching sharding function + */ +export function parseShardFun (str: string): Shard { + str = str.trim() + + if (str.length === 0) { + throw new Error('empty shard string') + } + + if (!str.startsWith(PREFIX)) { + throw new Error(`invalid or no path prefix: ${str}`) + } + + const parts = str.slice(PREFIX.length).split('/') + const version = parts[0] + + if (version !== 'v1') { + throw new Error(`expect 'v1' version, got '${version}'`) + } + + const name = parts[1] + + if (parts[2] == null || parts[2] === '') { + throw new Error('missing param') + } + + const param = parseInt(parts[2], 10) + + switch (name) { + case 'prefix': + return new Prefix(param) + case 'suffix': + return new Suffix(param) + case 'next-to-last': + return new NextToLast(param) + default: + throw new Error(`unkown sharding function: ${name}`) + } +} + +export const readShardFun = async (path: string | Uint8Array, store: Datastore): Promise => { + const key = new Key(path).child(new Key(SHARDING_FN)) + // @ts-expect-error + const get = typeof store.getRaw === 'function' ? store.getRaw.bind(store) : store.get.bind(store) + const res = await get(key) + return parseShardFun(new TextDecoder().decode(res ?? '').trim()) +} diff --git a/packages/datastore-core/src/sharding.ts b/packages/datastore-core/src/sharding.ts new file mode 100644 index 00000000..06e191dc --- /dev/null +++ b/packages/datastore-core/src/sharding.ts @@ -0,0 +1,143 @@ +import { Batch, Key, KeyQuery, KeyQueryFilter, Pair, Query, QueryFilter } from 'interface-datastore' +import { + readShardFun, + SHARDING_FN +} from './shard.js' +import { BaseDatastore } from './base.js' +import { KeyTransformDatastore } from './keytransform.js' +import * as Errors from './errors.js' +import type { Shard } from './index.js' +import type { Datastore } from 'interface-datastore' +import type { AbortOptions, AwaitIterable } from 'interface-store' + +const shardKey = new Key(SHARDING_FN) + +/** + * Backend independent abstraction of go-ds-flatfs. + * + * Wraps another datastore such that all values are stored + * sharded according to the given sharding function. + */ +export class ShardingDatastore extends BaseDatastore { + private readonly child: KeyTransformDatastore + private shard: Shard + + constructor (store: Datastore, shard: Shard) { + super() + + this.child = new KeyTransformDatastore(store, { + convert: this._convertKey.bind(this), + invert: this._invertKey.bind(this) + }) + this.shard = shard + } + + async open (): Promise { + this.shard = await ShardingDatastore.create(this.child, this.shard) + } + + _convertKey (key: Key): Key { + const s = key.toString() + if (s === shardKey.toString()) { + return key + } + + const parent = new Key(this.shard.fun(s)) + return parent.child(key) + } + + _invertKey (key: Key): Key { + const s = key.toString() + if (s === shardKey.toString()) { + return key + } + return Key.withNamespaces(key.list().slice(1)) + } + + static async create (store: Datastore, shard?: Shard): Promise { + const hasShard = await store.has(shardKey) + + if (!hasShard) { + if (shard == null) { + throw Errors.dbOpenFailedError(Error('Shard is required when datastore doesn\'t have a shard key already.')) + } + + await store.put(shardKey, new TextEncoder().encode(shard.toString() + '\n')) + } + + if (shard == null) { + shard = await readShardFun('/', store) + } + + // test shards + const diskShard = await readShardFun('/', store) + const a = diskShard.toString() + const b = shard.toString() + + if (a !== b) { + throw new Error(`specified fun ${b} does not match repo shard fun ${a}`) + } + + return diskShard + } + + async put (key: Key, val: Uint8Array, options?: AbortOptions): Promise { + await this.child.put(key, val, options) + + return key + } + + async get (key: Key, options?: AbortOptions): Promise { + return await this.child.get(key, options) + } + + async has (key: Key, options?: AbortOptions): Promise { + return await this.child.has(key, options) + } + + async delete (key: Key, options?: AbortOptions): Promise { + await this.child.delete(key, options) + } + + async * putMany (source: AwaitIterable, options: AbortOptions = {}): AsyncIterable { + yield * this.child.putMany(source, options) + } + + async * getMany (source: AwaitIterable, options: AbortOptions = {}): AsyncIterable { + yield * this.child.getMany(source, options) + } + + async * deleteMany (source: AwaitIterable, options: AbortOptions = {}): AsyncIterable { + yield * this.child.deleteMany(source, options) + } + + batch (): Batch { + return this.child.batch() + } + + query (q: Query, options?: AbortOptions): AsyncIterable { + const omitShard: QueryFilter = ({ key }) => key.toString() !== shardKey.toString() + + const tq: Query = { + ...q, + filters: [ + omitShard + ].concat(q.filters ?? []) + } + + return this.child.query(tq, options) + } + + queryKeys (q: KeyQuery, options?: AbortOptions): AsyncIterable { + const omitShard: KeyQueryFilter = (key) => key.toString() !== shardKey.toString() + + const tq: KeyQuery = { + ...q, + filters: [ + omitShard + ].concat(q.filters ?? []) + } + + return this.child.queryKeys(tq, options) + } +} diff --git a/packages/datastore-core/src/tiered.ts b/packages/datastore-core/src/tiered.ts new file mode 100644 index 00000000..c42084d8 --- /dev/null +++ b/packages/datastore-core/src/tiered.ts @@ -0,0 +1,153 @@ +import { BaseDatastore } from './base.js' +import * as Errors from './errors.js' +import { logger } from '@libp2p/logger' +import { pushable } from 'it-pushable' +import drain from 'it-drain' +import type { Batch, Datastore, Key, KeyQuery, Pair, Query } from 'interface-datastore' +import type { AbortOptions, AwaitIterable } from 'interface-store' + +const log = logger('datastore:core:tiered') + +/** + * A datastore that can combine multiple stores. Puts and deletes + * will write through to all datastores. Has and get will + * try each store sequentially. Query will always try the + * last one first. + * + */ +export class TieredDatastore extends BaseDatastore { + private readonly stores: Datastore[] + + constructor (stores: Datastore[]) { + super() + + this.stores = stores.slice() + } + + async put (key: Key, value: Uint8Array, options?: AbortOptions): Promise { + try { + await Promise.all(this.stores.map(async store => { await store.put(key, value, options) })) + return key + } catch (err: any) { + throw Errors.dbWriteFailedError(err) + } + } + + async get (key: Key, options?: AbortOptions): Promise { + for (const store of this.stores) { + try { + const res = await store.get(key, options) + if (res != null) return res + } catch (err) { + log.error(err) + } + } + throw Errors.notFoundError() + } + + async has (key: Key, options?: AbortOptions): Promise { + for (const s of this.stores) { + if (await s.has(key, options)) { + return true + } + } + + return false + } + + async delete (key: Key, options?: AbortOptions): Promise { + try { + await Promise.all(this.stores.map(async store => { await store.delete(key, options) })) + } catch (err: any) { + throw Errors.dbDeleteFailedError(err) + } + } + + async * putMany (source: AwaitIterable, options: AbortOptions = {}): AsyncIterable { + let error: Error | undefined + const pushables = this.stores.map(store => { + const source = pushable({ + objectMode: true + }) + + drain(store.putMany(source, options)) + .catch(err => { + // store threw while putting, make sure we bubble the error up + error = err + }) + + return source + }) + + try { + for await (const pair of source) { + if (error != null) { + throw error + } + + pushables.forEach(p => p.push(pair)) + + yield pair.key + } + } finally { + pushables.forEach(p => p.end()) + } + } + + async * deleteMany (source: AwaitIterable, options: AbortOptions = {}): AsyncIterable { + let error: Error | undefined + const pushables = this.stores.map(store => { + const source = pushable({ + objectMode: true + }) + + drain(store.deleteMany(source, options)) + .catch(err => { + // store threw while deleting, make sure we bubble the error up + error = err + }) + + return source + }) + + try { + for await (const key of source) { + if (error != null) { + throw error + } + + pushables.forEach(p => p.push(key)) + + yield key + } + } finally { + pushables.forEach(p => p.end()) + } + } + + batch (): Batch { + const batches = this.stores.map(store => store.batch()) + + return { + put: (key, value) => { + batches.forEach(b => { b.put(key, value) }) + }, + delete: (key) => { + batches.forEach(b => { b.delete(key) }) + }, + commit: async (options) => { + for (const batch of batches) { + await batch.commit(options) + } + } + } + } + + query (q: Query, options?: AbortOptions): AwaitIterable { + return this.stores[this.stores.length - 1].query(q, options) + } + + queryKeys (q: KeyQuery, options?: AbortOptions): AwaitIterable { + return this.stores[this.stores.length - 1].queryKeys(q, options) + } +} diff --git a/packages/datastore-core/src/utils.ts b/packages/datastore-core/src/utils.ts new file mode 100644 index 00000000..540bea05 --- /dev/null +++ b/packages/datastore-core/src/utils.ts @@ -0,0 +1,5 @@ + +export const replaceStartWith = (s: string, r: string): string => { + const matcher = new RegExp('^' + r) + return s.replace(matcher, '') +} diff --git a/packages/datastore-core/test/keytransform.spec.ts b/packages/datastore-core/test/keytransform.spec.ts new file mode 100644 index 00000000..265266b2 --- /dev/null +++ b/packages/datastore-core/test/keytransform.spec.ts @@ -0,0 +1,52 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import all from 'it-all' +import { Key } from 'interface-datastore/key' +import { MemoryDatastore } from '../src/memory.js' +import { KeyTransformDatastore } from '../src/keytransform.js' + +describe('KeyTransformDatastore', () => { + it('basic', async () => { + const mStore = new MemoryDatastore() + const transform = { + convert (key: Key): Key { + return new Key('/abc').child(key) + }, + invert (key: Key): Key { + const l = key.list() + if (l[0] !== 'abc') { + throw new Error('missing prefix, convert failed?') + } + return Key.withNamespaces(l.slice(1)) + } + } + + const kStore = new KeyTransformDatastore(mStore, transform) + + const keys = [ + 'foo', + 'foo/bar', + 'foo/bar/baz', + 'foo/barb', + 'foo/bar/bazb', + 'foo/bar/baz/barb' + ].map((s) => new Key(s)) + await Promise.all(keys.map(async (key) => { await kStore.put(key, key.uint8Array()) })) + const kResults = Promise.all(keys.map(async (key) => await kStore.get(key))) + const mResults = Promise.all(keys.map(async (key) => await mStore.get(new Key('abc').child(key)))) + const results = await Promise.all([kResults, mResults]) + expect(results[0]).to.eql(results[1]) + + const mRes = await all(mStore.query({})) + const kRes = await all(kStore.query({})) + expect(kRes).to.have.length(mRes.length) + + mRes.forEach((a, i) => { + const kA = a.key + const kB = kRes[i].key + expect(transform.invert(kA)).to.eql(kB) + expect(kA).to.eql(transform.convert(kB)) + }) + }) +}) diff --git a/packages/datastore-core/test/memory.spec.ts b/packages/datastore-core/test/memory.spec.ts new file mode 100644 index 00000000..8011d6b0 --- /dev/null +++ b/packages/datastore-core/test/memory.spec.ts @@ -0,0 +1,15 @@ +/* eslint-env mocha */ + +import { MemoryDatastore } from '../src/memory.js' +import { interfaceDatastoreTests } from 'interface-datastore-tests' + +describe('Memory', () => { + describe('interface-datastore', () => { + interfaceDatastoreTests({ + setup () { + return new MemoryDatastore() + }, + teardown () {} + }) + }) +}) diff --git a/packages/datastore-core/test/mount.spec.ts b/packages/datastore-core/test/mount.spec.ts new file mode 100644 index 00000000..a30e101c --- /dev/null +++ b/packages/datastore-core/test/mount.spec.ts @@ -0,0 +1,137 @@ +/* eslint-env mocha */ +/* eslint max-nested-callbacks: ["error", 8] */ + +import { expect, assert } from 'aegir/chai' +import all from 'it-all' +import { Key } from 'interface-datastore/key' +import { MemoryDatastore } from '../src/memory.js' +import { MountDatastore } from '../src/mount.js' +import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' +import { interfaceDatastoreTests } from 'interface-datastore-tests' +import { KeyTransformDatastore } from '../src/keytransform.js' +import type { Datastore } from 'interface-datastore' + +const stripPrefixDatastore = (datastore: Datastore, prefix: Key): Datastore => { + return new KeyTransformDatastore( + datastore, { + convert: (key) => { + if (!prefix.isAncestorOf(key)) { + throw new Error(`Expected prefix: (${prefix.toString()}) in key: ${key.toString()}`) + } + + return Key.withNamespaces(key.namespaces().slice(prefix.namespaces().length)) + }, + invert: (key) => Key.withNamespaces([...prefix.namespaces(), ...key.namespaces()]) + }) +} + +describe('MountDatastore', () => { + it('put - no mount', async () => { + const m = new MountDatastore([]) + try { + await m.put(new Key('hello'), uint8ArrayFromString('foo')) + assert(false, 'Failed to throw error on no mount') + } catch (err) { + expect(err).to.be.an('Error') + } + }) + + it('put - wrong mount', async () => { + const m = new MountDatastore([{ + datastore: stripPrefixDatastore(new MemoryDatastore(), new Key('cool')), + prefix: new Key('cool') + }]) + try { + await m.put(new Key('/fail/hello'), uint8ArrayFromString('foo')) + assert(false, 'Failed to throw error on wrong mount') + } catch (err) { + expect(err).to.be.an('Error') + } + }) + + it('put', async () => { + const mds = new MemoryDatastore() + const m = new MountDatastore([{ + datastore: stripPrefixDatastore(mds, new Key('cool')), + prefix: new Key('cool') + }]) + + const val = uint8ArrayFromString('hello') + await m.put(new Key('/cool/hello'), val) + const res = await mds.get(new Key('/hello')) + expect(res).to.eql(val) + }) + + it('get', async () => { + const mds = new MemoryDatastore() + const m = new MountDatastore([{ + datastore: stripPrefixDatastore(mds, new Key('cool')), + prefix: new Key('cool') + }]) + + const val = uint8ArrayFromString('hello') + await mds.put(new Key('/hello'), val) + const res = await m.get(new Key('/cool/hello')) + expect(res).to.eql(val) + }) + + it('has', async () => { + const mds = new MemoryDatastore() + const m = new MountDatastore([{ + datastore: stripPrefixDatastore(mds, new Key('cool')), + prefix: new Key('cool') + }]) + + const val = uint8ArrayFromString('hello') + await mds.put(new Key('/hello'), val) + const exists = await m.has(new Key('/cool/hello')) + expect(exists).to.eql(true) + }) + + it('delete', async () => { + const mds = new MemoryDatastore() + const m = new MountDatastore([{ + datastore: stripPrefixDatastore(mds, new Key('cool')), + prefix: new Key('cool') + }]) + + const val = uint8ArrayFromString('hello') + await m.put(new Key('/cool/hello'), val) + await m.delete(new Key('/cool/hello')) + let exists = await m.has(new Key('/cool/hello')) + expect(exists).to.eql(false) + exists = await mds.has(new Key('/hello')) + expect(exists).to.eql(false) + }) + + it('query simple', async () => { + const mds = new MemoryDatastore() + const m = new MountDatastore([{ + datastore: stripPrefixDatastore(mds, new Key('cool')), + prefix: new Key('cool') + }]) + + const val = uint8ArrayFromString('hello') + await m.put(new Key('/cool/hello'), val) + const res = await all(m.query({ prefix: '/cool' })) + expect(res).to.eql([{ key: new Key('/cool/hello'), value: val }]) + }) + + describe('interface-datastore', () => { + interfaceDatastoreTests({ + setup () { + return new MountDatastore([{ + prefix: new Key('/a'), + datastore: new MemoryDatastore() + }, { + prefix: new Key('/z'), + datastore: new MemoryDatastore() + }, { + prefix: new Key('/q'), + datastore: new MemoryDatastore() + }]) + }, + teardown () { } + }) + }) +}) diff --git a/packages/datastore-core/test/namespace.spec.ts b/packages/datastore-core/test/namespace.spec.ts new file mode 100644 index 00000000..a81c677c --- /dev/null +++ b/packages/datastore-core/test/namespace.spec.ts @@ -0,0 +1,58 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import all from 'it-all' +import { Key } from 'interface-datastore/key' +import { MemoryDatastore } from '../src/memory.js' +import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' +import { NamespaceDatastore } from '../src/namespace.js' +import { interfaceDatastoreTests } from 'interface-datastore-tests' + +describe('NamespaceDatastore', () => { + const prefixes = [ + 'abc', + '' + ] + prefixes.forEach((prefix) => it(`basic '${prefix}'`, async () => { + const mStore = new MemoryDatastore() + const store = new NamespaceDatastore(mStore, new Key(prefix)) + + const keys = [ + 'foo', + 'foo/bar', + 'foo/bar/baz', + 'foo/barb', + 'foo/bar/bazb', + 'foo/bar/baz/barb' + ].map((s) => new Key(s)) + + await Promise.all(keys.map(async key => { await store.put(key, uint8ArrayFromString(key.toString())) })) + const nResults = Promise.all(keys.map(async (key) => await store.get(key))) + const mResults = Promise.all(keys.map(async (key) => await mStore.get(new Key(prefix).child(key)))) + const results = await Promise.all([nResults, mResults]) + const mRes = await all(mStore.query({})) + const nRes = await all(store.query({})) + + expect(nRes).to.have.length(mRes.length) + + mRes.forEach((a, i) => { + const kA = a.key + const kB = nRes[i].key + expect(store.transform.invert(kA)).to.eql(kB) + expect(kA).to.eql(store.transform.convert(kB)) + }) + + expect(results[0]).to.eql(results[1]) + })) + + prefixes.forEach((prefix) => { + describe(`interface-datastore: '${prefix}'`, () => { + interfaceDatastoreTests({ + setup () { + return new NamespaceDatastore(new MemoryDatastore(), new Key(prefix)) + }, + async teardown () { } + }) + }) + }) +}) diff --git a/packages/datastore-core/test/shard.spec.ts b/packages/datastore-core/test/shard.spec.ts new file mode 100644 index 00000000..f1a7b229 --- /dev/null +++ b/packages/datastore-core/test/shard.spec.ts @@ -0,0 +1,103 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import { + Prefix, + Suffix, + NextToLast, + parseShardFun +} from '../src/shard.js' + +describe('shard', () => { + it('prefix', () => { + expect( + new Prefix(2).fun('hello') + ).to.eql( + 'he' + ) + expect( + new Prefix(2).fun('h') + ).to.eql( + 'h_' + ) + + expect( + new Prefix(2).toString() + ).to.eql( + '/repo/flatfs/shard/v1/prefix/2' + ) + }) + + it('suffix', () => { + expect( + new Suffix(2).fun('hello') + ).to.eql( + 'lo' + ) + expect( + new Suffix(2).fun('h') + ).to.eql( + '_h' + ) + + expect( + new Suffix(2).toString() + ).to.eql( + '/repo/flatfs/shard/v1/suffix/2' + ) + }) + + it('next-to-last', () => { + expect( + new NextToLast(2).fun('hello') + ).to.eql( + 'll' + ) + expect( + new NextToLast(3).fun('he') + ).to.eql( + '__h' + ) + + expect( + new NextToLast(2).toString() + ).to.eql( + '/repo/flatfs/shard/v1/next-to-last/2' + ) + }) +}) + +describe('parsesShardFun', () => { + it('errors', () => { + const errors = [ + '', + 'shard/v1/next-to-last/2', + '/repo/flatfs/shard/v2/next-to-last/2', + '/repo/flatfs/shard/v1/other/2', + '/repo/flatfs/shard/v1/next-to-last/' + ] + + errors.forEach((input) => { + expect( + () => parseShardFun(input) + ).to.throw() + }) + }) + + it('success', () => { + const success = [ + 'prefix', + 'suffix', + 'next-to-last' + ] + + success.forEach((name) => { + const n = Math.floor(Math.random() * 100) + expect( + parseShardFun( + `/repo/flatfs/shard/v1/${name}/${n}` + ).name + ).to.eql(name) + }) + }) +}) diff --git a/packages/datastore-core/test/sharding.spec.ts b/packages/datastore-core/test/sharding.spec.ts new file mode 100644 index 00000000..4266019b --- /dev/null +++ b/packages/datastore-core/test/sharding.spec.ts @@ -0,0 +1,65 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import { Key } from 'interface-datastore/key' +import { MemoryDatastore } from '../src/memory.js' +import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' +import { toString as uint8ArrayToString } from 'uint8arrays/to-string' +import { + NextToLast, + SHARDING_FN +} from '../src/shard.js' +import { + ShardingDatastore +} from '../src/sharding.js' +import { interfaceDatastoreTests } from 'interface-datastore-tests' + +describe('ShardingDatastore', () => { + it('create', async () => { + const ms = new MemoryDatastore() + const shard = new NextToLast(2) + const store = new ShardingDatastore(ms, shard) + await store.open() + const res = await Promise.all([ + store.get(new Key(SHARDING_FN)) + ]) + expect(uint8ArrayToString(res[0])).to.eql(shard.toString() + '\n') + }) + + it('open - empty', () => { + const ms = new MemoryDatastore() + // @ts-expect-error + const store = new ShardingDatastore(ms) + return expect(store.open()) + .to.eventually.be.rejected() + .with.property('code', 'ERR_DB_OPEN_FAILED') + }) + + it('open - existing', () => { + const ms = new MemoryDatastore() + const shard = new NextToLast(2) + const store = new ShardingDatastore(ms, shard) + + return expect(store.open()).to.eventually.be.fulfilled() + }) + + it('basics', async () => { + const ms = new MemoryDatastore() + const shard = new NextToLast(2) + const store = new ShardingDatastore(ms, shard) + await store.open() + await store.put(new Key('hello'), uint8ArrayFromString('test')) + const res = await ms.get(new Key('ll').child(new Key('hello'))) + expect(res).to.eql(uint8ArrayFromString('test')) + }) + + describe('interface-datastore', () => { + interfaceDatastoreTests({ + setup () { + const shard = new NextToLast(2) + return new ShardingDatastore(new MemoryDatastore(), shard) + }, + teardown () { } + }) + }) +}) diff --git a/packages/datastore-core/test/tiered.spec.ts b/packages/datastore-core/test/tiered.spec.ts new file mode 100644 index 00000000..9dade7d9 --- /dev/null +++ b/packages/datastore-core/test/tiered.spec.ts @@ -0,0 +1,72 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import { Key } from 'interface-datastore/key' +import { MemoryDatastore } from '../src/memory.js' +import { TieredDatastore } from '../src/tiered.js' +import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' +import { interfaceDatastoreTests } from 'interface-datastore-tests' +import type { Datastore } from 'interface-datastore' + +/** + * @typedef {import('interface-datastore').Datastore} Datastore + */ + +describe('Tiered', () => { + describe('all stores', () => { + const ms: Datastore[] = [] + let store: TieredDatastore + beforeEach(() => { + ms.push(new MemoryDatastore()) + ms.push(new MemoryDatastore()) + store = new TieredDatastore(ms) + }) + + it('put', async () => { + const k = new Key('hello') + const v = uint8ArrayFromString('world') + await store.put(k, v) + const res = await Promise.all([ms[0].get(k), ms[1].get(k)]) + res.forEach((val) => { + expect(val).to.be.eql(v) + }) + }) + + it('get and has, where available', async () => { + const k = new Key('hello') + const v = uint8ArrayFromString('world') + await ms[1].put(k, v) + const val = await store.get(k) + expect(val).to.be.eql(v) + const exists = await store.has(k) + expect(exists).to.be.eql(true) + }) + + it('has - key not found', async () => { + expect(await store.has(new Key('hello1'))).to.be.eql(false) + }) + + it('has and delete', async () => { + const k = new Key('hello') + const v = uint8ArrayFromString('world') + await store.put(k, v) + let res = await Promise.all([ms[0].has(k), ms[1].has(k)]) + expect(res).to.be.eql([true, true]) + await store.delete(k) + res = await Promise.all([ms[0].has(k), ms[1].has(k)]) + expect(res).to.be.eql([false, false]) + }) + }) + + describe('inteface-datastore-single', () => { + interfaceDatastoreTests({ + setup () { + return new TieredDatastore([ + new MemoryDatastore(), + new MemoryDatastore() + ]) + }, + teardown () { } + }) + }) +}) diff --git a/packages/datastore-core/test/utils.spec.ts b/packages/datastore-core/test/utils.spec.ts new file mode 100644 index 00000000..5cb851fb --- /dev/null +++ b/packages/datastore-core/test/utils.spec.ts @@ -0,0 +1,20 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import * as utils from '../src/utils.js' + +describe('utils', () => { + it('replaceStartWith', () => { + expect( + utils.replaceStartWith('helloworld', 'hello') + ).to.eql( + 'world' + ) + + expect( + utils.replaceStartWith('helloworld', 'world') + ).to.eql( + 'helloworld' + ) + }) +}) diff --git a/packages/datastore-core/tsconfig.json b/packages/datastore-core/tsconfig.json new file mode 100644 index 00000000..73cb3ab8 --- /dev/null +++ b/packages/datastore-core/tsconfig.json @@ -0,0 +1,21 @@ +{ + "extends": "aegir/src/config/tsconfig.aegir.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": [ + "src", + "test" + ], + "references": [ + { + "path": "../interface-datastore" + }, + { + "path": "../interface-datastore-tests" + }, + { + "path": "../interface-store" + } + ] +} diff --git a/packages/datastore-fs/CHANGELOG.md b/packages/datastore-fs/CHANGELOG.md new file mode 100644 index 00000000..cbecfd91 --- /dev/null +++ b/packages/datastore-fs/CHANGELOG.md @@ -0,0 +1,283 @@ +## [9.0.1](https://github.com/ipfs/js-datastore-fs/compare/v9.0.0...v9.0.1) (2023-03-23) + + +### Dependencies + +* update interface-store to 5.x.x ([#183](https://github.com/ipfs/js-datastore-fs/issues/183)) ([803df0b](https://github.com/ipfs/js-datastore-fs/commit/803df0bc96860352957fb06fee109a705124a273)) + +## [9.0.0](https://github.com/ipfs/js-datastore-fs/compare/v8.0.0...v9.0.0) (2023-03-14) + + +### ⚠ BREAKING CHANGES + +* interface-datastore 8.x.x has removed the open and close methods as these are implementation specific + +### Bug Fixes + +* fix exports map ([4332006](https://github.com/ipfs/js-datastore-fs/commit/4332006aba3e8e67c80144b110cc81042ce07e03)) + + +### Dependencies + +* update interface-datastore to 8.x.x ([#180](https://github.com/ipfs/js-datastore-fs/issues/180)) ([2ef5f5c](https://github.com/ipfs/js-datastore-fs/commit/2ef5f5cf021b1876cb83c53bafebb5f153c77462)) + + +### Trivial Changes + +* update project config ([62adc6b](https://github.com/ipfs/js-datastore-fs/commit/62adc6b238e64c6040afb4887bf6999de42cde08)) + +## [8.0.0](https://github.com/ipfs/js-datastore-fs/compare/v7.0.0...v8.0.0) (2022-08-12) + + +### ⚠ BREAKING CHANGES + +* this module used to be dual published as CJS/ESM, now it is just ESM + +### Features + +* publish module as ESM only ([#139](https://github.com/ipfs/js-datastore-fs/issues/139)) ([5896e57](https://github.com/ipfs/js-datastore-fs/commit/5896e57fbba4ed47e0ede2ae140f8e757c928148)) + + +### Trivial Changes + +* Update .github/workflows/stale.yml [skip ci] ([94f7d36](https://github.com/ipfs/js-datastore-fs/commit/94f7d369a9b3285d7be79e794f90fdcc1e80f704)) + +## [7.0.0](https://github.com/ipfs/js-datastore-fs/compare/v6.0.1...v7.0.0) (2022-01-19) + + +### ⚠ BREAKING CHANGES + +* updates project config to use unified ci + +### Trivial Changes + +* switch to unified ci ([#113](https://github.com/ipfs/js-datastore-fs/issues/113)) ([5e306e2](https://github.com/ipfs/js-datastore-fs/commit/5e306e260727d6fdfdfbd75aae84ca48bb11592f)) + +## [6.0.1](https://github.com/ipfs/js-datastore-fs/compare/v6.0.0...v6.0.1) (2021-09-09) + + + +# [6.0.0](https://github.com/ipfs/js-datastore-fs/compare/v5.0.2...v6.0.0) (2021-09-08) + + +### chore + +* convert to esm ([#103](https://github.com/ipfs/js-datastore-fs/issues/103)) ([314d130](https://github.com/ipfs/js-datastore-fs/commit/314d13051665c42be8a4bcc51c19ce2d5dd45346)) + + +### BREAKING CHANGES + +* deep imports are no longer possible + + + +## [5.0.2](https://github.com/ipfs/js-datastore-fs/compare/v5.0.1...v5.0.2) (2021-07-23) + + +### Features + +* parallel block writes ([#97](https://github.com/ipfs/js-datastore-fs/issues/97)) ([ceb5cfd](https://github.com/ipfs/js-datastore-fs/commit/ceb5cfd98487147888623b587f1480660eb5722e)) + + + +## [5.0.1](https://github.com/ipfs/js-datastore-fs/compare/v5.0.0...v5.0.1) (2021-07-10) + + + +# [5.0.0](https://github.com/ipfs/js-datastore-fs/compare/v4.0.1...v5.0.0) (2021-07-06) + + +### chore + +* update deps ([79b185f](https://github.com/ipfs/js-datastore-fs/commit/79b185f05b1f2bd2e4e0f75c44dad375e6c5f2dc)) + + +### BREAKING CHANGES + +* uses new interface-datastore interfaces + + + +## [4.0.1](https://github.com/ipfs/js-datastore-fs/compare/v4.0.0...v4.0.1) (2021-05-04) + + + +# [4.0.0](https://github.com/ipfs/js-datastore-fs/compare/v3.0.2...v4.0.0) (2021-04-15) + + +### Features + +* split .query into .query and .queryKeys ([#82](https://github.com/ipfs/js-datastore-fs/issues/82)) ([9f2a64b](https://github.com/ipfs/js-datastore-fs/commit/9f2a64be2714fc3ee3780f6f6a24a5c8424e157b)) + + + +## [3.0.2](https://github.com/ipfs/js-datastore-fs/compare/v3.0.1...v3.0.2) (2021-04-14) + + + +## [3.0.1](https://github.com/ipfs/js-datastore-fs/compare/v3.0.0...v3.0.1) (2021-04-06) + + + +# [3.0.0](https://github.com/ipfs/js-datastore-fs/compare/v2.0.2...v3.0.0) (2021-01-22) + + +### Features + +* types ([#62](https://github.com/ipfs/js-datastore-fs/issues/62)) ([de519dd](https://github.com/ipfs/js-datastore-fs/commit/de519dd6b8c0b892e827fb2d26cde3239358eaf6)) + + + +## [2.0.2](https://github.com/ipfs/js-datastore-fs/compare/v2.0.1...v2.0.2) (2020-11-09) + + + + +## [2.0.1](https://github.com/ipfs/js-datastore-fs/compare/v2.0.0...v2.0.1) (2020-08-15) + + +### Bug Fixes + +* only remove extension from path when it is non-empty ([#47](https://github.com/ipfs/js-datastore-fs/issues/47)) ([9e3e042](https://github.com/ipfs/js-datastore-fs/commit/9e3e042)) + + + + +# [2.0.0](https://github.com/ipfs/js-datastore-fs/compare/v1.1.0...v2.0.0) (2020-07-29) + + +### Bug Fixes + +* remove node buffers ([#44](https://github.com/ipfs/js-datastore-fs/issues/44)) ([887b762](https://github.com/ipfs/js-datastore-fs/commit/887b762)) + + +### BREAKING CHANGES + +* only uses Uint8Arrays internally + + + + +# [1.1.0](https://github.com/ipfs/js-datastore-fs/compare/v1.0.0...v1.1.0) (2020-05-07) + + +### Bug Fixes + +* **ci:** add empty commit to fix lint checks on master ([d3e2732](https://github.com/ipfs/js-datastore-fs/commit/d3e2732)) + + +### Features + +* add streaming/cancellable API ([#39](https://github.com/ipfs/js-datastore-fs/issues/39)) ([5232c1c](https://github.com/ipfs/js-datastore-fs/commit/5232c1c)) + + + + +# [1.0.0](https://github.com/ipfs/js-datastore-fs/compare/v0.9.1...v1.0.0) (2020-04-28) + + +### Bug Fixes + +* linter ([ac7235d](https://github.com/ipfs/js-datastore-fs/commit/ac7235d)) + + + + +## [0.9.1](https://github.com/ipfs/js-datastore-fs/compare/v0.9.0...v0.9.1) (2019-09-09) + + +### Bug Fixes + +* handle concurrent writes on windows ([d5c8e4f](https://github.com/ipfs/js-datastore-fs/commit/d5c8e4f)) + + + + +# [0.9.0](https://github.com/ipfs/js-datastore-fs/compare/v0.8.0...v0.9.0) (2019-05-29) + + + + +# [0.8.0](https://github.com/ipfs/js-datastore-fs/compare/v0.7.0...v0.8.0) (2019-01-24) + + +### Performance Improvements + +* use fast-write-atomic instead of write-file-atomic ([bf677b4](https://github.com/ipfs/js-datastore-fs/commit/bf677b4)) + + + + +# [0.7.0](https://github.com/ipfs/js-datastore-fs/compare/v0.6.0...v0.7.0) (2018-10-25) + + + + +# [0.6.0](https://github.com/ipfs/js-datastore-fs/compare/v0.5.0...v0.6.0) (2018-09-19) + + +### Features + +* add basic error codes ([c0fb50b](https://github.com/ipfs/js-datastore-fs/commit/c0fb50b)) + + + + +# [0.5.0](https://github.com/ipfs/js-datastore-fs/compare/v0.4.2...v0.5.0) (2018-05-28) + + +### Bug Fixes + +* add basic flowtype for glob to fix flow runner ([ef6c1f0](https://github.com/ipfs/js-datastore-fs/commit/ef6c1f0)) +* prevent delete from sending multiple args in callback ([0940452](https://github.com/ipfs/js-datastore-fs/commit/0940452)) + + + + +## [0.4.2](https://github.com/ipfs/js-datastore-fs/compare/v0.4.1...v0.4.2) (2017-12-05) + + + + +## [0.4.1](https://github.com/ipfs/js-datastore-fs/compare/v0.4.0...v0.4.1) (2017-11-06) + + + + +# [0.4.0](https://github.com/ipfs/js-datastore-fs/compare/v0.3.0...v0.4.0) (2017-11-04) + + +### Bug Fixes + +* exclude node_modules and include flow-typed ([#5](https://github.com/ipfs/js-datastore-fs/issues/5)) ([1507a6b](https://github.com/ipfs/js-datastore-fs/commit/1507a6b)) +* Windows support ([#7](https://github.com/ipfs/js-datastore-fs/issues/7)) ([e7c8f25](https://github.com/ipfs/js-datastore-fs/commit/e7c8f25)), closes [#6](https://github.com/ipfs/js-datastore-fs/issues/6) + + + + +# [0.3.0](https://github.com/ipfs/js-datastore-fs/compare/v0.2.0...v0.3.0) (2017-07-23) + + + + +# [0.2.0](https://github.com/ipfs/js-datastore-fs/compare/v0.1.1...v0.2.0) (2017-03-23) + + +### Features + +* add open method ([#2](https://github.com/ipfs/js-datastore-fs/issues/2)) ([99c7409](https://github.com/ipfs/js-datastore-fs/commit/99c7409)) + + + + +## [0.1.1](https://github.com/ipfs/js-datastore-fs/compare/v0.1.0...v0.1.1) (2017-03-17) + + +### Bug Fixes + +* do not return values when not expected ([f9f1979](https://github.com/ipfs/js-datastore-fs/commit/f9f1979)) + + + + +# 0.1.0 (2017-03-15) diff --git a/packages/datastore-fs/LICENSE b/packages/datastore-fs/LICENSE new file mode 100644 index 00000000..20ce483c --- /dev/null +++ b/packages/datastore-fs/LICENSE @@ -0,0 +1,4 @@ +This project is dual licensed under MIT and Apache-2.0. + +MIT: https://www.opensource.org/licenses/mit +Apache-2.0: https://www.apache.org/licenses/license-2.0 diff --git a/packages/datastore-fs/LICENSE-APACHE b/packages/datastore-fs/LICENSE-APACHE new file mode 100644 index 00000000..14478a3b --- /dev/null +++ b/packages/datastore-fs/LICENSE-APACHE @@ -0,0 +1,5 @@ +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/packages/datastore-fs/LICENSE-MIT b/packages/datastore-fs/LICENSE-MIT new file mode 100644 index 00000000..72dc60d8 --- /dev/null +++ b/packages/datastore-fs/LICENSE-MIT @@ -0,0 +1,19 @@ +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/packages/datastore-fs/README.md b/packages/datastore-fs/README.md new file mode 100644 index 00000000..15d6bce5 --- /dev/null +++ b/packages/datastore-fs/README.md @@ -0,0 +1,53 @@ +# datastore-fs + +[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) +[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/js-stores.svg?style=flat-square)](https://codecov.io/gh/ipfs/js-stores) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/js-stores/js-test-and-release.yml?branch=master\&style=flat-square)](https://github.com/ipfs/js-stores/actions/workflows/js-test-and-release.yml?query=branch%3Amaster) + +> Datastore implementation with file system backend + +## Table of contents + +- [Install](#install) +- [Usage](#usage) +- [API Docs](#api-docs) +- [License](#license) +- [Contribute](#contribute) + +## Install + +```console +$ npm i datastore-fs +``` + +## Usage + +```js +import { FSDatastore } from 'datastore-fs' + +const store = new FSDatastore('path/to/store') +``` + +## API Docs + +- + +## License + +Licensed under either of + +- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) +- MIT ([LICENSE-MIT](LICENSE-MIT) / ) + +## Contribute + +Contributions welcome! Please check out [the issues](https://github.com/ipfs/js-stores/issues). + +Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. + +Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) diff --git a/packages/datastore-fs/package.json b/packages/datastore-fs/package.json new file mode 100644 index 00000000..0518f0bb --- /dev/null +++ b/packages/datastore-fs/package.json @@ -0,0 +1,160 @@ +{ + "name": "datastore-fs", + "version": "9.0.1", + "description": "Datastore implementation with file system backend", + "author": "Friedel Ziegelmayer", + "license": "Apache-2.0 OR MIT", + "homepage": "https://github.com/ipfs/js-stores/tree/master/packages/datastore-fs#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/ipfs/js-stores.git" + }, + "bugs": { + "url": "https://github.com/ipfs/js-stores/issues" + }, + "keywords": [ + "datastore", + "fs", + "interface", + "ipfs", + "key-value" + ], + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + }, + "type": "module", + "types": "./dist/src/index.d.ts", + "files": [ + "src", + "dist", + "!dist/test", + "!**/*.tsbuildinfo" + ], + "exports": { + ".": { + "types": "./dist/src/index.d.ts", + "import": "./dist/src/index.js" + } + }, + "eslintConfig": { + "extends": "ipfs", + "parserOptions": { + "sourceType": "module" + } + }, + "release": { + "branches": [ + "master" + ], + "plugins": [ + [ + "@semantic-release/commit-analyzer", + { + "preset": "conventionalcommits", + "releaseRules": [ + { + "breaking": true, + "release": "major" + }, + { + "revert": true, + "release": "patch" + }, + { + "type": "feat", + "release": "minor" + }, + { + "type": "fix", + "release": "patch" + }, + { + "type": "docs", + "release": "patch" + }, + { + "type": "test", + "release": "patch" + }, + { + "type": "deps", + "release": "patch" + }, + { + "scope": "no-release", + "release": false + } + ] + } + ], + [ + "@semantic-release/release-notes-generator", + { + "preset": "conventionalcommits", + "presetConfig": { + "types": [ + { + "type": "feat", + "section": "Features" + }, + { + "type": "fix", + "section": "Bug Fixes" + }, + { + "type": "chore", + "section": "Trivial Changes" + }, + { + "type": "docs", + "section": "Documentation" + }, + { + "type": "deps", + "section": "Dependencies" + }, + { + "type": "test", + "section": "Tests" + } + ] + } + } + ], + "@semantic-release/changelog", + "@semantic-release/npm", + "@semantic-release/github", + "@semantic-release/git" + ] + }, + "scripts": { + "clean": "aegir clean", + "lint": "aegir lint", + "build": "aegir build --bundle false", + "release": "aegir release", + "test": "aegir test -t node -t electron-main", + "test:node": "aegir test -t node", + "test:electron-main": "aegir test -t electron-main", + "dep-check": "aegir dep-check", + "docs": "aegir docs" + }, + "dependencies": { + "datastore-core": "^9.0.0", + "fast-write-atomic": "^0.2.0", + "interface-datastore": "^8.0.0", + "interface-store": "^5.0.0", + "it-glob": "^2.0.1", + "it-map": "^2.0.1", + "it-parallel-batch": "^2.0.1" + }, + "devDependencies": { + "@types/mkdirp": "^2.0.0", + "aegir": "^38.1.7", + "interface-datastore-tests": "^5.0.0", + "ipfs-utils": "^9.0.4" + }, + "typedoc": { + "entryPoint": "./src/index.ts" + } +} diff --git a/packages/datastore-fs/src/index.ts b/packages/datastore-fs/src/index.ts new file mode 100644 index 00000000..a5a54236 --- /dev/null +++ b/packages/datastore-fs/src/index.ts @@ -0,0 +1,286 @@ +import fs from 'node:fs/promises' +import glob from 'it-glob' +import path from 'node:path' +import { promisify } from 'util' +import { + Key, KeyQuery, Pair, Query +} from 'interface-datastore' +import { + BaseDatastore, Errors +} from 'datastore-core' +import map from 'it-map' +import parallel from 'it-parallel-batch' +// @ts-expect-error no types +import fwa from 'fast-write-atomic' +import type { AwaitIterable } from 'interface-store' + +const writeAtomic = promisify(fwa) + +/** + * Write a file atomically + */ +async function writeFile (path: string, contents: Uint8Array): Promise { + try { + await writeAtomic(path, contents) + } catch (err: any) { + if (err.code === 'EPERM' && err.syscall === 'rename') { + // fast-write-atomic writes a file to a temp location before renaming it. + // On Windows, if the final file already exists this error is thrown. + // No such error is thrown on Linux/Mac + // Make sure we can read & write to this file + await fs.access(path, fs.constants.F_OK | fs.constants.W_OK) + + // The file was created by another context - this means there were + // attempts to write the same block by two different function calls + return + } + + throw err + } +} + +export interface FsDatastoreInit { + createIfMissing?: boolean + errorIfExists?: boolean + extension?: string + putManyConcurrency?: number + getManyConcurrency?: number + deleteManyConcurrency?: number +} + +/** + * A datastore backed by the file system. + * + * Keys need to be sanitized before use, as they are written + * to the file system as is. + */ +export class FsDatastore extends BaseDatastore { + public path: string + private readonly createIfMissing: boolean + private readonly errorIfExists: boolean + private readonly extension: string + private readonly deleteManyConcurrency: number + private readonly getManyConcurrency: number + private readonly putManyConcurrency: number + + constructor (location: string, init: FsDatastoreInit = {}) { + super() + + this.path = path.resolve(location) + this.createIfMissing = init.createIfMissing ?? true + this.errorIfExists = init.errorIfExists ?? false + this.extension = init.extension ?? '.data' + this.deleteManyConcurrency = init.deleteManyConcurrency ?? 50 + this.getManyConcurrency = init.getManyConcurrency ?? 50 + this.putManyConcurrency = init.putManyConcurrency ?? 50 + } + + async open (): Promise { + try { + await fs.access(this.path, fs.constants.F_OK | fs.constants.W_OK) + + if (this.errorIfExists) { + throw Errors.dbOpenFailedError(new Error(`Datastore directory: ${this.path} already exists`)) + } + return + } catch (err: any) { + if (err.code === 'ENOENT') { + if (this.createIfMissing) { + await fs.mkdir(this.path, { recursive: true }) + return + } else { + throw Errors.notFoundError(new Error(`Datastore directory: ${this.path} does not exist`)) + } + } + + throw err + } + } + + async close (): Promise { + + } + + /** + * Calculate the directory and file name for a given key. + */ + _encode (key: Key): { dir: string, file: string } { + const parent = key.parent().toString() + const dir = path.join(this.path, parent) + const name = key.toString().slice(parent.length) + const file = path.join(dir, name + this.extension) + + return { + dir, + file + } + } + + /** + * Calculate the original key, given the file name. + */ + _decode (file: string): Key { + const ext = this.extension + if (path.extname(file) !== ext) { + throw new Error(`Invalid extension: ${path.extname(file)}`) + } + + const keyname = file + .slice(this.path.length, -ext.length) + .split(path.sep) + .join('/') + + return new Key(keyname) + } + + /** + * Store the given value under the key + */ + async put (key: Key, val: Uint8Array): Promise { + const parts = this._encode(key) + + try { + await fs.mkdir(parts.dir, { + recursive: true + }) + await writeFile(parts.file, val) + + return key + } catch (err: any) { + throw Errors.dbWriteFailedError(err) + } + } + + async * putMany (source: AwaitIterable): AsyncIterable { + yield * parallel( + map(source, ({ key, value }) => { + return async () => { + await this.put(key, value) + + return key + } + }), + this.putManyConcurrency + ) + } + + /** + * Read from the file system + */ + async get (key: Key): Promise { + const parts = this._encode(key) + let data + try { + data = await fs.readFile(parts.file) + } catch (err: any) { + throw Errors.notFoundError(err) + } + return data + } + + async * getMany (source: AwaitIterable): AsyncIterable { + yield * parallel( + map(source, key => { + return async () => { + return { + key, + value: await this.get(key) + } + } + }), + this.getManyConcurrency + ) + } + + async * deleteMany (source: AwaitIterable): AsyncIterable { + yield * parallel( + map(source, key => { + return async () => { + await this.delete(key) + + return key + } + }), + this.deleteManyConcurrency + ) + } + + /** + * Check for the existence of the given key + */ + async has (key: Key): Promise { + const parts = this._encode(key) + + try { + await fs.access(parts.file) + } catch (err: any) { + return false + } + return true + } + + /** + * Delete the record under the given key + */ + async delete (key: Key): Promise { + const parts = this._encode(key) + try { + await fs.unlink(parts.file) + } catch (err: any) { + if (err.code === 'ENOENT') { + return + } + + throw Errors.dbDeleteFailedError(err) + } + } + + async * _all (q: Query): AsyncIterable { + let prefix = q.prefix ?? '**' + + // strip leading slashes + prefix = prefix.replace(/^\/+/, '') + + const pattern = `${prefix}/*${this.extension}` + .split(path.sep) + .join('/') + const files = glob(this.path, pattern, { + absolute: true + }) + + for await (const file of files) { + try { + const buf = await fs.readFile(file) + + const pair: Pair = { + key: this._decode(file), + value: buf + } + + yield pair + } catch (err: any) { + // if keys are removed from the datastore while the query is + // running, we may encounter missing files. + if (err.code !== 'ENOENT') { + throw err + } + } + } + } + + async * _allKeys (q: KeyQuery): AsyncIterable { + let prefix = q.prefix ?? '**' + + // strip leading slashes + prefix = prefix.replace(/^\/+/, '') + + const pattern = `${prefix}/*${this.extension}` + .split(path.sep) + .join('/') + const files = glob(this.path, pattern, { + absolute: true + }) + + yield * map(files, f => this._decode(f)) + } +} diff --git a/packages/datastore-fs/test/index.spec.ts b/packages/datastore-fs/test/index.spec.ts new file mode 100644 index 00000000..96d09637 --- /dev/null +++ b/packages/datastore-fs/test/index.spec.ts @@ -0,0 +1,173 @@ +/* eslint-env mocha */ +import { expect } from 'aegir/chai' +import path from 'node:path' +import fs from 'node:fs/promises' +import { Key } from 'interface-datastore' +import { ShardingDatastore, shard } from 'datastore-core' +import { interfaceDatastoreTests } from 'interface-datastore-tests' +import { FsDatastore } from '../src/index.js' +import tempdir from 'ipfs-utils/src/temp-dir.js' + +const utf8Encoder = new TextEncoder() + +describe('FsDatastore', () => { + describe('construction', () => { + it('defaults - folder missing', async () => { + const dir = tempdir() + + await expect( + (async () => { + const store = new FsDatastore(dir) + await store.open() + })() + ).to.eventually.be.undefined() + }) + + it('defaults - folder exists', async () => { + const dir = tempdir() + await fs.mkdir(dir, { + recursive: true + }) + + await expect( + (async () => { + const store = new FsDatastore(dir) + await store.open() + })() + ).to.eventually.be.undefined() + }) + }) + + describe('open', () => { + it('createIfMissing: false - folder missing', async () => { + const dir = tempdir() + const store = new FsDatastore(dir, { createIfMissing: false }) + await expect(store.open()).to.eventually.be.rejected + .with.property('code', 'ERR_NOT_FOUND') + }) + + it('errorIfExists: true - folder exists', async () => { + const dir = tempdir() + await fs.mkdir(dir, { + recursive: true + }) + const store = new FsDatastore(dir, { errorIfExists: true }) + await expect(store.open()).to.eventually.be.rejected + .with.property('code', 'ERR_DB_OPEN_FAILED') + }) + }) + + it('_encode and _decode', async () => { + const dir = tempdir() + const fs = new FsDatastore(dir) + await fs.open() + + expect( + fs._encode(new Key('hello/world')) + ).to.eql({ + dir: path.join(dir, 'hello'), + file: path.join(dir, 'hello', 'world.data') + }) + + expect( + fs._decode(fs._encode(new Key('hello/world/test:other')).file) + ).to.eql( + new Key('hello/world/test:other') + ) + }) + + it('deleting files', async () => { + const dir = tempdir() + const fs = new FsDatastore(dir) + await fs.open() + const key = new Key('1234') + + await fs.put(key, Uint8Array.from([0, 1, 2, 3])) + await fs.delete(key) + + try { + await fs.get(key) + throw new Error('Should have errored') + } catch (err: any) { + expect(err.code).to.equal('ERR_NOT_FOUND') + } + }) + + it('deleting non-existent files', async () => { + const dir = tempdir() + const fs = new FsDatastore(dir) + await fs.open() + const key = new Key('5678') + + await fs.delete(key) + + try { + await fs.get(key) + throw new Error('Should have errored') + } catch (err: any) { + expect(err.code).to.equal('ERR_NOT_FOUND') + } + }) + + it('sharding files', async () => { + const dir = tempdir() + const fstore = new FsDatastore(dir) + await fstore.open() + await ShardingDatastore.create(fstore, new shard.NextToLast(2)) + + const file = await fs.readFile(path.join(dir, shard.SHARDING_FN + '.data')) + expect(file.toString()).to.be.eql('/repo/flatfs/shard/v1/next-to-last/2\n') + + await fs.rm(dir, { + recursive: true + }) + }) + + describe('interface-datastore', () => { + interfaceDatastoreTests({ + async setup () { + const store = new FsDatastore(tempdir()) + await store.open() + + return store + }, + async teardown (store) { + await fs.rm(store.path, { + recursive: true + }) + } + }) + }) + + describe('interface-datastore (sharding(fs))', () => { + interfaceDatastoreTests({ + async setup () { + const store = new FsDatastore(tempdir()) + await store.open() + + const shardedStore = new ShardingDatastore(store, new shard.NextToLast(2)) + await shardedStore.open() + + return shardedStore + }, + teardown () { + + } + }) + }) + + it('can survive concurrent writes', async () => { + const dir = tempdir() + const fstore = new FsDatastore(dir) + const key = new Key('CIQGFTQ7FSI2COUXWWLOQ45VUM2GUZCGAXLWCTOKKPGTUWPXHBNIVOY') + const value = utf8Encoder.encode('Hello world') + + await Promise.all( + new Array(100).fill(0).map(async () => { await fstore.put(key, value) }) + ) + + const res = await fstore.get(key) + + expect(res).to.deep.equal(value) + }) +}) diff --git a/packages/datastore-fs/tsconfig.json b/packages/datastore-fs/tsconfig.json new file mode 100644 index 00000000..e30932c0 --- /dev/null +++ b/packages/datastore-fs/tsconfig.json @@ -0,0 +1,24 @@ +{ + "extends": "aegir/src/config/tsconfig.aegir.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": [ + "src", + "test" + ], + "references": [ + { + "path": "../datastore-core" + }, + { + "path": "../interface-datastore" + }, + { + "path": "../interface-datastore-tests" + }, + { + "path": "../interface-store" + } + ] +} diff --git a/packages/datastore-idb/.aegir.js b/packages/datastore-idb/.aegir.js new file mode 100644 index 00000000..22cff143 --- /dev/null +++ b/packages/datastore-idb/.aegir.js @@ -0,0 +1,3 @@ +export default { + bundlesize: { maxSize: '16kB' } +} diff --git a/packages/datastore-idb/CHANGELOG.md b/packages/datastore-idb/CHANGELOG.md new file mode 100644 index 00000000..81a97050 --- /dev/null +++ b/packages/datastore-idb/CHANGELOG.md @@ -0,0 +1,234 @@ +## [2.0.1](https://github.com/ipfs/js-datastore-idb/compare/v2.0.0...v2.0.1) (2023-03-23) + + +### Bug Fixes + +* update interface-store to 5.x.x ([#100](https://github.com/ipfs/js-datastore-idb/issues/100)) ([4c918fc](https://github.com/ipfs/js-datastore-idb/commit/4c918fcef352f381810d18225e679202a873f028)) + + +### Trivial Changes + +* update benchmark ([#99](https://github.com/ipfs/js-datastore-idb/issues/99)) ([49aa042](https://github.com/ipfs/js-datastore-idb/commit/49aa0429e0c07f1c53c8031567e3b77f154daa49)) + +## [2.0.0](https://github.com/ipfs/js-datastore-idb/compare/v1.1.0...v2.0.0) (2023-03-14) + + +### ⚠ BREAKING CHANGES + +* this module has been converted to typescript, updated to the latest interface-datastore version and is ESM-only + +### Features + +* convert to typescript ([#98](https://github.com/ipfs/js-datastore-idb/issues/98)) ([a34262f](https://github.com/ipfs/js-datastore-idb/commit/a34262fa979145a8defcb0ff8a0de2e0cf0ef54b)), closes [#6](https://github.com/ipfs/js-datastore-idb/issues/6) + + +### Documentation + +* fix link ([417af5b](https://github.com/ipfs/js-datastore-idb/commit/417af5b92518a8f81d3aa2b89e1755312453420f)) + + +### Trivial Changes + +* **deps-dev:** bump aegir from 22.1.0 to 24.0.0 ([f71f236](https://github.com/ipfs/js-datastore-idb/commit/f71f2360427b7d334b346cc97e8e6b461696de55)) +* **deps-dev:** bump aegir from 24.0.0 to 25.0.0 ([2479fe6](https://github.com/ipfs/js-datastore-idb/commit/2479fe628c871ab1db3e48b8acc12a302575a16c)) +* Update .github/workflows/stale.yml [skip ci] ([c113be2](https://github.com/ipfs/js-datastore-idb/commit/c113be22c50bdbbef14a2c7c2fb265ee008d9562)) +* update aegir ([5f00f1b](https://github.com/ipfs/js-datastore-idb/commit/5f00f1b8791b7e578275ed60f202417153311497)) + + +# [1.1.0](https://github.com/ipfs/js-datastore-idb/compare/v1.0.2...v1.1.0) (2020-05-07) + + +### Features + +* add streaming/cancellable API ([6d32b7f](https://github.com/ipfs/js-datastore-idb/commit/6d32b7f)) + + + + +## [1.0.2](https://github.com/ipfs/js-datastore-idb/compare/v1.0.1...v1.0.2) (2020-04-24) + + + + +## [1.0.1](https://github.com/ipfs/js-datastore-idb/compare/v1.0.0...v1.0.1) (2020-04-23) + + +### Bug Fixes + +* add bundle size config ([1fef7be](https://github.com/ipfs/js-datastore-idb/commit/1fef7be)) +* protect open and close ([0695ad6](https://github.com/ipfs/js-datastore-idb/commit/0695ad6)) + + + + +# 1.0.0 (2020-04-08) + + +### Bug Fixes + +* add guard for un open db ([d06332d](https://github.com/ipfs/js-datastore-idb/commit/d06332d)) +* add options ([2074f00](https://github.com/ipfs/js-datastore-idb/commit/2074f00)) +* benchmarks and perf tweaks ([71d6804](https://github.com/ipfs/js-datastore-idb/commit/71d6804)) +* skip node tests ([d8fd6e8](https://github.com/ipfs/js-datastore-idb/commit/d8fd6e8)) +* syntax tweaks ([7906f08](https://github.com/ipfs/js-datastore-idb/commit/7906f08)) + + +### Features + +* idb datastore ([336c7e7](https://github.com/ipfs/js-datastore-idb/commit/336c7e7)) +* improve query by a lot ([2ba5203](https://github.com/ipfs/js-datastore-idb/commit/2ba5203)) + + + + +## [0.14.1](https://github.com/ipfs/js-datastore-level/compare/v0.14.0...v0.14.1) (2020-01-14) + + +### Bug Fixes + +* leveldb iterator memory leak ([#26](https://github.com/ipfs/js-datastore-level/issues/26)) ([e503c1a](https://github.com/ipfs/js-datastore-level/commit/e503c1a)), closes [/github.com/Level/leveldown/blob/d3453fbde4d2a8aa04d9091101c25c999649069b/binding.cc#L545](https://github.com//github.com/Level/leveldown/blob/d3453fbde4d2a8aa04d9091101c25c999649069b/binding.cc/issues/L545) + + +### Performance Improvements + +* optimize prefix search ([#25](https://github.com/ipfs/js-datastore-level/issues/25)) ([8efa812](https://github.com/ipfs/js-datastore-level/commit/8efa812)) + + + + +# [0.14.0](https://github.com/ipfs/js-datastore-level/compare/v0.13.0...v0.14.0) (2019-11-29) + + + + +# [0.13.0](https://github.com/ipfs/js-datastore-level/compare/v0.12.1...v0.13.0) (2019-11-29) + + +### Bug Fixes + +* init db in overridable method for easier extending ([#21](https://github.com/ipfs/js-datastore-level/issues/21)) ([b21428c](https://github.com/ipfs/js-datastore-level/commit/b21428c)) + + + + +## [0.12.1](https://github.com/ipfs/js-datastore-level/compare/v0.12.0...v0.12.1) (2019-06-26) + + +### Bug Fixes + +* swap leveldown/level.js for level ([#20](https://github.com/ipfs/js-datastore-level/issues/20)) ([d16e212](https://github.com/ipfs/js-datastore-level/commit/d16e212)) + + + + +# [0.12.0](https://github.com/ipfs/js-datastore-level/compare/v0.11.0...v0.12.0) (2019-05-29) + + +### Bug Fixes + +* remove unused var ([74d4a36](https://github.com/ipfs/js-datastore-level/commit/74d4a36)) +* tests ([601599d](https://github.com/ipfs/js-datastore-level/commit/601599d)) + + + + +# [0.11.0](https://github.com/ipfs/js-datastore-level/compare/v0.10.0...v0.11.0) (2019-04-29) + + + + +# [0.10.0](https://github.com/ipfs/js-datastore-level/compare/v0.9.0...v0.10.0) (2018-10-24) + + + + +# [0.9.0](https://github.com/ipfs/js-datastore-level/compare/v0.8.0...v0.9.0) (2018-09-19) + + +### Features + +* add basic error codes ([02a5146](https://github.com/ipfs/js-datastore-level/commit/02a5146)) + + + + +# [0.8.0](https://github.com/ipfs/js-datastore-level/compare/v0.7.0...v0.8.0) (2018-05-29) + + +### Bug Fixes + +* add test and fix constructor ([396f657](https://github.com/ipfs/js-datastore-level/commit/396f657)) +* update binary encoding for levelup 2 ([a5d7378](https://github.com/ipfs/js-datastore-level/commit/a5d7378)) +* upgrade level libs to resolve node 10 failure ([a427eca](https://github.com/ipfs/js-datastore-level/commit/a427eca)) + + + + +# [0.7.0](https://github.com/ipfs/js-datastore-level/compare/v0.6.0...v0.7.0) (2017-11-06) + + +### Bug Fixes + +* Windows interop ([#4](https://github.com/ipfs/js-datastore-level/issues/4)) ([5d67042](https://github.com/ipfs/js-datastore-level/commit/5d67042)) + + + + +# [0.6.0](https://github.com/ipfs/js-datastore-level/compare/v0.5.0...v0.6.0) (2017-07-23) + + + + +# [0.5.0](https://github.com/ipfs/js-datastore-level/compare/v0.4.2...v0.5.0) (2017-07-22) + + + + +## [0.4.2](https://github.com/ipfs/js-datastore-level/compare/v0.4.0...v0.4.2) (2017-05-24) + + +### Bug Fixes + +* Object.assign is now evil and no longer is behaving as spec says when Webpacked ([5e40f3b](https://github.com/ipfs/js-datastore-level/commit/5e40f3b)) +* Object.assign is now evil and no longer is behaving as spec says when Webpacked ([c3f50ec](https://github.com/ipfs/js-datastore-level/commit/c3f50ec)) + + + + +## [0.4.1](https://github.com/ipfs/js-datastore-level/compare/v0.4.0...v0.4.1) (2017-05-24) + + +### Bug Fixes + +* Object.assign is now evil and no longer is behaving as spec says when Webpacked ([077bbc1](https://github.com/ipfs/js-datastore-level/commit/077bbc1)) + + + + +# [0.4.0](https://github.com/ipfs/js-datastore-level/compare/v0.3.0...v0.4.0) (2017-05-23) + + + + +# [0.3.0](https://github.com/ipfs/js-datastore-level/compare/v0.2.0...v0.3.0) (2017-03-23) + + + + +# [0.2.0](https://github.com/ipfs/js-datastore-level/compare/v0.1.0...v0.2.0) (2017-03-23) + + +### Features + +* add open method ([fd12c6b](https://github.com/ipfs/js-datastore-level/commit/fd12c6b)) + + + + +# 0.1.0 (2017-03-15) + + +### Bug Fixes + +* key handling ([682f8b3](https://github.com/ipfs/js-datastore-level/commit/682f8b3)) +* working interop with go ([f5e03c6](https://github.com/ipfs/js-datastore-level/commit/f5e03c6)) diff --git a/packages/datastore-idb/LICENSE b/packages/datastore-idb/LICENSE new file mode 100644 index 00000000..20ce483c --- /dev/null +++ b/packages/datastore-idb/LICENSE @@ -0,0 +1,4 @@ +This project is dual licensed under MIT and Apache-2.0. + +MIT: https://www.opensource.org/licenses/mit +Apache-2.0: https://www.apache.org/licenses/license-2.0 diff --git a/packages/datastore-idb/LICENSE-APACHE b/packages/datastore-idb/LICENSE-APACHE new file mode 100644 index 00000000..14478a3b --- /dev/null +++ b/packages/datastore-idb/LICENSE-APACHE @@ -0,0 +1,5 @@ +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/packages/datastore-idb/LICENSE-MIT b/packages/datastore-idb/LICENSE-MIT new file mode 100644 index 00000000..72dc60d8 --- /dev/null +++ b/packages/datastore-idb/LICENSE-MIT @@ -0,0 +1,19 @@ +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/packages/datastore-idb/README.md b/packages/datastore-idb/README.md new file mode 100644 index 00000000..ee85abb2 --- /dev/null +++ b/packages/datastore-idb/README.md @@ -0,0 +1,53 @@ +# datastore-idb + +[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) +[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/js-stores.svg?style=flat-square)](https://codecov.io/gh/ipfs/js-stores) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/js-stores/js-test-and-release.yml?branch=master\&style=flat-square)](https://github.com/ipfs/js-stores/actions/workflows/js-test-and-release.yml?query=branch%3Amaster) + +> Datastore implementation with IndexedDB backend. + +## Table of contents + +- [Install](#install) + - [Browser ` +``` + +## API Docs + +- + +## License + +Licensed under either of + +- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) +- MIT ([LICENSE-MIT](LICENSE-MIT) / ) + +## Contribute + +Contributions welcome! Please check out [the issues](https://github.com/ipfs/js-stores/issues). + +Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. + +Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) diff --git a/packages/datastore-idb/benchmark/datastore-level/package.json b/packages/datastore-idb/benchmark/datastore-level/package.json new file mode 100644 index 00000000..67cd6769 --- /dev/null +++ b/packages/datastore-idb/benchmark/datastore-level/package.json @@ -0,0 +1,21 @@ +{ + "name": "benchmarks-datastore-level", + "version": "1.0.0", + "main": "index.js", + "private": true, + "type": "module", + "scripts": { + "clean": "aegir clean", + "build": "aegir build --bundle false", + "lint": "aegir lint", + "dep-check": "aegir dep-check", + "start": "npm run build && playwright-test dist/src/index.js --runner benchmark" + }, + "devDependencies": { + "datastore-level": "^10.0.1", + "datastore-idb": "file:../../", + "multiformats": "^11.0.1", + "playwright-test": "^8.2.0", + "tinybench": "^2.4.0" + } +} diff --git a/packages/datastore-idb/benchmark/datastore-level/src/index.ts b/packages/datastore-idb/benchmark/datastore-level/src/index.ts new file mode 100644 index 00000000..c0316e49 --- /dev/null +++ b/packages/datastore-idb/benchmark/datastore-level/src/index.ts @@ -0,0 +1,161 @@ +/* eslint-disable no-console */ + +import { Bench } from 'tinybench' +import { IDBDatastore } from 'datastore-idb' +import { LevelDatastore } from 'datastore-level' +import { Key } from 'interface-datastore' + +const RESULT_PRECISION = 2 +const BLOCKS = 1000 + +async function testPut (): Promise { + console.info('simple put') + + const suite = new Bench({ + iterations: 1, + time: 100 + }) + suite.add('level put', async function () { + for (let i = 0; i < BLOCKS; i++) { + // @ts-expect-error + await this.store.put(new Key(`/z/block-${i}`), Uint8Array.from([0, 1, 2, 3, 4, i])) + } + }, { + beforeEach: async function () { + // @ts-expect-error + this.store = new LevelDatastore('hello-level-put') + // @ts-expect-error + await this.store.open() + }, + afterEach: async function () { + // @ts-expect-error + await this.store.close() + } + }) + suite.add('idb put', async function () { + for (let i = 0; i < BLOCKS; i++) { + // @ts-expect-error + await this.store.put(new Key(`/z/block-${i}`), Uint8Array.from([0, 1, 2, 3, 4, i])) + } + }, { + beforeEach: async function () { + // @ts-expect-error + this.store = new IDBDatastore('hello-idb-put') + // @ts-expect-error + await this.store.open() + }, + afterEach: async function () { + // @ts-expect-error + await this.store.close() + } + }) + + await suite.run() + + console.table(suite.tasks.sort((a, b) => { // eslint-disable-line no-console + const resultA = a.result?.hz ?? 0 + const resultB = b.result?.hz ?? 0 + + if (resultA === resultB) { + return 0 + } + + if (resultA < resultB) { + return 1 + } + + return -1 + }).map(({ name, result }) => ({ + Implementation: name, + 'ops/s': parseFloat(result?.hz.toFixed(RESULT_PRECISION) ?? '0'), + 'ms/op': parseFloat(result?.period.toFixed(RESULT_PRECISION) ?? '0'), + runs: result?.samples.length + }))) +} + +async function testGet (): Promise { + console.info('simple get') + + const suite = new Bench({ + iterations: 1, + time: 100 + }) + suite.add('level get', async function () { + for (let i = 0; i < BLOCKS; i++) { + // @ts-expect-error + await this.store.get(new Key(`/z/block-${i}`)) + } + }, { + beforeEach: async function () { + // @ts-expect-error + this.store = new LevelDatastore('hello-level-get') + + // @ts-expect-error + await this.store.open() + + for (let i = 0; i < BLOCKS; i++) { + // @ts-expect-error + await this.store.put(new Key(`/z/block-${i}`), Uint8Array.from([0, 1, 2, 3, 4, i])) + } + }, + afterEach: async function () { + // @ts-expect-error + await this.store.close() + } + }) + suite.add('idb get', async function () { + for (let i = 0; i < BLOCKS; i++) { + // @ts-expect-error + await this.store.get(new Key(`/z/block-${i}`)) + } + }, { + beforeEach: async function () { + // @ts-expect-error + this.store = new IDBDatastore('hello-idb-get') + + // @ts-expect-error + await this.store.open() + + for (let i = 0; i < BLOCKS; i++) { + // @ts-expect-error + await this.store.put(new Key(`/z/block-${i}`), Uint8Array.from([0, 1, 2, 3, 4, i])) + } + }, + afterEach: async function () { + // @ts-expect-error + await this.store.close() + } + }) + + await suite.run() + + console.table(suite.tasks.sort((a, b) => { // eslint-disable-line no-console + const resultA = a.result?.hz ?? 0 + const resultB = b.result?.hz ?? 0 + + if (resultA === resultB) { + return 0 + } + + if (resultA < resultB) { + return 1 + } + + return -1 + }).map(({ name, result }) => ({ + Implementation: name, + 'ops/s': parseFloat(result?.hz.toFixed(RESULT_PRECISION) ?? '0'), + 'ms/op': parseFloat(result?.period.toFixed(RESULT_PRECISION) ?? '0'), + runs: result?.samples.length + }))) +} + +async function main () { + await testPut() + await testGet() +} + +main().catch(err => { + console.error(err) // eslint-disable-line no-console + process.exit(1) +}) diff --git a/packages/datastore-idb/benchmark/datastore-level/tsconfig.json b/packages/datastore-idb/benchmark/datastore-level/tsconfig.json new file mode 100644 index 00000000..fee64009 --- /dev/null +++ b/packages/datastore-idb/benchmark/datastore-level/tsconfig.json @@ -0,0 +1,13 @@ +{ + "extends": "aegir/src/config/tsconfig.aegir.json", + "compilerOptions": { + "outDir": "dist", + "target": "ES2022", + "module": "ES2022", + "lib": ["ES2022", "DOM", "DOM.Iterable"] + }, + "include": [ + "src", + "test" + ] +} diff --git a/packages/datastore-idb/package.json b/packages/datastore-idb/package.json new file mode 100644 index 00000000..b4a8625f --- /dev/null +++ b/packages/datastore-idb/package.json @@ -0,0 +1,160 @@ +{ + "name": "datastore-idb", + "version": "2.0.1", + "description": "Datastore implementation with IndexedDB backend.", + "author": "Hugo Dias ", + "license": "Apache-2.0 OR MIT", + "homepage": "https://github.com/ipfs/js-stores/tree/master/packages/datastore-idb#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/ipfs/js-stores.git" + }, + "bugs": { + "url": "https://github.com/ipfs/js-stores/issues" + }, + "keywords": [ + "browser", + "datastore", + "idb", + "interface", + "ipfs", + "key-value" + ], + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + }, + "type": "module", + "types": "./dist/src/index.d.ts", + "files": [ + "src", + "dist", + "!dist/test", + "!**/*.tsbuildinfo" + ], + "exports": { + ".": { + "types": "./dist/src/index.d.ts", + "import": "./dist/src/index.js" + } + }, + "eslintConfig": { + "extends": "ipfs", + "parserOptions": { + "sourceType": "module", + "project": [ + "tsconfig.json", + "benchmarks/datastore-level/tsconfig.json" + ] + } + }, + "release": { + "branches": [ + "master" + ], + "plugins": [ + [ + "@semantic-release/commit-analyzer", + { + "preset": "conventionalcommits", + "releaseRules": [ + { + "breaking": true, + "release": "major" + }, + { + "revert": true, + "release": "patch" + }, + { + "type": "feat", + "release": "minor" + }, + { + "type": "fix", + "release": "patch" + }, + { + "type": "docs", + "release": "patch" + }, + { + "type": "test", + "release": "patch" + }, + { + "type": "deps", + "release": "patch" + }, + { + "scope": "no-release", + "release": false + } + ] + } + ], + [ + "@semantic-release/release-notes-generator", + { + "preset": "conventionalcommits", + "presetConfig": { + "types": [ + { + "type": "feat", + "section": "Features" + }, + { + "type": "fix", + "section": "Bug Fixes" + }, + { + "type": "chore", + "section": "Trivial Changes" + }, + { + "type": "docs", + "section": "Documentation" + }, + { + "type": "deps", + "section": "Dependencies" + }, + { + "type": "test", + "section": "Tests" + } + ] + } + } + ], + "@semantic-release/changelog", + "@semantic-release/npm", + "@semantic-release/github", + "@semantic-release/git" + ] + }, + "scripts": { + "test": "aegir test -t browser -t webworker", + "test:browser": "aegir test -t browser", + "test:webworker": "aegir test -t webworker", + "build": "aegir build", + "lint": "aegir lint", + "release": "aegir release", + "dep-check": "aegir dep-check", + "docs": "aegir docs" + }, + "dependencies": { + "idb": "^7.1.1", + "interface-datastore": "^8.0.0", + "it-filter": "^2.0.1", + "it-sort": "^2.0.1" + }, + "devDependencies": { + "aegir": "^38.1.7", + "datastore-core": "^9.0.0", + "interface-datastore-tests": "^5.0.0" + }, + "typedoc": { + "entryPoint": "./src/index.ts" + } +} diff --git a/packages/datastore-idb/src/index.ts b/packages/datastore-idb/src/index.ts new file mode 100644 index 00000000..3548b829 --- /dev/null +++ b/packages/datastore-idb/src/index.ts @@ -0,0 +1,230 @@ +import { openDB, deleteDB, IDBPDatabase } from 'idb' +import { Batch, Key, KeyQuery, Pair, Query } from 'interface-datastore' +import { Errors, BaseDatastore } from 'datastore-core' +import filter from 'it-filter' +import sort from 'it-sort' + +export interface IDBDatastoreInit { + /** + * A prefix to use for all database keys (default: '') + */ + prefix?: string + + /** + * The database version (default: 1) + */ + version?: number +} + +export class IDBDatastore extends BaseDatastore { + private readonly location: string + private readonly version: number + private db?: IDBPDatabase + + constructor (location: string, init: IDBDatastoreInit = {}) { + super() + + this.location = `${init.prefix ?? ''}${location}` + this.version = init.version ?? 1 + } + + async open (): Promise { + try { + const location = this.location + + this.db = await openDB(location, this.version, { + upgrade (db) { + db.createObjectStore(location) + } + }) + } catch (err: any) { + throw Errors.dbOpenFailedError(err) + } + } + + async close (): Promise { + this.db?.close() + } + + async put (key: Key, val: Uint8Array): Promise { + if (this.db == null) { + throw new Error('Datastore needs to be opened.') + } + + try { + await this.db.put(this.location, val, key.toString()) + + return key + } catch (err: any) { + throw Errors.dbWriteFailedError(err) + } + } + + async get (key: Key): Promise { + if (this.db == null) { + throw new Error('Datastore needs to be opened.') + } + + let val: Uint8Array | undefined + + try { + val = await this.db.get(this.location, key.toString()) + } catch (err: any) { + throw Errors.dbReadFailedError(err) + } + + if (val === undefined) { + throw Errors.notFoundError() + } + + return val + } + + async has (key: Key): Promise { + if (this.db == null) { + throw new Error('Blockstore needs to be opened.') + } + + try { + return Boolean(await this.db.getKey(this.location, key.toString())) + } catch (err: any) { + throw Errors.dbReadFailedError(err) + } + } + + async delete (key: Key): Promise { + if (this.db == null) { + throw new Error('Datastore needs to be opened.') + } + + try { + await this.db.delete(this.location, key.toString()) + } catch (err: any) { + throw Errors.dbWriteFailedError(err) + } + } + + batch (): Batch { + const puts: Pair[] = [] + const dels: Key[] = [] + + return { + put (key, value) { + puts.push({ key, value }) + }, + delete (key) { + dels.push(key) + }, + commit: async () => { + if (this.db == null) { + throw new Error('Datastore needs to be opened.') + } + + const tx = this.db.transaction(this.location, 'readwrite') + + try { + const ops = puts.filter(({ key }) => { + // don't put a key we are about to delete + return dels.find(delKey => delKey.toString() === key.toString()) == null + }) + .map(put => { + return async () => { + await tx.store.put(put.value, put.key.toString()) + } + }) + .concat(dels.map(key => { + return async () => { + await tx.store.delete(key.toString()) + } + })) + .concat(async () => { + await tx.done + }) + + await Promise.all(ops.map(async op => { await op() })) + } catch { + tx.abort() + } + } + } + } + + async * query (q: Query): AsyncIterable { + let it = this.#queryIt(q, (key, value) => { + return { key, value } + }) + + if (Array.isArray(q.filters)) { + it = q.filters.reduce((it, f) => filter(it, f), it) + } + + if (Array.isArray(q.orders)) { + it = q.orders.reduce((it, f) => sort(it, f), it) + } + + yield * it + } + + async * queryKeys (q: KeyQuery): AsyncIterable { + let it = this.#queryIt(q, (key) => key) + + if (Array.isArray(q.filters)) { + it = q.filters.reduce((it, f) => filter(it, f), it) + } + + if (Array.isArray(q.orders)) { + it = q.orders.reduce((it, f) => sort(it, f), it) + } + + yield * it + } + + async * #queryIt (q: { prefix?: string, offset?: number, limit?: number }, transform: (key: Key, value: Uint8Array) => T): AsyncIterable { + if (this.db == null) { + throw new Error('Datastore needs to be opened.') + } + + let yielded = 0 + let index = -1 + + for (const key of await this.db.getAllKeys(this.location)) { + if (q.prefix != null && !key.toString().startsWith(q.prefix)) { // eslint-disable-line @typescript-eslint/no-base-to-string + continue + } + + if (q.limit != null && yielded === q.limit) { + return + } + + index++ + + if (q.offset != null && index < q.offset) { + continue + } + + const k = new Key(key.toString()) // eslint-disable-line @typescript-eslint/no-base-to-string + let value: Uint8Array | undefined + + try { + value = await this.get(k) + } catch (err: any) { + if (err.code !== 'ERR_NOT_FOUND') { + throw err + } + continue + } + + if (value == null) { + continue + } + + yield transform(k, value) + + yielded++ + } + } + + async destroy (): Promise { + await deleteDB(this.location) + } +} diff --git a/packages/datastore-idb/test/index.spec.ts b/packages/datastore-idb/test/index.spec.ts new file mode 100644 index 00000000..04146e90 --- /dev/null +++ b/packages/datastore-idb/test/index.spec.ts @@ -0,0 +1,124 @@ +/* eslint-env mocha */ + +import { MountDatastore } from 'datastore-core' +import { Key } from 'interface-datastore' +import { IDBDatastore } from '../src/index.js' +import { expect } from 'aegir/chai' +import { interfaceDatastoreTests } from 'interface-datastore-tests' + +describe('IndexedDB Datastore', function () { + describe('interface-datastore (idb)', () => { + interfaceDatastoreTests({ + async setup () { + const store = new IDBDatastore(`hello-${Date.now()}`) + await store.open() + return store + }, + async teardown (store) { + await store.close() + await store.destroy() + } + }) + }) + + describe('interface-datastore (mount(idb, idb, idb))', () => { + interfaceDatastoreTests({ + async setup () { + const one = new IDBDatastore(`one-${Date.now()}`) + const two = new IDBDatastore(`two-${Date.now()}`) + const three = new IDBDatastore(`three-${Date.now()}`) + + await one.open() + await two.open() + await three.open() + + const d = new MountDatastore([ + { + prefix: new Key('/a'), + datastore: one + }, + { + prefix: new Key('/q'), + datastore: two + }, + { + prefix: new Key('/z'), + datastore: three + } + ]) + + return d + }, + teardown () { + + } + }) + }) + + describe('concurrency', () => { + let store: IDBDatastore + + beforeEach(async () => { + store = new IDBDatastore('hello') + await store.open() + }) + + it('should not explode under unreasonable load', function (done) { + this.timeout(10000) + + const updater = setInterval(async () => { // eslint-disable-line @typescript-eslint/no-misused-promises + try { + const key = new Key(`/a-${Date.now()}`) + + await store.put(key, Uint8Array.from([0, 1, 2, 3])) + await store.has(key) + await store.get(key) + } catch (err) { + clearInterval(updater) + clearInterval(mutatorQuery) + clearInterval(readOnlyQuery) + done(err) + } + }, 0) + + const mutatorQuery = setInterval(async () => { // eslint-disable-line @typescript-eslint/no-misused-promises + try { + for await (const { key } of store.query({})) { + await store.get(key) + + const otherKey = new Key(`/b-${Date.now()}`) + const otherValue = Uint8Array.from([0, 1, 2, 3]) + await store.put(otherKey, otherValue) + const res = await store.get(otherKey) + expect(res).to.deep.equal(otherValue) + } + } catch (err) { + clearInterval(updater) + clearInterval(mutatorQuery) + clearInterval(readOnlyQuery) + done(err) + } + }, 0) + + const readOnlyQuery = setInterval(async () => { // eslint-disable-line @typescript-eslint/no-misused-promises + try { + for await (const { key } of store.query({})) { + await store.has(key) + } + } catch (err) { + clearInterval(updater) + clearInterval(mutatorQuery) + clearInterval(readOnlyQuery) + done(err) + } + }, 0) + + setTimeout(() => { + clearInterval(updater) + clearInterval(mutatorQuery) + clearInterval(readOnlyQuery) + done() + }, 5000) + }) + }) +}) diff --git a/packages/datastore-idb/tsconfig.json b/packages/datastore-idb/tsconfig.json new file mode 100644 index 00000000..58cae45c --- /dev/null +++ b/packages/datastore-idb/tsconfig.json @@ -0,0 +1,21 @@ +{ + "extends": "aegir/src/config/tsconfig.aegir.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": [ + "src", + "test" + ], + "references": [ + { + "path": "../datastore-core" + }, + { + "path": "../interface-datastore" + }, + { + "path": "../interface-datastore-tests" + } + ] +} diff --git a/packages/datastore-level/.aegir.js b/packages/datastore-level/.aegir.js new file mode 100644 index 00000000..ed4824f9 --- /dev/null +++ b/packages/datastore-level/.aegir.js @@ -0,0 +1,6 @@ +/** @type {import('aegir').PartialOptions} */ +export default { + build: { + bundlesizeMax: '67KB' + } +} diff --git a/packages/datastore-level/CHANGELOG.md b/packages/datastore-level/CHANGELOG.md new file mode 100644 index 00000000..c728aec4 --- /dev/null +++ b/packages/datastore-level/CHANGELOG.md @@ -0,0 +1,356 @@ +## [10.0.2](https://github.com/ipfs/js-datastore-level/compare/v10.0.1...v10.0.2) (2023-03-23) + + +### Dependencies + +* update interface-store to 5.x.x ([#176](https://github.com/ipfs/js-datastore-level/issues/176)) ([2c89f37](https://github.com/ipfs/js-datastore-level/commit/2c89f371a03019d5e811bbd7893abb11fe0ea46f)) + +## [10.0.1](https://github.com/ipfs/js-datastore-level/compare/v10.0.0...v10.0.1) (2023-03-14) + + +### Bug Fixes + +* update project config ([#173](https://github.com/ipfs/js-datastore-level/issues/173)) ([57a69c7](https://github.com/ipfs/js-datastore-level/commit/57a69c79324cecd5966000e65b4734997c183c17)) + +## [10.0.0](https://github.com/ipfs/js-datastore-level/compare/v9.0.4...v10.0.0) (2023-03-13) + + +### ⚠ BREAKING CHANGES + +* this module now implements interface-datastore@8.x.x + +### Dependencies + +* update to interface-datastore 8.x.x ([#172](https://github.com/ipfs/js-datastore-level/issues/172)) ([178d235](https://github.com/ipfs/js-datastore-level/commit/178d235254805f2abdd919e0860bd86af5b48582)) + +## [9.0.4](https://github.com/ipfs/js-datastore-level/compare/v9.0.3...v9.0.4) (2022-11-03) + + +### Dependencies + +* bump it-map from 1.0.6 to 2.0.0 ([#136](https://github.com/ipfs/js-datastore-level/issues/136)) ([049045a](https://github.com/ipfs/js-datastore-level/commit/049045a4aced05840ec7abce235f3e64c83d42bd)) + +## [9.0.3](https://github.com/ipfs/js-datastore-level/compare/v9.0.2...v9.0.3) (2022-11-03) + + +### Dependencies + +* bump it-sort from 1.0.1 to 2.0.0 ([#137](https://github.com/ipfs/js-datastore-level/issues/137)) ([4ab7b70](https://github.com/ipfs/js-datastore-level/commit/4ab7b700642ad09548c3580129de9bb53270c33d)) + +## [9.0.2](https://github.com/ipfs/js-datastore-level/compare/v9.0.1...v9.0.2) (2022-11-03) + + +### Dependencies + +* bump it-filter from 1.0.3 to 2.0.0 ([#138](https://github.com/ipfs/js-datastore-level/issues/138)) ([2b13ed0](https://github.com/ipfs/js-datastore-level/commit/2b13ed04723922093cd2a15fff0a04de15a59f69)) +* bump it-take from 1.0.2 to 2.0.0 ([#139](https://github.com/ipfs/js-datastore-level/issues/139)) ([1052cdd](https://github.com/ipfs/js-datastore-level/commit/1052cdd68dbee91d2adfab578b04f741448b22ef)) +* **dev:** bump @ipld/dag-cbor from 7.0.3 to 8.0.0 ([#142](https://github.com/ipfs/js-datastore-level/issues/142)) ([1180956](https://github.com/ipfs/js-datastore-level/commit/11809565db0b80ae307f8ce219402600f9ce5745)) +* **dev:** bump multiformats from 9.9.0 to 10.0.2 ([#141](https://github.com/ipfs/js-datastore-level/issues/141)) ([8db833b](https://github.com/ipfs/js-datastore-level/commit/8db833bc97122334bb21bd3de6b52018b4c3e816)) + +## [9.0.1](https://github.com/ipfs/js-datastore-level/compare/v9.0.0...v9.0.1) (2022-08-14) + + +### Bug Fixes + +* restore open options and support old level iterators ([#132](https://github.com/ipfs/js-datastore-level/issues/132)) ([78e4911](https://github.com/ipfs/js-datastore-level/commit/78e4911458371740f2c482c7b1907ec6c15d61d9)) + +## [9.0.0](https://github.com/ipfs/js-datastore-level/compare/v8.0.0...v9.0.0) (2022-08-12) + + +### ⚠ BREAKING CHANGES + +* this module used to be published as ESM/CJS now it is just ESM + +### Features + +* publish as ESM only ([#131](https://github.com/ipfs/js-datastore-level/issues/131)) ([0d3b6ab](https://github.com/ipfs/js-datastore-level/commit/0d3b6ab61b23c20587059b01e5446d9638fb569b)) + + +### Trivial Changes + +* Update .github/workflows/stale.yml [skip ci] ([f5b456e](https://github.com/ipfs/js-datastore-level/commit/f5b456e022d2b00edad0d913bbcc517ce0b90218)) + +## [8.0.0](https://github.com/ipfs/js-datastore-level/compare/v7.0.1...v8.0.0) (2022-01-19) + + +### ⚠ BREAKING CHANGES + +* updates project config to use unified ci + +### Trivial Changes + +* switch to unified ci ([#99](https://github.com/ipfs/js-datastore-level/issues/99)) ([8344486](https://github.com/ipfs/js-datastore-level/commit/8344486d6971e0b2f49ceaa6dbc63f6469e8ed12)) + +## [7.0.1](https://github.com/ipfs/js-datastore-level/compare/v7.0.0...v7.0.1) (2021-09-09) + + + +# [7.0.0](https://github.com/ipfs/js-datastore-level/compare/v6.0.2...v7.0.0) (2021-09-08) + + +### chore + +* switch to ESM ([#87](https://github.com/ipfs/js-datastore-level/issues/87)) ([798e995](https://github.com/ipfs/js-datastore-level/commit/798e9958967c2fc279110eebe75e78523560f903)) + + +### BREAKING CHANGES + +* deep imports/requires are no longer possible + + + +## [6.0.2](https://github.com/ipfs/js-datastore-level/compare/v6.0.1...v6.0.2) (2021-07-30) + + +### Features + +* add level options ([#82](https://github.com/ipfs/js-datastore-level/issues/82)) ([1a3e060](https://github.com/ipfs/js-datastore-level/commit/1a3e060d12bcebc8b5bdc8123d03b23bbc13de59)) + + + +## [6.0.1](https://github.com/ipfs/js-datastore-level/compare/v6.0.0...v6.0.1) (2021-07-10) + + + +# [6.0.0](https://github.com/ipfs/js-datastore-level/compare/v5.0.1...v6.0.0) (2021-07-06) + + +### chore + +* update deps ([7e20056](https://github.com/ipfs/js-datastore-level/commit/7e20056b13db55e74c5fde986ec9afe186bba414)) + + +### BREAKING CHANGES + +* uses new interface-datastore interfaces + + + +## [5.0.1](https://github.com/ipfs/js-datastore-level/compare/v5.0.0...v5.0.1) (2021-04-19) + + + +# [5.0.0](https://github.com/ipfs/js-datastore-level/compare/v4.0.0...v5.0.0) (2021-04-15) + + +### Features + +* split .query into .query and .queryKeys ([#70](https://github.com/ipfs/js-datastore-level/issues/70)) ([39ba735](https://github.com/ipfs/js-datastore-level/commit/39ba735c591524270740b49bfaa09fa4bcbb11d0)) + + + +# [4.0.0](https://github.com/ipfs/js-datastore-level/compare/v3.0.0...v4.0.0) (2021-01-29) + + +### chore + +* **deps:** bump level from 5.0.1 to 6.0.1 ([#31](https://github.com/ipfs/js-datastore-level/issues/31)) ([06853bd](https://github.com/ipfs/js-datastore-level/commit/06853bd389f1f0c8cd00d12219040a903ed48633)) + + +### BREAKING CHANGES + +* **deps:** requires an upgrade to existing datastores created in the browser with level-js@4 or below + + + +# [3.0.0](https://github.com/ipfs/js-datastore-level/compare/v2.0.0...v3.0.0) (2021-01-22) + + +### Bug Fixes + +* fix constructor ([#58](https://github.com/ipfs/js-datastore-level/issues/58)) ([621e425](https://github.com/ipfs/js-datastore-level/commit/621e42569d8c31c3d2b7311a8abd2594fa6621bd)) + + +### Features + +* types ([#53](https://github.com/ipfs/js-datastore-level/issues/53)) ([51cd55e](https://github.com/ipfs/js-datastore-level/commit/51cd55e34aa5139dd9dfdb8966df5283e3c5a324)) + + + + +# [2.0.0](https://github.com/ipfs/js-datastore-level/compare/v1.1.0...v2.0.0) (2020-07-29) + + +### Bug Fixes + +* remove node buffers ([#39](https://github.com/ipfs/js-datastore-level/issues/39)) ([19fe886](https://github.com/ipfs/js-datastore-level/commit/19fe886)) + + +### BREAKING CHANGES + +* remove node buffers in favour of Uint8Arrays + + + + +# [1.1.0](https://github.com/ipfs/js-datastore-level/compare/v1.0.0...v1.1.0) (2020-05-07) + + +### Bug Fixes + +* **ci:** add empty commit to fix lint checks on master ([60d14c0](https://github.com/ipfs/js-datastore-level/commit/60d14c0)) + + +### Features + +* add streaming/cancellable API ([#34](https://github.com/ipfs/js-datastore-level/issues/34)) ([6bfb51a](https://github.com/ipfs/js-datastore-level/commit/6bfb51a)) + + + + +# [1.0.0](https://github.com/ipfs/js-datastore-level/compare/v0.14.1...v1.0.0) (2020-04-28) + + + + +## [0.14.1](https://github.com/ipfs/js-datastore-level/compare/v0.14.0...v0.14.1) (2020-01-14) + + +### Bug Fixes + +* leveldb iterator memory leak ([#26](https://github.com/ipfs/js-datastore-level/issues/26)) ([e503c1a](https://github.com/ipfs/js-datastore-level/commit/e503c1a)), closes [/github.com/Level/leveldown/blob/d3453fbde4d2a8aa04d9091101c25c999649069b/binding.cc#L545](https://github.com//github.com/Level/leveldown/blob/d3453fbde4d2a8aa04d9091101c25c999649069b/binding.cc/issues/L545) + + +### Performance Improvements + +* optimize prefix search ([#25](https://github.com/ipfs/js-datastore-level/issues/25)) ([8efa812](https://github.com/ipfs/js-datastore-level/commit/8efa812)) + + + + +# [0.14.0](https://github.com/ipfs/js-datastore-level/compare/v0.13.0...v0.14.0) (2019-11-29) + + + + +# [0.13.0](https://github.com/ipfs/js-datastore-level/compare/v0.12.1...v0.13.0) (2019-11-29) + + +### Bug Fixes + +* init db in overridable method for easier extending ([#21](https://github.com/ipfs/js-datastore-level/issues/21)) ([b21428c](https://github.com/ipfs/js-datastore-level/commit/b21428c)) + + + + +## [0.12.1](https://github.com/ipfs/js-datastore-level/compare/v0.12.0...v0.12.1) (2019-06-26) + + +### Bug Fixes + +* swap leveldown/level.js for level ([#20](https://github.com/ipfs/js-datastore-level/issues/20)) ([d16e212](https://github.com/ipfs/js-datastore-level/commit/d16e212)) + + + + +# [0.12.0](https://github.com/ipfs/js-datastore-level/compare/v0.11.0...v0.12.0) (2019-05-29) + + +### Bug Fixes + +* remove unused var ([74d4a36](https://github.com/ipfs/js-datastore-level/commit/74d4a36)) +* tests ([601599d](https://github.com/ipfs/js-datastore-level/commit/601599d)) + + + + +# [0.11.0](https://github.com/ipfs/js-datastore-level/compare/v0.10.0...v0.11.0) (2019-04-29) + + + + +# [0.10.0](https://github.com/ipfs/js-datastore-level/compare/v0.9.0...v0.10.0) (2018-10-24) + + + + +# [0.9.0](https://github.com/ipfs/js-datastore-level/compare/v0.8.0...v0.9.0) (2018-09-19) + + +### Features + +* add basic error codes ([02a5146](https://github.com/ipfs/js-datastore-level/commit/02a5146)) + + + + +# [0.8.0](https://github.com/ipfs/js-datastore-level/compare/v0.7.0...v0.8.0) (2018-05-29) + + +### Bug Fixes + +* add test and fix constructor ([396f657](https://github.com/ipfs/js-datastore-level/commit/396f657)) +* update binary encoding for levelup 2 ([a5d7378](https://github.com/ipfs/js-datastore-level/commit/a5d7378)) +* upgrade level libs to resolve node 10 failure ([a427eca](https://github.com/ipfs/js-datastore-level/commit/a427eca)) + + + + +# [0.7.0](https://github.com/ipfs/js-datastore-level/compare/v0.6.0...v0.7.0) (2017-11-06) + + +### Bug Fixes + +* Windows interop ([#4](https://github.com/ipfs/js-datastore-level/issues/4)) ([5d67042](https://github.com/ipfs/js-datastore-level/commit/5d67042)) + + + + +# [0.6.0](https://github.com/ipfs/js-datastore-level/compare/v0.5.0...v0.6.0) (2017-07-23) + + + + +# [0.5.0](https://github.com/ipfs/js-datastore-level/compare/v0.4.2...v0.5.0) (2017-07-22) + + + + +## [0.4.2](https://github.com/ipfs/js-datastore-level/compare/v0.4.0...v0.4.2) (2017-05-24) + + +### Bug Fixes + +* Object.assign is now evil and no longer is behaving as spec says when Webpacked ([5e40f3b](https://github.com/ipfs/js-datastore-level/commit/5e40f3b)) +* Object.assign is now evil and no longer is behaving as spec says when Webpacked ([c3f50ec](https://github.com/ipfs/js-datastore-level/commit/c3f50ec)) + + + + +## [0.4.1](https://github.com/ipfs/js-datastore-level/compare/v0.4.0...v0.4.1) (2017-05-24) + + +### Bug Fixes + +* Object.assign is now evil and no longer is behaving as spec says when Webpacked ([077bbc1](https://github.com/ipfs/js-datastore-level/commit/077bbc1)) + + + + +# [0.4.0](https://github.com/ipfs/js-datastore-level/compare/v0.3.0...v0.4.0) (2017-05-23) + + + + +# [0.3.0](https://github.com/ipfs/js-datastore-level/compare/v0.2.0...v0.3.0) (2017-03-23) + + + + +# [0.2.0](https://github.com/ipfs/js-datastore-level/compare/v0.1.0...v0.2.0) (2017-03-23) + + +### Features + +* add open method ([fd12c6b](https://github.com/ipfs/js-datastore-level/commit/fd12c6b)) + + + + +# 0.1.0 (2017-03-15) + + +### Bug Fixes + +* key handling ([682f8b3](https://github.com/ipfs/js-datastore-level/commit/682f8b3)) +* working interop with go ([f5e03c6](https://github.com/ipfs/js-datastore-level/commit/f5e03c6)) diff --git a/packages/datastore-level/LICENSE b/packages/datastore-level/LICENSE new file mode 100644 index 00000000..20ce483c --- /dev/null +++ b/packages/datastore-level/LICENSE @@ -0,0 +1,4 @@ +This project is dual licensed under MIT and Apache-2.0. + +MIT: https://www.opensource.org/licenses/mit +Apache-2.0: https://www.apache.org/licenses/license-2.0 diff --git a/packages/datastore-level/LICENSE-APACHE b/packages/datastore-level/LICENSE-APACHE new file mode 100644 index 00000000..14478a3b --- /dev/null +++ b/packages/datastore-level/LICENSE-APACHE @@ -0,0 +1,5 @@ +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/packages/datastore-level/LICENSE-MIT b/packages/datastore-level/LICENSE-MIT new file mode 100644 index 00000000..72dc60d8 --- /dev/null +++ b/packages/datastore-level/LICENSE-MIT @@ -0,0 +1,19 @@ +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/packages/datastore-level/README.md b/packages/datastore-level/README.md new file mode 100644 index 00000000..d4d19a6a --- /dev/null +++ b/packages/datastore-level/README.md @@ -0,0 +1,94 @@ +# datastore-level + +[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) +[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/js-stores.svg?style=flat-square)](https://codecov.io/gh/ipfs/js-stores) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/js-stores/js-test-and-release.yml?branch=master\&style=flat-square)](https://github.com/ipfs/js-stores/actions/workflows/js-test-and-release.yml?query=branch%3Amaster) + +> Datastore implementation with level(up|down) backend + +## Table of contents + +- [Install](#install) + - [Browser ` +``` + +## Usage + +```js +import { LevelDatastore } from 'datastore-level' + +// Default using level as backend for node or the browser +const store = new LevelDatastore('path/to/store') + +// another leveldown compliant backend like memory-level +const memStore = new LevelDatastore( + new MemoryLevel({ + keyEncoding: 'utf8', + valueEncoding: 'view' + }) +) +``` + +### Browser Shimming Leveldown + +`LevelStore` uses the `level` module to automatically use `level` if a modern bundler is used which can detect bundle targets based on the `pkg.browser` property in your `package.json`. + +If you are using a bundler that does not support `pkg.browser`, you will need to handle the shimming yourself, as was the case with versions of `LevelStore` 0.7.0 and earlier. + +### Database names + +`level-js@3` changed the database prefix from `IDBWrapper-` to `level-js-`, so please specify the old prefix if you wish to continue using databases created using `datastore-level` prior to `v0.12.0`. E.g. + +```javascript +import leveljs from 'level-js' +import browserStore = new LevelDatastore( + new Level('my/db/name', { + prefix: 'IDBWrapper-' + }) +}) +``` + +More information: [https://github.com/Level/level-js/blob/master/UPGRADING.md#new-database-prefix](https://github.com/Level/level-js/blob/99831913e905d19e5f6dee56d512b7264fbed7bd/UPGRADING.md#new-database-prefix) + +## API Docs + +- + +## License + +Licensed under either of + +- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) +- MIT ([LICENSE-MIT](LICENSE-MIT) / ) + +## Contribute + +Contributions welcome! Please check out [the issues](https://github.com/ipfs/js-stores/issues). + +Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. + +Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) diff --git a/packages/datastore-level/package.json b/packages/datastore-level/package.json new file mode 100644 index 00000000..98bbd64e --- /dev/null +++ b/packages/datastore-level/package.json @@ -0,0 +1,165 @@ +{ + "name": "datastore-level", + "version": "10.0.2", + "description": "Datastore implementation with level(up|down) backend", + "author": "Friedel Ziegelmayer", + "license": "Apache-2.0 OR MIT", + "homepage": "https://github.com/ipfs/js-stores/tree/master/packages/datastore-level#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/ipfs/js-stores.git" + }, + "bugs": { + "url": "https://github.com/ipfs/js-stores/issues" + }, + "keywords": [ + "datastore", + "interface", + "ipfs", + "key-value", + "leveldb", + "leveldown", + "levelup" + ], + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + }, + "type": "module", + "types": "./dist/src/index.d.ts", + "files": [ + "src", + "dist", + "!dist/test", + "!**/*.tsbuildinfo" + ], + "exports": { + ".": { + "types": "./dist/src/index.d.ts", + "import": "./dist/src/index.js" + } + }, + "eslintConfig": { + "extends": "ipfs", + "parserOptions": { + "sourceType": "module" + } + }, + "release": { + "branches": [ + "master" + ], + "plugins": [ + [ + "@semantic-release/commit-analyzer", + { + "preset": "conventionalcommits", + "releaseRules": [ + { + "breaking": true, + "release": "major" + }, + { + "revert": true, + "release": "patch" + }, + { + "type": "feat", + "release": "minor" + }, + { + "type": "fix", + "release": "patch" + }, + { + "type": "docs", + "release": "patch" + }, + { + "type": "test", + "release": "patch" + }, + { + "type": "deps", + "release": "patch" + }, + { + "scope": "no-release", + "release": false + } + ] + } + ], + [ + "@semantic-release/release-notes-generator", + { + "preset": "conventionalcommits", + "presetConfig": { + "types": [ + { + "type": "feat", + "section": "Features" + }, + { + "type": "fix", + "section": "Bug Fixes" + }, + { + "type": "chore", + "section": "Trivial Changes" + }, + { + "type": "docs", + "section": "Documentation" + }, + { + "type": "deps", + "section": "Dependencies" + }, + { + "type": "test", + "section": "Tests" + } + ] + } + } + ], + "@semantic-release/changelog", + "@semantic-release/npm", + "@semantic-release/github", + "@semantic-release/git" + ] + }, + "scripts": { + "clean": "aegir clean", + "lint": "aegir lint", + "build": "aegir build", + "release": "aegir release", + "test": "aegir test", + "test:node": "aegir test -t node --cov", + "test:chrome": "aegir test -t browser --cov", + "test:chrome-webworker": "aegir test -t webworker", + "test:firefox": "aegir test -t browser -- --browser firefox", + "test:firefox-webworker": "aegir test -t webworker -- --browser firefox", + "test:electron-main": "aegir test -t electron-main", + "dep-check": "aegir dep-check" + }, + "dependencies": { + "datastore-core": "^9.0.0", + "interface-datastore": "^8.0.0", + "it-filter": "^2.0.1", + "it-map": "^2.0.1", + "it-sort": "^2.0.1", + "it-take": "^2.0.0", + "level": "^8.0.0" + }, + "devDependencies": { + "aegir": "^38.1.7", + "interface-datastore-tests": "^5.0.0", + "ipfs-utils": "^9.0.4", + "memory-level": "^1.0.0" + }, + "typedoc": { + "entryPoint": "./src/index.ts" + } +} diff --git a/packages/datastore-level/src/index.ts b/packages/datastore-level/src/index.ts new file mode 100644 index 00000000..405e85ef --- /dev/null +++ b/packages/datastore-level/src/index.ts @@ -0,0 +1,263 @@ +import { Batch, Key, KeyQuery, Pair, Query } from 'interface-datastore' +import { BaseDatastore, Errors } from 'datastore-core' +import filter from 'it-filter' +import map from 'it-map' +import take from 'it-take' +import sort from 'it-sort' +import { Level } from 'level' +import type { DatabaseOptions, OpenOptions, IteratorOptions } from 'level' + +interface BatchPut { + type: 'put' + key: string + value: Uint8Array +} + +interface BatchDel { + type: 'del' + key: string +} + +type BatchOp = BatchPut | BatchDel + +/** + * A datastore backed by leveldb + */ +export class LevelDatastore extends BaseDatastore { + public db: Level + private readonly opts: OpenOptions + + constructor (path: string | Level, opts: DatabaseOptions & OpenOptions = {}) { + super() + + this.db = typeof path === 'string' + ? new Level(path, { + ...opts, + keyEncoding: 'utf8', + valueEncoding: 'view' + }) + : path + + this.opts = { + createIfMissing: true, + compression: false, // same default as go + ...opts + } + } + + async open (): Promise { + try { + await this.db.open(this.opts) + } catch (err: any) { + throw Errors.dbOpenFailedError(err) + } + } + + async put (key: Key, value: Uint8Array): Promise { + try { + await this.db.put(key.toString(), value) + + return key + } catch (err: any) { + throw Errors.dbWriteFailedError(err) + } + } + + async get (key: Key): Promise { + let data + try { + data = await this.db.get(key.toString()) + } catch (err: any) { + if (err.notFound != null) { + throw Errors.notFoundError(err) + } + + throw Errors.dbWriteFailedError(err) + } + return data + } + + async has (key: Key): Promise { + try { + await this.db.get(key.toString()) + } catch (err: any) { + if (err.notFound != null) { + return false + } + + throw err + } + return true + } + + async delete (key: Key): Promise { + try { + await this.db.del(key.toString()) + } catch (err: any) { + throw Errors.dbDeleteFailedError(err) + } + } + + async close (): Promise { + await this.db.close() + } + + batch (): Batch { + const ops: BatchOp[] = [] + + return { + put: (key, value) => { + ops.push({ + type: 'put', + key: key.toString(), + value + }) + }, + delete: (key) => { + ops.push({ + type: 'del', + key: key.toString() + }) + }, + commit: async () => { + if (this.db.batch == null) { + throw new Error('Batch operations unsupported by underlying Level') + } + + await this.db.batch(ops) + } + } + } + + query (q: Query): AsyncIterable { + let it = this._query({ + values: true, + prefix: q.prefix + }) + + if (Array.isArray(q.filters)) { + it = q.filters.reduce((it, f) => filter(it, f), it) + } + + if (Array.isArray(q.orders)) { + it = q.orders.reduce((it, f) => sort(it, f), it) + } + + const { offset, limit } = q + if (offset != null) { + let i = 0 + it = filter(it, () => i++ >= offset) + } + + if (limit != null) { + it = take(it, limit) + } + + return it + } + + queryKeys (q: KeyQuery): AsyncIterable { + let it = map(this._query({ + values: false, + prefix: q.prefix + }), ({ key }) => key) + + if (Array.isArray(q.filters)) { + it = q.filters.reduce((it, f) => filter(it, f), it) + } + + if (Array.isArray(q.orders)) { + it = q.orders.reduce((it, f) => sort(it, f), it) + } + + const { offset, limit } = q + if (offset != null) { + let i = 0 + it = filter(it, () => i++ >= offset) + } + + if (limit != null) { + it = take(it, limit) + } + + return it + } + + _query (opts: { values: boolean, prefix?: string }): AsyncIterable { + const iteratorOpts: IteratorOptions = { + keys: true, + keyEncoding: 'buffer', + values: opts.values + } + + // Let the db do the prefix matching + if (opts.prefix != null) { + const prefix = opts.prefix.toString() + // Match keys greater than or equal to `prefix` and + iteratorOpts.gte = prefix + // less than `prefix` + \xFF (hex escape sequence) + iteratorOpts.lt = prefix + '\xFF' + } + + const iterator = this.db.iterator(iteratorOpts) + + if (iterator[Symbol.asyncIterator] != null) { + return levelIteratorToIterator(iterator) + } + + // @ts-expect-error support older level + if (iterator.next != null && iterator.end != null) { + // @ts-expect-error support older level + return oldLevelIteratorToIterator(iterator) + } + + throw new Error('Level returned incompatible iterator') + } +} + +async function * levelIteratorToIterator (li: AsyncIterable<[string, Uint8Array]> & { close: () => Promise }): AsyncIterable { + for await (const [key, value] of li) { + yield { key: new Key(key, false), value } + } + + await li.close() +} + +interface OldLevelIterator { + next: (cb: (err: Error, key: string | Uint8Array | null, value: any) => void) => void + end: (cb: (err: Error) => void) => void +} + +function oldLevelIteratorToIterator (li: OldLevelIterator): AsyncIterable { + return { + [Symbol.asyncIterator] () { + return { + next: async () => await new Promise((resolve, reject) => { + li.next((err, key, value) => { + if (err != null) { + reject(err); return + } + if (key == null) { + li.end(err => { + if (err != null) { + reject(err) + return + } + resolve({ done: true, value: undefined }) + }); return + } + resolve({ done: false, value: { key: new Key(key, false), value } }) + }) + }), + return: async () => await new Promise((resolve, reject) => { + li.end(err => { + if (err != null) { + reject(err); return + } + resolve({ done: true, value: undefined }) + }) + }) + } + } + } +} diff --git a/packages/datastore-level/test/browser.ts b/packages/datastore-level/test/browser.ts new file mode 100644 index 00000000..e86adacd --- /dev/null +++ b/packages/datastore-level/test/browser.ts @@ -0,0 +1,33 @@ +/* eslint-env mocha */ + +import { MountDatastore } from 'datastore-core' +import { Key } from 'interface-datastore/key' +import { LevelDatastore } from '../src/index.js' +import { interfaceDatastoreTests } from 'interface-datastore-tests' + +describe('LevelDatastore', () => { + describe('interface-datastore (leveljs)', () => { + interfaceDatastoreTests({ + setup: () => new LevelDatastore(`hello-${Math.random()}`), + teardown: () => {} + }) + }) + + describe('interface-datastore (mount(leveljs, leveljs, leveljs))', () => { + interfaceDatastoreTests({ + setup () { + return new MountDatastore([{ + prefix: new Key('/a'), + datastore: new LevelDatastore(`one-${Math.random()}`) + }, { + prefix: new Key('/q'), + datastore: new LevelDatastore(`two-${Math.random()}`) + }, { + prefix: new Key('/z'), + datastore: new LevelDatastore(`three-${Math.random()}`) + }]) + }, + teardown () {} + }) + }) +}) diff --git a/packages/datastore-level/test/fixtures/test-level-iterator-destroy.ts b/packages/datastore-level/test/fixtures/test-level-iterator-destroy.ts new file mode 100644 index 00000000..ac945639 --- /dev/null +++ b/packages/datastore-level/test/fixtures/test-level-iterator-destroy.ts @@ -0,0 +1,17 @@ +import { Key } from 'interface-datastore/key' +import { LevelDatastore } from '../../src/index.js' +import tempdir from 'ipfs-utils/src/temp-dir.js' + +async function testLevelIteratorDestroy (): Promise { + const store = new LevelDatastore(tempdir()) + await store.open() + await store.put(new Key(`/test/key${Date.now()}`), new TextEncoder().encode(`TESTDATA${Date.now()}`)) + for await (const d of store.query({})) { + console.log(d) // eslint-disable-line no-console + } +} + +// Will exit with: +// > Assertion failed: (ended_), function ~Iterator, file ../binding.cc, line 546. +// If iterator gets destroyed (in c++ land) and .end() was not called on it. +void testLevelIteratorDestroy() diff --git a/packages/datastore-level/test/index.spec.ts b/packages/datastore-level/test/index.spec.ts new file mode 100644 index 00000000..b4f58e7f --- /dev/null +++ b/packages/datastore-level/test/index.spec.ts @@ -0,0 +1,63 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import { MemoryLevel } from 'memory-level' +import { Level } from 'level' +import { LevelDatastore } from '../src/index.js' +import tempdir from 'ipfs-utils/src/temp-dir.js' +import { interfaceDatastoreTests } from 'interface-datastore-tests' + +describe('LevelDatastore', () => { + describe('initialization', () => { + it('should default to a leveldown database', async () => { + const levelStore = new LevelDatastore(`${tempdir()}/init-default-${Date.now()}`) + await levelStore.open() + + expect(levelStore.db).to.be.an.instanceOf(Level) + }) + + it('should be able to override the database', async () => { + const levelStore = new LevelDatastore( + // @ts-expect-error MemoryLevel does not implement the same interface as Level + new MemoryLevel({ + keyEncoding: 'utf8', + valueEncoding: 'view' + }) + ) + + await levelStore.open() + + expect(levelStore.db).to.be.an.instanceOf(MemoryLevel) + }) + }) + + describe('interface-datastore MemoryLevel', () => { + interfaceDatastoreTests({ + async setup () { + const store = new LevelDatastore( + // @ts-expect-error MemoryLevel does not implement the same interface as Level + new MemoryLevel({ + keyEncoding: 'utf8', + valueEncoding: 'view' + }) + ) + await store.open() + + return store + }, + teardown () {} + }) + }) + + describe('interface-datastore Level', () => { + interfaceDatastoreTests({ + async setup () { + const store = new LevelDatastore(tempdir()) + await store.open() + + return store + }, + teardown () {} + }) + }) +}) diff --git a/packages/datastore-level/test/node.ts b/packages/datastore-level/test/node.ts new file mode 100644 index 00000000..075af85f --- /dev/null +++ b/packages/datastore-level/test/node.ts @@ -0,0 +1,77 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import path from 'path' +import { Key } from 'interface-datastore/key' +import { MountDatastore } from 'datastore-core' +import childProcess from 'child_process' +import { interfaceDatastoreTests } from 'interface-datastore-tests' +import { LevelDatastore } from '../src/index.js' +import tempdir from 'ipfs-utils/src/temp-dir.js' + +describe('LevelDatastore', () => { + describe('interface-datastore (leveldown)', () => { + interfaceDatastoreTests({ + async setup () { + const store = new LevelDatastore(tempdir()) + await store.open() + + return store + }, + teardown () {} + }) + }) + + describe('interface-datastore (mount(leveldown, leveldown, leveldown))', () => { + interfaceDatastoreTests({ + async setup () { + return new MountDatastore( + await Promise.all( + ['/a', '/q', '/z'].map(async prefix => { + const datastore = new LevelDatastore(tempdir()) + await datastore.open() + + return { + prefix: new Key(prefix), + datastore + } + }) + ) + ) + }, + async teardown () { + + } + }) + }) + + // The `.end()` method MUST be called on LevelDB iterators or they remain open, + // leaking memory. + // + // This test exposes this problem by causing an error to be thrown on process + // exit when an iterator is open AND leveldb is not closed. + // + // Normally when leveldb is closed it'll automatically clean up open iterators + // but if you don't close the store this error will occur: + // + // > Assertion failed: (ended_), function ~Iterator, file ../binding.cc, line 546. + // + // This is thrown by a destructor function for iterator objects that asserts + // the iterator has ended before cleanup. + // + // https://github.com/Level/leveldown/blob/d3453fbde4d2a8aa04d9091101c25c999649069b/binding.cc#L545 + it('should not leave iterators open and leak memory', (done) => { + const cp = childProcess.fork(path.join(process.cwd(), '/dist/test/fixtures/test-level-iterator-destroy'), { stdio: 'pipe' }) + + let out = '' + const { stdout, stderr } = cp + stdout?.on('data', d => { out = `${out}${d}` }) + stderr?.on('data', d => { out = `${out}${d}` }) + + cp.on('exit', code => { + expect(code).to.equal(0) + expect(out).to.not.include('Assertion failed: (ended_)') + done() + }) + }) +}) diff --git a/packages/datastore-level/tsconfig.json b/packages/datastore-level/tsconfig.json new file mode 100644 index 00000000..031d7aae --- /dev/null +++ b/packages/datastore-level/tsconfig.json @@ -0,0 +1,21 @@ +{ + "extends": "aegir/src/config/tsconfig.aegir.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": [ + "test", + "src" + ], + "references": [ + { + "path": "../datastore-core" + }, + { + "path": "../interface-datastore" + }, + { + "path": "../interface-datastore-tests" + } + ] +} diff --git a/packages/datastore-s3/CHANGELOG.md b/packages/datastore-s3/CHANGELOG.md new file mode 100644 index 00000000..8508e37b --- /dev/null +++ b/packages/datastore-s3/CHANGELOG.md @@ -0,0 +1,222 @@ +## [11.0.0](https://github.com/ipfs/js-datastore-s3/compare/v10.0.1...v11.0.0) (2023-03-23) + + +### ⚠ BREAKING CHANGES + +* this module is now ESM-only and uses the v3 @aws-sdk/s3-client + +### Features + +* convert to typescript and publish as ESM only ([#106](https://github.com/ipfs/js-datastore-s3/issues/106)) ([0f372e1](https://github.com/ipfs/js-datastore-s3/commit/0f372e180ae22d0c53eaeedcf3975007e1f12466)) + +## [10.0.1](https://github.com/ipfs/js-datastore-s3/compare/v10.0.0...v10.0.1) (2022-10-18) + + +### Dependencies + +* bump it-filter from 1.0.3 to 2.0.0 ([#79](https://github.com/ipfs/js-datastore-s3/issues/79)) ([edb6264](https://github.com/ipfs/js-datastore-s3/commit/edb6264e61c0bcde9e10afb66f80077dca3ad769)) +* bump it-to-buffer from 2.0.2 to 3.0.0 ([#80](https://github.com/ipfs/js-datastore-s3/issues/80)) ([fa9bf96](https://github.com/ipfs/js-datastore-s3/commit/fa9bf963ab0a14bb43d729fd9aecf67e7c9cc437)) +* bump uint8arrays from 3.1.1 to 4.0.2 ([#78](https://github.com/ipfs/js-datastore-s3/issues/78)) ([53af2c6](https://github.com/ipfs/js-datastore-s3/commit/53af2c61cfa72e7e3aa823e6fd41da0c91d99027)) +* bump uint8arrays to 4.0.3 ([40a1444](https://github.com/ipfs/js-datastore-s3/commit/40a1444c37d4f71c73f4bfc8dd5a131691ac4391)) + +## [10.0.0](https://github.com/ipfs/js-datastore-s3/compare/v9.0.0...v10.0.0) (2022-08-12) + + +### ⚠ BREAKING CHANGES + +* this module used to be published as ESM/CJS now it is just ESM + +### Features + +* publish as ESM only ([#75](https://github.com/ipfs/js-datastore-s3/issues/75)) ([dca5704](https://github.com/ipfs/js-datastore-s3/commit/dca57045fa52498245c6e85c2c03cf9a6a9ff177)) + + +### Trivial Changes + +* Update .github/workflows/stale.yml [skip ci] ([b6cbc38](https://github.com/ipfs/js-datastore-s3/commit/b6cbc38b2d8d9dfad58bcaefd21621cd48345263)) + +## [9.0.0](https://github.com/ipfs/js-datastore-s3/compare/v8.0.0...v9.0.0) (2022-01-19) + + +### ⚠ BREAKING CHANGES + +* updates config to use unified ci + +### Trivial Changes + +* fix IPFS-101 example URL ([#40](https://github.com/ipfs/js-datastore-s3/issues/40)) ([0044df3](https://github.com/ipfs/js-datastore-s3/commit/0044df35dbc6fd2a2b01eaaf9e9432796fd9525e)) +* switch to unified ci ([#41](https://github.com/ipfs/js-datastore-s3/issues/41)) ([a969b40](https://github.com/ipfs/js-datastore-s3/commit/a969b404293ca6122ce8d26c000e36724e8186fd)) + +# [8.0.0](https://github.com/ipfs/js-datastore-s3/compare/v7.0.0...v8.0.0) (2021-09-09) + + +### chore + +* switch to esm ([#37](https://github.com/ipfs/js-datastore-s3/issues/37)) ([25f4dce](https://github.com/ipfs/js-datastore-s3/commit/25f4dceaf3e6678756b4c93b56a082c0282cc9f6)) + + +### BREAKING CHANGES + +* only uses named exports + + + +# [7.0.0](https://github.com/ipfs/js-datastore-s3/compare/v6.0.0...v7.0.0) (2021-08-20) + + + +# [6.0.0](https://github.com/ipfs/js-datastore-s3/compare/v4.0.0...v6.0.0) (2021-07-06) + + +### chore + +* update deps ([be4fec6](https://github.com/ipfs/js-datastore-s3/commit/be4fec68ba1854acab5d2eec24f2719f685546fd)) + + +### Features + +* split .query into .query and .queryKeys ([#34](https://github.com/ipfs/js-datastore-s3/issues/34)) ([29423d1](https://github.com/ipfs/js-datastore-s3/commit/29423d13acc4ca137f9708578fe50764b33e0970)) + + +### BREAKING CHANGES + +* uses new interface-datastore types + + + +# [5.0.0](https://github.com/ipfs/js-datastore-s3/compare/v4.0.0...v5.0.0) (2021-04-15) + + +### Features + +* split .query into .query and .queryKeys ([#34](https://github.com/ipfs/js-datastore-s3/issues/34)) ([29423d1](https://github.com/ipfs/js-datastore-s3/commit/29423d13acc4ca137f9708578fe50764b33e0970)) + + + +# [4.0.0](https://github.com/ipfs/js-datastore-s3/compare/v3.0.0...v4.0.0) (2021-04-12) + + + + +# [3.0.0](https://github.com/ipfs/js-datastore-s3/compare/v2.0.0...v3.0.0) (2020-09-22) + + +### Bug Fixes + +* convert input to buffers before passing to aws-sdk ([#30](https://github.com/ipfs/js-datastore-s3/issues/30)) ([b844c63](https://github.com/ipfs/js-datastore-s3/commit/b844c63)) + + +### BREAKING CHANGES + +* - Returns Uint8Arrays only where before it was node Buffers + +* chore: configure pin store + +* docs: update example + +Co-authored-by: Jacob Heun + + + + +# [2.0.0](https://github.com/ipfs/js-datastore-s3/compare/v1.0.0...v2.0.0) (2020-06-19) + + + + +# [1.0.0](https://github.com/ipfs/js-datastore-s3/compare/v0.3.0...v1.0.0) (2020-05-08) + + +### Bug Fixes + +* **ci:** add empty commit to fix lint checks on master ([4251456](https://github.com/ipfs/js-datastore-s3/commit/4251456)) + + +### Features + +* adds interface-datastore streaming api ([6c74394](https://github.com/ipfs/js-datastore-s3/commit/6c74394)) + + + + +# [0.3.0](https://github.com/ipfs/js-datastore-s3/compare/v0.2.4...v0.3.0) (2019-08-15) + + +### Code Refactoring + +* callbacks -> async / await ([#17](https://github.com/ipfs/js-datastore-s3/issues/17)) ([629dba7](https://github.com/ipfs/js-datastore-s3/commit/629dba7)) + + +### BREAKING CHANGES + +* All places in the API that used callbacks are now replaced with async/await + + + + +## [0.2.4](https://github.com/ipfs/js-datastore-s3/compare/v0.2.3...v0.2.4) (2019-03-27) + + +### Bug Fixes + +* **create-repo:** pass sub paths in repo to each store ([1113c61](https://github.com/ipfs/js-datastore-s3/commit/1113c61)) + + + + +## [0.2.3](https://github.com/ipfs/js-datastore-s3/compare/v0.2.2...v0.2.3) (2019-02-14) + + +### Bug Fixes + +* aws-sdk should be a peer dependency ([836355c](https://github.com/ipfs/js-datastore-s3/commit/836355c)) + + + + +## [0.2.2](https://github.com/ipfs/js-datastore-s3/compare/v0.2.1...v0.2.2) (2019-02-14) + + +### Features + +* add createRepo utility ([0f5021c](https://github.com/ipfs/js-datastore-s3/commit/0f5021c)) + + + + +## [0.2.1](https://github.com/ipfs/js-datastore-s3/compare/v0.2.0...v0.2.1) (2019-02-07) + + +### Bug Fixes + +* use once to prevent multiple callback calls ([db99ae8](https://github.com/ipfs/js-datastore-s3/commit/db99ae8)) + + +### Features + +* have the s3 lock cleanup gracefully ([7f6b2c8](https://github.com/ipfs/js-datastore-s3/commit/7f6b2c8)) + + + + +# 0.2.0 (2018-10-01) + + +### Bug Fixes + +* **flow:** make flow pass and fix query abort call ([46e8e5e](https://github.com/ipfs/js-datastore-s3/commit/46e8e5e)) +* add windows support ([feaed0d](https://github.com/ipfs/js-datastore-s3/commit/feaed0d)) +* linting ([e00974f](https://github.com/ipfs/js-datastore-s3/commit/e00974f)) +* resolve an issue where a new repo wouldnt init properly ([104d6e9](https://github.com/ipfs/js-datastore-s3/commit/104d6e9)) + + +### Features + +* add basic error codes and update test ([#8](https://github.com/ipfs/js-datastore-s3/issues/8)) ([31ba28a](https://github.com/ipfs/js-datastore-s3/commit/31ba28a)) +* add querying and make all tests pass ([0c89c78](https://github.com/ipfs/js-datastore-s3/commit/0c89c78)) +* initial featureset aside from querying ([b710421](https://github.com/ipfs/js-datastore-s3/commit/b710421)) + + + + +# 0.1.0 (2018-05-07) diff --git a/packages/datastore-s3/LICENSE b/packages/datastore-s3/LICENSE new file mode 100644 index 00000000..20ce483c --- /dev/null +++ b/packages/datastore-s3/LICENSE @@ -0,0 +1,4 @@ +This project is dual licensed under MIT and Apache-2.0. + +MIT: https://www.opensource.org/licenses/mit +Apache-2.0: https://www.apache.org/licenses/license-2.0 diff --git a/packages/datastore-s3/LICENSE-APACHE b/packages/datastore-s3/LICENSE-APACHE new file mode 100644 index 00000000..14478a3b --- /dev/null +++ b/packages/datastore-s3/LICENSE-APACHE @@ -0,0 +1,5 @@ +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/packages/datastore-s3/LICENSE-MIT b/packages/datastore-s3/LICENSE-MIT new file mode 100644 index 00000000..72dc60d8 --- /dev/null +++ b/packages/datastore-s3/LICENSE-MIT @@ -0,0 +1,19 @@ +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/packages/datastore-s3/README.md b/packages/datastore-s3/README.md new file mode 100644 index 00000000..72023e81 --- /dev/null +++ b/packages/datastore-s3/README.md @@ -0,0 +1,79 @@ +# datastore-s3 + +[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) +[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/js-stores.svg?style=flat-square)](https://codecov.io/gh/ipfs/js-stores) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/js-stores/js-test-and-release.yml?branch=master\&style=flat-square)](https://github.com/ipfs/js-stores/actions/workflows/js-test-and-release.yml?query=branch%3Amaster) + +> IPFS datastore implementation backed by s3 + +## Table of contents + +- [Install](#install) + - [Browser ` +``` + +## Usage + +If the flag `createIfMissing` is not set or is false, then the bucket must be created prior to using datastore-s3. Please see the AWS docs for information on how to configure the S3 instance. A bucket name is required to be set at the s3 instance level, see the below example. + +```js +import S3 from 'aws-sdk/clients/s3.js' +import { S3Datastore } from 'datastore-s3' + +const s3Instance = new S3({ params: { Bucket: 'my-ipfs-bucket' } }) +const store = new S3Datastore('.ipfs/datastore', { + s3: s3Instance + createIfMissing: false +}) +``` + +### Create a Repo + +See [examples/full-s3-repo](./examples/full-s3-repo) for how to quickly create an S3 backed repo using the `createRepo` convenience function. + +### Examples + +You can see examples of S3 backed ipfs in the [examples folder](examples/) + +## API Docs + +- + +## License + +Licensed under either of + +- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) +- MIT ([LICENSE-MIT](LICENSE-MIT) / ) + +## Contribute + +Contributions welcome! Please check out [the issues](https://github.com/ipfs/js-stores/issues). + +Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. + +Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) diff --git a/packages/datastore-s3/examples/helia/README.md b/packages/datastore-s3/examples/helia/README.md new file mode 100644 index 00000000..b9bd8027 --- /dev/null +++ b/packages/datastore-s3/examples/helia/README.md @@ -0,0 +1,25 @@ +Use with Helia +====== + +This example uses a Datastore S3 instance to serve as the entire backend for Helia. + +## Running +The S3 parameters must be updated with an existing Bucket and credentials with access to it: +```js +// Configure S3 as normal +const s3 = new S3({ + region: 'region', + credentials: { + accessKeyId: 'myaccesskey', + secretAccessKey: 'mysecretkey' + } +}) + +const datastore = new DatastoreS3(s3, 'my-bucket') +``` + +Once the S3 instance has its needed data, you can run the example: +``` +npm install +node index.js +``` diff --git a/packages/datastore-s3/examples/helia/index.js b/packages/datastore-s3/examples/helia/index.js new file mode 100644 index 00000000..0de17895 --- /dev/null +++ b/packages/datastore-s3/examples/helia/index.js @@ -0,0 +1,55 @@ +import { createHelia } from 'helia' +import { unixfs } from '@helia/unixfs' +import toBuffer from 'it-to-buffer' +import { S3 } from '@aws-sdk/client-s3' +import { DatastoreS3 } from 'datastore-s3' + +async function main () { + // Configure S3 as normal + const s3 = new S3({ + region: 'region', + credentials: { + accessKeyId: 'myaccesskey', + secretAccessKey: 'mysecretkey' + } + }) + + const datastore = new DatastoreS3(s3, 'my-bucket') + + // Create a new Helia node with our S3 backed Repo + console.log('Start Helia') + const node = await createHelia({ + datastore + }) + + // Test out the repo by sending and fetching some data + console.log('Helia is ready') + + try { + const fs = unixfs(helia) + + // Let's add a file to Helia + const cid = await fs.addBytes(Uint8Array.from([0, 1, 2, 3])) + + console.log('\nAdded file:', cid) + + // Log out the added files metadata and cat the file from IPFS + const data = await toBuffer(fs.cat(cid)) + + // Print out the files contents to console + console.log(`\nFetched file content containing ${data.byteLength} bytes`) + } catch (err) { + // Log out the error + console.log('File Processing Error:', err) + } + // After everything is done, shut the node down + // We don't need to worry about catching errors here + console.log('\n\nStopping the node') + await node.stop() +} + +main() + .catch(err => { + console.error(err) + process.exit(1) + }) diff --git a/packages/datastore-s3/examples/helia/package.json b/packages/datastore-s3/examples/helia/package.json new file mode 100644 index 00000000..0d887034 --- /dev/null +++ b/packages/datastore-s3/examples/helia/package.json @@ -0,0 +1,19 @@ +{ + "name": "helia", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "author": "", + "license": "ISC", + "dependencies": { + "@aws-sdk/client-s3": "^3.297.0", + "@helia/unixfs": "^1.2.0", + "datastore-s3": "../../", + "helia": "^1.0.0", + "it-to-buffer": "^3.0.1" + } +} diff --git a/packages/datastore-s3/package.json b/packages/datastore-s3/package.json new file mode 100644 index 00000000..ecf2cf76 --- /dev/null +++ b/packages/datastore-s3/package.json @@ -0,0 +1,162 @@ +{ + "name": "datastore-s3", + "version": "11.0.0", + "description": "IPFS datastore implementation backed by s3", + "license": "Apache-2.0 OR MIT", + "homepage": "https://github.com/ipfs/js-stores/tree/master/packages/datastore-s3#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/ipfs/js-stores.git" + }, + "bugs": { + "url": "https://github.com/ipfs/js-stores/issues" + }, + "keywords": [ + "datastore", + "interface", + "ipfs", + "key-value", + "s3" + ], + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + }, + "type": "module", + "types": "./dist/src/index.d.ts", + "files": [ + "src", + "dist", + "!dist/test", + "!**/*.tsbuildinfo" + ], + "exports": { + ".": { + "types": "./dist/src/index.d.ts", + "import": "./src/index.js" + } + }, + "eslintConfig": { + "extends": "ipfs", + "parserOptions": { + "sourceType": "module" + } + }, + "release": { + "branches": [ + "master" + ], + "plugins": [ + [ + "@semantic-release/commit-analyzer", + { + "preset": "conventionalcommits", + "releaseRules": [ + { + "breaking": true, + "release": "major" + }, + { + "revert": true, + "release": "patch" + }, + { + "type": "feat", + "release": "minor" + }, + { + "type": "fix", + "release": "patch" + }, + { + "type": "docs", + "release": "patch" + }, + { + "type": "test", + "release": "patch" + }, + { + "type": "deps", + "release": "patch" + }, + { + "scope": "no-release", + "release": false + } + ] + } + ], + [ + "@semantic-release/release-notes-generator", + { + "preset": "conventionalcommits", + "presetConfig": { + "types": [ + { + "type": "feat", + "section": "Features" + }, + { + "type": "fix", + "section": "Bug Fixes" + }, + { + "type": "chore", + "section": "Trivial Changes" + }, + { + "type": "docs", + "section": "Documentation" + }, + { + "type": "deps", + "section": "Dependencies" + }, + { + "type": "test", + "section": "Tests" + } + ] + } + } + ], + "@semantic-release/changelog", + "@semantic-release/npm", + "@semantic-release/github", + "@semantic-release/git" + ] + }, + "scripts": { + "clean": "aegir clean", + "lint": "aegir lint", + "build": "aegir build", + "release": "aegir release", + "test": "aegir test", + "test:node": "aegir test -t node --cov", + "test:chrome": "aegir test -t browser --cov", + "test:chrome-webworker": "aegir test -t webworker", + "test:firefox": "aegir test -t browser -- --browser firefox", + "test:firefox-webworker": "aegir test -t webworker -- --browser firefox", + "test:electron-main": "aegir test -t electron-main", + "dep-check": "aegir dep-check" + }, + "dependencies": { + "@aws-sdk/client-s3": "^3.297.0", + "datastore-core": "^9.0.0", + "interface-datastore": "^8.0.0", + "interface-store": "^5.0.0", + "it-filter": "^2.0.1", + "it-to-buffer": "^3.0.0", + "uint8arrays": "^4.0.2" + }, + "devDependencies": { + "aegir": "^38.1.7", + "interface-datastore-tests": "^5.0.0", + "p-defer": "^4.0.0", + "sinon": "^15.0.2" + }, + "typedoc": { + "entryPoint": "./src/index.ts" + } +} diff --git a/packages/datastore-s3/src/index.ts b/packages/datastore-s3/src/index.ts new file mode 100644 index 00000000..bb261a3b --- /dev/null +++ b/packages/datastore-s3/src/index.ts @@ -0,0 +1,296 @@ +import filter from 'it-filter' +import { Key, KeyQuery, Pair, Query } from 'interface-datastore' +import { BaseDatastore } from 'datastore-core/base' +import * as Errors from 'datastore-core/errors' +import { fromString as unint8arrayFromString } from 'uint8arrays' +import toBuffer from 'it-to-buffer' +import type { S3 } from '@aws-sdk/client-s3' +import type { AbortOptions } from 'interface-store' +import { + PutObjectCommand, + CreateBucketCommand, + GetObjectCommand, + HeadObjectCommand, + DeleteObjectCommand, + ListObjectsV2Command +} from '@aws-sdk/client-s3' + +export interface S3DatastoreInit { + /** + * An optional path to use within the bucket for all files - this setting can + * affect S3 performance as it does internal sharding based on 'prefixes' - + * these can be delimited by '/' so it's often better to wrap this datastore in + * a sharding datastore which will generate prefixed datastore keys for you. + * + * See - https://docs.aws.amazon.com/AmazonS3/latest/userguide/optimizing-performance.html + * and https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-prefixes.html + */ + path?: string + + /** + * Whether to try to create the bucket if it is missing when `.open` is called + */ + createIfMissing?: boolean +} + +/** + * A datastore backed by AWS S3 + */ +export class S3Datastore extends BaseDatastore { + public path?: string + public createIfMissing: boolean + private readonly s3: S3 + private readonly bucket: string + + constructor (s3: S3, bucket: string, init?: S3DatastoreInit) { + super() + + if (s3 == null) { + throw new Error('An S3 instance must be supplied. See the datastore-s3 README for examples.') + } + + if (bucket == null) { + throw new Error('An bucket must be supplied. See the datastore-s3 README for examples.') + } + + this.path = init?.path + this.s3 = s3 + this.bucket = bucket + this.createIfMissing = init?.createIfMissing ?? false + } + + /** + * Returns the full key which includes the path to the ipfs store + */ + _getFullKey (key: Key): string { + // Avoid absolute paths with s3 + return [this.path, key.toString()].filter(Boolean).join('/').replace(/\/\/+/g, '/') + } + + /** + * Store the given value under the key. + */ + async put (key: Key, val: Uint8Array, options?: AbortOptions): Promise { + try { + // @ts-expect-error the AWS AbortSignal types are different to the @types/node version + await this.s3.send( + new PutObjectCommand({ + Bucket: this.bucket, + Key: this._getFullKey(key), + Body: val + }), { + abortSignal: options?.signal + } + ) + + return key + } catch (err: any) { + throw Errors.dbWriteFailedError(err) + } + } + + /** + * Read from s3 + */ + async get (key: Key, options?: AbortOptions): Promise { + try { + const data = await this.s3.send( + new GetObjectCommand({ + Bucket: this.bucket, + Key: this._getFullKey(key) + // @ts-expect-error the AWS AbortSignal types are different to the @types/node version + }, { + abortSignal: options?.signal + }) + ) + + if (data.Body == null) { + throw new Error('Response had no body') + } + + // If a body was returned, ensure it's a Uint8Array + if (data.Body instanceof Uint8Array) { + return data.Body + } + + if (typeof data.Body === 'string') { + return unint8arrayFromString(data.Body) + } + + if (data.Body instanceof Blob) { + const buf = await data.Body.arrayBuffer() + + return new Uint8Array(buf, 0, buf.byteLength) + } + + // @ts-expect-error s3 types define their own Blob as an empty interface + return await toBuffer(data.Body) + } catch (err: any) { + if (err.statusCode === 404) { + throw Errors.notFoundError(err) + } + throw err + } + } + + /** + * Check for the existence of the given key + */ + async has (key: Key, options?: AbortOptions): Promise { + try { + // @ts-expect-error the AWS AbortSignal types are different to the @types/node version + await this.s3.send( + new HeadObjectCommand({ + Bucket: this.bucket, + Key: this._getFullKey(key) + }), { + abortSignal: options?.signal + } + ) + + return true + } catch (err: any) { + // doesn't exist and permission policy includes s3:ListBucket + if (err.$metadata?.httpStatusCode === 404) { + return false + } + + // doesn't exist, permission policy does not include s3:ListBucket + if (err.$metadata?.httpStatusCode === 403) { + return false + } + + throw err + } + } + + /** + * Delete the record under the given key + */ + async delete (key: Key, options?: AbortOptions): Promise { + try { + // @ts-expect-error the AWS AbortSignal types are different to the @types/node version + await this.s3.send( + new DeleteObjectCommand({ + Bucket: this.bucket, + Key: this._getFullKey(key) + }), { + abortSignal: options?.signal + } + ) + } catch (err: any) { + throw Errors.dbDeleteFailedError(err) + } + } + + /** + * Recursively fetches all keys from s3 + */ + async * _listKeys (params: { Prefix?: string, StartAfter?: string }, options?: AbortOptions): AsyncIterable { + try { + // @ts-expect-error the AWS AbortSignal types are different to the @types/node version + const data = await this.s3.send( + new ListObjectsV2Command({ + Bucket: this.bucket, + ...params + }), { + abortSignal: options?.signal + } + ) + + if (options?.signal?.aborted === true) { + return + } + + if (data == null || data.Contents == null) { + throw new Error('Not found') + } + + for (const d of data.Contents) { + if (d.Key == null) { + throw new Error('Not found') + } + + // Remove the path from the key + yield new Key(d.Key.slice((this.path ?? '').length), false) + } + + // If we didn't get all records, recursively query + if (data.IsTruncated === true) { + // If NextMarker is absent, use the key from the last result + params.StartAfter = data.Contents[data.Contents.length - 1].Key + + // recursively fetch keys + yield * this._listKeys(params) + } + } catch (err: any) { + throw new Error(err.code) + } + } + + async * _all (q: Query, options?: AbortOptions): AsyncIterable { + for await (const key of this._allKeys({ prefix: q.prefix }, options)) { + try { + const res: Pair = { + key, + value: await this.get(key, options) + } + + yield res + } catch (err: any) { + // key was deleted while we are iterating over the results + if (err.statusCode !== 404) { + throw err + } + } + } + } + + async * _allKeys (q: KeyQuery, options?: AbortOptions): AsyncIterable { + const prefix = [this.path, q.prefix ?? ''].filter(Boolean).join('/').replace(/\/\/+/g, '/') + + // Get all the keys via list object, recursively as needed + let it = this._listKeys({ + Prefix: prefix + }, options) + + if (q.prefix != null) { + it = filter(it, k => k.toString().startsWith(`${q.prefix ?? ''}`)) + } + + yield * it + } + + /** + * This will check the s3 bucket to ensure access and existence + */ + async open (options?: AbortOptions): Promise { + try { + // @ts-expect-error the AWS AbortSignal types are different to the @types/node version + await this.s3.send( + new HeadObjectCommand({ + Bucket: this.bucket, + Key: this.path ?? '' + }), { + abortSignal: options?.signal + } + ) + } catch (err: any) { + if (err.statusCode !== 404) { + if (this.createIfMissing) { + // @ts-expect-error the AWS AbortSignal types are different to the @types/node version + await this.s3.send( + new CreateBucketCommand({ + Bucket: this.bucket + }), { + abortSignal: options?.signal + } + ) + return + } + + throw Errors.dbOpenFailedError(err) + } + } + } +} diff --git a/packages/datastore-s3/test/index.spec.ts b/packages/datastore-s3/test/index.spec.ts new file mode 100644 index 00000000..007850cc --- /dev/null +++ b/packages/datastore-s3/test/index.spec.ts @@ -0,0 +1,227 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import sinon from 'sinon' +import { Key } from 'interface-datastore' +import { CreateBucketCommand, PutObjectCommand, HeadObjectCommand, S3, GetObjectCommand } from '@aws-sdk/client-s3' +import defer from 'p-defer' +import { interfaceDatastoreTests } from 'interface-datastore-tests' + +import { s3Resolve, s3Reject, S3Error, s3Mock } from './utils/s3-mock.js' +import { S3Datastore } from '../src/index.js' + +describe('S3Datastore', () => { + describe('construction', () => { + it('requires an s3', () => { + expect( + // @ts-expect-error missing params + () => new S3Datastore() + ).to.throw() + }) + + it('requires a bucket', () => { + const s3 = new S3({ region: 'REGION' }) + expect( + // @ts-expect-error missing params + () => new S3Datastore(s3) + ).to.throw() + }) + + it('createIfMissing defaults to false', () => { + const s3 = new S3({ region: 'REGION' }) + const store = new S3Datastore(s3, 'test') + expect(store.createIfMissing).to.equal(false) + }) + + it('createIfMissing can be set to true', () => { + const s3 = new S3({ region: 'REGION' }) + const store = new S3Datastore(s3, 'test', { createIfMissing: true }) + expect(store.createIfMissing).to.equal(true) + }) + }) + + describe('put', () => { + it('should include the path in the key', async () => { + const s3 = new S3({ region: 'REGION' }) + const store = new S3Datastore(s3, 'test', { + path: '.ipfs/datastore' + }) + + const deferred = defer() + + sinon.replace(s3, 'send', (command: PutObjectCommand) => { + deferred.resolve(command) + return s3Resolve(null) + }) + + await store.put(new Key('/z/key'), new TextEncoder().encode('test data')) + + const command = await deferred.promise + expect(command).to.have.nested.property('input.Key', '.ipfs/datastore/z/key') + }) + + it('should return a standard error when the put fails', async () => { + const s3 = new S3({ region: 'REGION' }) + const store = new S3Datastore(s3, 'test') + + sinon.replace(s3, 'send', (command: any) => { + if (command.constructor.name === 'PutObjectCommand') { + return s3Reject(new Error('bad things happened')) + } + + return s3Reject(new S3Error('UnknownCommand')) + }) + + await expect(store.put(new Key('/z/key'), new TextEncoder().encode('test data'))).to.eventually.rejected + .with.property('code', 'ERR_DB_WRITE_FAILED') + }) + }) + + describe('get', () => { + it('should include the path in the fetch key', async () => { + const s3 = new S3({ region: 'REGION' }) + const store = new S3Datastore(s3, 'test', { + path: '.ipfs/datastore' + }) + const buf = new TextEncoder().encode('test') + + const deferred = defer() + + sinon.replace(s3, 'send', (command: any) => { + if (command.constructor.name === 'GetObjectCommand') { + deferred.resolve(command) + return s3Resolve({ Body: buf }) + } + + return s3Reject(new S3Error('UnknownCommand')) + }) + + const value = await store.get(new Key('/z/key')) + + expect(value).to.equalBytes(buf) + + const getObjectCommand = await deferred.promise + expect(getObjectCommand).to.have.nested.property('input.Key', '.ipfs/datastore/z/key') + }) + + it('should return a standard not found error code if the key isn\'t found', async () => { + const s3 = new S3({ region: 'REGION' }) + const store = new S3Datastore(s3, 'test') + + sinon.replace(s3, 'send', (command: any) => { + if (command.constructor.name === 'GetObjectCommand') { + return s3Reject(new S3Error('NotFound', 404)) + } + + return s3Reject(new S3Error('UnknownCommand')) + }) + + await expect(store.get(new Key('/z/key'))).to.eventually.rejected + .with.property('code', 'ERR_NOT_FOUND') + }) + }) + + describe('delete', () => { + it('should return a standard delete error if deletion fails', async () => { + const s3 = new S3({ region: 'REGION' }) + const store = new S3Datastore(s3, 'test') + + sinon.replace(s3, 'send', (command: any) => { + if (command.constructor.name === 'DeleteObjectCommand') { + return s3Reject(new Error('bad things')) + } + + return s3Reject(new S3Error('UnknownCommand')) + }) + + await expect(store.delete(new Key('/z/key'))).to.eventually.rejected + .with.property('code', 'ERR_DB_DELETE_FAILED') + }) + }) + + describe('open', () => { + it('should create the bucket when missing if createIfMissing is true', async () => { + const s3 = new S3({ region: 'REGION' }) + const store = new S3Datastore(s3, 'test', { createIfMissing: true }) + + // 1. On the first call upload will fail with a NoSuchBucket error + // 2. This should result in the `createBucket` standin being called + // 3. upload is then called a second time and it passes + + const bucketTested = defer() + const bucketCreated = defer() + + sinon.replace(s3, 'send', (command: any) => { + if (command.constructor.name === 'HeadObjectCommand') { + bucketTested.resolve(command) + return s3Reject(new S3Error('NoSuchBucket')) + } + + if (command.constructor.name === 'CreateBucketCommand') { + bucketCreated.resolve(command) + return s3Resolve(null) + } + + return s3Reject(new S3Error('UnknownCommand')) + }) + + await store.open() + + const headObjectCommand = await bucketTested.promise + expect(headObjectCommand).to.have.nested.property('input.Bucket', 'test') + + const createBucketCommand = await bucketCreated.promise + expect(createBucketCommand).to.have.nested.property('input.Bucket', 'test') + }) + + it('should not create the bucket when missing if createIfMissing is false', async () => { + const s3 = new S3({ region: 'REGION' }) + const store = new S3Datastore(s3, 'test') + + const bucketTested = defer() + + sinon.replace(s3, 'send', (command: any) => { + if (command.constructor.name === 'HeadObjectCommand') { + bucketTested.resolve(command) + return s3Reject(new S3Error('NoSuchBucket')) + } + + return s3Reject(new S3Error('UnknownCommand')) + }) + + await expect(store.open()).to.eventually.rejected + .with.property('code', 'ERR_DB_OPEN_FAILED') + + const headObjectCommand = await bucketTested.promise + expect(headObjectCommand).to.have.nested.property('input.Bucket', 'test') + }) + + it('should return a standard open error if the head request fails with an unknown error', async () => { + const s3 = new S3({ region: 'REGION' }) + const store = new S3Datastore(s3, 'test') + + sinon.replace(s3, 'send', (command: any) => { + if (command.constructor.name === 'HeadObjectCommand') { + return s3Reject(new Error('bad things')) + } + + return s3Reject(new S3Error('UnknownCommand')) + }) + + await expect(store.open()).to.eventually.rejected + .with.property('code', 'ERR_DB_OPEN_FAILED') + }) + }) + + describe('interface-datastore', () => { + interfaceDatastoreTests({ + setup () { + const s3 = s3Mock(new S3({ region: 'REGION' })) + + return new S3Datastore(s3, 'test') + }, + teardown () { + } + }) + }) +}) diff --git a/packages/datastore-s3/test/utils/s3-mock.ts b/packages/datastore-s3/test/utils/s3-mock.ts new file mode 100644 index 00000000..6c4bf01f --- /dev/null +++ b/packages/datastore-s3/test/utils/s3-mock.ts @@ -0,0 +1,87 @@ +import sinon from 'sinon' +import type { S3 } from '@aws-sdk/client-s3' + +export class S3Error extends Error { + public code: string + public statusCode?: number + public $metadata?: { httpStatusCode: number } + + constructor (message: string, code?: number) { + super(message) + this.code = message + this.statusCode = code + + this.$metadata = { + httpStatusCode: code ?? 200 + } + } +} + +export const s3Resolve = (res?: any): any => { + return Promise.resolve(res) +} + +export const s3Reject = (err: T): any => { + return Promise.reject(err) +} + +/** + * Mocks out the s3 calls made by datastore-s3 + */ +export function s3Mock (s3: S3): S3 { + const mocks: any = {} + const storage: Map = new Map() + + mocks.send = sinon.replace(s3, 'send', (command) => { + const commandName = command.constructor.name + const input: any = command.input + + if (commandName === 'PutObjectCommand') { + storage.set(input.Key, input.Body) + return s3Resolve({}) + } + + if (commandName === 'HeadObjectCommand') { + if (storage.has(input.Key)) { + return s3Resolve({}) + } + + return s3Reject(new S3Error('NotFound', 404)) + } + + if (commandName === 'GetObjectCommand') { + if (!storage.has(input.Key)) { + return s3Reject(new S3Error('NotFound', 404)) + } + + return s3Resolve({ + Body: storage.get(input.Key) + }) + } + + if (commandName === 'DeleteObjectCommand') { + storage.delete(input.Key) + return s3Resolve({}) + } + + if (commandName === 'ListObjectsV2Command') { + const results: { Contents: Array<{ Key: string }> } = { + Contents: [] + } + + for (const k of storage.keys()) { + if (k.startsWith(`${input.Prefix ?? ''}`)) { + results.Contents.push({ + Key: k + }) + } + } + + return s3Resolve(results) + } + + return s3Reject(new S3Error('UnknownCommand', 400)) + }) + + return s3 +} diff --git a/packages/datastore-s3/tsconfig.json b/packages/datastore-s3/tsconfig.json new file mode 100644 index 00000000..e30932c0 --- /dev/null +++ b/packages/datastore-s3/tsconfig.json @@ -0,0 +1,24 @@ +{ + "extends": "aegir/src/config/tsconfig.aegir.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": [ + "src", + "test" + ], + "references": [ + { + "path": "../datastore-core" + }, + { + "path": "../interface-datastore" + }, + { + "path": "../interface-datastore-tests" + }, + { + "path": "../interface-store" + } + ] +} diff --git a/packages/interface-blockstore-tests/README.md b/packages/interface-blockstore-tests/README.md index 0362e5f6..9df460db 100644 --- a/packages/interface-blockstore-tests/README.md +++ b/packages/interface-blockstore-tests/README.md @@ -2,8 +2,8 @@ [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) -[![codecov](https://img.shields.io/codecov/c/github/ipfs/js-ipfs-interfaces.svg?style=flat-square)](https://codecov.io/gh/ipfs/js-ipfs-interfaces) -[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/js-ipfs-interfaces/js-test-and-release.yml?branch=master\&style=flat-square)](https://github.com/ipfs/js-ipfs-interfaces/actions/workflows/js-test-and-release.yml?query=branch%3Amaster) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/js-stores.svg?style=flat-square)](https://codecov.io/gh/ipfs/js-stores) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/js-stores/js-test-and-release.yml?branch=master\&style=flat-square)](https://github.com/ipfs/js-stores/actions/workflows/js-test-and-release.yml?query=branch%3Amaster) > Compliance tests for the blockstore interface @@ -41,7 +41,7 @@ describe('MyBlockstore', () => { ## API Docs -- +- ## License @@ -52,7 +52,7 @@ Licensed under either of ## Contribute -Contributions welcome! Please check out [the issues](https://github.com/ipfs/js-ipfs-interfaces/issues). +Contributions welcome! Please check out [the issues](https://github.com/ipfs/js-stores/issues). Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. diff --git a/packages/interface-blockstore-tests/package.json b/packages/interface-blockstore-tests/package.json index dfc2dfb2..a5022319 100644 --- a/packages/interface-blockstore-tests/package.json +++ b/packages/interface-blockstore-tests/package.json @@ -3,13 +3,13 @@ "version": "6.0.0", "description": "Compliance tests for the blockstore interface", "license": "Apache-2.0 OR MIT", - "homepage": "https://github.com/ipfs/js-ipfs-interfaces/tree/master/packages/interface-blockstore-tests#readme", + "homepage": "https://github.com/ipfs/js-stores/tree/master/packages/interface-blockstore-tests#readme", "repository": { "type": "git", - "url": "git+https://github.com/ipfs/js-ipfs-interfaces.git" + "url": "git+https://github.com/ipfs/js-stores.git" }, "bugs": { - "url": "https://github.com/ipfs/js-ipfs-interfaces/issues" + "url": "https://github.com/ipfs/js-stores/issues" }, "keywords": [ "blockstore", @@ -138,7 +138,7 @@ "interface-blockstore": "^5.0.0", "it-all": "^2.0.0", "it-drain": "^2.0.0", - "multiformats": "^11.0.0", + "multiformats": "^11.0.2", "uint8arrays": "^4.0.2" }, "typedoc": { diff --git a/packages/interface-blockstore/README.md b/packages/interface-blockstore/README.md index db2597b6..ad7905fe 100644 --- a/packages/interface-blockstore/README.md +++ b/packages/interface-blockstore/README.md @@ -2,8 +2,8 @@ [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) -[![codecov](https://img.shields.io/codecov/c/github/ipfs/js-ipfs-interfaces.svg?style=flat-square)](https://codecov.io/gh/ipfs/js-ipfs-interfaces) -[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/js-ipfs-interfaces/js-test-and-release.yml?branch=master\&style=flat-square)](https://github.com/ipfs/js-ipfs-interfaces/actions/workflows/js-test-and-release.yml?query=branch%3Amaster) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/js-stores.svg?style=flat-square)](https://codecov.io/gh/ipfs/js-stores) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/js-stores/js-test-and-release.yml?branch=master\&style=flat-square)](https://github.com/ipfs/js-stores/actions/workflows/js-test-and-release.yml?query=branch%3Amaster) > An interface for storing and retrieving blocks @@ -22,7 +22,7 @@ $ npm i interface-blockstore ## API Docs -- +- ## License @@ -33,7 +33,7 @@ Licensed under either of ## Contribute -Contributions welcome! Please check out [the issues](https://github.com/ipfs/js-ipfs-interfaces/issues). +Contributions welcome! Please check out [the issues](https://github.com/ipfs/js-stores/issues). Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. diff --git a/packages/interface-blockstore/package.json b/packages/interface-blockstore/package.json index ee67261e..44cf109d 100644 --- a/packages/interface-blockstore/package.json +++ b/packages/interface-blockstore/package.json @@ -3,13 +3,13 @@ "version": "5.1.1", "description": "An interface for storing and retrieving blocks", "license": "Apache-2.0 OR MIT", - "homepage": "https://github.com/ipfs/js-ipfs-interfaces/tree/master/packages/interface-blockstore#readme", + "homepage": "https://github.com/ipfs/js-stores/tree/master/packages/interface-blockstore#readme", "repository": { "type": "git", - "url": "git+https://github.com/ipfs/js-ipfs-interfaces.git" + "url": "git+https://github.com/ipfs/js-stores.git" }, "bugs": { - "url": "https://github.com/ipfs/js-ipfs-interfaces/issues" + "url": "https://github.com/ipfs/js-stores/issues" }, "engines": { "node": ">=16.0.0", @@ -128,10 +128,10 @@ }, "dependencies": { "interface-store": "^5.0.0", - "multiformats": "^11.0.0" + "multiformats": "^11.0.2" }, "devDependencies": { - "aegir": "^38.1.0" + "aegir": "^38.1.7" }, "typedoc": { "entryPoint": "./src/index.ts" diff --git a/packages/interface-datastore-tests/README.md b/packages/interface-datastore-tests/README.md index 5d34eac4..13a8c95f 100644 --- a/packages/interface-datastore-tests/README.md +++ b/packages/interface-datastore-tests/README.md @@ -2,8 +2,8 @@ [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) -[![codecov](https://img.shields.io/codecov/c/github/ipfs/js-ipfs-interfaces.svg?style=flat-square)](https://codecov.io/gh/ipfs/js-ipfs-interfaces) -[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/js-ipfs-interfaces/js-test-and-release.yml?branch=master\&style=flat-square)](https://github.com/ipfs/js-ipfs-interfaces/actions/workflows/js-test-and-release.yml?query=branch%3Amaster) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/js-stores.svg?style=flat-square)](https://codecov.io/gh/ipfs/js-stores) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/js-stores/js-test-and-release.yml?branch=master\&style=flat-square)](https://github.com/ipfs/js-stores/actions/workflows/js-test-and-release.yml?query=branch%3Amaster) > Compliance tests for the datastore interface @@ -41,7 +41,7 @@ describe('MyDatastore', () => { ## API Docs -- +- ## License @@ -52,7 +52,7 @@ Licensed under either of ## Contribute -Contributions welcome! Please check out [the issues](https://github.com/ipfs/js-ipfs-interfaces/issues). +Contributions welcome! Please check out [the issues](https://github.com/ipfs/js-stores/issues). Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. diff --git a/packages/interface-datastore-tests/package.json b/packages/interface-datastore-tests/package.json index c3b10660..f4c78fce 100644 --- a/packages/interface-datastore-tests/package.json +++ b/packages/interface-datastore-tests/package.json @@ -3,13 +3,13 @@ "version": "5.0.0", "description": "Compliance tests for the datastore interface", "license": "Apache-2.0 OR MIT", - "homepage": "https://github.com/ipfs/js-ipfs-interfaces/tree/master/packages/interface-datastore-tests#readme", + "homepage": "https://github.com/ipfs/js-stores/tree/master/packages/interface-datastore-tests#readme", "repository": { "type": "git", - "url": "git+https://github.com/ipfs/js-ipfs-interfaces.git" + "url": "git+https://github.com/ipfs/js-stores.git" }, "bugs": { - "url": "https://github.com/ipfs/js-ipfs-interfaces/issues" + "url": "https://github.com/ipfs/js-stores/issues" }, "keywords": [ "datastore", diff --git a/packages/interface-datastore/README.md b/packages/interface-datastore/README.md index 39107f5a..de06de8f 100644 --- a/packages/interface-datastore/README.md +++ b/packages/interface-datastore/README.md @@ -2,8 +2,8 @@ [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) -[![codecov](https://img.shields.io/codecov/c/github/ipfs/js-ipfs-interfaces.svg?style=flat-square)](https://codecov.io/gh/ipfs/js-ipfs-interfaces) -[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/js-ipfs-interfaces/js-test-and-release.yml?branch=master\&style=flat-square)](https://github.com/ipfs/js-ipfs-interfaces/actions/workflows/js-test-and-release.yml?query=branch%3Amaster) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/js-stores.svg?style=flat-square)](https://codecov.io/gh/ipfs/js-stores) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/js-stores/js-test-and-release.yml?branch=master\&style=flat-square)](https://github.com/ipfs/js-stores/actions/workflows/js-test-and-release.yml?query=branch%3Amaster) > datastore interface @@ -127,7 +127,7 @@ Also, every namespace can be parameterized to embed relevant object information. ## API Docs -- +- ## License @@ -138,7 +138,7 @@ Licensed under either of ## Contribute -Contributions welcome! Please check out [the issues](https://github.com/ipfs/js-ipfs-interfaces/issues). +Contributions welcome! Please check out [the issues](https://github.com/ipfs/js-stores/issues). Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. diff --git a/packages/interface-datastore/package.json b/packages/interface-datastore/package.json index 86c99289..693d325d 100644 --- a/packages/interface-datastore/package.json +++ b/packages/interface-datastore/package.json @@ -3,13 +3,13 @@ "version": "8.1.2", "description": "datastore interface", "license": "Apache-2.0 OR MIT", - "homepage": "https://github.com/ipfs/js-ipfs-interfaces/tree/master/packages/interface-datastore#readme", + "homepage": "https://github.com/ipfs/js-stores/tree/master/packages/interface-datastore#readme", "repository": { "type": "git", - "url": "git+https://github.com/ipfs/js-ipfs-interfaces.git" + "url": "git+https://github.com/ipfs/js-stores.git" }, "bugs": { - "url": "https://github.com/ipfs/js-ipfs-interfaces/issues" + "url": "https://github.com/ipfs/js-stores/issues" }, "keywords": [ "datastore", @@ -165,7 +165,7 @@ "uint8arrays": "^4.0.2" }, "devDependencies": { - "aegir": "^38.1.0" + "aegir": "^38.1.7" }, "typedoc": { "entryPoint": "./src/index.ts" diff --git a/packages/interface-store/README.md b/packages/interface-store/README.md index b78b0f23..9bd9409a 100644 --- a/packages/interface-store/README.md +++ b/packages/interface-store/README.md @@ -2,8 +2,8 @@ [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) -[![codecov](https://img.shields.io/codecov/c/github/ipfs/js-ipfs-interfaces.svg?style=flat-square)](https://codecov.io/gh/ipfs/js-ipfs-interfaces) -[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/js-ipfs-interfaces/js-test-and-release.yml?branch=master\&style=flat-square)](https://github.com/ipfs/js-ipfs-interfaces/actions/workflows/js-test-and-release.yml?query=branch%3Amaster) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/js-stores.svg?style=flat-square)](https://codecov.io/gh/ipfs/js-stores) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/js-stores/js-test-and-release.yml?branch=master\&style=flat-square)](https://github.com/ipfs/js-stores/actions/workflows/js-test-and-release.yml?query=branch%3Amaster) > A generic interface for storing and retrieving data @@ -22,7 +22,7 @@ $ npm i interface-store ## API Docs -- +- ## License @@ -33,7 +33,7 @@ Licensed under either of ## Contribute -Contributions welcome! Please check out [the issues](https://github.com/ipfs/js-ipfs-interfaces/issues). +Contributions welcome! Please check out [the issues](https://github.com/ipfs/js-stores/issues). Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. diff --git a/packages/interface-store/package.json b/packages/interface-store/package.json index f250456f..9af40127 100644 --- a/packages/interface-store/package.json +++ b/packages/interface-store/package.json @@ -3,13 +3,13 @@ "version": "5.0.1", "description": "A generic interface for storing and retrieving data", "license": "Apache-2.0 OR MIT", - "homepage": "https://github.com/ipfs/js-ipfs-interfaces/tree/master/packages/interface-store#readme", + "homepage": "https://github.com/ipfs/js-stores/tree/master/packages/interface-store#readme", "repository": { "type": "git", - "url": "git+https://github.com/ipfs/js-ipfs-interfaces.git" + "url": "git+https://github.com/ipfs/js-stores.git" }, "bugs": { - "url": "https://github.com/ipfs/js-ipfs-interfaces/issues" + "url": "https://github.com/ipfs/js-stores/issues" }, "engines": { "node": ">=16.0.0", @@ -130,7 +130,7 @@ "release": "aegir release" }, "devDependencies": { - "aegir": "^38.1.0" + "aegir": "^38.1.7" }, "typedoc": { "entryPoint": "./src/index.ts"