From 5b5d9886dd0825217a7861438a9bdc09b49b7dac Mon Sep 17 00:00:00 2001 From: Ivan Zhang Date: Fri, 10 Nov 2023 16:29:50 -0800 Subject: [PATCH 01/20] add transformers.js --- README.md | 6 +- package-lock.json | 1138 +++++++++++++++++++++++++++++++++++++-- package.json | 12 +- src/VectorStorage.ts | 155 ++++-- src/common/constants.ts | 3 +- src/types/IVSOptions.ts | 3 +- 6 files changed, 1225 insertions(+), 92 deletions(-) diff --git a/README.md b/README.md index 5126d2b..4f76e80 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Vector Storage -Vector Storage is a lightweight and efficient vector database that stores document vectors in the browser's IndexedDB. This package allows you to perform semantic similarity searches on text documents using vector embeddings. Semantic search refers to the ability to understand the meaning and context of text documents and queries, enabling more accurate and relevant search results. Vector Storage leverages OpenAI embeddings to convert text documents into vectors and provides an interface for searching similar documents based on cosine similarity. +Vector Storage is a lightweight and efficient vector database that stores document vectors in the browser's IndexedDB. This package allows you to perform semantic similarity searches on text documents using vector embeddings. Semantic search refers to the ability to understand the meaning and context of text documents and queries, enabling more accurate and relevant search results. Vector Storage leverages OpenAI or Transformer.js embeddings (see compatible models [here](https://huggingface.co/models?library=transformers.js)) to convert text documents into vectors and provides an interface for searching similar documents based on cosine similarity. ## Features @@ -71,7 +71,7 @@ interface IVSOptions { openAIApiKey: string; // The OpenAI API key used for generating embeddings. maxSizeInMB?: number; // The maximum size of the storage in megabytes. Defaults to 2GB debounceTime?: number; // The debounce time in milliseconds for saving to IndexedDB. Defaults to 0. - openaiModel?: string; // The OpenAI model used for generating embeddings. Defaults to 'text-embedding-ada-002'. + openaiModel?: string; // The OpenAI model used for generating embeddings. } ``` @@ -131,4 +131,4 @@ Please ensure that your code follows the project's coding style and that all tes This project is licensed under the MIT License. See the LICENSE file for the full license text. -Copyright (c) Nitai Aharoni. All rights reserved. \ No newline at end of file +Copyright (c) Nitai Aharoni. All rights reserved. diff --git a/package-lock.json b/package-lock.json index 58bb753..003e577 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,14 +1,15 @@ { - "name": "vector-storage", - "version": "1.0.54", + "name": "react-embeddings-search", + "version": "0.01", "lockfileVersion": 2, "requires": true, "packages": { "": { - "name": "vector-storage", - "version": "1.0.54", + "name": "react-embeddings-search", + "version": "0.01", "license": "MIT", "dependencies": { + "@xenova/transformers": "^2.8.0", "dexie": "^3.2.3", "idb": "^7.1.1" }, @@ -167,6 +168,60 @@ "node": ">= 8" } }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", + "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", + "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", + "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1", + "@protobufjs/inquire": "^1.1.0" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==" + }, + "node_modules/@protobufjs/inquire": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", + "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", + "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" + }, "node_modules/@types/json-schema": { "version": "7.0.11", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.11.tgz", @@ -185,11 +240,15 @@ "integrity": "sha512-r22s9tAS7imvBt2lyHC9B8AGwWnXaYb1tY09oyLkXDs4vArpYJzw09nj8MLx5VfciBPGIb+ZwG0ssYnEPJxn/g==", "dev": true }, + "node_modules/@types/long": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.2.tgz", + "integrity": "sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==" + }, "node_modules/@types/node": { "version": "18.16.13", "resolved": "https://registry.npmjs.org/@types/node/-/node-18.16.13.tgz", - "integrity": "sha512-uZRomboV1vBL61EBXneL4j9/hEn+1Yqa4LQdpGrKmXFyJmVfWc9JV9+yb2AlnOnuaDnb2PDO3hC6/LKmzJxP1A==", - "dev": true + "integrity": "sha512-uZRomboV1vBL61EBXneL4j9/hEn+1Yqa4LQdpGrKmXFyJmVfWc9JV9+yb2AlnOnuaDnb2PDO3hC6/LKmzJxP1A==" }, "node_modules/@types/semver": { "version": "7.5.0", @@ -385,6 +444,18 @@ "url": "https://opencollective.com/typescript-eslint" } }, + "node_modules/@xenova/transformers": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/@xenova/transformers/-/transformers-2.8.0.tgz", + "integrity": "sha512-+6/cC/iJd7A+B+C3OPuiIw3dhisdUqBRD/f8CAoo2wYfHofF9zn46pH/dTLAOPoI2px7L9+bvJoXSKO18TBj2Q==", + "dependencies": { + "onnxruntime-web": "1.14.0", + "sharp": "^0.32.0" + }, + "optionalDependencies": { + "onnxruntime-node": "1.14.0" + } + }, "node_modules/acorn": { "version": "8.8.2", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.2.tgz", @@ -551,12 +622,46 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/b4a": { + "version": "1.6.4", + "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.6.4.tgz", + "integrity": "sha512-fpWrvyVHEKyeEvbKZTVOeZF3VSKKWtJxFIxX/jaVPf+cLbGUSitjb49pHLqPV2BUNNZ0LcoeEGfE/YCpyDYHIw==" + }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", "dev": true }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, "node_modules/brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -579,6 +684,29 @@ "node": ">=8" } }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, "node_modules/builtins": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/builtins/-/builtins-5.0.1.tgz", @@ -627,11 +755,27 @@ "url": "https://github.com/chalk/chalk?sponsor=1" } }, + "node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" + }, + "node_modules/color": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/color/-/color-4.2.3.tgz", + "integrity": "sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==", + "dependencies": { + "color-convert": "^2.0.1", + "color-string": "^1.9.0" + }, + "engines": { + "node": ">=12.5.0" + } + }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dev": true, "dependencies": { "color-name": "~1.1.4" }, @@ -642,8 +786,16 @@ "node_modules/color-name": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, + "node_modules/color-string": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz", + "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==", + "dependencies": { + "color-name": "^1.0.0", + "simple-swizzle": "^0.2.2" + } }, "node_modules/comment-parser": { "version": "1.3.1", @@ -692,6 +844,28 @@ } } }, + "node_modules/decompress-response": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", + "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", + "dependencies": { + "mimic-response": "^3.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "engines": { + "node": ">=4.0.0" + } + }, "node_modules/deep-is": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", @@ -714,6 +888,14 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/detect-libc": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.2.tgz", + "integrity": "sha512-UX6sGumvvqSaXgdKGUsgZWqcUyIXZ/vZTrlRT/iobiKhGL0zL4d3osHj3uqllWJK+i+sixDS/3COVEOFbupFyw==", + "engines": { + "node": ">=8" + } + }, "node_modules/dexie": { "version": "3.2.3", "resolved": "https://registry.npmjs.org/dexie/-/dexie-3.2.3.tgz", @@ -746,6 +928,14 @@ "node": ">=6.0.0" } }, + "node_modules/end-of-stream": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "dependencies": { + "once": "^1.4.0" + } + }, "node_modules/es-abstract": { "version": "1.21.2", "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.21.2.tgz", @@ -1498,6 +1688,14 @@ "node": ">=0.10.0" } }, + "node_modules/expand-template": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", + "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", + "engines": { + "node": ">=6" + } + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -1510,6 +1708,11 @@ "integrity": "sha512-xJuoT5+L99XlZ8twedaRf6Ax2TgQVxvgZOYoPKqZufmJib0tL2tegPBOZb1pVNgIhlqDlA0eO0c3wBvQcmzx4w==", "dev": true }, + "node_modules/fast-fifo": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", + "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==" + }, "node_modules/fast-glob": { "version": "3.2.12", "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.12.tgz", @@ -1612,6 +1815,11 @@ "node": "^10.12.0 || >=12.0.0" } }, + "node_modules/flatbuffers": { + "version": "1.12.0", + "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-1.12.0.tgz", + "integrity": "sha512-c7CZADjRcl6j0PlvFy0ZqXQ67qSEZfrVPynmnL+2zPc+NtMvrF8Y0QceMo7QqnSPc7+uWjUIAbvCQ5WIKlMVdQ==" + }, "node_modules/flatted": { "version": "3.2.7", "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.2.7.tgz", @@ -1627,6 +1835,11 @@ "is-callable": "^1.1.3" } }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" + }, "node_modules/fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -1696,6 +1909,11 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/github-from-package": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", + "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==" + }, "node_modules/glob": { "version": "7.2.3", "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", @@ -1796,6 +2014,11 @@ "integrity": "sha512-bzh50DW9kTPM00T8y4o8vQg89Di9oLJVLW/KaOGIXJWP/iqCN6WKYkbNOF04vFLJhwcpYUh9ydh/+5vpOqV4YQ==", "dev": true }, + "node_modules/guid-typescript": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/guid-typescript/-/guid-typescript-1.0.9.tgz", + "integrity": "sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==" + }, "node_modules/has": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", @@ -1882,6 +2105,25 @@ "resolved": "https://registry.npmjs.org/idb/-/idb-7.1.1.tgz", "integrity": "sha512-gchesWBzyvGHRO9W8tzUWFDycow5gwjvFKfyV9FF32Y7F50yZMp7mP+T2mJIWFx49zicqyC4uefHM17o6xKIVQ==" }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, "node_modules/ignore": { "version": "5.2.4", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.4.tgz", @@ -1929,8 +2171,12 @@ "node_modules/inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "dev": true + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "node_modules/ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==" }, "node_modules/internal-slot": { "version": "1.0.5", @@ -1960,6 +2206,11 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-arrayish": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", + "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==" + }, "node_modules/is-bigint": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/is-bigint/-/is-bigint-1.0.4.tgz", @@ -2278,11 +2529,15 @@ "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", "dev": true }, + "node_modules/long": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz", + "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==" + }, "node_modules/lru-cache": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "dev": true, "dependencies": { "yallist": "^4.0.0" }, @@ -2312,6 +2567,17 @@ "node": ">=8.6" } }, + "node_modules/mimic-response": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", + "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/minimatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", @@ -2328,17 +2594,26 @@ "version": "1.2.8", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", - "dev": true, "funding": { "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" + }, "node_modules/ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", "dev": true }, + "node_modules/napi-build-utils": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz", + "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==" + }, "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -2351,6 +2626,22 @@ "integrity": "sha512-Tj+HTDSJJKaZnfiuw+iaF9skdPpTo2GtEly5JHnWV/hfv2Qj/9RKsGISQtLh2ox3l5EAGw487hnBee0sIJ6v2g==", "dev": true }, + "node_modules/node-abi": { + "version": "3.51.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.51.0.tgz", + "integrity": "sha512-SQkEP4hmNWjlniS5zdnfIXTk1x7Ome85RDzHlTbBtzE97Gfwz/Ipw4v/Ryk20DWIy3yCNVLVlGKApCnmvYoJbA==", + "dependencies": { + "semver": "^7.3.5" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/node-addon-api": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-6.1.0.tgz", + "integrity": "sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA==" + }, "node_modules/object-inspect": { "version": "1.12.3", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.3.tgz", @@ -2408,11 +2699,50 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "dev": true, "dependencies": { "wrappy": "1" } }, + "node_modules/onnx-proto": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/onnx-proto/-/onnx-proto-4.0.4.tgz", + "integrity": "sha512-aldMOB3HRoo6q/phyB6QRQxSt895HNNw82BNyZ2CMh4bjeKv7g/c+VpAFtJuEMVfYLMbRx61hbuqnKceLeDcDA==", + "dependencies": { + "protobufjs": "^6.8.8" + } + }, + "node_modules/onnxruntime-common": { + "version": "1.14.0", + "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.14.0.tgz", + "integrity": "sha512-3LJpegM2iMNRX2wUmtYfeX/ytfOzNwAWKSq1HbRrKc9+uqG/FsEA0bbKZl1btQeZaXhC26l44NWpNUeXPII7Ew==" + }, + "node_modules/onnxruntime-node": { + "version": "1.14.0", + "resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.14.0.tgz", + "integrity": "sha512-5ba7TWomIV/9b6NH/1x/8QEeowsb+jBEvFzU6z0T4mNsFwdPqXeFUM7uxC6QeSRkEbWu3qEB0VMjrvzN/0S9+w==", + "optional": true, + "os": [ + "win32", + "darwin", + "linux" + ], + "dependencies": { + "onnxruntime-common": "~1.14.0" + } + }, + "node_modules/onnxruntime-web": { + "version": "1.14.0", + "resolved": "https://registry.npmjs.org/onnxruntime-web/-/onnxruntime-web-1.14.0.tgz", + "integrity": "sha512-Kcqf43UMfW8mCydVGcX9OMXI2VN17c0p6XvR7IPSZzBf/6lteBzXHvcEVWDPmCKuGombl997HgLqj91F11DzXw==", + "dependencies": { + "flatbuffers": "^1.12.0", + "guid-typescript": "^1.0.9", + "long": "^4.0.0", + "onnx-proto": "^4.0.4", + "onnxruntime-common": "~1.14.0", + "platform": "^1.3.6" + } + }, "node_modules/optionator": { "version": "0.9.1", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.1.tgz", @@ -2526,6 +2856,62 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/platform": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/platform/-/platform-1.3.6.tgz", + "integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==" + }, + "node_modules/prebuild-install": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.1.tgz", + "integrity": "sha512-jAXscXWMcCK8GgCoHOfIr0ODh5ai8mj63L2nWrjuAgXE6tDyYGnx4/8o/rCgU+B4JSyZBKbeZqzhtwtC3ovxjw==", + "dependencies": { + "detect-libc": "^2.0.0", + "expand-template": "^2.0.3", + "github-from-package": "0.0.0", + "minimist": "^1.2.3", + "mkdirp-classic": "^0.5.3", + "napi-build-utils": "^1.0.1", + "node-abi": "^3.3.0", + "pump": "^3.0.0", + "rc": "^1.2.7", + "simple-get": "^4.0.0", + "tar-fs": "^2.0.0", + "tunnel-agent": "^0.6.0" + }, + "bin": { + "prebuild-install": "bin.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/prebuild-install/node_modules/tar-fs": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz", + "integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==", + "dependencies": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "node_modules/prebuild-install/node_modules/tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "dependencies": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", @@ -2563,6 +2949,40 @@ "node": ">=6.0.0" } }, + "node_modules/protobufjs": { + "version": "6.11.4", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.11.4.tgz", + "integrity": "sha512-5kQWPaJHi1WoCpjTGszzQ32PG2F4+wRY6BmAT4Vfw56Q2FZ4YZzK20xUYQH4YkfehY1e6QSICrJquM6xXZNcrw==", + "hasInstallScript": true, + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/long": "^4.0.1", + "@types/node": ">=13.7.0", + "long": "^4.0.0" + }, + "bin": { + "pbjs": "bin/pbjs", + "pbts": "bin/pbts" + } + }, + "node_modules/pump": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", + "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, "node_modules/punycode": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz", @@ -2592,6 +3012,46 @@ } ] }, + "node_modules/queue-tick": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/queue-tick/-/queue-tick-1.0.1.tgz", + "integrity": "sha512-kJt5qhMxoszgU/62PLP1CJytzd2NKetjSRnyuj31fDd3Rlcz3fzlFdFLD1SItunPwyqEOkca6GbV612BWfaBag==" + }, + "node_modules/rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "dependencies": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "bin": { + "rc": "cli.js" + } + }, + "node_modules/rc/node_modules/strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/regexp.prototype.flags": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.0.tgz", @@ -2705,6 +3165,25 @@ "queue-microtask": "^1.2.2" } }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, "node_modules/safe-regex-test": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/safe-regex-test/-/safe-regex-test-1.0.0.tgz", @@ -2720,10 +3199,9 @@ } }, "node_modules/semver": { - "version": "7.5.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.1.tgz", - "integrity": "sha512-Wvss5ivl8TMRZXXESstBA4uR5iXgEN/VC5/sOcuXdVLzcdkz4HWetIoRfG5gb5X+ij/G9rw9YoGn3QoQ8OCSpw==", - "dev": true, + "version": "7.5.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", + "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", "dependencies": { "lru-cache": "^6.0.0" }, @@ -2734,6 +3212,28 @@ "node": ">=10" } }, + "node_modules/sharp": { + "version": "0.32.6", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.32.6.tgz", + "integrity": "sha512-KyLTWwgcR9Oe4d9HwCwNM2l7+J0dUQwn/yf7S0EnTtb0eVS4RxO0eUSvxPtzT4F3SY+C4K6fqdv/DO27sJ/v/w==", + "hasInstallScript": true, + "dependencies": { + "color": "^4.2.3", + "detect-libc": "^2.0.2", + "node-addon-api": "^6.1.0", + "prebuild-install": "^7.1.1", + "semver": "^7.5.4", + "simple-get": "^4.0.1", + "tar-fs": "^3.0.4", + "tunnel-agent": "^0.6.0" + }, + "engines": { + "node": ">=14.15.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, "node_modules/shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", @@ -2769,6 +3269,57 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/simple-concat": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", + "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/simple-get": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", + "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "decompress-response": "^6.0.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } + }, + "node_modules/simple-swizzle": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", + "integrity": "sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==", + "dependencies": { + "is-arrayish": "^0.3.1" + } + }, "node_modules/slash": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", @@ -2803,6 +3354,23 @@ "dev": true, "peer": true }, + "node_modules/streamx": { + "version": "2.15.4", + "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.15.4.tgz", + "integrity": "sha512-uSXKl88bibiUCQ1eMpItRljCzDENcDx18rsfDmV79r0e/ThfrAwxG4Y2FarQZ2G4/21xcOKmFFd1Hue+ZIDwHw==", + "dependencies": { + "fast-fifo": "^1.1.0", + "queue-tick": "^1.0.1" + } + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, "node_modules/string.prototype.trim": { "version": "1.2.7", "resolved": "https://registry.npmjs.org/string.prototype.trim/-/string.prototype.trim-1.2.7.tgz", @@ -2905,6 +3473,26 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/tar-fs": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.0.4.tgz", + "integrity": "sha512-5AFQU8b9qLfZCX9zp2duONhPmZv0hGYiBPJsyUdqMjzq/mqVpy/rEUSeHk1+YitmxugaptgBh5oDGU3VsAJq4w==", + "dependencies": { + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^3.1.5" + } + }, + "node_modules/tar-stream": { + "version": "3.1.6", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.1.6.tgz", + "integrity": "sha512-B/UyjYwPpMBv+PaFSWAmtYjwdrlEaZQEhMIBFNC5oEG8lpiW8XjcSdmEaClj28ArfKScKHs2nshz3k2le6crsg==", + "dependencies": { + "b4a": "^1.6.4", + "fast-fifo": "^1.2.0", + "streamx": "^2.15.0" + } + }, "node_modules/text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", @@ -2956,6 +3544,17 @@ "typescript": ">=2.8.0 || >= 3.2.0-dev || >= 3.3.0-dev || >= 3.4.0-dev || >= 3.5.0-dev || >= 3.6.0-dev || >= 3.6.0-beta || >= 3.7.0-dev || >= 3.7.0-beta" } }, + "node_modules/tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", + "dependencies": { + "safe-buffer": "^5.0.1" + }, + "engines": { + "node": "*" + } + }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", @@ -3031,6 +3630,11 @@ "punycode": "^2.1.0" } }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -3094,14 +3698,12 @@ "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "dev": true + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" }, "node_modules/yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "dev": true + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" }, "node_modules/yocto-queue": { "version": "0.1.0", @@ -3216,6 +3818,60 @@ "fastq": "^1.6.0" } }, + "@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==" + }, + "@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==" + }, + "@protobufjs/codegen": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", + "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==" + }, + "@protobufjs/eventemitter": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", + "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==" + }, + "@protobufjs/fetch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", + "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", + "requires": { + "@protobufjs/aspromise": "^1.1.1", + "@protobufjs/inquire": "^1.1.0" + } + }, + "@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==" + }, + "@protobufjs/inquire": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", + "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==" + }, + "@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==" + }, + "@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==" + }, + "@protobufjs/utf8": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", + "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" + }, "@types/json-schema": { "version": "7.0.11", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.11.tgz", @@ -3234,11 +3890,15 @@ "integrity": "sha512-r22s9tAS7imvBt2lyHC9B8AGwWnXaYb1tY09oyLkXDs4vArpYJzw09nj8MLx5VfciBPGIb+ZwG0ssYnEPJxn/g==", "dev": true }, + "@types/long": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.2.tgz", + "integrity": "sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==" + }, "@types/node": { "version": "18.16.13", "resolved": "https://registry.npmjs.org/@types/node/-/node-18.16.13.tgz", - "integrity": "sha512-uZRomboV1vBL61EBXneL4j9/hEn+1Yqa4LQdpGrKmXFyJmVfWc9JV9+yb2AlnOnuaDnb2PDO3hC6/LKmzJxP1A==", - "dev": true + "integrity": "sha512-uZRomboV1vBL61EBXneL4j9/hEn+1Yqa4LQdpGrKmXFyJmVfWc9JV9+yb2AlnOnuaDnb2PDO3hC6/LKmzJxP1A==" }, "@types/semver": { "version": "7.5.0", @@ -3345,6 +4005,16 @@ "eslint-visitor-keys": "^3.3.0" } }, + "@xenova/transformers": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/@xenova/transformers/-/transformers-2.8.0.tgz", + "integrity": "sha512-+6/cC/iJd7A+B+C3OPuiIw3dhisdUqBRD/f8CAoo2wYfHofF9zn46pH/dTLAOPoI2px7L9+bvJoXSKO18TBj2Q==", + "requires": { + "onnxruntime-node": "1.14.0", + "onnxruntime-web": "1.14.0", + "sharp": "^0.32.0" + } + }, "acorn": { "version": "8.8.2", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.2.tgz", @@ -3457,12 +4127,32 @@ "integrity": "sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw==", "dev": true }, + "b4a": { + "version": "1.6.4", + "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.6.4.tgz", + "integrity": "sha512-fpWrvyVHEKyeEvbKZTVOeZF3VSKKWtJxFIxX/jaVPf+cLbGUSitjb49pHLqPV2BUNNZ0LcoeEGfE/YCpyDYHIw==" + }, "balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", "dev": true }, + "base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" + }, + "bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "requires": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -3482,6 +4172,15 @@ "fill-range": "^7.0.1" } }, + "buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "requires": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, "builtins": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/builtins/-/builtins-5.0.1.tgz", @@ -3518,11 +4217,24 @@ "supports-color": "^7.1.0" } }, + "chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" + }, + "color": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/color/-/color-4.2.3.tgz", + "integrity": "sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==", + "requires": { + "color-convert": "^2.0.1", + "color-string": "^1.9.0" + } + }, "color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dev": true, "requires": { "color-name": "~1.1.4" } @@ -3530,8 +4242,16 @@ "color-name": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, + "color-string": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz", + "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==", + "requires": { + "color-name": "^1.0.0", + "simple-swizzle": "^0.2.2" + } }, "comment-parser": { "version": "1.3.1", @@ -3566,6 +4286,19 @@ "ms": "2.1.2" } }, + "decompress-response": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", + "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", + "requires": { + "mimic-response": "^3.1.0" + } + }, + "deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==" + }, "deep-is": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", @@ -3582,6 +4315,11 @@ "object-keys": "^1.1.1" } }, + "detect-libc": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.2.tgz", + "integrity": "sha512-UX6sGumvvqSaXgdKGUsgZWqcUyIXZ/vZTrlRT/iobiKhGL0zL4d3osHj3uqllWJK+i+sixDS/3COVEOFbupFyw==" + }, "dexie": { "version": "3.2.3", "resolved": "https://registry.npmjs.org/dexie/-/dexie-3.2.3.tgz", @@ -3605,6 +4343,14 @@ "esutils": "^2.0.2" } }, + "end-of-stream": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "requires": { + "once": "^1.4.0" + } + }, "es-abstract": { "version": "1.21.2", "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.21.2.tgz", @@ -4141,6 +4887,11 @@ "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", "dev": true }, + "expand-template": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", + "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==" + }, "fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -4153,6 +4904,11 @@ "integrity": "sha512-xJuoT5+L99XlZ8twedaRf6Ax2TgQVxvgZOYoPKqZufmJib0tL2tegPBOZb1pVNgIhlqDlA0eO0c3wBvQcmzx4w==", "dev": true }, + "fast-fifo": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", + "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==" + }, "fast-glob": { "version": "3.2.12", "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.12.tgz", @@ -4236,6 +4992,11 @@ "rimraf": "^3.0.2" } }, + "flatbuffers": { + "version": "1.12.0", + "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-1.12.0.tgz", + "integrity": "sha512-c7CZADjRcl6j0PlvFy0ZqXQ67qSEZfrVPynmnL+2zPc+NtMvrF8Y0QceMo7QqnSPc7+uWjUIAbvCQ5WIKlMVdQ==" + }, "flatted": { "version": "3.2.7", "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.2.7.tgz", @@ -4251,6 +5012,11 @@ "is-callable": "^1.1.3" } }, + "fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" + }, "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -4302,6 +5068,11 @@ "get-intrinsic": "^1.1.1" } }, + "github-from-package": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", + "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==" + }, "glob": { "version": "7.2.3", "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", @@ -4372,6 +5143,11 @@ "integrity": "sha512-bzh50DW9kTPM00T8y4o8vQg89Di9oLJVLW/KaOGIXJWP/iqCN6WKYkbNOF04vFLJhwcpYUh9ydh/+5vpOqV4YQ==", "dev": true }, + "guid-typescript": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/guid-typescript/-/guid-typescript-1.0.9.tgz", + "integrity": "sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==" + }, "has": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", @@ -4428,6 +5204,11 @@ "resolved": "https://registry.npmjs.org/idb/-/idb-7.1.1.tgz", "integrity": "sha512-gchesWBzyvGHRO9W8tzUWFDycow5gwjvFKfyV9FF32Y7F50yZMp7mP+T2mJIWFx49zicqyC4uefHM17o6xKIVQ==" }, + "ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==" + }, "ignore": { "version": "5.2.4", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.4.tgz", @@ -4463,8 +5244,12 @@ "inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "dev": true + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==" }, "internal-slot": { "version": "1.0.5", @@ -4488,6 +5273,11 @@ "is-typed-array": "^1.1.10" } }, + "is-arrayish": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", + "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==" + }, "is-bigint": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/is-bigint/-/is-bigint-1.0.4.tgz", @@ -4706,11 +5496,15 @@ "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", "dev": true }, + "long": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz", + "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==" + }, "lru-cache": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "dev": true, "requires": { "yallist": "^4.0.0" } @@ -4731,6 +5525,11 @@ "picomatch": "^2.3.1" } }, + "mimic-response": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", + "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==" + }, "minimatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", @@ -4743,8 +5542,12 @@ "minimist": { "version": "1.2.8", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", - "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", - "dev": true + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==" + }, + "mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" }, "ms": { "version": "2.1.2", @@ -4752,6 +5555,11 @@ "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", "dev": true }, + "napi-build-utils": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz", + "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==" + }, "natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -4764,6 +5572,19 @@ "integrity": "sha512-Tj+HTDSJJKaZnfiuw+iaF9skdPpTo2GtEly5JHnWV/hfv2Qj/9RKsGISQtLh2ox3l5EAGw487hnBee0sIJ6v2g==", "dev": true }, + "node-abi": { + "version": "3.51.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.51.0.tgz", + "integrity": "sha512-SQkEP4hmNWjlniS5zdnfIXTk1x7Ome85RDzHlTbBtzE97Gfwz/Ipw4v/Ryk20DWIy3yCNVLVlGKApCnmvYoJbA==", + "requires": { + "semver": "^7.3.5" + } + }, + "node-addon-api": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-6.1.0.tgz", + "integrity": "sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA==" + }, "object-inspect": { "version": "1.12.3", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.3.tgz", @@ -4803,11 +5624,45 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "dev": true, "requires": { "wrappy": "1" } }, + "onnx-proto": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/onnx-proto/-/onnx-proto-4.0.4.tgz", + "integrity": "sha512-aldMOB3HRoo6q/phyB6QRQxSt895HNNw82BNyZ2CMh4bjeKv7g/c+VpAFtJuEMVfYLMbRx61hbuqnKceLeDcDA==", + "requires": { + "protobufjs": "^6.8.8" + } + }, + "onnxruntime-common": { + "version": "1.14.0", + "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.14.0.tgz", + "integrity": "sha512-3LJpegM2iMNRX2wUmtYfeX/ytfOzNwAWKSq1HbRrKc9+uqG/FsEA0bbKZl1btQeZaXhC26l44NWpNUeXPII7Ew==" + }, + "onnxruntime-node": { + "version": "1.14.0", + "resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.14.0.tgz", + "integrity": "sha512-5ba7TWomIV/9b6NH/1x/8QEeowsb+jBEvFzU6z0T4mNsFwdPqXeFUM7uxC6QeSRkEbWu3qEB0VMjrvzN/0S9+w==", + "optional": true, + "requires": { + "onnxruntime-common": "~1.14.0" + } + }, + "onnxruntime-web": { + "version": "1.14.0", + "resolved": "https://registry.npmjs.org/onnxruntime-web/-/onnxruntime-web-1.14.0.tgz", + "integrity": "sha512-Kcqf43UMfW8mCydVGcX9OMXI2VN17c0p6XvR7IPSZzBf/6lteBzXHvcEVWDPmCKuGombl997HgLqj91F11DzXw==", + "requires": { + "flatbuffers": "^1.12.0", + "guid-typescript": "^1.0.9", + "long": "^4.0.0", + "onnx-proto": "^4.0.4", + "onnxruntime-common": "~1.14.0", + "platform": "^1.3.6" + } + }, "optionator": { "version": "0.9.1", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.1.tgz", @@ -4885,6 +5740,55 @@ "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", "dev": true }, + "platform": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/platform/-/platform-1.3.6.tgz", + "integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==" + }, + "prebuild-install": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.1.tgz", + "integrity": "sha512-jAXscXWMcCK8GgCoHOfIr0ODh5ai8mj63L2nWrjuAgXE6tDyYGnx4/8o/rCgU+B4JSyZBKbeZqzhtwtC3ovxjw==", + "requires": { + "detect-libc": "^2.0.0", + "expand-template": "^2.0.3", + "github-from-package": "0.0.0", + "minimist": "^1.2.3", + "mkdirp-classic": "^0.5.3", + "napi-build-utils": "^1.0.1", + "node-abi": "^3.3.0", + "pump": "^3.0.0", + "rc": "^1.2.7", + "simple-get": "^4.0.0", + "tar-fs": "^2.0.0", + "tunnel-agent": "^0.6.0" + }, + "dependencies": { + "tar-fs": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz", + "integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==", + "requires": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "requires": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + } + } + } + }, "prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", @@ -4907,6 +5811,35 @@ "fast-diff": "^1.1.2" } }, + "protobufjs": { + "version": "6.11.4", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.11.4.tgz", + "integrity": "sha512-5kQWPaJHi1WoCpjTGszzQ32PG2F4+wRY6BmAT4Vfw56Q2FZ4YZzK20xUYQH4YkfehY1e6QSICrJquM6xXZNcrw==", + "requires": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/long": "^4.0.1", + "@types/node": ">=13.7.0", + "long": "^4.0.0" + } + }, + "pump": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", + "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "requires": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, "punycode": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz", @@ -4919,6 +5852,39 @@ "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", "dev": true }, + "queue-tick": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/queue-tick/-/queue-tick-1.0.1.tgz", + "integrity": "sha512-kJt5qhMxoszgU/62PLP1CJytzd2NKetjSRnyuj31fDd3Rlcz3fzlFdFLD1SItunPwyqEOkca6GbV612BWfaBag==" + }, + "rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "requires": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "dependencies": { + "strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==" + } + } + }, + "readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + }, "regexp.prototype.flags": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.0.tgz", @@ -4984,6 +5950,11 @@ "queue-microtask": "^1.2.2" } }, + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + }, "safe-regex-test": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/safe-regex-test/-/safe-regex-test-1.0.0.tgz", @@ -4996,14 +5967,28 @@ } }, "semver": { - "version": "7.5.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.1.tgz", - "integrity": "sha512-Wvss5ivl8TMRZXXESstBA4uR5iXgEN/VC5/sOcuXdVLzcdkz4HWetIoRfG5gb5X+ij/G9rw9YoGn3QoQ8OCSpw==", - "dev": true, + "version": "7.5.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", + "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", "requires": { "lru-cache": "^6.0.0" } }, + "sharp": { + "version": "0.32.6", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.32.6.tgz", + "integrity": "sha512-KyLTWwgcR9Oe4d9HwCwNM2l7+J0dUQwn/yf7S0EnTtb0eVS4RxO0eUSvxPtzT4F3SY+C4K6fqdv/DO27sJ/v/w==", + "requires": { + "color": "^4.2.3", + "detect-libc": "^2.0.2", + "node-addon-api": "^6.1.0", + "prebuild-install": "^7.1.1", + "semver": "^7.5.4", + "simple-get": "^4.0.1", + "tar-fs": "^3.0.4", + "tunnel-agent": "^0.6.0" + } + }, "shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", @@ -5030,6 +6015,29 @@ "object-inspect": "^1.9.0" } }, + "simple-concat": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", + "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==" + }, + "simple-get": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", + "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==", + "requires": { + "decompress-response": "^6.0.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } + }, + "simple-swizzle": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", + "integrity": "sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==", + "requires": { + "is-arrayish": "^0.3.1" + } + }, "slash": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", @@ -5061,6 +6069,23 @@ "dev": true, "peer": true }, + "streamx": { + "version": "2.15.4", + "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.15.4.tgz", + "integrity": "sha512-uSXKl88bibiUCQ1eMpItRljCzDENcDx18rsfDmV79r0e/ThfrAwxG4Y2FarQZ2G4/21xcOKmFFd1Hue+ZIDwHw==", + "requires": { + "fast-fifo": "^1.1.0", + "queue-tick": "^1.0.1" + } + }, + "string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "requires": { + "safe-buffer": "~5.2.0" + } + }, "string.prototype.trim": { "version": "1.2.7", "resolved": "https://registry.npmjs.org/string.prototype.trim/-/string.prototype.trim-1.2.7.tgz", @@ -5130,6 +6155,26 @@ "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", "dev": true }, + "tar-fs": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.0.4.tgz", + "integrity": "sha512-5AFQU8b9qLfZCX9zp2duONhPmZv0hGYiBPJsyUdqMjzq/mqVpy/rEUSeHk1+YitmxugaptgBh5oDGU3VsAJq4w==", + "requires": { + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^3.1.5" + } + }, + "tar-stream": { + "version": "3.1.6", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.1.6.tgz", + "integrity": "sha512-B/UyjYwPpMBv+PaFSWAmtYjwdrlEaZQEhMIBFNC5oEG8lpiW8XjcSdmEaClj28ArfKScKHs2nshz3k2le6crsg==", + "requires": { + "b4a": "^1.6.4", + "fast-fifo": "^1.2.0", + "streamx": "^2.15.0" + } + }, "text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", @@ -5172,6 +6217,14 @@ "tslib": "^1.8.1" } }, + "tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", + "requires": { + "safe-buffer": "^5.0.1" + } + }, "type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", @@ -5225,6 +6278,11 @@ "punycode": "^2.1.0" } }, + "util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" + }, "which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -5270,14 +6328,12 @@ "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "dev": true + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" }, "yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "dev": true + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" }, "yocto-queue": { "version": "0.1.0", diff --git a/package.json b/package.json index 1f63bca..74a00b7 100644 --- a/package.json +++ b/package.json @@ -1,12 +1,12 @@ { - "name": "vector-storage", - "version": "1.0.54", - "description": "A lightweight and efficient vector database for storing and searching text embeddings in the browser's local storage. The package uses OpenAI's API to generate embeddings for text documents and provides functionality for similarity search, filtering, and automatic management of local storage space. It is designed for use cases where client-side storage and retrieval of text embeddings are needed.", + "name": "react-embeddings-search", + "version": "0.01", + "description": "A lightweight and efficient vector database for storing and searching text embeddings in the browser's local storage. The package uses OpenAI's API or Transformer.js models to generate embeddings for text documents and provides functionality for similarity search, filtering, and automatic management of local storage space. It is designed for use cases where client-side storage and retrieval of text embeddings are needed.", "main": "dist/index.js", "types": "dist/index.d.ts", "repository": { "type": "git", - "url": "https://github.com/nitaiaharoni1/vector-storage.git" + "url": "https://github.com/aivanzhang/ai-search" }, "keywords": [ "vector", @@ -23,6 +23,7 @@ "search", "similarity", "OpenAI", + "transformers", "API" ], "scripts": { @@ -31,9 +32,10 @@ "package": "npm run lint && npm run build && npm run bump && npm publish", "lint": "eslint --ext .ts src" }, - "author": "Nitai Aharoni", + "author": "Ivan Zhang", "license": "MIT", "dependencies": { + "@xenova/transformers": "^2.8.0", "dexie": "^3.2.3", "idb": "^7.1.1" }, diff --git a/src/VectorStorage.ts b/src/VectorStorage.ts index 69c9cae..cea3848 100644 --- a/src/VectorStorage.ts +++ b/src/VectorStorage.ts @@ -1,28 +1,39 @@ -import { ICreateEmbeddingResponse } from './types/ICreateEmbeddingResponse'; -import { IDBPDatabase, openDB } from 'idb'; -import { IVSDocument, IVSSimilaritySearchItem } from './types/IVSDocument'; -import { IVSOptions } from './types/IVSOptions'; -import { IVSSimilaritySearchParams } from './types/IVSSimilaritySearchParams'; -import { constants } from './common/constants'; -import { filterDocuments, getObjectSizeInMB } from './common/helpers'; +import { ICreateEmbeddingResponse } from "./types/ICreateEmbeddingResponse"; +import { IDBPDatabase, openDB } from "idb"; +import { Pipeline, pipeline, env } from "@xenova/transformers"; +import { IVSDocument, IVSSimilaritySearchItem } from "./types/IVSDocument"; +import { IVSOptions } from "./types/IVSOptions"; +import { IVSSimilaritySearchParams } from "./types/IVSSimilaritySearchParams"; +import { constants } from "./common/constants"; +import { filterDocuments, getObjectSizeInMB } from "./common/helpers"; export class VectorStorage { private db!: IDBPDatabase; private documents: Array> = []; private readonly maxSizeInMB: number; private readonly debounceTime: number; - private readonly openaiModel: string; + private readonly openaiModel?: string; private readonly openaiApiKey?: string; + private readonly transfomersModel?: string; + private transformersPipeline?: Pipeline; private readonly embedTextsFn: (texts: string[]) => Promise; constructor(options: IVSOptions = {}) { this.maxSizeInMB = options.maxSizeInMB ?? constants.DEFAULT_MAX_SIZE_IN_MB; this.debounceTime = options.debounceTime ?? constants.DEFAULT_DEBOUNCE_TIME; - this.openaiModel = options.openaiModel ?? constants.DEFAULT_OPENAI_MODEL; + this.openaiModel = options.openaiModel; this.embedTextsFn = options.embedTextsFn ?? this.embedTexts; // Use the custom function if provided, else use the default one this.openaiApiKey = options.openAIApiKey; - if (!this.openaiApiKey && !options.embedTextsFn) { - console.error('VectorStorage: pass as an option either an OpenAI API key or a custom embedTextsFn function.'); + this.transfomersModel = options.transfomersModel; + + if ( + !this.openaiApiKey && + !options.embedTextsFn && + !options.transfomersModel + ) { + console.error( + "VectorStorage: pass as an option either an OpenAI API key, a custom embedTextsFn function, or a Transformer.js model." + ); } else { this.loadFromIndexDbStorage(); } @@ -41,9 +52,12 @@ export class VectorStorage { return docs[0]; } - public async addTexts(texts: string[], metadatas: T[]): Promise>> { + public async addTexts( + texts: string[], + metadatas: T[] + ): Promise>> { if (texts.length !== metadatas.length) { - throw new Error('The lengths of texts and metadata arrays must match.'); + throw new Error("The lengths of texts and metadata arrays must match."); } const docs: Array> = texts.map((text, index) => ({ metadata: metadatas[index], @@ -63,9 +77,16 @@ export class VectorStorage { const queryEmbedding = await this.embedText(query); const queryMagnitude = await this.calculateMagnitude(queryEmbedding); const filteredDocuments = filterDocuments(this.documents, filterOptions); - const scoresPairs: Array<[IVSDocument, number]> = this.calculateSimilarityScores(filteredDocuments, queryEmbedding, queryMagnitude); + const scoresPairs: Array<[IVSDocument, number]> = + this.calculateSimilarityScores( + filteredDocuments, + queryEmbedding, + queryMagnitude + ); const sortedPairs = scoresPairs.sort((a, b) => b[1] - a[1]); - const results = sortedPairs.slice(0, k).map((pair) => ({ ...pair[0], score: pair[1] })); + const results = sortedPairs + .slice(0, k) + .map((pair) => ({ ...pair[0], score: pair[1] })); this.updateHitCounters(results); if (results.length > 0) { this.removeDocsLRU(); @@ -84,30 +105,41 @@ export class VectorStorage { } private async initDB(): Promise> { - return await openDB('VectorStorageDatabase', undefined, { + return await openDB("VectorStorageDatabase", undefined, { upgrade(db) { - const documentStore = db.createObjectStore('documents', { + const documentStore = db.createObjectStore("documents", { autoIncrement: true, - keyPath: 'id', + keyPath: "id", }); - documentStore.createIndex('text', 'text', { unique: true }); - documentStore.createIndex('metadata', 'metadata'); - documentStore.createIndex('timestamp', 'timestamp'); - documentStore.createIndex('vector', 'vector'); - documentStore.createIndex('vectorMag', 'vectorMag'); - documentStore.createIndex('hits', 'hits'); + documentStore.createIndex("text", "text", { unique: true }); + documentStore.createIndex("metadata", "metadata"); + documentStore.createIndex("timestamp", "timestamp"); + documentStore.createIndex("vector", "vector"); + documentStore.createIndex("vectorMag", "vectorMag"); + documentStore.createIndex("hits", "hits"); }, }); } - private async addDocuments(documents: Array>): Promise>> { + private async loadTransformersModel(): Promise { + env.allowLocalModels = false; + return await pipeline("feature-extraction", this.transfomersModel!); + } + + private async addDocuments( + documents: Array> + ): Promise>> { // filter out already existing documents - const newDocuments = documents.filter((doc) => !this.documents.some((d) => d.text === doc.text)); + const newDocuments = documents.filter( + (doc) => !this.documents.some((d) => d.text === doc.text) + ); // If there are no new documents, return an empty array if (newDocuments.length === 0) { return []; } - const newVectors = await this.embedTextsFn(newDocuments.map((doc) => doc.text)); + const newVectors = await this.embedTextsFn( + newDocuments.map((doc) => doc.text) + ); // Assign vectors and precompute vector magnitudes for new documents newDocuments.forEach((doc, index) => { doc.vector = newVectors[index]; @@ -122,6 +154,30 @@ export class VectorStorage { } private async embedTexts(texts: string[]): Promise { + if (this.transfomersModel) { + return await this.embedTextsTransformers(texts); + } else { + return await this.embedTextsOpenAI(texts); + } + } + + private async embedTextsTransformers(texts: string[]): Promise { + if (!this.transformersPipeline) { + this.transformersPipeline = await this.loadTransformersModel(); + } + const embeddings: number[][] = []; + for (const text of texts) { + // eslint-disable-next-line no-await-in-loop + const embedding = await this.transformersPipeline(text, { + normalize: true, + pooling: "mean", + }); + embeddings.push(embedding.data); + } + return embeddings; + } + + private async embedTextsOpenAI(texts: string[]): Promise { const response = await fetch(constants.OPENAI_API_URL, { body: JSON.stringify({ input: texts, @@ -129,9 +185,9 @@ export class VectorStorage { }), headers: { Authorization: `Bearer ${this.openaiApiKey}`, - 'Content-Type': 'application/json', + "Content-Type": "application/json", }, - method: 'POST', + method: "POST", }); if (!response.ok) { @@ -147,14 +203,27 @@ export class VectorStorage { } private calculateMagnitude(embedding: number[]): number { - const queryMagnitude = Math.sqrt(embedding.reduce((sum, val) => sum + val * val, 0)); + const queryMagnitude = Math.sqrt( + embedding.reduce((sum, val) => sum + val * val, 0) + ); return queryMagnitude; } - private calculateSimilarityScores(filteredDocuments: Array>, queryVector: number[], queryMagnitude: number): Array<[IVSDocument, number]> { + private calculateSimilarityScores( + filteredDocuments: Array>, + queryVector: number[], + queryMagnitude: number + ): Array<[IVSDocument, number]> { return filteredDocuments.map((doc) => { - const dotProduct = doc.vector!.reduce((sum, val, i) => sum + val * queryVector[i], 0); - let score = getCosineSimilarityScore(dotProduct, doc.vectorMag!, queryMagnitude); + const dotProduct = doc.vector!.reduce( + (sum, val, i) => sum + val * queryVector[i], + 0 + ); + let score = getCosineSimilarityScore( + dotProduct, + doc.vectorMag!, + queryMagnitude + ); score = normalizeScore(score); // Normalize the score return [doc, score]; }); @@ -170,7 +239,7 @@ export class VectorStorage { if (!this.db) { this.db = await this.initDB(); } - this.documents = await this.db.getAll('documents'); + this.documents = await this.db.getAll("documents"); this.removeDocsLRU(); } @@ -179,22 +248,24 @@ export class VectorStorage { this.db = await this.initDB(); } try { - const tx = this.db.transaction('documents', 'readwrite'); - await tx.objectStore('documents').clear(); + const tx = this.db.transaction("documents", "readwrite"); + await tx.objectStore("documents").clear(); for (const doc of this.documents) { // eslint-disable-next-line no-await-in-loop - await tx.objectStore('documents').put(doc); + await tx.objectStore("documents").put(doc); } await tx.done; } catch (error: any) { - console.error('Failed to save to IndexedDB:', error.message); + console.error("Failed to save to IndexedDB:", error.message); } } private removeDocsLRU(): void { if (getObjectSizeInMB(this.documents) > this.maxSizeInMB) { // Sort documents by hit counter (ascending) and then by timestamp (ascending) - this.documents.sort((a, b) => (a.hits ?? 0) - (b.hits ?? 0) || a.timestamp - b.timestamp); + this.documents.sort( + (a, b) => (a.hits ?? 0) - (b.hits ?? 0) || a.timestamp - b.timestamp + ); // Remove documents until the size is below the limit while (getObjectSizeInMB(this.documents) > this.maxSizeInMB) { @@ -208,7 +279,11 @@ function calcVectorMagnitude(doc: IVSDocument): number { return Math.sqrt(doc.vector!.reduce((sum, val) => sum + val * val, 0)); } -function getCosineSimilarityScore(dotProduct: number, magnitudeA: number, magnitudeB: number): number { +function getCosineSimilarityScore( + dotProduct: number, + magnitudeA: number, + magnitudeB: number +): number { return dotProduct / (magnitudeA * magnitudeB); } diff --git a/src/common/constants.ts b/src/common/constants.ts index f723275..b43e5f9 100644 --- a/src/common/constants.ts +++ b/src/common/constants.ts @@ -2,6 +2,5 @@ export const constants = { DEFAULT_DEBOUNCE_TIME: 0, DEFAULT_MAX_SIZE_IN_MB: 2048, - DEFAULT_OPENAI_MODEL: 'text-embedding-ada-002', - OPENAI_API_URL: 'https://api.openai.com/v1/embeddings', + OPENAI_API_URL: "https://api.openai.com/v1/embeddings", }; diff --git a/src/types/IVSOptions.ts b/src/types/IVSOptions.ts index 1f4222b..7b01b69 100644 --- a/src/types/IVSOptions.ts +++ b/src/types/IVSOptions.ts @@ -2,6 +2,7 @@ export interface IVSOptions { openAIApiKey?: string; // The OpenAI API key used for generating embeddings. maxSizeInMB?: number; // The maximum size of the storage in megabytes. Defaults to 4.8. Cannot exceed 5. debounceTime?: number; // The debounce time in milliseconds for saving to local storage. Defaults to 0. - openaiModel?: string; // The OpenAI model used for generating embeddings. Defaults to 'text-embedding-ada-002'. + openaiModel?: string; // The OpenAI model used for generating embeddings. + transfomersModel?: string; // The HuggingFace model used for generating embeddings. embedTextsFn?: (texts: string[]) => Promise; // Option for custom embedding function } From 0703b2b04779454a25019d6101d852a5319a5af0 Mon Sep 17 00:00:00 2001 From: Ivan Zhang Date: Fri, 10 Nov 2023 16:37:49 -0800 Subject: [PATCH 02/20] update for eslint --- src/VectorStorage.ts | 126 ++++++++++++++-------------------------- src/common/constants.ts | 2 +- 2 files changed, 43 insertions(+), 85 deletions(-) diff --git a/src/VectorStorage.ts b/src/VectorStorage.ts index cea3848..6f208fa 100644 --- a/src/VectorStorage.ts +++ b/src/VectorStorage.ts @@ -1,11 +1,11 @@ -import { ICreateEmbeddingResponse } from "./types/ICreateEmbeddingResponse"; -import { IDBPDatabase, openDB } from "idb"; -import { Pipeline, pipeline, env } from "@xenova/transformers"; -import { IVSDocument, IVSSimilaritySearchItem } from "./types/IVSDocument"; -import { IVSOptions } from "./types/IVSOptions"; -import { IVSSimilaritySearchParams } from "./types/IVSSimilaritySearchParams"; -import { constants } from "./common/constants"; -import { filterDocuments, getObjectSizeInMB } from "./common/helpers"; +import { ICreateEmbeddingResponse } from './types/ICreateEmbeddingResponse'; +import { IDBPDatabase, openDB } from 'idb'; +import { IVSDocument, IVSSimilaritySearchItem } from './types/IVSDocument'; +import { IVSOptions } from './types/IVSOptions'; +import { IVSSimilaritySearchParams } from './types/IVSSimilaritySearchParams'; +import { Pipeline, env, pipeline } from '@xenova/transformers'; +import { constants } from './common/constants'; +import { filterDocuments, getObjectSizeInMB } from './common/helpers'; export class VectorStorage { private db!: IDBPDatabase; @@ -26,14 +26,8 @@ export class VectorStorage { this.openaiApiKey = options.openAIApiKey; this.transfomersModel = options.transfomersModel; - if ( - !this.openaiApiKey && - !options.embedTextsFn && - !options.transfomersModel - ) { - console.error( - "VectorStorage: pass as an option either an OpenAI API key, a custom embedTextsFn function, or a Transformer.js model." - ); + if (!this.openaiApiKey && !options.embedTextsFn && !options.transfomersModel) { + console.error('VectorStorage: pass as an option either an OpenAI API key, a custom embedTextsFn function, or a Transformer.js model.'); } else { this.loadFromIndexDbStorage(); } @@ -52,12 +46,9 @@ export class VectorStorage { return docs[0]; } - public async addTexts( - texts: string[], - metadatas: T[] - ): Promise>> { + public async addTexts(texts: string[], metadatas: T[]): Promise>> { if (texts.length !== metadatas.length) { - throw new Error("The lengths of texts and metadata arrays must match."); + throw new Error('The lengths of texts and metadata arrays must match.'); } const docs: Array> = texts.map((text, index) => ({ metadata: metadatas[index], @@ -77,16 +68,9 @@ export class VectorStorage { const queryEmbedding = await this.embedText(query); const queryMagnitude = await this.calculateMagnitude(queryEmbedding); const filteredDocuments = filterDocuments(this.documents, filterOptions); - const scoresPairs: Array<[IVSDocument, number]> = - this.calculateSimilarityScores( - filteredDocuments, - queryEmbedding, - queryMagnitude - ); + const scoresPairs: Array<[IVSDocument, number]> = this.calculateSimilarityScores(filteredDocuments, queryEmbedding, queryMagnitude); const sortedPairs = scoresPairs.sort((a, b) => b[1] - a[1]); - const results = sortedPairs - .slice(0, k) - .map((pair) => ({ ...pair[0], score: pair[1] })); + const results = sortedPairs.slice(0, k).map((pair) => ({ ...pair[0], score: pair[1] })); this.updateHitCounters(results); if (results.length > 0) { this.removeDocsLRU(); @@ -105,41 +89,35 @@ export class VectorStorage { } private async initDB(): Promise> { - return await openDB("VectorStorageDatabase", undefined, { + return await openDB('VectorStorageDatabase', undefined, { upgrade(db) { - const documentStore = db.createObjectStore("documents", { + const documentStore = db.createObjectStore('documents', { autoIncrement: true, - keyPath: "id", + keyPath: 'id', }); - documentStore.createIndex("text", "text", { unique: true }); - documentStore.createIndex("metadata", "metadata"); - documentStore.createIndex("timestamp", "timestamp"); - documentStore.createIndex("vector", "vector"); - documentStore.createIndex("vectorMag", "vectorMag"); - documentStore.createIndex("hits", "hits"); + documentStore.createIndex('text', 'text', { unique: true }); + documentStore.createIndex('metadata', 'metadata'); + documentStore.createIndex('timestamp', 'timestamp'); + documentStore.createIndex('vector', 'vector'); + documentStore.createIndex('vectorMag', 'vectorMag'); + documentStore.createIndex('hits', 'hits'); }, }); } private async loadTransformersModel(): Promise { env.allowLocalModels = false; - return await pipeline("feature-extraction", this.transfomersModel!); + return await pipeline('feature-extraction', this.transfomersModel); } - private async addDocuments( - documents: Array> - ): Promise>> { + private async addDocuments(documents: Array>): Promise>> { // filter out already existing documents - const newDocuments = documents.filter( - (doc) => !this.documents.some((d) => d.text === doc.text) - ); + const newDocuments = documents.filter((doc) => !this.documents.some((d) => d.text === doc.text)); // If there are no new documents, return an empty array if (newDocuments.length === 0) { return []; } - const newVectors = await this.embedTextsFn( - newDocuments.map((doc) => doc.text) - ); + const newVectors = await this.embedTextsFn(newDocuments.map((doc) => doc.text)); // Assign vectors and precompute vector magnitudes for new documents newDocuments.forEach((doc, index) => { doc.vector = newVectors[index]; @@ -156,9 +134,8 @@ export class VectorStorage { private async embedTexts(texts: string[]): Promise { if (this.transfomersModel) { return await this.embedTextsTransformers(texts); - } else { - return await this.embedTextsOpenAI(texts); } + return await this.embedTextsOpenAI(texts); } private async embedTextsTransformers(texts: string[]): Promise { @@ -170,7 +147,7 @@ export class VectorStorage { // eslint-disable-next-line no-await-in-loop const embedding = await this.transformersPipeline(text, { normalize: true, - pooling: "mean", + pooling: 'mean', }); embeddings.push(embedding.data); } @@ -185,9 +162,9 @@ export class VectorStorage { }), headers: { Authorization: `Bearer ${this.openaiApiKey}`, - "Content-Type": "application/json", + 'Content-Type': 'application/json', }, - method: "POST", + method: 'POST', }); if (!response.ok) { @@ -203,27 +180,14 @@ export class VectorStorage { } private calculateMagnitude(embedding: number[]): number { - const queryMagnitude = Math.sqrt( - embedding.reduce((sum, val) => sum + val * val, 0) - ); + const queryMagnitude = Math.sqrt(embedding.reduce((sum, val) => sum + val * val, 0)); return queryMagnitude; } - private calculateSimilarityScores( - filteredDocuments: Array>, - queryVector: number[], - queryMagnitude: number - ): Array<[IVSDocument, number]> { + private calculateSimilarityScores(filteredDocuments: Array>, queryVector: number[], queryMagnitude: number): Array<[IVSDocument, number]> { return filteredDocuments.map((doc) => { - const dotProduct = doc.vector!.reduce( - (sum, val, i) => sum + val * queryVector[i], - 0 - ); - let score = getCosineSimilarityScore( - dotProduct, - doc.vectorMag!, - queryMagnitude - ); + const dotProduct = doc.vector!.reduce((sum, val, i) => sum + val * queryVector[i], 0); + let score = getCosineSimilarityScore(dotProduct, doc.vectorMag!, queryMagnitude); score = normalizeScore(score); // Normalize the score return [doc, score]; }); @@ -239,7 +203,7 @@ export class VectorStorage { if (!this.db) { this.db = await this.initDB(); } - this.documents = await this.db.getAll("documents"); + this.documents = await this.db.getAll('documents'); this.removeDocsLRU(); } @@ -248,24 +212,22 @@ export class VectorStorage { this.db = await this.initDB(); } try { - const tx = this.db.transaction("documents", "readwrite"); - await tx.objectStore("documents").clear(); + const tx = this.db.transaction('documents', 'readwrite'); + await tx.objectStore('documents').clear(); for (const doc of this.documents) { // eslint-disable-next-line no-await-in-loop - await tx.objectStore("documents").put(doc); + await tx.objectStore('documents').put(doc); } await tx.done; } catch (error: any) { - console.error("Failed to save to IndexedDB:", error.message); + console.error('Failed to save to IndexedDB:', error.message); } } private removeDocsLRU(): void { if (getObjectSizeInMB(this.documents) > this.maxSizeInMB) { // Sort documents by hit counter (ascending) and then by timestamp (ascending) - this.documents.sort( - (a, b) => (a.hits ?? 0) - (b.hits ?? 0) || a.timestamp - b.timestamp - ); + this.documents.sort((a, b) => (a.hits ?? 0) - (b.hits ?? 0) || a.timestamp - b.timestamp); // Remove documents until the size is below the limit while (getObjectSizeInMB(this.documents) > this.maxSizeInMB) { @@ -279,11 +241,7 @@ function calcVectorMagnitude(doc: IVSDocument): number { return Math.sqrt(doc.vector!.reduce((sum, val) => sum + val * val, 0)); } -function getCosineSimilarityScore( - dotProduct: number, - magnitudeA: number, - magnitudeB: number -): number { +function getCosineSimilarityScore(dotProduct: number, magnitudeA: number, magnitudeB: number): number { return dotProduct / (magnitudeA * magnitudeB); } diff --git a/src/common/constants.ts b/src/common/constants.ts index b43e5f9..67799d0 100644 --- a/src/common/constants.ts +++ b/src/common/constants.ts @@ -2,5 +2,5 @@ export const constants = { DEFAULT_DEBOUNCE_TIME: 0, DEFAULT_MAX_SIZE_IN_MB: 2048, - OPENAI_API_URL: "https://api.openai.com/v1/embeddings", + OPENAI_API_URL: 'https://api.openai.com/v1/embeddings', }; From e744483978e76f292bc2b9e8b9690c04031b524d Mon Sep 17 00:00:00 2001 From: Ivan Zhang Date: Fri, 10 Nov 2023 16:46:30 -0800 Subject: [PATCH 03/20] update package.json version --- .prettierrc | 16 ++++++++++++++++ package.json | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 .prettierrc diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 0000000..f5c010a --- /dev/null +++ b/.prettierrc @@ -0,0 +1,16 @@ +{ + "prettier/prettier": [ + "error", + { + "singleQuote": true, + "printWidth": 180, + "trailingComma": "all", + "bracketSpacing": true, + "bracketSameLine": true, + "quoteProps": "as-needed", + "jsxSingleQuote": true, + "jsxBracketSameLine": false, + "jsxBracketSpacing": true + } + ] +} diff --git a/package.json b/package.json index 74a00b7..588c73a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "react-embeddings-search", - "version": "0.01", + "version": "0.0.1", "description": "A lightweight and efficient vector database for storing and searching text embeddings in the browser's local storage. The package uses OpenAI's API or Transformer.js models to generate embeddings for text documents and provides functionality for similarity search, filtering, and automatic management of local storage space. It is designed for use cases where client-side storage and retrieval of text embeddings are needed.", "main": "dist/index.js", "types": "dist/index.d.ts", From 0a58f1ed33aadf7745f03db5d205336c8109e410 Mon Sep 17 00:00:00 2001 From: Ivan Zhang Date: Fri, 10 Nov 2023 16:46:36 -0800 Subject: [PATCH 04/20] 0.0.2 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 003e577..9a9946e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "react-embeddings-search", - "version": "0.01", + "version": "0.0.2", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "react-embeddings-search", - "version": "0.01", + "version": "0.0.2", "license": "MIT", "dependencies": { "@xenova/transformers": "^2.8.0", diff --git a/package.json b/package.json index 588c73a..279ce5e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "react-embeddings-search", - "version": "0.0.1", + "version": "0.0.2", "description": "A lightweight and efficient vector database for storing and searching text embeddings in the browser's local storage. The package uses OpenAI's API or Transformer.js models to generate embeddings for text documents and provides functionality for similarity search, filtering, and automatic management of local storage space. It is designed for use cases where client-side storage and retrieval of text embeddings are needed.", "main": "dist/index.js", "types": "dist/index.d.ts", From cde7cc46ea0ae860e162698e47191ebe880dec49 Mon Sep 17 00:00:00 2001 From: Ivan Zhang Date: Fri, 10 Nov 2023 16:47:23 -0800 Subject: [PATCH 05/20] 0.0.3 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 9a9946e..883d621 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "react-embeddings-search", - "version": "0.0.2", + "version": "0.0.3", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "react-embeddings-search", - "version": "0.0.2", + "version": "0.0.3", "license": "MIT", "dependencies": { "@xenova/transformers": "^2.8.0", diff --git a/package.json b/package.json index 279ce5e..1bedcac 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "react-embeddings-search", - "version": "0.0.2", + "version": "0.0.3", "description": "A lightweight and efficient vector database for storing and searching text embeddings in the browser's local storage. The package uses OpenAI's API or Transformer.js models to generate embeddings for text documents and provides functionality for similarity search, filtering, and automatic management of local storage space. It is designed for use cases where client-side storage and retrieval of text embeddings are needed.", "main": "dist/index.js", "types": "dist/index.d.ts", From 344b75c9181b79a598a0fb1826e640f7a575f72a Mon Sep 17 00:00:00 2001 From: Ivan Zhang Date: Fri, 10 Nov 2023 16:48:04 -0800 Subject: [PATCH 06/20] 0.0.4 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 883d621..4434938 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "react-embeddings-search", - "version": "0.0.3", + "version": "0.0.4", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "react-embeddings-search", - "version": "0.0.3", + "version": "0.0.4", "license": "MIT", "dependencies": { "@xenova/transformers": "^2.8.0", diff --git a/package.json b/package.json index 1bedcac..89890bd 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "react-embeddings-search", - "version": "0.0.3", + "version": "0.0.4", "description": "A lightweight and efficient vector database for storing and searching text embeddings in the browser's local storage. The package uses OpenAI's API or Transformer.js models to generate embeddings for text documents and provides functionality for similarity search, filtering, and automatic management of local storage space. It is designed for use cases where client-side storage and retrieval of text embeddings are needed.", "main": "dist/index.js", "types": "dist/index.d.ts", From 90283db8e0189299d5519d3a5e71b10289e4e2e9 Mon Sep 17 00:00:00 2001 From: Ivan Zhang Date: Fri, 10 Nov 2023 16:49:02 -0800 Subject: [PATCH 07/20] 0.0.5 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 4434938..8f87b3f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "react-embeddings-search", - "version": "0.0.4", + "version": "0.0.5", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "react-embeddings-search", - "version": "0.0.4", + "version": "0.0.5", "license": "MIT", "dependencies": { "@xenova/transformers": "^2.8.0", diff --git a/package.json b/package.json index 89890bd..d6b7fec 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "react-embeddings-search", - "version": "0.0.4", + "version": "0.0.5", "description": "A lightweight and efficient vector database for storing and searching text embeddings in the browser's local storage. The package uses OpenAI's API or Transformer.js models to generate embeddings for text documents and provides functionality for similarity search, filtering, and automatic management of local storage space. It is designed for use cases where client-side storage and retrieval of text embeddings are needed.", "main": "dist/index.js", "types": "dist/index.d.ts", From 7db99cb69865b64b8b69ff6546e882bb2ddedea2 Mon Sep 17 00:00:00 2001 From: Ivan Zhang Date: Fri, 10 Nov 2023 16:49:46 -0800 Subject: [PATCH 08/20] update version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index d6b7fec..588c73a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "react-embeddings-search", - "version": "0.0.5", + "version": "0.0.1", "description": "A lightweight and efficient vector database for storing and searching text embeddings in the browser's local storage. The package uses OpenAI's API or Transformer.js models to generate embeddings for text documents and provides functionality for similarity search, filtering, and automatic management of local storage space. It is designed for use cases where client-side storage and retrieval of text embeddings are needed.", "main": "dist/index.js", "types": "dist/index.d.ts", From 0f6b4333d3c708ea2f8c01122aea5ae17224fc8c Mon Sep 17 00:00:00 2001 From: Ivan Zhang Date: Fri, 10 Nov 2023 16:49:52 -0800 Subject: [PATCH 09/20] 0.0.2 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 8f87b3f..9a9946e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "react-embeddings-search", - "version": "0.0.5", + "version": "0.0.2", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "react-embeddings-search", - "version": "0.0.5", + "version": "0.0.2", "license": "MIT", "dependencies": { "@xenova/transformers": "^2.8.0", diff --git a/package.json b/package.json index 588c73a..279ce5e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "react-embeddings-search", - "version": "0.0.1", + "version": "0.0.2", "description": "A lightweight and efficient vector database for storing and searching text embeddings in the browser's local storage. The package uses OpenAI's API or Transformer.js models to generate embeddings for text documents and provides functionality for similarity search, filtering, and automatic management of local storage space. It is designed for use cases where client-side storage and retrieval of text embeddings are needed.", "main": "dist/index.js", "types": "dist/index.d.ts", From b61887ab89aba65194430ad8f0bf555d9921ebd2 Mon Sep 17 00:00:00 2001 From: Ivan Zhang Date: Fri, 10 Nov 2023 16:50:56 -0800 Subject: [PATCH 10/20] update version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 279ce5e..bf41457 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "react-embeddings-search", - "version": "0.0.2", + "version": "0.0.0", "description": "A lightweight and efficient vector database for storing and searching text embeddings in the browser's local storage. The package uses OpenAI's API or Transformer.js models to generate embeddings for text documents and provides functionality for similarity search, filtering, and automatic management of local storage space. It is designed for use cases where client-side storage and retrieval of text embeddings are needed.", "main": "dist/index.js", "types": "dist/index.d.ts", From 9f0adcb5581ce066f319b9fbd5bbeafffbcbaeec Mon Sep 17 00:00:00 2001 From: Ivan Zhang Date: Fri, 10 Nov 2023 16:51:01 -0800 Subject: [PATCH 11/20] 0.0.1 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 9a9946e..b0edee5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "react-embeddings-search", - "version": "0.0.2", + "version": "0.0.1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "react-embeddings-search", - "version": "0.0.2", + "version": "0.0.1", "license": "MIT", "dependencies": { "@xenova/transformers": "^2.8.0", diff --git a/package.json b/package.json index bf41457..588c73a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "react-embeddings-search", - "version": "0.0.0", + "version": "0.0.1", "description": "A lightweight and efficient vector database for storing and searching text embeddings in the browser's local storage. The package uses OpenAI's API or Transformer.js models to generate embeddings for text documents and provides functionality for similarity search, filtering, and automatic management of local storage space. It is designed for use cases where client-side storage and retrieval of text embeddings are needed.", "main": "dist/index.js", "types": "dist/index.d.ts", From 3c003efa85ee46b5ea13a873af8d54876e871b5b Mon Sep 17 00:00:00 2001 From: Ivan Zhang Date: Fri, 10 Nov 2023 16:51:49 -0800 Subject: [PATCH 12/20] 0.0.2 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index b0edee5..9a9946e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "react-embeddings-search", - "version": "0.0.1", + "version": "0.0.2", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "react-embeddings-search", - "version": "0.0.1", + "version": "0.0.2", "license": "MIT", "dependencies": { "@xenova/transformers": "^2.8.0", diff --git a/package.json b/package.json index 588c73a..279ce5e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "react-embeddings-search", - "version": "0.0.1", + "version": "0.0.2", "description": "A lightweight and efficient vector database for storing and searching text embeddings in the browser's local storage. The package uses OpenAI's API or Transformer.js models to generate embeddings for text documents and provides functionality for similarity search, filtering, and automatic management of local storage space. It is designed for use cases where client-side storage and retrieval of text embeddings are needed.", "main": "dist/index.js", "types": "dist/index.d.ts", From a1730b023a5cfb5d8f43c4c7286c0753f342a1ff Mon Sep 17 00:00:00 2001 From: Ivan Zhang Date: Fri, 10 Nov 2023 16:52:44 -0800 Subject: [PATCH 13/20] update version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 279ce5e..bf41457 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "react-embeddings-search", - "version": "0.0.2", + "version": "0.0.0", "description": "A lightweight and efficient vector database for storing and searching text embeddings in the browser's local storage. The package uses OpenAI's API or Transformer.js models to generate embeddings for text documents and provides functionality for similarity search, filtering, and automatic management of local storage space. It is designed for use cases where client-side storage and retrieval of text embeddings are needed.", "main": "dist/index.js", "types": "dist/index.d.ts", From 2c187a0c0d91953da535982757cbc4afc2ed28fe Mon Sep 17 00:00:00 2001 From: Ivan Zhang Date: Fri, 10 Nov 2023 16:52:51 -0800 Subject: [PATCH 14/20] 0.0.1 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 9a9946e..b0edee5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "react-embeddings-search", - "version": "0.0.2", + "version": "0.0.1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "react-embeddings-search", - "version": "0.0.2", + "version": "0.0.1", "license": "MIT", "dependencies": { "@xenova/transformers": "^2.8.0", diff --git a/package.json b/package.json index bf41457..588c73a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "react-embeddings-search", - "version": "0.0.0", + "version": "0.0.1", "description": "A lightweight and efficient vector database for storing and searching text embeddings in the browser's local storage. The package uses OpenAI's API or Transformer.js models to generate embeddings for text documents and provides functionality for similarity search, filtering, and automatic management of local storage space. It is designed for use cases where client-side storage and retrieval of text embeddings are needed.", "main": "dist/index.js", "types": "dist/index.d.ts", From 8af3f1fe7212777238053c78de7789ae6727adb0 Mon Sep 17 00:00:00 2001 From: Ivan Zhang Date: Fri, 10 Nov 2023 16:55:55 -0800 Subject: [PATCH 15/20] update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 4f76e80..57c54cd 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # Vector Storage +Forked from [vector-storage](https://github.com/nitaiaharoni1/vector-storage) with patches to allow for Transformer.js embeddings. + Vector Storage is a lightweight and efficient vector database that stores document vectors in the browser's IndexedDB. This package allows you to perform semantic similarity searches on text documents using vector embeddings. Semantic search refers to the ability to understand the meaning and context of text documents and queries, enabling more accurate and relevant search results. Vector Storage leverages OpenAI or Transformer.js embeddings (see compatible models [here](https://huggingface.co/models?library=transformers.js)) to convert text documents into vectors and provides an interface for searching similar documents based on cosine similarity. ## Features From 77c8e1b017672bf80823cd9128cc070e2e5adb34 Mon Sep 17 00:00:00 2001 From: Ivan Zhang Date: Fri, 10 Nov 2023 16:56:09 -0800 Subject: [PATCH 16/20] 0.0.2 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index b0edee5..9a9946e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "react-embeddings-search", - "version": "0.0.1", + "version": "0.0.2", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "react-embeddings-search", - "version": "0.0.1", + "version": "0.0.2", "license": "MIT", "dependencies": { "@xenova/transformers": "^2.8.0", diff --git a/package.json b/package.json index 588c73a..279ce5e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "react-embeddings-search", - "version": "0.0.1", + "version": "0.0.2", "description": "A lightweight and efficient vector database for storing and searching text embeddings in the browser's local storage. The package uses OpenAI's API or Transformer.js models to generate embeddings for text documents and provides functionality for similarity search, filtering, and automatic management of local storage space. It is designed for use cases where client-side storage and retrieval of text embeddings are needed.", "main": "dist/index.js", "types": "dist/index.d.ts", From f1fe9b30da5370f6489df07e005e71734f071b5b Mon Sep 17 00:00:00 2001 From: Ivan Zhang Date: Fri, 10 Nov 2023 16:56:15 -0800 Subject: [PATCH 17/20] 0.0.3 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 9a9946e..883d621 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "react-embeddings-search", - "version": "0.0.2", + "version": "0.0.3", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "react-embeddings-search", - "version": "0.0.2", + "version": "0.0.3", "license": "MIT", "dependencies": { "@xenova/transformers": "^2.8.0", diff --git a/package.json b/package.json index 279ce5e..1bedcac 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "react-embeddings-search", - "version": "0.0.2", + "version": "0.0.3", "description": "A lightweight and efficient vector database for storing and searching text embeddings in the browser's local storage. The package uses OpenAI's API or Transformer.js models to generate embeddings for text documents and provides functionality for similarity search, filtering, and automatic management of local storage space. It is designed for use cases where client-side storage and retrieval of text embeddings are needed.", "main": "dist/index.js", "types": "dist/index.d.ts", From 699d5f07b40c40bcabebfc37206b30b6b5e67ba5 Mon Sep 17 00:00:00 2001 From: Ivan Zhang Date: Fri, 10 Nov 2023 16:56:36 -0800 Subject: [PATCH 18/20] version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 1bedcac..279ce5e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "react-embeddings-search", - "version": "0.0.3", + "version": "0.0.2", "description": "A lightweight and efficient vector database for storing and searching text embeddings in the browser's local storage. The package uses OpenAI's API or Transformer.js models to generate embeddings for text documents and provides functionality for similarity search, filtering, and automatic management of local storage space. It is designed for use cases where client-side storage and retrieval of text embeddings are needed.", "main": "dist/index.js", "types": "dist/index.d.ts", From 2e982f4fdcc6df793ec7d58e812788d824a6deac Mon Sep 17 00:00:00 2001 From: Ivan Zhang Date: Fri, 10 Nov 2023 16:59:14 -0800 Subject: [PATCH 19/20] revert pkg.json --- package.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 279ce5e..1564141 100644 --- a/package.json +++ b/package.json @@ -1,12 +1,12 @@ { - "name": "react-embeddings-search", - "version": "0.0.2", - "description": "A lightweight and efficient vector database for storing and searching text embeddings in the browser's local storage. The package uses OpenAI's API or Transformer.js models to generate embeddings for text documents and provides functionality for similarity search, filtering, and automatic management of local storage space. It is designed for use cases where client-side storage and retrieval of text embeddings are needed.", + "name": "vector-storage", + "version": "1.0.54", + "description": "A lightweight and efficient vector database for storing and searching text embeddings in the browser's local storage. The package uses OpenAI's API to generate embeddings for text documents and provides functionality for similarity search, filtering, and automatic management of local storage space. It is designed for use cases where client-side storage and retrieval of text embeddings are needed.", "main": "dist/index.js", "types": "dist/index.d.ts", "repository": { "type": "git", - "url": "https://github.com/aivanzhang/ai-search" + "url": "https://github.com/nitaiaharoni1/vector-storage.git" }, "keywords": [ "vector", @@ -32,7 +32,7 @@ "package": "npm run lint && npm run build && npm run bump && npm publish", "lint": "eslint --ext .ts src" }, - "author": "Ivan Zhang", + "author": "Nitai Aharoni", "license": "MIT", "dependencies": { "@xenova/transformers": "^2.8.0", From 0ec053f623ff661682e1e31fd6bda94b25ee9139 Mon Sep 17 00:00:00 2001 From: Ivan <40708241+aivanzhang@users.noreply.github.com> Date: Fri, 10 Nov 2023 17:01:33 -0800 Subject: [PATCH 20/20] Delete .prettierrc --- .prettierrc | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 .prettierrc diff --git a/.prettierrc b/.prettierrc deleted file mode 100644 index f5c010a..0000000 --- a/.prettierrc +++ /dev/null @@ -1,16 +0,0 @@ -{ - "prettier/prettier": [ - "error", - { - "singleQuote": true, - "printWidth": 180, - "trailingComma": "all", - "bracketSpacing": true, - "bracketSameLine": true, - "quoteProps": "as-needed", - "jsxSingleQuote": true, - "jsxBracketSameLine": false, - "jsxBracketSpacing": true - } - ] -}