Adds Mandarin tokenizer

packages/tokenizers/.gitignore

@@ -0,0 +1 @@
+dist/

packages/tokenizers/LICENSE.md

@@ -0,0 +1,13 @@
+Copyright 2023 OramaSearch Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

packages/tokenizers/README.md

@@ -0,0 +1,25 @@
+# Orama Tokenizers
+
+This package provides support for additional tokenizers for the Orama Search Engine.
+
+Available tokenizers:
+- Mandarin
+- Japanese (work in progress)
+- Korean (work in progress)
+
+Usage:
+
+```js
+import init, { tokenize } from '@orama/tokenizes/mandarin'
+
+await init() // Initialize the WebAssembly module
+
+const hmm = true // Use Hidden Markov Model or not (suggested: true)
+const tokens = await tokenize('我是一名软件工程师', hmm)
+
+console.log(tokens)
+// ["我","是","一名","软件","工程师"]
+```
+
+# License
+[Apache 2.0](/LICENSE.md)

packages/tokenizers/package.json

@@ -0,0 +1,47 @@
+{
+  "name": "@orama/tokenizers",
+  "version": "2.0.5",
+  "type": "module",
+  "description": "Additional tokenizers for Orama",
+  "sideEffects": false,
+  "exports": {
+    "./mandarin": {
+      "types": "./dist/tokenizer_mandarin.d.ts",
+      "import": "./dist/tokenizer_mandarin.js"
+    }
+  },
+  "files": ["dist"],
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/oramasearch/orama"
+  },
+  "bugs": {
+    "url": "https://github.com/oramasearch/orama"
+  },
+  "scripts": {
+    "build": "node ./scripts/build.mjs",
+    "lint": "exit 0;",
+    "test": "exit 0;",
+    "ci": "npm run test"
+  },
+  "keywords": [
+    "full-text search",
+    "search",
+    "fuzzy search",
+    "typo-tolerant search",
+    "full-text",
+    "vector search",
+    "stemming",
+    "tokenizers"
+  ],
+  "author": {
+    "name": "Michele Riva",
+    "email": "[email protected]",
+    "url": "https://github.com/MicheleRiva",
+    "author": true
+  },
+  "license": "Apache-2.0",
+  "engines": {
+    "node": ">= 18.0.0"
+  }
+}

packages/tokenizers/scripts/build.mjs

@@ -0,0 +1,27 @@
+import path from 'node:path'
+import fs from 'node:fs'
+import childProcess from 'node:child_process'
+
+if (process.env.GITHUB_ACTIONS) {
+  console.log('Skipping build in CI')
+  process.exit(0)
+}
+
+const outdirBaseURL = new URL('../dist', import.meta.url).pathname
+const tokenizersBaseURL = new URL('../src', import.meta.url).pathname
+
+const mandarinTokenizerPath = path.join(tokenizersBaseURL, 'tokenizer-mandarin')
+const mandarinTokenizerWasmPath = path.join(mandarinTokenizerPath, 'pkg')
+const mandarinTokenizerDistPath = path.join(tokenizersBaseURL, '../dist/tokenizer-mandarin')
+
+if (fs.existsSync(outdirBaseURL)) {
+  fs.rmdirSync(outdirBaseURL, { recursive: true })
+}
+
+fs.mkdirSync(outdirBaseURL)
+
+childProcess.execSync(`cd ${mandarinTokenizerPath} && wasm-pack build --target web`)
+
+fs.cpSync(mandarinTokenizerWasmPath, mandarinTokenizerDistPath, {
+  recursive: true
+})

packages/tokenizers/src/tokenizer-mandarin/.gitignore

@@ -0,0 +1,2 @@
+target
+pkg