Use wangfenjin/simple as fts5 tokenizer

docs/dev.md

@@ -0,0 +1,12 @@
+Grab libsimple-osx-x64.zip from [wangfenjin/simple/releases](https://github.com/wangfenjin/simple/releases) and put `libsimple.dylib` into `db/` of the project root.
+
+```sql
+.mode box
+
+; list triggers
+select * from sqlite_master where type = 'trigger';
+
+select rowid, * from migration;
+; manual reset migration version
+update migration set version = 5 where rowid = 1;
+```

src/lib/server/db/bookmark.db.ts

@@ -70,11 +70,15 @@ export function getBookmarks(
   }
 
   if (text) {
-    join.push('bookmark_fts on (bookmark_fts.rowid = b.id)');
-    where.push('bookmark_fts match ?');
+    // join.push('bookmark_fts on (bookmark_fts.rowid = b.id)');
+    // where.push('bookmark_fts match ?');
+    join.push('bookmark_fts_v2 on (bookmark_fts_v2.rowid = b.id)');
+    where.push('bookmark_fts_v2 match simple_query(?)');
+
     params.push(text);
 
-    tokens.orderBy = 'bookmark_fts.rank';
+    // tokens.orderBy = 'bookmark_fts.rank';
+    tokens.orderBy = 'bookmark_fts_v2.rank';
   } else {
     tokens.orderBy = 'b.updatedAt desc, b.id desc';
   }

src/lib/server/db/common.db.ts

@@ -1,3 +1,5 @@
+import path from 'node:path';
+
 import assert from 'assert';
 import Sqlite from 'better-sqlite3';
 
@@ -8,6 +10,7 @@ import * as v1 from './migrations/v01.migration';
 import * as v2 from './migrations/v02.migration';
 import * as v4 from './migrations/v04.migration';
 import * as v5 from './migrations/v05.migration';
+import * as v6 from './migrations/v06.migration';
 
 const DATABASE_STATE: Record<string, { db: Sqlite.Database; migrated?: boolean }> = {};
 
@@ -16,13 +19,17 @@ const migrations = [
   { version: 2, mod: v2 },
   { version: 4, mod: v4 },
   { version: 5, mod: v5 },
+  { version: 6, mod: v6 },
 ];
 
 export const lite = (filepath = DATABASE_PATH, shouldMigrate = true, verbose = false) => {
   if (DATABASE_STATE[filepath]) return DATABASE_STATE[filepath].db;
   const db = new Sqlite(filepath, {
     ...(verbose ? { verbose: console.log } : undefined),
   });
+
+  db.loadExtension(path.join('db', 'libsimple'));
+
   DATABASE_STATE[filepath] = { db };
   if (shouldMigrate && !DATABASE_STATE[filepath].migrated) {
     db.pragma('journal_mode = WAL');
@@ -67,6 +74,7 @@ function migrate(db: Sqlite.Database) {
       if (m.version > version) {
         m.mod.up(db);
         db.prepare('update migration set version = @version where rowid = 1').run({ version: m.version });
+        logger.warn('database migrated to version %s', m.version);
       }
     }
   });

src/lib/server/db/migrations/v06.migration.ts

@@ -0,0 +1,83 @@
+import type { Database } from 'better-sqlite3';
+
+import { trigger } from '../builder';
+
+export const version = 6;
+
+// migrate to fts to use https://github.com/wangfenjin/simple
+// the new table is bookmark_fts_v2
+// the previous table bookmark_fts will be dropped
+// fts related triggers will be dropped and recreated
+
+export const up = (db: Database) => {
+  db.prepare(
+    `create virtual table if not exists bookmark_fts_v2 USING fts5 (title,desc,url,content=bookmark,content_rowid=id,tokenize='simple')`,
+  ).run();
+  db.prepare(`insert into bookmark_fts_v2(bookmark_fts_v2) values('rebuild')`).run();
+  db.prepare(`drop trigger if exists bookmark_insert_trigger`).run();
+  db.prepare(
+    trigger.create({
+      name: 'bookmark_insert_trigger_6_1',
+      ifNotExists: true,
+      event: 'after insert',
+      table: 'bookmark',
+      statements: [`update user set bookmarkCount = bookmarkCount + 1 where id = new.userId`],
+    }),
+  ).run();
+
+  db.prepare(
+    trigger.create({
+      name: 'bookmark_insert_trigger_6_2',
+      ifNotExists: true,
+      event: 'after insert',
+      table: 'bookmark',
+      statements: [`insert into bookmark_fts_v2(rowid,title,desc,url) values (new.id,new.title,new.desc,new.url)`],
+    }),
+  ).run();
+
+  db.prepare(`drop trigger if exists bookmark_delete_trigger`).run();
+  db.prepare(
+    trigger.create({
+      name: 'bookmark_delete_trigger_6_1',
+      ifNotExists: true,
+      event: 'after delete',
+      table: 'bookmark',
+      statements: [
+        `insert into bookmark_fts_v2 (bookmark_fts_v2,rowid,title,desc,url) values ('delete',old.id,old.title,old.desc,old.url)`,
+      ],
+    }),
+  ).run();
+  db.prepare(
+    trigger.create({
+      name: 'bookmark_delete_trigger_6_2',
+      ifNotExists: true,
+      event: 'after delete',
+      table: 'bookmark',
+      statements: [`delete from bookmark_tag where bookmarkId = old.id`],
+    }),
+  ).run();
+  db.prepare(
+    trigger.create({
+      name: 'bookmark_delete_trigger_6_3',
+      ifNotExists: true,
+      event: 'after delete',
+      table: 'bookmark',
+      statements: [`update user set bookmarkCount = bookmarkCount - 1 where id = old.userId`],
+    }),
+  ).run();
+
+  db.prepare(`drop trigger if exists bookmark_update_trigger`).run();
+  db.prepare(
+    trigger.create({
+      name: 'bookmark_update_trigger_6_0',
+      ifNotExists: true,
+      event: 'after update',
+      table: 'bookmark',
+      statements: [
+        `insert into bookmark_fts_v2 (bookmark_fts_v2,rowid,title,desc,url) values ('delete',old.id,old.title,old.desc,old.url)`,
+        `insert into bookmark_fts_v2(rowid,title,desc,url) values (new.id,new.title,new.desc,new.url)`,
+      ],
+    }),
+  ).run();
+  db.prepare(`drop table if exists bookmark_fts`).run();
+};