Determine language per author

src/algos.ts

@@ -5,7 +5,7 @@ import {
   OutputSchema as AlgoOutput,
 } from './lexicon/types/app/bsky/feed/getFeedSkeleton';
 import { PostSchema } from './db/schema';
-import { LANGS_HEBREW, LANGS_YIDDISH } from './util/hebrew';
+import { HEBREW_LOOKALIKES, LANGS_HEBREW, LANGS_YIDDISH } from './util/hebrew';
 import { FILTERED_USERS } from './util/userlists';
 import { AppContext } from './context';
 
@@ -49,12 +49,25 @@ function createLanguageFeed(
   return async (ctx: AppContext, params: QueryParams, actor?: string) => {
     let builder = ctx.db
       .selectFrom('post')
-      .select(['indexedAt', 'uri'])
-      .where('language', 'in', languages)
-      .where('author', 'not in', FILTERED_USERS)
-      .orderBy('indexedAt', 'desc')
-      .orderBy('cid', 'desc')
-      .limit(params.limit);
+      .select(['post.indexedAt', 'post.uri'])
+      .where('post.author', 'not in', FILTERED_USERS)
+      .orderBy('post.indexedAt', 'desc')
+      .orderBy('post.cid', 'desc')
+      .limit(params.limit)
+
+      // Solve the "author who always posts in a specific language whose posts still get misidentified" problem
+      .innerJoin('author_language', (eb) =>
+        eb.onRef('post.author', '=', 'author_language.author'),
+      )
+      .where((eb) =>
+        eb.or([
+          eb('post.language', 'in', languages),
+          eb.and([
+            eb('post.language', 'in', HEBREW_LOOKALIKES),
+            eb('author_language.language', 'in', languages),
+          ]),
+        ]),
+      );
 
     if (includeReplies) {
       if (actor) {
@@ -63,8 +76,8 @@ function createLanguageFeed(
         if (blocklist && blocklist.length > 0) {
           builder = builder.where((eb) =>
             eb.or([
-              eb('replyTo', 'is', null),
-              eb('replyTo', 'not in', (eb) =>
+              eb('post.replyTo', 'is', null),
+              eb('post.replyTo', 'not in', (eb) =>
                 eb
                   .selectFrom('post')
                   .select('uri')

src/db/cron.ts

@@ -0,0 +1,14 @@
+import { sql } from 'kysely';
+import { Database } from '.';
+import { interval } from 'ix/asynciterable';
+
+export async function refreshAuthorLanguage(
+  db: Database,
+  refreshIntervalMs: number,
+) {
+  for await (const _ of interval(refreshIntervalMs)) {
+    await sql`REFRESH MATERIALIZED VIEW CONCURRENTLY author_language;`.execute(
+      db,
+    );
+  }
+}

src/db/migrations.ts

@@ -1,4 +1,5 @@
 import { Kysely, MigrationProvider, sql } from 'kysely';
+import { DatabaseSchema } from './schema';
 
 export const migrationProvider: MigrationProvider = {
   async getMigrations() {
@@ -11,6 +12,7 @@ export const migrationProvider: MigrationProvider = {
       '006': addIndexOnAuthor,
       '007': optimizeIndexes,
       '008': removeLanguageDefault,
+      '009': createAuthorLanguageView,
     };
   },
 };
@@ -121,3 +123,26 @@ const removeLanguageDefault = {
       .execute();
   },
 };
+
+const createAuthorLanguageView = {
+  async up(db: Kysely<DatabaseSchema>) {
+    await db.schema
+      .createView('author_language')
+      .materialized()
+      .as(
+        sql`
+SELECT
+  author,
+  CASE 
+    WHEN COUNT(*) <= 10 THEN NULL
+    WHEN 1.0 * COUNT(*) FILTER (WHERE language IN ('he', 'iw')) / COUNT(*) > 0.6 THEN 'he'
+    WHEN 1.0 * COUNT(*) FILTER (WHERE language = 'yi') / COUNT(*) > 0.6 THEN 'yi'
+    ELSE NULL
+  END AS "language"
+FROM post
+GROUP BY author;
+      `,
+      )
+      .execute();
+  },
+};

src/db/schema.ts

@@ -1,6 +1,7 @@
 export type DatabaseSchema = {
   post: PostSchema;
   sub_state: SubStateSchema;
+  author_language: AuthorLanguageViewSchema;
 };
 
 export type PostSchema = {
@@ -17,3 +18,8 @@ export type SubStateSchema = {
   service: string;
   cursor: number;
 };
+
+export type AuthorLanguageViewSchema = {
+  author: string;
+  language: string | null;
+};

src/util/hebrew/index.ts

@@ -17,6 +17,11 @@ export function hasHebrewLetters(text: string) {
 export const LANGS_HEBREW = ['he', 'iw'];
 export const LANGS_YIDDISH = ['yi'];
 export const LANG_UNKNOWN = 'unknown';
+export const HEBREW_LOOKALIKES = [
+  ...LANGS_HEBREW,
+  ...LANGS_YIDDISH,
+  LANG_UNKNOWN,
+];
 
 const classifier = new FastText.Classifier(path.join(__dirname, 'model.ftz'));
 const indexer_language_detections = new Counter({