Fix/word_similarity - master

migration/1715086559930-add_pg_trgm_indexes.ts

@@ -0,0 +1,29 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+export class AddPgTrgmIndexes1715086559930 implements MigrationInterface {
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      'CREATE INDEX if not exists trgm_idx_project_title ON project USING GIN (title gin_trgm_ops);',
+    );
+    await queryRunner.query(
+      'CREATE INDEX if not exists trgm_idx_project_description ON project USING GIN (description gin_trgm_ops);',
+    );
+    await queryRunner.query(
+      'CREATE INDEX if not exists trgm_idx_project_impact_location ON project USING GIN ("impactLocation" gin_trgm_ops);',
+    );
+    await queryRunner.query(
+      'CREATE INDEX if not exists trgm_idx_user_name ON public.user USING GIN ("name" gin_trgm_ops);',
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query('drop index if exists trgm_idx_project_title;');
+    await queryRunner.query(
+      'drop index if exists trgm_idx_project_description;',
+    );
+    await queryRunner.query(
+      'drop index if exists trgm_idx_project_impact_location;',
+    );
+    await queryRunner.query('drop index if exists trgm_idx_user_name;');
+  }
+}

src/entities/project.ts

@@ -210,6 +210,7 @@ export class Project extends BaseEntity {
   @Column({ nullable: true })
   image?: string;
 
+  @Index('trgm_idx_project_impact_location', { synchronize: false })
   @Field({ nullable: true })
   @Column({ nullable: true })
   impactLocation?: string;
@@ -606,6 +607,7 @@ export class ProjectUpdate extends BaseEntity {
   @PrimaryGeneratedColumn()
   readonly id: number;
 
+  @Index('trgm_idx_project_title', { synchronize: false })
   @Field(_type => String)
   @Column()
   title: string;
@@ -667,6 +669,7 @@ export class ProjectUpdate extends BaseEntity {
   @Column('text', { nullable: true })
   organizationWebsite: string;
 
+  @Index('trgm_idx_project_description', { synchronize: false })
   @Field({ nullable: true })
   @Column('text', { nullable: true })
   organizationDescription: string;

src/entities/user.ts

@@ -4,6 +4,7 @@ import {
   Column,
   CreateDateColumn,
   Entity,
+  Index,
   JoinTable,
   ManyToMany,
   OneToMany,
@@ -86,6 +87,7 @@ export class User extends BaseEntity {
   @Column({ nullable: true })
   lastName?: string;
 
+  @Index('trgm_idx_user_name', { synchronize: false })
   @Field(_type => String, { nullable: true })
   @Column({ nullable: true })
   name?: string;

src/resolvers/projectResolver.ts

@@ -40,7 +40,6 @@ import { Donation } from '../entities/donation';
 import { ProjectImage } from '../entities/projectImage';
 import { ApolloContext } from '../types/ApolloContext';
 import { publicSelectionFields, User } from '../entities/user';
-import config from '../config';
 import { Context } from '../context';
 import SentryLogger from '../sentryLogger';
 import {
@@ -319,40 +318,27 @@ export class ProjectResolver {
     searchTerm?: string,
   ) {
     if (!searchTerm) return query;
-    const similarityThreshold =
-      Number(config.get('PROJECT_SEARCH_SIMILARITY_THRESHOLD')) || 0.4;
-
-    return query.andWhere(
-      new Brackets(qb => {
-        qb.where(
-          'WORD_SIMILARITY(project.title, :searchTerm) > :similarityThreshold',
-          {
-            searchTerm: `%${searchTerm}%`,
-            similarityThreshold,
-          },
+
+    return (
+      query
+        // For future use! This is the way to use similarity in TypeORM
+        // .addSelect('similarity(project.title, :searchTerm)', 'title_slm')
+        // .addSelect('similarity(project.description, :searchTerm)', 'desc_slm')
+        // .addSelect('similarity(project.impactLocation, :searchTerm)', 'loc_slm')
+        // .setParameter('searchTerm', searchTerm)
+        .andWhere(
+          new Brackets(qb => {
+            qb.where('project.title % :searchTerm ', {
+              searchTerm,
+            })
+              .orWhere('project.description % :searchTerm ', {
+                searchTerm,
+              })
+              .orWhere('project.impactLocation % :searchTerm', {
+                searchTerm,
+              });
+          }),
         )
-          .orWhere(
-            'WORD_SIMILARITY(project.description, :searchTerm) > :similarityThreshold',
-            {
-              searchTerm: `%${searchTerm}%`,
-              similarityThreshold,
-            },
-          )
-          .orWhere(
-            'WORD_SIMILARITY(project.impactLocation, :searchTerm) > :similarityThreshold',
-            {
-              searchTerm: `%${searchTerm}%`,
-              similarityThreshold,
-            },
-          )
-          .orWhere(
-            'WORD_SIMILARITY(user.name, :searchTerm) > :similarityThreshold',
-            {
-              searchTerm: `%${searchTerm}%`,
-              similarityThreshold,
-            },
-          );
-      }),
     );
   }
 

src/server/bootstrap.ts

@@ -98,6 +98,8 @@ export async function bootstrap() {
 
     Container.set(DataSource, AppDataSource.getDataSource());
 
+    await setDatabaseParameters(AppDataSource.getDataSource());
+
     const dropSchema = config.get('DROP_DATABASE') === 'true';
     if (dropSchema) {
       // eslint-disable-next-line no-console
@@ -449,3 +451,16 @@ export async function bootstrap() {
     await initializeCronJobs();
   }
 }
+
+async function setDatabaseParameters(ds: DataSource) {
+  await setPgTrgmParameters(ds);
+}
+
+async function setPgTrgmParameters(ds: DataSource) {
+  const similarityThreshold =
+    Number(config.get('PROJECT_SEARCH_SIMILARITY_THRESHOLD')) || 0.1;
+  await ds.query(`SET pg_trgm.similarity_threshold TO ${similarityThreshold};`);
+  await ds.query(
+    `SET pg_trgm.word_similarity_threshold TO ${similarityThreshold};`,
+  );
+}

test/pre-test-scripts.ts

@@ -41,6 +41,8 @@ import { createDonationethUser1701756190381 } from '../migration/1701756190381-c
 import { ChainType } from '../src/types/network';
 import { COINGECKO_TOKEN_IDS } from '../src/adapters/price/CoingeckoPriceAdapter';
 import { ProjectActualMatchinView151713700147145 } from '../migration/1713700147145-project_actual_matchin_view_15';
+import { EnablePgTrgmExtension1713859866338 } from '../migration/1713859866338-enable_pg_trgm_extension';
+import { AddPgTrgmIndexes1715086559930 } from '../migration/1715086559930-add_pg_trgm_indexes';
 
 async function seedDb() {
   await seedUsers();
@@ -479,6 +481,8 @@ async function runMigrations() {
     );
     await new createDonationethUser1701756190381().up(queryRunner);
     await new ProjectActualMatchinView151713700147145().up(queryRunner);
+    await new EnablePgTrgmExtension1713859866338().up(queryRunner);
+    await new AddPgTrgmIndexes1715086559930().up(queryRunner);
   } finally {
     await queryRunner.release();
   }