Fix/word similarity - staging (#1546)

* Hotfix db improvements (#1523) (#1524) * add extra configurations for postgresql connections * add master and slave replication strategy for typeorm Co-authored-by: CarlosQ96 <[email protected]> * Fix/db replica production (#1525) * Hotfix db improvements (#1523) * add extra configurations for postgresql connections * add master and slave replication strategy for typeorm * Define db read only configs --------- Co-authored-by: CarlosQ96 <[email protected]> * update comment * Hotfix latency issues for prod (#1529) * Hotfix staging fix latency (#1528) * add project donation summary view entity * convert projectQueries to querybuilder * add cache to projectDonationSummary queries * add configurable cache to slow queries * remove massive recurring donation log * add await for project queries * Add informative logs for draft donation service job (#1537) * Fix eslint errors * Fix/master test (#1541) * Fixed master test issue * Returned test to master pipeline * Comment executing donation summary view --------- Co-authored-by: Mohammad Ranjbar Z <[email protected]> * Fixed word similarity issue * Removed unused import --------- Co-authored-by: CarlosQ96 <[email protected]> Co-authored-by: Carlos <[email protected]> Co-authored-by: mohammadranjbarz <[email protected]>
Giveth · May 7, 2024 · 7db0975 · 7db0975
1 parent 7467f9b
commit 7db0975
Show file tree

Hide file tree

Showing 6 changed files with 73 additions and 34 deletions.
diff --git a/migration/1715086559930-add_pg_trgm_indexes.ts b/migration/1715086559930-add_pg_trgm_indexes.ts
@@ -0,0 +1,29 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+export class AddPgTrgmIndexes1715086559930 implements MigrationInterface {
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      'CREATE INDEX if not exists trgm_idx_project_title ON project USING GIN (title gin_trgm_ops);',
+    );
+    await queryRunner.query(
+      'CREATE INDEX if not exists trgm_idx_project_description ON project USING GIN (description gin_trgm_ops);',
+    );
+    await queryRunner.query(
+      'CREATE INDEX if not exists trgm_idx_project_impact_location ON project USING GIN ("impactLocation" gin_trgm_ops);',
+    );
+    await queryRunner.query(
+      'CREATE INDEX if not exists trgm_idx_user_name ON public.user USING GIN ("name" gin_trgm_ops);',
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query('drop index if exists trgm_idx_project_title;');
+    await queryRunner.query(
+      'drop index if exists trgm_idx_project_description;',
+    );
+    await queryRunner.query(
+      'drop index if exists trgm_idx_project_impact_location;',
+    );
+    await queryRunner.query('drop index if exists trgm_idx_user_name;');
+  }
+}
diff --git a/src/entities/project.ts b/src/entities/project.ts
@@ -210,6 +210,7 @@ export class Project extends BaseEntity {
   @Column({ nullable: true })
   image?: string;
 
+  @Index('trgm_idx_project_impact_location', { synchronize: false })
   @Field({ nullable: true })
   @Column({ nullable: true })
   impactLocation?: string;
@@ -606,6 +607,7 @@ export class ProjectUpdate extends BaseEntity {
   @PrimaryGeneratedColumn()
   readonly id: number;
 
+  @Index('trgm_idx_user_name', { synchronize: false })
   @Field(_type => String)
   @Column()
   title: string;
@@ -667,6 +669,7 @@ export class ProjectUpdate extends BaseEntity {
   @Column('text', { nullable: true })
   organizationWebsite: string;
 
+  @Index('trgm_idx_project_description', { synchronize: false })
   @Field({ nullable: true })
   @Column('text', { nullable: true })
   organizationDescription: string;

diff --git a/src/entities/user.ts b/src/entities/user.ts
@@ -4,6 +4,7 @@ import {
   Column,
   CreateDateColumn,
   Entity,
+  Index,
   JoinTable,
   ManyToMany,
   OneToMany,
@@ -86,6 +87,7 @@ export class User extends BaseEntity {
   @Column({ nullable: true })
   lastName?: string;
 
+  @Index('trgm_idx_project_impact_location', { synchronize: false })
   @Field(_type => String, { nullable: true })
   @Column({ nullable: true })
   name?: string;

diff --git a/src/resolvers/projectResolver.ts b/src/resolvers/projectResolver.ts
@@ -40,7 +40,6 @@ import { Donation } from '../entities/donation';
 import { ProjectImage } from '../entities/projectImage';
 import { ApolloContext } from '../types/ApolloContext';
 import { publicSelectionFields, User } from '../entities/user';
-import config from '../config';
 import { Context } from '../context';
 import SentryLogger from '../sentryLogger';
 import {
@@ -319,40 +318,27 @@ export class ProjectResolver {
     searchTerm?: string,
   ) {
     if (!searchTerm) return query;
-    const similarityThreshold =
-      Number(config.get('PROJECT_SEARCH_SIMILARITY_THRESHOLD')) || 0.4;
-
-    return query.andWhere(
-      new Brackets(qb => {
-        qb.where(
-          'WORD_SIMILARITY(project.title, :searchTerm) > :similarityThreshold',
-          {
-            searchTerm: `%${searchTerm}%`,
-            similarityThreshold,
-          },
+
+    return (
+      query
+        // For future use! This is the way to use similarity in TypeORM
+        // .addSelect('similarity(project.title, :searchTerm)', 'title_slm')
+        // .addSelect('similarity(project.description, :searchTerm)', 'desc_slm')
+        // .addSelect('similarity(project.impactLocation, :searchTerm)', 'loc_slm')
+        // .setParameter('searchTerm', searchTerm)
+        .andWhere(
+          new Brackets(qb => {
+            qb.where('project.title % :searchTerm ', {
+              searchTerm,
+            })
+              .orWhere('project.description % :searchTerm ', {
+                searchTerm,
+              })
+              .orWhere('project.impactLocation % :searchTerm', {
+                searchTerm,
+              });
+          }),
         )
-          .orWhere(
-            'WORD_SIMILARITY(project.description, :searchTerm) > :similarityThreshold',
-            {
-              searchTerm: `%${searchTerm}%`,
-              similarityThreshold,
-            },
-          )
-          .orWhere(
-            'WORD_SIMILARITY(project.impactLocation, :searchTerm) > :similarityThreshold',
-            {
-              searchTerm: `%${searchTerm}%`,
-              similarityThreshold,
-            },
-          )
-          .orWhere(
-            'WORD_SIMILARITY(user.name, :searchTerm) > :similarityThreshold',
-            {
-              searchTerm: `%${searchTerm}%`,
-              similarityThreshold,
-            },
-          );
-      }),
     );
   }
 

diff --git a/src/server/bootstrap.ts b/src/server/bootstrap.ts
@@ -98,6 +98,8 @@ export async function bootstrap() {
 
     Container.set(DataSource, AppDataSource.getDataSource());
 
+    await setDatabaseParameters(AppDataSource.getDataSource());
+
     const dropSchema = config.get('DROP_DATABASE') === 'true';
     if (dropSchema) {
       // eslint-disable-next-line no-console
@@ -445,3 +447,16 @@ export async function bootstrap() {
     await initializeCronJobs();
   }
 }
+
+async function setDatabaseParameters(ds: DataSource) {
+  await setPgTrgmParameters(ds);
+}
+
+async function setPgTrgmParameters(ds: DataSource) {
+  const similarityThreshold =
+    Number(config.get('PROJECT_SEARCH_SIMILARITY_THRESHOLD')) || 0.1;
+  await ds.query(`SET pg_trgm.similarity_threshold TO ${similarityThreshold};`);
+  await ds.query(
+    `SET pg_trgm.word_similarity_threshold TO ${similarityThreshold};`,
+  );
+}
diff --git a/test/pre-test-scripts.ts b/test/pre-test-scripts.ts
@@ -41,6 +41,8 @@ import { createDonationethUser1701756190381 } from '../migration/1701756190381-c
 import { ChainType } from '../src/types/network';
 import { COINGECKO_TOKEN_IDS } from '../src/adapters/price/CoingeckoPriceAdapter';
 import { ProjectActualMatchinView151713700147145 } from '../migration/1713700147145-project_actual_matchin_view_15';
+import { EnablePgTrgmExtension1713859866338 } from '../migration/1713859866338-enable_pg_trgm_extension';
+import { AddPgTrgmIndexes1715086559930 } from '../migration/1715086559930-add_pg_trgm_indexes';
 
 async function seedDb() {
   await seedUsers();
@@ -479,6 +481,8 @@ async function runMigrations() {
     );
     await new createDonationethUser1701756190381().up(queryRunner);
     await new ProjectActualMatchinView151713700147145().up(queryRunner);
+    await new EnablePgTrgmExtension1713859866338().up(queryRunner);
+    await new AddPgTrgmIndexes1715086559930().up(queryRunner);
   } finally {
     await queryRunner.release();
   }