From 7db097511b45028ab4c8f84adc299481ec42b946 Mon Sep 17 00:00:00 2001 From: Amin Latifi Date: Tue, 7 May 2024 14:15:38 +0000 Subject: [PATCH] Fix/word similarity - staging (#1546) * Hotfix db improvements (#1523) (#1524) * add extra configurations for postgresql connections * add master and slave replication strategy for typeorm Co-authored-by: CarlosQ96 <92376054+CarlosQ96@users.noreply.github.com> * Fix/db replica production (#1525) * Hotfix db improvements (#1523) * add extra configurations for postgresql connections * add master and slave replication strategy for typeorm * Define db read only configs --------- Co-authored-by: CarlosQ96 <92376054+CarlosQ96@users.noreply.github.com> * update comment * Hotfix latency issues for prod (#1529) * Hotfix staging fix latency (#1528) * add project donation summary view entity * convert projectQueries to querybuilder * add cache to projectDonationSummary queries * add configurable cache to slow queries * remove massive recurring donation log * add await for project queries * Add informative logs for draft donation service job (#1537) * Fix eslint errors * Fix/master test (#1541) * Fixed master test issue * Returned test to master pipeline * Comment executing donation summary view --------- Co-authored-by: Mohammad Ranjbar Z * Fixed word similarity issue * Removed unused import --------- Co-authored-by: CarlosQ96 <92376054+CarlosQ96@users.noreply.github.com> Co-authored-by: Carlos Co-authored-by: mohammadranjbarz --- .../1715086559930-add_pg_trgm_indexes.ts | 29 ++++++++++ src/entities/project.ts | 3 ++ src/entities/user.ts | 2 + src/resolvers/projectResolver.ts | 54 +++++++------------ src/server/bootstrap.ts | 15 ++++++ test/pre-test-scripts.ts | 4 ++ 6 files changed, 73 insertions(+), 34 deletions(-) create mode 100644 migration/1715086559930-add_pg_trgm_indexes.ts diff --git a/migration/1715086559930-add_pg_trgm_indexes.ts b/migration/1715086559930-add_pg_trgm_indexes.ts new file mode 100644 index 000000000..57890bf58 --- /dev/null +++ b/migration/1715086559930-add_pg_trgm_indexes.ts @@ -0,0 +1,29 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class AddPgTrgmIndexes1715086559930 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + 'CREATE INDEX if not exists trgm_idx_project_title ON project USING GIN (title gin_trgm_ops);', + ); + await queryRunner.query( + 'CREATE INDEX if not exists trgm_idx_project_description ON project USING GIN (description gin_trgm_ops);', + ); + await queryRunner.query( + 'CREATE INDEX if not exists trgm_idx_project_impact_location ON project USING GIN ("impactLocation" gin_trgm_ops);', + ); + await queryRunner.query( + 'CREATE INDEX if not exists trgm_idx_user_name ON public.user USING GIN ("name" gin_trgm_ops);', + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query('drop index if exists trgm_idx_project_title;'); + await queryRunner.query( + 'drop index if exists trgm_idx_project_description;', + ); + await queryRunner.query( + 'drop index if exists trgm_idx_project_impact_location;', + ); + await queryRunner.query('drop index if exists trgm_idx_user_name;'); + } +} diff --git a/src/entities/project.ts b/src/entities/project.ts index 6076ed6cd..6ff2984ae 100644 --- a/src/entities/project.ts +++ b/src/entities/project.ts @@ -210,6 +210,7 @@ export class Project extends BaseEntity { @Column({ nullable: true }) image?: string; + @Index('trgm_idx_project_impact_location', { synchronize: false }) @Field({ nullable: true }) @Column({ nullable: true }) impactLocation?: string; @@ -606,6 +607,7 @@ export class ProjectUpdate extends BaseEntity { @PrimaryGeneratedColumn() readonly id: number; + @Index('trgm_idx_user_name', { synchronize: false }) @Field(_type => String) @Column() title: string; @@ -667,6 +669,7 @@ export class ProjectUpdate extends BaseEntity { @Column('text', { nullable: true }) organizationWebsite: string; + @Index('trgm_idx_project_description', { synchronize: false }) @Field({ nullable: true }) @Column('text', { nullable: true }) organizationDescription: string; diff --git a/src/entities/user.ts b/src/entities/user.ts index ed26692b0..7954e76aa 100644 --- a/src/entities/user.ts +++ b/src/entities/user.ts @@ -4,6 +4,7 @@ import { Column, CreateDateColumn, Entity, + Index, JoinTable, ManyToMany, OneToMany, @@ -86,6 +87,7 @@ export class User extends BaseEntity { @Column({ nullable: true }) lastName?: string; + @Index('trgm_idx_project_impact_location', { synchronize: false }) @Field(_type => String, { nullable: true }) @Column({ nullable: true }) name?: string; diff --git a/src/resolvers/projectResolver.ts b/src/resolvers/projectResolver.ts index 6af8c1a95..fb38730ed 100644 --- a/src/resolvers/projectResolver.ts +++ b/src/resolvers/projectResolver.ts @@ -40,7 +40,6 @@ import { Donation } from '../entities/donation'; import { ProjectImage } from '../entities/projectImage'; import { ApolloContext } from '../types/ApolloContext'; import { publicSelectionFields, User } from '../entities/user'; -import config from '../config'; import { Context } from '../context'; import SentryLogger from '../sentryLogger'; import { @@ -319,40 +318,27 @@ export class ProjectResolver { searchTerm?: string, ) { if (!searchTerm) return query; - const similarityThreshold = - Number(config.get('PROJECT_SEARCH_SIMILARITY_THRESHOLD')) || 0.4; - - return query.andWhere( - new Brackets(qb => { - qb.where( - 'WORD_SIMILARITY(project.title, :searchTerm) > :similarityThreshold', - { - searchTerm: `%${searchTerm}%`, - similarityThreshold, - }, + + return ( + query + // For future use! This is the way to use similarity in TypeORM + // .addSelect('similarity(project.title, :searchTerm)', 'title_slm') + // .addSelect('similarity(project.description, :searchTerm)', 'desc_slm') + // .addSelect('similarity(project.impactLocation, :searchTerm)', 'loc_slm') + // .setParameter('searchTerm', searchTerm) + .andWhere( + new Brackets(qb => { + qb.where('project.title % :searchTerm ', { + searchTerm, + }) + .orWhere('project.description % :searchTerm ', { + searchTerm, + }) + .orWhere('project.impactLocation % :searchTerm', { + searchTerm, + }); + }), ) - .orWhere( - 'WORD_SIMILARITY(project.description, :searchTerm) > :similarityThreshold', - { - searchTerm: `%${searchTerm}%`, - similarityThreshold, - }, - ) - .orWhere( - 'WORD_SIMILARITY(project.impactLocation, :searchTerm) > :similarityThreshold', - { - searchTerm: `%${searchTerm}%`, - similarityThreshold, - }, - ) - .orWhere( - 'WORD_SIMILARITY(user.name, :searchTerm) > :similarityThreshold', - { - searchTerm: `%${searchTerm}%`, - similarityThreshold, - }, - ); - }), ); } diff --git a/src/server/bootstrap.ts b/src/server/bootstrap.ts index 5dfa2b249..e4b0334d1 100644 --- a/src/server/bootstrap.ts +++ b/src/server/bootstrap.ts @@ -98,6 +98,8 @@ export async function bootstrap() { Container.set(DataSource, AppDataSource.getDataSource()); + await setDatabaseParameters(AppDataSource.getDataSource()); + const dropSchema = config.get('DROP_DATABASE') === 'true'; if (dropSchema) { // eslint-disable-next-line no-console @@ -445,3 +447,16 @@ export async function bootstrap() { await initializeCronJobs(); } } + +async function setDatabaseParameters(ds: DataSource) { + await setPgTrgmParameters(ds); +} + +async function setPgTrgmParameters(ds: DataSource) { + const similarityThreshold = + Number(config.get('PROJECT_SEARCH_SIMILARITY_THRESHOLD')) || 0.1; + await ds.query(`SET pg_trgm.similarity_threshold TO ${similarityThreshold};`); + await ds.query( + `SET pg_trgm.word_similarity_threshold TO ${similarityThreshold};`, + ); +} diff --git a/test/pre-test-scripts.ts b/test/pre-test-scripts.ts index 1051749eb..b33f1c2f8 100644 --- a/test/pre-test-scripts.ts +++ b/test/pre-test-scripts.ts @@ -41,6 +41,8 @@ import { createDonationethUser1701756190381 } from '../migration/1701756190381-c import { ChainType } from '../src/types/network'; import { COINGECKO_TOKEN_IDS } from '../src/adapters/price/CoingeckoPriceAdapter'; import { ProjectActualMatchinView151713700147145 } from '../migration/1713700147145-project_actual_matchin_view_15'; +import { EnablePgTrgmExtension1713859866338 } from '../migration/1713859866338-enable_pg_trgm_extension'; +import { AddPgTrgmIndexes1715086559930 } from '../migration/1715086559930-add_pg_trgm_indexes'; async function seedDb() { await seedUsers(); @@ -479,6 +481,8 @@ async function runMigrations() { ); await new createDonationethUser1701756190381().up(queryRunner); await new ProjectActualMatchinView151713700147145().up(queryRunner); + await new EnablePgTrgmExtension1713859866338().up(queryRunner); + await new AddPgTrgmIndexes1715086559930().up(queryRunner); } finally { await queryRunner.release(); }