From 88ad0b1ed07204d8b2318d269a30b756d8a4d75d Mon Sep 17 00:00:00 2001 From: Ruben Date: Mon, 9 Dec 2024 18:50:55 +0100 Subject: [PATCH 1/2] Prototype find duplicates on the fly --- .../registration-profile.component.html | 76 ++++ .../registration-profile.component.ts | 22 ++ .../services/programs-service-api.service.ts | 24 ++ .../migration/1733229093775-duplicate-view.ts | 60 +++ ...1733401228093-registration_unique_pairs.ts | 45 +++ ...5114-take-into-account-mark-as-unique-2.ts | 38 ++ .../program-attributes.service.ts | 1 + .../const/filter-operation.const.ts | 4 + .../registration-unique-pair.entity.ts | 22 ++ .../registration/registration-view.entity.ts | 41 ++ .../registration/registrations.controller.ts | 46 +++ .../src/registration/registrations.module.ts | 2 + .../src/registration/registrations.service.ts | 162 +++++++- .../registrations-pagination.service.ts | 3 +- .../src/scripts/seed-mock-helpers.ts | 24 ++ .../src/scripts/seed-multiple-nlrc-mock.ts | 1 + .../scripts/sql/mock-introduce-duplicates.sql | 26 ++ .../scripts/sql/mock-make-phone-unique.sql | 2 +- .../seed-data/program/program-nlrc-pv.json | 17 +- services/121-service/swagger.json | 361 +++++------------- 20 files changed, 701 insertions(+), 276 deletions(-) create mode 100644 services/121-service/src/migration/1733229093775-duplicate-view.ts create mode 100644 services/121-service/src/migration/1733401228093-registration_unique_pairs.ts create mode 100644 services/121-service/src/migration/1733766255114-take-into-account-mark-as-unique-2.ts create mode 100644 services/121-service/src/registration/registration-unique-pair.entity.ts create mode 100644 services/121-service/src/scripts/sql/mock-introduce-duplicates.sql diff --git a/interfaces/Portal/src/app/components/registration-profile/registration-profile.component.html b/interfaces/Portal/src/app/components/registration-profile/registration-profile.component.html index f4b9839d86..63da3c6957 100644 --- a/interfaces/Portal/src/app/components/registration-profile/registration-profile.component.html +++ b/interfaces/Portal/src/app/components/registration-profile/registration-profile.component.html @@ -8,6 +8,82 @@ [sizeMd]="5" [sizeLg]="4" > + + + Duplicate Registrations + + + + + Duplicate Registration ID + Value + Attribute Name + Action + + + + + + {{ duplicate.duplicateRegistrationProgramId }} + + + {{ duplicate.value }} + {{ duplicate.name }} + + + Mark as Unique + + + + + + + + + + + Fuzzy Duplicate Registrations + + + + + Fuzzy Match Score + Duplicate Registration ID + Value + Attribute Name + + + + {{ fuzzyDuplicate.fuzzyMatchScore }} + + + {{ fuzzyDuplicate.duplicateRegistrationProgramId }} + + + {{ fuzzyDuplicate.value }} + {{ fuzzyDuplicate.name }} + + + + + + event.type === EventEnum.registrationStatusChange, ); + const duplicateObject = await this.programsService.getRegistrationDuplicate( + this.program.id, + this.person.referenceId, + ); + this.duplicates = duplicateObject.duplicates; + this.fuzzyDuplicates = duplicateObject.fuzzyDuplicates; + console.log( + '🚀 ~ RegistrationProfileComponent ~ ngOnInit ~ this.duplicates:', + this.duplicates, + ); } public canViewPhysicalCards(programId: number): boolean { @@ -64,6 +77,15 @@ export class RegistrationProfileComponent implements OnInit { ]); } + public async postUnique(referenceIdDuplicate: string) { + await this.programsService.postUnique( + this.program.id, + this.person.referenceId, + referenceIdDuplicate, + ); + window.location.reload(); + } + public fspHasPhysicalCardSupport( fspName: Person['financialServiceProviderName'], ): boolean { diff --git a/interfaces/Portal/src/app/services/programs-service-api.service.ts b/interfaces/Portal/src/app/services/programs-service-api.service.ts index 5caa76f021..492aba63eb 100644 --- a/interfaces/Portal/src/app/services/programs-service-api.service.ts +++ b/interfaces/Portal/src/app/services/programs-service-api.service.ts @@ -537,6 +537,30 @@ export class ProgramsServiceApiService { ); } + public getRegistrationDuplicate( + programId: number, + referenceId: string, + ): Promise { + return this.apiService.get( + environment.url_121_service_api, + `/programs/${programId}/registrations/${referenceId}/duplicates`, + ); + } + + public postUnique( + programId: number, + referenceId1: string, + referenceId2: string, + ): Promise { + return this.apiService.post( + environment.url_121_service_api, + `/programs/${programId}/registrations/${referenceId1}/uniques`, + { + referenceId: referenceId2, + }, + ); + } + public async getUpdateWalletAndCards( programId: number, referenceId: string, diff --git a/services/121-service/src/migration/1733229093775-duplicate-view.ts b/services/121-service/src/migration/1733229093775-duplicate-view.ts new file mode 100644 index 0000000000..d0c1f49ee5 --- /dev/null +++ b/services/121-service/src/migration/1733229093775-duplicate-view.ts @@ -0,0 +1,60 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class DuplicateView1733229093775 implements MigrationInterface { + name = 'DuplicateView1733229093775'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `DELETE FROM "121-service"."typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "schema" = $3`, + ['VIEW', 'registration_view', '121-service'], + ); + await queryRunner.query(`CREATE EXTENSION IF NOT EXISTS pg_trgm`); // For similarity postgres function + await queryRunner.query(`CREATE EXTENSION IF NOT EXISTS fuzzystrmatch;`); // For levenshtein distance + await queryRunner.query(`DROP VIEW "121-service"."registration_view"`); + await queryRunner.query(`CREATE VIEW "121-service"."registration_view" AS SELECT "registration"."id" AS "id", "registration"."created" AS "registrationCreated", "registration"."programId" AS "programId", "registration"."registrationStatus" AS "status", "registration"."referenceId" AS "referenceId", "registration"."phoneNumber" AS "phoneNumber", "registration"."preferredLanguage" AS "preferredLanguage", "registration"."inclusionScore" AS "inclusionScore", "registration"."paymentAmountMultiplier" AS "paymentAmountMultiplier", "registration"."maxPayments" AS "maxPayments", "registration"."paymentCount" AS "paymentCount", "registration"."scope" AS "scope", "fspconfig"."label" AS "programFinancialServiceProviderConfigurationLabel", CAST(CONCAT('PA #',registration."registrationProgramId") as VARCHAR) AS "personAffectedSequence", registration."registrationProgramId" AS "registrationProgramId", TO_CHAR("registration"."created",'yyyy-mm-dd') AS "registrationCreatedDate", fspconfig."name" AS "programFinancialServiceProviderConfigurationName", fspconfig."id" AS "programFinancialServiceProviderConfigurationId", fspconfig."financialServiceProviderName" AS "financialServiceProviderName", "registration"."maxPayments" - "registration"."paymentCount" AS "paymentCountRemaining", COALESCE("message"."type" || ': ' || "message"."status",'no messages yet') AS "lastMessageStatus", (CASE + WHEN "registration"."id" IN ( + SELECT + d1."registrationId" + FROM + "121-service".registration_attribute_data d1 + JOIN ( + SELECT + "programRegistrationAttributeId", + value + FROM + "121-service".registration_attribute_data rad + LEFT JOIN + "121-service".program_registration_attribute pra ON pra.id = rad."programRegistrationAttributeId" + WHERE + value != '' AND pra."duplicateCheck" + GROUP BY + "programRegistrationAttributeId", + value + HAVING + COUNT(*) > 1 + ) d2 + ON + d1."programRegistrationAttributeId" = d2."programRegistrationAttributeId" + AND d1.value = d2.value + ) THEN TRUE + ELSE FALSE + END) AS "isDuplicate" FROM "121-service"."registration" "registration" LEFT JOIN "121-service"."program_financial_service_provider_configuration" "fspconfig" ON "fspconfig"."id"="registration"."programFinancialServiceProviderConfigurationId" LEFT JOIN "121-service"."latest_message" "latestMessage" ON "latestMessage"."registrationId"="registration"."id" LEFT JOIN "121-service"."twilio_message" "message" ON "message"."id"="latestMessage"."messageId" ORDER BY "registration"."registrationProgramId" ASC`); + await queryRunner.query( + `INSERT INTO "121-service"."typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES (DEFAULT, $1, DEFAULT, $2, $3, $4)`, + [ + '121-service', + 'VIEW', + 'registration_view', + 'SELECT "registration"."id" AS "id", "registration"."created" AS "registrationCreated", "registration"."programId" AS "programId", "registration"."registrationStatus" AS "status", "registration"."referenceId" AS "referenceId", "registration"."phoneNumber" AS "phoneNumber", "registration"."preferredLanguage" AS "preferredLanguage", "registration"."inclusionScore" AS "inclusionScore", "registration"."paymentAmountMultiplier" AS "paymentAmountMultiplier", "registration"."maxPayments" AS "maxPayments", "registration"."paymentCount" AS "paymentCount", "registration"."scope" AS "scope", "fspconfig"."label" AS "programFinancialServiceProviderConfigurationLabel", CAST(CONCAT(\'PA #\',registration."registrationProgramId") as VARCHAR) AS "personAffectedSequence", registration."registrationProgramId" AS "registrationProgramId", TO_CHAR("registration"."created",\'yyyy-mm-dd\') AS "registrationCreatedDate", fspconfig."name" AS "programFinancialServiceProviderConfigurationName", fspconfig."id" AS "programFinancialServiceProviderConfigurationId", fspconfig."financialServiceProviderName" AS "financialServiceProviderName", "registration"."maxPayments" - "registration"."paymentCount" AS "paymentCountRemaining", COALESCE("message"."type" || \': \' || "message"."status",\'no messages yet\') AS "lastMessageStatus", (CASE\n WHEN "registration"."id" IN (\n SELECT\n d1."registrationId"\n FROM\n "121-service".registration_attribute_data d1\n JOIN (\n SELECT\n "programRegistrationAttributeId",\n value\n FROM\n "121-service".registration_attribute_data rad\n LEFT JOIN\n "121-service".program_registration_attribute pra ON pra.id = rad.id\n WHERE\n value != \'\' AND pra."duplicateCheck"\n GROUP BY\n "programRegistrationAttributeId",\n value\n HAVING\n COUNT(*) > 1\n ) d2\n ON\n d1."programRegistrationAttributeId" = d2."programRegistrationAttributeId"\n AND d1.value = d2.value\n ) THEN TRUE\n ELSE FALSE\n END) AS "isDuplicate" FROM "121-service"."registration" "registration" LEFT JOIN "121-service"."program_financial_service_provider_configuration" "fspconfig" ON "fspconfig"."id"="registration"."programFinancialServiceProviderConfigurationId" LEFT JOIN "121-service"."latest_message" "latestMessage" ON "latestMessage"."registrationId"="registration"."id" LEFT JOIN "121-service"."twilio_message" "message" ON "message"."id"="latestMessage"."messageId" ORDER BY "registration"."registrationProgramId" ASC', + ], + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `DELETE FROM "121-service"."typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "schema" = $3`, + ['VIEW', 'registration_view', '121-service'], + ); + await queryRunner.query(`DROP VIEW "121-service"."registration_view"`); + } +} diff --git a/services/121-service/src/migration/1733401228093-registration_unique_pairs.ts b/services/121-service/src/migration/1733401228093-registration_unique_pairs.ts new file mode 100644 index 0000000000..eb8033729b --- /dev/null +++ b/services/121-service/src/migration/1733401228093-registration_unique_pairs.ts @@ -0,0 +1,45 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class RegistrationUniquePairs1733401228093 + implements MigrationInterface +{ + name = 'RegistrationUniquePairs1733401228093'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `CREATE TABLE "121-service"."registration_unique_pairs" ("id" SERIAL NOT NULL, "created" TIMESTAMP NOT NULL DEFAULT now(), "updated" TIMESTAMP NOT NULL DEFAULT now(), "registrationSmallerId" integer NOT NULL, "registrationLargerId" integer NOT NULL, CONSTRAINT "UQ_baca30bd87b6df409d332675954" UNIQUE ("registrationSmallerId", "registrationLargerId"), CONSTRAINT "PK_953444295386f708a5f10d4a2df" PRIMARY KEY ("id"))`, + ); + await queryRunner.query( + `CREATE INDEX "IDX_093e666fd0931ed9dadbe3b81d" ON "121-service"."registration_unique_pairs" ("created") `, + ); + await queryRunner.query( + `CREATE INDEX "IDX_027ea55217d7fe593fba98fcd1" ON "121-service"."registration_unique_pairs" ("registrationSmallerId") `, + ); + await queryRunner.query( + `CREATE INDEX "IDX_835621467e670d1d4e51d7a850" ON "121-service"."registration_unique_pairs" ("registrationLargerId") `, + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `DELETE FROM "121-service"."typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "schema" = $3`, + ['VIEW', 'registration_view', '121-service'], + ); + await queryRunner.query(`DROP VIEW "121-service"."registration_view"`); + await queryRunner.query( + `DROP INDEX "121-service"."IDX_835621467e670d1d4e51d7a850"`, + ); + await queryRunner.query( + `DROP INDEX "121-service"."IDX_027ea55217d7fe593fba98fcd1"`, + ); + await queryRunner.query( + `DROP INDEX "121-service"."IDX_093e666fd0931ed9dadbe3b81d"`, + ); + await queryRunner.query( + `DROP TABLE "121-service"."registration_unique_pairs"`, + ); + await queryRunner.query( + `CREATE INDEX "registration_attribute_data_value_idx" ON "121-service"."registration_attribute_data" ("value") `, + ); + } +} diff --git a/services/121-service/src/migration/1733766255114-take-into-account-mark-as-unique-2.ts b/services/121-service/src/migration/1733766255114-take-into-account-mark-as-unique-2.ts new file mode 100644 index 0000000000..9467ded859 --- /dev/null +++ b/services/121-service/src/migration/1733766255114-take-into-account-mark-as-unique-2.ts @@ -0,0 +1,38 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class TakeIntoAccountMarkAsUnique21733766255114 + implements MigrationInterface +{ + name = 'TakeIntoAccountMarkAsUnique21733766255114'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `DELETE FROM "121-service"."typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "schema" = $3`, + ['VIEW', 'registration_view', '121-service'], + ); + await queryRunner.query(`DROP VIEW "121-service"."registration_view"`); + await queryRunner.query(`CREATE VIEW "121-service"."registration_view" AS SELECT "registration"."id" AS "id", "registration"."created" AS "registrationCreated", "registration"."programId" AS "programId", "registration"."registrationStatus" AS "status", "registration"."referenceId" AS "referenceId", "registration"."phoneNumber" AS "phoneNumber", "registration"."preferredLanguage" AS "preferredLanguage", "registration"."inclusionScore" AS "inclusionScore", "registration"."paymentAmountMultiplier" AS "paymentAmountMultiplier", "registration"."maxPayments" AS "maxPayments", "registration"."paymentCount" AS "paymentCount", "registration"."scope" AS "scope", "fspconfig"."label" AS "programFinancialServiceProviderConfigurationLabel", CAST(CONCAT('PA #',registration."registrationProgramId") as VARCHAR) AS "personAffectedSequence", registration."registrationProgramId" AS "registrationProgramId", TO_CHAR("registration"."created",'yyyy-mm-dd') AS "registrationCreatedDate", fspconfig."name" AS "programFinancialServiceProviderConfigurationName", fspconfig."id" AS "programFinancialServiceProviderConfigurationId", fspconfig."financialServiceProviderName" AS "financialServiceProviderName", "registration"."maxPayments" - "registration"."paymentCount" AS "paymentCountRemaining", COALESCE("message"."type" || ': ' || "message"."status",'no messages yet') AS "lastMessageStatus", + (CASE + WHEN dup."registrationId" IS NOT NULL THEN TRUE + ELSE FALSE + END) + AS "isDuplicate" FROM "121-service"."registration" "registration" LEFT JOIN "121-service"."program_financial_service_provider_configuration" "fspconfig" ON "fspconfig"."id"="registration"."programFinancialServiceProviderConfigurationId" LEFT JOIN "121-service"."latest_message" "latestMessage" ON "latestMessage"."registrationId"="registration"."id" LEFT JOIN "121-service"."twilio_message" "message" ON "message"."id"="latestMessage"."messageId" LEFT JOIN (SELECT distinct d1."registrationId" FROM "121-service"."registration_attribute_data" "d1" INNER JOIN "121-service"."registration_attribute_data" "d2" ON d1."programRegistrationAttributeId" = d2."programRegistrationAttributeId" AND "d1"."value" = "d2"."value" AND d1."registrationId" != d2."registrationId" INNER JOIN "121-service"."program_registration_attribute" "pra" ON d1."programRegistrationAttributeId" = "pra"."id" WHERE "d1"."value" != '' AND pra."duplicateCheck" = true AND NOT EXISTS (SELECT 1 FROM "121-service".registration_unique_pairs rup WHERE (rup."registrationSmallerId" = LEAST(d1."registrationId", d2."registrationId") AND rup."registrationLargerId" = GREATEST(d1."registrationId", d2."registrationId")))) "dup" ON "registration"."id" = dup."registrationId" ORDER BY "registration"."registrationProgramId" ASC`); + await queryRunner.query( + `INSERT INTO "121-service"."typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES (DEFAULT, $1, DEFAULT, $2, $3, $4)`, + [ + '121-service', + 'VIEW', + 'registration_view', + 'SELECT "registration"."id" AS "id", "registration"."created" AS "registrationCreated", "registration"."programId" AS "programId", "registration"."registrationStatus" AS "status", "registration"."referenceId" AS "referenceId", "registration"."phoneNumber" AS "phoneNumber", "registration"."preferredLanguage" AS "preferredLanguage", "registration"."inclusionScore" AS "inclusionScore", "registration"."paymentAmountMultiplier" AS "paymentAmountMultiplier", "registration"."maxPayments" AS "maxPayments", "registration"."paymentCount" AS "paymentCount", "registration"."scope" AS "scope", "fspconfig"."label" AS "programFinancialServiceProviderConfigurationLabel", CAST(CONCAT(\'PA #\',registration."registrationProgramId") as VARCHAR) AS "personAffectedSequence", registration."registrationProgramId" AS "registrationProgramId", TO_CHAR("registration"."created",\'yyyy-mm-dd\') AS "registrationCreatedDate", fspconfig."name" AS "programFinancialServiceProviderConfigurationName", fspconfig."id" AS "programFinancialServiceProviderConfigurationId", fspconfig."financialServiceProviderName" AS "financialServiceProviderName", "registration"."maxPayments" - "registration"."paymentCount" AS "paymentCountRemaining", COALESCE("message"."type" || \': \' || "message"."status",\'no messages yet\') AS "lastMessageStatus", \n (CASE\n WHEN dup."registrationId" IS NOT NULL THEN TRUE\n ELSE FALSE\n END)\n AS "isDuplicate" FROM "121-service"."registration" "registration" LEFT JOIN "121-service"."program_financial_service_provider_configuration" "fspconfig" ON "fspconfig"."id"="registration"."programFinancialServiceProviderConfigurationId" LEFT JOIN "121-service"."latest_message" "latestMessage" ON "latestMessage"."registrationId"="registration"."id" LEFT JOIN "121-service"."twilio_message" "message" ON "message"."id"="latestMessage"."messageId" LEFT JOIN (SELECT distinct d1."registrationId" FROM "121-service"."registration_attribute_data" "d1" INNER JOIN "121-service"."registration_attribute_data" "d2" ON d1."programRegistrationAttributeId" = d2."programRegistrationAttributeId" AND "d1"."value" = "d2"."value" AND d1."registrationId" != d2."registrationId" INNER JOIN "121-service"."program_registration_attribute" "pra" ON d1."programRegistrationAttributeId" = "pra"."id" WHERE "d1"."value" != \'\' AND pra."duplicateCheck" = true AND NOT EXISTS (SELECT 1 FROM "121-service".registration_unique_pairs rup WHERE (rup."registrationSmallerId" = LEAST(d1."registrationId", d2."registrationId") AND rup."registrationLargerId" = GREATEST(d1."registrationId", d2."registrationId")))) "dup" ON "registration"."id" = dup."registrationId" ORDER BY "registration"."registrationProgramId" ASC', + ], + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `DELETE FROM "121-service"."typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "schema" = $3`, + ['VIEW', 'registration_view', '121-service'], + ); + await queryRunner.query(`DROP VIEW "121-service"."registration_view"`); + } +} diff --git a/services/121-service/src/program-attributes/program-attributes.service.ts b/services/121-service/src/program-attributes/program-attributes.service.ts index 40580e819b..455f5219a6 100644 --- a/services/121-service/src/program-attributes/program-attributes.service.ts +++ b/services/121-service/src/program-attributes/program-attributes.service.ts @@ -34,6 +34,7 @@ export class ProgramAttributesService { 'inclusionScore', 'paymentAmountMultiplier', 'financialServiceProvider', + 'isDuplicate', ]; const paAttributesNameArray = program['paTableAttributes'].map( (paAttribute: Attribute) => paAttribute.name, diff --git a/services/121-service/src/registration/const/filter-operation.const.ts b/services/121-service/src/registration/const/filter-operation.const.ts index 557c4cc6a1..2627b490da 100644 --- a/services/121-service/src/registration/const/filter-operation.const.ts +++ b/services/121-service/src/registration/const/filter-operation.const.ts @@ -14,6 +14,8 @@ export const AllowedFilterOperatorsNumber = [ FilterOperator.NULL, ]; +export const AllowedFilterOperatorsBoolean = [FilterOperator.EQ]; + const dataSearchableColumn = 'data.value'; const basePaginateConfigRegistrationView: PaginateConfig = { @@ -37,6 +39,7 @@ const basePaginateConfigRegistrationView: PaginateConfig 'paymentCount', 'paymentCountRemaining', 'lastMessageStatus', + 'isDuplicate', 'data.value', ], filterableColumns: { @@ -59,6 +62,7 @@ const basePaginateConfigRegistrationView: PaginateConfig paymentCountRemaining: AllowedFilterOperatorsNumber, personAffectedSequence: AllowedFilterOperatorsString, lastMessageStatus: AllowedFilterOperatorsString, + isDuplicate: AllowedFilterOperatorsBoolean, }, }; diff --git a/services/121-service/src/registration/registration-unique-pair.entity.ts b/services/121-service/src/registration/registration-unique-pair.entity.ts new file mode 100644 index 0000000000..d601247287 --- /dev/null +++ b/services/121-service/src/registration/registration-unique-pair.entity.ts @@ -0,0 +1,22 @@ +import { Column, Entity, Index, JoinColumn, Relation, Unique } from 'typeorm'; + +import { Base121Entity } from '@121-service/src/base.entity'; +import { RegistrationEntity } from '@121-service/src/registration/registration.entity'; + +// TODO needs unique constraint on the combination of registrationSmallerId and registrationLargerId + +@Entity('registration_unique_pairs') +@Unique(['registrationSmallerId', 'registrationLargerId']) +export class RegistrationUniquePairEntity extends Base121Entity { + @JoinColumn({ name: 'registrationSmallerId' }) + public registrationWithSmallerId: Relation; + @Index() + @Column({ type: 'int', nullable: false }) + public registrationSmallerId: number; + + @JoinColumn({ name: 'registrationLargerId' }) + public registrationWithLargerId: Relation; + @Index() + @Column({ type: 'int', nullable: false }) + public registrationLargerId: number; +} diff --git a/services/121-service/src/registration/registration-view.entity.ts b/services/121-service/src/registration/registration-view.entity.ts index 8aa418d02e..aae7741314 100644 --- a/services/121-service/src/registration/registration-view.entity.ts +++ b/services/121-service/src/registration/registration-view.entity.ts @@ -81,6 +81,44 @@ import { LocalizedString } from '@121-service/src/shared/types/localized-string. .addSelect( `COALESCE(message.type || ': ' || message.status,'no messages yet')`, 'lastMessageStatus', + ) + + .addSelect( + ` + (CASE + WHEN dup."registrationId" IS NOT NULL THEN TRUE + ELSE FALSE + END) + `, + 'isDuplicate', + ) + .leftJoin( + (qb) => + qb + .select('distinct d1."registrationId"') + .from('registration_attribute_data', 'd1') + .innerJoin( + 'registration_attribute_data', + 'd2', + 'd1."programRegistrationAttributeId" = d2."programRegistrationAttributeId" AND d1.value = d2.value AND d1."registrationId" != d2."registrationId"', + ) + .innerJoin( + 'program_registration_attribute', + 'pra', + 'd1."programRegistrationAttributeId" = pra.id', + ) + .where("d1.value != ''") + .andWhere('pra."duplicateCheck" = true') + .andWhere( + 'NOT EXISTS (' + + 'SELECT 1 ' + + 'FROM "121-service".registration_unique_pairs rup ' + + 'WHERE (rup."registrationSmallerId" = LEAST(d1."registrationId", d2."registrationId") ' + + 'AND rup."registrationLargerId" = GREATEST(d1."registrationId", d2."registrationId"))' + + ')', + ), + 'dup', + 'registration.id = dup."registrationId"', ), }) export class RegistrationViewEntity { @@ -152,6 +190,9 @@ export class RegistrationViewEntity { @ViewColumn() public scope: string; + @ViewColumn() + public isDuplicate: boolean; + @OneToMany( () => RegistrationAttributeDataEntity, (registrationData) => registrationData.registration, diff --git a/services/121-service/src/registration/registrations.controller.ts b/services/121-service/src/registration/registrations.controller.ts index 3c5f50a615..655db1dd6f 100644 --- a/services/121-service/src/registration/registrations.controller.ts +++ b/services/121-service/src/registration/registrations.controller.ts @@ -876,4 +876,50 @@ export class RegistrationsController { return registrationEntity; } + + @ApiTags('registration') + @AuthenticatedUser({ permissions: [PermissionEnum.RegistrationPersonalREAD] }) + @ApiOperation({ + summary: '[SCOPED] Gets duplicates fro registration', + }) + @ApiParam({ name: 'programId', required: true, type: 'integer' }) + @ApiParam({ name: 'referenceId', required: true, type: 'string' }) + @ApiResponse({ + status: HttpStatus.OK, + description: 'Duplicates', + }) + @Get('programs/:programId/registrations/:referenceId/duplicates') + public async getDuplicates( + @Param('referenceId') referenceId: string, + @Param('programId', ParseIntPipe) + programId: number, + ): Promise { + return await this.registrationsService.getDuplicates( + referenceId, + programId, + ); + } + + @AuthenticatedUser({ permissions: [PermissionEnum.RegistrationPersonalREAD] }) + @ApiOperation({ + summary: '[SCOPED] Post uniques for registration', + }) + @ApiParam({ name: 'programId', required: true, type: 'integer' }) + @ApiParam({ name: 'referenceId', required: true, type: 'string' }) + @ApiResponse({ + status: HttpStatus.OK, + description: 'Uniques', + }) + @Post('programs/:programId/registrations/:referenceId/uniques') + public async markRegistrationAsUnique( + @Param('referenceId') referenceId: string, + @Param('programId', ParseIntPipe) programId: number, + @Body() body: ReferenceIdDto, + ): Promise { + return await this.registrationsService.markRegistrationAsUnique( + referenceId, + body.referenceId, + programId, + ); + } } diff --git a/services/121-service/src/registration/registrations.module.ts b/services/121-service/src/registration/registrations.module.ts index 0a307bbc3c..6c488849cd 100644 --- a/services/121-service/src/registration/registrations.module.ts +++ b/services/121-service/src/registration/registrations.module.ts @@ -28,6 +28,7 @@ import { RegistrationUtilsModule } from '@121-service/src/registration/modules/r import { RegistrationUpdateProcessor } from '@121-service/src/registration/processsors/registrations-update.processor'; import { RegistrationEntity } from '@121-service/src/registration/registration.entity'; import { RegistrationAttributeDataEntity } from '@121-service/src/registration/registration-attribute-data.entity'; +import { RegistrationUniquePairEntity } from '@121-service/src/registration/registration-unique-pair.entity'; import { RegistrationsController } from '@121-service/src/registration/registrations.controller'; import { RegistrationsService } from '@121-service/src/registration/registrations.service'; import { RegistrationScopedRepository } from '@121-service/src/registration/repositories/registration-scoped.repository'; @@ -88,6 +89,7 @@ import { createScopedRepositoryProvider } from '@121-service/src/utils/scope/cre createScopedRepositoryProvider(NoteEntity), createScopedRepositoryProvider(TransactionEntity), createScopedRepositoryProvider(EventEntity), + createScopedRepositoryProvider(RegistrationUniquePairEntity), ], controllers: [RegistrationsController], exports: [ diff --git a/services/121-service/src/registration/registrations.service.ts b/services/121-service/src/registration/registrations.service.ts index 46b54f33b6..7bf164199b 100644 --- a/services/121-service/src/registration/registrations.service.ts +++ b/services/121-service/src/registration/registrations.service.ts @@ -1,4 +1,4 @@ -import { HttpException, HttpStatus, Injectable } from '@nestjs/common'; +import { HttpException, HttpStatus, Inject, Injectable } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; import { Equal, FindOneOptions, Repository } from 'typeorm'; @@ -49,16 +49,20 @@ import { RegistrationDataService } from '@121-service/src/registration/modules/r import { RegistrationDataScopedRepository } from '@121-service/src/registration/modules/registration-data/repositories/registration-data.scoped.repository'; import { RegistrationUtilsService } from '@121-service/src/registration/modules/registration-utilts/registration-utils.service'; import { RegistrationEntity } from '@121-service/src/registration/registration.entity'; +import { RegistrationAttributeDataEntity } from '@121-service/src/registration/registration-attribute-data.entity'; +import { RegistrationUniquePairEntity } from '@121-service/src/registration/registration-unique-pair.entity'; import { RegistrationScopedRepository } from '@121-service/src/registration/repositories/registration-scoped.repository'; import { RegistrationViewScopedRepository } from '@121-service/src/registration/repositories/registration-view-scoped.repository'; import { InclusionScoreService } from '@121-service/src/registration/services/inclusion-score.service'; import { RegistrationsImportService } from '@121-service/src/registration/services/registrations-import.service'; import { RegistrationsPaginationService } from '@121-service/src/registration/services/registrations-pagination.service'; import { RegistrationsInputValidator } from '@121-service/src/registration/validators/registrations-input-validator'; +import { ScopedRepository } from '@121-service/src/scoped.repository'; import { PermissionEnum } from '@121-service/src/user/enum/permission.enum'; import { UserEntity } from '@121-service/src/user/user.entity'; import { UserService } from '@121-service/src/user/user.service'; import { convertToScopedOptions } from '@121-service/src/utils/scope/createFindWhereOptions.helper'; +import { getScopedRepositoryProviderName } from '@121-service/src/utils/scope/createScopedRepositoryProvider.helper'; @Injectable() export class RegistrationsService { @@ -85,6 +89,8 @@ export class RegistrationsService { private readonly programFinancialServiceProviderConfigurationRepository: ProgramFinancialServiceProviderConfigurationRepository, private readonly registrationDataScopedRepository: RegistrationDataScopedRepository, private readonly registrationsInputValidator: RegistrationsInputValidator, + @Inject(getScopedRepositoryProviderName(RegistrationUniquePairEntity)) + private readonly registrationUniquePairScopedRepository: ScopedRepository, ) {} // This methods can be used to get the same formattted data as the pagination query using referenceId @@ -1164,4 +1170,158 @@ export class RegistrationsService { }); await this.sendContactInformationToIntersolve(registration); } + + public async getDuplicates( + referenceId: string, + programId: number, + ): Promise { + const registration = await this.getRegistrationOrThrow({ + referenceId, + programId, + }); + const registrationId = registration.id; + console.log('🚀 ~ RegistrationsService ~ registrationId:', registrationId); + + console.time('duplicates'); + const duplicates = await this.registrationDataScopedRepository + .createQueryBuilder('d1') + .select([ + 'd1."registrationId" as "registrationId"', + 'r2."referenceId" AS "duplicateReferenceId"', + 'r2."registrationProgramId" AS "duplicateRegistrationProgramId"', + 'd1.programRegistrationAttributeId as "programRegistrationAttributeId"', + 'd1.value as value', + 'pra.name AS "name"', + ]) + .innerJoin( + RegistrationAttributeDataEntity, + 'd2', + 'd1.programRegistrationAttributeId = d2.programRegistrationAttributeId AND d1.value = d2.value AND d1.registrationId != d2.registrationId', // Ensure different registrationId but same programRegistrationAttributeId and value + ) + .leftJoin(RegistrationEntity, 'r2', 'd2.registrationId = r2.id') + .leftJoin( + ProgramRegistrationAttributeEntity, + 'pra', + 'd1.programRegistrationAttributeId = pra.id', + ) + .andWhere('d1.registrationId = :registrationId', { registrationId }) + .andWhere('pra."duplicateCheck" = true') + .andWhere( + 'NOT EXISTS (' + + 'SELECT 1 ' + + 'FROM "121-service".registration_unique_pairs rup ' + + 'WHERE rup."registrationSmallerId" = LEAST(d1."registrationId", d2."registrationId") ' + + 'AND rup."registrationLargerId" = GREATEST(d1."registrationId", d2."registrationId")' + + ')', + ) + .orderBy('d1.programRegistrationAttributeId', 'ASC') + .getRawMany(); + console.timeEnd('duplicates'); + console.log('🚀 ~ RegistrationsService ~ result:', duplicates); + + console.time('fuzzyMatch levenshtein'); + const fuzzyQbLeven = await this.registrationDataScopedRepository + .createQueryBuilder('d1') + .select([ + 'd1."registrationId" as "registrationId"', + 'r2."registrationProgramId" AS "duplicateRegistrationProgramId"', + 'd1.programRegistrationAttributeId as "programRegistrationAttributeId"', + 'd1.value as value', + 'pra.name AS "name"', + 'levenshtein(d1.value, d2.value) AS "fuzzyMatchScore"', + ]) + .innerJoin( + RegistrationAttributeDataEntity, + 'd2', + 'd1.programRegistrationAttributeId = d2.programRegistrationAttributeId AND d1.registrationId != d2.registrationId', // Ensure different registrationId but same programRegistrationAttributeId + ) + .leftJoin(RegistrationEntity, 'r2', 'd2.registrationId = r2.id') + .leftJoin( + ProgramRegistrationAttributeEntity, + 'pra', + 'd1.programRegistrationAttributeId = pra.id', + ) + .andWhere('d1.registrationId = :registrationId', { registrationId }) + .andWhere('pra."duplicateCheck" = true') + // .andWhere('similarity(d1.value, d2.value) > 0.7') + .andWhere('levenshtein(d1.value, d2.value) <= 3') // Use <= for distance threshold + .orderBy('d1.programRegistrationAttributeId', 'ASC'); + + const fuzzyLevenshteinDuplicates = await fuzzyQbLeven.getRawMany(); + console.log( + '🚀 ~ RegistrationsService ~ fuzzyLevenshteinDuplicates:', + fuzzyLevenshteinDuplicates, + ); + console.timeEnd('fuzzyMatch levenshtein'); + + console.time('fuzzyMatch fuzzySimilarity'); + const fuzzySimilarityDuplicates = + await this.registrationDataScopedRepository + .createQueryBuilder('d1') + .select([ + 'd1."registrationId" as "registrationId"', + 'r2."registrationProgramId" AS "duplicateRegistrationProgramId"', + 'd1.programRegistrationAttributeId as "programRegistrationAttributeId"', + 'd1.value as value', + 'pra.name AS "name"', + 'similarity(d1.value, d2.value) AS "fuzzyMatchScore"', + ]) + .innerJoin( + RegistrationAttributeDataEntity, + 'd2', + 'd1.programRegistrationAttributeId = d2.programRegistrationAttributeId AND d1.registrationId != d2.registrationId', // Ensure different registrationId but same programRegistrationAttributeId + ) + .leftJoin(RegistrationEntity, 'r2', 'd2.registrationId = r2.id') + .leftJoin( + ProgramRegistrationAttributeEntity, + 'pra', + 'd1.programRegistrationAttributeId = pra.id', + ) + .andWhere('d1.registrationId = :registrationId', { registrationId }) + .andWhere('pra."duplicateCheck" = true') + .andWhere('similarity(d1.value, d2.value) > 0.7') + .orderBy('d1.programRegistrationAttributeId', 'ASC') + .getRawMany(); + console.timeEnd('fuzzyMatch fuzzySimilarity'); + console.log( + '🚀 ~ RegistrationsService ~ fuzzySimilarityDuplicates:', + fuzzySimilarityDuplicates, + ); + + return { + duplicates, + fuzzyDuplicates: fuzzyLevenshteinDuplicates, + }; + } + + public async markRegistrationAsUnique( + referenceId1: string, + referenceId2: string, + programId: number, + ) { + const registration1 = await this.getRegistrationOrThrow({ + referenceId: referenceId1, + programId, + }); + const registration2 = await this.getRegistrationOrThrow({ + referenceId: referenceId2, + programId, + }); + const registrationUniquePair = new RegistrationUniquePairEntity(); + if (registration1.id < registration2.id) { + registrationUniquePair.registrationWithSmallerId = registration1; + registrationUniquePair.registrationSmallerId = registration1.id; + registrationUniquePair.registrationWithLargerId = registration2; + registrationUniquePair.registrationLargerId = registration2.id; + } else { + registrationUniquePair.registrationWithSmallerId = registration2; + registrationUniquePair.registrationSmallerId = registration2.id; + registrationUniquePair.registrationWithLargerId = registration1; + registrationUniquePair.registrationLargerId = registration1.id; + } + + await this.registrationUniquePairScopedRepository.save( + registrationUniquePair, + ); + } } diff --git a/services/121-service/src/registration/services/registrations-pagination.service.ts b/services/121-service/src/registration/services/registrations-pagination.service.ts index ca77f8ed2a..bf85922fe3 100644 --- a/services/121-service/src/registration/services/registrations-pagination.service.ts +++ b/services/121-service/src/registration/services/registrations-pagination.service.ts @@ -68,6 +68,7 @@ export class RegistrationsPaginationService { noLimit: boolean, queryBuilder?: ScopedQueryBuilder, ): Promise { + console.time('getPaginate'); // Deep clone query here to prevent mutation out of this function query = structuredClone(query); @@ -190,7 +191,7 @@ export class RegistrationsPaginationService { hasPersonalReadPermission, programId, }); - + console.timeEnd('getPaginate'); return { ...result, data, diff --git a/services/121-service/src/scripts/seed-mock-helpers.ts b/services/121-service/src/scripts/seed-mock-helpers.ts index fd2eb86d25..54c305d762 100644 --- a/services/121-service/src/scripts/seed-mock-helpers.ts +++ b/services/121-service/src/scripts/seed-mock-helpers.ts @@ -4,6 +4,7 @@ import fs from 'fs'; import path from 'path'; import { AppDataSource } from '@121-service/src/appdatasource'; +import { ProgramRegistrationAttributeEntity } from '@121-service/src/programs/program-registration-attribute.entity'; import { RegistrationStatusEnum } from '@121-service/src/registration/enum/registration-status.enum'; import { CustomHttpService } from '@121-service/src/shared/services/custom-http.service'; import { AxiosCallsService } from '@121-service/src/utils/axios/axios-calls.service'; @@ -252,6 +253,29 @@ export class SeedMockHelper { console.log(`**Done updating latest message**`); } + public async introduceDuplicates(): Promise { + console.time('introduceDuplicates'); + const selectProgramRegistrationAttributesWithDuplicateCheck = + await this.dataSource.manager + .getRepository(ProgramRegistrationAttributeEntity) + .createQueryBuilder('program_registration_attribute') + .select('id') + .where('"duplicateCheck" = true') + .getRawMany(); + + for (const { + id, + } of selectProgramRegistrationAttributesWithDuplicateCheck) { + const queryIntroduceDuplicates = readSqlFile( + '../../src/scripts/sql/mock-introduce-duplicates.sql', + ); + // TODO: Could not get proper parameter to work here so resorted to string replace + const qWithParam = queryIntroduceDuplicates.replace('$1', id); + await this.dataSource.query(qWithParam); + } + console.timeEnd('introduceDuplicates'); + } + public async updateSequenceNumbers(): Promise { console.log('**Updating sequence numbers.**'); const tables = await this.dataSource.query(` diff --git a/services/121-service/src/scripts/seed-multiple-nlrc-mock.ts b/services/121-service/src/scripts/seed-multiple-nlrc-mock.ts index 03caaf4116..441f4e442d 100644 --- a/services/121-service/src/scripts/seed-multiple-nlrc-mock.ts +++ b/services/121-service/src/scripts/seed-multiple-nlrc-mock.ts @@ -67,6 +67,7 @@ export class SeedMultipleNLRCMockData implements InterfaceScript { await this.seedMockHelper.multiplyTransactions(nrPayments); await this.seedMockHelper.multiplyMessages(powerNrMessages); await this.seedMockHelper.updateSequenceNumbers(); + await this.seedMockHelper.introduceDuplicates(); } private async seedRegistrationForProgram( diff --git a/services/121-service/src/scripts/sql/mock-introduce-duplicates.sql b/services/121-service/src/scripts/sql/mock-introduce-duplicates.sql new file mode 100644 index 0000000000..4bb15f2133 --- /dev/null +++ b/services/121-service/src/scripts/sql/mock-introduce-duplicates.sql @@ -0,0 +1,26 @@ +WITH RankedRecords AS ( + SELECT id, value, "programRegistrationAttributeId", "registrationId", + ROW_NUMBER() OVER (PARTITION BY "programRegistrationAttributeId" ORDER BY id) AS rn + FROM "121-service".registration_attribute_data + WHERE "programRegistrationAttributeId" = $1 +), +MockDuplicates AS ( + UPDATE "121-service".registration_attribute_data rad1 + SET value = rr2.value + FROM RankedRecords rr1 + JOIN RankedRecords rr2 ON rr2.rn = rr1.rn + 1 + WHERE rad1.id = rr1.id + AND rr1.rn % 30 = 0 + AND rr1."programRegistrationAttributeId" = rr2."programRegistrationAttributeId" + RETURNING rr1."registrationId" AS duplicate1, rr2."registrationId" AS duplicate2 +), +RandomSample AS ( + SELECT DISTINCT LEAST(duplicate1, duplicate2) AS registrationSmallerId, + GREATEST(duplicate1, duplicate2) AS registrationLargerId + FROM MockDuplicates + WHERE RANDOM() < 0.05 +) +INSERT INTO "121-service".registration_unique_pairs ("registrationSmallerId", "registrationLargerId") +SELECT registrationSmallerId, registrationLargerId +FROM RandomSample +ON CONFLICT DO NOTHING; diff --git a/services/121-service/src/scripts/sql/mock-make-phone-unique.sql b/services/121-service/src/scripts/sql/mock-make-phone-unique.sql index 2ace1bb3ee..78843f5efc 100644 --- a/services/121-service/src/scripts/sql/mock-make-phone-unique.sql +++ b/services/121-service/src/scripts/sql/mock-make-phone-unique.sql @@ -1,6 +1,6 @@ update "121-service".registration_attribute_data set "value" = CAST(10000000000 + floor(random() * 90000000000) AS bigint) - WHERE "programRegistrationAttributeId" IN (SELECT id FROM "121-service".program_registration_attribute WHERE "name" = 'phoneNumber' OR "name" = 'whatsappPhoneNumber'); + WHERE "programRegistrationAttributeId" IN (SELECT id FROM "121-service".program_registration_attribute WHERE "name" = 'phoneNumber' OR "name" = 'whatsappPhoneNumber' OR "name" = 'nationalId' or "name" = 'fullName'); ; update "121-service".registration r set "phoneNumber" = rd."value" diff --git a/services/121-service/src/seed-data/program/program-nlrc-pv.json b/services/121-service/src/seed-data/program/program-nlrc-pv.json index 4d05e607c4..c08e390b0b 100644 --- a/services/121-service/src/seed-data/program/program-nlrc-pv.json +++ b/services/121-service/src/seed-data/program/program-nlrc-pv.json @@ -38,7 +38,8 @@ "export": ["all-people-affected", "included"], "scoring": {}, "editableInPortal": true, - "showInPeopleAffectedTable": false + "showInPeopleAffectedTable": false, + "duplicateCheck": false }, { "name": "phoneNumber", @@ -69,9 +70,21 @@ "export": ["all-people-affected", "included"], "type": "tel", "options": null, - "duplicateCheck": true, + "duplicateCheck": false, "showInPeopleAffectedTable": true }, + { + "name": "nationalId", + "label": { + "en": "Nationaal Id" + }, + "export": ["all-people-affected", "included"], + "type": "text", + "pattern": ".+", + "duplicateCheck": false, + "showInPeopleAffectedTable": true, + "editableInPortal": true + }, { "name": "addressStreet", "label": { diff --git a/services/121-service/swagger.json b/services/121-service/swagger.json index 870ad3a84c..34ba7ff0b4 100644 --- a/services/121-service/swagger.json +++ b/services/121-service/swagger.json @@ -23,17 +23,13 @@ { "method": "put", "path": "/api/roles/{userRoleId}", - "params": [ - "userRoleId" - ], + "params": ["userRoleId"], "returnType": "UserRoleResponseDTO" }, { "method": "delete", "path": "/api/roles/{userRoleId}", - "params": [ - "userRoleId" - ], + "params": ["userRoleId"], "returnType": "UserRoleResponseDTO" }, { @@ -70,17 +66,13 @@ { "method": "delete", "path": "/api/users/{userId}", - "params": [ - "userId" - ], + "params": ["userId"], "returnType": "UserEntity" }, { "method": "patch", "path": "/api/users/{userId}", - "params": [ - "userId" - ] + "params": ["userId"] }, { "method": "get", @@ -90,53 +82,36 @@ { "method": "get", "path": "/api/programs/{programId}/users/search", - "params": [ - "programId", - "username" - ] + "params": ["programId", "username"] }, { "method": "get", "path": "/api/programs/{programId}/users/{userId}", - "params": [ - "programId", - "userId" - ], + "params": ["programId", "userId"], "returnType": "AssignmentResponseDTO" }, { "method": "put", "path": "/api/programs/{programId}/users/{userId}", - "params": [ - "programId", - "userId" - ], + "params": ["programId", "userId"], "returnType": "AssignmentResponseDTO" }, { "method": "patch", "path": "/api/programs/{programId}/users/{userId}", - "params": [ - "programId", - "userId" - ], + "params": ["programId", "userId"], "returnType": "AssignmentResponseDTO" }, { "method": "delete", "path": "/api/programs/{programId}/users/{userId}", - "params": [ - "programId", - "userId" - ], + "params": ["programId", "userId"], "returnType": "AssignmentResponseDTO" }, { "method": "get", "path": "/api/programs/{programId}/users", - "params": [ - "programId" - ] + "params": ["programId"] }, { "method": "post", @@ -154,100 +129,67 @@ { "method": "post", "path": "/api/scripts/duplicate-registrations", - "params": [ - "mockPowerNumberRegistrations" - ] + "params": ["mockPowerNumberRegistrations"] }, { "method": "get", "path": "/api/notifications/{programId}/message-templates", - "params": [ - "programId" - ] + "params": ["programId"] }, { "method": "post", "path": "/api/notifications/{programId}/message-templates", - "params": [ - "programId" - ] + "params": ["programId"] }, { "method": "patch", "path": "/api/notifications/{programId}/message-templates/{type}/{language}", - "params": [ - "language", - "type", - "programId" - ], + "params": ["language", "type", "programId"], "returnType": "MessageTemplateEntity" }, { "method": "delete", "path": "/api/notifications/{programId}/message-templates/{type}", - "params": [ - "language", - "type", - "programId" - ], + "params": ["language", "type", "programId"], "returnType": "DeleteResult" }, { "method": "get", "path": "/api/programs/{programId}", - "params": [ - "programId", - "formatProgramReturnDto" - ] + "params": ["programId", "formatProgramReturnDto"] }, { "method": "delete", "path": "/api/programs/{programId}", - "params": [ - "programId" - ] + "params": ["programId"] }, { "method": "patch", "path": "/api/programs/{programId}", - "params": [ - "programId" - ], + "params": ["programId"], "returnType": "ProgramReturnDto" }, { "method": "post", "path": "/api/programs", - "params": [ - "importFromKobo", - "koboToken", - "koboAssetId" - ], + "params": ["importFromKobo", "koboToken", "koboAssetId"], "returnType": "ProgramEntity" }, { "method": "post", "path": "/api/programs/{programId}/registration-attributes", - "params": [ - "programId" - ] + "params": ["programId"] }, { "method": "patch", "path": "/api/programs/{programId}/registration-attributes/{programRegistrationAttributeName}", - "params": [ - "programId", - "programRegistrationAttributeName" - ], + "params": ["programId", "programRegistrationAttributeName"], "returnType": "ProgramRegistrationAttributeEntity" }, { "method": "delete", "path": "/api/programs/{programId}/registration-attributes/{programRegistrationAttributeId}", - "params": [ - "programId", - "programRegistrationAttributeId" - ] + "params": ["programId", "programRegistrationAttributeId"] }, { "method": "get", @@ -262,25 +204,18 @@ { "method": "get", "path": "/api/programs/{programId}/financial-service-providers/intersolve-visa/funding-wallet", - "params": [ - "programId" - ] + "params": ["programId"] }, { "method": "get", "path": "/api/programs/{programId}/actions", - "params": [ - "programId", - "actionType" - ], + "params": ["programId", "actionType"], "returnType": "ActionReturnDto" }, { "method": "post", "path": "/api/programs/{programId}/actions", - "params": [ - "programId" - ], + "params": ["programId"], "returnType": "ActionReturnDto" }, { @@ -292,94 +227,58 @@ { "method": "get", "path": "/api/financial-service-providers/{financialServiceProviderName}", - "params": [ - "financialServiceProviderName" - ], + "params": ["financialServiceProviderName"], "returnType": "FinancialServiceProviderDto" }, { "method": "get", "path": "/api/programs/{programId}/financial-service-provider-configurations", - "params": [ - "programId" - ] + "params": ["programId"] }, { "method": "post", "path": "/api/programs/{programId}/financial-service-provider-configurations", - "params": [ - "programId" - ] + "params": ["programId"] }, { "method": "patch", "path": "/api/programs/{programId}/financial-service-provider-configurations/{name}", - "params": [ - "programId", - "name" - ] + "params": ["programId", "name"] }, { "method": "delete", "path": "/api/programs/{programId}/financial-service-provider-configurations/{name}", - "params": [ - "programId", - "name" - ] + "params": ["programId", "name"] }, { "method": "post", "path": "/api/programs/{programId}/financial-service-provider-configurations/{name}/properties", - "params": [ - "programId", - "name" - ] + "params": ["programId", "name"] }, { "method": "patch", "path": "/api/programs/{programId}/financial-service-provider-configurations/{name}/properties/{propertyName}", - "params": [ - "programId", - "name", - "propertyName" - ] + "params": ["programId", "name", "propertyName"] }, { "method": "delete", "path": "/api/programs/{programId}/financial-service-provider-configurations/{name}/properties/{propertyName}", - "params": [ - "programId", - "name", - "propertyName" - ] + "params": ["programId", "name", "propertyName"] }, { "method": "get", "path": "/api/programs/{programId}/transactions", - "params": [ - "programId", - "referenceId", - "payment" - ] + "params": ["programId", "referenceId", "payment"] }, { "method": "get", "path": "/api/programs/{programId}/events", - "params": [ - "programId", - "format", - "toDate", - "fromDate", - "referenceId" - ] + "params": ["programId", "format", "toDate", "fromDate", "referenceId"] }, { "method": "get", "path": "/api/programs/{programId}/registrations/{registrationId}/events", - "params": [ - "registrationId", - "programId" - ] + "params": ["registrationId", "programId"] }, { "method": "patch", @@ -410,55 +309,37 @@ { "method": "get", "path": "/api/notifications/whatsapp/templates/{sessionId}", - "params": [ - "sessionId" - ] + "params": ["sessionId"] }, { "method": "get", "path": "/api/notifications/imageCode/{secret}", - "params": [ - "secret" - ] + "params": ["secret"] }, { "method": "get", "path": "/api/programs/{programId}/financial-service-providers/intersolve-voucher/vouchers", - "params": [ - "programId", - "referenceId", - "payment" - ] + "params": ["programId", "referenceId", "payment"] }, { "method": "get", "path": "/api/programs/{programId}/financial-service-providers/intersolve-voucher/vouchers/balance", - "params": [ - "programId", - "referenceId", - "payment" - ] + "params": ["programId", "referenceId", "payment"] }, { "method": "get", "path": "/api/programs/{programId}/financial-service-providers/intersolve-voucher/instructions", - "params": [ - "programId" - ] + "params": ["programId"] }, { "method": "post", "path": "/api/programs/{programId}/financial-service-providers/intersolve-voucher/instructions", - "params": [ - "programId" - ] + "params": ["programId"] }, { "method": "post", "path": "/api/programs/{programId}/financial-service-providers/intersolve-voucher/batch-jobs", - "params": [ - "programId" - ] + "params": ["programId"] }, { "method": "post", @@ -499,6 +380,7 @@ "filter.paymentCountRemaining", "filter.personAffectedSequence", "filter.lastMessageStatus", + "filter.isDuplicate", "filter.failedPayment", "filter.waitingPayment", "filter.successPayment", @@ -522,37 +404,27 @@ { "method": "get", "path": "/api/programs/{programId}/metrics/payment-state-sums", - "params": [ - "programId" - ] + "params": ["programId"] }, { "method": "get", "path": "/api/programs/{programId}/metrics/program-stats-summary", - "params": [ - "programId" - ] + "params": ["programId"] }, { "method": "get", "path": "/api/programs/{programId}/metrics/registration-status", - "params": [ - "programId" - ] + "params": ["programId"] }, { "method": "post", "path": "/api/programs/{programId}/registrations/import", - "params": [ - "programId" - ] + "params": ["programId"] }, { "method": "post", "path": "/api/programs/{programId}/registrations", - "params": [ - "programId" - ] + "params": ["programId"] }, { "method": "get", @@ -578,6 +450,7 @@ "filter.paymentCountRemaining", "filter.personAffectedSequence", "filter.lastMessageStatus", + "filter.isDuplicate", "filter.failedPayment", "filter.waitingPayment", "filter.successPayment", @@ -590,9 +463,7 @@ { "method": "patch", "path": "/api/programs/{programId}/registrations", - "params": [ - "programId" - ] + "params": ["programId"] }, { "method": "delete", @@ -619,6 +490,7 @@ "filter.paymentCountRemaining", "filter.personAffectedSequence", "filter.lastMessageStatus", + "filter.isDuplicate", "filter.failedPayment", "filter.waitingPayment", "filter.successPayment", @@ -632,9 +504,7 @@ { "method": "get", "path": "/api/programs/{programId}/registrations/import/template", - "params": [ - "programId" - ] + "params": ["programId"] }, { "method": "patch", @@ -661,6 +531,7 @@ "filter.paymentCountRemaining", "filter.personAffectedSequence", "filter.lastMessageStatus", + "filter.isDuplicate", "filter.failedPayment", "filter.waitingPayment", "filter.successPayment", @@ -674,17 +545,12 @@ { "method": "patch", "path": "/api/programs/{programId}/registrations/{referenceId}", - "params": [ - "programId", - "referenceId" - ] + "params": ["programId", "referenceId"] }, { "method": "get", "path": "/api/registrations", - "params": [ - "phonenumber" - ] + "params": ["phonenumber"] }, { "method": "post", @@ -711,6 +577,7 @@ "filter.paymentCountRemaining", "filter.personAffectedSequence", "filter.lastMessageStatus", + "filter.isDuplicate", "filter.failedPayment", "filter.waitingPayment", "filter.successPayment", @@ -724,77 +591,59 @@ { "method": "get", "path": "/api/programs/{programId}/registrations/{referenceId}/messages", - "params": [ - "referenceId", - "programId" - ] + "params": ["referenceId", "programId"] }, { "method": "get", "path": "/api/programs/{programId}/registrations/referenceid/{paId}", - "params": [ - "paId", - "programId" - ] + "params": ["paId", "programId"] }, { "method": "post", "path": "/api/programs/{programId}/registrations/{referenceId}/financial-service-providers/intersolve-visa/wallet/cards", - "params": [ - "programId", - "referenceId" - ] + "params": ["programId", "referenceId"] }, { "method": "patch", "path": "/api/programs/{programId}/registrations/{referenceId}/financial-service-providers/intersolve-visa/wallet/cards/{tokenCode}", - "params": [ - "programId", - "referenceId", - "tokenCode", - "pause" - ] + "params": ["programId", "referenceId", "tokenCode", "pause"] }, { "method": "patch", "path": "/api/programs/{programId}/registrations/{referenceId}/financial-service-providers/intersolve-visa/wallet", - "params": [ - "referenceId", - "programId" - ], + "params": ["referenceId", "programId"], "returnType": "IntersolveVisaWalletDto" }, { "method": "get", "path": "/api/programs/{programId}/registrations/{referenceId}/financial-service-providers/intersolve-visa/wallet", - "params": [ - "referenceId", - "programId" - ], + "params": ["referenceId", "programId"], "returnType": "IntersolveVisaWalletDto" }, { "method": "post", "path": "/api/programs/{programId}/registrations/{referenceId}/financial-service-providers/intersolve-visa/contact-information", - "params": [ - "programId", - "referenceId" - ] + "params": ["programId", "referenceId"] }, { "method": "get", "path": "/api/programs/{programId}/registrations/{id}", - "params": [ - "programId", - "id" - ] + "params": ["programId", "id"] + }, + { + "method": "get", + "path": "/api/programs/{programId}/registrations/{referenceId}/duplicates", + "params": ["referenceId", "programId"] + }, + { + "method": "post", + "path": "/api/programs/{programId}/registrations/{referenceId}/uniques", + "params": ["referenceId", "programId"] }, { "method": "get", "path": "/api/programs/{programId}/payments", - "params": [ - "programId" - ] + "params": ["programId"] }, { "method": "post", @@ -821,6 +670,7 @@ "filter.paymentCountRemaining", "filter.personAffectedSequence", "filter.lastMessageStatus", + "filter.isDuplicate", "filter.failedPayment", "filter.waitingPayment", "filter.successPayment", @@ -834,56 +684,38 @@ { "method": "patch", "path": "/api/programs/{programId}/payments", - "params": [ - "programId" - ] + "params": ["programId"] }, { "method": "get", "path": "/api/programs/{programId}/payments/status", - "params": [ - "programId" - ] + "params": ["programId"] }, { "method": "get", "path": "/api/programs/{programId}/payments/{payment}/status", - "params": [ - "programId", - "payment" - ] + "params": ["programId", "payment"] }, { "method": "get", "path": "/api/programs/{programId}/payments/{payment}", - "params": [ - "payment", - "programId" - ], + "params": ["payment", "programId"], "returnType": "PaymentReturnDto" }, { "method": "get", "path": "/api/programs/{programId}/payments/{payment}/fsp-instructions", - "params": [ - "programId", - "payment" - ] + "params": ["programId", "payment"] }, { "method": "get", "path": "/api/programs/{programId}/payments/fsp-reconciliation/import-template", - "params": [ - "programId" - ] + "params": ["programId"] }, { "method": "post", "path": "/api/programs/{programId}/payments/{payment}/fsp-reconciliation", - "params": [ - "programId", - "payment" - ] + "params": ["programId", "payment"] }, { "method": "post", @@ -898,17 +730,13 @@ { "method": "get", "path": "/api/programs/{programId}/financial-service-providers/commercial-bank-ethiopia/account-enquiries", - "params": [ - "programId" - ], + "params": ["programId"], "returnType": "CommercialBankEthiopiaValidationReportDto" }, { "method": "post", "path": "/api/programs/{programId}/financial-service-providers/commercial-bank-ethiopia/account-enquiries/validation", - "params": [ - "programId" - ] + "params": ["programId"] }, { "method": "post", @@ -933,25 +761,16 @@ { "method": "post", "path": "/api/programs/{programId}/registrations/{referenceId}/notes", - "params": [ - "programId", - "referenceId" - ] + "params": ["programId", "referenceId"] }, { "method": "get", "path": "/api/programs/{programId}/registrations/{referenceId}/notes", - "params": [ - "programId", - "referenceId" - ] + "params": ["programId", "referenceId"] }, { "method": "get", "path": "/api/programs/{programId}/registrations/{registrationId}/activities", - "params": [ - "programId", - "registrationId" - ] + "params": ["programId", "registrationId"] } -] \ No newline at end of file +] From 3c1a93af21c6142bba900c7a06a30d7ba9739e2c Mon Sep 17 00:00:00 2001 From: jannisvisser Date: Fri, 13 Dec 2024 14:42:24 +0100 Subject: [PATCH 2/2] Update services/121-service/src/registration/registrations.service.ts --- services/121-service/src/registration/registrations.service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/121-service/src/registration/registrations.service.ts b/services/121-service/src/registration/registrations.service.ts index 7bf164199b..e32ebbded3 100644 --- a/services/121-service/src/registration/registrations.service.ts +++ b/services/121-service/src/registration/registrations.service.ts @@ -1218,7 +1218,7 @@ export class RegistrationsService { .getRawMany(); console.timeEnd('duplicates'); console.log('🚀 ~ RegistrationsService ~ result:', duplicates); - +// do not use levenshtein, but other alternative in this method console.time('fuzzyMatch levenshtein'); const fuzzyQbLeven = await this.registrationDataScopedRepository .createQueryBuilder('d1')