BC-7561 - a bit of cleanup

this commit changes failed deletion and counted pending requests
 to also be older than configured threshold

apps/server/src/modules/deletion/api/uc/deletion-request.uc.ts

@@ -76,42 +76,39 @@ export class DeletionRequestUc implements IEventHandler<DataDeletedEvent> {
 
 	async executeDeletionRequests(limit?: number): Promise<void> {
 		this.logger.debug({ action: 'executeDeletionRequests', limit });
-		const maxAmountOfDeletionRequestsDoConcurrently = this.configService.get<number>(
-			'ADMIN_API__MAX_CONCURRENT_DELETION_REQUESTS'
-		);
-		const callsDelayMilliseconds = this.configService.get<number>('ADMIN_API__DELETION_DELAY_MILLISECONDS');
-		let tasks: DeletionRequest[] = [];
+
+		const max = this.configService.get<number>('ADMIN_API__MAX_CONCURRENT_DELETION_REQUESTS');
+
+		let deletionRequests: DeletionRequest[] = [];
 
 		do {
-			const numberOfDeletionRequestsWithStatusPending =
-				// eslint-disable-next-line no-await-in-loop
-				await this.deletionRequestService.countPendingDeletionRequests();
-			const numberOfDeletionRequestsToProccess =
-				maxAmountOfDeletionRequestsDoConcurrently - numberOfDeletionRequestsWithStatusPending;
+			// eslint-disable-next-line no-await-in-loop
+			const pendingCount = await this.deletionRequestService.countPendingDeletionRequests();
+			limit = limit ?? max - pendingCount;
+
 			this.logger.debug({
 				action: 'numberItemsWithStatusPending, amountWillingToTake',
-				numberOfDeletionRequestsWithStatusPending,
-				numberOfDeletionRequestsToProccess,
+				numberOfDeletionRequestsWithStatusPending: pendingCount,
+				numberOfDeletionRequestsToProccess: limit,
 			});
 			// eslint-disable-next-line no-await-in-loop
-			if (numberOfDeletionRequestsToProccess > 0) {
+			if (limit > 0) {
 				// eslint-disable-next-line no-await-in-loop
-				tasks = await this.deletionRequestService.findAllItemsToExecute(numberOfDeletionRequestsToProccess);
+				deletionRequests = await this.deletionRequestService.findAllItemsToExecute(limit);
+
+				this.logger.debug({ action: 'processing deletion request', deletionRequests });
 				// eslint-disable-next-line no-await-in-loop
 				await Promise.all(
-					tasks.map(async (req) => {
+					deletionRequests.map(async (req) => {
 						await this.executeDeletionRequest(req);
 					})
 				);
 			}
-			// short sleep mode to give time for deletion process to do their work
-			if (callsDelayMilliseconds && callsDelayMilliseconds > 0) {
-				// eslint-disable-next-line no-await-in-loop
-				await new Promise((resolve) => {
-					setTimeout(resolve, callsDelayMilliseconds);
-				});
-			}
-		} while (tasks.length > 0);
+
+			// eslint-disable-next-line no-await-in-loop
+			await this.sleep();
+		} while (deletionRequests.length > 0);
+
 		this.logger.debug({ action: 'deletion process completed' });
 	}
 
@@ -150,4 +147,16 @@ export class DeletionRequestUc implements IEventHandler<DataDeletedEvent> {
 			await this.deletionRequestService.markDeletionRequestAsFailed(deletionRequest.id);
 		}
 	}
+
+	// short sleep mode to give time for deletion process to do its work
+	private async sleep() {
+		const delay = this.configService.get<number>('ADMIN_API__DELETION_DELAY_MILLISECONDS');
+		if (delay > 0) {
+			return new Promise((resolve) => {
+				setTimeout(resolve, delay);
+			});
+		}
+
+		return Promise.resolve();
+	}
 }

apps/server/src/modules/deletion/domain/service/deletion-request.service.ts

@@ -9,10 +9,16 @@ import { DeletionConfig } from '../../deletion.config';
 
 @Injectable()
 export class DeletionRequestService {
+	private modificationThreshold: Date;
+
 	constructor(
 		private readonly deletionRequestRepo: DeletionRequestRepo,
 		private readonly configService: ConfigService<DeletionConfig, true>
-	) {}
+	) {
+		const threshold = this.configService.get<number>('ADMIN_API__MODIFICATION_THRESHOLD_MS');
+		const modificationThreshold = new Date(Date.now() - threshold);
+		this.modificationThreshold = modificationThreshold;
+	}
 
 	async createDeletionRequest(
 		targetRefId: EntityId,
@@ -41,15 +47,14 @@ export class DeletionRequestService {
 		return deletionRequest;
 	}
 
-	async findAllItemsToExecute(limit?: number): Promise<DeletionRequest[]> {
-		const threshold = this.configService.get<number>('ADMIN_API__MODIFICATION_THRESHOLD_MS');
-		const itemsToDelete: DeletionRequest[] = await this.deletionRequestRepo.findAllItemsToExecution(threshold, limit);
+	async findAllItemsToExecute(limit: number): Promise<DeletionRequest[]> {
+		const deletionRequests = await this.deletionRequestRepo.findAllItemsToExecution(this.modificationThreshold, limit);
 
-		return itemsToDelete;
+		return deletionRequests;
 	}
 
 	async countPendingDeletionRequests(): Promise<number> {
-		const numberItemsWithStatusPending: number = await this.deletionRequestRepo.countPendingDeletionRequests();
+		const numberItemsWithStatusPending: number = await this.deletionRequestRepo.countPendingDeletionRequests(this.modificationThreshold);
 
 		return numberItemsWithStatusPending;
 	}

apps/server/src/modules/deletion/repo/deletion-request.repo.ts

@@ -6,6 +6,7 @@ import { DeletionRequest } from '../domain/do';
 import { DeletionRequestEntity } from './entity';
 import { DeletionRequestMapper } from './mapper';
 import { DeletionRequestScope } from './scope';
+import { StatusModel } from '../domain/types';
 
 @Injectable()
 export class DeletionRequestRepo {
@@ -31,11 +32,15 @@ export class DeletionRequestRepo {
 		await this.em.flush();
 	}
 
-	async findAllItemsToExecution(threshold: number, limit?: number): Promise<DeletionRequest[]> {
-		const currentDate = new Date();
-		const modificationThreshold = new Date(Date.now() - threshold);
-		const scope = new DeletionRequestScope().byDeleteAfter(currentDate).byStatus(modificationThreshold);
-		const order = { createdAt: SortOrder.desc };
+	async findAllItemsToExecution(olderThan: Date, limit: number): Promise<DeletionRequest[]> {
+		const scope = new DeletionRequestScope();
+		scope.byDeleteAfter(new Date());
+		scope.byStatus(StatusModel.REGISTERED, olderThan);
+		// TODO failed and hanging pending should be handled differently
+		scope.byStatus(StatusModel.FAILED, olderThan);
+		scope.byStatus(StatusModel.PENDING, olderThan);
+
+		const order = { createdAt: SortOrder.desc }; // TODO why decending order? Should it not be ascending?
 
 		const [deletionRequestEntities] = await this.em.findAndCount(DeletionRequestEntity, scope.query, {
 			limit,
@@ -47,8 +52,9 @@ export class DeletionRequestRepo {
 		return mapped;
 	}
 
-	async countPendingDeletionRequests(): Promise<number> {
-		const scope = new DeletionRequestScope().byStatusPending();
+	async countPendingDeletionRequests(olderThan: Date): Promise<number> {
+		const scope = new DeletionRequestScope();
+		scope.byStatus(StatusModel.PENDING, olderThan);
 
 		const numberItemsWithStatusPending: number = await this.em.count(DeletionRequestEntity, scope.query);
 

apps/server/src/modules/deletion/repo/scope/deletion-request-scope.ts

@@ -9,21 +9,12 @@ export class DeletionRequestScope extends Scope<DeletionRequestEntity> {
 		return this;
 	}
 
-	byStatus(fifteenMinutesAgo: Date): this {
-		this.addQuery({
-			$or: [
-				{ status: StatusModel.FAILED },
-				{
-					$and: [{ status: [StatusModel.REGISTERED, StatusModel.PENDING] }, { updatedAt: { $lt: fifteenMinutesAgo } }],
-				},
-			],
-		});
-
-		return this;
-	}
-
-	byStatusPending(): this {
-		this.addQuery({ status: [StatusModel.PENDING] });
+	byStatus(status: StatusModel, olderThan: Date): this {
+		if (olderThan) {
+			this.addQuery({ $and: [{ status }, { updatedAt: { $lt: olderThan } }] });
+		} else {
+			this.addQuery({ status });
+		}
 
 		return this;
 	}