From 37a3a4f1d8a32374be087b989daa761aab7fc366 Mon Sep 17 00:00:00 2001 From: AlfredRosenthal Date: Wed, 30 Nov 2022 11:03:41 -0800 Subject: [PATCH 01/11] removed extra status causing problems --- api/src/services/validation-service.ts | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/api/src/services/validation-service.ts b/api/src/services/validation-service.ts index 6f76c46d5f..82a8348ad4 100644 --- a/api/src/services/validation-service.ts +++ b/api/src/services/validation-service.ts @@ -106,10 +106,11 @@ export class ValidationService extends DBService { // Parse Archive into JSON file for custom validation await this.parseDWCToJSON(submissionId, dwcPrep.archive); + + await this.templateScrapeAndUploadOccurrences(submissionId); + // insert validated status await this.submissionRepository.insertSubmissionStatus(submissionId, SUBMISSION_STATUS_TYPE.TEMPLATE_VALIDATED); - - await this.templateScrapeAndUploadOccurrences(submissionId); } catch (error) { if (error instanceof SubmissionError) { await this.errorService.insertSubmissionError(submissionId, error); @@ -127,17 +128,14 @@ export class ValidationService extends DBService { // template validation await this.templateValidation(submissionPrep.xlsx, surveyId); - // insert template validated status - await this.submissionRepository.insertSubmissionStatus(submissionId, SUBMISSION_STATUS_TYPE.TEMPLATE_VALIDATED); - // template transformation await this.templateTransformation(submissionId, submissionPrep.xlsx, submissionPrep.s3InputKey, surveyId); - // insert template validated status - await this.submissionRepository.insertSubmissionStatus(submissionId, SUBMISSION_STATUS_TYPE.TEMPLATE_TRANSFORMED); - // occurrence scraping await this.templateScrapeAndUploadOccurrences(submissionId); + + // insert template validated status + await this.submissionRepository.insertSubmissionStatus(submissionId, SUBMISSION_STATUS_TYPE.TEMPLATE_TRANSFORMED); } catch (error) { if (error instanceof SubmissionError) { await this.errorService.insertSubmissionError(submissionId, error); @@ -421,6 +419,7 @@ export class ValidationService extends DBService { s3OutputKey: string, xlsxCsv: XLSXCSV ) { + console.log("PERSIST TRANSFORMATION RESULTS") // Build the archive zip file const dwcArchiveZip = new AdmZip(); fileBuffers.forEach((file) => dwcArchiveZip.addFile(`${file.name}.csv`, file.buffer)); From db4c6b6e3c75f997dc0502b2e330ec4a83f2410d Mon Sep 17 00:00:00 2001 From: AlfredRosenthal Date: Wed, 30 Nov 2022 17:15:23 -0800 Subject: [PATCH 02/11] clean up --- api/src/services/validation-service.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/api/src/services/validation-service.ts b/api/src/services/validation-service.ts index 82a8348ad4..d61920a295 100644 --- a/api/src/services/validation-service.ts +++ b/api/src/services/validation-service.ts @@ -106,7 +106,7 @@ export class ValidationService extends DBService { // Parse Archive into JSON file for custom validation await this.parseDWCToJSON(submissionId, dwcPrep.archive); - + await this.templateScrapeAndUploadOccurrences(submissionId); // insert validated status @@ -419,7 +419,6 @@ export class ValidationService extends DBService { s3OutputKey: string, xlsxCsv: XLSXCSV ) { - console.log("PERSIST TRANSFORMATION RESULTS") // Build the archive zip file const dwcArchiveZip = new AdmZip(); fileBuffers.forEach((file) => dwcArchiveZip.addFile(`${file.name}.csv`, file.buffer)); From bf2a4275c4c0a4f9b9665714a2d8c582c7e8aeff Mon Sep 17 00:00:00 2001 From: AlfredRosenthal Date: Wed, 30 Nov 2022 17:24:50 -0800 Subject: [PATCH 03/11] fixed test --- api/src/services/validation-service.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/services/validation-service.test.ts b/api/src/services/validation-service.test.ts index 4bb9bc0061..6b54747211 100644 --- a/api/src/services/validation-service.test.ts +++ b/api/src/services/validation-service.test.ts @@ -766,7 +766,7 @@ describe('ValidationService', () => { expect(validate).to.be.calledOnce; expect(transform).to.be.calledOnce; expect(upload).to.be.calledOnce; - expect(status).to.be.calledTwice; + expect(status).to.be.calledOnce; }); it('should insert submission error', async () => { From 54c1cdba885fa64134ae4136db30dd8abe85d75b Mon Sep 17 00:00:00 2001 From: AlfredRosenthal Date: Thu, 1 Dec 2022 13:51:22 -0800 Subject: [PATCH 04/11] modified sql to properly display --- .../queries/survey/survey-occurrence-queries.ts | 2 +- api/src/services/validation-service.ts | 14 ++++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/api/src/queries/survey/survey-occurrence-queries.ts b/api/src/queries/survey/survey-occurrence-queries.ts index e023075f8b..4ee6624d0e 100644 --- a/api/src/queries/survey/survey-occurrence-queries.ts +++ b/api/src/queries/survey/survey-occurrence-queries.ts @@ -185,7 +185,7 @@ export const getLatestSurveyOccurrenceSubmissionSQL = (surveyId: number): SQLSta WHERE os.survey_id = ${surveyId} ORDER BY - os.event_timestamp DESC + os.event_timestamp DESC, ss.submission_status_id DESC LIMIT 1 ; `; diff --git a/api/src/services/validation-service.ts b/api/src/services/validation-service.ts index d61920a295..87a3579d8e 100644 --- a/api/src/services/validation-service.ts +++ b/api/src/services/validation-service.ts @@ -98,6 +98,10 @@ export class ValidationService extends DBService { const csvState = this.validateDWC(dwcPrep.archive); // update submission await this.persistValidationResults(csvState.csv_state, csvState.media_state); + + // insert validated status + await this.submissionRepository.insertSubmissionStatus(submissionId, SUBMISSION_STATUS_TYPE.TEMPLATE_VALIDATED); + await this.occurrenceService.updateSurveyOccurrenceSubmission( submissionId, dwcPrep.archive.rawFile.fileName, @@ -109,8 +113,6 @@ export class ValidationService extends DBService { await this.templateScrapeAndUploadOccurrences(submissionId); - // insert validated status - await this.submissionRepository.insertSubmissionStatus(submissionId, SUBMISSION_STATUS_TYPE.TEMPLATE_VALIDATED); } catch (error) { if (error instanceof SubmissionError) { await this.errorService.insertSubmissionError(submissionId, error); @@ -128,14 +130,18 @@ export class ValidationService extends DBService { // template validation await this.templateValidation(submissionPrep.xlsx, surveyId); + // insert template validated status + await this.submissionRepository.insertSubmissionStatus(submissionId, SUBMISSION_STATUS_TYPE.TEMPLATE_VALIDATED); + // template transformation await this.templateTransformation(submissionId, submissionPrep.xlsx, submissionPrep.s3InputKey, surveyId); + // insert template transformed status + await this.submissionRepository.insertSubmissionStatus(submissionId, SUBMISSION_STATUS_TYPE.TEMPLATE_TRANSFORMED); + // occurrence scraping await this.templateScrapeAndUploadOccurrences(submissionId); - // insert template validated status - await this.submissionRepository.insertSubmissionStatus(submissionId, SUBMISSION_STATUS_TYPE.TEMPLATE_TRANSFORMED); } catch (error) { if (error instanceof SubmissionError) { await this.errorService.insertSubmissionError(submissionId, error); From 5cef0abb545c316f583755bc3e0b0d5691d8845a Mon Sep 17 00:00:00 2001 From: AlfredRosenthal Date: Thu, 1 Dec 2022 14:20:11 -0800 Subject: [PATCH 05/11] ran fix --- api/src/services/validation-service.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/api/src/services/validation-service.ts b/api/src/services/validation-service.ts index 87a3579d8e..4a29122ec0 100644 --- a/api/src/services/validation-service.ts +++ b/api/src/services/validation-service.ts @@ -98,7 +98,7 @@ export class ValidationService extends DBService { const csvState = this.validateDWC(dwcPrep.archive); // update submission await this.persistValidationResults(csvState.csv_state, csvState.media_state); - + // insert validated status await this.submissionRepository.insertSubmissionStatus(submissionId, SUBMISSION_STATUS_TYPE.TEMPLATE_VALIDATED); @@ -112,7 +112,6 @@ export class ValidationService extends DBService { await this.parseDWCToJSON(submissionId, dwcPrep.archive); await this.templateScrapeAndUploadOccurrences(submissionId); - } catch (error) { if (error instanceof SubmissionError) { await this.errorService.insertSubmissionError(submissionId, error); @@ -141,7 +140,6 @@ export class ValidationService extends DBService { // occurrence scraping await this.templateScrapeAndUploadOccurrences(submissionId); - } catch (error) { if (error instanceof SubmissionError) { await this.errorService.insertSubmissionError(submissionId, error); From 3b0cd8b06342d0f783c15243b41d18a40ec711e1 Mon Sep 17 00:00:00 2001 From: AlfredRosenthal Date: Thu, 1 Dec 2022 15:05:56 -0800 Subject: [PATCH 06/11] formatting --- api/src/services/validation-service.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/src/services/validation-service.ts b/api/src/services/validation-service.ts index 4a29122ec0..e4fc628d7f 100644 --- a/api/src/services/validation-service.ts +++ b/api/src/services/validation-service.ts @@ -99,14 +99,14 @@ export class ValidationService extends DBService { // update submission await this.persistValidationResults(csvState.csv_state, csvState.media_state); - // insert validated status - await this.submissionRepository.insertSubmissionStatus(submissionId, SUBMISSION_STATUS_TYPE.TEMPLATE_VALIDATED); - await this.occurrenceService.updateSurveyOccurrenceSubmission( submissionId, dwcPrep.archive.rawFile.fileName, dwcPrep.s3InputKey - ); + ); + + // insert validated status + await this.submissionRepository.insertSubmissionStatus(submissionId, SUBMISSION_STATUS_TYPE.TEMPLATE_VALIDATED); // Parse Archive into JSON file for custom validation await this.parseDWCToJSON(submissionId, dwcPrep.archive); From caed627917f9106afe4beb83859da67574b21e84 Mon Sep 17 00:00:00 2001 From: AlfredRosenthal Date: Thu, 1 Dec 2022 15:12:45 -0800 Subject: [PATCH 07/11] fixed test --- api/src/services/validation-service.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/services/validation-service.test.ts b/api/src/services/validation-service.test.ts index 6b54747211..4bb9bc0061 100644 --- a/api/src/services/validation-service.test.ts +++ b/api/src/services/validation-service.test.ts @@ -766,7 +766,7 @@ describe('ValidationService', () => { expect(validate).to.be.calledOnce; expect(transform).to.be.calledOnce; expect(upload).to.be.calledOnce; - expect(status).to.be.calledOnce; + expect(status).to.be.calledTwice; }); it('should insert submission error', async () => { From b4d38be50caf07983d9f9d64020b73a6a2ec6d18 Mon Sep 17 00:00:00 2001 From: AlfredRosenthal Date: Thu, 1 Dec 2022 15:20:00 -0800 Subject: [PATCH 08/11] ran fix --- api/src/services/validation-service.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/src/services/validation-service.ts b/api/src/services/validation-service.ts index e4fc628d7f..a97ee1ee11 100644 --- a/api/src/services/validation-service.ts +++ b/api/src/services/validation-service.ts @@ -103,8 +103,8 @@ export class ValidationService extends DBService { submissionId, dwcPrep.archive.rawFile.fileName, dwcPrep.s3InputKey - ); - + ); + // insert validated status await this.submissionRepository.insertSubmissionStatus(submissionId, SUBMISSION_STATUS_TYPE.TEMPLATE_VALIDATED); From de15f1bb6622cc6f0c4f5969329ceab462500973 Mon Sep 17 00:00:00 2001 From: AlfredRosenthal Date: Thu, 1 Dec 2022 16:30:11 -0800 Subject: [PATCH 09/11] build From 67a350d5678ce4820b10be05798ca8053aa8418a Mon Sep 17 00:00:00 2001 From: AlfredRosenthal Date: Fri, 2 Dec 2022 09:50:42 -0800 Subject: [PATCH 10/11] build From 953b2c9c7e513f7dfa042ac0cebc0954728a8487 Mon Sep 17 00:00:00 2001 From: AlfredRosenthal Date: Fri, 2 Dec 2022 11:08:10 -0800 Subject: [PATCH 11/11] build