diff --git a/api/src/constants/status.ts b/api/src/constants/status.ts index 25379476ea..53874880fa 100644 --- a/api/src/constants/status.ts +++ b/api/src/constants/status.ts @@ -92,7 +92,8 @@ export enum SUBMISSION_MESSAGE_TYPE { 'INVALID_MEDIA' = 'Media is invalid', 'INVALID_XLSX_CSV' = 'Media is not a valid XLSX CSV file.', 'UNSUPPORTED_FILE_TYPE' = 'File submitted is not a supported type', - 'NON_UNIQUE_KEY' = 'Duplicate Key(s) found in file.' + 'NON_UNIQUE_KEY' = 'Duplicate Key(s) found in file.', + 'MISMATCHED_TEMPLATE_SURVEY_SPECIES' = 'Mismatched template with survey focal species' } export enum MESSAGE_CLASS_NAME { diff --git a/api/src/paths/project/{projectId}/survey/{surveyId}/observation/submission/get.test.ts b/api/src/paths/project/{projectId}/survey/{surveyId}/observation/submission/get.test.ts index 7afc44a212..7149793ec9 100644 --- a/api/src/paths/project/{projectId}/survey/{surveyId}/observation/submission/get.test.ts +++ b/api/src/paths/project/{projectId}/survey/{surveyId}/observation/submission/get.test.ts @@ -1,13 +1,19 @@ import chai, { expect } from 'chai'; import { describe } from 'mocha'; +import OpenAPIRequestValidator, { OpenAPIRequestValidatorArgs } from 'openapi-request-validator'; +import OpenAPIResponseValidator, { OpenAPIResponseValidatorArgs } from 'openapi-response-validator'; import sinon from 'sinon'; import sinonChai from 'sinon-chai'; -import SQL from 'sql-template-strings'; -import { SUBMISSION_MESSAGE_TYPE } from '../../../../../../../constants/status'; +import { + MESSAGE_CLASS_NAME, + SUBMISSION_MESSAGE_TYPE, + SUBMISSION_STATUS_TYPE +} from '../../../../../../../constants/status'; import * as db from '../../../../../../../database/db'; -import { HTTPError } from '../../../../../../../errors/http-error'; -import survey_queries from '../../../../../../../queries/survey'; -import { IGetLatestSurveyOccurrenceSubmission } from '../../../../../../../repositories/survey-repository'; +import { + IGetLatestSurveyOccurrenceSubmission, + SurveyRepository +} from '../../../../../../../repositories/survey-repository'; import { SurveyService } from '../../../../../../../services/survey-service'; import { getMockDBConnection } from '../../../../../../../__mocks__/db'; import * as observationSubmission from './get'; @@ -42,21 +48,777 @@ describe('getObservationSubmission', () => { sinon.restore(); }); - it('should throw a 400 error when no surveyId is provided', async () => { - sinon.stub(db, 'getDBConnection').returns(dbConnectionObj); - - try { - const result = observationSubmission.getOccurrenceSubmission(); - await result( - { ...sampleReq, params: { ...sampleReq.params, surveyId: null } }, - (null as unknown) as any, - (null as unknown) as any - ); - expect.fail(); - } catch (actualError) { - expect((actualError as HTTPError).status).to.equal(400); - expect((actualError as HTTPError).message).to.equal('Missing required path param `surveyId`'); - } + describe('openApiScheme', () => { + const requestSchema = (observationSubmission.GET.apiDoc as unknown) as OpenAPIRequestValidatorArgs; + const responseSchema = (observationSubmission.GET.apiDoc as unknown) as OpenAPIResponseValidatorArgs; + + describe('request validation', () => { + const requestValidator = new OpenAPIRequestValidator(requestSchema); + + describe('should throw an error when', () => { + describe('projectId', () => { + it('is missing', () => { + const request = { + headers: { 'content-type': 'application/json' }, + params: { + surveyId: 5 + } + }; + + const response = requestValidator.validateRequest(request); + expect(response.status).to.equal(400); + expect(response.errors.length).to.equal(1); + expect(response.errors[0].path).to.equal('projectId'); + expect(response.errors[0].message).to.equal("must have required property 'projectId'"); + }); + + it('is null', () => { + const request = { + headers: { 'content-type': 'application/json' }, + params: { + projectId: null, + surveyId: 5 + } + }; + + const response = requestValidator.validateRequest(request); + expect(response.status).to.equal(400); + expect(response.errors.length).to.equal(1); + expect(response.errors[0].path).to.equal('projectId'); + expect(response.errors[0].message).to.equal('must be number'); + }); + + it('is not a number', () => { + const request = { + headers: { 'content-type': 'application/json' }, + params: { + projectId: '12', + surveyId: 5 + } + }; + + const response = requestValidator.validateRequest(request); + console.log(JSON.stringify(response)); + expect(response.status).to.equal(400); + expect(response.errors.length).to.equal(1); + expect(response.errors[0].path).to.equal('projectId'); + expect(response.errors[0].message).to.equal('must be number'); + }); + + it('is less than 1', () => { + const request = { + headers: { 'content-type': 'application/json' }, + params: { + projectId: 0, + surveyId: 5 + } + }; + + const response = requestValidator.validateRequest(request); + expect(response.status).to.equal(400); + expect(response.errors.length).to.equal(1); + expect(response.errors[0].path).to.equal('projectId'); + expect(response.errors[0].message).to.equal('must be >= 1'); + }); + }); + + describe('surveyId', () => { + it('is missing', () => { + const request = { + headers: { 'content-type': 'application/json' }, + params: { + projectId: 2 + } + }; + + const response = requestValidator.validateRequest(request); + expect(response.status).to.equal(400); + expect(response.errors.length).to.equal(1); + expect(response.errors[0].path).to.equal('surveyId'); + expect(response.errors[0].message).to.equal("must have required property 'surveyId'"); + }); + + it('is null', () => { + const request = { + headers: { 'content-type': 'application/json' }, + params: { + projectId: 2, + surveyId: null + } + }; + + const response = requestValidator.validateRequest(request); + expect(response.status).to.equal(400); + expect(response.errors.length).to.equal(1); + expect(response.errors[0].path).to.equal('surveyId'); + expect(response.errors[0].message).to.equal('must be number'); + }); + + it('is not a number', () => { + const request = { + headers: { 'content-type': 'application/json' }, + params: { + projectId: 2, + surveyId: '15' + } + }; + + const response = requestValidator.validateRequest(request); + expect(response.status).to.equal(400); + expect(response.errors.length).to.equal(1); + expect(response.errors[0].path).to.equal('surveyId'); + expect(response.errors[0].message).to.equal('must be number'); + }); + + it('is less than 1', () => { + const request = { + headers: { 'content-type': 'application/json' }, + params: { + projectId: 2, + surveyId: 0 + } + }; + + const response = requestValidator.validateRequest(request); + + expect(response.status).to.equal(400); + expect(response.errors.length).to.equal(1); + expect(response.errors[0].path).to.equal('surveyId'); + expect(response.errors[0].message).to.equal('must be >= 1'); + }); + }); + }); + + describe('should succeed when', () => { + it('is provided with valid params', () => { + const request = { + headers: { 'content-type': 'application/json' }, + params: { + projectId: 2, + surveyId: 5 + } + }; + + const response = requestValidator.validateRequest(request); + + expect(response).to.equal(undefined); + }); + }); + }); + + describe('response validation', () => { + const responseValidator = new OpenAPIResponseValidator(responseSchema); + + describe('should throw an error when', () => { + it('returns a non-object response', () => { + const apiResponse = 'test-response'; + const response = responseValidator.validateResponse(200, apiResponse); + + expect(response.message).to.equal('The response was not valid.'); + expect(response.errors.length).to.equal(1); + expect(response.errors[0].path).to.equal('response'); + expect(response.errors[0].message).to.equal('must be object,null'); + }); + + describe('id', () => { + it('is missing', () => { + const apiResponse = { + inputFileName: 'filename.xlsx', + status: 'validation-status', + isValidating: false, + messageTypes: [] + }; + + const response = responseValidator.validateResponse(200, apiResponse); + expect(response.message).to.equal('The response was not valid.'); + expect(response.errors.length).to.equal(1); + expect(response.errors[0].path).to.equal('response'); + expect(response.errors[0].message).to.equal("must have required property 'id'"); + }); + + it('is null', () => { + const apiResponse = { + id: null, + inputFileName: 'filename.xlsx', + status: 'validation-status', + isValidating: false, + messageTypes: [] + }; + + const response = responseValidator.validateResponse(200, apiResponse); + expect(response.message).to.equal('The response was not valid.'); + expect(response.errors[0].path).to.equal('id'); + expect(response.errors[0].message).to.equal('must be number'); + }); + + it('is not a number', () => { + const apiResponse = { + id: '12', + inputFileName: 'filename.xlsx', + status: 'validation-status', + isValidating: false, + messageTypes: [] + }; + + const response = responseValidator.validateResponse(200, apiResponse); + expect(response.message).to.equal('The response was not valid.'); + expect(response.errors[0].path).to.equal('id'); + expect(response.errors[0].message).to.equal('must be number'); + }); + + it('is less than 1', () => { + const apiResponse = { + id: 0, + inputFileName: 'filename.xlsx', + status: 'validation-status', + isValidating: false, + messageTypes: [] + }; + + const response = responseValidator.validateResponse(200, apiResponse); + expect(response.message).to.equal('The response was not valid.'); + expect(response.errors[0].path).to.equal('id'); + expect(response.errors[0].message).to.equal('must be >= 1'); + }); + }); + + describe('inputFileName', () => { + it('is missing', () => { + const apiResponse = { + id: 1, + status: 'validation-status', + isValidating: false, + messageTypes: [] + }; + + const response = responseValidator.validateResponse(200, apiResponse); + expect(response.message).to.equal('The response was not valid.'); + expect(response.errors.length).to.equal(1); + expect(response.errors[0].path).to.equal('response'); + expect(response.errors[0].message).to.equal("must have required property 'inputFileName'"); + }); + + it('is null', () => { + const apiResponse = { + id: 1, + inputFileName: null, + status: 'validation-status', + isValidating: false, + messageTypes: [] + }; + + const response = responseValidator.validateResponse(200, apiResponse); + expect(response.message).to.equal('The response was not valid.'); + expect(response.errors.length).to.equal(1); + expect(response.errors[0].path).to.equal('inputFileName'); + expect(response.errors[0].message).to.equal('must be string'); + }); + + it('is not a string', () => { + const apiResponse = { + id: 1, + inputFileName: { filename: 'filename' }, + status: 'validation-status', + isValidating: false, + messageTypes: [] + }; + + const response = responseValidator.validateResponse(200, apiResponse); + expect(response.message).to.equal('The response was not valid.'); + expect(response.errors.length).to.equal(1); + expect(response.errors[0].path).to.equal('inputFileName'); + expect(response.errors[0].message).to.equal('must be string'); + }); + }); + + describe('status', () => { + it('is missing', () => { + const apiResponse = { + id: 1, + inputFileName: 'filename.xlsx', + isValidating: false, + messageTypes: [] + }; + + const response = responseValidator.validateResponse(200, apiResponse); + expect(response.message).to.equal('The response was not valid.'); + expect(response.errors.length).to.equal(1); + expect(response.errors[0].path).to.equal('response'); + expect(response.errors[0].message).to.equal("must have required property 'status'"); + }); + + it('is not a string', () => { + const apiResponse = { + id: 1, + inputFileName: 'filename.xlsx', + status: { status: 'status' }, + isValidating: false, + messageTypes: [] + }; + + const response = responseValidator.validateResponse(200, apiResponse); + expect(response.message).to.equal('The response was not valid.'); + expect(response.errors.length).to.equal(1); + expect(response.errors[0].path).to.equal('status'); + expect(response.errors[0].message).to.equal('must be string,null'); + }); + }); + + describe('isValidating', () => { + it('is missing', () => { + const apiResponse = { + id: 1, + inputFileName: 'filename.xlsx', + status: 'validation-status', + messageTypes: [] + }; + + const response = responseValidator.validateResponse(200, apiResponse); + expect(response.message).to.equal('The response was not valid.'); + expect(response.errors.length).to.equal(1); + expect(response.errors[0].path).to.equal('response'); + expect(response.errors[0].message).to.equal("must have required property 'isValidating'"); + }); + + it('is not a bool', () => { + const apiResponse = { + id: 1, + inputFileName: 'filename.xlsx', + status: 'validation-status', + isValidating: 'true', + messageTypes: [] + }; + + const response = responseValidator.validateResponse(200, apiResponse); + expect(response.message).to.equal('The response was not valid.'); + expect(response.errors.length).to.equal(1); + expect(response.errors[0].path).to.equal('isValidating'); + expect(response.errors[0].message).to.equal('must be boolean'); + }); + }); + + describe('messageTypes', () => { + it('is missing', () => { + const apiResponse = { + id: 1, + inputFileName: 'filename.xlsx', + isValidating: false, + status: 'validation-status' + }; + + const response = responseValidator.validateResponse(200, apiResponse); + expect(response.message).to.equal('The response was not valid.'); + expect(response.errors.length).to.equal(1); + expect(response.errors[0].path).to.equal('response'); + expect(response.errors[0].message).to.equal("must have required property 'messageTypes'"); + }); + + it('is not an array', () => { + const apiResponse = { + id: 1, + inputFileName: 'filename.xlsx', + status: 'validation-status', + isValidating: false, + messageTypes: 'message-types' + }; + + const response = responseValidator.validateResponse(200, apiResponse); + expect(response.message).to.equal('The response was not valid.'); + expect(response.errors.length).to.equal(1); + expect(response.errors[0].path).to.equal('messageTypes'); + expect(response.errors[0].message).to.equal('must be array'); + }); + + describe('messageType', () => { + it('is not an object', () => { + const apiResponse = { + id: 1, + inputFileName: 'filename.xlsx', + status: 'validation-status', + isValidating: false, + messageTypes: ['message-type'] + }; + + const response = responseValidator.validateResponse(200, apiResponse); + expect(response.message).to.equal('The response was not valid.'); + expect(response.errors.length).to.equal(1); + expect(response.errors[0].path).to.equal('messageTypes/0'); + expect(response.errors[0].message).to.equal('must be object'); + }); + + describe('severityLabel', () => { + it('is missing', () => { + const apiResponse = { + id: 1, + inputFileName: 'filename.xlsx', + status: 'validation-status', + isValidating: false, + messageTypes: [ + { + messageTypeLabel: 'type-label', + messageStatus: 'message-status', + messages: [] + } + ] + }; + + const response = responseValidator.validateResponse(200, apiResponse); + expect(response.message).to.equal('The response was not valid.'); + expect(response.errors.length).to.equal(1); + expect(response.errors[0].path).to.equal('messageTypes/0'); + expect(response.errors[0].message).to.equal("must have required property 'severityLabel'"); + }); + + it('is not a string', () => { + const apiResponse = { + id: 1, + inputFileName: 'filename.xlsx', + status: 'validation-status', + isValidating: false, + messageTypes: [ + { + severityLabel: { label: 'label ' }, + messageTypeLabel: 'type-label', + messageStatus: 'message-status', + messages: [] + } + ] + }; + + const response = responseValidator.validateResponse(200, apiResponse); + expect(response.message).to.equal('The response was not valid.'); + expect(response.errors.length).to.equal(1); + expect(response.errors[0].path).to.equal('messageTypes/0/severityLabel'); + expect(response.errors[0].message).to.equal('must be string'); + }); + }); + + describe('messageStatus', () => { + it('is missing', () => { + const apiResponse = { + id: 1, + inputFileName: 'filename.xlsx', + status: 'validation-status', + isValidating: false, + messageTypes: [ + { + severityLabel: 'severity-label', + messageStatus: 'message-status', + messages: [] + } + ] + }; + + const response = responseValidator.validateResponse(200, apiResponse); + expect(response.message).to.equal('The response was not valid.'); + expect(response.errors.length).to.equal(1); + expect(response.errors[0].path).to.equal('messageTypes/0'); + expect(response.errors[0].message).to.equal("must have required property 'messageTypeLabel'"); + }); + + it('is not a string', () => { + const apiResponse = { + id: 1, + inputFileName: 'filename.xlsx', + status: 'validation-status', + isValidating: false, + messageTypes: [ + { + severityLabel: 'severity-label', + messageTypeLabel: { label: 'label ' }, + messageStatus: 'message-status', + messages: [] + } + ] + }; + + const response = responseValidator.validateResponse(200, apiResponse); + expect(response.message).to.equal('The response was not valid.'); + expect(response.errors.length).to.equal(1); + expect(response.errors[0].path).to.equal('messageTypes/0/messageTypeLabel'); + expect(response.errors[0].message).to.equal('must be string'); + }); + }); + + describe('messageStatus', () => { + it('is missing', () => { + const apiResponse = { + id: 1, + inputFileName: 'filename.xlsx', + status: 'validation-status', + isValidating: false, + messageTypes: [ + { + severityLabel: 'severity-label', + messageTypeLabel: 'type-label', + messages: [] + } + ] + }; + + const response = responseValidator.validateResponse(200, apiResponse); + expect(response.message).to.equal('The response was not valid.'); + expect(response.errors.length).to.equal(1); + expect(response.errors[0].path).to.equal('messageTypes/0'); + expect(response.errors[0].message).to.equal("must have required property 'messageStatus'"); + }); + + it('is not a string', () => { + const apiResponse = { + id: 1, + inputFileName: 'filename.xlsx', + status: 'validation-status', + isValidating: false, + messageTypes: [ + { + severityLabel: 'severity-label', + messageTypeLabel: 'type-label', + messageStatus: { status: 'status' }, + messages: [] + } + ] + }; + + const response = responseValidator.validateResponse(200, apiResponse); + expect(response.message).to.equal('The response was not valid.'); + expect(response.errors.length).to.equal(1); + expect(response.errors[0].path).to.equal('messageTypes/0/messageStatus'); + expect(response.errors[0].message).to.equal('must be string'); + }); + }); + + describe('messages', () => { + it('is missing', () => { + const apiResponse = { + id: 1, + inputFileName: 'filename.xlsx', + status: 'validation-status', + isValidating: false, + messageTypes: [ + { + severityLabel: 'severity-label', + messageTypeLabel: 'type-label', + messageStatus: 'message-status' + } + ] + }; + + const response = responseValidator.validateResponse(200, apiResponse); + expect(response.message).to.equal('The response was not valid.'); + expect(response.errors.length).to.equal(1); + expect(response.errors[0].path).to.equal('messageTypes/0'); + expect(response.errors[0].message).to.equal("must have required property 'messages'"); + }); + + it('is not an array', () => { + const apiResponse = { + id: 1, + inputFileName: 'filename.xlsx', + status: 'validation-status', + isValidating: false, + messageTypes: [ + { + severityLabel: 'severity-label', + messageTypeLabel: 'type-label', + messageStatus: 'message-status', + messages: 'messages' + } + ] + }; + + const response = responseValidator.validateResponse(200, apiResponse); + expect(response.message).to.equal('The response was not valid.'); + expect(response.errors.length).to.equal(1); + expect(response.errors[0].path).to.equal('messageTypes/0/messages'); + expect(response.errors[0].message).to.equal('must be array'); + }); + + describe('message', () => { + it('is not an object', () => { + const apiResponse = { + id: 1, + inputFileName: 'filename.xlsx', + status: 'validation-status', + isValidating: false, + messageTypes: [ + { + severityLabel: 'severity-label', + messageTypeLabel: 'type-label', + messageStatus: 'message-status', + messages: ['messages'] + } + ] + }; + + const response = responseValidator.validateResponse(200, apiResponse); + expect(response.message).to.equal('The response was not valid.'); + expect(response.errors.length).to.equal(1); + expect(response.errors[0].path).to.equal('messageTypes/0/messages/0'); + expect(response.errors[0].message).to.equal('must be object'); + }); + + it('id is missing', () => { + const apiResponse = { + id: 1, + inputFileName: 'filename.xlsx', + status: 'validation-status', + isValidating: false, + messageTypes: [ + { + severityLabel: 'severity-label', + messageTypeLabel: 'type-label', + messageStatus: 'message-status', + messages: [ + { + message: 'test-message' + } + ] + } + ] + }; + + const response = responseValidator.validateResponse(200, apiResponse); + expect(response.message).to.equal('The response was not valid.'); + expect(response.errors.length).to.equal(1); + expect(response.errors[0].path).to.equal('messageTypes/0/messages/0'); + expect(response.errors[0].message).to.equal("must have required property 'id'"); + }); + + it('id is not number', () => { + const apiResponse = { + id: 1, + inputFileName: 'filename.xlsx', + status: 'validation-status', + isValidating: false, + messageTypes: [ + { + severityLabel: 'severity-label', + messageTypeLabel: 'type-label', + messageStatus: 'message-status', + messages: [ + { + id: '12', + message: 'test-message' + } + ] + } + ] + }; + + const response = responseValidator.validateResponse(200, apiResponse); + expect(response.message).to.equal('The response was not valid.'); + expect(response.errors.length).to.equal(1); + expect(response.errors[0].path).to.equal('messageTypes/0/messages/0/id'); + expect(response.errors[0].message).to.equal('must be number'); + }); + + it('message is missing', () => { + const apiResponse = { + id: 1, + inputFileName: 'filename.xlsx', + status: 'validation-status', + isValidating: false, + messageTypes: [ + { + severityLabel: 'severity-label', + messageTypeLabel: 'type-label', + messageStatus: 'message-status', + messages: [ + { + id: 1 + } + ] + } + ] + }; + + const response = responseValidator.validateResponse(200, apiResponse); + expect(response.message).to.equal('The response was not valid.'); + expect(response.errors.length).to.equal(1); + expect(response.errors[0].path).to.equal('messageTypes/0/messages/0'); + expect(response.errors[0].message).to.equal("must have required property 'message'"); + }); + + it('message is not string', () => { + const apiResponse = { + id: 1, + inputFileName: 'filename.xlsx', + status: 'validation-status', + isValidating: false, + messageTypes: [ + { + severityLabel: 'severity-label', + messageTypeLabel: 'type-label', + messageStatus: 'message-status', + messages: [ + { + id: 1, + message: { test: 'test-message' } + } + ] + } + ] + }; + + const response = responseValidator.validateResponse(200, apiResponse); + expect(response.message).to.equal('The response was not valid.'); + expect(response.errors.length).to.equal(1); + expect(response.errors[0].path).to.equal('messageTypes/0/messages/0/message'); + expect(response.errors[0].message).to.equal('must be string'); + }); + }); + }); + }); + }); + }); + + describe('should succeed when', () => { + it('returns a null response', () => { + const apiResponse = null; + + const response = responseValidator.validateResponse(200, apiResponse); + expect(response).to.equal(undefined); + }); + + it('status is null', () => { + const apiResponse = { + id: 1, + status: null, + inputFileName: 'filename.xlsx', + isValidating: false, + messageTypes: [] + }; + + const response = responseValidator.validateResponse(200, apiResponse); + expect(response).to.equal(undefined); + }); + + it('has valid response values', () => { + const apiResponse = { + id: 1, + inputFileName: 'filename.xlsx', + status: 'validation-status', + isValidating: false, + messageTypes: [ + { + severityLabel: 'severity-label', + messageTypeLabel: 'type-label', + messageStatus: 'message-status', + messages: [ + { + id: 1, + message: 'test-message' + } + ] + } + ] + }; + + const response = responseValidator.validateResponse(200, apiResponse); + expect(response).to.equal(undefined); + }); + }); + }); }); it('should return an observation submission, on success with no rejected files', async () => { @@ -70,8 +832,7 @@ describe('getObservationSubmission', () => { sinon.stub(SurveyService.prototype, 'getLatestSurveyOccurrenceSubmission').resolves(({ id: 13, input_file_name: 'dwca_moose.zip', - submission_status_type_name: 'Darwin Core Validated', - message: 'string' + submission_status_type_name: 'Darwin Core Validated' } as unknown) as IGetLatestSurveyOccurrenceSubmission); const result = observationSubmission.getOccurrenceSubmission(); @@ -82,38 +843,17 @@ describe('getObservationSubmission', () => { id: 13, inputFileName: 'dwca_moose.zip', status: 'Darwin Core Validated', - messages: [] + isValidating: true, + messageTypes: [] }); }); it('should return an observation submission on success, with rejected files', async () => { - const mockQuery = sinon.stub(); - - mockQuery.resolves({ - rows: [ - { - errorCode: SUBMISSION_MESSAGE_TYPE.MISSING_REQUIRED_HEADER, - id: 1, - message: 'occurrence.txt - Missing Required Header - associatedTaxa - Missing required header', - status: 'Rejected', - type: 'Error' - }, - { - errorCode: SUBMISSION_MESSAGE_TYPE.MISSING_REQUIRED_HEADER, - id: 2, - message: 'occurrence.txt - Missing Required Header - associatedTaxa - Missing required header', - status: 'Rejected', - type: 'Error' - } - ] - }); - sinon.stub(db, 'getDBConnection').returns({ ...dbConnectionObj, systemUserId: () => { return 20; - }, - query: mockQuery + } }); sinon.stub(SurveyService.prototype, 'getLatestSurveyOccurrenceSubmission').resolves(({ @@ -122,7 +862,22 @@ describe('getObservationSubmission', () => { submission_status_type_name: 'Rejected' } as unknown) as IGetLatestSurveyOccurrenceSubmission); - sinon.stub(survey_queries, 'getOccurrenceSubmissionMessagesSQL').returns(SQL`something`); + sinon.stub(SurveyRepository.prototype, 'getOccurrenceSubmissionMessages').resolves([ + { + id: 1, + message: 'occurrence.txt - Missing Required Header - associatedTaxa - Missing required header', + status: SUBMISSION_STATUS_TYPE.REJECTED, + type: SUBMISSION_MESSAGE_TYPE.MISSING_REQUIRED_HEADER, + class: MESSAGE_CLASS_NAME.ERROR + }, + { + id: 2, + message: 'occurrence.txt - Missing Required Header - associatedTaxa - Missing required header', + status: SUBMISSION_STATUS_TYPE.REJECTED, + type: SUBMISSION_MESSAGE_TYPE.MISSING_REQUIRED_HEADER, + class: MESSAGE_CLASS_NAME.ERROR + } + ]); const result = observationSubmission.getOccurrenceSubmission(); @@ -132,20 +887,22 @@ describe('getObservationSubmission', () => { id: 13, inputFileName: 'dwca_moose.zip', status: 'Rejected', - messages: [ - { - errorCode: SUBMISSION_MESSAGE_TYPE.MISSING_REQUIRED_HEADER, - id: 1, - message: 'occurrence.txt - Missing Required Header - associatedTaxa - Missing required header', - status: 'Rejected', - type: 'Error' - }, + isValidating: false, + messageTypes: [ { - errorCode: SUBMISSION_MESSAGE_TYPE.MISSING_REQUIRED_HEADER, - id: 2, - message: 'occurrence.txt - Missing Required Header - associatedTaxa - Missing required header', - status: 'Rejected', - type: 'Error' + severityLabel: 'Error', + messageTypeLabel: 'Missing Required Header', + messageStatus: 'Rejected', + messages: [ + { + id: 1, + message: 'occurrence.txt - Missing Required Header - associatedTaxa - Missing required header' + }, + { + id: 2, + message: 'occurrence.txt - Missing Required Header - associatedTaxa - Missing required header' + } + ] } ] }); diff --git a/api/src/paths/project/{projectId}/survey/{surveyId}/observation/submission/get.ts b/api/src/paths/project/{projectId}/survey/{surveyId}/observation/submission/get.ts index e12e012661..167b6c3ad0 100644 --- a/api/src/paths/project/{projectId}/survey/{surveyId}/observation/submission/get.ts +++ b/api/src/paths/project/{projectId}/survey/{surveyId}/observation/submission/get.ts @@ -3,10 +3,8 @@ import { Operation } from 'express-openapi'; import { PROJECT_ROLE } from '../../../../../../../constants/roles'; import { SUBMISSION_STATUS_TYPE } from '../../../../../../../constants/status'; import { getDBConnection } from '../../../../../../../database/db'; -import { HTTP400 } from '../../../../../../../errors/http-error'; -import { queries } from '../../../../../../../queries/queries'; import { authorizeRequestHandler } from '../../../../../../../request-handlers/security/authorization'; -import { SurveyService } from '../../../../../../../services/survey-service'; +import { IMessageTypeGroup, SurveyService } from '../../../../../../../services/survey-service'; import { getLogger } from '../../../../../../../utils/logger'; const defaultLog = getLogger('/api/project/{projectId}/survey/{surveyId}/observation/submission/get'); @@ -39,7 +37,8 @@ GET.apiDoc = { in: 'path', name: 'projectId', schema: { - type: 'number' + type: 'number', + minimum: 1 }, required: true }, @@ -47,7 +46,8 @@ GET.apiDoc = { in: 'path', name: 'surveyId', schema: { - type: 'number' + type: 'number', + minimum: 1 }, required: true } @@ -60,9 +60,11 @@ GET.apiDoc = { schema: { type: 'object', nullable: true, + required: ['id', 'inputFileName', 'status', 'isValidating', 'messageTypes'], properties: { id: { - type: 'number' + type: 'number', + minimum: 1 }, inputFileName: { description: 'The file name of the submission', @@ -73,12 +75,50 @@ GET.apiDoc = { nullable: true, type: 'string' }, - messages: { - description: 'The validation status messages of the observation submission', + isValidating: { + description: 'True if the submission has not yet been validated, false otherwise', + type: 'boolean' + }, + messageTypes: { + description: 'An array containing all submission messages grouped by message type', type: 'array', items: { type: 'object', - description: 'A validation status message of the observation submission' + required: ['severityLabel', 'messageTypeLabel', 'messageStatus', 'messages'], + properties: { + severityLabel: { + type: 'string', + description: + 'The label of the "class" or severity of this type of message, e.g. "Error", "Warning", "Notice", etc.' + }, + messageTypeLabel: { + type: 'string', + description: 'The name of the type of error pertaining to this submission' + }, + messageStatus: { + type: 'string', + description: 'The resulting status of the submission as a consequence of the error' + }, + messages: { + type: 'array', + description: 'The array of submission messages belonging to this type of message', + items: { + type: 'object', + description: 'A submission message object belonging to a particular message type group', + required: ['id', 'message'], + properties: { + id: { + type: 'number', + description: 'The ID of this submission message' + }, + message: { + type: 'string', + description: 'The actual message which describes the concern in detail' + } + } + } + } + } } } } @@ -106,11 +146,11 @@ GET.apiDoc = { export function getOccurrenceSubmission(): RequestHandler { return async (req, res) => { - defaultLog.debug({ label: 'Get an occurrence submission', message: 'params', req_params: req.params }); - - if (!req.params.surveyId) { - throw new HTTP400('Missing required path param `surveyId`'); - } + defaultLog.debug({ + label: 'getOccurrenceSubmission', + description: 'Gets an occurrence submission', + req_params: req.params + }); const connection = getDBConnection(req['keycloak_token']); @@ -118,54 +158,35 @@ export function getOccurrenceSubmission(): RequestHandler { await connection.open(); const surveyService = new SurveyService(connection); - const response = await surveyService.getLatestSurveyOccurrenceSubmission(Number(req.params.surveyId)); + const occurrenceSubmission = await surveyService.getLatestSurveyOccurrenceSubmission(Number(req.params.surveyId)); - // Ensure we only retrieve the latest occurrence submission record if it has not been soft deleted - if (!response || response.delete_timestamp) { + if (!occurrenceSubmission || occurrenceSubmission.delete_timestamp) { + // Ensure we only retrieve the latest occurrence submission record if it has not been soft deleted return res.status(200).json(null); } - let messageList: any[] = []; - - const errorStatus = response.submission_status_type_name; - - if ( - errorStatus === SUBMISSION_STATUS_TYPE.REJECTED || - errorStatus === SUBMISSION_STATUS_TYPE.SYSTEM_ERROR || - errorStatus === SUBMISSION_STATUS_TYPE.FAILED_OCCURRENCE_PREPARATION || - errorStatus === SUBMISSION_STATUS_TYPE.FAILED_VALIDATION || - errorStatus === SUBMISSION_STATUS_TYPE.FAILED_TRANSFORMED || - errorStatus === SUBMISSION_STATUS_TYPE.FAILED_PROCESSING_OCCURRENCE_DATA - ) { - const occurrence_submission_id = response.id; - - const getSubmissionErrorListSQLStatement = queries.survey.getOccurrenceSubmissionMessagesSQL( - Number(occurrence_submission_id) - ); - - if (!getSubmissionErrorListSQLStatement) { - throw new HTTP400('Failed to build SQL getOccurrenceSubmissionMessagesSQL statement'); - } - - const submissionErrorListData = await connection.query( - getSubmissionErrorListSQLStatement.text, - getSubmissionErrorListSQLStatement.values - ); - - messageList = (submissionErrorListData && submissionErrorListData.rows) || []; - } - - await connection.commit(); - const getOccurrenceSubmissionData = - (response && { - id: response.id, - inputFileName: response.input_file_name, - status: response.submission_status_type_name, - messages: messageList - }) || - null; - - return res.status(200).json(getOccurrenceSubmissionData); + const hasAdditionalOccurrenceSubmissionMessages = + occurrenceSubmission.submission_status_type_name && + [ + SUBMISSION_STATUS_TYPE.REJECTED, + SUBMISSION_STATUS_TYPE.SYSTEM_ERROR, + SUBMISSION_STATUS_TYPE.FAILED_OCCURRENCE_PREPARATION, + SUBMISSION_STATUS_TYPE.FAILED_VALIDATION, + SUBMISSION_STATUS_TYPE.FAILED_TRANSFORMED, + SUBMISSION_STATUS_TYPE.FAILED_PROCESSING_OCCURRENCE_DATA + ].includes(occurrenceSubmission.submission_status_type_name); + + const messageTypes: IMessageTypeGroup[] = hasAdditionalOccurrenceSubmissionMessages + ? await surveyService.getOccurrenceSubmissionMessages(Number(occurrenceSubmission.id)) + : []; + + return res.status(200).json({ + id: occurrenceSubmission.id, + inputFileName: occurrenceSubmission.input_file_name, + status: occurrenceSubmission.submission_status_type_name || null, + isValidating: !hasAdditionalOccurrenceSubmissionMessages, + messageTypes + }); } catch (error) { defaultLog.error({ label: 'getOccurrenceSubmission', message: 'error', error }); await connection.rollback(); diff --git a/api/src/paths/project/{projectId}/survey/{surveyId}/observation/submission/upload.test.ts b/api/src/paths/project/{projectId}/survey/{surveyId}/observation/submission/upload.test.ts index b5565d6fec..6e7f08e44a 100644 --- a/api/src/paths/project/{projectId}/survey/{surveyId}/observation/submission/upload.test.ts +++ b/api/src/paths/project/{projectId}/survey/{surveyId}/observation/submission/upload.test.ts @@ -2,10 +2,9 @@ import chai, { expect } from 'chai'; import { describe } from 'mocha'; import sinon from 'sinon'; import sinonChai from 'sinon-chai'; -import SQL from 'sql-template-strings'; import * as db from '../../../../../../../database/db'; import { HTTPError } from '../../../../../../../errors/http-error'; -import survey_queries from '../../../../../../../queries/survey'; +import { SurveyService } from '../../../../../../../services/survey-service'; import * as file_utils from '../../../../../../../utils/file-utils'; import { getMockDBConnection, getRequestHandlerMocks } from '../../../../../../../__mocks__/db'; import * as upload from './upload'; @@ -203,11 +202,13 @@ describe('uploadObservationSubmission', () => { systemUserId: () => { return 20; }, - query: mockQuery + knex: mockQuery }); sinon.stub(file_utils, 'scanFileForVirus').resolves(true); - sinon.stub(survey_queries, 'insertSurveyOccurrenceSubmissionSQL').returns(SQL`some query`); + sinon + .stub(SurveyService.prototype, 'insertSurveyOccurrenceSubmission') + .resolves({ submissionId: (undefined as unknown) as number }); const requestHandler = upload.uploadMedia(); @@ -220,52 +221,6 @@ describe('uploadObservationSubmission', () => { } }); - it('should throw a 400 error when it fails to get the update SQL', async () => { - const dbConnectionObj = getMockDBConnection(); - - const { mockReq, mockRes, mockNext } = getRequestHandlerMocks(); - - mockReq.params = { - projectId: '1', - surveyId: '2' - }; - mockReq.files = [ - { - fieldname: 'media', - originalname: 'test.txt', - encoding: '7bit', - mimetype: 'text/plain', - size: 340 - } - ] as any; - - const mockQuery = sinon.stub(); - - mockQuery.onCall(0).resolves({ rowCount: 1, rows: [{ id: 1 }] }); - - sinon.stub(db, 'getDBConnection').returns({ - ...dbConnectionObj, - systemUserId: () => { - return 20; - }, - query: mockQuery - }); - - sinon.stub(file_utils, 'scanFileForVirus').resolves(true); - sinon.stub(survey_queries, 'insertSurveyOccurrenceSubmissionSQL').returns(SQL`some query`); - sinon.stub(survey_queries, 'updateSurveyOccurrenceSubmissionSQL').returns(null); - - const requestHandler = upload.uploadMedia(); - - try { - await requestHandler(mockReq, mockRes, mockNext); - expect.fail(); - } catch (actualError) { - expect((actualError as HTTPError).status).to.equal(400); - expect((actualError as HTTPError).message).to.equal('Failed to build SQL update statement'); - } - }); - it('should throw a 400 error when it fails to get the update the record in the database', async () => { const dbConnectionObj = getMockDBConnection(); @@ -287,20 +242,18 @@ describe('uploadObservationSubmission', () => { const mockQuery = sinon.stub(); - mockQuery.onCall(0).resolves({ rowCount: 1, rows: [{ id: 1 }] }); - mockQuery.onCall(1).resolves(null); + mockQuery.onCall(0).resolves({ rowCount: 1, rows: [{ submissionId: 1 }] }); + mockQuery.onCall(1).resolves({ rowCount: 1, rows: [{ submissionId: undefined }] }); sinon.stub(db, 'getDBConnection').returns({ ...dbConnectionObj, systemUserId: () => { return 20; }, - query: mockQuery + knex: mockQuery }); sinon.stub(file_utils, 'scanFileForVirus').resolves(true); - sinon.stub(survey_queries, 'insertSurveyOccurrenceSubmissionSQL').returns(SQL`some query`); - sinon.stub(survey_queries, 'updateSurveyOccurrenceSubmissionSQL').returns(SQL`some query`); const requestHandler = upload.uploadMedia(); @@ -334,20 +287,18 @@ describe('uploadObservationSubmission', () => { const mockQuery = sinon.stub(); - mockQuery.onCall(0).resolves({ rowCount: 1, rows: [{ id: 1 }] }); - mockQuery.onCall(1).resolves({ rowCount: 1, rows: [{ id: 1 }] }); + mockQuery.onCall(0).resolves({ rowCount: 1, rows: [{ submissionId: 1 }] }); + mockQuery.onCall(1).resolves({ rowCount: 1, rows: [{ submissionId: 1 }] }); sinon.stub(db, 'getDBConnection').returns({ ...dbConnectionObj, systemUserId: () => { return 20; }, - query: mockQuery + knex: mockQuery }); sinon.stub(file_utils, 'scanFileForVirus').resolves(true); - sinon.stub(survey_queries, 'insertSurveyOccurrenceSubmissionSQL').returns(SQL`some query`); - sinon.stub(survey_queries, 'updateSurveyOccurrenceSubmissionSQL').returns(SQL`some query`); sinon.stub(file_utils, 'uploadFileToS3').rejects('Failed to insert occurrence submission data'); const requestHandler = upload.uploadMedia(); @@ -385,21 +336,19 @@ describe('uploadObservationSubmission', () => { const mockQuery = sinon.stub(); - mockQuery.onCall(0).resolves({ rowCount: 1, rows: [{ id: 1 }] }); - mockQuery.onCall(1).resolves({ rowCount: 1, rows: [{ id: 1 }] }); - mockQuery.onCall(2).resolves({ rowCount: 1, rows: [{ id: 1 }] }); + mockQuery.onCall(0).resolves({ rowCount: 1, rows: [{ submissionId: 1 }] }); + mockQuery.onCall(1).resolves({ rowCount: 1, rows: [{ submissionId: 1 }] }); + mockQuery.onCall(2).resolves({ rowCount: 1, rows: [{ submissionId: 1 }] }); sinon.stub(db, 'getDBConnection').returns({ ...dbConnectionObj, systemUserId: () => { return 20; }, - query: mockQuery + knex: mockQuery }); sinon.stub(file_utils, 'scanFileForVirus').resolves(true); - sinon.stub(survey_queries, 'insertSurveyOccurrenceSubmissionSQL').returns(SQL`some query`); - sinon.stub(survey_queries, 'updateSurveyOccurrenceSubmissionSQL').returns(SQL`some query`); sinon.stub(file_utils, 'uploadFileToS3').resolves({ key: 'projects/1/surveys/1/test.txt' } as any); const requestHandler = upload.uploadMedia(); @@ -407,49 +356,4 @@ describe('uploadObservationSubmission', () => { await requestHandler(mockReq, mockRes, mockNext); expect(mockRes.statusValue).to.equal(200); }); - - it('should throw a 400 error when it fails to get the insertSurveyOccurrenceSubmissionSQL SQL', async () => { - const dbConnectionObj = getMockDBConnection(); - - const { mockReq, mockRes, mockNext } = getRequestHandlerMocks(); - - mockReq.params = { - projectId: '1', - surveyId: '2' - }; - mockReq.files = [ - { - fieldname: 'media', - originalname: 'test.txt', - encoding: '7bit', - mimetype: 'text/plain', - size: 340 - } - ] as any; - - const mockQuery = sinon.stub(); - - mockQuery.onCall(0).resolves({ rowCount: 1, rows: [{ id: 1 }] }); - - sinon.stub(db, 'getDBConnection').returns({ - ...dbConnectionObj, - systemUserId: () => { - return 20; - }, - query: mockQuery - }); - - sinon.stub(file_utils, 'scanFileForVirus').resolves(true); - sinon.stub(survey_queries, 'insertSurveyOccurrenceSubmissionSQL').returns(null); - - const requestHandler = upload.uploadMedia(); - - try { - await requestHandler(mockReq, mockRes, mockNext); - expect.fail(); - } catch (actualError) { - expect((actualError as HTTPError).status).to.equal(400); - expect((actualError as HTTPError).message).to.equal('Failed to build SQL insert statement'); - } - }); }); diff --git a/api/src/paths/project/{projectId}/survey/{surveyId}/observation/submission/upload.ts b/api/src/paths/project/{projectId}/survey/{surveyId}/observation/submission/upload.ts index c15fca4b64..4555db88d3 100644 --- a/api/src/paths/project/{projectId}/survey/{surveyId}/observation/submission/upload.ts +++ b/api/src/paths/project/{projectId}/survey/{surveyId}/observation/submission/upload.ts @@ -3,8 +3,8 @@ import { Operation } from 'express-openapi'; import { PROJECT_ROLE } from '../../../../../../../constants/roles'; import { getDBConnection, IDBConnection } from '../../../../../../../database/db'; import { HTTP400 } from '../../../../../../../errors/http-error'; -import { queries } from '../../../../../../../queries/queries'; import { authorizeRequestHandler } from '../../../../../../../request-handlers/security/authorization'; +import { SurveyService } from '../../../../../../../services/survey-service'; import { generateS3FileKey, scanFileForVirus, uploadFileToS3 } from '../../../../../../../utils/file-utils'; import { getLogger } from '../../../../../../../utils/logger'; @@ -152,7 +152,7 @@ export function uploadMedia(): RequestHandler { connection ); - const submissionId = response.rows[0].id; + const { submissionId } = response; const inputKey = generateS3FileKey({ projectId: Number(req.params.projectId), @@ -199,24 +199,20 @@ export const insertSurveyOccurrenceSubmission = async ( source: string, inputFileName: string, connection: IDBConnection -): Promise => { - const insertSqlStatement = queries.survey.insertSurveyOccurrenceSubmissionSQL({ +): Promise<{ submissionId: number }> => { + const surveyService = new SurveyService(connection); + + const response = await surveyService.insertSurveyOccurrenceSubmission({ surveyId, source, inputFileName }); - if (!insertSqlStatement) { - throw new HTTP400('Failed to build SQL insert statement'); - } - - const insertResponse = await connection.query(insertSqlStatement.text, insertSqlStatement.values); - - if (!insertResponse.rowCount) { + if (!response.submissionId) { throw new HTTP400('Failed to insert survey occurrence submission record'); } - return insertResponse; + return response; }; /** @@ -231,18 +227,14 @@ export const updateSurveyOccurrenceSubmissionWithKey = async ( submissionId: number, inputKey: string, connection: IDBConnection -): Promise => { - const updateSqlStatement = queries.survey.updateSurveyOccurrenceSubmissionSQL({ submissionId, inputKey }); - - if (!updateSqlStatement) { - throw new HTTP400('Failed to build SQL update statement'); - } +): Promise<{ submissionId: number }> => { + const surveyService = new SurveyService(connection); - const updateResponse = await connection.query(updateSqlStatement.text, updateSqlStatement.values); + const response = await surveyService.updateSurveyOccurrenceSubmission({ submissionId, inputKey }); - if (!updateResponse || !updateResponse.rowCount) { + if (!response.submissionId) { throw new HTTP400('Failed to update survey occurrence submission record'); } - return updateResponse; + return response; }; diff --git a/api/src/paths/xlsx/process.ts b/api/src/paths/xlsx/process.ts index aff6b0957a..2adcea6cba 100644 --- a/api/src/paths/xlsx/process.ts +++ b/api/src/paths/xlsx/process.ts @@ -8,6 +8,7 @@ import { authorizeRequestHandler } from '../../request-handlers/security/authori import { ErrorService } from '../../services/error-service'; import { ValidationService } from '../../services/validation-service'; import { getLogger } from '../../utils/logger'; +import { SubmissionError } from '../../utils/submission-error'; const defaultLog = getLogger('paths/xlsx/process'); @@ -111,10 +112,19 @@ export function processFile(): RequestHandler { await connection.open(); const validationService = new ValidationService(connection); - // process the raw template data - await validationService.processXLSXFile(submissionId, surveyId); - // process the resulting transformed dwc data - await validationService.processDWCFile(submissionId); + + try { + // process the raw template data + await validationService.processXLSXFile(submissionId, surveyId); + // process the resulting transformed dwc data + await validationService.processDWCFile(submissionId); + } catch (error: any) { + // Since submission errors are caught by the validation service and persisted in the database, anything + // outside of a submission message should be thrown here. + if (!(error instanceof SubmissionError)) { + throw error; + } + } await connection.commit(); } catch (error) { diff --git a/api/src/queries/queries.ts b/api/src/queries/queries.ts index d3523e5d44..25f0fd4937 100644 --- a/api/src/queries/queries.ts +++ b/api/src/queries/queries.ts @@ -5,7 +5,6 @@ import project from './project'; import projectParticipation from './project-participation'; import search from './search'; import spatial from './spatial'; -import survey from './survey'; import users from './users'; export const queries = { @@ -16,6 +15,5 @@ export const queries = { projectParticipation, search, spatial, - survey, users }; diff --git a/api/src/queries/survey/index.ts b/api/src/queries/survey/index.ts deleted file mode 100644 index a04eab7066..0000000000 --- a/api/src/queries/survey/index.ts +++ /dev/null @@ -1,5 +0,0 @@ -import * as surveyOccurrence from './survey-occurrence-queries'; - -export default { - ...surveyOccurrence -}; diff --git a/api/src/queries/survey/survey-occurrence-queries.test.ts b/api/src/queries/survey/survey-occurrence-queries.test.ts deleted file mode 100644 index d9055f7673..0000000000 --- a/api/src/queries/survey/survey-occurrence-queries.test.ts +++ /dev/null @@ -1,151 +0,0 @@ -import { expect } from 'chai'; -import { describe } from 'mocha'; -import { - deleteOccurrenceSubmissionSQL, - getOccurrenceSubmissionMessagesSQL, - insertSurveyOccurrenceSubmissionSQL, - updateSurveyOccurrenceSubmissionSQL -} from './survey-occurrence-queries'; - -describe('insertSurveyOccurrenceSubmissionSQL', () => { - it('returns null response when null surveyId provided', () => { - const response = insertSurveyOccurrenceSubmissionSQL({ - surveyId: (null as unknown) as number, - source: 'fileSource', - inputKey: 'fileKey' - }); - - expect(response).to.be.null; - }); - - it('returns null response when null source provided', () => { - const response = insertSurveyOccurrenceSubmissionSQL({ - surveyId: 1, - source: (null as unknown) as string, - inputKey: 'fileKey' - }); - - expect(response).to.be.null; - }); - - it('returns non null response when all valid params provided without inputKey', () => { - const response = insertSurveyOccurrenceSubmissionSQL({ - surveyId: 1, - source: 'fileSource', - inputFileName: 'inputFileName', - outputFileName: 'outputFileName', - outputKey: 'outputfileKey' - }); - - expect(response).to.not.be.null; - }); - - it('returns non null response when all valid params provided with inputKey', () => { - const response = insertSurveyOccurrenceSubmissionSQL({ - surveyId: 1, - source: 'fileSource', - inputFileName: 'inputFileName', - inputKey: 'inputfileKey', - outputFileName: 'outputFileName', - outputKey: 'outputfileKey' - }); - - expect(response).to.not.be.null; - }); -}); - -describe('deleteOccurrenceSubmissionSQL', () => { - it('returns null response when null submissionId provided', () => { - const response = deleteOccurrenceSubmissionSQL((null as unknown) as number); - - expect(response).to.be.null; - }); - - it('returns non null response when valid params provided', () => { - const response = deleteOccurrenceSubmissionSQL(1); - - expect(response).to.not.be.null; - }); -}); - -describe('updateSurveyOccurrenceSubmissionSQL', () => { - it('returns null response when null surveyId provided', () => { - const response = updateSurveyOccurrenceSubmissionSQL({ - submissionId: (null as unknown) as number, - inputKey: 'fileKey' - }); - - expect(response).to.be.null; - }); - - it('returns null response when null key provided', () => { - const response = updateSurveyOccurrenceSubmissionSQL({ - submissionId: 1, - inputKey: (null as unknown) as string - }); - - expect(response).to.be.null; - }); - - it('returns non null response when valid params provided', () => { - const response = updateSurveyOccurrenceSubmissionSQL({ - submissionId: 1, - inputKey: 'fileKey', - inputFileName: 'fileName', - outputFileName: 'outputFileName', - outputKey: 'outputKey' - }); - - expect(response).to.not.be.null; - }); - - it('returns non null response when valid params provided without inputKey', () => { - const response = updateSurveyOccurrenceSubmissionSQL({ - submissionId: 1, - inputKey: 'fileKey' - }); - - expect(response).to.not.be.null; - }); - - it('returns non null response when valid params provided without inputFileName', () => { - const response = updateSurveyOccurrenceSubmissionSQL({ - submissionId: 1, - inputFileName: 'fileName' - }); - - expect(response).to.not.be.null; - }); - - it('returns non null response when valid params provided without outputFileName', () => { - const response = updateSurveyOccurrenceSubmissionSQL({ - submissionId: 1, - outputFileName: 'outputFileName' - }); - - expect(response).to.not.be.null; - }); - - it('returns non null response when valid params provided without outputKey', () => { - const response = updateSurveyOccurrenceSubmissionSQL({ - submissionId: 1, - outputKey: 'outputKey' - }); - - expect(response).to.not.be.null; - }); -}); - -describe('getOccurrenceSubmissionMessagesSQL', () => { - it('returns null response when null occurrenceSubmissionId provided', () => { - const response = getOccurrenceSubmissionMessagesSQL((null as unknown) as number); - - expect(response).to.be.null; - }); - - it('returns non null response when valid params provided', () => { - const response = getOccurrenceSubmissionMessagesSQL(1); - - expect(response).to.not.be.null; - }); -}); diff --git a/api/src/queries/survey/survey-occurrence-queries.ts b/api/src/queries/survey/survey-occurrence-queries.ts deleted file mode 100644 index 6dc627c7dd..0000000000 --- a/api/src/queries/survey/survey-occurrence-queries.ts +++ /dev/null @@ -1,204 +0,0 @@ -import { SQL, SQLStatement } from 'sql-template-strings'; -import { - AppendSQLColumn, - appendSQLColumns, - AppendSQLColumnsEqualValues, - appendSQLColumnsEqualValues, - AppendSQLValue, - appendSQLValues -} from '../../utils/sql-utils'; - -/** - * SQL query to insert a survey occurrence submission row. - * - * @param {number} surveyId - * @param {string} source - * @param {string} inputFileName - * @param {(number | null)} templateMethodologyId - * @return {*} {(SQLStatement | null)} - */ -export const insertSurveyOccurrenceSubmissionSQL = (data: { - surveyId: number; - source: string; - inputFileName?: string; - inputKey?: string; - outputFileName?: string; - outputKey?: string; -}): SQLStatement | null => { - if (!data || !data.surveyId || !data.source) { - return null; - } - - const dataKeys = Object.keys(data); - - const columnItems: AppendSQLColumn[] = []; - const valueItems: AppendSQLValue[] = []; - - if (dataKeys.includes('inputFileName')) { - columnItems.push({ columnName: 'input_file_name' }); - valueItems.push({ columnValue: data.inputFileName }); - } - - if (dataKeys.includes('inputKey')) { - columnItems.push({ columnName: 'input_key' }); - valueItems.push({ columnValue: data.inputKey }); - } - - if (dataKeys.includes('outputFileName')) { - columnItems.push({ columnName: 'output_file_name' }); - valueItems.push({ columnValue: data.outputFileName }); - } - - if (dataKeys.includes('outputKey')) { - columnItems.push({ columnName: 'output_key' }); - valueItems.push({ columnValue: data.outputKey }); - } - - const sqlStatement: SQLStatement = SQL` - INSERT INTO occurrence_submission ( - survey_id, - source, - event_timestamp, - `; - - appendSQLColumns(sqlStatement, columnItems); - - sqlStatement.append(SQL` - ) VALUES ( - ${data.surveyId}, - ${data.source}, - now(), - `); - - appendSQLValues(sqlStatement, valueItems); - - sqlStatement.append(SQL` - ) - RETURNING - occurrence_submission_id as id; - `); - - return sqlStatement; -}; - -/** - * SQL query to update a survey occurrence submission row. - * - * @param {{ - * submissionId: number; - * inputKey?: string; - * outputFileName?: string; - * outputKey?: string; - * }} data - * @return {*} {(SQLStatement | null)} - */ -export const updateSurveyOccurrenceSubmissionSQL = (data: { - submissionId: number; - inputFileName?: string; - inputKey?: string; - outputFileName?: string; - outputKey?: string; -}): SQLStatement | null => { - if (!data.submissionId || (!data.inputFileName && !data.inputKey && !data.outputFileName && !data.outputKey)) { - return null; - } - - const dataKeys = Object.keys(data); - - const items: AppendSQLColumnsEqualValues[] = []; - - if (dataKeys.includes('inputFileName')) { - items.push({ columnName: 'input_file_name', columnValue: data.inputFileName }); - } - - if (dataKeys.includes('inputKey')) { - items.push({ columnName: 'input_key', columnValue: data.inputKey }); - } - - if (dataKeys.includes('outputFileName')) { - items.push({ columnName: 'output_file_name', columnValue: data.outputFileName }); - } - - if (dataKeys.includes('outputKey')) { - items.push({ columnName: 'output_key', columnValue: data.outputKey }); - } - - const sqlStatement: SQLStatement = SQL` - UPDATE occurrence_submission - SET - `; - - appendSQLColumnsEqualValues(sqlStatement, items); - - sqlStatement.append(SQL` - WHERE - occurrence_submission_id = ${data.submissionId} - RETURNING occurrence_submission_id as id; - `); - - return sqlStatement; -}; - -/** - * SQL query to soft delete the occurrence submission entry by ID - * - * @param {number} occurrenceSubmissionId - * @returns {SQLStatement} sql query object - */ -export const deleteOccurrenceSubmissionSQL = (occurrenceSubmissionId: number): SQLStatement | null => { - if (!occurrenceSubmissionId) { - return null; - } - - return SQL` - UPDATE occurrence_submission - SET delete_timestamp = now() - WHERE occurrence_submission_id = ${occurrenceSubmissionId}; - `; -}; - -/** - * SQL query to get the list of messages for an occurrence submission. - * - * @param {number} occurrenceSubmissionId - * @returns {SQLStatement} sql query object - */ -export const getOccurrenceSubmissionMessagesSQL = (occurrenceSubmissionId: number): SQLStatement | null => { - if (!occurrenceSubmissionId) { - return null; - } - - return SQL` - SELECT - sm.submission_message_id as id, - smt.name as type, - sst.name as status, - smc.name as class, - sm.message - FROM - occurrence_submission as os - LEFT OUTER JOIN - submission_status as ss - ON - os.occurrence_submission_id = ss.occurrence_submission_id - LEFT OUTER JOIN - submission_status_type as sst - ON - sst.submission_status_type_id = ss.submission_status_type_id - LEFT OUTER JOIN - submission_message as sm - ON - sm.submission_status_id = ss.submission_status_id - LEFT OUTER JOIN - submission_message_type as smt - ON - smt.submission_message_type_id = sm.submission_message_type_id - LEFT OUTER JOIN - submission_message_class smc - ON - smc.submission_message_class_id = smt.submission_message_class_id - WHERE - os.occurrence_submission_id = ${occurrenceSubmissionId} - ORDER BY sm.submission_message_id; - `; -}; diff --git a/api/src/repositories/survey-repository.test.ts b/api/src/repositories/survey-repository.test.ts index 2049905979..c12bba1e11 100644 --- a/api/src/repositories/survey-repository.test.ts +++ b/api/src/repositories/survey-repository.test.ts @@ -15,7 +15,11 @@ import { GetSurveyPurposeAndMethodologyData } from '../models/survey-view'; import { getMockDBConnection } from '../__mocks__/db'; -import { SurveyRepository } from './survey-repository'; +import { + IObservationSubmissionInsertDetails, + IObservationSubmissionUpdateDetails, + SurveyRepository +} from './survey-repository'; chai.use(sinonChai); @@ -799,4 +803,133 @@ describe('SurveyRepository', () => { } }); }); + + describe('getOccurrenceSubmissionMessages', () => { + it('should return result', async () => { + const mockResponse = ({ + rows: [ + { + id: 1, + type: 'type', + status: 'status', + class: 'class', + message: 'message' + } + ], + rowCount: 1 + } as any) as Promise>; + + const dbConnection = getMockDBConnection({ sql: () => mockResponse }); + + const repository = new SurveyRepository(dbConnection); + + const response = await repository.getOccurrenceSubmissionMessages(1); + + expect(response).to.eql([ + { + id: 1, + type: 'type', + status: 'status', + class: 'class', + message: 'message' + } + ]); + }); + + it('should return empty array', async () => { + const mockResponse = ({ rows: [], rowCount: 0 } as any) as Promise>; + const dbConnection = getMockDBConnection({ sql: () => mockResponse }); + + const repository = new SurveyRepository(dbConnection); + + const response = await repository.getOccurrenceSubmissionMessages(1); + + expect(response).to.eql([]); + }); + }); + + describe('insertSurveyOccurrenceSubmission', () => { + it('should return result', async () => { + const mockResponse = ({ rows: [{ submissionId: 1 }], rowCount: 1 } as any) as Promise>; + const dbConnection = getMockDBConnection({ knex: () => mockResponse }); + + const repository = new SurveyRepository(dbConnection); + + const response = await repository.insertSurveyOccurrenceSubmission({ + surveyId: 1 + } as IObservationSubmissionInsertDetails); + + expect(response).to.eql({ submissionId: 1 }); + }); + + it('should throw an error', async () => { + const mockResponse = (undefined as any) as Promise>; + const dbConnection = getMockDBConnection({ knex: () => mockResponse }); + + const repository = new SurveyRepository(dbConnection); + + try { + await repository.insertSurveyOccurrenceSubmission({ surveyId: 1 } as IObservationSubmissionInsertDetails); + expect.fail(); + } catch (error) { + expect((error as Error).message).to.equal('Failed to insert survey occurrence submission'); + } + }); + }); + + describe('updateSurveyOccurrenceSubmission', () => { + it('should return result', async () => { + const mockResponse = ({ rows: [{ submissionId: 1 }], rowCount: 1 } as any) as Promise>; + const dbConnection = getMockDBConnection({ knex: () => mockResponse }); + + const repository = new SurveyRepository(dbConnection); + + const response = await repository.updateSurveyOccurrenceSubmission({ + submissionId: 1 + } as IObservationSubmissionUpdateDetails); + + expect(response).to.eql({ submissionId: 1 }); + }); + + it('should throw an error', async () => { + const mockResponse = (undefined as any) as Promise>; + const dbConnection = getMockDBConnection({ knex: () => mockResponse }); + + const repository = new SurveyRepository(dbConnection); + + try { + await repository.updateSurveyOccurrenceSubmission({ submissionId: 1 } as IObservationSubmissionUpdateDetails); + expect.fail(); + } catch (error) { + expect((error as Error).message).to.equal('Failed to update survey occurrence submission'); + } + }); + }); + + describe('deleteOccurrenceSubmission', () => { + it('should return 1 upon success', async () => { + const mockResponse = ({ rows: [{ submissionId: 2 }], rowCount: 1 } as any) as Promise>; + const dbConnection = getMockDBConnection({ knex: () => mockResponse }); + + const repository = new SurveyRepository(dbConnection); + + const response = await repository.deleteOccurrenceSubmission(2); + + expect(response).to.eql(1); + }); + + it('should throw an error upon failure', async () => { + const mockResponse = ({ rows: [], rowCount: 0 } as any) as Promise>; + const dbConnection = getMockDBConnection({ knex: () => mockResponse }); + + const repository = new SurveyRepository(dbConnection); + + try { + await await repository.deleteOccurrenceSubmission(2); + expect.fail(); + } catch (error) { + expect((error as Error).message).to.equal('Failed to delete survey occurrence submission'); + } + }); + }); }); diff --git a/api/src/repositories/survey-repository.ts b/api/src/repositories/survey-repository.ts index 84306c3610..78a48b1842 100644 --- a/api/src/repositories/survey-repository.ts +++ b/api/src/repositories/survey-repository.ts @@ -1,4 +1,5 @@ import SQL from 'sql-template-strings'; +import { MESSAGE_CLASS_NAME, SUBMISSION_MESSAGE_TYPE, SUBMISSION_STATUS_TYPE } from '../constants/status'; import { getKnex } from '../database/db'; import { ApiExecuteSQLError } from '../errors/api-error'; import { PostProprietorData, PostSurveyObject } from '../models/survey-create'; @@ -13,6 +14,7 @@ import { GetSurveyPurposeAndMethodologyData } from '../models/survey-view'; import { queries } from '../queries/queries'; +import { getLogger } from '../utils/logger'; import { BaseRepository } from './base-repository'; export interface IGetSpeciesData { @@ -32,13 +34,40 @@ export interface IGetLatestSurveyOccurrenceSubmission { output_file_name: string; submission_status_id: number; submission_status_type_id: number; - submission_status_type_name: string; + submission_status_type_name?: SUBMISSION_STATUS_TYPE; submission_message_id: number; submission_message_type_id: number; message: string; submission_message_type_name: string; } +export interface IOccurrenceSubmissionMessagesResponse { + id: number; + class: MESSAGE_CLASS_NAME; + type: SUBMISSION_MESSAGE_TYPE; + status: SUBMISSION_STATUS_TYPE; + message: string; +} + +export interface IObservationSubmissionInsertDetails { + surveyId: number; + source: string; + inputFileName?: string; + inputKey?: string; + outputFileName?: string; + outputKey?: string; +} + +export interface IObservationSubmissionUpdateDetails { + submissionId: number; + inputFileName?: string; + inputKey?: string; + outputFileName?: string; + outputKey?: string; +} + +const defaultLog = getLogger('repositories/survey-repository'); + export class SurveyRepository extends BaseRepository { async deleteSurvey(surveyId: number): Promise { const sqlStatement = SQL`call api_delete_survey(${surveyId})`; @@ -345,6 +374,59 @@ export class SurveyRepository extends BaseRepository { return result; } + /** + * SQL query to get the list of messages for an occurrence submission. + * + * @param {number} submissionId The ID of the submission + * @returns {*} Promise Promsie resolving the array of submission messages + */ + async getOccurrenceSubmissionMessages(submissionId: number): Promise { + const sqlStatement = SQL` + SELECT + sm.submission_message_id as id, + smt.name as type, + sst.name as status, + smc.name as class, + sm.message + FROM + occurrence_submission as os + LEFT OUTER JOIN + submission_status as ss + ON + os.occurrence_submission_id = ss.occurrence_submission_id + LEFT OUTER JOIN + submission_status_type as sst + ON + sst.submission_status_type_id = ss.submission_status_type_id + LEFT OUTER JOIN + submission_message as sm + ON + sm.submission_status_id = ss.submission_status_id + LEFT OUTER JOIN + submission_message_type as smt + ON + smt.submission_message_type_id = sm.submission_message_type_id + LEFT OUTER JOIN + submission_message_class smc + ON + smc.submission_message_class_id = smt.submission_message_class_id + WHERE + os.occurrence_submission_id = ${submissionId} + ORDER BY sm.submission_message_id; + `; + + const response = await this.connection.sql(sqlStatement); + + if (!response?.rows) { + throw new ApiExecuteSQLError('Failed to get occurrence submission messages', [ + 'SurveyRepository->getOccurrenceSubmissionMessages', + 'response was null or undefined, expected response != null' + ]); + } + + return response.rows; + } + async getSummaryResultId(surveyId: number): Promise { const sqlStatement = SQL` SELECT @@ -815,4 +897,100 @@ export class SurveyRepository extends BaseRepository { await this.connection.sql(sqlStatement); } + + /** + * Inserts a survey occurrence submission row. + * + * @param {IObservationSubmissionInsertDetails} submission The details of the submission + * @return {*} {Promise<{ submissionId: number }>} Promise resolving the ID of the submission upon successful insertion + */ + async insertSurveyOccurrenceSubmission( + submission: IObservationSubmissionInsertDetails + ): Promise<{ submissionId: number }> { + defaultLog.debug({ label: 'insertSurveyOccurrenceSubmission', submission }); + const queryBuilder = getKnex() + .table('occurrence_submission') + .insert({ + input_file_name: submission.inputFileName, + input_key: submission.inputKey, + output_file_name: submission.outputFileName, + output_key: submission.outputKey, + survey_id: submission.surveyId, + source: submission.source, + event_timestamp: `now()` + }) + .returning('occurrence_submission_id as submissionId'); + + const response = await this.connection.knex<{ submissionId: number }>(queryBuilder); + + if (!response || response.rowCount !== 1) { + throw new ApiExecuteSQLError('Failed to insert survey occurrence submission', [ + 'ErrorRepository->insertSurveyOccurrenceSubmission', + 'rowCount was null or undefined, expected rowCount = 1' + ]); + } + + return response.rows[0]; + } + + /** + * Updates a survey occurrence submission with the given details. + * + * @param {IObservationSubmissionUpdateDetails} submission The details of the submission to be updated + * @return {*} {Promise<{ submissionId: number }>} Promise resolving the ID of the submission upon successfully updating it + */ + async updateSurveyOccurrenceSubmission( + submission: IObservationSubmissionUpdateDetails + ): Promise<{ submissionId: number }> { + defaultLog.debug({ label: 'updateSurveyOccurrenceSubmission', submission }); + const queryBuilder = getKnex() + .table('occurrence_submission') + .update({ + input_file_name: submission.inputFileName, + input_key: submission.inputKey, + output_file_name: submission.outputFileName, + output_key: submission.outputKey + }) + .where('occurrence_submission_id', submission.submissionId) + .returning('occurrence_submission_id as submissionId'); + + const response = await this.connection.knex<{ submissionId: number }>(queryBuilder); + + if (!response || response.rowCount !== 1) { + throw new ApiExecuteSQLError('Failed to update survey occurrence submission', [ + 'ErrorRepository->updateSurveyOccurrenceSubmission', + 'rowCount was null or undefined, expected rowCount = 1' + ]); + } + + return response.rows[0]; + } + + /** + * Soft-deletes an occurrence submission. + * + * @param {number} submissionId The ID of the submission to soft delete + * @returns {*} {number} The row count of the affected records, namely `1` if the delete succeeds, `0` if it does not + */ + async deleteOccurrenceSubmission(submissionId: number): Promise { + defaultLog.debug({ label: 'deleteOccurrenceSubmission', submissionId }); + const queryBuilder = getKnex() + .table('occurrence_submission') + .update({ + delete_timestamp: `now()` + }) + .where('occurrence_submission_id', submissionId) + .returning('occurrence_submission_id as submissionId'); + + const response = await this.connection.knex<{ submissionId: number }>(queryBuilder); + + if (!response || response.rowCount !== 1) { + throw new ApiExecuteSQLError('Failed to delete survey occurrence submission', [ + 'ErrorRepository->deleteOccurrenceSubmission', + 'rowCount was null or undefined, expected rowCount = 1' + ]); + } + + return response.rowCount; + } } diff --git a/api/src/repositories/validation-repository.test.ts b/api/src/repositories/validation-repository.test.ts index d2c395c28b..6f5a464543 100644 --- a/api/src/repositories/validation-repository.test.ts +++ b/api/src/repositories/validation-repository.test.ts @@ -31,6 +31,7 @@ describe('ValidationRepository', () => { rows: [ { template_methodology_species_id: 1, + wldtaxonomic_units_id: '10', validation: '{}', transform: '{}' } diff --git a/api/src/repositories/validation-repository.ts b/api/src/repositories/validation-repository.ts index 26fe775beb..dbac8c6e25 100644 --- a/api/src/repositories/validation-repository.ts +++ b/api/src/repositories/validation-repository.ts @@ -1,11 +1,14 @@ import SQL from 'sql-template-strings'; +import { SUBMISSION_MESSAGE_TYPE, SUBMISSION_STATUS_TYPE } from '../constants/status'; import { getKnex } from '../database/db'; import { HTTP400 } from '../errors/http-error'; import { TransformSchema } from '../utils/media/xlsx/transformation/xlsx-transform-schema-parser'; +import { MessageError, SubmissionError } from '../utils/submission-error'; import { BaseRepository } from './base-repository'; export interface ITemplateMethodologyData { template_methodology_species_id: number; + wldtaxonomic_units_id: string; validation: string; transform: TransformSchema; } @@ -28,31 +31,61 @@ export class ValidationRepository extends BaseRepository { surveySpecies: number[] ): Promise { const templateRow = await this.getTemplateNameVersionId(templateName, templateVersion); - // TODO throw proper error if `templateRow` or `templateRow.template_id` are empty + + const failedToFindValidationRulesError = new SubmissionError({ + status: SUBMISSION_STATUS_TYPE.FAILED_VALIDATION, + messages: [ + new MessageError( + SUBMISSION_MESSAGE_TYPE.FAILED_GET_VALIDATION_RULES, + `Could not find any validation schema associated with Template Name "${templateName}" and Template Version "${templateVersion}".` + ) + ] + }); + + // No template validation record is found for the given template name and version + if (!templateRow) { + throw failedToFindValidationRulesError; + } + const queryBuilder = getKnex() - .select( - 'template_methodology_species.template_methodology_species_id', - 'template_methodology_species.validation', - 'template_methodology_species.transform' - ) + .select('template_methodology_species_id', 'wldtaxonomic_units_id', 'validation', 'transform') .from('template_methodology_species') - .where('template_methodology_species.template_id', templateRow.template_id) - .and.whereIn( - 'template_methodology_species.wldtaxonomic_units_id', - (Array.isArray(surveySpecies) && surveySpecies) || [surveySpecies] - ) + .where('template_id', templateRow.template_id) .and.where(function (qb) { - qb.or.where('template_methodology_species.field_method_id', surveyFieldMethodId); - qb.or.where('template_methodology_species.field_method_id', null); + qb.or.where('field_method_id', surveyFieldMethodId); + qb.or.where('field_method_id', null); }); const response = await this.connection.knex(queryBuilder); + // Querying the template methodology species table fails if (!response || !response.rows) { throw new HTTP400('Failed to query template methodology species table'); } - return response.rows[0]; + // Failure to find a template methodology species record for this template name and verion; Should yield a validation error. + if (response.rows.length === 0) { + throw failedToFindValidationRulesError; + } + + // Some template methodology species records are found for this template name and version, but none are associated with this + // particular surveySpecies, indicating that the wrong focal species was likely selected. + if (!response.rows.some((row) => surveySpecies.includes(Number(row.wldtaxonomic_units_id)))) { + throw new SubmissionError({ + status: SUBMISSION_STATUS_TYPE.FAILED_VALIDATION, + messages: [ + new MessageError( + SUBMISSION_MESSAGE_TYPE.MISMATCHED_TEMPLATE_SURVEY_SPECIES, + 'The focal species imported from this template does not match the focal species selected for this survey.' + ) + ] + }); + } + + // Return the first result among all records that match on the given surveySpecies. + return response.rows.filter((row) => { + return surveySpecies.includes(Number(row.wldtaxonomic_units_id)); + })[0]; } /** diff --git a/api/src/services/survey-service.test.ts b/api/src/services/survey-service.test.ts index 37351b2f15..648c0bf2e6 100644 --- a/api/src/services/survey-service.test.ts +++ b/api/src/services/survey-service.test.ts @@ -3,7 +3,8 @@ import { describe } from 'mocha'; import { QueryResult } from 'pg'; import sinon from 'sinon'; import sinonChai from 'sinon-chai'; -import { ApiGeneralError } from '../errors/api-error'; +import { MESSAGE_CLASS_NAME, SUBMISSION_MESSAGE_TYPE, SUBMISSION_STATUS_TYPE } from '../constants/status'; +import { ApiExecuteSQLError, ApiGeneralError } from '../errors/api-error'; import { GetReportAttachmentsData } from '../models/project-view'; import { PostProprietorData, PostSurveyObject } from '../models/survey-create'; import { PutSurveyObject, PutSurveyPermitData } from '../models/survey-update'; @@ -885,4 +886,181 @@ describe('SurveyService', () => { expect(response).to.eql(undefined); }); }); + + describe('getOccurrenceSubmissionMessages', () => { + it('should return empty array if no messages are found', async () => { + const dbConnection = getMockDBConnection(); + const service = new SurveyService(dbConnection); + + const repoStub = sinon.stub(SurveyRepository.prototype, 'getOccurrenceSubmissionMessages').resolves([]); + + const response = await service.getOccurrenceSubmissionMessages(1); + + expect(repoStub).to.be.calledOnce; + expect(response).to.eql([]); + }); + + it('should successfully group messages by message type', async () => { + const dbConnection = getMockDBConnection(); + const service = new SurveyService(dbConnection); + + const repoStub = sinon.stub(SurveyRepository.prototype, 'getOccurrenceSubmissionMessages').resolves([ + { + id: 1, + type: SUBMISSION_MESSAGE_TYPE.DUPLICATE_HEADER, + status: SUBMISSION_STATUS_TYPE.FAILED_VALIDATION, + class: MESSAGE_CLASS_NAME.ERROR, + message: 'message 1' + }, + { + id: 2, + type: SUBMISSION_MESSAGE_TYPE.DUPLICATE_HEADER, + status: SUBMISSION_STATUS_TYPE.FAILED_VALIDATION, + class: MESSAGE_CLASS_NAME.ERROR, + message: 'message 2' + }, + { + id: 3, + type: SUBMISSION_MESSAGE_TYPE.MISSING_RECOMMENDED_HEADER, + status: SUBMISSION_STATUS_TYPE.SUBMITTED, + class: MESSAGE_CLASS_NAME.WARNING, + message: 'message 3' + }, + { + id: 4, + type: SUBMISSION_MESSAGE_TYPE.MISCELLANEOUS, + status: SUBMISSION_STATUS_TYPE.SUBMITTED, + class: MESSAGE_CLASS_NAME.NOTICE, + message: 'message 4' + } + ]); + + const response = await service.getOccurrenceSubmissionMessages(1); + + expect(repoStub).to.be.calledOnce; + expect(response).to.eql([ + { + severityLabel: 'Error', + messageTypeLabel: 'Duplicate Header', + messageStatus: 'Failed to validate', + messages: [ + { id: 1, message: 'message 1' }, + { id: 2, message: 'message 2' } + ] + }, + { + severityLabel: 'Warning', + messageTypeLabel: 'Missing Recommended Header', + messageStatus: 'Submitted', + messages: [{ id: 3, message: 'message 3' }] + }, + { + severityLabel: 'Notice', + messageTypeLabel: 'Miscellaneous', + messageStatus: 'Submitted', + messages: [{ id: 4, message: 'message 4' }] + } + ]); + }); + }); + + describe('insertSurveyOccurrenceSubmission', () => { + it('should return submissionId upon success', async () => { + const dbConnection = getMockDBConnection(); + const service = new SurveyService(dbConnection); + + const repoStub = sinon + .stub(SurveyRepository.prototype, 'insertSurveyOccurrenceSubmission') + .resolves({ submissionId: 1 }); + + const response = await service.insertSurveyOccurrenceSubmission({ + surveyId: 1, + source: 'Test' + }); + + expect(repoStub).to.be.calledOnce; + expect(response).to.eql({ submissionId: 1 }); + }); + + it('should throw an error', async () => { + const dbConnection = getMockDBConnection(); + const service = new SurveyService(dbConnection); + + sinon + .stub(SurveyRepository.prototype, 'insertSurveyOccurrenceSubmission') + .throws(new ApiExecuteSQLError('Failed to insert survey occurrence submission')); + + try { + await service.insertSurveyOccurrenceSubmission({ + surveyId: 1, + source: 'Test' + }); + expect.fail(); + } catch (actualError) { + expect((actualError as ApiGeneralError).message).to.equal('Failed to insert survey occurrence submission'); + } + }); + }); + + describe('updateSurveyOccurrenceSubmission', () => { + it('should return submissionId upon success', async () => { + const dbConnection = getMockDBConnection(); + const service = new SurveyService(dbConnection); + + const repoStub = sinon + .stub(SurveyRepository.prototype, 'updateSurveyOccurrenceSubmission') + .resolves({ submissionId: 1 }); + + const response = await service.updateSurveyOccurrenceSubmission({ submissionId: 1 }); + + expect(repoStub).to.be.calledOnce; + expect(response).to.eql({ submissionId: 1 }); + }); + + it('should throw an error', async () => { + const dbConnection = getMockDBConnection(); + const service = new SurveyService(dbConnection); + + sinon + .stub(SurveyRepository.prototype, 'updateSurveyOccurrenceSubmission') + .throws(new ApiExecuteSQLError('Failed to update survey occurrence submission')); + + try { + await service.updateSurveyOccurrenceSubmission({ submissionId: 1 }); + expect.fail(); + } catch (actualError) { + expect((actualError as ApiGeneralError).message).to.equal('Failed to update survey occurrence submission'); + } + }); + }); + + describe('deleteOccurrenceSubmission', () => { + it('should return 1 upon success', async () => { + const dbConnection = getMockDBConnection(); + const service = new SurveyService(dbConnection); + + const repoStub = sinon.stub(SurveyRepository.prototype, 'deleteOccurrenceSubmission').resolves(1); + + const response = await service.deleteOccurrenceSubmission(2); + + expect(repoStub).to.be.calledOnce; + expect(response).to.eql(1); + }); + + it('should throw an error upon failure', async () => { + const dbConnection = getMockDBConnection(); + const service = new SurveyService(dbConnection); + + sinon + .stub(SurveyRepository.prototype, 'deleteOccurrenceSubmission') + .throws(new ApiExecuteSQLError('Failed to delete survey occurrence submission')); + + try { + await service.deleteOccurrenceSubmission(2); + expect.fail(); + } catch (actualError) { + expect((actualError as ApiGeneralError).message).to.equal('Failed to delete survey occurrence submission'); + } + }); + }); }); diff --git a/api/src/services/survey-service.ts b/api/src/services/survey-service.ts index 943d64c765..620ad5f98f 100644 --- a/api/src/services/survey-service.ts +++ b/api/src/services/survey-service.ts @@ -1,3 +1,4 @@ +import { MESSAGE_CLASS_NAME, SUBMISSION_MESSAGE_TYPE, SUBMISSION_STATUS_TYPE } from '../constants/status'; import { IDBConnection } from '../database/db'; import { PostProprietorData, PostSurveyObject } from '../models/survey-create'; import { PutSurveyObject } from '../models/survey-update'; @@ -16,11 +17,27 @@ import { SurveySupplementaryData } from '../models/survey-view'; import { AttachmentRepository } from '../repositories/attachment-repository'; -import { IGetLatestSurveyOccurrenceSubmission, SurveyRepository } from '../repositories/survey-repository'; +import { + IGetLatestSurveyOccurrenceSubmission, + IObservationSubmissionInsertDetails, + IObservationSubmissionUpdateDetails, + IOccurrenceSubmissionMessagesResponse, + SurveyRepository +} from '../repositories/survey-repository'; +import { getLogger } from '../utils/logger'; import { DBService } from './db-service'; import { PermitService } from './permit-service'; import { TaxonomyService } from './taxonomy-service'; +const defaultLog = getLogger('services/survey-service'); + +export interface IMessageTypeGroup { + severityLabel: MESSAGE_CLASS_NAME; + messageTypeLabel: SUBMISSION_MESSAGE_TYPE; + messageStatus: SUBMISSION_STATUS_TYPE; + messages: { id: number; message: string }[]; +} + export class SurveyService extends DBService { attachmentRepository: AttachmentRepository; surveyRepository: SurveyRepository; @@ -67,13 +84,13 @@ export class SurveyService extends DBService { } async getSurveySupplementaryDataById(surveyId: number): Promise { - const [occurrenceSubmissionId, summaryResultId] = await Promise.all([ + const [submissionId, summaryResultId] = await Promise.all([ this.getOccurrenceSubmissionId(surveyId), this.getSummaryResultId(surveyId) ]); return { - occurrence_submission: occurrenceSubmissionId, + occurrence_submission: submissionId, summary_result: summaryResultId }; } @@ -128,6 +145,40 @@ export class SurveyService extends DBService { return this.surveyRepository.getLatestSurveyOccurrenceSubmission(surveyId); } + /** + * Retrieves all submission messages by the given submission ID, then groups them based on the message type. + * @param {number} submissionId The ID of the submission + * @returns {*} {Promise} Promise resolving the array of message groups containing the submission messages + */ + async getOccurrenceSubmissionMessages(submissionId: number): Promise { + const messages = await this.surveyRepository.getOccurrenceSubmissionMessages(submissionId); + defaultLog.debug({ label: 'getOccurrenceSubmissionMessages', submissionId, messages }); + + return messages.reduce((typeGroups: IMessageTypeGroup[], message: IOccurrenceSubmissionMessagesResponse) => { + const groupIndex = typeGroups.findIndex((group) => { + return group.messageTypeLabel === message.type; + }); + + const messageObject = { + id: message.id, + message: message.message + }; + + if (groupIndex < 0) { + typeGroups.push({ + severityLabel: message.class, + messageTypeLabel: message.type, + messageStatus: message.status, + messages: [messageObject] + }); + } else { + typeGroups[groupIndex].messages.push(messageObject); + } + + return typeGroups; + }, []); + } + async getSummaryResultId(surveyId: number): Promise { return this.surveyRepository.getSummaryResultId(surveyId); } @@ -403,4 +454,38 @@ export class SurveyService extends DBService { async deleteSurvey(surveyId: number): Promise { return this.surveyRepository.deleteSurvey(surveyId); } + + /** + * Inserts a survey occurrence submission row. + * + * @param {IObservationSubmissionInsertDetails} submission The details of the submission + * @return {*} {Promise<{ submissionId: number }>} Promise resolving the ID of the submission upon successful insertion + */ + async insertSurveyOccurrenceSubmission( + submission: IObservationSubmissionInsertDetails + ): Promise<{ submissionId: number }> { + return this.surveyRepository.insertSurveyOccurrenceSubmission(submission); + } + + /** + * Updates a survey occurrence submission with the given details. + * + * @param {IObservationSubmissionUpdateDetails} submission The details of the submission to be updated + * @return {*} {Promise<{ submissionId: number }>} Promise resolving the ID of the submission upon successfully updating it + */ + async updateSurveyOccurrenceSubmission( + submission: IObservationSubmissionUpdateDetails + ): Promise<{ submissionId: number }> { + return this.surveyRepository.updateSurveyOccurrenceSubmission(submission); + } + + /** + * Soft-deletes an occurrence submission. + * + * @param {number} submissionId The ID of the submission to soft delete + * @returns {*} {number} The row count of the affected records, namely `1` if the delete succeeds, `0` if it does not + */ + async deleteOccurrenceSubmission(submissionId: number): Promise { + return this.surveyRepository.deleteOccurrenceSubmission(submissionId); + } } diff --git a/api/src/services/validation-service.test.ts b/api/src/services/validation-service.test.ts index a4957ecf45..c6fa312658 100644 --- a/api/src/services/validation-service.test.ts +++ b/api/src/services/validation-service.test.ts @@ -76,6 +76,7 @@ describe('ValidationService', () => { const service = mockService(); sinon.stub(ValidationService.prototype, 'getTemplateMethodologySpeciesRecord').resolves({ template_methodology_species_id: 1, + wldtaxonomic_units_id: '1', validation: '{}', transform: ('{}' as unknown) as TransformSchema }); @@ -113,6 +114,7 @@ describe('ValidationService', () => { const service = mockService(); sinon.stub(ValidationService.prototype, 'getTemplateMethodologySpeciesRecord').resolves({ template_methodology_species_id: 1, + wldtaxonomic_units_id: '1', validation: '{}', transform: ('{}' as unknown) as TransformSchema }); @@ -564,7 +566,7 @@ describe('ValidationService', () => { it('should run without issue', async () => { const service = mockService(); const mockPrep = { - s3InputKey: '', + s3OutputKey: '', archive: new DWCArchive(new ArchiveFile('test', 'application/zip', Buffer.from([]), [buildFile('test', {})])) }; const mockState = { @@ -595,7 +597,7 @@ describe('ValidationService', () => { it('should insert submission error from prep failure', async () => { const service = mockService(); const mockPrep = { - s3InputKey: '', + s3OutputKey: '', archive: new DWCArchive(new ArchiveFile('test', 'application/zip', Buffer.from([]), [buildFile('test', {})])) }; const mockState = { @@ -615,19 +617,23 @@ describe('ValidationService', () => { .throws(SubmissionErrorFromMessageType(SUBMISSION_MESSAGE_TYPE.FAILED_UPDATE_OCCURRENCE_SUBMISSION)); const insertError = sinon.stub(service.errorService, 'insertSubmissionError').resolves(); - await service.processDWCFile(1); - expect(prep).to.be.calledOnce; - expect(state).to.be.calledOnce; - expect(persistResults).to.be.calledOnce; - expect(update).to.be.calledOnce; + try { + await service.processDWCFile(1); + expect.fail(); + } catch (error) { + expect(prep).to.be.calledOnce; + expect(state).to.be.calledOnce; + expect(persistResults).to.be.calledOnce; + expect(update).to.be.calledOnce; - expect(insertError).to.be.calledOnce; + expect(insertError).to.be.calledOnce; + } }); it('should throw unrecognized error', async () => { const service = mockService(); const mockPrep = { - s3InputKey: '', + s3OutputKey: '', archive: new DWCArchive(new ArchiveFile('test', 'application/zip', Buffer.from([]), [buildFile('test', {})])) }; const mockState = { @@ -700,11 +706,15 @@ describe('ValidationService', () => { sinon.stub(service, 'templateScrapeAndUploadOccurrences').resolves(); sinon.stub(service.submissionRepository, 'insertSubmissionStatus').resolves(); - await service.processXLSXFile(1, 1); - expect(prep).to.be.calledOnce; - expect(validate).to.be.calledOnce; - expect(transform).to.be.calledOnce; - expect(insertError).to.be.calledOnce; + try { + await service.processXLSXFile(1, 1); + expect.fail(); + } catch { + expect(prep).to.be.calledOnce; + expect(validate).to.be.calledOnce; + expect(transform).to.be.calledOnce; + expect(insertError).to.be.calledOnce; + } }); it('should throw unrecognized error', async () => { @@ -748,7 +758,7 @@ describe('ValidationService', () => { const prep = sinon.stub(service, 'prepDWCArchive').returns(archive); const results = await service.dwcPreparation(1); - expect(results.s3InputKey).to.not.be.empty; + expect(results.s3OutputKey).to.not.be.empty; expect(occurrence).to.be.calledOnce; expect(s3).to.be.calledOnce; expect(prep).to.be.calledOnce; diff --git a/api/src/services/validation-service.ts b/api/src/services/validation-service.ts index 3e62366515..be93c45d57 100644 --- a/api/src/services/validation-service.ts +++ b/api/src/services/validation-service.ts @@ -51,6 +51,7 @@ export class ValidationService extends DBService { } async transformFile(submissionId: number, surveyId: number) { + defaultLog.debug({ label: 'transformFile', submissionId, surveyId }); try { const submissionPrep = await this.templatePreparation(submissionId); await this.templateTransformation(submissionId, submissionPrep.xlsx, submissionPrep.s3InputKey, surveyId); @@ -67,6 +68,7 @@ export class ValidationService extends DBService { } async validateFile(submissionId: number, surveyId: number) { + defaultLog.debug({ label: 'validateFile', submissionId, surveyId }); try { const submissionPrep = await this.templatePreparation(submissionId); await this.templateValidation(submissionPrep.xlsx, surveyId); @@ -83,6 +85,7 @@ export class ValidationService extends DBService { } async processDWCFile(submissionId: number) { + defaultLog.debug({ label: 'processDWCFile', submissionId }); try { // prep dwc const dwcPrep = await this.dwcPreparation(submissionId); @@ -94,7 +97,7 @@ export class ValidationService extends DBService { await this.occurrenceService.updateSurveyOccurrenceSubmission( submissionId, dwcPrep.archive.rawFile.fileName, - dwcPrep.s3InputKey + dwcPrep.s3OutputKey ); // insert validated status @@ -109,13 +112,14 @@ export class ValidationService extends DBService { defaultLog.debug({ label: 'processDWCFile', message: 'error', error }); if (error instanceof SubmissionError) { await this.errorService.insertSubmissionError(submissionId, error); - } else { - throw error; } + + throw error; } } async processXLSXFile(submissionId: number, surveyId: number) { + defaultLog.debug({ label: 'processXLSXFile', submissionId, surveyId }); try { // template preparation const submissionPrep = await this.templatePreparation(submissionId); @@ -135,13 +139,14 @@ export class ValidationService extends DBService { defaultLog.debug({ label: 'processXLSXFile', message: 'error', error }); if (error instanceof SubmissionError) { await this.errorService.insertSubmissionError(submissionId, error); - } else { - throw error; } + + throw error; } } validateDWC(archive: DWCArchive): ICsvMediaState { + defaultLog.debug({ label: 'validateDWC' }); try { const validationSchema = {}; const rules = this.getValidationRules(validationSchema); @@ -156,14 +161,15 @@ export class ValidationService extends DBService { } } - async dwcPreparation(submissionId: number): Promise<{ archive: DWCArchive; s3InputKey: string }> { + async dwcPreparation(submissionId: number): Promise<{ archive: DWCArchive; s3OutputKey: string }> { + defaultLog.debug({ label: 'dwcPreparation', submissionId }); try { const occurrenceSubmission = await this.occurrenceService.getOccurrenceSubmission(submissionId); - const s3InputKey = occurrenceSubmission.output_key; - const s3File = await getFileFromS3(s3InputKey); + const s3OutputKey = occurrenceSubmission.output_key; + const s3File = await getFileFromS3(s3OutputKey); const archive = this.prepDWCArchive(s3File); - return { archive, s3InputKey }; + return { archive, s3OutputKey }; } catch (error) { if (error instanceof SubmissionError) { error.setStatus(SUBMISSION_STATUS_TYPE.FAILED_PROCESSING_OCCURRENCE_DATA); @@ -173,6 +179,7 @@ export class ValidationService extends DBService { } async templatePreparation(submissionId: number): Promise<{ s3InputKey: string; xlsx: XLSXCSV }> { + defaultLog.debug({ label: 'templatePreparation', submissionId }); try { const occurrenceSubmission = await this.occurrenceService.getOccurrenceSubmission(submissionId); const s3InputKey = occurrenceSubmission.input_key; @@ -189,6 +196,7 @@ export class ValidationService extends DBService { } async templateScrapeAndUploadOccurrences(submissionId: number) { + defaultLog.debug({ label: 'templateScrapeAndUploadOccurrences', submissionId }); try { await this.spatialService.runSpatialTransforms(submissionId); } catch (error) { @@ -200,6 +208,7 @@ export class ValidationService extends DBService { } async templateValidation(xlsx: XLSXCSV, surveyId: number) { + defaultLog.debug({ label: 'templateValidation' }); try { const schema = await this.getValidationSchema(xlsx, surveyId); const schemaParser = this.getValidationRules(schema); @@ -214,6 +223,7 @@ export class ValidationService extends DBService { } async templateTransformation(submissionId: number, xlsx: XLSXCSV, s3InputKey: string, surveyId: number) { + defaultLog.debug({ label: 'templateTransformation' }); try { const xlsxSchema = await this.getTransformationSchema(xlsx, surveyId); const fileBuffer = this.transformXLSX(xlsx.workbook.rawWorkbook, xlsxSchema); @@ -426,6 +436,13 @@ export class ValidationService extends DBService { s3OutputKey: string, xlsxCsv: XLSXCSV ) { + defaultLog.debug({ + label: 'persistTransformationResults', + submissionId, + fileBuffers, + s3OutputKey + }); + // Build the archive zip file const dwcArchiveZip = new AdmZip(); fileBuffers.forEach((file) => dwcArchiveZip.addFile(`${file.name}.csv`, file.buffer)); diff --git a/app/src/features/surveys/view/SurveyObservations.tsx b/app/src/features/surveys/view/SurveyObservations.tsx index 196f8a3aba..e913c2ab3d 100644 --- a/app/src/features/surveys/view/SurveyObservations.tsx +++ b/app/src/features/surveys/view/SurveyObservations.tsx @@ -24,8 +24,12 @@ import { IUploadHandler } from 'components/file-upload/FileUploadItem'; import { H2ButtonToolbar } from 'components/toolbar/ActionToolbars'; import { DialogContext } from 'contexts/dialogContext'; import { useBiohubApi } from 'hooks/useBioHubApi'; +import useDataLoader from 'hooks/useDataLoader'; import { useInterval } from 'hooks/useInterval'; -import { IGetObservationSubmissionResponse } from 'interfaces/useObservationApi.interface'; +import { + IUploadObservationSubmissionResponse, + ObservationSubmissionMessageSeverityLabel +} from 'interfaces/useObservationApi.interface'; import React, { useCallback, useContext, useEffect, useState } from 'react'; import { useParams } from 'react-router'; @@ -44,177 +48,42 @@ const useStyles = makeStyles((theme: Theme) => ({ } })); -export enum ClassGrouping { - NOTICE = 'Notice', - ERROR = 'Error', - WARNING = 'Warning' -} - -const finalStatus = [ - 'Rejected', - 'Darwin Core Validated', - 'Template Validated', - 'Template Transformed', - 'System Error', - 'Failed to prepare submission', - 'Media is not valid', - 'Failed to validate', - 'Failed to transform', - 'Failed to process occurrence data' -]; - -export enum SUBMISSION_STATUS_TYPE { - SUBMITTED = 'Submitted', - 'TEMPLATE_VALIDATED' = 'Template Validated', - 'DARWIN_CORE_VALIDATED' = 'Darwin Core Validated', - 'TEMPLATE_TRANSFORMED' = 'Template Transformed', - 'SUBMISSION_DATA_INGESTED' = 'Submission Data Ingested', - 'SECURED' = 'Secured', - 'AWAITING CURRATION' = 'Awaiting Curration', - 'REJECTED' = 'Rejected', - 'ON HOLD' = 'On Hold', - 'SYSTEM_ERROR' = 'System Error', - - //Failure - 'FAILED_OCCURRENCE_PREPARATION' = 'Failed to prepare submission', - 'INVALID_MEDIA' = 'Media is not valid', - 'FAILED_VALIDATION' = 'Failed to validate', - 'FAILED_TRANSFORMED' = 'Failed to transform', - 'FAILED_PROCESSING_OCCURRENCE_DATA' = 'Failed to process occurrence data' -} - -export enum SUBMISSION_MESSAGE_TYPE { - //message types that match the submission_message_type table, and API - - 'DUPLICATE_HEADER' = 'Duplicate Header', - 'UNKNOWN_HEADER' = 'Unknown Header', - 'MISSING_REQUIRED_HEADER' = 'Missing Required Header', - 'MISSING_RECOMMENDED_HEADER' = 'Missing Recommended Header', - 'MISCELLANEOUS' = 'Miscellaneous', - 'MISSING_REQUIRED_FIELD' = 'Missing Required Field', - 'UNEXPECTED_FORMAT' = 'Unexpected Format', - 'OUT_OF_RANGE' = 'Out of Range', - 'INVALID_VALUE' = 'Invalid Value', - 'MISSING_VALIDATION_SCHEMA' = 'Missing Validation Schema', - 'ERROR' = 'Error', - 'PARSE_ERROR' = 'Parse error', - - 'FAILED_GET_OCCURRENCE' = 'Failed to Get Occurrence Submission', - 'FAILED_GET_FILE_FROM_S3' = 'Failed to get file from S3', - 'FAILED_UPLOAD_FILE_TO_S3' = 'Failed to upload file to S3', - 'FAILED_PARSE_SUBMISSION' = 'Failed to parse submission', - 'FAILED_PREP_DWC_ARCHIVE' = 'Failed to prep DarwinCore Archive', - 'FAILED_PREP_XLSX' = 'Failed to prep XLSX', - 'FAILED_PERSIST_PARSE_ERRORS' = 'Failed to persist parse errors', - 'FAILED_GET_VALIDATION_RULES' = 'Failed to get validation rules.', - 'FAILED_GET_TRANSFORMATION_RULES' = 'Failed to get transformation rules', - 'FAILED_PERSIST_TRANSFORMATION_RESULTS' = 'Failed to persist transformation results', - 'FAILED_TRANSFORM_XLSX' = 'Failed to transform XLSX', - 'FAILED_VALIDATE_DWC_ARCHIVE' = 'Failed to validate DarwinCore Archive', - 'FAILED_PERSIST_VALIDATION_RESULTS' = 'Failed to persist validation results', - 'FAILED_UPDATE_OCCURRENCE_SUBMISSION' = 'Failed to update occurrence submission', - 'FAILED_TO_GET_TRANSFORM_SCHEMA' = 'Unable to get transform schema for submission', - 'INVALID_MEDIA' = 'Media is invalid', - 'UNSUPPORTED_FILE_TYPE' = 'File submitted is not a supported type', - 'DANGLING_PARENT_CHILD_KEY' = 'Missing Child Key from Parent', - 'NON_UNIQUE_KEY' = 'Duplicate Key(s) found in file.' -} - const SurveyObservations: React.FC = (props) => { const biohubApi = useBiohubApi(); const urlParams = useParams(); - // const config = useContext(ConfigContext); + const dialogContext = useContext(DialogContext); + const classes = useStyles(); + const [openImportObservations, setOpenImportObservations] = useState(false); + const [willRefreshOnClose, setWillRefreshOnClose] = useState(false); const projectId = Number(urlParams['id']); const surveyId = Number(urlParams['survey_id']); - const [occurrenceSubmissionId, setOccurrenceSubmissionId] = useState(null); - const [openImportObservations, setOpenImportObservations] = useState(false); - - const classes = useStyles(); - - const importObservations = (): IUploadHandler => { - return (file, cancelToken, handleFileUploadProgress) => { - return biohubApi.observation - .uploadObservationSubmission(projectId, surveyId, file, cancelToken, handleFileUploadProgress) - .then((result) => { - if (!result || !result.submissionId) { - return; - } - if (file.type === 'application/x-zip-compressed' || file.type === 'application/zip') { - biohubApi.observation.processDWCFile(projectId, result.submissionId).then(() => { - props.refresh(); - }); - } else { - biohubApi.observation.processOccurrences(projectId, result.submissionId, surveyId); - } - }); - }; - }; + const submissionDataLoader = useDataLoader(() => biohubApi.observation.getObservationSubmission(projectId, surveyId)); - const [submissionStatus, setSubmissionStatus] = useState(null); - const [isLoading, setIsLoading] = useState(true); - const [isValidating, setIsValidating] = useState(false); - const [isPolling, setIsPolling] = useState(false); - const [pollingTime, setPollingTime] = useState(0); + submissionDataLoader.load(); - const dialogContext = useContext(DialogContext); - - const fetchObservationSubmission = useCallback(async () => { - const submission = await biohubApi.observation.getObservationSubmission(projectId, surveyId); + const refreshSubmission = submissionDataLoader.refresh; + const occurrenceSubmission = submissionDataLoader.data; + const occurrenceSubmissionId = occurrenceSubmission?.id; + const submissionMessageTypes = occurrenceSubmission?.messageTypes || []; + const submissionExists = Boolean(occurrenceSubmission); - setSubmissionStatus(() => { - setIsLoading(false); - if (submission) { - if (finalStatus.includes(submission.status)) { - setIsValidating(false); - setIsPolling(false); - - setPollingTime(null); - } else { - setIsValidating(true); - setIsPolling(true); - } - - setOccurrenceSubmissionId(submission.id); - } - - return submission; - }); - }, [biohubApi.observation, projectId, surveyId]); - - useInterval(fetchObservationSubmission, pollingTime, 60000); + const submissionPollingInterval = useInterval(refreshSubmission, 5000, 60000); useEffect(() => { - if (isLoading) { - fetchObservationSubmission(); + if (submissionExists) { + submissionPollingInterval.enable(); + } else { + submissionPollingInterval.disable(); } + }, [submissionExists, submissionPollingInterval]); - if (isPolling && !pollingTime) { - setPollingTime(5000); - } - }, [ - biohubApi, - isLoading, - fetchObservationSubmission, - isPolling, - pollingTime, - isValidating, - submissionStatus, - projectId, - surveyId - ]); - - const softDeleteSubmission = async () => { - if (!occurrenceSubmissionId) { - return; + useEffect(() => { + if (occurrenceSubmission?.isValidating === false) { + submissionPollingInterval.disable(); } - - await biohubApi.observation.deleteObservationSubmission(projectId, surveyId, occurrenceSubmissionId); - - await props.refresh(); - fetchObservationSubmission(); - }; + }, [occurrenceSubmission, submissionPollingInterval]); const defaultUploadYesNoDialogProps = { dialogTitle: 'Upload Observation Data', @@ -233,23 +102,67 @@ const SurveyObservations: React.FC = (props) => { 'Are you sure you want to delete the current observation data? Your observation will be removed from this survey.' }; - const showUploadDialog = () => { - if (submissionStatus) { - // already have observation data, prompt user to confirm override + const importObservations = (): IUploadHandler => { + return async (file, cancelToken, handleFileUploadProgress) => { + return biohubApi.observation + .uploadObservationSubmission(projectId, surveyId, file, cancelToken, handleFileUploadProgress) + .then((result: IUploadObservationSubmissionResponse) => { + if (!result || !result.submissionId) { + return; + } + + if (file.type === 'application/x-zip-compressed' || file.type === 'application/zip') { + biohubApi.observation.processDWCFile(projectId, result.submissionId).then(props.refresh); + } else { + biohubApi.observation.processOccurrences(projectId, result.submissionId, surveyId); + } + }) + .finally(() => { + setWillRefreshOnClose(true); + }); + }; + }; + + function handleOpenImportObservations() { + setOpenImportObservations(true); + setWillRefreshOnClose(false); + } + + function handleCloseImportObservations() { + if (willRefreshOnClose) { + refreshSubmission(); + } + + setOpenImportObservations(false); + } + + function softDeleteSubmission() { + if (!occurrenceSubmissionId) { + return; + } + + biohubApi.observation.deleteObservationSubmission(projectId, surveyId, occurrenceSubmissionId).then(() => { + props.refresh(); + refreshSubmission(); + }); + } + + function showUploadDialog() { + if (submissionExists) { dialogContext.setYesNoDialog({ ...defaultUploadYesNoDialogProps, open: true, onYes: () => { - setOpenImportObservations(true); + handleOpenImportObservations(); dialogContext.setYesNoDialog({ open: false }); } }); } else { - setOpenImportObservations(true); + handleOpenImportObservations(); } - }; + } - const showDeleteDialog = () => { + function showDeleteDialog() { dialogContext.setYesNoDialog({ ...defaultDeleteYesNoDialogProps, open: true, @@ -258,170 +171,65 @@ const SurveyObservations: React.FC = (props) => { dialogContext.setYesNoDialog({ open: false }); } }); - }; + } - // Action prop for the Alert MUI component to render the delete icon and associated actions const submissionAlertAction = () => ( - viewFileContents()}> + - showDeleteDialog()}> + ); - type MessageGrouping = { [key: string]: { type: string[]; label: string } }; - - const messageGrouping: MessageGrouping = { - mandatory: { - type: [ - SUBMISSION_MESSAGE_TYPE.MISSING_REQUIRED_FIELD, - SUBMISSION_MESSAGE_TYPE.MISSING_REQUIRED_HEADER, - SUBMISSION_MESSAGE_TYPE.DUPLICATE_HEADER, - SUBMISSION_MESSAGE_TYPE.DANGLING_PARENT_CHILD_KEY, - SUBMISSION_MESSAGE_TYPE.NON_UNIQUE_KEY - ], - label: 'Mandatory fields have not been filled out' - }, - recommended: { - type: [SUBMISSION_MESSAGE_TYPE.MISSING_RECOMMENDED_HEADER], - label: 'Recommended fields have not been filled out' - }, - value_not_from_list: { - type: [SUBMISSION_MESSAGE_TYPE.INVALID_VALUE], - label: "Values have not been selected from the field's dropdown list" - }, - unsupported_header: { - type: [SUBMISSION_MESSAGE_TYPE.UNKNOWN_HEADER], - label: 'Column headers are not supported' - }, - out_of_range: { - type: [SUBMISSION_MESSAGE_TYPE.OUT_OF_RANGE], - label: 'Values are out of range' - }, - formatting_errors: { - type: [SUBMISSION_MESSAGE_TYPE.UNEXPECTED_FORMAT], - label: 'Unexpected formats in the values provided' - }, - miscellaneous: { type: [SUBMISSION_MESSAGE_TYPE.MISCELLANEOUS], label: 'Miscellaneous errors exist in your file' }, - system_error: { - type: [ - SUBMISSION_MESSAGE_TYPE.FAILED_GET_FILE_FROM_S3, - SUBMISSION_MESSAGE_TYPE.ERROR, - SUBMISSION_MESSAGE_TYPE.PARSE_ERROR, - SUBMISSION_MESSAGE_TYPE.FAILED_GET_OCCURRENCE, - SUBMISSION_MESSAGE_TYPE.FAILED_UPLOAD_FILE_TO_S3, - SUBMISSION_MESSAGE_TYPE.FAILED_PARSE_SUBMISSION, - SUBMISSION_MESSAGE_TYPE.FAILED_PREP_DWC_ARCHIVE, - SUBMISSION_MESSAGE_TYPE.FAILED_PREP_XLSX, - SUBMISSION_MESSAGE_TYPE.FAILED_PERSIST_PARSE_ERRORS, - SUBMISSION_MESSAGE_TYPE.FAILED_GET_VALIDATION_RULES, - SUBMISSION_MESSAGE_TYPE.FAILED_GET_TRANSFORMATION_RULES, - SUBMISSION_MESSAGE_TYPE.FAILED_PERSIST_TRANSFORMATION_RESULTS, - SUBMISSION_MESSAGE_TYPE.FAILED_TRANSFORM_XLSX, - SUBMISSION_MESSAGE_TYPE.FAILED_VALIDATE_DWC_ARCHIVE, - SUBMISSION_MESSAGE_TYPE.FAILED_PERSIST_VALIDATION_RESULTS, - SUBMISSION_MESSAGE_TYPE.FAILED_UPDATE_OCCURRENCE_SUBMISSION, - SUBMISSION_MESSAGE_TYPE.FAILED_TO_GET_TRANSFORM_SCHEMA, - SUBMISSION_MESSAGE_TYPE.UNSUPPORTED_FILE_TYPE, - SUBMISSION_MESSAGE_TYPE.INVALID_MEDIA, - SUBMISSION_MESSAGE_TYPE.MISSING_VALIDATION_SCHEMA - ], - label: 'Contact your system administrator' + const openFileContents = useCallback(() => { + if (!occurrenceSubmissionId) { + return; } - }; - - type SubmissionErrors = { [key: string]: string[] }; - type SubmissionWarnings = { [key: string]: string[] }; - - const submissionErrors: SubmissionErrors = {}; - const submissionWarnings: SubmissionWarnings = {}; - const messageList = submissionStatus?.messages; - - if (messageList) { - Object.entries(messageGrouping).forEach(([key, value]) => { - messageList.forEach((message) => { - if (value.type.includes(message.type)) { - if (message.class === ClassGrouping.ERROR) { - if (!submissionErrors[key]) { - submissionErrors[key] = []; - } - submissionErrors[key].push(message.message); - } - - if (message.class === ClassGrouping.WARNING) { - if (!submissionWarnings[key]) { - submissionWarnings[key] = []; - } - - submissionWarnings[key].push(message.message); - } - } + biohubApi.survey + .getObservationSubmissionSignedURL(projectId, surveyId, occurrenceSubmissionId) + .then((objectUrl: string) => { + window.open(objectUrl); + }) + .catch((_err: any) => { + return; }); - }); + }, [biohubApi.survey, occurrenceSubmissionId, projectId, surveyId]); + + if (!submissionExists && submissionDataLoader.isLoading) { + return ; } - const viewFileContents = async () => { - if (!occurrenceSubmissionId) { - return; - } + type AlertSeverityLevel = 'error' | 'info' | 'success' | 'warning'; - let response; + const alertSeverityFromSeverityLabel = (severity: ObservationSubmissionMessageSeverityLabel): AlertSeverityLevel => { + switch (severity) { + case 'Warning': + return 'warning'; - try { - response = await biohubApi.survey.getObservationSubmissionSignedURL(projectId, surveyId, occurrenceSubmissionId); - } catch { - return; - } + case 'Error': + return 'error'; - if (!response) { - return; + case 'Notice': + default: + return 'info'; } - - window.open(response); }; - if (isLoading) { - return ; - } - - type severityLevel = 'error' | 'info' | 'success' | 'warning' | undefined; - - function displayAlertBox(severityLevel: severityLevel, iconName: string, fileName: string, message: string) { - return ( - } severity={severityLevel} action={submissionAlertAction()}> - - viewFileContents()}> - {fileName} - - - {message} - - ); - } - - function displayMessages(list: SubmissionErrors | SubmissionWarnings, msgGroup: MessageGrouping, iconName: string) { - return ( - - {Object.entries(list).map(([key, value], index) => ( - - {msgGroup[key].label} - - {value.map((message: string, index2: number) => { - return ( -
  • - {message} -
    • - ); - })} -     -     - ))} -     - ); + let submissionStatusIcon = occurrenceSubmission?.isValidating ? mdiClockOutline : mdiFileOutline; + let submissionStatusSeverity: AlertSeverityLevel = 'info'; + + if (submissionMessageTypes.some((messageType) => messageType.severityLabel === 'Error')) { + submissionStatusIcon = mdiAlertCircleOutline; + submissionStatusSeverity = 'error'; + } else if (submissionMessageTypes.some((messageType) => messageType.severityLabel === 'Warning')) { + submissionStatusIcon = mdiAlertCircleOutline; + submissionStatusSeverity = 'warning'; + } else if (submissionMessageTypes.some((messageType) => messageType.severityLabel === 'Notice')) { + submissionStatusIcon = mdiInformationOutline; } return ( @@ -436,83 +244,71 @@ const SurveyObservations: React.FC = (props) => { buttonOnClick={() => showUploadDialog()} /> - + - {!submissionStatus && ( + {!submissionExists ? ( <> No Observation Data.   - setOpenImportObservations(true)}>Click Here to Import + Click Here to Import - )} - - {!isValidating && submissionStatus?.status === SUBMISSION_STATUS_TYPE.SYSTEM_ERROR && ( - - {displayAlertBox( - 'error', - mdiAlertCircleOutline, - submissionStatus.inputFileName, - SUBMISSION_STATUS_TYPE.SYSTEM_ERROR - )} - - - Resolve the following errors in your local file and re-import. - - - - {displayMessages(submissionErrors, messageGrouping, mdiAlertCircleOutline)} - {displayMessages(submissionWarnings, messageGrouping, mdiInformationOutline)} - - - )} - - {!isValidating && - (submissionStatus?.status === SUBMISSION_STATUS_TYPE.REJECTED || - submissionStatus?.status === SUBMISSION_STATUS_TYPE.FAILED_OCCURRENCE_PREPARATION || - submissionStatus?.status === SUBMISSION_STATUS_TYPE.INVALID_MEDIA || - submissionStatus?.status === SUBMISSION_STATUS_TYPE.FAILED_VALIDATION || - submissionStatus?.status === SUBMISSION_STATUS_TYPE.FAILED_TRANSFORMED || - submissionStatus?.status === SUBMISSION_STATUS_TYPE.FAILED_PROCESSING_OCCURRENCE_DATA) && ( - - {displayAlertBox( - 'error', - mdiAlertCircleOutline, - submissionStatus.inputFileName, - `Validation error: ${submissionStatus?.status}` - )} - - - Resolve the following errors in your local file and re-import. - - - - {displayMessages(submissionErrors, messageGrouping, mdiAlertCircleOutline)} - {displayMessages(submissionWarnings, messageGrouping, mdiInformationOutline)} + ) : ( + <> + } + severity={submissionStatusSeverity} + action={submissionAlertAction()}> + + + {occurrenceSubmission?.inputFileName} + - - )} - - {!isValidating && - submissionStatus && - (submissionStatus.status === SUBMISSION_STATUS_TYPE.DARWIN_CORE_VALIDATED || - submissionStatus.status === SUBMISSION_STATUS_TYPE.TEMPLATE_VALIDATED || - submissionStatus.status === SUBMISSION_STATUS_TYPE.TEMPLATE_TRANSFORMED) && ( - {displayAlertBox('info', mdiFileOutline, submissionStatus.inputFileName, '')} - )} - - {isValidating && submissionStatus && ( - - {displayAlertBox( - 'info', - mdiClockOutline, - submissionStatus?.inputFileName, - 'Validating observation data. Please wait ...' + + {occurrenceSubmission?.isValidating + ? 'Validating observation data. Please wait...' + : occurrenceSubmission?.status} + + + + {!occurrenceSubmission?.isValidating && ( + <> + {submissionStatusSeverity === 'error' && ( + + + Resolve the following errors in your local file and re-import. + + + )} + + {submissionMessageTypes.length > 0 && ( + + {submissionMessageTypes.map((messageType) => { + return ( + + + {messageType.messageTypeLabel} + + + {messageType.messages.map((messageObject: { id: number; message: string }) => { + return ( +
  • + {messageObject.message} +
    • + ); + })} +     +     + ); + })} +     + )} + )} -     + )}     @@ -520,11 +316,7 @@ const SurveyObservations: React.FC = (props) => { { - setOpenImportObservations(false); - setIsPolling(true); - setIsLoading(true); - }}> + onClose={handleCloseImportObservations}> void, disable: () => void}} `enable` and `disable` callbacks which start or stop polling. */ export const useInterval = ( callback: (() => any) | null | undefined, period: number | null | undefined, timeout?: number -): void => { +): { enable: () => void; disable: () => void } => { const savedCallback = useRef(callback); + const interval = useRef(undefined); + const intervalTimeout = useRef(undefined); - useEffect(() => { - savedCallback.current = callback; - }, [callback]); - - useEffect(() => { + const enable = useCallback(() => { if (!period || !savedCallback?.current) { return; } - const interval = setInterval(() => savedCallback?.current?.(), period); + clearIntervals(); - let intervalTimeout: NodeJS.Timeout | undefined; + interval.current = setInterval(() => savedCallback?.current?.(), period); - if (timeout) { - intervalTimeout = setTimeout(() => clearInterval(interval), timeout); + if (timeout && interval.current) { + intervalTimeout.current = setTimeout(() => clearInterval(Number(interval.current)), timeout); } + }, [period, timeout]); - return () => { - clearInterval(interval); + const clearIntervals = () => { + if (interval) { + clearInterval(Number(interval.current)); + } + + if (intervalTimeout) { + clearTimeout(Number(intervalTimeout.current)); + } + }; - if (intervalTimeout) { - clearTimeout(intervalTimeout); - } + useEffect(() => { + savedCallback.current = callback; + }, [callback]); + + useEffect(() => { + enable(); + + return () => { + clearIntervals(); }; - }, [period, timeout]); + }, [enable, period, timeout]); + + return { enable, disable: clearIntervals }; }; diff --git a/app/src/interfaces/useObservationApi.interface.ts b/app/src/interfaces/useObservationApi.interface.ts index 6ee9142f0f..e853ab2601 100644 --- a/app/src/interfaces/useObservationApi.interface.ts +++ b/app/src/interfaces/useObservationApi.interface.ts @@ -10,12 +10,13 @@ export interface IGetSubmissionCSVForViewResponse { data: IGetSubmissionCSVForViewItem[]; } +export type ObservationSubmissionMessageSeverityLabel = 'Notice' | 'Error' | 'Warning'; + interface IGetObservationSubmissionResponseMessages { - id: number; - status: string; - class: string; - type: string; - message: string; + severityLabel: ObservationSubmissionMessageSeverityLabel; + messageTypeLabel: string; + messageStatus: string; + messages: { id: number; message: string }[]; } /** @@ -27,8 +28,9 @@ interface IGetObservationSubmissionResponseMessages { export interface IGetObservationSubmissionResponse { id: number; inputFileName: string; - status: string; - messages: IGetObservationSubmissionResponseMessages[]; + status?: string; + isValidating: boolean; + messageTypes: IGetObservationSubmissionResponseMessages[]; } export interface IGetObservationSubmissionErrorListResponse { diff --git a/database/src/migrations/20230123000000_submission_message_types.ts b/database/src/migrations/20230123000000_submission_message_types.ts new file mode 100644 index 0000000000..ab20007e72 --- /dev/null +++ b/database/src/migrations/20230123000000_submission_message_types.ts @@ -0,0 +1,47 @@ +import { Knex } from 'knex'; + +const DB_SCHEMA = process.env.DB_SCHEMA; + +const submissionMessageTypes = [ + { + name: 'Mismatched template with survey focal species', + description: + 'The species associated with the template does not match the focal species associated with this survey.', + class: 'Error' + } +]; + +/** + * Add new submission message types. + * + * @export + * @param {Knex} knex + * @return {*} {Promise} + */ +export async function up(knex: Knex): Promise { + await knex.raw(` + SET schema '${DB_SCHEMA}'; + set search_path = ${DB_SCHEMA},public; + `); + + for (const message of submissionMessageTypes) { + await knex.raw(` + INSERT INTO + ${DB_SCHEMA}.submission_message_type (name, record_effective_date, description, submission_message_class_id) + VALUES ( + '${message.name}', now(), '${message.description}', ( + SELECT + submission_message_class_id + FROM + submission_message_class + WHERE + name = '${message.class}' + ) + ); + `); + } +} + +export async function down(knex: Knex): Promise { + await knex.raw(``); +}