BHBC-2140: Release to Test

api/src/json-schema/validation-schema.ts

@@ -139,6 +139,9 @@ export const submissionValidationSchema = {
       title: 'Column Validation',
       description: 'The validators that can be applied against a column within a file/sheet.',
       anyOf: [
+        {
+          $ref: '#/$defs/column_required_validator'
+        },
         {
           $ref: '#/$defs/column_format_validator'
         },
@@ -331,6 +334,25 @@ export const submissionValidationSchema = {
       },
       additionalProperties: false
     },
+    column_required_validator: {
+      description: 'Validates that this column value is not empty',
+      type: 'object',
+      properties: {
+        column_required_validator: {
+          type: 'object',
+          properties: {
+            name: {
+              type: 'string'
+            },
+            description: {
+              type: 'string'
+            }
+          },
+          additionalProperties: false
+        }
+      },
+      additionalProperties: false
+    },
     column_format_validator: {
       description: 'Validates that this column value matches a regex',
       type: 'object',

api/src/paths/project/{projectId}/survey/{surveyId}/observation/submission/get.test.ts

@@ -98,7 +98,6 @@ describe('getObservationSubmission', () => {
             };
 
             const response = requestValidator.validateRequest(request);
-            console.log(JSON.stringify(response));
             expect(response.status).to.equal(400);
             expect(response.errors.length).to.equal(1);
             expect(response.errors[0].path).to.equal('projectId');

api/src/paths/project/{projectId}/survey/{surveyId}/observation/submission/get.ts

@@ -185,7 +185,8 @@ export function getOccurrenceSubmission(): RequestHandler {
           [
             // Submission statuses for validation completion
             SUBMISSION_STATUS_TYPE.TEMPLATE_VALIDATED,
-            SUBMISSION_STATUS_TYPE.DARWIN_CORE_VALIDATED
+            SUBMISSION_STATUS_TYPE.DARWIN_CORE_VALIDATED,
+            SUBMISSION_STATUS_TYPE.TEMPLATE_TRANSFORMED
           ].includes(occurrenceSubmission.submission_status_type_name));
 
       const messageTypes: IMessageTypeGroup[] = willFetchAdditionalMessages

api/src/paths/xlsx/process.test.ts

@@ -167,13 +167,11 @@ describe('xlsx/process', () => {
       mockReq['keycloak_token'] = 'token';
 
       const processXLSXFileStub = sinon.stub(ValidationService.prototype, 'processXLSXFile').resolves();
-      const processDWCFileStub = sinon.stub(ValidationService.prototype, 'processDWCFile').resolves();
 
       const requestHandler = process.processFile();
       await requestHandler(mockReq, mockRes, mockNext);
       expect(mockRes.statusValue).to.equal(200);
       expect(processXLSXFileStub).to.have.been.calledOnceWith(mockReq.body.occurrence_submission_id);
-      expect(processDWCFileStub).to.have.been.calledOnceWith(mockReq.body.occurrence_submission_id);
       expect(mockRes.jsonValue).to.eql({ status: 'success' });
     });
 
@@ -209,38 +207,6 @@ describe('xlsx/process', () => {
       }
     });
 
-    it('catches an error on processDWCFile', async () => {
-      const dbConnectionObj = getMockDBConnection({ rollback: sinon.stub(), release: sinon.stub() });
-      sinon.stub(db, 'getDBConnection').returns(dbConnectionObj);
-
-      const processXLSXFileStub = sinon.stub(ValidationService.prototype, 'processXLSXFile').resolves();
-      const processDWCFileStub = sinon
-        .stub(ValidationService.prototype, 'processDWCFile')
-        .throws(new Error('test processDWCFile error'));
-      const errorServiceStub = sinon.stub(ErrorService.prototype, 'insertSubmissionStatus').resolves();
-
-      const { mockReq, mockRes, mockNext } = getRequestHandlerMocks();
-      mockReq['keycloak_token'] = 'token';
-
-      mockReq.body = {
-        occurrence_submission_id: '123-456-789'
-      };
-
-      const requestHandler = process.processFile();
-
-      try {
-        await requestHandler(mockReq, mockRes, mockNext);
-        expect.fail();
-      } catch (actualError) {
-        expect(processXLSXFileStub).to.have.been.calledOnceWith(mockReq.body.occurrence_submission_id);
-        expect(processDWCFileStub).to.have.been.calledOnceWith(mockReq.body.occurrence_submission_id);
-        expect(errorServiceStub).to.have.been.calledOnce;
-        expect(dbConnectionObj.rollback).to.have.been.calledOnce;
-        expect(dbConnectionObj.release).to.have.been.calledOnce;
-        expect((actualError as Error).message).to.equal('test processDWCFile error');
-      }
-    });
-
     it('catches an error on insertSubmissionStatus', async () => {
       const dbConnectionObj = getMockDBConnection({ rollback: sinon.stub(), release: sinon.stub() });
       sinon.stub(db, 'getDBConnection').returns(dbConnectionObj);

api/src/paths/xlsx/process.ts

@@ -8,7 +8,6 @@ import { authorizeRequestHandler } from '../../request-handlers/security/authori
 import { ErrorService } from '../../services/error-service';
 import { ValidationService } from '../../services/validation-service';
 import { getLogger } from '../../utils/logger';
-import { SubmissionError } from '../../utils/submission-error';
 
 const defaultLog = getLogger('paths/xlsx/process');
 
@@ -113,18 +112,8 @@ export function processFile(): RequestHandler {
 
       const validationService = new ValidationService(connection);
 
-      try {
-        // process the raw template data
-        await validationService.processXLSXFile(submissionId, surveyId);
-        // process the resulting transformed dwc data
-        await validationService.processDWCFile(submissionId);
-      } catch (error: any) {
-        // Since submission errors are caught by the validation service and persisted in the database, anything
-        // outside of a submission message should be thrown here.
-        if (!(error instanceof SubmissionError)) {
-          throw error;
-        }
-      }
+      // process the raw template data
+      await validationService.processXLSXFile(submissionId, surveyId);
 
       await connection.commit();
     } catch (error) {

api/src/services/dwc-service.test.ts

@@ -2,12 +2,9 @@ import chai, { expect } from 'chai';
 import { describe } from 'mocha';
 import sinon from 'sinon';
 import sinonChai from 'sinon-chai';
-import { ApiError } from '../errors/api-error';
-import { IOccurrenceSubmission } from '../repositories/occurrence-repository';
 import * as spatial_utils from '../utils/spatial-utils';
 import { getMockDBConnection } from '../__mocks__/db';
 import { DwCService } from './dwc-service';
-import { OccurrenceService } from './occurrence-service';
 import { TaxonomyService } from './taxonomy-service';
 chai.use(sinonChai);
 
@@ -117,26 +114,6 @@ describe('DwCService', () => {
       expect(newJson).to.eql(jsonObject);
     });
 
-    it('throws an error if veratimCoordinates cannot be parsed ', async () => {
-      const dbConnectionObj = getMockDBConnection();
-
-      const dwcService = new DwCService(dbConnectionObj);
-      const jsonObject = {
-        item_with_depth_1: {
-          item_with_depth_2: [{ verbatimCoordinates: '12 12314 12241' }]
-        }
-      };
-
-      sinon.stub(spatial_utils, 'parseUTMString').returns(null);
-
-      try {
-        await dwcService.decorateLatLong(jsonObject);
-        expect.fail();
-      } catch (actualError) {
-        expect((actualError as ApiError).message).to.equal('Failed to parse UTM String');
-      }
-    });
-
     it('succeeds and decorates Lat Long', async () => {
       const dbConnectionObj = getMockDBConnection();
 
@@ -167,7 +144,8 @@ describe('DwCService', () => {
     });
   });
 
-  describe('decorateDWCASourceData', () => {
+  //TODO:  this needs to be examined thoroughly
+  describe('decorateDwCJSON', () => {
     afterEach(() => {
       sinon.restore();
     });
@@ -177,28 +155,18 @@ describe('DwCService', () => {
 
       const dwcService = new DwCService(dbConnectionObj);
 
-      const getOccurrenceSubmissionStub = sinon
-        .stub(OccurrenceService.prototype, 'getOccurrenceSubmission')
-        .resolves(({ id: 1, darwin_core_source: { id: 2 } } as unknown) as IOccurrenceSubmission);
-
       const decorateLatLongStub = sinon
         .stub(DwCService.prototype, 'decorateLatLong')
         .resolves({ id: 2, lat: 1, long: 2 });
       const decorateTaxonIDsStub = sinon
         .stub(DwCService.prototype, 'decorateTaxonIDs')
         .resolves({ id: 2, lat: 1, long: 2, taxonID: 3 });
 
-      const updateDWCSourceForOccurrenceSubmissionStub = sinon
-        .stub(OccurrenceService.prototype, 'updateDWCSourceForOccurrenceSubmission')
-        .resolves(1);
-
-      const response = await dwcService.decorateDWCASourceData(1);
+      const response = await dwcService.decorateDwCJSON({});
 
-      expect(response).to.eql(true);
-      expect(getOccurrenceSubmissionStub).to.be.calledOnce;
+      expect(response).to.eql({ id: 2, lat: 1, long: 2, taxonID: 3 });
       expect(decorateLatLongStub).to.be.calledOnce;
       expect(decorateTaxonIDsStub).to.be.calledOnce;
-      expect(updateDWCSourceForOccurrenceSubmissionStub).to.be.calledOnce;
     });
   });
 });

api/src/services/dwc-service.ts

@@ -1,10 +1,8 @@
 import jsonpatch, { Operation } from 'fast-json-patch';
 import { JSONPath } from 'jsonpath-plus';
 import { IDBConnection } from '../database/db';
-import { ApiError, ApiErrorType } from '../errors/api-error';
 import { parseUTMString, utmToLatLng } from '../utils/spatial-utils';
 import { DBService } from './db-service';
-import { OccurrenceService } from './occurrence-service';
 import { TaxonomyService } from './taxonomy-service';
 /**
  * Service to produce DWC data for a project.
@@ -72,27 +70,18 @@ export class DwCService extends DBService {
   }
 
   /**
-   * Run all Decoration functions on DWCA Source Data
+   * Decorates the DwC json object
    *
-   * @param {number} occurrenceSubmissionId
-   * @return {*}  {Promise<boolean>}
+   * @param {Record<any, any>} jsonObject
+   * @return {*}  {Promise<Record<any, any>>}
    * @memberof DwCService
    */
-  async decorateDWCASourceData(occurrenceSubmissionId: number): Promise<boolean> {
-    const occurrenceService = new OccurrenceService(this.connection);
-
-    const submission = await occurrenceService.getOccurrenceSubmission(occurrenceSubmissionId);
-    const jsonObject = submission.darwin_core_source;
-
+  async decorateDwCJSON(jsonObject: Record<any, any>): Promise<Record<any, any>> {
     const latlongDec = await this.decorateLatLong(jsonObject);
-    const taxonAndLatLongDec = await this.decorateTaxonIDs(latlongDec);
 
-    const response = await occurrenceService.updateDWCSourceForOccurrenceSubmission(
-      occurrenceSubmissionId,
-      JSON.stringify(taxonAndLatLongDec)
-    );
+    const taxonDec = await this.decorateTaxonIDs(latlongDec);
 
-    return !!response;
+    return taxonDec;
   }
 
   /**
@@ -110,7 +99,6 @@ export class DwCService extends DBService {
     });
 
     const patchOperations: Operation[] = [];
-    const errors: string[] = [];
 
     pathsToPatch.forEach(async (item: any) => {
       if (
@@ -125,7 +113,6 @@ export class DwCService extends DBService {
       const verbatimCoordinates = parseUTMString(item.value['verbatimCoordinates']);
 
       if (!verbatimCoordinates) {
-        errors.push('Failed to parse UTM String');
         return;
       }
 
@@ -146,10 +133,6 @@ export class DwCService extends DBService {
       patchOperations.push(decimalLatitudePatch, decimalLongitudePatch);
     });
 
-    if (errors.length) {
-      throw new ApiError(ApiErrorType.UNKNOWN, errors.join());
-    }
-
     return jsonpatch.applyPatch(jsonObject, patchOperations).newDocument;
   }
 }

api/src/services/occurrence-service.test.ts

@@ -68,12 +68,12 @@ describe('OccurrenceService', () => {
     });
   });
 
-  describe('updateSurveyOccurrenceSubmission', () => {
+  describe('updateSurveyOccurrenceSubmissionWithOutputKey', () => {
     it('should return a submission id', async () => {
       const service = mockService();
       sinon.stub(OccurrenceRepository.prototype, 'updateSurveyOccurrenceSubmissionWithOutputKey').resolves({});
 
-      const result = await service.updateSurveyOccurrenceSubmission(1, 'file name', 'key');
+      const result = await service.updateSurveyOccurrenceSubmissionWithOutputKey(1, 'file name', 'key');
       expect(result).to.be.eql({});
     });
   });

api/src/services/occurrence-service.ts

@@ -57,7 +57,11 @@ export class OccurrenceService extends DBService {
    * @param {string} key
    * @return {*} {Promise<any>}
    */
-  async updateSurveyOccurrenceSubmission(submissionId: number, fileName: string, key: string): Promise<any> {
+  async updateSurveyOccurrenceSubmissionWithOutputKey(
+    submissionId: number,
+    fileName: string,
+    key: string
+  ): Promise<any> {
     return this.occurrenceRepository.updateSurveyOccurrenceSubmissionWithOutputKey(submissionId, fileName, key);
   }