Hook up BioHub validated end-point to store errors and be callable by the workflow engine (XLSX)

api/src/paths/dwc/validate.test.ts

@@ -0,0 +1,175 @@
+import chai, { expect } from 'chai';
+import { describe } from 'mocha';
+import sinon from 'sinon';
+import sinonChai from 'sinon-chai';
+import * as validate from './validate';
+import * as db from '../../database/db';
+import * as survey_occurrence_queries from '../../queries/survey/survey-occurrence-queries';
+import SQL from 'sql-template-strings';
+import * as file_utils from '../../utils/file-utils';
+import { GetObjectOutput } from 'aws-sdk/clients/s3';
+
+chai.use(sinonChai);
+
+const dbConnectionObj = {
+  systemUserId: () => {
+    return 20;
+  },
+  open: async () => {
+    // do nothing
+  },
+  release: () => {
+    // do nothing
+  },
+  commit: async () => {
+    // do nothing
+  },
+  rollback: async () => {
+    // do nothing
+  },
+  query: async () => {
+    // do nothing
+  }
+};
+
+const sampleReq = {
+  keycloak_token: {},
+  body: {
+    occurrence_submission_id: 1
+  }
+} as any;
+
+describe('getSubmissionS3Key', () => {
+  const sampleReq = {
+    keycloak_token: {},
+    body: {
+      occurrence_submission_id: 1
+    }
+  } as any;
+
+  afterEach(() => {
+    sinon.restore();
+  });
+
+  it('should throw a 400 error when no occurrence submission id is provided', async () => {
+    sinon.stub(db, 'getDBConnection').returns(dbConnectionObj);
+
+    try {
+      const result = validate.getSubmissionS3Key();
+      await result(
+        { ...sampleReq, body: { ...sampleReq.body, occurrence_submission_id: null } },
+        (null as unknown) as any,
+        (null as unknown) as any
+      );
+      expect.fail();
+    } catch (actualError) {
+      expect(actualError.status).to.equal(400);
+      expect(actualError.message).to.equal('Missing required body param `occurrence_submission_id`.');
+    }
+  });
+
+  it('should throw a 400 error when no sql statement returned for getSurveyOccurrenceSubmissionSQL', async () => {
+    sinon.stub(db, 'getDBConnection').returns(dbConnectionObj);
+    sinon.stub(survey_occurrence_queries, 'getSurveyOccurrenceSubmissionSQL').returns(null);
+
+    try {
+      const result = validate.getSubmissionS3Key();
+
+      await result(sampleReq, (null as unknown) as any, (null as unknown) as any);
+      expect.fail();
+    } catch (actualError) {
+      expect(actualError.status).to.equal(400);
+      expect(actualError.message).to.equal('Failed to build SQL get statement');
+    }
+  });
+
+  it('should throw a 400 error when no rows returned', async () => {
+    const mockQuery = sinon.stub();
+
+    mockQuery.resolves({
+      rows: []
+    });
+
+    sinon.stub(db, 'getDBConnection').returns({
+      ...dbConnectionObj,
+      query: mockQuery
+    });
+
+    sinon.stub(survey_occurrence_queries, 'getSurveyOccurrenceSubmissionSQL').returns(SQL`something`);
+
+    try {
+      const result = validate.getSubmissionS3Key();
+
+      await result(sampleReq, (null as unknown) as any, (null as unknown) as any);
+      expect.fail();
+    } catch (actualError) {
+      expect(actualError.status).to.equal(400);
+      expect(actualError.message).to.equal('Failed to get survey occurrence submission');
+    }
+  });
+
+  it('should set the s3 key in the request on success', async () => {
+    const nextSpy = sinon.spy();
+    const mockQuery = sinon.stub();
+
+    mockQuery.resolves({
+      rows: [{ key: 'somekey' }]
+    });
+
+    sinon.stub(db, 'getDBConnection').returns({
+      ...dbConnectionObj,
+      query: mockQuery
+    });
+
+    sinon.stub(survey_occurrence_queries, 'getSurveyOccurrenceSubmissionSQL').returns(SQL`something`);
+
+    const result = validate.getSubmissionS3Key();
+    await result(sampleReq, (null as unknown) as any, nextSpy as any);
+
+    expect(sampleReq.s3Key).to.equal('somekey');
+    expect(nextSpy).to.have.been.called;
+  });
+});
+
+describe('getSubmissionFileFromS3', () => {
+  const updatedSampleReq = { ...sampleReq, s3Key: 'somekey' };
+
+  afterEach(() => {
+    sinon.restore();
+  });
+
+  it('should throw a 500 error when no file in S3', async () => {
+    sinon.stub(db, 'getDBConnection').returns(dbConnectionObj);
+    sinon.stub(file_utils, 'getFileFromS3').resolves(undefined);
+
+    try {
+      const result = validate.getSubmissionFileFromS3();
+      await result(updatedSampleReq, (null as unknown) as any, (null as unknown) as any);
+      expect.fail();
+    } catch (actualError) {
+      expect(actualError.status).to.equal(500);
+      expect(actualError.message).to.equal('Failed to get occurrence submission file');
+    }
+  });
+
+  it('should set the s3 file in the request on success', async () => {
+    const file = {
+      fieldname: 'media',
+      originalname: 'test.txt',
+      encoding: '7bit',
+      mimetype: 'text/plain',
+      size: 340
+    };
+
+    const nextSpy = sinon.spy();
+
+    sinon.stub(db, 'getDBConnection').returns(dbConnectionObj);
+    sinon.stub(file_utils, 'getFileFromS3').resolves(file as GetObjectOutput);
+
+    const result = validate.getSubmissionFileFromS3();
+    await result(sampleReq, (null as unknown) as any, nextSpy as any);
+
+    expect(sampleReq.s3File).to.eql(file);
+    expect(nextSpy).to.have.been.called;
+  });
+});

api/src/paths/dwc/validate.ts

@@ -31,58 +31,68 @@ export const POST: Operation = [
   persistParseErrors(),
   getValidationRules(),
   validateDWCArchive(),
-  persistValidationResults()
+  persistValidationResults({ initialSubmissionStatusType: 'Darwin Core Validated' })
 ];
 
-POST.apiDoc = {
-  description: 'Validates a Darwin Core (DWC) Archive survey observation submission.',
-  tags: ['survey', 'observation', 'dwc'],
-  security: [
-    {
-      Bearer: [SYSTEM_ROLE.SYSTEM_ADMIN, SYSTEM_ROLE.PROJECT_ADMIN]
-    }
-  ],
-  requestBody: {
-    description: 'Request body',
-    content: {
-      'application/json': {
-        schema: {
-          type: 'object',
-          required: ['occurrence_submission_id'],
-          properties: {
-            occurrence_submission_id: {
-              description: 'A survey occurrence submission ID',
-              type: 'number',
-              example: 1
+export const getValidateAPIDoc = (basicDescription: string, successDescription: string, tags: string[]) => {
+  return {
+    description: basicDescription,
+    tags: tags,
+    security: [
+      {
+        Bearer: [SYSTEM_ROLE.SYSTEM_ADMIN, SYSTEM_ROLE.PROJECT_ADMIN]
+      }
+    ],
+    requestBody: {
+      description: 'Request body',
+      content: {
+        'application/json': {
+          schema: {
+            type: 'object',
+            required: ['occurrence_submission_id'],
+            properties: {
+              occurrence_submission_id: {
+                description: 'A survey occurrence submission ID',
+                type: 'number',
+                example: 1
+              }
             }
           }
         }
       }
-    }
-  },
-  responses: {
-    200: {
-      description: 'Validate Darwin Core (DWC) Archive survey observation submission OK'
-    },
-    400: {
-      $ref: '#/components/responses/400'
-    },
-    401: {
-      $ref: '#/components/responses/401'
-    },
-    403: {
-      $ref: '#/components/responses/401'
-    },
-    500: {
-      $ref: '#/components/responses/500'
     },
-    default: {
-      $ref: '#/components/responses/default'
+    responses: {
+      200: {
+        description: successDescription
+      },
+      400: {
+        $ref: '#/components/responses/400'
+      },
+      401: {
+        $ref: '#/components/responses/401'
+      },
+      403: {
+        $ref: '#/components/responses/401'
+      },
+      500: {
+        $ref: '#/components/responses/500'
+      },
+      default: {
+        $ref: '#/components/responses/default'
+      }
     }
-  }
+  };
 };
 
-function getSubmissionS3Key(): RequestHandler {
+POST.apiDoc = {
+  ...getValidateAPIDoc(
+    'Validates a Darwin Core (DWC) Archive survey observation submission.',
+    'Validate Darwin Core (DWC) Archive survey observation submission OK',
+    ['survey', 'observation', 'dwc']
+  )
+};
+
+export function getSubmissionS3Key(): RequestHandler {
   return async (req, res, next) => {
     defaultLog.debug({ label: 'getSubmissionS3Key', message: 'params', files: req.body });
 
@@ -123,7 +133,7 @@ function getSubmissionS3Key(): RequestHandler {
   };
 }
 
-function getSubmissionFileFromS3(): RequestHandler {
+export function getSubmissionFileFromS3(): RequestHandler {
   return async (req, res, next) => {
     defaultLog.debug({ label: 'getSubmissionFileFromS3', message: 'params', files: req.body });
 
@@ -179,13 +189,13 @@ function prepDWCArchive(): RequestHandler {
   };
 }
 
-function persistParseErrors(): RequestHandler {
+export function persistParseErrors(): RequestHandler {
   return async (req, res, next) => {
     const parseError = req['parseError'];
 
     if (!parseError) {
       // no errors to persist, skip to next step
-      next();
+      return next();
     }
 
     defaultLog.debug({ label: 'persistParseErrors', message: 'parseError', parseError });
@@ -287,7 +297,7 @@ function validateDWCArchive(): RequestHandler {
   };
 }
 
-function persistValidationResults(): RequestHandler {
+export function persistValidationResults(statusTypeObject: any): RequestHandler {
   return async (req, res) => {
     defaultLog.debug({ label: 'persistValidationResults', message: 'validationResults' });
 
@@ -299,7 +309,7 @@ function persistValidationResults(): RequestHandler {
 
       await connection.open();
 
-      let submissionStatusType = 'Darwin Core Validated';
+      let submissionStatusType = statusTypeObject.initialSubmissionStatusType;
       if (mediaState?.some((item) => !item.isValid) || csvState?.some((item) => !item.isValid)) {
         // At least 1 error exists
         submissionStatusType = 'Rejected';

api/src/paths/project/{projectId}/survey/{surveyId}/observation/submission/{submissionId}/view.test.ts

@@ -204,7 +204,7 @@ describe('getObservationSubmissionCSVForView', () => {
     }
   });
 
-  it('should return data on success with xslx file (empty)', async () => {
+  it('should return data on success with xlsx file (empty)', async () => {
     const mockQuery = sinon.stub();
 
     mockQuery.resolves({

api/src/paths/project/{projectId}/survey/{surveyId}/observation/submission/{submissionId}/view.ts

@@ -8,8 +8,8 @@ import { HTTP400, HTTP500 } from '../../../../../../../../errors/CustomError';
 import { getSurveyOccurrenceSubmissionSQL } from '../../../../../../../../queries/survey/survey-occurrence-queries';
 import { generateS3FileKey, getFileFromS3 } from '../../../../../../../../utils/file-utils';
 import { getLogger } from '../../../../../../../../utils/logger';
-import { XLSXCSV } from '../../../../../../../../utils/media/csv/csv-file';
 import { DWCArchive } from '../../../../../../../../utils/media/csv/dwc/dwc-archive-file';
+import { XLSXCSV } from '../../../../../../../../utils/media/csv/csv-file';
 import { MediaFile } from '../../../../../../../../utils/media/media-file';
 import { parseUnknownMedia } from '../../../../../../../../utils/media/media-utils';
 
@@ -158,10 +158,10 @@ export function getObservationSubmissionCSVForView(): RequestHandler {
 
       // Get the worksheets for the file based on type
       if (parsedMedia instanceof MediaFile) {
-        // xslx
-        const xlsxCSV = new XLSXCSV(parsedMedia);
+        // xlsx csv
+        const xlsxCsv = new XLSXCSV(parsedMedia);
 
-        worksheets = xlsxCSV.workbook.worksheets;
+        worksheets = xlsxCsv.workbook.worksheets;
       } else {
         // dwc archive
         const dwcArchive = new DWCArchive(parsedMedia);