SIMSBIOHUB-656: Import CSV Markings

api/src/errors/http-error.ts

@@ -1,4 +1,5 @@
 import { DatabaseError } from 'pg';
+import { CSVError } from '../utils/csv-utils/csv-config-validation.interface';
 import { ApiError } from './api-error';
 import { BaseError } from './base-error';
 
@@ -10,10 +11,20 @@
   INTERNAL_SERVER_ERROR = 'Internal Server Error'
 }
 
+export enum HTTPCustomErrorType {
+  CSV_VALIDATION_ERROR = 'CSV Validation Error'
+}
+
 export class HTTPError extends BaseError {
   status: number;
 
-  constructor(name: HTTPErrorType, status: number, message: string, errors?: (string | object)[], stack?: string) {
+  constructor(
+    name: HTTPErrorType | HTTPCustomErrorType,
+    status: number,
+    message: string,
+    errors?: (string | object)[],
+    stack?: string
+  ) {
     super(name, message, errors, stack);
 
     this.status = status;
@@ -85,6 +96,19 @@
   }
 }
 
+/**
+ * A HTTP `422 CSV Validation Error` error.
+ *
+ * @export
+ * @class CSVValidationError
+ * @extends {HTTPError}
+ */
+export class HTTP422CSVValidationError extends HTTPError {
+  constructor(message: string, errors: CSVError[]) {
+    super(HTTPCustomErrorType.CSV_VALIDATION_ERROR, 422, message, errors);
+  }
+}
+
 /**
  * Ensures that the incoming error is converted into an `HTTPError` if it is not one already.
  * If `error` is a `HTTPError`, then change nothing and return it.

api/src/openapi/schemas/csv.ts

@@ -8,7 +8,7 @@ export const CSVErrorSchema: OpenAPIV3.SchemaObject = {
   title: 'CSV validation error object',
   type: 'object',
   additionalProperties: false,
-  required: ['error', 'solution', 'row'],
+  required: ['error', 'solution', 'values', 'cell', 'header', 'row'],
   properties: {
     error: {
       description: 'The error message',
@@ -21,21 +21,73 @@ export const CSVErrorSchema: OpenAPIV3.SchemaObject = {
     values: {
       description: 'The list of allowed values if applicable',
       type: 'array',
+      nullable: true,
       items: {
-        oneOf: [{ type: 'string' }, { type: 'number' }]
+        oneOf: [
+          {
+            type: 'string'
+          },
+          {
+            type: 'number'
+          }
+        ]
       }
     },
     cell: {
       description: 'The CSV cell value',
-      oneOf: [{ type: 'string' }, { type: 'number' }]
+      oneOf: [
+        {
+          type: 'string'
+        },
+        {
+          type: 'number'
+        }
+      ],
+      nullable: true
     },
     header: {
       description: 'The header name used in the CSV file',
-      type: 'string'
+      type: 'string',
+      nullable: true
     },
     row: {
       description: 'The row index the error occurred. Header row index 0. First data row index 1.',
       type: 'number'
     }
   }
 };
+
+/**
+ * CSV validation error response schema
+ *
+ */
+export const CSVValidationErrorResponse: OpenAPIV3.ResponseObject = {
+  description: 'CSV validation errors response',
+  content: {
+    'application/json': {
+      schema: {
+        description: 'CSV validation error response object',
+        required: ['name', 'status', 'message', 'errors'],
+        properties: {
+          name: {
+            description: 'Error name',
+            type: 'string'
+          },
+          status: {
+            description: 'HTTP status code',
+            type: 'number'
+          },
+          message: {
+            description: 'Error message',
+            type: 'string'
+          },
+          errors: {
+            type: 'array',
+            description: 'List of CSV errors which occurred during validation',
+            items: CSVErrorSchema
+          }
+        }
+      }
+    }
+  }
+};

api/src/paths/project/{projectId}/survey/{surveyId}/critters/import.test.ts

@@ -16,6 +16,8 @@ describe('importCsv', () => {
 
     const importCSVWorksheetStub = sinon.stub(ImportCrittersService.prototype, 'importCSVWorksheet');
 
+    importCSVWorksheetStub.resolves([]);
+
     const mockFile = { originalname: 'test.csv', mimetype: 'test.csv', buffer: Buffer.alloc(1) } as Express.Multer.File;
 
     const { mockReq, mockRes, mockNext } = getRequestHandlerMocks();

api/src/paths/project/{projectId}/survey/{surveyId}/critters/import.ts

@@ -2,9 +2,12 @@
 import { Operation } from 'express-openapi';
 import { PROJECT_PERMISSION, SYSTEM_ROLE } from '../../../../../../constants/roles';
 import { getDBConnection } from '../../../../../../database/db';
+import { HTTP422CSVValidationError } from '../../../../../../errors/http-error';
+import { CSVValidationErrorResponse } from '../../../../../../openapi/schemas/csv';
 import { csvFileSchema } from '../../../../../../openapi/schemas/file';
 import { authorizeRequestHandler } from '../../../../../../request-handlers/security/authorization';
 import { ImportCrittersService } from '../../../../../../services/import-services/critter/import-critters-service';
+import { CSV_ERROR_MESSAGE } from '../../../../../../utils/csv-utils/csv-config-validation.interface';
 import { getLogger } from '../../../../../../utils/logger';
 import { parseMulterFile } from '../../../../../../utils/media/media-utils';
 import { getFileFromRequest } from '../../../../../../utils/request';
@@ -94,6 +97,7 @@
     403: {
       $ref: '#/components/responses/403'
     },
+    422: CSVValidationErrorResponse,
     500: {
       $ref: '#/components/responses/500'
     },
@@ -123,7 +127,11 @@
 
       const importService = new ImportCrittersService(connection, worksheet, surveyId);
 
-      await importService.importCSVWorksheet();
+      const errors = await importService.importCSVWorksheet();
+
+      if (errors.length) {
+        throw new HTTP422CSVValidationError(CSV_ERROR_MESSAGE, errors);
+      }
 
       await connection.commit();
 

api/src/paths/project/{projectId}/survey/{surveyId}/critters/markings/import.ts

@@ -2,13 +2,16 @@
 import { Operation } from 'express-openapi';
 import { PROJECT_PERMISSION, SYSTEM_ROLE } from '../../../../../../../constants/roles';
 import { getDBConnection } from '../../../../../../../database/db';
+import { HTTP422CSVValidationError } from '../../../../../../../errors/http-error';
+import { CSVValidationErrorResponse } from '../../../../../../../openapi/schemas/csv';
 import { csvFileSchema } from '../../../../../../../openapi/schemas/file';
 import { authorizeRequestHandler } from '../../../../../../../request-handlers/security/authorization';
-import { importCSV } from '../../../../../../../services/import-services/import-csv';
-import { ImportMarkingsStrategy } from '../../../../../../../services/import-services/marking/import-markings-strategy';
+import { ImportMarkingsService } from '../../../../../../../services/import-services/marking/import-markings-service';
+import { CSV_ERROR_MESSAGE } from '../../../../../../../utils/csv-utils/csv-config-validation.interface';
 import { getLogger } from '../../../../../../../utils/logger';
 import { parseMulterFile } from '../../../../../../../utils/media/media-utils';
 import { getFileFromRequest } from '../../../../../../../utils/request';
+import { constructXLSXWorkbook, getDefaultWorksheet } from '../../../../../../../utils/xlsx-utils/worksheet-utils';
 
 const defaultLog = getLogger('/api/project/{projectId}/survey/{surveyId}/markings/import');
 
@@ -85,21 +88,7 @@
   },
   responses: {
     201: {
-      description: 'Marking import success.',
-      content: {
-        'application/json': {
-          schema: {
-            type: 'object',
-            additionalProperties: false,
-            properties: {
-              markingsCreated: {
-                description: 'Number of Critterbase markings created.',
-                type: 'integer'
-              }
-            }
-          }
-        }
-      }
+      description: 'Marking import success.'
     },
     400: {
       $ref: '#/components/responses/400'
@@ -110,6 +99,7 @@
     403: {
       $ref: '#/components/responses/403'
     },
+    422: CSVValidationErrorResponse,
     500: {
       $ref: '#/components/responses/500'
     },
@@ -131,17 +121,23 @@
 
     const connection = getDBConnection(req.keycloak_token);
 
+    const mediaFile = parseMulterFile(rawFile);
+    const worksheet = getDefaultWorksheet(constructXLSXWorkbook(mediaFile));
+
     try {
       await connection.open();
 
-      const importCsvMarkingsStrategy = new ImportMarkingsStrategy(connection, surveyId);
+      const importMarkings = new ImportMarkingsService(connection, worksheet, surveyId);
+
+      const errors = await importMarkings.importCSVWorksheet();
 
-      // Pass CSV file and importer as dependencies
-      const markingsCreated = await importCSV(parseMulterFile(rawFile), importCsvMarkingsStrategy);
+      if (errors.length) {
+        throw new HTTP422CSVValidationError(CSV_ERROR_MESSAGE, errors);
+      }
 
       await connection.commit();
 
-      return res.status(201).json({ markingsCreated });
+      return res.status(201).send();
     } catch (error) {
       defaultLog.error({ label: 'importMarkingsCSV', message: 'error', error });
       await connection.rollback();