diff --git a/api/src/errors/http-error.ts b/api/src/errors/http-error.ts index dd2a649eb1..576fa32a67 100644 --- a/api/src/errors/http-error.ts +++ b/api/src/errors/http-error.ts @@ -1,4 +1,5 @@ import { DatabaseError } from 'pg'; +import { CSVError } from '../utils/csv-utils/csv-config-validation.interface'; import { ApiError } from './api-error'; import { BaseError } from './base-error'; @@ -10,10 +11,20 @@ export enum HTTPErrorType { INTERNAL_SERVER_ERROR = 'Internal Server Error' } +export enum HTTPCustomErrorType { + CSV_VALIDATION_ERROR = 'CSV Validation Error' +} + export class HTTPError extends BaseError { status: number; - constructor(name: HTTPErrorType, status: number, message: string, errors?: (string | object)[], stack?: string) { + constructor( + name: HTTPErrorType | HTTPCustomErrorType, + status: number, + message: string, + errors?: (string | object)[], + stack?: string + ) { super(name, message, errors, stack); this.status = status; @@ -85,6 +96,19 @@ export class HTTP500 extends HTTPError { } } +/** + * A HTTP `422 CSV Validation Error` error. + * + * @export + * @class CSVValidationError + * @extends {HTTPError} + */ +export class HTTP422CSVValidationError extends HTTPError { + constructor(message: string, errors: CSVError[]) { + super(HTTPCustomErrorType.CSV_VALIDATION_ERROR, 422, message, errors); + } +} + /** * Ensures that the incoming error is converted into an `HTTPError` if it is not one already. * If `error` is a `HTTPError`, then change nothing and return it. diff --git a/api/src/openapi/schemas/csv.ts b/api/src/openapi/schemas/csv.ts index 8c18600e35..051b900e88 100644 --- a/api/src/openapi/schemas/csv.ts +++ b/api/src/openapi/schemas/csv.ts @@ -8,7 +8,7 @@ export const CSVErrorSchema: OpenAPIV3.SchemaObject = { title: 'CSV validation error object', type: 'object', additionalProperties: false, - required: ['error', 'solution', 'row'], + required: ['error', 'solution', 'values', 'cell', 'header', 'row'], properties: { error: { description: 'The error message', @@ -21,17 +21,34 @@ export const CSVErrorSchema: OpenAPIV3.SchemaObject = { values: { description: 'The list of allowed values if applicable', type: 'array', + nullable: true, items: { - oneOf: [{ type: 'string' }, { type: 'number' }] + oneOf: [ + { + type: 'string' + }, + { + type: 'number' + } + ] } }, cell: { description: 'The CSV cell value', - oneOf: [{ type: 'string' }, { type: 'number' }] + oneOf: [ + { + type: 'string' + }, + { + type: 'number' + } + ], + nullable: true }, header: { description: 'The header name used in the CSV file', - type: 'string' + type: 'string', + nullable: true }, row: { description: 'The row index the error occurred. Header row index 0. First data row index 1.', @@ -39,3 +56,38 @@ export const CSVErrorSchema: OpenAPIV3.SchemaObject = { } } }; + +/** + * CSV validation error response schema + * + */ +export const CSVValidationErrorResponse: OpenAPIV3.ResponseObject = { + description: 'CSV validation errors response', + content: { + 'application/json': { + schema: { + description: 'CSV validation error response object', + required: ['name', 'status', 'message', 'errors'], + properties: { + name: { + description: 'Error name', + type: 'string' + }, + status: { + description: 'HTTP status code', + type: 'number' + }, + message: { + description: 'Error message', + type: 'string' + }, + errors: { + type: 'array', + description: 'List of CSV errors which occurred during validation', + items: CSVErrorSchema + } + } + } + } + } +}; diff --git a/api/src/paths/project/{projectId}/survey/{surveyId}/critters/import.test.ts b/api/src/paths/project/{projectId}/survey/{surveyId}/critters/import.test.ts index 6c24e5b70f..50316c5fea 100644 --- a/api/src/paths/project/{projectId}/survey/{surveyId}/critters/import.test.ts +++ b/api/src/paths/project/{projectId}/survey/{surveyId}/critters/import.test.ts @@ -16,6 +16,8 @@ describe('importCsv', () => { const importCSVWorksheetStub = sinon.stub(ImportCrittersService.prototype, 'importCSVWorksheet'); + importCSVWorksheetStub.resolves([]); + const mockFile = { originalname: 'test.csv', mimetype: 'test.csv', buffer: Buffer.alloc(1) } as Express.Multer.File; const { mockReq, mockRes, mockNext } = getRequestHandlerMocks(); diff --git a/api/src/paths/project/{projectId}/survey/{surveyId}/critters/import.ts b/api/src/paths/project/{projectId}/survey/{surveyId}/critters/import.ts index f3b83117fa..8f0de88d54 100644 --- a/api/src/paths/project/{projectId}/survey/{surveyId}/critters/import.ts +++ b/api/src/paths/project/{projectId}/survey/{surveyId}/critters/import.ts @@ -2,9 +2,12 @@ import { RequestHandler } from 'express'; import { Operation } from 'express-openapi'; import { PROJECT_PERMISSION, SYSTEM_ROLE } from '../../../../../../constants/roles'; import { getDBConnection } from '../../../../../../database/db'; +import { HTTP422CSVValidationError } from '../../../../../../errors/http-error'; +import { CSVValidationErrorResponse } from '../../../../../../openapi/schemas/csv'; import { csvFileSchema } from '../../../../../../openapi/schemas/file'; import { authorizeRequestHandler } from '../../../../../../request-handlers/security/authorization'; import { ImportCrittersService } from '../../../../../../services/import-services/critter/import-critters-service'; +import { CSV_ERROR_MESSAGE } from '../../../../../../utils/csv-utils/csv-config-validation.interface'; import { getLogger } from '../../../../../../utils/logger'; import { parseMulterFile } from '../../../../../../utils/media/media-utils'; import { getFileFromRequest } from '../../../../../../utils/request'; @@ -94,6 +97,7 @@ POST.apiDoc = { 403: { $ref: '#/components/responses/403' }, + 422: CSVValidationErrorResponse, 500: { $ref: '#/components/responses/500' }, @@ -123,7 +127,11 @@ export function importCritterCSV(): RequestHandler { const importService = new ImportCrittersService(connection, worksheet, surveyId); - await importService.importCSVWorksheet(); + const errors = await importService.importCSVWorksheet(); + + if (errors.length) { + throw new HTTP422CSVValidationError(CSV_ERROR_MESSAGE, errors); + } await connection.commit(); diff --git a/api/src/paths/project/{projectId}/survey/{surveyId}/critters/markings/import.ts b/api/src/paths/project/{projectId}/survey/{surveyId}/critters/markings/import.ts index ae13127315..5f9bb80c01 100644 --- a/api/src/paths/project/{projectId}/survey/{surveyId}/critters/markings/import.ts +++ b/api/src/paths/project/{projectId}/survey/{surveyId}/critters/markings/import.ts @@ -2,13 +2,16 @@ import { RequestHandler } from 'express'; import { Operation } from 'express-openapi'; import { PROJECT_PERMISSION, SYSTEM_ROLE } from '../../../../../../../constants/roles'; import { getDBConnection } from '../../../../../../../database/db'; +import { HTTP422CSVValidationError } from '../../../../../../../errors/http-error'; +import { CSVValidationErrorResponse } from '../../../../../../../openapi/schemas/csv'; import { csvFileSchema } from '../../../../../../../openapi/schemas/file'; import { authorizeRequestHandler } from '../../../../../../../request-handlers/security/authorization'; -import { importCSV } from '../../../../../../../services/import-services/import-csv'; -import { ImportMarkingsStrategy } from '../../../../../../../services/import-services/marking/import-markings-strategy'; +import { ImportMarkingsService } from '../../../../../../../services/import-services/marking/import-markings-service'; +import { CSV_ERROR_MESSAGE } from '../../../../../../../utils/csv-utils/csv-config-validation.interface'; import { getLogger } from '../../../../../../../utils/logger'; import { parseMulterFile } from '../../../../../../../utils/media/media-utils'; import { getFileFromRequest } from '../../../../../../../utils/request'; +import { constructXLSXWorkbook, getDefaultWorksheet } from '../../../../../../../utils/xlsx-utils/worksheet-utils'; const defaultLog = getLogger('/api/project/{projectId}/survey/{surveyId}/markings/import'); @@ -85,21 +88,7 @@ POST.apiDoc = { }, responses: { 201: { - description: 'Marking import success.', - content: { - 'application/json': { - schema: { - type: 'object', - additionalProperties: false, - properties: { - markingsCreated: { - description: 'Number of Critterbase markings created.', - type: 'integer' - } - } - } - } - } + description: 'Marking import success.' }, 400: { $ref: '#/components/responses/400' @@ -110,6 +99,7 @@ POST.apiDoc = { 403: { $ref: '#/components/responses/403' }, + 422: CSVValidationErrorResponse, 500: { $ref: '#/components/responses/500' }, @@ -131,17 +121,23 @@ export function importCsv(): RequestHandler { const connection = getDBConnection(req.keycloak_token); + const mediaFile = parseMulterFile(rawFile); + const worksheet = getDefaultWorksheet(constructXLSXWorkbook(mediaFile)); + try { await connection.open(); - const importCsvMarkingsStrategy = new ImportMarkingsStrategy(connection, surveyId); + const importMarkings = new ImportMarkingsService(connection, worksheet, surveyId); + + const errors = await importMarkings.importCSVWorksheet(); - // Pass CSV file and importer as dependencies - const markingsCreated = await importCSV(parseMulterFile(rawFile), importCsvMarkingsStrategy); + if (errors.length) { + throw new HTTP422CSVValidationError(CSV_ERROR_MESSAGE, errors); + } await connection.commit(); - return res.status(201).json({ markingsCreated }); + return res.status(201).send(); } catch (error) { defaultLog.error({ label: 'importMarkingsCSV', message: 'error', error }); await connection.rollback(); diff --git a/api/src/services/import-services/critter/critter-header-configs.test.ts b/api/src/services/import-services/critter/critter-header-configs.test.ts index d965a70a2f..3c92d379c1 100644 --- a/api/src/services/import-services/critter/critter-header-configs.test.ts +++ b/api/src/services/import-services/critter/critter-header-configs.test.ts @@ -11,12 +11,17 @@ import { getCritterSexCellValidator, getWlhIDCellValidator } from './critter-header-configs'; +import { CritterCSVStaticHeader } from './import-critters-service'; -const mockConfig: CSVConfig = { +const mockConfig: CSVConfig = { staticHeadersConfig: { - ALIAS: { aliases: [] } + ITIS_TSN: { aliases: ['TAXON', 'SPECIES', 'TSN'] }, + ALIAS: { aliases: ['NICKNAME', 'NAME', 'ANIMAL_ID'] }, + SEX: { aliases: [], optional: true }, + WLH_ID: { aliases: ['WILDLIFE_HEALTH_ID', 'WILD LIFE HEALTH ID', 'WLHID'], optional: true }, + DESCRIPTION: { aliases: ['COMMENTS', 'COMMENT', 'NOTES'], optional: true } }, - ignoreDynamicHeaders: true + ignoreDynamicHeaders: false }; describe('critter-header-configs', () => { @@ -29,7 +34,13 @@ describe('critter-header-configs', () => { new CSVConfigUtils(xlsx.utils.json_to_sheet([]), mockConfig) ); - const result = critterAliasValidator({ cell: badCellValue, row: {}, header: 'ALIAS', rowIndex: 0 }); + const result = critterAliasValidator({ + cell: badCellValue, + row: {}, + header: 'ALIAS', + rowIndex: 0, + mutateCell: badCellValue + }); expect(result.length).to.be.equal(1); } @@ -42,7 +53,13 @@ describe('critter-header-configs', () => { const critterAliasValidator = getCritterAliasCellValidator(surveyAliases, configUtils); - const result = critterAliasValidator({ cell: 'alias4', row: {}, header: 'ALIAS', rowIndex: 0 }); + const result = critterAliasValidator({ + cell: 'alias4', + row: {}, + header: 'ALIAS', + rowIndex: 0, + mutateCell: 'alias4' + }); expect(result).to.be.deep.equal([]); }); @@ -54,7 +71,13 @@ describe('critter-header-configs', () => { const critterAliasValidator = getCritterAliasCellValidator(surveyAliases, configUtils); - const result = critterAliasValidator({ cell: 'alias1', row: {}, header: 'ALIAS', rowIndex: 0 }); + const result = critterAliasValidator({ + cell: 'alias1', + row: {}, + header: 'ALIAS', + rowIndex: 0, + mutateCell: 'alias1' + }); expect(result).to.be.deep.equal([ { @@ -71,7 +94,13 @@ describe('critter-header-configs', () => { const critterAliasValidator = getCritterAliasCellValidator(surveyAliases, configUtils); - const result = critterAliasValidator({ cell: 'alias3', row: {}, header: 'ALIAS', rowIndex: 0 }); + const result = critterAliasValidator({ + cell: 'alias3', + row: {}, + header: 'ALIAS', + rowIndex: 0, + mutateCell: 'alias3' + }); expect(result).to.be.deep.equal([ { @@ -98,7 +127,13 @@ describe('critter-header-configs', () => { const cellValues = ['unit', undefined]; for (const cell of cellValues) { - const result = cellValidator({ cell: cell, row: { ITIS_TSN: 1 }, header: 'HEADER', rowIndex: 0 }); + const result = cellValidator({ + cell: cell, + row: { ITIS_TSN: 1 }, + header: 'HEADER', + rowIndex: 0, + mutateCell: cell + }); expect(result).to.be.deep.equal([]); } @@ -116,7 +151,13 @@ describe('critter-header-configs', () => { new CSVConfigUtils(xlsx.utils.json_to_sheet([]), mockConfig) ); - const result = cellValidator({ cell: 'unit', row: { ITIS_TSN: 2 }, header: 'HEADER', rowIndex: 0 }); + const result = cellValidator({ + cell: 'unit', + row: { ITIS_TSN: 2 }, + header: 'HEADER', + rowIndex: 0, + mutateCell: 'unit' + }); expect(result[0].error).to.be.equal('Collection units not found for TSN: 2'); }); @@ -133,7 +174,13 @@ describe('critter-header-configs', () => { new CSVConfigUtils(xlsx.utils.json_to_sheet([]), mockConfig) ); - const result = cellValidator({ cell: 'unit', row: { ITIS_TSN: 1 }, header: 'HEADER2', rowIndex: 0 }); + const result = cellValidator({ + cell: 'unit', + row: { ITIS_TSN: 1 }, + header: 'HEADER2', + rowIndex: 0, + mutateCell: 'unit' + }); expect(result[0].error).to.be.equal('Invalid collection category header'); }); @@ -150,7 +197,13 @@ describe('critter-header-configs', () => { new CSVConfigUtils(xlsx.utils.json_to_sheet([]), mockConfig) ); - const result = cellValidator({ cell: 'unit2', row: { ITIS_TSN: 1 }, header: 'HEADER', rowIndex: 0 }); + const result = cellValidator({ + cell: 'unit2', + row: { ITIS_TSN: 1 }, + header: 'HEADER', + rowIndex: 0, + mutateCell: 'unit2' + }); expect(result[0].error).to.be.equal('Invalid collection unit cell value'); }); @@ -163,7 +216,7 @@ describe('critter-header-configs', () => { new CSVConfigUtils(xlsx.utils.json_to_sheet([]), mockConfig) ); - const result = cellSetter({ cell: '', row: {}, header: 'HEADER', rowIndex: 0 }); + const result = cellSetter({ cell: '', row: {}, header: 'HEADER', rowIndex: 0, mutateCell: '' }); expect(result).to.be.equal(undefined); }); @@ -180,7 +233,13 @@ describe('critter-header-configs', () => { new CSVConfigUtils(xlsx.utils.json_to_sheet([]), mockConfig) ); - const result = cellSetter({ cell: 'unit', row: { ITIS_TSN: 1 }, header: 'HEADER', rowIndex: 0 }); + const result = cellSetter({ + cell: 'unit', + row: { ITIS_TSN: 1 }, + header: 'HEADER', + rowIndex: 0, + mutateCell: 'unit' + }); expect(result).to.be.equal('uuid'); }); @@ -200,7 +259,13 @@ describe('critter-header-configs', () => { const cellValues = ['male', 'MALE', undefined]; for (const cell of cellValues) { - const result = cellValidator({ cell: cell, row: { ITIS_TSN: 1 }, header: 'HEADER', rowIndex: 0 }); + const result = cellValidator({ + cell: cell, + row: { ITIS_TSN: 1 }, + header: 'HEADER', + rowIndex: 0, + mutateCell: cell + }); expect(result).to.be.deep.equal([]); } @@ -219,7 +284,13 @@ describe('critter-header-configs', () => { const cellValues = ['', 0]; for (const cell of cellValues) { - const result = cellValidator({ cell: cell, row: { ITIS_TSN: 1 }, header: 'HEADER', rowIndex: 0 }); + const result = cellValidator({ + cell: cell, + row: { ITIS_TSN: 1 }, + header: 'HEADER', + rowIndex: 0, + mutateCell: cell + }); expect(result.length).to.be.equal(1); } @@ -235,7 +306,13 @@ describe('critter-header-configs', () => { new CSVConfigUtils(xlsx.utils.json_to_sheet([]), mockConfig) ); - const result = cellValidator({ cell: 'male', row: { ITIS_TSN: 2 }, header: 'HEADER', rowIndex: 0 }); + const result = cellValidator({ + cell: 'male', + row: { ITIS_TSN: 2 }, + header: 'HEADER', + rowIndex: 0, + mutateCell: 'male' + }); expect(result[0].error).to.be.equal('Sex is not a supported attribute for TSN: 2'); }); @@ -250,7 +327,13 @@ describe('critter-header-configs', () => { new CSVConfigUtils(xlsx.utils.json_to_sheet([]), mockConfig) ); - const result = cellValidator({ cell: 'maled', row: { ITIS_TSN: 1 }, header: 'HEADER', rowIndex: 0 }); + const result = cellValidator({ + cell: 'maled', + row: { ITIS_TSN: 1 }, + header: 'HEADER', + rowIndex: 0, + mutateCell: 'maled' + }); expect(result[0].error).to.be.equal('Sex cell value is invalid'); }); @@ -268,7 +351,13 @@ describe('critter-header-configs', () => { new CSVConfigUtils(xlsx.utils.json_to_sheet([]), mockConfig) ); - const result = cellSetter({ cell: 'MALE', row: { ITIS_TSN: 1 }, header: 'HEADER', rowIndex: 0 }); + const result = cellSetter({ + cell: 'MALE', + row: { ITIS_TSN: 1 }, + header: 'HEADER', + rowIndex: 0, + mutateCell: 'MALE' + }); expect(result).to.be.equal('uuid'); }); @@ -278,7 +367,13 @@ describe('critter-header-configs', () => { it('should return an empty array if the cell is valid', () => { const wlhIDValidator = getWlhIDCellValidator(new CSVConfigUtils(xlsx.utils.json_to_sheet([]), mockConfig)); - const result = wlhIDValidator({ cell: '10-01111', row: {}, header: 'HEADER', rowIndex: 0 }); + const result = wlhIDValidator({ + cell: '10-01111', + row: {}, + header: 'HEADER', + rowIndex: 0, + mutateCell: '10-01111' + }); expect(result).to.be.deep.equal([]); }); @@ -286,7 +381,7 @@ describe('critter-header-configs', () => { it('should return no errors when cell is undefined', () => { const wlhIDValidator = getWlhIDCellValidator(new CSVConfigUtils(xlsx.utils.json_to_sheet([]), mockConfig)); - const result = wlhIDValidator({ cell: undefined, row: {}, header: 'HEADER', rowIndex: 0 }); + const result = wlhIDValidator({ cell: undefined, row: {}, header: 'HEADER', rowIndex: 0, mutateCell: undefined }); expect(result).to.be.deep.equal([]); }); @@ -297,7 +392,7 @@ describe('critter-header-configs', () => { const badWlhIds = ['100111', '1-011111', '100-222', '21-']; badWlhIds.forEach((badWlhId) => { - const result = wlhIDValidator({ cell: badWlhId, row: {}, header: 'HEADER', rowIndex: 0 }); + const result = wlhIDValidator({ cell: badWlhId, row: {}, header: 'HEADER', rowIndex: 0, mutateCell: badWlhId }); expect(result).to.be.deep.equal([ { diff --git a/api/src/services/import-services/critter/import-critters-service.test.ts b/api/src/services/import-services/critter/import-critters-service.test.ts index 3a4015ba93..6bd614c9eb 100644 --- a/api/src/services/import-services/critter/import-critters-service.test.ts +++ b/api/src/services/import-services/critter/import-critters-service.test.ts @@ -183,7 +183,7 @@ describe('ImportCrittersService', () => { const sexHeaderConfig = await service._getSexHeaderConfig(); - expect(getTaxonMeasurementsStub).to.have.been.calledWithExactly(1234); + expect(getTaxonMeasurementsStub).to.have.been.calledWithExactly('1234'); expect(getSexCellValidatorStub).to.have.been.calledWithExactly( new NestedRecord({ 1234: { male: 'maleUUID', female: 'femaleUUID' } @@ -224,7 +224,7 @@ describe('ImportCrittersService', () => { const config = await service._getCollectionUnitDynamicHeaderConfig(); - expect(findTaxonCollectionUnitsStub).to.have.been.calledOnceWithExactly(1234); + expect(findTaxonCollectionUnitsStub).to.have.been.calledOnceWithExactly('1234'); expect(getCollectionUnitCellValidatorStub).to.have.been.calledWithExactly( new NestedRecord({ 1234: { category: { unit: 'uuid' } } }), diff --git a/api/src/services/import-services/critter/import-critters-service.ts b/api/src/services/import-services/critter/import-critters-service.ts index c714f56530..aa889b01b5 100644 --- a/api/src/services/import-services/critter/import-critters-service.ts +++ b/api/src/services/import-services/critter/import-critters-service.ts @@ -5,7 +5,12 @@ import { IDBConnection } from '../../../database/db'; import { ApiGeneralError } from '../../../errors/api-error'; import { CSVConfigUtils } from '../../../utils/csv-utils/csv-config-utils'; import { validateCSVWorksheet } from '../../../utils/csv-utils/csv-config-validation'; -import { CSVConfig, CSVHeaderConfig, CSVRowValidated } from '../../../utils/csv-utils/csv-config-validation.interface'; +import { + CSVConfig, + CSVError, + CSVHeaderConfig, + CSVRowValidated +} from '../../../utils/csv-utils/csv-config-validation.interface'; import { getDescriptionCellValidator, getTsnCellValidator } from '../../../utils/csv-utils/csv-header-configs'; import { getLogger } from '../../../utils/logger'; import { NestedRecord } from '../../../utils/nested-record'; @@ -85,15 +90,15 @@ export class ImportCrittersService extends DBService { * * @async * @throws {ApiGeneralError} - If unable to fully insert records into Critterbase - * @returns {*} {Promise} List of inserted survey critter ids + * @returns {*} {Promise} List of CSV errors encountered during import */ - async importCSVWorksheet(): Promise { + async importCSVWorksheet(): Promise { const config = await this.getCSVConfig(); const { errors, rows } = validateCSVWorksheet(this.worksheet, config); if (errors.length) { - throw new ApiGeneralError('Failed to validate CSV', errors); + return errors; } const payloads = this._getImportPayloads(rows); @@ -112,6 +117,8 @@ export class ImportCrittersService extends DBService { // Add Critters to SIMS survey await this.surveyCritterService.addCrittersToSurvey(this.surveyId, payloads.simsPayload); + + return []; } /** @@ -201,7 +208,7 @@ export class ImportCrittersService extends DBService { * @returns {*} {Promise} The TSN header config */ async _getTsnHeaderConfig(): Promise { - const rowTsns = this.configUtils.getUniqueCellValues('ITIS_TSN'); + const rowTsns = this.configUtils.getUniqueCellValues('ITIS_TSN').map((tsn) => String(tsn)); const taxonomy = await this.platformService.getTaxonomyByTsns(rowTsns); const allowedTsns = new Set(taxonomy.map((taxon) => taxon.tsn)); @@ -242,7 +249,7 @@ export class ImportCrittersService extends DBService { async _getSexHeaderConfig(): Promise { const rowDictionary = new NestedRecord(); - const rowTsns = this.configUtils.getUniqueCellValues('ITIS_TSN'); + const rowTsns = this.configUtils.getUniqueCellValues('ITIS_TSN').map((tsn) => String(tsn)); const measurements = await Promise.all(rowTsns.map((tsn) => this.critterbaseService.getTaxonMeasurements(tsn))); measurements.forEach((measurement, index) => { @@ -273,7 +280,7 @@ export class ImportCrittersService extends DBService { */ async _getCollectionUnitDynamicHeaderConfig(): Promise { const rowDictionary = new NestedRecord(); - const rowTsns = this.configUtils.getUniqueCellValues('ITIS_TSN'); + const rowTsns = this.configUtils.getUniqueCellValues('ITIS_TSN').map((tsn) => String(tsn)); // Get the collection units for all the tsns in the worksheet const collectionUnits = await Promise.all( rowTsns.map((tsn) => this.critterbaseService.findTaxonCollectionUnits(tsn)) diff --git a/api/src/services/import-services/marking/import-markings-service.test.ts b/api/src/services/import-services/marking/import-markings-service.test.ts new file mode 100644 index 0000000000..43a75370f8 --- /dev/null +++ b/api/src/services/import-services/marking/import-markings-service.test.ts @@ -0,0 +1,201 @@ +import chai, { expect } from 'chai'; +import sinon from 'sinon'; +import sinonChai from 'sinon-chai'; +import { WorkSheet } from 'xlsx'; +import * as csv from '../../../utils/csv-utils/csv-config-validation'; +import { CSVConfig } from '../../../utils/csv-utils/csv-config-validation.interface'; +import { NestedRecord } from '../../../utils/nested-record'; +import { getMockDBConnection } from '../../../__mocks__/db'; +import { IAsSelectLookup } from '../../critterbase-service'; +import { ImportMarkingsService } from './import-markings-service'; + +chai.use(sinonChai); + +describe('import-markings-service', () => { + beforeEach(() => { + sinon.restore(); + }); + + describe('constructor', () => { + it('should create an instance of ImportMarkingsService', () => { + const mockConnection = getMockDBConnection(); + const worksheet = {} as WorkSheet; + const surveyId = 1; + + const service = new ImportMarkingsService(mockConnection, worksheet, surveyId); + + expect(service).to.be.instanceof(ImportMarkingsService); + }); + }); + + describe('importCSVWorksheet', () => { + it('should import the CSV worksheet', async () => { + const mockConnection = getMockDBConnection(); + const worksheet = {} as WorkSheet; + const surveyId = 1; + + const service = new ImportMarkingsService(mockConnection, worksheet, surveyId); + + const mockCSVConfig = {} as CSVConfig; + const mockGetConfig = sinon.stub(service, 'getCSVConfig').resolves(mockCSVConfig); + const bulkCreateStub = sinon.stub(service.surveyCritterService.critterbaseService, 'bulkCreate').resolves(); + + const mockValidate = sinon.stub(csv, 'validateCSVWorksheet').returns({ + errors: [], + rows: [ + { + ALIAS: 'uuid', + CAPTURE_DATE: 'uuid2', + BODY_LOCATION: 'ear', + MARKING_TYPE: 'tag', + IDENTIFIER: 'id', + PRIMARY_COLOUR: 'red', + SECONDARY_COLOUR: 'blue', + DESCRIPTION: 'comments' + } + ] + }); + + await service.importCSVWorksheet(); + + expect(mockGetConfig).to.have.been.called; + expect(mockValidate).to.have.been.calledOnceWithExactly(worksheet, mockCSVConfig); + expect(bulkCreateStub).to.have.been.calledOnceWithExactly({ + markings: [ + { + critter_id: 'uuid', + capture_id: 'uuid2', + body_location: 'ear', + marking_type: 'tag', + identifier: 'id', + primary_colour: 'red', + secondary_colour: 'blue', + comment: 'comments' + } + ] + }); + }); + + it('should return CSV Validation error if rows fail validation', async () => { + const mockConnection = getMockDBConnection(); + const worksheet = {} as WorkSheet; + const surveyId = 1; + + const service = new ImportMarkingsService(mockConnection, worksheet, surveyId); + + const mockCSVConfig = {} as CSVConfig; + const mockGetConfig = sinon.stub(service, 'getCSVConfig').resolves(mockCSVConfig); + + const mockValidate = sinon.stub(csv, 'validateCSVWorksheet').returns({ + errors: [{ error: 'error', solution: 'solution', values: [] }], + rows: [] + }); + + const errors = await service.importCSVWorksheet(); + + expect(mockGetConfig).to.have.been.called; + expect(mockValidate).to.have.been.calledOnceWithExactly(worksheet, mockCSVConfig); + expect(errors).to.deep.equal([{ error: 'error', solution: 'solution', values: [] }]); + }); + }); + + describe('getCSVConfig', () => { + it('should return a CSVConfig object', async () => { + const mockConnection = getMockDBConnection(); + const worksheet = {} as WorkSheet; + const surveyId = 1; + + const service = new ImportMarkingsService(mockConnection, worksheet, surveyId); + + const mockAliasMap = new Map(); + const mockDictionary = new NestedRecord(); + const mockMarkingTypes = [{ value: 'type1' }, { value: 'type2' }] as IAsSelectLookup[]; + const mockColours = [{ value: 'colour1' }, { value: 'colour2' }] as IAsSelectLookup[]; + + const surveyAliasMapStub = sinon + .stub(service.surveyCritterService, 'getSurveyCritterAliasMap') + .resolves(mockAliasMap); + + const bodyLocationDictionaryStub = sinon.stub(service, '_getBodyLocationDictionary').resolves(mockDictionary); + + const markingTypesStub = sinon + .stub(service.surveyCritterService.critterbaseService, 'getMarkingTypes') + .resolves(mockMarkingTypes); + + const coloursStub = sinon + .stub(service.surveyCritterService.critterbaseService, 'getColours') + .resolves(mockColours); + + expect(surveyAliasMapStub).to.not.have.been.calledOnceWithExactly(surveyId); + expect(bodyLocationDictionaryStub).to.not.have.been.calledOnceWithExactly(mockAliasMap); + expect(markingTypesStub).to.not.have.been.calledOnceWithExactly(); + expect(coloursStub).to.not.have.been.calledOnceWithExactly(); + + const config = await service.getCSVConfig(); + + expect(config.dynamicHeadersConfig).to.be.equal(undefined); + expect(config.staticHeadersConfig).to.have.keys([ + 'ALIAS', + 'CAPTURE_DATE', + 'CAPTURE_TIME', + 'BODY_LOCATION', + 'MARKING_TYPE', + 'IDENTIFIER', + 'PRIMARY_COLOUR', + 'SECONDARY_COLOUR', + 'DESCRIPTION' + ]); + }); + }); + + describe('_getBodyLocationDictionary', () => { + it('should return a dictionary of critter tsn -> body location -> body_location_id', async () => { + const mockConnection = getMockDBConnection(); + + const service = new ImportMarkingsService(mockConnection, {}, 1); + + const getCellValues = sinon.stub(service.utils, 'getUniqueCellValues'); + const getTaxonBodyLocationsStub = sinon.stub( + service.surveyCritterService.critterbaseService, + 'getTaxonBodyLocations' + ); + + getCellValues.returns(['steve', 'brule']); + + getTaxonBodyLocationsStub.onCall(0).resolves([ + { + id: 'A', + value: 'ear', + key: 'key' + } + ]); + + getTaxonBodyLocationsStub.onCall(1).resolves([ + { + id: 'B', + value: 'tail', + key: 'key' + } + ]); + + const dictionary = await service._getBodyLocationDictionary( + new Map([ + ['steve', { itis_tsn: 2 }], + ['brule', { itis_tsn: 3 }] + ] as any) + ); + + expect(dictionary).to.be.instanceof(NestedRecord); + expect(dictionary).to.deep.equal( + new NestedRecord({ + 2: { + ear: 'A' + }, + 3: { + tail: 'B' + } + }) + ); + }); + }); +}); diff --git a/api/src/services/import-services/marking/import-markings-service.ts b/api/src/services/import-services/marking/import-markings-service.ts new file mode 100644 index 0000000000..fb7b4c716c --- /dev/null +++ b/api/src/services/import-services/marking/import-markings-service.ts @@ -0,0 +1,201 @@ +import { WorkSheet } from 'xlsx'; +import { IDBConnection } from '../../../database/db'; +import { CSVConfigUtils } from '../../../utils/csv-utils/csv-config-utils'; +import { validateCSVWorksheet } from '../../../utils/csv-utils/csv-config-validation'; +import { CSVConfig, CSVError } from '../../../utils/csv-utils/csv-config-validation.interface'; +import { + getDescriptionCellValidator, + getTimeCellSetter, + getTimeCellValidator +} from '../../../utils/csv-utils/csv-header-configs'; +import { getLogger } from '../../../utils/logger'; +import { NestedRecord } from '../../../utils/nested-record'; +import { ICritterDetailed } from '../../critterbase-service'; +import { DBService } from '../../db-service'; +import { SurveyCritterService } from '../../survey-critter-service'; +import { + getMarkingAliasCellValidator, + getMarkingBodyLocationCellValidator, + getMarkingCaptureDateCellValidator, + getMarkingColourCellValidator, + getMarkingIdentifierCellValidator, + getMarkingTypeCellValidator +} from './marking-header-configs'; + +const defaultLog = getLogger('services/import/import-markings-service'); + +// Marking CSV static headers +export type MarkingCSVStaticHeader = + | 'ALIAS' + | 'CAPTURE_DATE' + | 'CAPTURE_TIME' + | 'BODY_LOCATION' + | 'MARKING_TYPE' + | 'IDENTIFIER' + | 'PRIMARY_COLOUR' + | 'SECONDARY_COLOUR' + | 'DESCRIPTION'; + +/** + * ImportMarkingsService - A service for importing Markings from a CSV into Critterbase. + * + * @class ImportMarkingsService + * @extends DBService + */ +export class ImportMarkingsService extends DBService { + worksheet: WorkSheet; + surveyId: number; + + surveyCritterService: SurveyCritterService; + utils: CSVConfigUtils; + + /** + * Construct an instance of ImportMarkingsService. + * + * @param {IDBConnection} connection - DB connection + * @param {string} surveyId + */ + constructor(connection: IDBConnection, worksheet: WorkSheet, surveyId: number) { + super(connection); + + const initialConfig: CSVConfig = { + staticHeadersConfig: { + ALIAS: { aliases: ['NICKNAME', 'ANIMAL'] }, + CAPTURE_DATE: { aliases: ['CAPTURE DATE', 'DATE'] }, + CAPTURE_TIME: { aliases: ['CAPTURE TIME', 'TIME'], optional: true }, + BODY_LOCATION: { aliases: ['BODY LOCATION'] }, + MARKING_TYPE: { aliases: ['MARKING TYPE', 'TYPE'], optional: true }, + IDENTIFIER: { aliases: ['ID'], optional: true }, + PRIMARY_COLOUR: { aliases: ['PRIMARY COLOUR'], optional: true }, + SECONDARY_COLOUR: { aliases: ['SECONDARY COLOUR'], optional: true }, + DESCRIPTION: { aliases: ['COMMENT', 'COMMENTS', 'NOTES'], optional: true } + }, + ignoreDynamicHeaders: false + }; + + this.worksheet = worksheet; + this.surveyId = surveyId; + + this.surveyCritterService = new SurveyCritterService(connection); + this.utils = new CSVConfigUtils(this.worksheet, initialConfig); + } + + /** + * Import a Marking CSV worksheet into Critterbase. + * + * @async + * @throws {ApiGeneralError} - If unable to fully insert records into Critterbase + * @returns {*} {Promise} List of CSV errors encountered during import + */ + async importCSVWorksheet(): Promise { + const config = await this.getCSVConfig(); + + const { errors, rows } = validateCSVWorksheet(this.worksheet, config); + + if (errors.length) { + return errors; + } + + const markings = rows.map((row) => ({ + critter_id: row.ALIAS, // ALIAS set to Critterbase critter_id + capture_id: row.CAPTURE_DATE, // CAPTURE_DATE set to Critterbase capture_id + body_location: row.BODY_LOCATION, // BODY_LOCATION set to Critterbase body_location_id + marking_type: row.MARKING_TYPE, + identifier: row.IDENTIFIER, + primary_colour: row.PRIMARY_COLOUR, + secondary_colour: row.SECONDARY_COLOUR, + comment: row.DESCRIPTION + })); + + defaultLog.debug({ label: 'import markings', markings }); + + await this.surveyCritterService.critterbaseService.bulkCreate({ markings }); + + return []; + } + + /** + * Get the CSV configuration for Markings. + * + * @returns {Promise>} The CSV configuration + */ + async getCSVConfig(): Promise> { + const surveyAliasMap = await this.surveyCritterService.getSurveyCritterAliasMap(this.surveyId); + const bodyLocationDictionary = await this._getBodyLocationDictionary(surveyAliasMap); + + const markingTypes = new Set( + (await this.surveyCritterService.critterbaseService.getMarkingTypes()).map((type) => type.value) + ); + + const colours = new Set( + (await this.surveyCritterService.critterbaseService.getColours()).map((colour) => colour.value) + ); + + this.utils.setStaticHeaderConfig('ALIAS', { + validateCell: getMarkingAliasCellValidator(surveyAliasMap) + }); + this.utils.setStaticHeaderConfig('CAPTURE_DATE', { + validateCell: getMarkingCaptureDateCellValidator(surveyAliasMap, this.utils) + }); + this.utils.setStaticHeaderConfig('CAPTURE_TIME', { + validateCell: getTimeCellValidator(), + setCellValue: getTimeCellSetter() + }); + this.utils.setStaticHeaderConfig('BODY_LOCATION', { + validateCell: getMarkingBodyLocationCellValidator(surveyAliasMap, bodyLocationDictionary, this.utils) + }); + this.utils.setStaticHeaderConfig('MARKING_TYPE', { + validateCell: getMarkingTypeCellValidator(markingTypes) + }); + this.utils.setStaticHeaderConfig('IDENTIFIER', { + validateCell: getMarkingIdentifierCellValidator() + }); + this.utils.setStaticHeaderConfig('PRIMARY_COLOUR', { + validateCell: getMarkingColourCellValidator(colours) + }); + this.utils.setStaticHeaderConfig('SECONDARY_COLOUR', { + validateCell: getMarkingColourCellValidator(colours) + }); + this.utils.setStaticHeaderConfig('DESCRIPTION', { + validateCell: getDescriptionCellValidator() + }); + + // Return the final CSV config + return this.utils.getConfig(); + } + + /** + * Get a dictionary of critter tsn -> body location -> body_location_id. + * + * @param {Map} surveyAliasMap - The survey alias map + * @returns {Promise>} The body location dictionary + */ + async _getBodyLocationDictionary(surveyAliasMap: Map): Promise> { + const dictionary = new NestedRecord(); + + const rowAliases = this.utils.getUniqueCellValues('ALIAS').map((alias) => String(alias).toLowerCase()); + const allTsns = rowAliases.map((alias) => surveyAliasMap.get(alias)?.itis_tsn).filter(Boolean) as number[]; + const uniqueTsns = [...new Set(allTsns)]; + + // Get body locations for each unique TSN (in parallel) + const taxonBodyLocationArrays = await Promise.all( + uniqueTsns.map((tsn) => this.surveyCritterService.critterbaseService.getTaxonBodyLocations(String(tsn))) + ); + + // Loop through each TSN and set the dictionary: tsn -> body location -> id + for (let index = 0; index < uniqueTsns.length; index++) { + const tsn = uniqueTsns[index]; + const bodyLocations = taxonBodyLocationArrays[index]; + + // set body location dictionary + for (const bodyLocation of bodyLocations) { + dictionary.set({ + path: [tsn, bodyLocation.value], + value: bodyLocation.id + }); + } + } + + return dictionary; + } +} diff --git a/api/src/services/import-services/marking/import-markings-strategy.interface.ts b/api/src/services/import-services/marking/import-markings-strategy.interface.ts deleted file mode 100644 index 0917f011b9..0000000000 --- a/api/src/services/import-services/marking/import-markings-strategy.interface.ts +++ /dev/null @@ -1,79 +0,0 @@ -import { z } from 'zod'; -import { IAsSelectLookup } from '../../critterbase-service'; - -/** - * Get CSV Marking schema. - * - * Note: This getter allows custom values to be injected for validation. - * - * Note: This could be updated to transform the string values into the primary keys - * to prevent Critterbase from having to translate / patch in incomming bulk values. - * - * @param {IAsSelectLookup[]} colours - Array of supported Critterbase colours - * @returns {*} Custom Zod schema for CSV Markings - */ -export const getCsvMarkingSchema = ( - colours: IAsSelectLookup[], - markingTypes: IAsSelectLookup[], - critterBodyLocationsMap: Map -) => { - const colourNames = colours.map((colour) => colour.value.toLowerCase()); - const markingTypeNames = markingTypes.map((markingType) => markingType.value.toLowerCase()); - - const coloursSet = new Set(colourNames); - const markingTypesSet = new Set(markingTypeNames); - - return z - .object({ - critter_id: z.string({ required_error: 'Unable to find matching survey critter with alias' }).uuid(), - capture_id: z.string({ required_error: 'Unable to find matching capture with date and time' }).uuid(), - body_location: z.string(), - marking_type: z - .string() - .refine( - (val) => markingTypesSet.has(val.toLowerCase()), - `Marking type not supported. Allowed values: ${markingTypeNames.join(', ')}` - ) - .optional(), - identifier: z.string().optional(), - primary_colour: z - .string() - .refine( - (val) => coloursSet.has(val.toLowerCase()), - `Colour not supported. Allowed values: ${colourNames.join(', ')}` - ) - .optional(), - secondary_colour: z - .string() - .refine( - (val) => coloursSet.has(val.toLowerCase()), - `Colour not supported. Allowed values: ${colourNames.join(', ')}` - ) - .optional(), - comment: z.string().optional() - }) - .superRefine((schema, ctx) => { - const bodyLocations = critterBodyLocationsMap.get(schema.critter_id); - if (!bodyLocations) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - message: 'No taxon body locations found for Critter' - }); - } else if ( - !bodyLocations.filter((location) => location.value.toLowerCase() === schema.body_location.toLowerCase()).length - ) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - message: `Invalid body location for Critter. Allowed values: ${bodyLocations - .map((bodyLocation) => bodyLocation.value) - .join(', ')}` - }); - } - }); -}; - -/** - * A validated CSV Marking object - * - */ -export type CsvMarking = z.infer>; diff --git a/api/src/services/import-services/marking/import-markings-strategy.test.ts b/api/src/services/import-services/marking/import-markings-strategy.test.ts deleted file mode 100644 index 074cc472f2..0000000000 --- a/api/src/services/import-services/marking/import-markings-strategy.test.ts +++ /dev/null @@ -1,249 +0,0 @@ -import { expect } from 'chai'; -import sinon from 'sinon'; -import { MediaFile } from '../../../utils/media/media-file'; -import * as worksheetUtils from '../../../utils/xlsx-utils/worksheet-utils'; -import { getMockDBConnection } from '../../../__mocks__/db'; -import { IBulkCreateResponse, ICritterDetailed } from '../../critterbase-service'; -import { importCSV } from '../import-csv'; -import { ImportMarkingsStrategy } from './import-markings-strategy'; -import { CsvMarking } from './import-markings-strategy.interface'; - -describe('ImportMarkingsStrategy', () => { - describe('importCSV marking worksheet', () => { - beforeEach(() => { - sinon.restore(); - }); - it('should validate successfully', async () => { - const worksheet = { - A1: { t: 's', v: 'CAPTURE_DATE' }, // testing order incorrect - B1: { t: 's', v: 'ALIAS' }, - C1: { t: 's', v: 'CAPTURE_TIME' }, - D1: { t: 's', v: 'BODY_LOCATION' }, - E1: { t: 's', v: 'MARKING_TYPE' }, - F1: { t: 's', v: 'IDENTIFIER' }, - G1: { t: 's', v: 'PRIMARY_COLOUR' }, - H1: { t: 's', v: 'SECONDARY_COLOUR' }, - I1: { t: 's', v: 'DESCRIPTION' }, // testing alias works - A2: { z: 'yyyy-mm-dd', t: 'd', v: new Date('2024-10-10T07:00:00.000Z'), w: '2024-10-10' }, - B2: { t: 's', v: 'Carl' }, - C2: { t: 's', v: '10:10:12' }, - D2: { t: 's', v: 'Left ear' }, // testing case insensitivity - E2: { t: 's', v: 'Ear tag' }, - F2: { t: 's', v: 'asdfasdf' }, - G2: { t: 's', v: 'red' }, - H2: { t: 's', v: 'blue' }, - I2: { t: 's', v: 'tagged' }, - '!ref': 'A1:I2' - }; - - const mockDBConnection = getMockDBConnection(); - - const strategy = new ImportMarkingsStrategy(mockDBConnection, 1); - - const getDefaultWorksheetStub = sinon.stub(worksheetUtils, 'getDefaultWorksheet'); - const critterbaseInsertStub = sinon.stub(strategy.surveyCritterService.critterbaseService, 'bulkCreate'); - const aliasMapStub = sinon.stub(strategy.surveyCritterService, 'getSurveyCritterAliasMap'); - const colourStub = sinon.stub(strategy.surveyCritterService.critterbaseService, 'getColours'); - const markingTypeStub = sinon.stub(strategy.surveyCritterService.critterbaseService, 'getMarkingTypes'); - const taxonBodyLocationStub = sinon.stub(strategy, 'getTaxonBodyLocationsCritterIdMap'); - - colourStub.resolves([ - { id: 'A', key: 'colour', value: 'red' }, - { id: 'B', key: 'colour', value: 'blue' } - ]); - - markingTypeStub.resolves([ - { id: 'C', key: 'markingType', value: 'ear tag' }, - { id: 'D', key: 'markingType', value: 'nose band' } - ]); - - taxonBodyLocationStub.resolves( - new Map([ - ['3647cdc9-6fe9-4c32-acfa-6096fe123c4a', [{ id: 'D', key: 'bodylocation', value: 'left ear' }]], - ['4540d43a-7ced-4216-b49e-2a972d25dfdc', [{ id: 'E', key: 'bodylocation', value: 'tail' }]] - ]) - ); - - getDefaultWorksheetStub.returns(worksheet); - aliasMapStub.resolves( - new Map([ - [ - 'carl', - { - critter_id: '3647cdc9-6fe9-4c32-acfa-6096fe123c4a', - captures: [ - { - capture_id: '4647cdc9-6fe9-4c32-acfa-6096fe123c4a', - capture_date: '2024-10-10', - capture_time: '10:10:12' - } - ] - } as ICritterDetailed - ], - [ - 'carlita', - { - critter_id: '3647cdc9-6fe9-4c32-acfa-6096fe123c4a', - captures: [ - { - capture_id: '5647cdc9-6fe9-4c32-acfa-6096fe123c4a', - capture_date: '2024-10-10', - capture_time: '10:10:10' - } - ] - } as ICritterDetailed - ] - ]) - ); - critterbaseInsertStub.resolves({ created: { markings: 2 } } as IBulkCreateResponse); - - try { - const data = await importCSV(new MediaFile('test', 'test', 'test' as unknown as Buffer), strategy); - expect(data).to.deep.equal(2); - } catch (error: any) { - expect.fail(error); - } - }); - }); - describe('getTaxonBodyLocationsCritterIdMap', () => { - it('should return a critter_id mapping of body locations', async () => { - const mockDBConnection = getMockDBConnection(); - const strategy = new ImportMarkingsStrategy(mockDBConnection, 1); - - const taxonBodyLocationsStub = sinon.stub( - strategy.surveyCritterService.critterbaseService, - 'getTaxonBodyLocations' - ); - const mockBodyLocationsA = [ - { id: 'A', key: 'column', value: 'Right Ear' }, - { id: 'B', key: 'column', value: 'Antlers' } - ]; - - const mockBodyLocationsB = [ - { id: 'C', key: 'column', value: 'Nose' }, - { id: 'D', key: 'column', value: 'Tail' } - ]; - - taxonBodyLocationsStub.onCall(0).resolves(mockBodyLocationsA); - taxonBodyLocationsStub.onCall(1).resolves(mockBodyLocationsB); - - const critterMap = await strategy.getTaxonBodyLocationsCritterIdMap([ - { critter_id: 'ACRITTER', itis_tsn: 1 }, - { critter_id: 'BCRITTER', itis_tsn: 2 }, - { critter_id: 'CCRITTER', itis_tsn: 2 } - ] as ICritterDetailed[]); - - expect(taxonBodyLocationsStub).to.have.been.calledTwice; - expect(taxonBodyLocationsStub.getCall(0).args[0]).to.be.eql('1'); - expect(taxonBodyLocationsStub.getCall(1).args[0]).to.be.eql('2'); - expect(critterMap).to.be.deep.equal( - new Map([ - ['ACRITTER', mockBodyLocationsA], - ['BCRITTER', mockBodyLocationsB], - ['CCRITTER', mockBodyLocationsB] - ]) - ); - }); - }); - - describe('validateRows', () => { - it('should validate the rows successfully', async () => { - const mockDBConnection = getMockDBConnection(); - const strategy = new ImportMarkingsStrategy(mockDBConnection, 1); - - const mockCritterA = { - critter_id: '4df8fd4c-4d7b-4142-8f03-92d8bf52d8cb', - itis_tsn: 1, - captures: [ - { capture_id: 'e9087545-5b1f-4b86-bf1d-a3372a7b33c7', capture_date: '10-10-2024', capture_time: '10:10:10' } - ] - } as ICritterDetailed; - - const mockCritterB = { - critter_id: '4540d43a-7ced-4216-b49e-2a972d25dfdc', - itis_tsn: 1, - captures: [ - { capture_id: '21f3c699-9017-455b-bd7d-49110ca4b586', capture_date: '10-10-2024', capture_time: '10:10:10' } - ] - } as ICritterDetailed; - - const aliasStub = sinon.stub(strategy.surveyCritterService, 'getSurveyCritterAliasMap'); - const colourStub = sinon.stub(strategy.surveyCritterService.critterbaseService, 'getColours'); - const markingTypeStub = sinon.stub(strategy.surveyCritterService.critterbaseService, 'getMarkingTypes'); - const taxonBodyLocationStub = sinon.stub(strategy, 'getTaxonBodyLocationsCritterIdMap'); - - aliasStub.resolves( - new Map([ - ['carl', mockCritterA], - ['carlita', mockCritterB] - ]) - ); - - colourStub.resolves([ - { id: 'A', key: 'colour', value: 'red' }, - { id: 'B', key: 'colour', value: 'blue' } - ]); - - markingTypeStub.resolves([ - { id: 'C', key: 'markingType', value: 'ear tag' }, - { id: 'D', key: 'markingType', value: 'nose band' } - ]); - - taxonBodyLocationStub.resolves( - new Map([ - ['4df8fd4c-4d7b-4142-8f03-92d8bf52d8cb', [{ id: 'D', key: 'bodylocation', value: 'ear' }]], - ['4540d43a-7ced-4216-b49e-2a972d25dfdc', [{ id: 'E', key: 'bodylocation', value: 'tail' }]] - ]) - ); - - const rows = [ - { - CAPTURE_DATE: '10-10-2024', - CAPTURE_TIME: '10:10:10', - ALIAS: 'carl', - BODY_LOCATION: 'Ear', - MARKING_TYPE: 'ear tag', - IDENTIFIER: 'identifier', - PRIMARY_COLOUR: 'Red', - SECONDARY_COLOUR: 'blue', - DESCRIPTION: 'comment' - } - ]; - - const validation = await strategy.validateRows(rows); - - if (!validation.success) { - expect.fail(); - } else { - expect(validation.success).to.be.true; - expect(validation.data).to.be.deep.equal([ - { - critter_id: '4df8fd4c-4d7b-4142-8f03-92d8bf52d8cb', - capture_id: 'e9087545-5b1f-4b86-bf1d-a3372a7b33c7', - body_location: 'Ear', - marking_type: 'ear tag', - identifier: 'identifier', - primary_colour: 'Red', - secondary_colour: 'blue', - comment: 'comment' - } - ]); - } - }); - }); - describe('insert', () => { - it('should return the count of inserted markings', async () => { - const mockDBConnection = getMockDBConnection(); - const strategy = new ImportMarkingsStrategy(mockDBConnection, 1); - - const bulkCreateStub = sinon.stub(strategy.surveyCritterService.critterbaseService, 'bulkCreate'); - - bulkCreateStub.resolves({ created: { markings: 1 } } as IBulkCreateResponse); - - const data = await strategy.insert([{ critter_id: 'id' } as unknown as CsvMarking]); - - expect(bulkCreateStub).to.have.been.calledWith({ markings: [{ critter_id: 'id' }] }); - expect(data).to.be.eql(1); - }); - }); -}); diff --git a/api/src/services/import-services/marking/import-markings-strategy.ts b/api/src/services/import-services/marking/import-markings-strategy.ts deleted file mode 100644 index 97dbf52237..0000000000 --- a/api/src/services/import-services/marking/import-markings-strategy.ts +++ /dev/null @@ -1,170 +0,0 @@ -import { z } from 'zod'; -import { IDBConnection } from '../../../database/db'; -import { getLogger } from '../../../utils/logger'; -import { CSV_COLUMN_ALIASES } from '../../../utils/xlsx-utils/column-aliases'; -import { generateColumnCellGetterFromColumnValidator } from '../../../utils/xlsx-utils/column-validator-utils'; -import { IXLSXCSVValidator } from '../../../utils/xlsx-utils/worksheet-utils'; -import { IAsSelectLookup, ICritterDetailed } from '../../critterbase-service'; -import { DBService } from '../../db-service'; -import { SurveyCritterService } from '../../survey-critter-service'; -import { CSVImportStrategy, Row } from '../import-csv.interface'; -import { findCapturesFromDateTime } from '../utils/datetime'; -import { CsvMarking, getCsvMarkingSchema } from './import-markings-strategy.interface'; - -const defaultLog = getLogger('services/import/import-markings-strategy'); - -/** - * - * @class ImportMarkingsStrategy - * @extends DBService - * @see CSVImport - * - */ -export class ImportMarkingsStrategy extends DBService implements CSVImportStrategy { - surveyCritterService: SurveyCritterService; - surveyId: number; - - /** - * An XLSX validation config for the standard columns of a Critterbase Marking CSV. - * - * Note: `satisfies` allows `keyof` to correctly infer keyof type, while also - * enforcing uppercase object keys. - */ - columnValidator = { - ALIAS: { type: 'string', aliases: CSV_COLUMN_ALIASES.ALIAS }, - CAPTURE_DATE: { type: 'date' }, - CAPTURE_TIME: { type: 'string', optional: true }, - BODY_LOCATION: { type: 'string', optional: true }, - MARKING_TYPE: { type: 'string', optional: true }, - IDENTIFIER: { type: 'string', optional: true }, - PRIMARY_COLOUR: { type: 'string', optional: true }, - SECONDARY_COLOUR: { type: 'string', optional: true }, - DESCRIPTION: { type: 'string', aliases: CSV_COLUMN_ALIASES.DESCRIPTION, optional: true } - } satisfies IXLSXCSVValidator; - - /** - * Construct an instance of ImportMarkingsStrategy. - * - * @param {IDBConnection} connection - DB connection - * @param {string} surveyId - */ - constructor(connection: IDBConnection, surveyId: number) { - super(connection); - - this.surveyId = surveyId; - - this.surveyCritterService = new SurveyCritterService(connection); - } - - /** - * Get taxon body locations Map from a list of Critters. - * - * @async - * @param {ICritterDetailed[]} critters - List of detailed critters - * @returns {Promise>} Critter id -> taxon body locations Map - */ - async getTaxonBodyLocationsCritterIdMap(critters: ICritterDetailed[]): Promise> { - const tsnBodyLocationsMap = new Map(); - const critterBodyLocationsMap = new Map(); - - const uniqueTsns = Array.from(new Set(critters.map((critter) => critter.itis_tsn))); - - // Only fetch body locations for unique tsns - const bodyLocations = await Promise.all( - uniqueTsns.map((tsn) => this.surveyCritterService.critterbaseService.getTaxonBodyLocations(String(tsn))) - ); - - // Loop through the flattened responses and set the body locations for each tsn - bodyLocations.flatMap((bodyLocationValues, idx) => { - tsnBodyLocationsMap.set(uniqueTsns[idx], bodyLocationValues); - }); - - // Now loop through the critters and assign the body locations to the critter id - for (const critter of critters) { - const tsnBodyLocations = tsnBodyLocationsMap.get(critter.itis_tsn); - if (tsnBodyLocations) { - critterBodyLocationsMap.set(critter.critter_id, tsnBodyLocations); - } - } - - return critterBodyLocationsMap; - } - - /** - * Validate the CSV rows against zod schema. - * - * @param {Row[]} rows - CSV rows - * @returns {*} - */ - async validateRows(rows: Row[]) { - // Generate type-safe cell getter from column validator - const getColumnCell = generateColumnCellGetterFromColumnValidator(this.columnValidator); - - // Get validation reference data - const [critterAliasMap, colours, markingTypes] = await Promise.all([ - this.surveyCritterService.getSurveyCritterAliasMap(this.surveyId), - this.surveyCritterService.critterbaseService.getColours(), - this.surveyCritterService.critterbaseService.getMarkingTypes() - ]); - - // Used to find critter_id -> taxon body location [] map - const rowCritters: ICritterDetailed[] = []; - - // Rows passed to validator - const rowsToValidate: Partial[] = []; - - for (const row of rows) { - let critterId, captureId; - - const alias = getColumnCell(row, 'ALIAS'); - - // If the alias is included attempt to retrieve the critter_id and capture_id for the row - if (alias.cell) { - const captureDate = getColumnCell(row, 'CAPTURE_DATE'); - const captureTime = getColumnCell(row, 'CAPTURE_TIME'); - - const critter = critterAliasMap.get(alias.cell.toLowerCase()); - - if (critter) { - // Find the capture_id from the date time columns - const captures = findCapturesFromDateTime(critter.captures, captureDate.cell, captureTime.cell); - captureId = captures.length === 1 ? captures[0].capture_id : undefined; - critterId = critter.critter_id; - rowCritters.push(critter); - } - } - - rowsToValidate.push({ - critter_id: critterId, // Found using alias - capture_id: captureId, // Found using capture date and time - body_location: getColumnCell(row, 'BODY_LOCATION').cell, - marking_type: getColumnCell(row, 'MARKING_TYPE').cell, - identifier: getColumnCell(row, 'IDENTIFIER').cell, - primary_colour: getColumnCell(row, 'PRIMARY_COLOUR').cell, - secondary_colour: getColumnCell(row, 'SECONDARY_COLOUR').cell, - comment: getColumnCell(row, 'DESCRIPTION').cell - }); - } - // Get the critter_id -> taxonBodyLocations[] Map - const critterBodyLocationsMap = await this.getTaxonBodyLocationsCritterIdMap(rowCritters); - - // Generate the zod schema with injected reference values - // This allows the zod schema to validate against Critterbase lookup values - return z.array(getCsvMarkingSchema(colours, markingTypes, critterBodyLocationsMap)).safeParseAsync(rowsToValidate); - } - - /** - * Insert markings into Critterbase. - * - * @async - * @param {CsvCapture[]} markings - List of CSV markings to create - * @returns {Promise} Number of created markings - */ - async insert(markings: CsvMarking[]): Promise { - const response = await this.surveyCritterService.critterbaseService.bulkCreate({ markings }); - - defaultLog.debug({ label: 'import markings', markings, insertedCount: response.created.markings }); - - return response.created.markings; - } -} diff --git a/api/src/services/import-services/marking/marking-header-configs.test.ts b/api/src/services/import-services/marking/marking-header-configs.test.ts new file mode 100644 index 0000000000..6fa7311bd3 --- /dev/null +++ b/api/src/services/import-services/marking/marking-header-configs.test.ts @@ -0,0 +1,340 @@ +import chai, { expect } from 'chai'; +import sinon from 'sinon'; +import sinonChai from 'sinon-chai'; +import { CSVConfigUtils } from '../../../utils/csv-utils/csv-config-utils'; +import { CSVConfig, CSVParams } from '../../../utils/csv-utils/csv-config-validation.interface'; +import { NestedRecord } from '../../../utils/nested-record'; +import { + getMarkingAliasCellValidator, + getMarkingBodyLocationCellValidator, + getMarkingCaptureDateCellValidator, + getMarkingColourCellValidator, + getMarkingIdentifierCellValidator, + getMarkingTypeCellValidator +} from './marking-header-configs'; + +chai.use(sinonChai); + +describe('marking-header-configs', () => { + beforeEach(() => { + sinon.restore(); + }); + + describe('getMarkingIdentifierCellValidator', () => { + it('should allow a string with a length between 1 and 50', () => { + const result = getMarkingIdentifierCellValidator()({ cell: 'string' } as CSVParams); + + expect(result).to.deep.equal([]); + }); + + it('should allow a number with a min value of 0', () => { + const result = getMarkingIdentifierCellValidator()({ cell: 0 } as CSVParams); + + expect(result).to.deep.equal([]); + }); + + it('should allow an optional cell', () => { + const result = getMarkingIdentifierCellValidator()({ cell: undefined } as CSVParams); + + expect(result).to.deep.equal([]); + }); + + it('should return a single error for these values', () => { + const badValues = ['', ' ', 'a'.repeat(51), -1]; + + badValues.forEach((cell) => { + const result = getMarkingIdentifierCellValidator()({ cell } as CSVParams); + + expect(result.length).to.deep.equal(1); + }); + }); + }); + + describe('getMarkingAliasCellValidator', () => { + it('should only allow values that exist in the surveyAliasMap', () => { + const surveyAliasMap: any = new Map([['alias', { captures: [{ capture_id: 'uuid' }] } as any]]); + + const result = getMarkingAliasCellValidator(surveyAliasMap)({ cell: 'ALIAS' } as CSVParams); + + expect(result).to.deep.equal([]); + }); + + it('should allow undefined', () => { + const surveyAliasMap: any = new Map([['alias', { captures: [{ capture_id: 'uuid' }] } as any]]); + + const result = getMarkingAliasCellValidator(surveyAliasMap)({ cell: undefined } as CSVParams); + + expect(result).to.deep.equal([]); + }); + + it('should update the mutateCell value to the critter_id', () => { + const surveyAliasMap = new Map([['alias', { critter_id: 'critter', captures: [{ capture_id: 'uuid' }] } as any]]); + + const params = { cell: 'ALIAS', mutateCell: 'ALIAS' } as CSVParams; + + const result = getMarkingAliasCellValidator(surveyAliasMap)(params); + + expect(params.mutateCell).to.deep.equal('critter'); + expect(result.length).to.deep.equal(0); + }); + + it('should return a single error if does not exist in the surveyAliasMap', () => { + const surveyAliasMap: any = new Map([['alias', 'survey']]); + + const result = getMarkingAliasCellValidator(surveyAliasMap)({ cell: 'bad' } as CSVParams); + + expect(result[0].error).to.contain('find a matching survey critter'); + }); + + it('should return a error if the critter has no captures', () => { + const surveyAliasMap: any = new Map([['alias', { captures: [] } as any]]); + + const result = getMarkingAliasCellValidator(surveyAliasMap)({ cell: 'alias' } as CSVParams); + + expect(result[0].error).to.contain('no captures'); + }); + }); + + describe('getMarkingTypeCellValidator', () => { + it('should only allow values from the markingTypes set', () => { + const markingTypes = new Set(['type']); + + const result = getMarkingTypeCellValidator(markingTypes)({ cell: 'TYPE' } as CSVParams); + + expect(result).to.deep.equal([]); + }); + + it('should allow undefined', () => { + const markingTypes = new Set(['type']); + + const result = getMarkingTypeCellValidator(markingTypes)({ cell: undefined } as CSVParams); + + expect(result).to.deep.equal([]); + }); + + it('should return a single error if the value is not in the markingTypes set', () => { + const markingTypes = new Set(['type']); + + const result = getMarkingTypeCellValidator(markingTypes)({ cell: 'bad' } as CSVParams); + + expect(result.length).to.be.equal(1); + }); + }); + + describe('getMarkingBodyLocationCellValidator', () => { + it('should return no errors for valid body locations', () => { + const dictionary = new NestedRecord({ 1: { location: 'uuid' } }); + const surveyAliasMap: any = new Map([['alias', { itis_tsn: 1 }]]); + + const getCellValueStub = sinon.stub().returns('alias'); + + const utils: any = { + getCellValue: getCellValueStub + }; + + const cellValidator = getMarkingBodyLocationCellValidator(surveyAliasMap, dictionary, utils); + + const result = cellValidator({ + mutateCell: 'body_location_id', + cell: 'location', + row: { ALIAS: 'alias' }, + header: '', + rowIndex: 0 + } as CSVParams); + + expect(result).to.deep.equal([]); + }); + + it('should return no errors when alias does not map to a survey critter', () => { + const dictionary = new NestedRecord({ 1: { location: 'uuid' } }); + const surveyAliasMap: any = new Map([['alias', { itis_tsn: 1 }]]); + + const getCellValueStub = sinon.stub().returns('alias2'); + + const utils: any = { + getCellValue: getCellValueStub + }; + + const cellValidator = getMarkingBodyLocationCellValidator(surveyAliasMap, dictionary, utils); + + const result = cellValidator({ + mutateCell: 'body_location_id', + cell: 'bad', + row: { ALIAS: 'invalidAlias' }, + header: '', + rowIndex: 0 + } as CSVParams); + + expect(result).to.deep.equal([]); + }); + + it('should return an error when tsn has no body locations', () => { + const dictionary = new NestedRecord({ 1: { location: 'uuid' } }); + const surveyAliasMap: any = new Map([['alias', { itis_tsn: 2 }]]); + + const getCellValueStub = sinon.stub().returns('alias'); + + const utils: any = { + getCellValue: getCellValueStub + }; + + const cellValidator = getMarkingBodyLocationCellValidator(surveyAliasMap, dictionary, utils); + + const result = cellValidator({ + mutateCell: 'body_location_id', + cell: 'bad', + row: { ALIAS: 'invalidAlias' }, + header: '', + rowIndex: 0 + } as CSVParams); + + expect(result[0].error).to.contain('body locations not found'); + }); + + it('should return a single error when invalid body location option', () => { + const dictionary = new NestedRecord({ 1: { location: 'uuid' } }); + const surveyAliasMap: any = new Map([['alias', { itis_tsn: 1 }]]); + + const getCellValueStub = sinon.stub().returns('alias'); + + const utils: any = { + getCellValue: getCellValueStub + }; + + const cellValidator = getMarkingBodyLocationCellValidator(surveyAliasMap, dictionary, utils); + + const result = cellValidator({ + mutateCell: 'body_location_id', + cell: 'bad', + row: { ALIAS: 'alias' }, + header: '', + rowIndex: 0 + } as CSVParams); + + expect(result[0].error).to.contain('Invalid taxon marking body location'); + }); + }); + + describe('getMarkingColourCellValidator', () => { + it('should return no errors for valid colours', () => { + const colours = new Set(['colour']); + + const result = getMarkingColourCellValidator(colours)({ cell: 'COLOUR' } as CSVParams); + + expect(result).to.deep.equal([]); + }); + + it('should return no errors for undefined', () => { + const colours = new Set(['colour']); + + const result = getMarkingColourCellValidator(colours)({ cell: undefined } as CSVParams); + + expect(result).to.deep.equal([]); + }); + + it('should return a single error if the value is not in the colours set', () => { + const colours = new Set(['colour']); + + const result = getMarkingColourCellValidator(colours)({ cell: 'bad' } as CSVParams); + + expect(result.length).to.be.equal(1); + }); + }); + + describe('getMarkingCaptureDateCellValidator', () => { + it('should return no errors when alias does not map to survey aliases', () => { + const surveyAliasMap: any = new Map([['alias', { captures: [{ capture_id: 'uuid' }] } as any]]); + const mockConfig: CSVConfig = { + staticHeadersConfig: { + ALIAS: { aliases: [] }, + CAPTURE_DATE: { aliases: [] }, + CAPTURE_TIME: { aliases: [] } + }, + ignoreDynamicHeaders: true + }; + const utils = new CSVConfigUtils({}, mockConfig); + + const result = getMarkingCaptureDateCellValidator( + surveyAliasMap, + utils + )({ cell: '2024-01-01', row: { ALIAS: 'bad', CAPTURE_TIME: '20:20:10' } } as unknown as CSVParams); + + expect(result).to.deep.equal([]); + }); + + it('should update the mutateCell value to the capture_id', () => { + const surveyAliasMap: any = new Map([ + ['alias', { captures: [{ capture_id: 'uuid', capture_date: '2021-01-01' }] } as any] + ]); + const mockConfig: CSVConfig = { + staticHeadersConfig: { + ALIAS: { aliases: [] }, + CAPTURE_DATE: { aliases: [] }, + CAPTURE_TIME: { aliases: [] } + }, + ignoreDynamicHeaders: true + }; + const utils = new CSVConfigUtils({}, mockConfig); + + const params = { cell: '2021-01-01', row: { ALIAS: 'alias' } } as unknown as CSVParams; + + const result = getMarkingCaptureDateCellValidator(surveyAliasMap, utils)(params); + + expect(params.mutateCell).to.deep.equal('uuid'); + expect(result.length).to.deep.equal(0); + }); + + it('should return error when capture not found for critter', () => { + const surveyAliasMap: any = new Map([ + ['alias', { captures: [{ capture_id: 'uuid', capture_date: '2021-01-01' }] } as any] + ]); + const mockConfig: CSVConfig = { + staticHeadersConfig: { + ALIAS: { aliases: [] }, + CAPTURE_DATE: { aliases: [] }, + CAPTURE_TIME: { aliases: [] } + }, + ignoreDynamicHeaders: true + }; + const utils = new CSVConfigUtils({}, mockConfig); + + const result = getMarkingCaptureDateCellValidator( + surveyAliasMap, + utils + )({ cell: '2024-01-01', row: { ALIAS: 'alias', CAPTURE_TIME: '20:20:10' } } as unknown as CSVParams); + + expect(result[0].error).to.contain('not found'); + }); + + it('should return error when multiple captures found for critter', () => { + const surveyAliasMap: any = new Map([ + [ + 'alias', + { + captures: [ + { capture_id: 'uuid', capture_date: '2021-01-01' }, + { capture_id: 'uuid2', capture_date: '2021-01-01' } + ] + } as any + ] + ]); + + const mockConfig: CSVConfig = { + staticHeadersConfig: { + ALIAS: { aliases: [] }, + CAPTURE_DATE: { aliases: [] }, + CAPTURE_TIME: { aliases: [] } + }, + ignoreDynamicHeaders: true + }; + const utils = new CSVConfigUtils({}, mockConfig); + + const result = getMarkingCaptureDateCellValidator( + surveyAliasMap, + utils + )({ cell: '2021-01-01', row: { ALIAS: 'alias' } } as unknown as CSVParams); + + expect(result[0].error).to.contain('ultiple captures found'); + }); + }); +}); diff --git a/api/src/services/import-services/marking/marking-header-configs.ts b/api/src/services/import-services/marking/marking-header-configs.ts new file mode 100644 index 0000000000..834b42de5b --- /dev/null +++ b/api/src/services/import-services/marking/marking-header-configs.ts @@ -0,0 +1,239 @@ +import { z } from 'zod'; +import { CSVConfigUtils } from '../../../utils/csv-utils/csv-config-utils'; +import { CSVCellValidator, CSVError, CSVParams } from '../../../utils/csv-utils/csv-config-validation.interface'; +import { validateZodCell } from '../../../utils/csv-utils/csv-header-configs'; +import { NestedRecord } from '../../../utils/nested-record'; +import { setToLowercase } from '../../../utils/string-utils'; +import { ICritterDetailed } from '../../critterbase-service'; +import { findCapturesFromDateTime } from '../utils/datetime'; +import { MarkingCSVStaticHeader } from './import-markings-service'; + +/** + * Get the marking identifier cell validator. + * + * Rules: + * 1. The cell can be a string with a length between 1 and 50 + * 2. The cell can be a number with a min value of 0 + * 3. The cell can be optional + * + * @returns {*} {CSVCellValidator} The validate cell callback + */ +export const getMarkingIdentifierCellValidator = (): CSVCellValidator => { + return (params: CSVParams) => { + return validateZodCell(params, z.union([z.string().trim().min(1).max(50), z.number().min(0)]).optional()); + }; +}; + +/** + * Get the marking alias cell validator. + * + * Note: Modifies the mutateCell value to the `critter_id` + * + * Rules: + * 1. The alias must exist in the surveyAliasMap ie: critter alias -> critter + * 2. The alias (critter) must have Critterbase captures + * + * @param {Map} surveyAliasMap The survey alias map + * @returns {*} {CSVCellValidator} The validate cell callback + */ +export const getMarkingAliasCellValidator = (surveyAliasMap: Map): CSVCellValidator => { + return (params: CSVParams) => { + if (params.cell === undefined) { + return []; + } + + const critter = surveyAliasMap.get(String(params.cell).toLowerCase()); + + if (!critter) { + return [ + { + error: `Unable to find a matching survey critter`, + solution: `Use a valid critter alias that exists in the Survey` + } + ]; + } + + // If the critter has no captures + if (critter.captures.length === 0) { + return [ + { + error: `Animal has no captures`, + solution: `Add captures to animal` + } + ]; + } + + // Set the critter id in the state for the setter + params.mutateCell = critter.critter_id; + + return []; + }; +}; + +/** + * Get the marking type cell validator. + * + * Rules: + * 1. The cell must be a valid marking type ie: exists in the markingTypes set + * + * @param {Set} markingTypes The marking types set (case insensitive) + * @returns {*} {CSVCellValidator} The validate cell callback + */ +export const getMarkingTypeCellValidator = (markingTypes: Set): CSVCellValidator => { + const markingTypesLowerCased = setToLowercase(markingTypes); + + return (params: CSVParams) => { + if (params.cell === undefined) { + return []; + } + + if (!markingTypesLowerCased.has(String(params.cell).toLowerCase())) { + return [ + { + error: `Marking type not supported`, + solution: `Use a valid marking type`, + values: Array.from(markingTypes) + } + ]; + } + + return []; + }; +}; + +/** + * Get the marking type cell setter. + * + * @param {Set} colours The colours set (case insensitive) + * @returns {*} {CSVCellSetter} The set cell callback + */ +export const getMarkingColourCellValidator = (colours: Set): CSVCellValidator => { + const coloursLowerCased = setToLowercase(colours); + + return (params: CSVParams) => { + if (params.cell === undefined) { + return []; + } + + if (coloursLowerCased.has(String(params.cell).toLowerCase())) { + return []; + } + + return [{ error: `Colour not supported`, solution: `Use a valid colour`, values: Array.from(colours) }]; + }; +}; + +/** + * Get the marking body location cell validator. + * + * Rules: + * 1. The cell must be a valid body location for the critter ie: exists in the rowDictionary + * + * @param {Map} surveyAliasMap The survey alias map + * @param {NestedRecord} rowDictionary The row dictionary + * @param {CSVConfigUtils} utils The CSV config utils + * @returns {*} {CSVCellValidator} The validate cell callback + */ +export const getMarkingBodyLocationCellValidator = ( + surveyAliasMap: Map, + rowDictionary: NestedRecord, + utils: CSVConfigUtils +): CSVCellValidator => { + return (params: CSVParams) => { + const rowAlias = String(utils.getCellValue('ALIAS', params.row)); + const aliasTsn = surveyAliasMap.get(rowAlias.toLowerCase())?.itis_tsn; + + // ALIAS header will catch this error + if (!aliasTsn) { + return []; + } + + const rowDictionaryAlias = rowDictionary.get(aliasTsn); + + if (!rowDictionaryAlias) { + return [ + { + error: `Taxon marking body locations not found for animal`, + solution: `Validate the taxon (TSN) is correct and it allows marking body locations` + } + ]; + } + + const bodyLocationCellValue = String(params.cell); + + const rowDictionaryBodyLocation = rowDictionary.get(aliasTsn, bodyLocationCellValue); + + if (!rowDictionaryBodyLocation) { + return [ + { + error: `Invalid taxon marking body location`, + solution: `Use valid taxon marking body location`, + values: Object.keys(rowDictionaryAlias) + } + ]; + } + + return []; + }; +}; + +/** + * Get the marking capture date cell validator. + * + * Note: Modifies the mutateCell value to the `capture_id` + * + * Rules: + * 1. The cell combined with the 'CAPTURE_TIME' must be a valid timestamp + * 2. The timestamp must map to a specific critter capture + * + * @param {Map} surveyAliasMap The survey alias map + * @param {CSVConfigUtils} utils The CSV config utils + * @returns {*} {CSVCellValidator} The validate cell callback + */ +export const getMarkingCaptureDateCellValidator = ( + surveyAliasMap: Map, + utils: CSVConfigUtils +): CSVCellValidator => { + return (params: CSVParams): CSVError[] => { + const cellErrors = validateZodCell(params, z.string().date()); + + if (cellErrors.length) { + return cellErrors; + } + + // Row meta data + const dateCellValue = String(params.cell); + const rowAlias = String(utils.getCellValue('ALIAS', params.row)); + const rowTime = utils.getCellValue('CAPTURE_TIME', params.row) as string; // casting to allow undefined + const aliasCritter = surveyAliasMap.get(rowAlias.toLowerCase()); + + // All alias errors need to be resolved before proceeding ie: alias not found + if (!aliasCritter) { + return []; + } + + const foundCaptures = findCapturesFromDateTime(aliasCritter.captures, dateCellValue, rowTime); + + // If unable to map the capture date and time to a specific critter capture + if (foundCaptures.length === 0) { + return [ + { + error: `Capture not found for animal using date AND time`, + solution: `Use a valid date and time to identify the capture` + } + ]; + } + + // If multiple captures found for the critter - data error + if (foundCaptures.length > 1) { + return [ + { error: `Multiple captures found for animal`, solution: `Use a unique date and time to identify the capture` } + ]; + } + + // Set the capture id in the state for the setter + params.mutateCell = foundCaptures[0].capture_id; + + return []; + }; +}; diff --git a/api/src/services/import-services/utils/datetime.ts b/api/src/services/import-services/utils/datetime.ts index d3787fa12c..cbcf6ee011 100644 --- a/api/src/services/import-services/utils/datetime.ts +++ b/api/src/services/import-services/utils/datetime.ts @@ -76,7 +76,7 @@ export const areDatesEqual = (_dateA: string, _dateB: string): boolean => { export const findCapturesFromDateTime = ( captures: T[], captureDate: string, - captureTime: string + captureTime?: string ): T[] => { return captures.filter((capture) => { return ( diff --git a/api/src/services/survey-critter-service.ts b/api/src/services/survey-critter-service.ts index db5470df70..4c7ad351f4 100644 --- a/api/src/services/survey-critter-service.ts +++ b/api/src/services/survey-critter-service.ts @@ -235,9 +235,9 @@ export class SurveyCritterService extends DBService { * * @async * @param {number} surveyId - * @returns {Promise>} Critter alias -> Detailed critter + * @returns {Promise>} Critter alias -> Detailed critter */ - async getSurveyCritterAliasMap(surveyId: number): Promise> { + async getSurveyCritterAliasMap(surveyId: number): Promise> { const critters = await this.getCritterbaseSurveyCritters(surveyId); // Create mapping of alias -> critter_id diff --git a/api/src/utils/csv-utils/csv-config-utils.test.ts b/api/src/utils/csv-utils/csv-config-utils.test.ts index c5622b51e1..6bcdc41143 100644 --- a/api/src/utils/csv-utils/csv-config-utils.test.ts +++ b/api/src/utils/csv-utils/csv-config-utils.test.ts @@ -1,5 +1,6 @@ import { expect } from 'chai'; import xlsx, { WorkSheet } from 'xlsx'; +import { WorksheetRowIndexSymbol } from '../xlsx-utils/worksheet-utils'; import { CSVConfigUtils } from './csv-config-utils'; import { CSVConfig } from './csv-config-validation.interface'; @@ -22,9 +23,13 @@ describe('CSVConfigUtils', () => { expect(utils).to.be.instanceOf(CSVConfigUtils); expect(utils._config).to.be.equal(mockConfig); expect(utils.worksheet).to.be.equal(worksheet); - expect(utils.worksheetRows).to.be.deep.equal([ - { TEST: 'cellValue', ALIASED_HEADER: 'cellValue2', DYNAMIC_HEADER: 'dynamicValue' } - ]); + + expect(utils.worksheetRows[0]).to.deep.equal({ + TEST: 'cellValue', + ALIASED_HEADER: 'cellValue2', + DYNAMIC_HEADER: 'dynamicValue', + [WorksheetRowIndexSymbol]: 1 + }); expect(utils.worksheetHeaders).to.be.deep.equal(['TEST', 'ALIASED_HEADER', 'DYNAMIC_HEADER']); expect(utils.worksheetAliasedStaticHeaders).to.be.deep.equal(['TEST', 'ALIASED_HEADER']); expect(utils.worksheetStaticHeaders).to.be.deep.equal(['TEST', 'TEST_ALIAS']); diff --git a/api/src/utils/csv-utils/csv-config-utils.ts b/api/src/utils/csv-utils/csv-config-utils.ts index 555a75af67..d0957e228b 100644 --- a/api/src/utils/csv-utils/csv-config-utils.ts +++ b/api/src/utils/csv-utils/csv-config-utils.ts @@ -1,7 +1,7 @@ import { countBy, difference } from 'lodash'; import { WorkSheet } from 'xlsx'; import { getHeadersUpperCase, getWorksheetRowObjects } from '../xlsx-utils/worksheet-utils'; -import { CSVConfig, CSVRow } from './csv-config-validation.interface'; +import { CSVCell, CSVConfig, CSVHeaderConfig, CSVRow } from './csv-config-validation.interface'; /** * CSV Config Utils - A collection of methods useful when building CSVConfigs @@ -10,12 +10,12 @@ import { CSVConfig, CSVRow } from './csv-config-validation.interface'; * @template StaticHeaderType - The static header type * @class CSVConfigUtils */ -export class CSVConfigUtils> { +export class CSVConfigUtils = Uppercase> { _config: CSVConfig; worksheet: WorkSheet; worksheetRows: CSVRow[]; - constructor(worksheet: WorkSheet, config: CSVConfig) { + constructor(worksheet: WorkSheet, config: CSVConfig) { this._config = config; this.worksheet = worksheet; this.worksheetRows = getWorksheetRowObjects(worksheet); @@ -24,10 +24,10 @@ export class CSVConfigUtils> { /** * The CSV config static headers. * - * @returns {Uppercase[]} - The config headers + * @returns {StaticHeaderType[]} - The config headers */ - get configStaticHeaders(): Uppercase[] { - return Object.keys(this._config.staticHeadersConfig) as Uppercase[]; + get configStaticHeaders(): StaticHeaderType[] { + return Object.keys(this._config.staticHeadersConfig) as StaticHeaderType[]; } /** @@ -118,16 +118,42 @@ export class CSVConfigUtils> { return difference(this.worksheetHeaders, this.worksheetAliasedStaticHeaders); } + /** + * Set a static header config. Injects the header config into the CSV static headers config. + * + * @param {StaticHeaderType} header - The header name + * @param {CSVHeaderConfig} headerConfig - The header config + * @returns {void} + */ + setStaticHeaderConfig(header: StaticHeaderType, headerConfig: CSVHeaderConfig): void { + this._config.staticHeadersConfig[header] = { ...this._config.staticHeadersConfig[header], ...headerConfig }; + } + + /** + * Get the final CSV config + * + * @returns {CSVConfig} - The CSV config + */ + getConfig(): CSVConfig { + for (const header of this.configStaticHeaders) { + if (!this._config.staticHeadersConfig[header].validateCell) { + throw new Error(`Invalid CSV config. Missing 'validateCell' for static header: ${header}`); + } + } + + return this._config; + } + /** * Get the cell value from a CSV row. * * @param {StaticHeaderType} header - The header name * @param {CSVRow} row - The CSV row - * @returns {unknown} - The cell value + * @returns {any} - The cell value */ - getCellValue(header: StaticHeaderType, row: CSVRow) { + getCellValue(header: StaticHeaderType, row: CSVRow): CSVCell { // Static header or dynamic header exact match - if (header in row) { + if ((header as Uppercase) in row) { return row[header]; } @@ -143,9 +169,9 @@ export class CSVConfigUtils> { * Get all the cell values from a static header. * * @param {StaticHeaderType} header - The header name - * @returns {unknown[]} - The cell values + * @returns {any[]} - The cell values */ - getCellValues(header: StaticHeaderType) { + getCellValues(header: StaticHeaderType): CSVCell[] { return this.worksheetRows.map((row) => this.getCellValue(header, row)); } @@ -153,9 +179,9 @@ export class CSVConfigUtils> { * Get all the unique cell values from a static header. * * @param {StaticHeaderType} header - The header name - * @returns {unknown[]} - The unique cell values + * @returns {any[]} - The unique cell values */ - getUniqueCellValues(header: StaticHeaderType) { + getUniqueCellValues(header: StaticHeaderType): CSVCell[] { return [...new Set(this.getCellValues(header))]; } @@ -166,7 +192,7 @@ export class CSVConfigUtils> { * @param {unknown} cell - The cell value * @returns {boolean} - Whether all the cell values are unique */ - isCellUnique(header: StaticHeaderType, cell: unknown) { + isCellUnique(header: StaticHeaderType, cell: unknown): boolean { const uniqueDictionary = countBy(this.getCellValues(header), (value) => String(value).toLowerCase()); const dictionaryKey = String(cell).toLowerCase(); return uniqueDictionary[dictionaryKey] === 1 || uniqueDictionary[dictionaryKey] === undefined; diff --git a/api/src/utils/csv-utils/csv-config-validation.interface.ts b/api/src/utils/csv-utils/csv-config-validation.interface.ts index 07cad6ee57..fd646219d3 100644 --- a/api/src/utils/csv-utils/csv-config-validation.interface.ts +++ b/api/src/utils/csv-utils/csv-config-validation.interface.ts @@ -1,3 +1,6 @@ +export const CSV_ERROR_MESSAGE = + 'CSV contains validation errors. Please check for formatting issues, missing fields, or invalid values and try again.'; + /** * The CSV configuration interface * @@ -93,11 +96,21 @@ export interface CSVHeaderConfig { */ export interface CSVParams { /** - * The cell value. + * The cell value. Readonly to prevent mutation during validation. + * + * Why? CSVUtils and related functions are expecting the initial non-modified cell value for calculations. + * + * Use the `setCellValue` callback or the CSVParams `this.mutateCell` to update the cell value. + * + * @type {unknown} + */ + readonly cell: unknown; + /** + * The mutatable cell value. * * @type {unknown} */ - cell: unknown; + mutateCell: unknown; /** * The row header name. The initial row key. * @@ -157,7 +170,7 @@ export interface CSVError { * * @type {(string[] | number[]) | undefined} */ - values?: string[] | number[]; + values?: string[] | number[] | null; /** * The cell value. * @@ -167,13 +180,13 @@ export interface CSVError { /** * The header name. * - * @type {string | undefined} + * @type {string | null | undefined} */ - header?: string; + header?: string | null; /** * The row index the error occurred. * - * Note: Header row index 0. First data row index 1. + * Note: Header row index 1. First data row index 2. * * @type {number} */ @@ -191,3 +204,5 @@ export type CSVRow = Record, any>; * */ export type CSVRowValidated> = Record; + +export type CSVCell = string | number | undefined; diff --git a/api/src/utils/csv-utils/csv-config-validation.test.ts b/api/src/utils/csv-utils/csv-config-validation.test.ts index 99e881955e..6c9110820f 100644 --- a/api/src/utils/csv-utils/csv-config-validation.test.ts +++ b/api/src/utils/csv-utils/csv-config-validation.test.ts @@ -2,6 +2,7 @@ import chai, { expect } from 'chai'; import sinon from 'sinon'; import sinonChai from 'sinon-chai'; import xlsx, { WorkSheet } from 'xlsx'; +import { WorksheetRowIndexSymbol } from '../xlsx-utils/worksheet-utils'; import { executeSetCellValue, executeValidateCell, @@ -93,7 +94,8 @@ describe('csv-config-validation', () => { solution: `Add all required columns to the file.`, header: 'ALIAS', values: ['ALIAS', 'ALIAS_2'], - row: 0 + cell: null, + row: 1 } ], rows: [] @@ -119,10 +121,12 @@ describe('csv-config-validation', () => { expect(result).to.deep.equal([ { - row: 0, + row: 1, error: 'No columns in the file', solution: 'Add column names. Did you accidentally include an empty first row above the columns?', - values: ['ALIAS'] + values: ['ALIAS'], + cell: null, + header: null } ]); }); @@ -135,9 +139,12 @@ describe('csv-config-validation', () => { expect(result).to.deep.equal([ { - row: 1, + row: 2, error: 'No rows in the file', - solution: 'Add rows. Did you accidentally import the wrong file?' + solution: 'Add rows. Did you accidentally import the wrong file?', + cell: null, + header: null, + values: null } ]); }); @@ -150,11 +157,12 @@ describe('csv-config-validation', () => { expect(result).to.deep.equal([ { - row: 0, + row: 1, error: 'A required column is missing', solution: `Add all required columns to the file.`, header: 'ALIAS', - values: ['ALIAS'] + values: ['ALIAS'], + cell: null } ]); }); @@ -179,10 +187,12 @@ describe('csv-config-validation', () => { expect(result).to.deep.equal([ { - row: 0, + row: 1, error: 'An unknown column is included in the file', solution: `Remove extra columns from the file.`, - header: 'UNKNOWN_HEADER' + header: 'UNKNOWN_HEADER', + cell: null, + values: null } ]); }); @@ -216,7 +226,8 @@ describe('csv-config-validation', () => { header: 'TEST', rowIndex: 0, row: { TEST: 'cellValue' }, - staticHeader: 'TEST' + staticHeader: 'TEST', + mutateCell: 'cellValue' }, { validateCell: validateCellStub, @@ -252,7 +263,8 @@ describe('csv-config-validation', () => { header: 'TEST_ALIAS', rowIndex: 0, row: { TEST_ALIAS: 'cellValue' }, - staticHeader: 'TEST' + staticHeader: 'TEST', + mutateCell: 'cellValue' }, { validateCell: validateCellStub, @@ -298,8 +310,9 @@ describe('csv-config-validation', () => { cell: 'cellValue', header: 'TEST_ALIAS', rowIndex: 0, - row: { TEST_ALIAS: 'cellValue', DYNAMIC_HEADER: 'dynamicValue' }, - staticHeader: 'TEST' + row: { TEST_ALIAS: 'cellValue', DYNAMIC_HEADER: 'dynamicValue', [WorksheetRowIndexSymbol]: 1 }, + staticHeader: 'TEST', + mutateCell: 'cellValue' }, { validateCell: staticValidateCellStub, @@ -312,8 +325,9 @@ describe('csv-config-validation', () => { cell: 'dynamicValue', header: 'DYNAMIC_HEADER', rowIndex: 0, - row: { TEST_ALIAS: 'cellValue', DYNAMIC_HEADER: 'dynamicValue' }, - staticHeader: undefined // Dynamic headers have no static header mapping + row: { TEST_ALIAS: 'cellValue', DYNAMIC_HEADER: 'dynamicValue', [WorksheetRowIndexSymbol]: 1 }, + staticHeader: undefined, // Dynamic headers have no static header mapping + mutateCell: 'dynamicValue' }, { validateCell: validateDynamicCellStub, @@ -333,8 +347,9 @@ describe('csv-config-validation', () => { cell: 'cellValue', header: 'TEST', rowIndex: 0, - row: { TEST: 'cellValue' }, - staticHeader: 'TEST' + row: { TEST: 'cellValue', [WorksheetRowIndexSymbol]: 1 }, + staticHeader: 'TEST', + mutateCell: 'cellValue' }; const headerConfig = { @@ -349,8 +364,8 @@ describe('csv-config-validation', () => { solution: 'solution', cell: 'cellValue', header: 'TEST', - row: 1, - values: undefined + row: 2, + values: null } ]); }); @@ -367,7 +382,8 @@ describe('csv-config-validation', () => { header: 'TEST', rowIndex: 0, row, - staticHeader: 'TEST' + staticHeader: 'TEST', + mutateCell: 'cellValue' }; const headerConfig = { @@ -392,7 +408,8 @@ describe('csv-config-validation', () => { header: 'TEST', rowIndex: 0, row, - staticHeader: 'NEW_KEY' + staticHeader: 'NEW_KEY', + mutateCell: 'cellValue' }; const headerConfig = { diff --git a/api/src/utils/csv-utils/csv-config-validation.ts b/api/src/utils/csv-utils/csv-config-validation.ts index 8b5347b862..7d4bb50b5e 100644 --- a/api/src/utils/csv-utils/csv-config-validation.ts +++ b/api/src/utils/csv-utils/csv-config-validation.ts @@ -1,5 +1,5 @@ import { WorkSheet } from 'xlsx'; -import { getWorksheetRowObjects } from '../xlsx-utils/worksheet-utils'; +import { getWorksheetRowObjects, WorksheetRowIndexSymbol } from '../xlsx-utils/worksheet-utils'; import { CSVConfigUtils } from './csv-config-utils'; import { CSVConfig, @@ -69,7 +69,9 @@ export const validateCSVHeaders = (worksheet: WorkSheet, config: CSVConfig): CSV error: 'No columns in the file', solution: 'Add column names. Did you accidentally include an empty first row above the columns?', values: configUtils.configStaticHeaders, - row: 0 + header: null, + cell: null, + row: 1 } ]; } @@ -79,7 +81,10 @@ export const validateCSVHeaders = (worksheet: WorkSheet, config: CSVConfig): CSV { error: 'No rows in the file', solution: 'Add rows. Did you accidentally import the wrong file?', - row: 1 + values: null, + header: null, + cell: null, + row: 2 } ]; } @@ -95,9 +100,10 @@ export const validateCSVHeaders = (worksheet: WorkSheet, config: CSVConfig): CSV csvErrors.push({ error: 'A required column is missing', solution: `Add all required columns to the file.`, - header: staticHeader, values: [staticHeader, ...config.staticHeadersConfig[staticHeader].aliases], - row: 0 + header: staticHeader, + cell: null, + row: 1 }); } } @@ -108,8 +114,10 @@ export const validateCSVHeaders = (worksheet: WorkSheet, config: CSVConfig): CSV csvErrors.push({ error: 'An unknown column is included in the file', solution: `Remove extra columns from the file.`, + values: null, header: unknownHeader, - row: 0 + cell: null, + row: 1 }); } } @@ -141,8 +149,9 @@ export const forEachCSVCell = ( const headerConfig = staticHeaderConfigMap.get(header) ?? config.dynamicHeadersConfig ?? {}; const cell = worksheetRow[header]; const params: CSVParams = { - cell, - header, + cell: cell, + mutateCell: cell, // Set the mutate cell to the cell value + header: header, row: worksheetRow, rowIndex: i, staticHeader: staticHeaderConfigMap.get(header)?.staticHeader @@ -167,17 +176,19 @@ export const forEachCSVCell = ( * @returns {*} {CSVRow[]} - The updated row */ export const executeSetCellValue = (params: CSVParams, headerConfig: CSVHeaderConfig, mutableRows: CSVRow[]) => { + const row = { ...mutableRows[params.rowIndex] }; + const headerKey = params.staticHeader?.toUpperCase() ?? params.header.toUpperCase(); - const cellValue = headerConfig?.setCellValue?.(params) ?? params.cell; + const cellValue = headerConfig?.setCellValue?.(params) ?? params.mutateCell; // Remove the aliased header if it is not the static header if (params.staticHeader && params.header !== params.staticHeader) { - delete params.row[params.header]; + delete row[params.header as Uppercase]; } - params.row[headerKey] = cellValue; + row[headerKey as Uppercase] = cellValue; - mutableRows[params.rowIndex] = params.row; + mutableRows[params.rowIndex] = row; }; /** @@ -206,10 +217,11 @@ export const executeValidateCell = ( mutableErrors.push({ error: error.error, solution: error.solution, - values: error.values, - cell: error.cell ?? params.cell, - header: error.header ?? params.header, - row: error.row ?? params.rowIndex + 1 // headers: 0, data row: 1 + values: error.values ?? null, + cell: (error.cell === undefined ? params.cell : error.cell) ?? null, // Use cell value if intentionally null + header: (error.header === undefined ? params.header : error.header) ?? null, // Use header value if intentionally null + // WorksheetRowIndexSymbol is the original row index from the worksheet ie: before filtering empty rows + row: error.row ?? params.row[WorksheetRowIndexSymbol] + 1 ?? params.rowIndex + 2 // headers: 1, data row: 2 }); }); } diff --git a/api/src/utils/csv-utils/csv-header-configs.test.ts b/api/src/utils/csv-utils/csv-header-configs.test.ts index 109f9ebc37..330d901879 100644 --- a/api/src/utils/csv-utils/csv-header-configs.test.ts +++ b/api/src/utils/csv-utils/csv-header-configs.test.ts @@ -25,7 +25,7 @@ describe('CSVHeaderConfigs', () => { const tsns = new Set([1, 2]); const tsnValidator = getTsnCellValidator(tsns); - const result = tsnValidator({ cell: 1, row: {}, header: 'HEADER', rowIndex: 0 }); + const result = tsnValidator({ cell: 1, row: {}, header: 'HEADER', rowIndex: 0, mutateCell: 1 }); expect(result).to.be.deep.equal([]); }); @@ -34,7 +34,7 @@ describe('CSVHeaderConfigs', () => { const tsns = new Set([1, 2]); const tsnValidator = getTsnCellValidator(tsns); - const result = tsnValidator({ cell: 3, row: {}, header: 'HEADER', rowIndex: 0 }); + const result = tsnValidator({ cell: 3, row: {}, header: 'HEADER', rowIndex: 0, mutateCell: 3 }); expect(result).to.be.deep.equal([ { @@ -49,7 +49,13 @@ describe('CSVHeaderConfigs', () => { it('should return an empty array if the cell is valid', () => { const descriptionValidator = getDescriptionCellValidator(); - const result = descriptionValidator({ cell: 'description', row: {}, header: 'HEADER', rowIndex: 0 }); + const result = descriptionValidator({ + cell: 'description', + row: {}, + header: 'HEADER', + rowIndex: 0, + mutateCell: 'description' + }); expect(result).to.be.deep.equal([]); }); @@ -60,7 +66,13 @@ describe('CSVHeaderConfigs', () => { for (const badDescription of badDescriptions) { const descriptionValidator = getDescriptionCellValidator(); - const result = descriptionValidator({ cell: badDescription, row: {}, header: 'HEADER', rowIndex: 0 }); + const result = descriptionValidator({ + cell: badDescription, + row: {}, + header: 'HEADER', + rowIndex: 0, + mutateCell: badDescription + }); expect(result.length).to.be.equal(1); } diff --git a/api/src/utils/csv-utils/csv-header-configs.ts b/api/src/utils/csv-utils/csv-header-configs.ts index a2f8c5bb5b..49c1b3eab8 100644 --- a/api/src/utils/csv-utils/csv-header-configs.ts +++ b/api/src/utils/csv-utils/csv-header-configs.ts @@ -1,5 +1,6 @@ import { z } from 'zod'; -import { CSVCellValidator, CSVError, CSVParams } from './csv-config-validation.interface'; +import { formatTimeString } from '../../services/import-services/utils/datetime'; +import { CSVCellSetter, CSVCellValidator, CSVError, CSVParams } from './csv-config-validation.interface'; /** * Utility function to validate a CSV cell using a Zod schema. @@ -64,3 +65,41 @@ export const getDescriptionCellValidator = (): CSVCellValidator => { return validateZodCell(params, z.string().trim().min(1).max(250).optional()); }; }; + +/** + * Get the time header cell validator. + * + * Rules: + * 1. The cell must be a valid 24-hour time format 'HH:mm:ss' or 'HH:mm' or undefined + * + * @returns {*} {CSVCellValidator} The validate cell callback + */ +export const getTimeCellValidator = (): CSVCellValidator => { + return (params: CSVParams) => { + if (params.cell === undefined || formatTimeString(String(params.cell))) { + return []; + } + + return [ + { + error: `Use a valid 24-hour time format 'HH:mm:ss' or 'HH:mm'`, + solution: `Update the cell value to match the expected format` + } + ]; + }; +}; + +/** + * Get the time header cell setter. + * + * @returns {*} {CSVCellSetter} The set cell callback + */ +export const getTimeCellSetter = (): CSVCellSetter => { + return (params: CSVParams) => { + if (params.cell === undefined) { + return undefined; + } + + return formatTimeString(String(params.cell)); + }; +}; diff --git a/api/src/utils/string-utils.ts b/api/src/utils/string-utils.ts index 94d4136d56..02aca03d38 100644 --- a/api/src/utils/string-utils.ts +++ b/api/src/utils/string-utils.ts @@ -52,3 +52,13 @@ export function numberOrNull(value: string | null | undefined): number | null { return Number(value); } + +/** + * Convert a Set of strings to lowercase. + * + * @param {Set} set The set of strings + * @return {*} {Set} The set of strings in lowercase + */ +export function setToLowercase(set: Set): Set { + return new Set([...set].map((value) => value.toLowerCase())); +} diff --git a/api/src/utils/xlsx-utils/worksheet-utils.ts b/api/src/utils/xlsx-utils/worksheet-utils.ts index 02b5e107df..38057e53ab 100644 --- a/api/src/utils/xlsx-utils/worksheet-utils.ts +++ b/api/src/utils/xlsx-utils/worksheet-utils.ts @@ -17,6 +17,8 @@ dayjs.extend(customParseFormat); const defaultLog = getLogger('src/utils/xlsx-utils/worksheet-utils'); +export const WorksheetRowIndexSymbol = Symbol('WorksheetRowIndex'); + export interface IXLSXCSVColumn { /** * Supported column cell types @@ -133,6 +135,7 @@ export const getWorksheetRows = (worksheet: xlsx.WorkSheet): string[][] => { for (let i = 1; i <= originalRange.e.r; i++) { const row = new Array(getHeadersUpperCase(worksheet).length); + let rowHasValues = false; for (let j = 0; j <= originalRange.e.c; j++) { @@ -161,6 +164,8 @@ export const getWorksheetRows = (worksheet: xlsx.WorkSheet): string[][] => { * Return an array of row value arrays. * * Note: The column headers will be transformed to UPPERCASE. + * Note: Rows with no non-empty cells will be excluded. + * Note: A `RowIndex` symbol will be added to each row object with the original row index. * * @example * [ @@ -168,37 +173,64 @@ export const getWorksheetRows = (worksheet: xlsx.WorkSheet): string[][] => { * "HEADER1": "value1", * "HEADER2": "value2", * "HEADER3": "value3" + * [RowIndex]: 1 * }, + * // Empty row 2 was excluded * { * "HEADER1": "value4", * "HEADER2": "value5", * "HEADER3": "value6" + * [RowIndex]: 3 * } * ] * * @export * @param {xlsx.WorkSheet} worksheet - * @return {*} {Record[]} + * @return {*} {Record[]} */ -export const getWorksheetRowObjects = (worksheet: xlsx.WorkSheet): Record[] => { - const ref = worksheet['!ref']; +export const getWorksheetRowObjects = (worksheet: xlsx.WorkSheet): Record[] => { + const originalRange = getWorksheetRange(worksheet); - if (!ref) { + if (!originalRange) { return []; } - const rowObjectsArray: Record[] = []; - const rows = getWorksheetRows(worksheet); const headers = getHeadersUpperCase(worksheet); - for (let i = 0; i < rows.length; i++) { - const rowObject: Record = {}; + const rowObjectsArray: Record[] = []; + + for (let i = 1; i <= originalRange.e.r; i++) { + const rowObject: Record = {}; + + let rowHasValues = false; + + for (let j = 0; j <= originalRange.e.c; j++) { + // Always add the header (key) to the row object + rowObject[headers[j]] = undefined; + + const cellAddress = { c: j, r: i }; + const cellRef = xlsx.utils.encode_cell(cellAddress); + const cell = worksheet[cellRef]; + + if (!cell) { + continue; + } + + // Set the cell value for the header, if the cell exists + rowObject[headers[j]] = trimCellWhitespace(replaceCellDates(cell)).v; - for (let j = 0; j < headers.length; j++) { - rowObject[headers[j]] = rows[i][j]; + // If at least one cell has a value, then the row is not empty + rowHasValues = true; } - rowObjectsArray.push(rowObject); + // Add the original row index to the row object + // Symbols are non-enumerable, so they will be `hidden` in the rowObject + rowObject[WorksheetRowIndexSymbol] = i; + + if (rowHasValues) { + // Add the row object to the array if it has at least one non-empty cell + rowObjectsArray.push(rowObject); + } } return rowObjectsArray; diff --git a/app/src/components/csv/CSVDropzoneSection.tsx b/app/src/components/csv/CSVDropzoneSection.tsx new file mode 100644 index 0000000000..030773bb93 --- /dev/null +++ b/app/src/components/csv/CSVDropzoneSection.tsx @@ -0,0 +1,40 @@ +import { Box } from '@mui/material'; +import Button from '@mui/material/Button'; +import { CSVErrorsTableContainer } from 'components/csv/CSVErrorsTableContainer'; +import HorizontalSplitFormComponent from 'components/fields/HorizontalSplitFormComponent'; +import { PropsWithChildren } from 'react'; +import { CSVError } from 'utils/csv-utils'; + +interface CSVDropzoneSectionProps { + title: string; + summary: string; + onDownloadTemplate: () => void; + errors: CSVError[]; +} + +/** + * A section that contains a dropzone for CSV files. + * Also renders a table to display errors that occured during the CSV file validation. + * + * @param {CSVDropzoneSectionProps} props + * @returns {*} {JSX.Element} + */ +export const CSVDropzoneSection = (props: PropsWithChildren) => { + return ( + + + + + + {props.children} + {props.errors.length > 0 ? : null} + + + ); +}; diff --git a/app/src/components/csv/CSVErrorsTable.tsx b/app/src/components/csv/CSVErrorsTable.tsx new file mode 100644 index 0000000000..bf97fc3c05 --- /dev/null +++ b/app/src/components/csv/CSVErrorsTable.tsx @@ -0,0 +1,99 @@ +import { GridColDef } from '@mui/x-data-grid'; +import { StyledDataGrid } from 'components/data-grid/StyledDataGrid'; +import { useMemo } from 'react'; +import { CSVError } from 'utils/csv-utils'; +import { v4 } from 'uuid'; +import { CSVErrorsTableOptionsMenu } from './CSVErrorsTableOptionsMenu'; + +interface CSVErrorsTableProps { + errors: CSVError[]; +} + +/** + * Renders a CSV errors table. + * + * @param {CSVErrorsTableProps} props + * @returns {*} {JSX.Element} + */ +export const CSVErrorsTable = (props: CSVErrorsTableProps) => { + const columns: GridColDef[] = [ + { + field: 'row', + headerName: 'Row', + description: 'Row number in the CSV file', + minWidth: 85 + }, + { + field: 'header', + headerName: 'Header', + description: 'Column header in the CSV file', + minWidth: 150, + maxWidth: 250, + renderCell: (params) => { + return params.value?.toUpperCase(); + } + }, + { + field: 'cell', + headerName: 'Cell', + description: 'The cell value in the CSV file', + minWidth: 85 + }, + { + field: 'error', + headerName: 'Error', + description: 'The error message', + flex: 1, + minWidth: 250, + resizable: true + }, + { + field: 'solution', + headerName: 'Solution', + description: 'The solution to the error', + flex: 1, + minWidth: 250, + resizable: true + }, + { + field: 'values', + headerName: 'Options', + description: 'The applicable cell values', + minWidth: 85, + renderCell: (params) => { + return params.value?.length ? : 'N/A'; + } + } + ]; + + const rows = useMemo(() => { + return props.errors.map((error) => { + return { + id: v4(), + ...error + }; + }); + }, [props.errors]); + + return ( + 'auto'} + rows={rows} + getRowId={(row) => row.id} + columns={columns} + pageSizeOptions={[10, 25, 50]} + rowSelection={false} + checkboxSelection={false} + sortingOrder={['asc', 'desc']} + initialState={{ + pagination: { + paginationModel: { + pageSize: 5 + } + } + }} + /> + ); +}; diff --git a/app/src/components/csv/CSVErrorsTableContainer.tsx b/app/src/components/csv/CSVErrorsTableContainer.tsx new file mode 100644 index 0000000000..954b3b8d74 --- /dev/null +++ b/app/src/components/csv/CSVErrorsTableContainer.tsx @@ -0,0 +1,47 @@ +import { Divider, Paper, Toolbar, Typography } from '@mui/material'; +import { Box, Stack } from '@mui/system'; +import { ReactElement } from 'react'; +import { CSVError } from 'utils/csv-utils'; +import { CSVErrorsTable } from './CSVErrorsTable'; + +interface CSVErrorsTableContainerProps { + errors: CSVError[]; + title?: ReactElement; +} + +/** + * Renders a CSV errors table with toolbar. + * + * @param {CSVErrorsTableContainerProps} props + * @returns {*} {JSX.Element} + */ +export const CSVErrorsTableContainer = (props: CSVErrorsTableContainerProps) => { + return ( + + + {props.title ?? ( + + CSV Errors Detected ‌ + + ({props.errors.length}) + + + )} + + + + + + + ); +}; diff --git a/app/src/components/csv/CSVErrorsTableOptionsMenu.tsx b/app/src/components/csv/CSVErrorsTableOptionsMenu.tsx new file mode 100644 index 0000000000..78f18a03a4 --- /dev/null +++ b/app/src/components/csv/CSVErrorsTableOptionsMenu.tsx @@ -0,0 +1,35 @@ +import { mdiChevronDown } from '@mdi/js'; +import Icon from '@mdi/react'; +import { Button, Menu, MenuItem } from '@mui/material'; +import { useState } from 'react'; + +interface CSVErrorsTableOptionsMenuProps { + options: string[]; +} + +/** + * Renders a CSV errors table options menu. + * + * @param {CSVErrorsTableOptionsMenuProps} props + * @returns {*} {JSX.Element} + */ +export const CSVErrorsTableOptionsMenu = (props: CSVErrorsTableOptionsMenuProps) => { + const [anchorEl, setAnchorEl] = useState(null); + + return ( + <> + + setAnchorEl(null)}> + {props.options.map((value) => ( + {value} + ))} + + + ); +}; diff --git a/app/src/components/file-upload/FileUploadItemContent.tsx b/app/src/components/file-upload/FileUploadItemContent.tsx index 93a002d221..edf67496e7 100644 --- a/app/src/components/file-upload/FileUploadItemContent.tsx +++ b/app/src/components/file-upload/FileUploadItemContent.tsx @@ -51,60 +51,51 @@ export const FileUploadItemContent = (props: FileUploadItemContentProps) => { const ProgressBar = props.ProgressBarComponent ?? FileUploadItemProgressBar; return ( - } + - - } + sx={{ flexWrap: 'wrap' }}> + + + + } + sx={{ + '& .MuiListItemText-primary': { + fontWeight: 700 + } + }} /> - - } - sx={{ - '& .MuiListItemText-primary': { - fontWeight: 700 - } - }} - /> - - - - {props.enableErrorDetails && ( - - + + - )} - + {props.enableErrorDetails && ( + + + + )} + + ); }; diff --git a/app/src/features/surveys/animals/profile/captures/import-captures/CreateCSVCapturesPage.tsx b/app/src/features/surveys/animals/profile/captures/import-captures/CreateCSVCapturesPage.tsx index 88abed1acf..4d5f26af97 100644 --- a/app/src/features/surveys/animals/profile/captures/import-captures/CreateCSVCapturesPage.tsx +++ b/app/src/features/surveys/animals/profile/captures/import-captures/CreateCSVCapturesPage.tsx @@ -1,5 +1,5 @@ import LoadingButton from '@mui/lab/LoadingButton'; -import { Box, Divider } from '@mui/material'; +import { Divider } from '@mui/material'; import Breadcrumbs from '@mui/material/Breadcrumbs'; import Button from '@mui/material/Button'; import Container from '@mui/material/Container'; @@ -8,7 +8,6 @@ import Paper from '@mui/material/Paper'; import Stack from '@mui/material/Stack'; import Typography from '@mui/material/Typography'; import axios, { AxiosProgressEvent } from 'axios'; -import HorizontalSplitFormComponent from 'components/fields/HorizontalSplitFormComponent'; import { UploadFileStatus } from 'components/file-upload/FileUploadItem'; import { FileUploadSingleItem } from 'components/file-upload/FileUploadSingleItem'; import PageHeader from 'components/layout/PageHeader'; @@ -18,14 +17,24 @@ import { SKIP_CONFIRMATION_DIALOG, useUnsavedChangesDialog } from 'hooks/useUnsa import { useCallback, useMemo, useState } from 'react'; import { Prompt, useHistory } from 'react-router'; import { Link as RouterLink } from 'react-router-dom'; +import { CSVError, isCSVValidationError } from 'utils/csv-utils'; import { downloadFile } from 'utils/file-utils'; import { getAxiosProgress } from 'utils/Utils'; +import { CSVDropzoneSection } from '../../../../../../components/csv/CSVDropzoneSection'; import { getCapturesCSVTemplate, getMarkingsCSVTemplate, getMeasurementsCSVTemplate } from './utils/templates'; type CSVFilesStatus = { - captures: { file: File | null; status: UploadFileStatus; progress: number; error?: string }; - measurements: { file: File | null; status: UploadFileStatus; progress: number; error?: string }; - markings: { file: File | null; status: UploadFileStatus; progress: number; error?: string }; + captures: { file: File | null; status: UploadFileStatus; progress: number; error?: string; errors: CSVError[] }; + measurements: { file: File | null; status: UploadFileStatus; progress: number; error?: string; errors: CSVError[] }; + markings: { file: File | null; status: UploadFileStatus; progress: number; error?: string; errors: CSVError[] }; +}; + +const INITIAL_FILE_STATE = { + file: null, + status: UploadFileStatus.PENDING, + progress: 0, + error: undefined, + errors: [] }; type UpdateFileState = { @@ -35,6 +44,7 @@ type UpdateFileState = { status?: UploadFileStatus; progress?: number; error?: string; + errors?: CSVError[]; }; /** @@ -57,9 +67,9 @@ export const CreateCSVCapturesPage = () => { // Initialize the file upload states const [files, setFiles] = useState({ - captures: { file: null, status: UploadFileStatus.PENDING, progress: 0 }, - measurements: { file: null, status: UploadFileStatus.PENDING, progress: 0 }, - markings: { file: null, status: UploadFileStatus.PENDING, progress: 0 } + captures: INITIAL_FILE_STATE, + measurements: INITIAL_FILE_STATE, + markings: INITIAL_FILE_STATE }); // When any of the files are uploading @@ -113,7 +123,13 @@ export const CreateCSVCapturesPage = () => { return UploadFileStatus.COMPLETE; // Return the final status to prevent race conditions with state } catch (error: any) { - handleFileState({ fileType, status: UploadFileStatus.FAILED, error: error.message ?? 'Unknown error' }); + handleFileState({ + fileType, + status: UploadFileStatus.FAILED, + progress: 100, + error: error.message ?? 'Unknown error', + errors: isCSVValidationError(error) ? error.errors : [] + }); return UploadFileStatus.FAILED; // Return the final status to prevent race conditions with state } @@ -182,6 +198,26 @@ export const CreateCSVCapturesPage = () => { history.push(`/admin/projects/${projectId}/surveys/${surveyId}/animals`); }; + /** + * Get the props for the file upload component + * + * @param {keyof CSVFilesStatus} fileType - The type of file to get the props for + * @returns {*} {FileUploadSingleItemProps} The props for the file upload component + */ + const getFileUploadProps = (fileType: keyof CSVFilesStatus) => { + return { + file: files[fileType].file, + status: files[fileType].status, + progress: files[fileType].progress, + error: files[fileType].error, + onStatus: (status: UploadFileStatus) => handleFileState({ fileType, status }), + onFile: (file: File | null) => handleFileState({ fileType, file }), + onError: (error: string) => handleFileState({ fileType, error }), + onCancel: () => handleFileState({ fileType, ...INITIAL_FILE_STATE }), + DropZoneProps: { acceptedFileExtensions: '.csv' } + }; + }; + return ( <> @@ -226,94 +262,31 @@ export const CreateCSVCapturesPage = () => { - - - - handleFileState({ fileType: 'captures', status })} - onFile={(file) => handleFileState({ fileType: 'captures', file })} - onError={(error) => handleFileState({ fileType: 'captures', error })} - onCancel={() => - handleFileState({ - fileType: 'captures', - status: UploadFileStatus.PENDING, - error: undefined, - progress: undefined - }) - } - DropZoneProps={{ acceptedFileExtensions: '.csv' }} - /> - - + downloadFile(getCapturesCSVTemplate(), 'SIMS-captures-template.csv')} + errors={files.captures.errors}> + + - - - - handleFileState({ fileType: 'measurements', status })} - onFile={(file) => handleFileState({ fileType: 'measurements', file })} - onError={(error) => handleFileState({ fileType: 'measurements', error })} - onCancel={() => - handleFileState({ - fileType: 'measurements', - status: UploadFileStatus.PENDING, - error: undefined, - progress: undefined - }) - } - DropZoneProps={{ acceptedFileExtensions: '.csv' }} - /> - - + downloadFile(getMeasurementsCSVTemplate(), 'SIMS-measurements-template.csv')} + errors={files.measurements.errors}> + + - - - - handleFileState({ fileType: 'markings', status })} - onFile={(file) => handleFileState({ fileType: 'markings', file })} - onError={(error) => handleFileState({ fileType: 'markings', error })} - onCancel={() => - handleFileState({ - fileType: 'markings', - status: UploadFileStatus.PENDING, - error: undefined, - progress: undefined - }) - } - DropZoneProps={{ acceptedFileExtensions: '.csv' }} - /> - - + downloadFile(getMarkingsCSVTemplate(), 'SIMS-markings-template.csv')} + errors={files.markings.errors}> + + diff --git a/app/src/features/surveys/animals/profile/captures/import-captures/utils/templates.ts b/app/src/features/surveys/animals/profile/captures/import-captures/utils/templates.ts index c6101422b3..e30e125a2f 100644 --- a/app/src/features/surveys/animals/profile/captures/import-captures/utils/templates.ts +++ b/app/src/features/surveys/animals/profile/captures/import-captures/utils/templates.ts @@ -1,4 +1,4 @@ -import { getCSVTemplate } from 'utils/file-utils'; +import { getCSVTemplate } from 'utils/csv-utils'; /** * Get CSV template for measurements. diff --git a/app/src/utils/csv-utils.ts b/app/src/utils/csv-utils.ts new file mode 100644 index 0000000000..953548f3bb --- /dev/null +++ b/app/src/utils/csv-utils.ts @@ -0,0 +1,37 @@ +const CSV_VALIDATION_ERROR = 'CSV Validation Error'; + +export interface CSVError { + error: string; + solution: string; + header: string | null; + cell: string | number | null; + values: Array | null; + row: number; +} + +export interface CSVValidationError { + name: typeof CSV_VALIDATION_ERROR; + message: string; + status: number; + errors: CSVError[]; +} + +/** + * Get CSV template from a list of column headers. + * + * @param {string[]} headers - CSV column headers + * @returns {string} Encoded CSV template + */ +export const getCSVTemplate = (headers: string[]) => { + return 'data:text/csv;charset=utf-8,' + headers.join(',') + '\n'; +}; + +/** + * Check if the error is a CSV validation error. + * + * @param {any} error - The error object to check + * @returns {boolean} True if the error is a CSV validation error + */ +export const isCSVValidationError = (error: any): error is CSVValidationError => { + return error && error.name === CSV_VALIDATION_ERROR && error.status === 422; +}; diff --git a/app/src/utils/file-utils.ts b/app/src/utils/file-utils.ts index 6fddb9881d..1321dbd913 100644 --- a/app/src/utils/file-utils.ts +++ b/app/src/utils/file-utils.ts @@ -1,13 +1,3 @@ -/** - * Get CSV template from a list of column headers. - * - * @param {string[]} headers - CSV column headers - * @returns {string} Encoded CSV template - */ -export const getCSVTemplate = (headers: string[]) => { - return 'data:text/csv;charset=utf-8,' + headers.join(',') + '\n'; -}; - /** * Download a file client side. *