Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

SIMS-BIOHUB-8 #1024

Merged
merged 17 commits into from
May 9, 2023
Merged

SIMS-BIOHUB-8 #1024

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
7a066b8
display list of missing sheets
anissa-agahchen Apr 27, 2023
450b125
test
anissa-agahchen Apr 27, 2023
ad01459
remove logs
anissa-agahchen Apr 27, 2023
65710ae
addressed feedback and fixed a bug
anissa-agahchen May 1, 2023
fab506b
fixed tests
anissa-agahchen May 1, 2023
10e8a31
Merge branch 'dev' of https://github.com/bcgov/biohubbc into SIM-BIOH…
anissa-agahchen May 1, 2023
6b44223
tweak
anissa-agahchen May 1, 2023
819520c
Merge branch 'dev' into SIM-BIOHUB-8
anissa-agahchen May 1, 2023
faaa6d8
apply fix to summary result validateXLSX
anissa-agahchen May 2, 2023
49775a5
fixed bug found by sonar cloud
anissa-agahchen May 2, 2023
16223f3
more sonar cloud feedback
anissa-agahchen May 2, 2023
2b99726
removed unused test artifact
anissa-agahchen May 2, 2023
f334257
adding the row number to the primary key
anissa-agahchen May 4, 2023
a3bbfec
addressed sonar cloud feedback
anissa-agahchen May 4, 2023
6c83299
fixes
anissa-agahchen May 8, 2023
ac902f8
Merge branch 'dev' of https://github.com/bcgov/biohubbc into SIM-BIOH…
anissa-agahchen May 8, 2023
f44ce84
remove parse error
anissa-agahchen May 8, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions api/src/json-schema/validation-schema.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,7 @@ export const submissionValidationSchema = {
title: 'File/Sheet Validation',
description: 'The validators that can be applied against a file/sheet within a submission file.',
anyOf: [
//below: column header validator
anissa-agahchen marked this conversation as resolved.
Show resolved Hide resolved
{
$ref: '#/$defs/file_required_columns_validator'
},
Expand All @@ -130,6 +131,7 @@ export const submissionValidationSchema = {
{
$ref: '#/$defs/file_valid_columns_validator'
},
//below: column cell validator
{
$ref: '#/$defs/file_column_unique_validator'
}
Expand Down Expand Up @@ -334,6 +336,7 @@ export const submissionValidationSchema = {
},
additionalProperties: false
},
//for the cell to have a certain rule, pairs with format/range, etc
column_required_validator: {
description: 'Validates that this column value is not empty',
type: 'object',
Expand Down
10 changes: 5 additions & 5 deletions api/src/services/summary-service.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -490,7 +490,7 @@ describe('SummaryService', () => {
]);

const getRules = sinon.stub(service, 'getValidationRules').resolves('');
const validate = sinon.stub(service, 'validateXLSX').resolves({});
const validate = sinon.stub(service, 'validateXLSX_summary').resolves({});
anissa-agahchen marked this conversation as resolved.
Show resolved Hide resolved
const persistResults = sinon.stub(service, 'persistSummaryValidationResults').resolves();

const logFoundValidation = sinon.stub(SummaryRepository.prototype, 'insertSummarySubmissionMessage').resolves();
Expand Down Expand Up @@ -520,7 +520,7 @@ describe('SummaryService', () => {
.stub(service, 'getSummaryTemplateSpeciesRecords')
.resolves([makeMockTemplateSpeciesRecord(1)]);
const getRules = sinon.stub(service, 'getValidationRules').resolves('');
const validate = sinon.stub(service, 'validateXLSX').resolves({});
const validate = sinon.stub(service, 'validateXLSX_summary').resolves({});
const persistResults = sinon.stub(service, 'persistSummaryValidationResults').resolves();

await service.summaryTemplateValidation(xlsxCsv, 1);
Expand All @@ -541,7 +541,7 @@ describe('SummaryService', () => {

const getValidation = sinon.stub(service, 'getSummaryTemplateSpeciesRecords').resolves(templateSpeciesRecords);
const getRules = sinon.stub(service, 'getValidationRules').resolves('');
const validate = sinon.stub(service, 'validateXLSX').resolves({});
const validate = sinon.stub(service, 'validateXLSX_summary').resolves({});
const persistResults = sinon.stub(service, 'persistSummaryValidationResults').resolves();

await service.summaryTemplateValidation(xlsxCsv, 1);
Expand Down Expand Up @@ -601,7 +601,7 @@ describe('SummaryService', () => {
}
});

it('should throw INVALID_MEDIA error if validateXLSX fails with invalid media', async () => {
it('should1 throw INVALID_MEDIA error if validateXLSX fails with invalid media', async () => {
const service = mockService();
const file = new MediaFile('test.txt', 'text/plain', Buffer.of(0));
const xlsxCsv = new XLSXCSV(file);
Expand Down Expand Up @@ -776,7 +776,7 @@ describe('SummaryService', () => {
const service = mockService();
const xlsx = new XLSXCSV(buildFile('test file', {}));
const parser = new ValidationSchemaParser({});
const response = await service.validateXLSX(xlsx, parser);
const response = await service.validateXLSX_summary(xlsx, parser);

expect(response.media_state.isValid).to.be.true;
expect(response.media_state.fileErrors).is.empty;
Expand Down
4 changes: 2 additions & 2 deletions api/src/services/summary-service.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -211,7 +211,7 @@ export class SummaryService extends DBService {
}

const schemaParser = this.getValidationRules(validationSchema);
const csvState = this.validateXLSX(xlsx, schemaParser);
const csvState = this.validateXLSX_summary(xlsx, schemaParser);
await this.persistSummaryValidationResults(csvState.csv_state, csvState.media_state);
} catch (error) {
if (error instanceof SubmissionError) {
Expand Down Expand Up @@ -300,7 +300,7 @@ export class SummaryService extends DBService {
* @param {ValidationSchemaParser} parser
* @returns {ICsvMediaState}
*/
validateXLSX(file: XLSXCSV, parser: ValidationSchemaParser): ICsvMediaState {
validateXLSX_summary(file: XLSXCSV, parser: ValidationSchemaParser): ICsvMediaState {
defaultLog.debug({ label: 'validateXLSX' });

// Run media validations
Expand Down
44 changes: 12 additions & 32 deletions api/src/services/validation-service.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -157,11 +157,11 @@ describe('ValidationService', () => {

const getValidation = sinon.stub(ValidationService.prototype, 'getValidationSchema').resolves('');
const getRules = sinon.stub(ValidationService.prototype, 'getValidationRules').resolves('');
const validate = sinon.stub(ValidationService.prototype, 'validateXLSX').resolves({});
const validate = sinon.stub(ValidationService.prototype, 'validateXLSX_occurrence').resolves({});
const persistResults = sinon.stub(ValidationService.prototype, 'persistValidationResults').resolves(true);

const service = mockService();
await service.templateValidation(xlsxCsv, 1);
await service.occurrenceTemplateValidation(xlsxCsv, 1);

expect(getValidation).to.be.calledOnce;
expect(getRules).to.be.calledOnce;
Expand All @@ -176,13 +176,13 @@ describe('ValidationService', () => {

sinon.stub(ValidationService.prototype, 'getValidationSchema').throws(new SubmissionError({}));
sinon.stub(ValidationService.prototype, 'getValidationRules').resolves({});
sinon.stub(ValidationService.prototype, 'validateXLSX').resolves({});
sinon.stub(ValidationService.prototype, 'validateXLSX_occurrence').resolves({});
sinon.stub(ValidationService.prototype, 'persistValidationResults').resolves(true);

try {
const dbConnection = getMockDBConnection();
const service = new ValidationService(dbConnection);
await service.templateValidation(xlsxCsv, 1);
await service.occurrenceTemplateValidation(xlsxCsv, 1);
expect.fail();
} catch (error) {
expect(error).to.be.instanceOf(SubmissionError);
Expand Down Expand Up @@ -505,7 +505,7 @@ describe('ValidationService', () => {
xlsx: new XLSXCSV(buildFile('test file', {}))
};
const prep = sinon.stub(service, 'templatePreparation').resolves(mockPrep);
const validation = sinon.stub(service, 'templateValidation').resolves();
const validation = sinon.stub(service, 'occurrenceTemplateValidation').resolves();
const submissionStatus = sinon.stub(service.submissionRepository, 'insertSubmissionStatus').resolves();

await service.validateFile(1, 1);
Expand All @@ -522,7 +522,7 @@ describe('ValidationService', () => {
};
const prep = sinon.stub(service, 'templatePreparation').resolves(mockPrep);
const validation = sinon
.stub(service, 'templateValidation')
.stub(service, 'occurrenceTemplateValidation')
.throws(SubmissionErrorFromMessageType(SUBMISSION_MESSAGE_TYPE.MISSING_VALIDATION_SCHEMA));
const submissionStatus = sinon.stub(service.submissionRepository, 'insertSubmissionStatus').resolves();
const insertError = sinon.stub(service.errorService, 'insertSubmissionError').resolves();
Expand All @@ -545,7 +545,7 @@ describe('ValidationService', () => {
xlsx: new XLSXCSV(buildFile('test file', {}))
};
const prep = sinon.stub(service, 'templatePreparation').resolves(mockPrep);
const validation = sinon.stub(service, 'templateValidation').throws(new Error());
const validation = sinon.stub(service, 'occurrenceTemplateValidation').throws(new Error());
const submissionStatus = sinon.stub(service.submissionRepository, 'insertSubmissionStatus').resolves();
const insertError = sinon.stub(service.errorService, 'insertSubmissionError').resolves();

Expand Down Expand Up @@ -707,7 +707,7 @@ describe('ValidationService', () => {
};

const prep = sinon.stub(service, 'templatePreparation').resolves(mockPrep);
const validate = sinon.stub(service, 'templateValidation').resolves();
const validate = sinon.stub(service, 'occurrenceTemplateValidation').resolves();
const transform = sinon
.stub(service, 'templateTransformation')
.throws(SubmissionErrorFromMessageType(SUBMISSION_MESSAGE_TYPE.FAILED_TRANSFORM_XLSX));
Expand All @@ -734,7 +734,7 @@ describe('ValidationService', () => {
};

const prep = sinon.stub(service, 'templatePreparation').resolves(mockPrep);
const validate = sinon.stub(service, 'templateValidation').resolves();
const validate = sinon.stub(service, 'occurrenceTemplateValidation').resolves();
const transform = sinon.stub(service, 'templateTransformation').throws();
const insertError = sinon.stub(service.errorService, 'insertSubmissionError').resolves();
sinon.stub(service, 'scrapeDwCAndUploadOccurrences').resolves();
Expand All @@ -759,7 +759,7 @@ describe('ValidationService', () => {
};

const prep = sinon.stub(service, 'templatePreparation').resolves(mockPrep);
const validate = sinon.stub(service, 'templateValidation').resolves();
const validate = sinon.stub(service, 'occurrenceTemplateValidation').resolves();
const status = sinon.stub(service.submissionRepository, 'insertSubmissionStatus').resolves();
const transform = sinon.stub(service, 'templateTransformation').resolves();
const decorate = sinon.stub(service.dwCService, 'decorateDwCJSON').resolves();
Expand Down Expand Up @@ -1026,32 +1026,12 @@ describe('ValidationService', () => {
const service = mockService();
const xlsx = new XLSXCSV(buildFile('test file', {}));
const parser = new ValidationSchemaParser({});
const response = await service.validateXLSX(xlsx, parser);
const response = await service.validateXLSX_occurrence(xlsx, parser);

expect(response.media_state.isValid).to.be.true;
expect(response.media_state.fileErrors).is.empty;
});

it('should throw Media is invalid error', async () => {
const service = mockService();
const mockMediaState = {
fileName: 'test file',
isValid: false
} as IMediaState;
const xlsx = new XLSXCSV(buildFile('test file', {}));
const parser = new ValidationSchemaParser({});

sinon.stub(XLSXCSV.prototype, 'getMediaState').returns(mockMediaState);

try {
await service.validateXLSX(xlsx, parser);
expect.fail();
} catch (error) {
expect(error).to.be.instanceOf(SubmissionError);
expect((error as SubmissionError).submissionMessages[0].type).to.be.eql(SUBMISSION_MESSAGE_TYPE.INVALID_MEDIA);
}
});

it('should return valid state object with content errors', async () => {
const service = mockService();
const mockState = {
Expand Down Expand Up @@ -1086,7 +1066,7 @@ describe('ValidationService', () => {
sinon.stub(DWCArchive.prototype, 'validateContent');
sinon.stub(XLSXCSV.prototype, 'getContentState').returns([mockState]);

const response = await service.validateXLSX(xlsx, parser);
const response = await service.validateXLSX_occurrence(xlsx, parser);
expect(response.csv_state).is.not.empty;
expect(response.csv_state[0].headerErrors).is.not.empty;
expect(response.csv_state[0].rowErrors).is.not.empty;
Expand Down
13 changes: 5 additions & 8 deletions api/src/services/validation-service.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ export class ValidationService extends DBService {
defaultLog.debug({ label: 'validateFile', submissionId, surveyId });
try {
const submissionPrep = await this.templatePreparation(submissionId);
await this.templateValidation(submissionPrep.xlsx, surveyId);
await this.occurrenceTemplateValidation(submissionPrep.xlsx, surveyId);

// insert template validated status
await this.submissionRepository.insertSubmissionStatus(submissionId, SUBMISSION_STATUS_TYPE.TEMPLATE_VALIDATED);
Expand Down Expand Up @@ -158,7 +158,7 @@ export class ValidationService extends DBService {
const submissionPrep = await this.templatePreparation(submissionId);

// Run template validations
await this.templateValidation(submissionPrep.xlsx, surveyId);
await this.occurrenceTemplateValidation(submissionPrep.xlsx, surveyId);

// Insert validation complete status
await this.submissionRepository.insertSubmissionStatus(submissionId, SUBMISSION_STATUS_TYPE.TEMPLATE_VALIDATED);
Expand Down Expand Up @@ -264,12 +264,12 @@ export class ValidationService extends DBService {
}
}

async templateValidation(xlsx: XLSXCSV, surveyId: number) {
async occurrenceTemplateValidation(xlsx: XLSXCSV, surveyId: number) {
defaultLog.debug({ label: 'templateValidation' });
try {
const schema = await this.getValidationSchema(xlsx, surveyId);
const schemaParser = this.getValidationRules(schema);
const csvState = this.validateXLSX(xlsx, schemaParser);
const csvState = this.validateXLSX_occurrence(xlsx, schemaParser);
await this.persistValidationResults(csvState.csv_state, csvState.media_state);
} catch (error) {
if (error instanceof SubmissionError) {
Expand Down Expand Up @@ -359,14 +359,11 @@ export class ValidationService extends DBService {
return validationSchemaParser;
}

validateXLSX(file: XLSXCSV, parser: ValidationSchemaParser) {
validateXLSX_occurrence(file: XLSXCSV, parser: ValidationSchemaParser) {
// Run media validations
file.validateMedia(parser);

const media_state = file.getMediaState();
anissa-agahchen marked this conversation as resolved.
Show resolved Hide resolved
if (!media_state.isValid) {
throw SubmissionErrorFromMessageType(SUBMISSION_MESSAGE_TYPE.INVALID_MEDIA);
}

// Run CSV content validations
file.validateContent(parser);
Expand Down