diff --git a/Makefile b/Makefile index ac108bcaa3..3becb04c73 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ export $(shell sed 's/=.*//' .env) .DEFAULT : help -.PHONY : setup close clean build run run-debug build-backend run-backend run-backend-debug build-web run-web run-web-debug database app api db-setup db-migrate db-rollback install test lint lint-fix format help +.PHONY : setup close clean build run run-debug build-backend run-backend run-backend-debug build-web run-web run-web-debug database app api db-setup db-migrate db-rollback n8n-setup n8n-export clamav install test lint lint-fix format help ## ------------------------------------------------------------------------------ ## Alias Commands @@ -43,6 +43,8 @@ db-rollback: | build-db-rollback run-db-rollback ## Performs all commands necess n8n-setup: | build-n8n-setup run-n8n-setup ## Performs all commands necessary to run the n8n setup n8n-export: | build-n8n-export run-n8n-export ## Performs all commands necessary to export the latest n8n credentials and workflows +clamav: | build-clamav run-clamav ## Performs all commands necessary to run clamav + ## ------------------------------------------------------------------------------ ## Setup/Cleanup Commands ## ------------------------------------------------------------------------------ @@ -256,6 +258,22 @@ run-n8n-export: ## Run the n8n export @echo "===============================================" @docker-compose -f docker-compose.yml up n8n_export +## ------------------------------------------------------------------------------ +## clamav commands +## ------------------------------------------------------------------------------ + +build-clamav: ## Build the clamav image + @echo "===============================================" + @echo "Make: build-clamav - building clamav image" + @echo "===============================================" + @docker-compose -f docker-compose.yml build clamav + +run-clamav: ## Run clamav + @echo "===============================================" + @echo "Make: run-clamav - running clamav" + @echo "===============================================" + @docker-compose -f docker-compose.yml up -d clamav + ## ------------------------------------------------------------------------------ ## Run `npm` commands for all projects ## ------------------------------------------------------------------------------ diff --git a/api/openshift/api.dc.yaml b/api/openshift/api.dc.yaml index 5735e38587..85a3d58b22 100644 --- a/api/openshift/api.dc.yaml +++ b/api/openshift/api.dc.yaml @@ -5,6 +5,8 @@ metadata: selfLink: '' name: biohubbc-api-dc parameters: + - name: ENABLE_FILE_VIRUS_SCAN + value: 'true' - name: NAME value: biohubbc-api - name: SUFFIX @@ -114,6 +116,8 @@ objects: value: ${HOST} - name: API_PORT value: ${API_PORT_DEFAULT} + - name: ENABLE_FILE_VIRUS_SCAN + value: ${ENABLE_FILE_VIRUS_SCAN} - name: DB_HOST value: ${DB_SERVICE_NAME} - name: DB_USER_API diff --git a/api/package-lock.json b/api/package-lock.json index af4fe16dd8..6cbb685c68 100644 --- a/api/package-lock.json +++ b/api/package-lock.json @@ -1754,6 +1754,11 @@ "integrity": "sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ==", "dev": true }, + "clamdjs": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/clamdjs/-/clamdjs-1.0.2.tgz", + "integrity": "sha512-gVnX5ySMULvwYL2ykZQnP4UK4nIK7ftG6z015drJyOFgWpsqXt1Hcq4fMyPwM8LLsxfgfYKLiZi288xuTfmZBQ==" + }, "class-utils": { "version": "0.3.6", "resolved": "https://registry.npmjs.org/class-utils/-/class-utils-0.3.6.tgz", diff --git a/api/package.json b/api/package.json index 396987651d..bed9fc0e46 100644 --- a/api/package.json +++ b/api/package.json @@ -35,6 +35,7 @@ "db-migrate": "~0.11.11", "db-migrate-pg": "~1.2.2", "xlsx": "~0.17.0", + "clamdjs": "~1.0.2", "express": "~4.17.1", "express-openapi": "~7.0.1", "fast-json-patch": "~3.0.0-1", diff --git a/api/src/paths/project/{projectId}/attachments/upload.test.ts b/api/src/paths/project/{projectId}/attachments/upload.test.ts index 9614e635b9..250e07aa7a 100644 --- a/api/src/paths/project/{projectId}/attachments/upload.test.ts +++ b/api/src/paths/project/{projectId}/attachments/upload.test.ts @@ -104,6 +104,8 @@ describe('uploadMedia', () => { } }); + sinon.stub(file_utils, 'scanFileForVirus').resolves(true); + try { const result = upload.uploadMedia(); @@ -115,7 +117,30 @@ describe('uploadMedia', () => { } }); - it('should return a list of file keys on success (with username and email)', async () => { + it('should throw a 400 error when file contains malicious content', async () => { + sinon.stub(db, 'getDBConnection').returns({ + ...dbConnectionObj, + systemUserId: () => { + return 20; + } + }); + + sinon.stub(file_utils, 'uploadFileToS3').resolves({ Key: '1/1/test.txt' } as any); + sinon.stub(project, 'upsertProjectAttachment').resolves(1); + sinon.stub(file_utils, 'scanFileForVirus').resolves(false); + + try { + const result = upload.uploadMedia(); + + await result(sampleReq, sampleRes as any, (null as unknown) as any); + expect.fail(); + } catch (actualError) { + expect(actualError.status).to.equal(400); + expect(actualError.message).to.equal('Malicious content detected, upload cancelled'); + } + }); + + it('should return file key on success (with username and email)', async () => { sinon.stub(db, 'getDBConnection').returns({ ...dbConnectionObj, systemUserId: () => { @@ -123,6 +148,7 @@ describe('uploadMedia', () => { } }); + sinon.stub(file_utils, 'scanFileForVirus').resolves(true); sinon.stub(file_utils, 'uploadFileToS3').resolves({ Key: '1/1/test.txt' } as any); sinon.stub(project, 'upsertProjectAttachment').resolves(1); @@ -130,6 +156,6 @@ describe('uploadMedia', () => { await result(sampleReq, sampleRes as any, (null as unknown) as any); - expect(actualResult).to.eql(['1/1/test.txt']); + expect(actualResult).to.eql('1/1/test.txt'); }); }); diff --git a/api/src/paths/project/{projectId}/attachments/upload.ts b/api/src/paths/project/{projectId}/attachments/upload.ts index c6948c1d2a..1ebea55842 100644 --- a/api/src/paths/project/{projectId}/attachments/upload.ts +++ b/api/src/paths/project/{projectId}/attachments/upload.ts @@ -1,12 +1,11 @@ 'use strict'; -import { ManagedUpload } from 'aws-sdk/clients/s3'; import { RequestHandler } from 'express'; import { Operation } from 'express-openapi'; import { SYSTEM_ROLE } from '../../../../constants/roles'; import { getDBConnection } from '../../../../database/db'; import { HTTP400 } from '../../../../errors/CustomError'; -import { generateS3FileKey, uploadFileToS3 } from '../../../../utils/file-utils'; +import { generateS3FileKey, scanFileForVirus, uploadFileToS3 } from '../../../../utils/file-utils'; import { getLogger } from '../../../../utils/logger'; import { upsertProjectAttachment } from '../../../project'; @@ -14,8 +13,8 @@ const defaultLog = getLogger('/api/project/{projectId}/attachments/upload'); export const POST: Operation = [uploadMedia()]; POST.apiDoc = { - description: 'Upload project-specific attachments.', - tags: ['attachments'], + description: 'Upload a project-specific attachment.', + tags: ['attachment'], security: [ { Bearer: [SYSTEM_ROLE.SYSTEM_ADMIN, SYSTEM_ROLE.PROJECT_ADMIN] @@ -29,19 +28,15 @@ POST.apiDoc = { } ], requestBody: { - description: 'Attachments upload post request object.', + description: 'Attachment upload post request object.', content: { 'multipart/form-data': { schema: { type: 'object', properties: { media: { - type: 'array', - description: 'An array of attachments to upload', - items: { - type: 'string', - format: 'binary' - } + type: 'string', + format: 'binary' } } } @@ -50,24 +45,12 @@ POST.apiDoc = { }, responses: { 200: { - description: 'Attachments upload response.', + description: 'Attachment upload response.', content: { 'application/json': { schema: { - type: 'array', - items: { - type: 'object', - properties: { - mediaKey: { - type: 'string', - description: 'The S3 unique key for this file.' - }, - lastModified: { - type: 'string', - description: 'The date the object was last modified' - } - } - } + type: 'string', + description: 'The S3 unique key for this file.' } } } @@ -99,12 +82,12 @@ export function uploadMedia(): RequestHandler { throw new HTTP400('Missing upload data'); } + const rawMediaFile: Express.Multer.File = rawMediaArray[0]; + defaultLog.debug({ label: 'uploadMedia', - message: 'files', - files: rawMediaArray.map((item) => { - return { ...item, buffer: 'Too big to print' }; - }) + message: 'file', + file: { ...rawMediaFile, buffer: 'Too big to print' } }); if (!req.params.projectId) { @@ -113,41 +96,38 @@ export function uploadMedia(): RequestHandler { const connection = getDBConnection(req['keycloak_token']); - // Insert file metadata into project_attachment table try { await connection.open(); - const insertProjectAttachmentsPromises = - rawMediaArray.map((file: Express.Multer.File) => - upsertProjectAttachment(file, Number(req.params.projectId), connection) - ) || []; + // Scan file for viruses using ClamAV + const virusScanResult = await scanFileForVirus(rawMediaFile); - await Promise.all([...insertProjectAttachmentsPromises]); + if (!virusScanResult) { + throw new HTTP400('Malicious content detected, upload cancelled'); + } - // Upload files to S3 - const s3UploadPromises: Promise[] = []; + // Insert file metadata into project_attachment table + await upsertProjectAttachment(rawMediaFile, Number(req.params.projectId), connection); - rawMediaArray.forEach((file: Express.Multer.File) => { - const key = generateS3FileKey({ - projectId: Number(req.params.projectId), - fileName: file.originalname - }); + // Upload file to S3 + const key = generateS3FileKey({ + projectId: Number(req.params.projectId), + fileName: rawMediaFile.originalname + }); - const metadata = { - filename: file.originalname, - username: (req['auth_payload'] && req['auth_payload'].preferred_username) || '', - email: (req['auth_payload'] && req['auth_payload'].email) || '' - }; + const metadata = { + filename: rawMediaFile.originalname, + username: (req['auth_payload'] && req['auth_payload'].preferred_username) || '', + email: (req['auth_payload'] && req['auth_payload'].email) || '' + }; - s3UploadPromises.push(uploadFileToS3(file, key, metadata)); - }); + const result = await uploadFileToS3(rawMediaFile, key, metadata); - const results = await Promise.all(s3UploadPromises); - defaultLog.debug({ label: 'uploadMedia', message: 'results', results }); + defaultLog.debug({ label: 'uploadMedia', message: 'result', result }); await connection.commit(); - return res.status(200).json(results.map((result) => result.Key)); + return res.status(200).json(result.Key); } catch (error) { defaultLog.debug({ label: 'uploadMedia', message: 'error', error }); await connection.rollback(); diff --git a/api/src/paths/project/{projectId}/survey/{surveyId}/attachments/upload.test.ts b/api/src/paths/project/{projectId}/survey/{surveyId}/attachments/upload.test.ts index f56ef0bbf8..f5bb6daf37 100644 --- a/api/src/paths/project/{projectId}/survey/{surveyId}/attachments/upload.test.ts +++ b/api/src/paths/project/{projectId}/survey/{surveyId}/attachments/upload.test.ts @@ -110,6 +110,8 @@ describe('uploadMedia', () => { } }); + sinon.stub(file_utils, 'scanFileForVirus').resolves(true); + try { const result = upload.uploadMedia(); @@ -121,7 +123,30 @@ describe('uploadMedia', () => { } }); - it('should return a list of file keys on success (with username and email)', async () => { + it('should throw a 400 error when file contains malicious content', async () => { + sinon.stub(db, 'getDBConnection').returns({ + ...dbConnectionObj, + systemUserId: () => { + return 20; + } + }); + + sinon.stub(file_utils, 'uploadFileToS3').resolves({ Key: '1/1/test.txt' } as any); + sinon.stub(upload, 'upsertSurveyAttachment').resolves(1); + sinon.stub(file_utils, 'scanFileForVirus').resolves(false); + + try { + const result = upload.uploadMedia(); + + await result(sampleReq, sampleRes as any, (null as unknown) as any); + expect.fail(); + } catch (actualError) { + expect(actualError.status).to.equal(400); + expect(actualError.message).to.equal('Malicious content detected, upload cancelled'); + } + }); + + it('should return file key on success (with username and email)', async () => { sinon.stub(db, 'getDBConnection').returns({ ...dbConnectionObj, systemUserId: () => { @@ -131,15 +156,16 @@ describe('uploadMedia', () => { sinon.stub(file_utils, 'uploadFileToS3').resolves({ Key: '1/1/test.txt' } as any); sinon.stub(upload, 'upsertSurveyAttachment').resolves(1); + sinon.stub(file_utils, 'scanFileForVirus').resolves(true); const result = upload.uploadMedia(); await result(sampleReq, sampleRes as any, (null as unknown) as any); - expect(actualResult).to.eql(['1/1/test.txt']); + expect(actualResult).to.eql('1/1/test.txt'); }); - it('should return a list of file keys on success (without username and email)', async () => { + it('should return file key on success (without username and email)', async () => { sinon.stub(db, 'getDBConnection').returns({ ...dbConnectionObj, systemUserId: () => { @@ -149,6 +175,7 @@ describe('uploadMedia', () => { sinon.stub(file_utils, 'uploadFileToS3').resolves({ Key: '1/1/test.txt' } as any); sinon.stub(upload, 'upsertSurveyAttachment').resolves(1); + sinon.stub(file_utils, 'scanFileForVirus').resolves(true); const result = upload.uploadMedia(); @@ -158,7 +185,7 @@ describe('uploadMedia', () => { (null as unknown) as any ); - expect(actualResult).to.eql(['1/1/test.txt']); + expect(actualResult).to.eql('1/1/test.txt'); }); }); diff --git a/api/src/paths/project/{projectId}/survey/{surveyId}/attachments/upload.ts b/api/src/paths/project/{projectId}/survey/{surveyId}/attachments/upload.ts index 5d303c25c3..f39512f10f 100644 --- a/api/src/paths/project/{projectId}/survey/{surveyId}/attachments/upload.ts +++ b/api/src/paths/project/{projectId}/survey/{surveyId}/attachments/upload.ts @@ -1,6 +1,5 @@ 'use strict'; -import { ManagedUpload } from 'aws-sdk/clients/s3'; import { RequestHandler } from 'express'; import { Operation } from 'express-openapi'; import { SYSTEM_ROLE } from '../../../../../../constants/roles'; @@ -11,15 +10,15 @@ import { postSurveyAttachmentSQL, putSurveyAttachmentSQL } from '../../../../../../queries/survey/survey-attachments-queries'; -import { generateS3FileKey, uploadFileToS3 } from '../../../../../../utils/file-utils'; +import { generateS3FileKey, scanFileForVirus, uploadFileToS3 } from '../../../../../../utils/file-utils'; import { getLogger } from '../../../../../../utils/logger'; const defaultLog = getLogger('/api/project/{projectId}/survey/{surveyId}/attachments/upload'); export const POST: Operation = [uploadMedia()]; POST.apiDoc = { - description: 'Upload survey-specific attachments.', - tags: ['attachments'], + description: 'Upload a survey-specific attachment.', + tags: ['attachment'], security: [ { Bearer: [SYSTEM_ROLE.SYSTEM_ADMIN, SYSTEM_ROLE.PROJECT_ADMIN] @@ -38,19 +37,15 @@ POST.apiDoc = { } ], requestBody: { - description: 'Attachments upload post request object.', + description: 'Attachment upload post request object.', content: { 'multipart/form-data': { schema: { type: 'object', properties: { media: { - type: 'array', - description: 'An array of attachments to upload', - items: { - type: 'string', - format: 'binary' - } + type: 'string', + format: 'binary' } } } @@ -59,24 +54,12 @@ POST.apiDoc = { }, responses: { 200: { - description: 'Attachments upload response.', + description: 'Attachment upload response.', content: { 'application/json': { schema: { - type: 'array', - items: { - type: 'object', - properties: { - mediaKey: { - type: 'string', - description: 'The S3 unique key for this file.' - }, - lastModified: { - type: 'string', - description: 'The date the object was last modified' - } - } - } + type: 'string', + description: 'The S3 unique key for this file.' } } } @@ -108,12 +91,12 @@ export function uploadMedia(): RequestHandler { throw new HTTP400('Missing upload data'); } + const rawMediaFile: Express.Multer.File = rawMediaArray[0]; + defaultLog.debug({ label: 'uploadMedia', message: 'files', - files: rawMediaArray.map((item) => { - return { ...item, buffer: 'Too big to print' }; - }) + files: { ...rawMediaFile, buffer: 'Too big to print' } }); if (!req.params.surveyId) { @@ -122,41 +105,39 @@ export function uploadMedia(): RequestHandler { const connection = getDBConnection(req['keycloak_token']); - // Insert file metadata into survey_attachment table try { await connection.open(); - const insertSurveyAttachmentsPromises = rawMediaArray.map((file: Express.Multer.File) => - upsertSurveyAttachment(file, Number(req.params.projectId), Number(req.params.surveyId), connection) - ); + // Scan file for viruses using ClamAV + const virusScanResult = await scanFileForVirus(rawMediaFile); - await Promise.all([...insertSurveyAttachmentsPromises]); + if (!virusScanResult) { + throw new HTTP400('Malicious content detected, upload cancelled'); + } - // Upload files to S3 - const s3UploadPromises: Promise[] = []; + // Insert file metadata into survey_attachment table + await upsertSurveyAttachment(rawMediaFile, Number(req.params.projectId), Number(req.params.surveyId), connection); - rawMediaArray.forEach((file: Express.Multer.File) => { - const key = generateS3FileKey({ - projectId: Number(req.params.projectId), - surveyId: Number(req.params.surveyId), - fileName: file.originalname - }); + // Upload file to S3 + const key = generateS3FileKey({ + projectId: Number(req.params.projectId), + surveyId: Number(req.params.surveyId), + fileName: rawMediaFile.originalname + }); - const metadata = { - filename: file.originalname, - username: (req['auth_payload'] && req['auth_payload'].preferred_username) || '', - email: (req['auth_payload'] && req['auth_payload'].email) || '' - }; + const metadata = { + filename: rawMediaFile.originalname, + username: (req['auth_payload'] && req['auth_payload'].preferred_username) || '', + email: (req['auth_payload'] && req['auth_payload'].email) || '' + }; - s3UploadPromises.push(uploadFileToS3(file, key, metadata)); - }); + const result = await uploadFileToS3(rawMediaFile, key, metadata); - const results = await Promise.all(s3UploadPromises); - defaultLog.debug({ label: 'uploadMedia', message: 'results', results }); + defaultLog.debug({ label: 'uploadMedia', message: 'result', result }); await connection.commit(); - return res.status(200).json(results.map((result) => result.Key)); + return res.status(200).json(result.Key); } catch (error) { defaultLog.debug({ label: 'uploadMedia', message: 'error', error }); await connection.rollback(); diff --git a/api/src/paths/project/{projectId}/survey/{surveyId}/observation/submission/upload.test.ts b/api/src/paths/project/{projectId}/survey/{surveyId}/observation/submission/upload.test.ts index 769daa58fe..6870bd050f 100644 --- a/api/src/paths/project/{projectId}/survey/{surveyId}/observation/submission/upload.test.ts +++ b/api/src/paths/project/{projectId}/survey/{surveyId}/observation/submission/upload.test.ts @@ -129,6 +129,7 @@ describe('uploadSubmission', () => { sinon.stub(db, 'getDBConnection').returns(dbConnectionObj); sinon.stub(survey_occurrence_queries, 'insertSurveyOccurrenceSubmissionSQL').returns(null); + sinon.stub(file_utils, 'scanFileForVirus').resolves(true); const result = upload.uploadMedia(); @@ -141,6 +142,27 @@ describe('uploadSubmission', () => { } }); + it('should throw a 400 error when file contains malicious content', async () => { + sinon.stub(db, 'getDBConnection').returns({ + ...dbConnectionObj, + systemUserId: () => { + return 20; + } + }); + + sinon.stub(file_utils, 'scanFileForVirus').resolves(false); + + const result = upload.uploadMedia(); + + try { + await result(mockReq, mockRes, mockNext); + expect.fail(); + } catch (actualError) { + expect(actualError.status).to.equal(400); + expect(actualError.message).to.equal('Malicious content detected, upload cancelled'); + } + }); + it('should throw a 400 error when it fails to insert a record in the database', async () => { const mockQuery = sinon.stub(); @@ -154,6 +176,7 @@ describe('uploadSubmission', () => { query: mockQuery }); + sinon.stub(file_utils, 'scanFileForVirus').resolves(true); sinon.stub(survey_occurrence_queries, 'insertSurveyOccurrenceSubmissionSQL').returns(SQL`some query`); const result = upload.uploadMedia(); @@ -180,6 +203,7 @@ describe('uploadSubmission', () => { query: mockQuery }); + sinon.stub(file_utils, 'scanFileForVirus').resolves(true); sinon.stub(survey_occurrence_queries, 'insertSurveyOccurrenceSubmissionSQL').returns(SQL`some query`); sinon.stub(survey_occurrence_queries, 'updateSurveyOccurrenceSubmissionWithKeySQL').returns(null); @@ -208,6 +232,7 @@ describe('uploadSubmission', () => { query: mockQuery }); + sinon.stub(file_utils, 'scanFileForVirus').resolves(true); sinon.stub(survey_occurrence_queries, 'insertSurveyOccurrenceSubmissionSQL').returns(SQL`some query`); sinon.stub(survey_occurrence_queries, 'updateSurveyOccurrenceSubmissionWithKeySQL').returns(SQL`some query`); @@ -235,6 +260,7 @@ describe('uploadSubmission', () => { query: mockQuery }); + sinon.stub(file_utils, 'scanFileForVirus').resolves(true); sinon.stub(survey_occurrence_queries, 'insertSurveyOccurrenceSubmissionSQL').returns(SQL`some query`); sinon.stub(survey_occurrence_queries, 'updateSurveyOccurrenceSubmissionWithKeySQL').returns(SQL`some query`); sinon.stub(file_utils, 'uploadFileToS3').rejects('Failed to insert occurrence submission data'); @@ -262,6 +288,7 @@ describe('uploadSubmission', () => { query: mockQuery }); + sinon.stub(file_utils, 'scanFileForVirus').resolves(true); sinon.stub(survey_occurrence_queries, 'insertSurveyOccurrenceSubmissionSQL').returns(SQL`some query`); sinon.stub(survey_occurrence_queries, 'updateSurveyOccurrenceSubmissionWithKeySQL').returns(SQL`some query`); diff --git a/api/src/paths/project/{projectId}/survey/{surveyId}/observation/submission/upload.ts b/api/src/paths/project/{projectId}/survey/{surveyId}/observation/submission/upload.ts index 91d1a6b9be..6562030bd8 100644 --- a/api/src/paths/project/{projectId}/survey/{surveyId}/observation/submission/upload.ts +++ b/api/src/paths/project/{projectId}/survey/{surveyId}/observation/submission/upload.ts @@ -9,7 +9,7 @@ import { updateSurveyOccurrenceSubmissionWithKeySQL, getTemplateMethodologySpeciesIdSQLStatement } from '../../../../../../../queries/survey/survey-occurrence-queries'; -import { generateS3FileKey, uploadFileToS3 } from '../../../../../../../utils/file-utils'; +import { generateS3FileKey, scanFileForVirus, uploadFileToS3 } from '../../../../../../../utils/file-utils'; import { getLogger } from '../../../../../../../utils/logger'; import { logRequest } from '../../../../../../../utils/path-utils'; @@ -121,6 +121,13 @@ export function uploadMedia(): RequestHandler { await connection.open(); + // Scan file for viruses using ClamAV + const virusScanResult = await scanFileForVirus(rawMediaFile); + + if (!virusScanResult) { + throw new HTTP400('Malicious content detected, upload cancelled'); + } + const templateMethodologyId = await getTemplateMethodologySpeciesIdStatement( Number(req.params.surveyId), rawMediaFile.originalname, @@ -144,7 +151,7 @@ export function uploadMedia(): RequestHandler { fileName: rawMediaFile.originalname }); - //query to update the record with the key before uploading the file + // Query to update the record with the key before uploading the file await updateSurveyOccurrenceSubmissionWithKey(submissionId, key, connection); await connection.commit(); diff --git a/api/src/types/clamdjs.d.ts b/api/src/types/clamdjs.d.ts new file mode 100644 index 0000000000..d82f3d9e8f --- /dev/null +++ b/api/src/types/clamdjs.d.ts @@ -0,0 +1,7 @@ +declare module 'clamdjs' { + interface ClamScanner { + scanBuffer: (buffer: string | Buffer, timeout: number, chunkSize: number) => Promise; + } + + export function createScanner(host: string, port: number): ClamScanner; +} diff --git a/api/src/utils/file-utils.ts b/api/src/utils/file-utils.ts index 6bf6380097..f2ba0e5cc9 100644 --- a/api/src/utils/file-utils.ts +++ b/api/src/utils/file-utils.ts @@ -1,7 +1,12 @@ import AWS from 'aws-sdk'; import { DeleteObjectOutput, GetObjectOutput, ManagedUpload, Metadata } from 'aws-sdk/clients/s3'; import { S3_ROLE } from '../constants/roles'; +import clamd from 'clamdjs'; +const scanner = + process.env.ENABLE_FILE_VIRUS_SCAN === 'true' + ? clamd.createScanner(process.env.CLAMAV_HOST || 'clamav', Number(process.env.CLAMAV_PORT) || 3310) + : null; const OBJECT_STORE_BUCKET_NAME = process.env.OBJECT_STORE_BUCKET_NAME || ''; const OBJECT_STORE_URL = process.env.OBJECT_STORE_URL || 'nrs.objectstore.gov.bc.ca'; const AWS_ENDPOINT = new AWS.Endpoint(OBJECT_STORE_URL); @@ -128,3 +133,20 @@ export function generateS3FileKey(options: IS3FileKey): string { return keyParts.join('/'); } + +export async function scanFileForVirus(file: Express.Multer.File): Promise { + // if virus scan is not to be performed/cannot be performed + if (!scanner) { + return true; + } + + const clamavScanResult = await scanner.scanBuffer(file.buffer, 3000, 1024 * 1024); + + // if virus found in file + if (clamavScanResult.includes('FOUND')) { + return false; + } + + // no virus found in file + return true; +} diff --git a/api/tsconfig.json b/api/tsconfig.json index 2ea774f749..809871d206 100644 --- a/api/tsconfig.json +++ b/api/tsconfig.json @@ -2,6 +2,7 @@ "compilerOptions": { "module": "commonjs", "lib": ["es2018"], + "baseUrl": "src", "outDir": "dist", "target": "es5", "sourceMap": true, @@ -21,7 +22,10 @@ "isolatedModules": true, "noFallthroughCasesInSwitch": true, "strict": true, - "typeRoots": ["node_modules/@types"] + "typeRoots": ["src/types", "node_modules/@types"], + "paths": { + "*": ["types/*"] + } }, "include": ["src"] } diff --git a/app/src/components/attachments/DropZone.tsx b/app/src/components/attachments/DropZone.tsx index eb45663370..9c6dc763c0 100644 --- a/app/src/components/attachments/DropZone.tsx +++ b/app/src/components/attachments/DropZone.tsx @@ -75,7 +75,7 @@ export const DropZone: React.FC = (props) Drag your files here, or Browse Files - {!!maxFileSize && maxFileSize !== Infinity && ( + {acceptedFileExtensions && ( {`Accepted file types: ${acceptedFileExtensions}`} diff --git a/app/src/components/attachments/FileUploadItem.test.tsx b/app/src/components/attachments/FileUploadItem.test.tsx index e3b91890e2..e7a4f8e4b7 100644 --- a/app/src/components/attachments/FileUploadItem.test.tsx +++ b/app/src/components/attachments/FileUploadItem.test.tsx @@ -30,7 +30,7 @@ describe('FileUploadItem', () => { }); await waitFor(() => { - expect(mockUploadHandler).toHaveBeenCalledWith([testFile], expect.any(Object), expect.any(Function)); + expect(mockUploadHandler).toHaveBeenCalledWith(testFile, expect.any(Object), expect.any(Function)); expect(getByText('testpng.txt')).toBeVisible(); @@ -76,7 +76,7 @@ describe('FileUploadItem', () => { }); await waitFor(() => { - expect(mockUploadHandler).toHaveBeenCalledWith([testFile], expect.any(Object), expect.any(Function)); + expect(mockUploadHandler).toHaveBeenCalledWith(testFile, expect.any(Object), expect.any(Function)); expect(getByText('testpng.txt')).toBeVisible(); @@ -118,7 +118,7 @@ describe('FileUploadItem', () => { }); await waitFor(() => { - expect(mockUploadHandler).toHaveBeenCalledWith([testFile], expect.any(Object), expect.any(Function)); + expect(mockUploadHandler).toHaveBeenCalledWith(testFile, expect.any(Object), expect.any(Function)); expect(getByText('testpng.txt')).toBeVisible(); diff --git a/app/src/components/attachments/FileUploadItem.tsx b/app/src/components/attachments/FileUploadItem.tsx index cb4837e37d..41f74c97c4 100644 --- a/app/src/components/attachments/FileUploadItem.tsx +++ b/app/src/components/attachments/FileUploadItem.tsx @@ -65,7 +65,7 @@ export interface IUploadFile { } export type IUploadHandler = ( - files: File[], + file: File, cancelToken: CancelTokenSource, handleFileUploadProgress: (progressEvent: ProgressEvent) => void ) => Promise; @@ -139,8 +139,10 @@ const FileUploadItem: React.FC = (props) => { onSuccess?.(response); }; - uploadHandler([file], cancelToken, handleFileUploadProgress) - .then(handleFileUploadSuccess, (error: APIError) => setError(error?.message)) + uploadHandler(file, cancelToken, handleFileUploadProgress) + .then(handleFileUploadSuccess, (error: APIError) => { + setError(error?.message); + }) .catch(); setStatus(UploadFileStatus.UPLOADING); diff --git a/app/src/features/projects/view/ProjectAttachments.tsx b/app/src/features/projects/view/ProjectAttachments.tsx index 7972fab406..d2402e526b 100644 --- a/app/src/features/projects/view/ProjectAttachments.tsx +++ b/app/src/features/projects/view/ProjectAttachments.tsx @@ -51,8 +51,8 @@ const ProjectAttachments: React.FC = () => { ); const uploadAttachments = (): IUploadHandler => { - return (files, cancelToken, handleFileUploadProgress) => { - return biohubApi.project.uploadProjectAttachments(projectId, files, cancelToken, handleFileUploadProgress); + return (file, cancelToken, handleFileUploadProgress) => { + return biohubApi.project.uploadProjectAttachments(projectId, file, cancelToken, handleFileUploadProgress); }; }; @@ -70,7 +70,13 @@ const ProjectAttachments: React.FC = () => { getAttachments(true); setOpenUploadAttachments(false); }}> - + diff --git a/app/src/features/surveys/view/SurveyAttachments.tsx b/app/src/features/surveys/view/SurveyAttachments.tsx index 277c6880bd..59e51fc66a 100644 --- a/app/src/features/surveys/view/SurveyAttachments.tsx +++ b/app/src/features/surveys/view/SurveyAttachments.tsx @@ -54,14 +54,8 @@ const SurveyAttachments: React.FC = () => { ); const uploadAttachments = (): IUploadHandler => { - return (files, cancelToken, handleFileUploadProgress) => { - return biohubApi.survey.uploadSurveyAttachments( - projectId, - surveyId, - files, - cancelToken, - handleFileUploadProgress - ); + return (file, cancelToken, handleFileUploadProgress) => { + return biohubApi.survey.uploadSurveyAttachments(projectId, surveyId, file, cancelToken, handleFileUploadProgress); }; }; diff --git a/app/src/features/surveys/view/SurveyObservations.tsx b/app/src/features/surveys/view/SurveyObservations.tsx index 486d3661a4..41ee7ad00d 100644 --- a/app/src/features/surveys/view/SurveyObservations.tsx +++ b/app/src/features/surveys/view/SurveyObservations.tsx @@ -84,9 +84,7 @@ const SurveyObservations = () => { const classes = useStyles(); const importObservations = (): IUploadHandler => { - return (files, cancelToken, handleFileUploadProgress) => { - const file = files[0]; - + return (file, cancelToken, handleFileUploadProgress) => { return biohubApi.observation .uploadObservationSubmission(projectId, surveyId, file, cancelToken, handleFileUploadProgress) .then((result) => { diff --git a/app/src/hooks/api/useProjectApi.test.ts b/app/src/hooks/api/useProjectApi.test.ts index fc10d30b6d..4dcc109c50 100644 --- a/app/src/hooks/api/useProjectApi.test.ts +++ b/app/src/hooks/api/useProjectApi.test.ts @@ -167,11 +167,11 @@ describe('useProjectApi', () => { type: 'text/plain' }); - mock.onPost(`/api/project/${projectId}/attachments/upload`).reply(200, ['result 1', 'result 2']); + mock.onPost(`/api/project/${projectId}/attachments/upload`).reply(200, 'result 1'); - const result = await useProjectApi(axios).uploadProjectAttachments(projectId, [file]); + const result = await useProjectApi(axios).uploadProjectAttachments(projectId, file); - expect(result).toEqual(['result 1', 'result 2']); + expect(result).toEqual('result 1'); }); it('createProject works as expected', async () => { diff --git a/app/src/hooks/api/useProjectApi.ts b/app/src/hooks/api/useProjectApi.ts index 30a3b42ae4..68f4bf6be5 100644 --- a/app/src/hooks/api/useProjectApi.ts +++ b/app/src/hooks/api/useProjectApi.ts @@ -154,20 +154,20 @@ const useProjectApi = (axios: AxiosInstance) => { * Upload project attachments. * * @param {number} projectId - * @param {File[]} files + * @param {File} file * @param {CancelTokenSource} [cancelTokenSource] * @param {(progressEvent: ProgressEvent) => void} [onProgress] * @return {*} {Promise} */ const uploadProjectAttachments = async ( projectId: number, - files: File[], + file: File, cancelTokenSource?: CancelTokenSource, onProgress?: (progressEvent: ProgressEvent) => void - ): Promise => { + ): Promise => { const req_message = new FormData(); - files.forEach((file) => req_message.append('media', file)); + req_message.append('media', file); const { data } = await axios.post(`/api/project/${projectId}/attachments/upload`, req_message, { cancelToken: cancelTokenSource?.token, diff --git a/app/src/hooks/api/useSurveyApi.test.ts b/app/src/hooks/api/useSurveyApi.test.ts index 3a8038e80f..558d3623b7 100644 --- a/app/src/hooks/api/useSurveyApi.test.ts +++ b/app/src/hooks/api/useSurveyApi.test.ts @@ -154,11 +154,11 @@ describe('useSurveyApi', () => { type: 'text/plain' }); - mock.onPost(`/api/project/${projectId}/survey/${surveyId}/attachments/upload`).reply(200, ['result 1', 'result 2']); + mock.onPost(`/api/project/${projectId}/survey/${surveyId}/attachments/upload`).reply(200, 'result 1'); - const result = await useSurveyApi(axios).uploadSurveyAttachments(projectId, surveyId, [file]); + const result = await useSurveyApi(axios).uploadSurveyAttachments(projectId, surveyId, file); - expect(result).toEqual(['result 1', 'result 2']); + expect(result).toEqual('result 1'); }); it('updateSurvey works as expected', async () => { diff --git a/app/src/hooks/api/useSurveyApi.ts b/app/src/hooks/api/useSurveyApi.ts index 0a6967f42b..97487bcd6f 100644 --- a/app/src/hooks/api/useSurveyApi.ts +++ b/app/src/hooks/api/useSurveyApi.ts @@ -98,7 +98,7 @@ const useSurveyApi = (axios: AxiosInstance) => { * * @param {number} projectId * @param {number} surveyId - * @param {File[]} files + * @param {File} file * @param {CancelTokenSource} [cancelTokenSource] * @param {(progressEvent: ProgressEvent) => void} [onProgress] * @return {*} {Promise} @@ -106,13 +106,13 @@ const useSurveyApi = (axios: AxiosInstance) => { const uploadSurveyAttachments = async ( projectId: number, surveyId: number, - files: File[], + file: File, cancelTokenSource?: CancelTokenSource, onProgress?: (progressEvent: ProgressEvent) => void - ): Promise => { + ): Promise => { const req_message = new FormData(); - files.forEach((file) => req_message.append('media', file)); + req_message.append('media', file); const { data } = await axios.post(`/api/project/${projectId}/survey/${surveyId}/attachments/upload`, req_message, { cancelToken: cancelTokenSource?.token, diff --git a/containers/clamav/README.md b/containers/clamav/README.md index 4c8a87669e..3bf7bfd384 100644 --- a/containers/clamav/README.md +++ b/containers/clamav/README.md @@ -12,7 +12,3 @@ Freshclam can be run within the container at any time to update the existing sig # Deployment The templates in the [openshift/templates](./openshift/templates) will build and deploy the app. Modify to suit your own environment. [openshift/templates/clamav-bc.yaml](./openshift/templates/clamav-bc.yaml) will create your builder image (ideally in your tools project), and [openshift/templates/clamav-dc.yaml](./openshift/templates/clamav-dc.yaml) will create the pod deployment. Modify the environment variables defined in both the build config and deployment config appropriately. - -# Client - -Sample client files can be found [here](./client) they are based on [clamav.js](https://github.com/yongtang/clamav.js). diff --git a/containers/clamav/client/README.md b/containers/clamav/client/README.md deleted file mode 100644 index 10581ba86a..0000000000 --- a/containers/clamav/client/README.md +++ /dev/null @@ -1,15 +0,0 @@ -# Running the Client - -To run the client against the ClamAV server, your first need to `oc port-forward` the pods port to your local. - -``` -oc login -oc project -oc port-forward 3310:3310 -``` - -# Client Examples -* `ping.js`: Makes sure you can reach your server -* `getversion.js`: Reports the version of your server -* `scandirectory.js`: Scans a directory (and all below) -* `scanfile.js`: Scans a single file diff --git a/containers/clamav/client/getversion.js b/containers/clamav/client/getversion.js deleted file mode 100644 index c092fabf7d..0000000000 --- a/containers/clamav/client/getversion.js +++ /dev/null @@ -1,10 +0,0 @@ -var clamav=require('clamav.js'); - -clamav.version(3310, '127.0.0.1', 1000, function(err, version) { - if (err) { - console.log('Version is not available['+err+']'); - } - else { - console.log('Version is ['+version+']'); - } -}); \ No newline at end of file diff --git a/containers/clamav/client/package-lock.json b/containers/clamav/client/package-lock.json deleted file mode 100644 index b2d565a13f..0000000000 --- a/containers/clamav/client/package-lock.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "name": "client", - "lockfileVersion": 2, - "requires": true, - "packages": { - "": { - "dependencies": { - "clamav.js": "^0.12.0" - } - }, - "node_modules/clamav.js": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/clamav.js/-/clamav.js-0.12.0.tgz", - "integrity": "sha1-9lTr0tEMcHy199RFOIWs2I0nMfc=" - } - }, - "dependencies": { - "clamav.js": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/clamav.js/-/clamav.js-0.12.0.tgz", - "integrity": "sha1-9lTr0tEMcHy199RFOIWs2I0nMfc=" - } - } -} diff --git a/containers/clamav/client/package.json b/containers/clamav/client/package.json deleted file mode 100644 index 7663382bc8..0000000000 --- a/containers/clamav/client/package.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "dependencies": { - "clamav.js": "^0.12.0" - } -} diff --git a/containers/clamav/client/ping.js b/containers/clamav/client/ping.js deleted file mode 100644 index e261bead8f..0000000000 --- a/containers/clamav/client/ping.js +++ /dev/null @@ -1,10 +0,0 @@ -var clamav=require('clamav.js'); - -clamav.ping(3310, '127.0.0.1', 1000, function(err) { - if (err) { - console.log('127.0.0.1:3310 is not available['+err+']'); - } - else { - console.log('127.0.0.1:3310 is alive'); - } -}); \ No newline at end of file diff --git a/containers/clamav/client/sample_files/RM_doggo.png b/containers/clamav/client/sample_files/RM_doggo.png deleted file mode 100644 index 3c9105577f..0000000000 Binary files a/containers/clamav/client/sample_files/RM_doggo.png and /dev/null differ diff --git a/containers/clamav/client/scandirectory.js b/containers/clamav/client/scandirectory.js deleted file mode 100644 index 3496fee1de..0000000000 --- a/containers/clamav/client/scandirectory.js +++ /dev/null @@ -1,14 +0,0 @@ -var clamav=require('clamav.js'); - -clamav.createScanner(3310, '127.0.0.1').scan('sample_files' - , function(err, object, malicious) { - if (err) { - console.log(object+': '+err); - } - else if (malicious) { - console.log(object+': '+malicious+' FOUND'); - } - else { - console.log(object+': OK'); - } -}); \ No newline at end of file diff --git a/containers/clamav/client/scanfile.js b/containers/clamav/client/scanfile.js deleted file mode 100644 index 4e965c3b52..0000000000 --- a/containers/clamav/client/scanfile.js +++ /dev/null @@ -1,16 +0,0 @@ -var fs=require('fs'); -var clamav=require('clamav.js'); - -var stream = fs.createReadStream('scandirectory.js'); -clamav.createScanner(3310, '127.0.0.1').scan(stream - , function(err, object, malicious) { - if (err) { - console.log(object.path+': '+err); - } - else if (malicious) { - console.log(object.path+': '+malicious+' FOUND'); - } - else { - console.log(object.path+': OK'); - } -}); \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 1e5db2940c..182cc862a4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -59,6 +59,9 @@ services: - MAX_UPLOAD_NUM_FILES=${MAX_UPLOAD_NUM_FILES} - MAX_UPLOAD_FILE_SIZE=${MAX_UPLOAD_FILE_SIZE} - LOG_LEVEL=${LOG_LEVEL} + - CLAMAV_PORT=${CLAMAV_PORT} + - CLAMAV_HOST=${CLAMAV_HOST} + - ENABLE_FILE_VIRUS_SCAN=${ENABLE_FILE_VIRUS_SCAN} volumes: - ./api:/opt/app-root/src - /opt/app-root/src/node_modules # prevents local node_modules overriding container node_modules @@ -68,6 +71,17 @@ services: - db - db_setup + # Build the clamav docker image + clamav: + image: mkodockx/docker-clamav:latest + container_name: ${DOCKER_PROJECT_NAME}-clamav-${DOCKER_NAMESPACE}-container + ports: + - ${CLAMAV_PORT}:${CLAMAV_PORT} + networks: + - local-network + depends_on: + - api + # Build the n8n nginx proxy docker image n8n_nginx: image: ${DOCKER_PROJECT_NAME}-n8n-nginx-${DOCKER_NAMESPACE}-img diff --git a/env_config/env.docker b/env_config/env.docker index 0d59507767..de7f766434 100644 --- a/env_config/env.docker +++ b/env_config/env.docker @@ -89,3 +89,10 @@ N8N_BASIC_AUTH_ACTIVE=true N8N_BASIC_AUTH_USER=n8n N8N_BASIC_AUTH_PASSWORD=n8n N8N_ENCRYPTION_KEY=secret + +# ------------------------------------------------------------------------------ +# Clamav - Virus scanning +# ------------------------------------------------------------------------------ +CLAMAV_PORT=3310 +CLAMAV_HOST=clamav +ENABLE_FILE_VIRUS_SCAN=