Updated File Upload to Auto Encode to UTF-8

tdrs-frontend/package.json

@@ -11,6 +11,7 @@
     "@uswds/uswds": "3.10.0",
     "axios": "^1.7.7",
     "classnames": "^2.5.1",
+    "detect-file-encoding-and-language": "^2.4.0",
     "file-type-checker": "^1.1.2",
     "history": "^5.3.0",
     "include-media": "^2.0.0",

tdrs-frontend/src/actions/reports.js

@@ -4,7 +4,6 @@ import axios from 'axios'
 import axiosInstance from '../axios-instance'
 import { logErrorToServer } from '../utils/eventLogger'
 import removeFileInputErrorState from '../utils/removeFileInputErrorState'
-import { fileUploadSections } from '../reducers/reports'
 
 const BACKEND_URL = process.env.REACT_APP_BACKEND_URL
 

tdrs-frontend/src/components/FileUpload/FileUpload.jsx

@@ -2,6 +2,7 @@ import React, { useRef, useEffect } from 'react'
 import PropTypes from 'prop-types'
 import { useDispatch, useSelector } from 'react-redux'
 import fileTypeChecker from 'file-type-checker'
+import languageEncoding from 'detect-file-encoding-and-language'
 
 import {
   clearError,
@@ -34,6 +35,86 @@ const INVALID_EXT_ERROR = (
   </>
 )
 
+// The package author suggests using a minimum of 500 words to determine the encoding. However, datafiles don't have
+// "words" so we're using bytes instead to determine the encoding. See: https://www.npmjs.com/package/detect-file-encoding-and-language
+const MIN_BYTES = 500
+
+/* istanbul ignore next */
+const tryGetUTF8EncodedFile = async function (fileBytes, file) {
+  // Create a small view of the file to determine the encoding.
+  const btyesView = new Uint8Array(fileBytes.slice(0, MIN_BYTES))
+  const blobView = new Blob([btyesView], { type: 'text/plain' })
+  try {
+    const fileInfo = await languageEncoding(blobView)
+    const bom = btyesView.slice(0, 3)
+    const hasBom = bom[0] === 0xef && bom[1] === 0xbb && bom[2] === 0xbf
+    if ((fileInfo && fileInfo.encoding !== 'UTF-8') || hasBom) {
+      const utf8Encoder = new TextEncoder()
+      const decoder = new TextDecoder(fileInfo.encoding)
+      const decodedString = decoder.decode(
+        hasBom ? fileBytes.slice(3) : fileBytes
+      )
+      const utf8Bytes = utf8Encoder.encode(decodedString)
+      return new File([utf8Bytes], file.name, file.options)
+    }
+    return file
+  } catch (error) {
+    // This is a last ditch fallback to ensure consistent functionality and also allows the unit tests to work in the
+    // same way they did before this change. When the unit tests (i.e. Node environment) call `languageEncoding` it
+    // expects a Buffer/string/URL object. When the browser calls `languageEncoding`, it expects a Blob/File object.
+    // There is not a convenient way or universal object to handle both cases. Thus, when the tests run the call to
+    // `languageEncoding`, it raises an exception and we return the file as is which is then dispatched as it would
+    // have been before this change.
+    console.error('Caught error while handling file encoding. Error:', error)
+    return file
+  }
+}
+
+const load = (file, section, input, dropTarget, dispatch) => {
+  const filereader = new FileReader()
+  const types = ['png', 'gif', 'jpeg']
+
+  return new Promise((resolve, reject) => {
+    filereader.onerror = () => {
+      filereader.abort()
+      reject(new Error('Problem parsing input file.'))
+    }
+
+    filereader.onload = () => {
+      let error = false
+      const re = /(\.txt|\.ms\d{2}|\.ts\d{2,3})$/i
+      if (!re.exec(file.name)) {
+        dispatch({
+          type: FILE_EXT_ERROR,
+          payload: {
+            error: { message: INVALID_EXT_ERROR },
+            section,
+          },
+        })
+        error = true
+      }
+
+      const isImg = fileTypeChecker.validateFileType(filereader.result, types)
+
+      if (!error && isImg) {
+        createFileInputErrorState(input, dropTarget)
+
+        dispatch({
+          type: SET_FILE_ERROR,
+          payload: {
+            error: { message: INVALID_FILE_ERROR },
+            section,
+          },
+        })
+        error = true
+      }
+
+      resolve({ result: filereader.result, error: error })
+    }
+    filereader.readAsArrayBuffer(file)
+  })
+}
+
 function FileUpload({ section, setLocalAlertState }) {
   // e.g. 'Aggregate Case Data' => 'aggregate-case-data'
   // The set of uploaded files in our Redux state
@@ -86,7 +167,7 @@ function FileUpload({ section, setLocalAlertState }) {
   }
   const inputRef = useRef(null)
 
-  const validateAndUploadFile = (event) => {
+  const validateAndUploadFile = async (event) => {
     setLocalAlertState({
       active: false,
       type: null,
@@ -101,51 +182,14 @@ function FileUpload({ section, setLocalAlertState }) {
     dispatch(clearError({ section }))
     dispatch(clearFile({ section }))
 
-    // Get the the first 4 bytes of the file with which to check file signatures
-    const blob = file.slice(0, 4)
-
     const input = inputRef.current
     const dropTarget = inputRef.current.parentNode
 
-    const filereader = new FileReader()
-
-    const types = ['png', 'gif', 'jpeg']
-    filereader.onload = () => {
-      const re = /(\.txt|\.ms\d{2}|\.ts\d{2,3})$/i
-      if (!re.exec(file.name)) {
-        dispatch({
-          type: FILE_EXT_ERROR,
-          payload: {
-            error: { message: INVALID_EXT_ERROR },
-            section,
-          },
-        })
-        return
-      }
-
-      const isImg = fileTypeChecker.validateFileType(filereader.result, types)
-
-      if (isImg) {
-        createFileInputErrorState(input, dropTarget)
-
-        dispatch({
-          type: SET_FILE_ERROR,
-          payload: {
-            error: { message: INVALID_FILE_ERROR },
-            section,
-          },
-        })
-      } else {
-        dispatch(
-          upload({
-            section,
-            file,
-          })
-        )
-      }
-    }
+    const { result } = await load(file, section, input, dropTarget, dispatch)
 
-    filereader.readAsArrayBuffer(blob)
+    // Get the correctly encoded file
+    const encodedFile = await tryGetUTF8EncodedFile(result, file)
+    dispatch(upload({ file: encodedFile, section }))
   }
 
   return (

tdrs-frontend/src/components/Reports/Reports.test.js

@@ -323,7 +323,11 @@ describe('Reports', () => {
         },
       })
     })
-    expect(store.dispatch).toHaveBeenCalledTimes(14)
+    await waitFor(() => expect(getByText('section1.txt')).toBeInTheDocument())
+    await waitFor(() => expect(getByText('section2.txt')).toBeInTheDocument())
+    await waitFor(() => expect(getByText('section3.txt')).toBeInTheDocument())
+    await waitFor(() => expect(getByText('section4.txt')).toBeInTheDocument())
+    expect(store.dispatch).toHaveBeenCalledTimes(18)
 
     // There should be 4 more dispatches upon making the submission,
     // one request to /reports for each file
@@ -524,10 +528,12 @@ describe('Reports', () => {
       })
 
       // add a file to be uploaded, but don't submit
-      fireEvent.change(getByLabelText('Section 1 - Active Case Data'), {
-        target: {
-          files: [makeTestFile('section1.txt')],
-        },
+      await waitFor(() => {
+        fireEvent.change(getByLabelText('Section 1 - Active Case Data'), {
+          target: {
+            files: [makeTestFile('section1.txt')],
+          },
+        })
       })
 
       await waitFor(() => expect(getByText('section1.txt')).toBeInTheDocument())
@@ -566,10 +572,12 @@ describe('Reports', () => {
       })
 
       // add a file to be uploaded, but don't submit
-      fireEvent.change(getByLabelText('Section 1 - Active Case Data'), {
-        target: {
-          files: [makeTestFile('section1.txt')],
-        },
+      await waitFor(() => {
+        fireEvent.change(getByLabelText('Section 1 - Active Case Data'), {
+          target: {
+            files: [makeTestFile('section1.txt')],
+          },
+        })
       })
 
       await waitFor(() => expect(getByText('section1.txt')).toBeInTheDocument())
@@ -617,10 +625,12 @@ describe('Reports', () => {
       })
 
       // add a file to be uploaded, but don't submit
-      fireEvent.change(getByLabelText('Section 1 - Active Case Data'), {
-        target: {
-          files: [makeTestFile('section1.txt')],
-        },
+      await waitFor(() => {
+        fireEvent.change(getByLabelText('Section 1 - Active Case Data'), {
+          target: {
+            files: [makeTestFile('section1.txt')],
+          },
+        })
       })
 
       await waitFor(() => expect(getByText('section1.txt')).toBeInTheDocument())