small clean up

elastic · Apr 8, 2020 · 8dbd6c6 · 8dbd6c6
1 parent a89359f
commit 8dbd6c6
Show file tree

Hide file tree

Showing 4 changed files with 18 additions and 20 deletions.
diff --git a/.../public/application/datavisualizer/file_based/components/import_view/importer/importer.ts b/.../public/application/datavisualizer/file_based/components/import_view/importer/importer.ts
@@ -20,7 +20,7 @@ import {
 const CHUNK_SIZE = 5000;
 const MAX_CHUNK_CHAR_COUNT = 1000000;
 const IMPORT_RETRIES = 5;
-const STRING_CHUNKS_MB = 50;
+const STRING_CHUNKS_MB = 100;
 
 export interface ImportConfig {
   settings: Settings;
@@ -35,7 +35,7 @@ export interface ImportResults {
   error?: any;
 }
 
-export interface ReadResponse {
+export interface CreateDocsResponse {
   success: boolean;
   remainder: number;
   docs: ImportDoc[];
@@ -56,9 +56,13 @@ export abstract class Importer {
   }
 
   public read(data: ArrayBuffer) {
-    const decoder = new TextDecoder('utf-8');
+    const decoder = new TextDecoder();
     const size = STRING_CHUNKS_MB * Math.pow(2, 20);
 
+    // chop the data up into 100MB chunks for processing.
+    // if the chop produces a partial line at the end, a character "remainder" count
+    // is returned which is used to roll the next chunk back that many chars so
+    // it is included in the next chunk.
     const parts = Math.ceil(data.byteLength / size);
     let remainder = 0;
     for (let i = 0; i < parts; i++) {
@@ -75,7 +79,7 @@ export abstract class Importer {
     return { success: true };
   }
 
-  protected abstract _createDocs(t: string): ReadResponse;
+  protected abstract _createDocs(t: string): CreateDocsResponse;
 
   public async initializeImport(index: string) {
     const settings = this._settings;

diff --git a/...application/datavisualizer/file_based/components/import_view/importer/message_importer.ts b/...application/datavisualizer/file_based/components/import_view/importer/message_importer.ts
@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { Importer, ImportConfig, ReadResponse } from './importer';
+import { Importer, ImportConfig, CreateDocsResponse } from './importer';
 import {
   Doc,
   FindFileStructureResponse,
@@ -33,7 +33,7 @@ export class MessageImporter extends Importer {
   // multiline_start_pattern regex
   // if it does, it is a legitimate end of line and can be pushed into the list,
   // if not, it must be a newline char inside a field value, so keep looking.
-  protected _createDocs(text: string): ReadResponse {
+  protected _createDocs(text: string): CreateDocsResponse {
     let remainder = 0;
     try {
       const docs: Doc[] = [];

diff --git a/.../application/datavisualizer/file_based/components/import_view/importer/ndjson_importer.ts b/.../application/datavisualizer/file_based/components/import_view/importer/ndjson_importer.ts
@@ -4,15 +4,15 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { Importer, ImportConfig } from './importer';
+import { Importer, ImportConfig, CreateDocsResponse } from './importer';
 import { FindFileStructureResponse } from '../../../../../../../common/types/file_datavisualizer';
 
 export class NdjsonImporter extends Importer {
   constructor(results: FindFileStructureResponse, settings: ImportConfig) {
     super(settings);
   }
 
-  protected _createDocs(json: string) {
+  protected _createDocs(json: string): CreateDocsResponse {
     let remainder = 0;
     try {
       const splitJson = json.split(/}\s*\n/);

diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/utils.ts b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/utils.ts
@@ -39,24 +39,18 @@ export function readFile(file: File) {
 
       reader.onload = (() => {
         return () => {
-          const decoder = new TextDecoder('utf-8');
-          if (reader.result === null || typeof reader.result === 'string') {
+          const decoder = new TextDecoder();
+          const data = reader.result;
+          if (data === null || typeof data === 'string') {
             return reject();
           }
           const size = UPLOAD_SIZE_MB * Math.pow(2, 20);
-          const fileContents = decoder.decode(reader.result.slice(0, size));
-          // let w = 0;
-          // const chunkSize = 250;
-          // let fileContents = '';
-          // while (w * chunkSize < reader.result.byteLength) {
-          //   const r = reader.result.slice(w * chunkSize, (w + 1) * chunkSize);
-          //   fileContents += decoder.decode(r);
-          //   w++;
-          // }
+          const fileContents = decoder.decode(data.slice(0, size));
+
           if (fileContents === '') {
             reject();
           } else {
-            resolve({ fileContents, data: reader.result });
+            resolve({ fileContents, data });
           }
         };
       })();