feat(upload): add posibility to disable s3 integrity check (md5 gener… (

#299) * feat(upload): add posibility to disable s3 integrity check (md5 generation) * refactor(picker): remove disableIntegrityCheck from wrong pleace in picker options * fix(storeOptions): add disableStorageKey to validator * fix(picker): fix problem with undefined name after crop file from cloud
filestack · Oct 29, 2019 · 207e4c7 · 207e4c7
1 parent bc29f1a
commit 207e4c7
Show file tree

Hide file tree

Showing 15 changed files with 202 additions and 57 deletions.
diff --git a/manual_tests/upload_new.ts b/manual_tests/upload_new.ts
@@ -20,16 +20,16 @@ import { Client } from './../src/lib/client';
 import * as Path from 'path';
 // import * as Sentry from '@sentry/node';
 
-// import { S3Uploader } from './../src/lib/api/upload/uploaders/s3';
-// import { getFile } from '../src/lib/api/upload';
+import { S3Uploader } from './../src/lib/api/upload/uploaders/s3';
+import { getFile } from '../src/lib/api/upload';
 
 const createFile = (size = 44320) => Buffer.alloc(size);
 // Sentry.init({ dsn: 'DSN' });
 
-const fs = new Client(process.env.API_KEY);
-fs.on('upload.error', (e) => {
-  console.log('uploadError', e);
-});
+// const fs = new Client(process.env.API_KEY);
+// fs.on('upload.error', (e) => {
+//   console.log('uploadError', e);
+// });
 
 // fs.multiupload(
 //   [
@@ -46,24 +46,27 @@ fs.on('upload.error', (e) => {
 //   console.dir(res, { depth: null });
 // });
 
-try {
-  fs.upload('./upload.js').then((res) => {
-    console.info('Upload done!', res);
-  });
-} catch (e) {
-  console.log(e.details);
-}
+// try {
+//   fs.upload('./upload.js').then((res) => {
+//     console.info('Upload done!', res);
+//   });
+// } catch (e) {
+//   console.log(e.details);
+// }
 
-// (async () => {
-//   const file = await getFile(createFile());
-//   const u = new S3Uploader({});
-//   u.setUrl('https://upload.rc.filepickerapp.com');
-//   u.setApikey('AbHASoTdORfqk8APJB72Wz');
-//   u.addFile(file);
+(async () => {
+  const file = await getFile(createFile());
+  file.name = 'test.txt';
 
-//   const res = await u.execute().catch((e) => {
-//     console.log('ERROR', e);
-//   });
+  const u = new S3Uploader({});
+  u.setUrl('https://upload.filestackapi.com');
+  u.setApikey('AHvhedybhQMqZOqRvZquez');
+  u.setIntegrityCheck(false);
+  u.addFile(file);
+
+  const res = await u.execute().catch((e) => {
+    console.log('ERROR', e);
+  });
 
-//   console.log(res);
-// })();
+  console.log(res);
+})();
diff --git a/package-lock.json b/package-lock.json
diff --git a/src/config.ts b/src/config.ts
@@ -18,7 +18,7 @@
 /**
  * @private
  */
-const PICKER_VERSION = '1.9.3';
+const PICKER_VERSION = '1.9.4';
 
 /**
  * @private

diff --git a/src/lib/api/upload/file.spec.ts b/src/lib/api/upload/file.spec.ts
@@ -97,6 +97,14 @@ describe('Api/Upload/File', () => {
     expect(part.md5).toEqual('Vp73JkK+D63XEdakaNaO4Q==');
   });
 
+  it('should not calc chunk md5 on disable param', async () => {
+    const meta = file.getPartMetadata(0, 2);
+    const part = await file.getPartByMetadata(meta, false);
+
+    expect(part.size).toEqual(2);
+    expect(part.md5).toEqual(undefined);
+  });
+
   it('should return chunk by part metadata and offset', async () => {
     const meta = file.getPartMetadata(0, 4);
     const chunk = await file.getChunkByMetadata(meta, 1, 2);
@@ -107,6 +115,16 @@ describe('Api/Upload/File', () => {
     expect(chunk.endByte).toEqual(3);
   });
 
+  it('should not calc chunk md5 on disable param', async () => {
+    const meta = file.getPartMetadata(0, 4);
+    const chunk = await file.getChunkByMetadata(meta, 1, 2, false);
+
+    expect(chunk.size).toEqual(2);
+    expect(chunk.md5).toEqual(undefined);
+    expect(chunk.startByte).toEqual(1);
+    expect(chunk.endByte).toEqual(3);
+  });
+
   it('should release file buffer', () => {
     file.release();
 

diff --git a/src/lib/api/upload/file.ts b/src/lib/api/upload/file.ts
@@ -41,7 +41,7 @@ export interface FilePartMetadata {
 
 export interface FilePart extends FilePartMetadata {
   buffer: Buffer | ArrayBuffer;
-  md5: string;
+  md5?: string;
 }
 
 export interface FileChunk extends FilePart {
@@ -176,7 +176,7 @@ export class File {
    * @returns {FilePartMetadata}
    * @memberof File
    */
-  public getPartMetadata (partNum: number, size): FilePartMetadata {
+  public getPartMetadata (partNum: number, size: number): FilePartMetadata {
     const startByte = size * partNum;
 
     if (startByte > this._file.size) {
@@ -200,13 +200,13 @@ export class File {
    * @returns {FilePart}
    * @memberof File
    */
-  public async getPartByMetadata(meta: FilePartMetadata): Promise<FilePart> {
+  public async getPartByMetadata(meta: FilePartMetadata, md5Enabled: boolean = true): Promise<FilePart> {
     let slice = await this._file.slice(meta.startByte, meta.endByte);
 
     return Promise.resolve({
       ...meta,
       buffer: slice,
-      md5: md5(slice),
+      md5: md5Enabled ? md5(slice) : undefined,
     });
   }
 
@@ -219,7 +219,7 @@ export class File {
    * @returns {FilePart}
    * @memberof File
    */
-  public async getChunkByMetadata(meta: FilePartMetadata, offset: number, chunkSize: number): Promise<FileChunk> {
+  public async getChunkByMetadata(meta: FilePartMetadata, offset: number, chunkSize: number, md5Enabled: boolean = true): Promise<FileChunk> {
     const startByte = meta.startByte + offset;
     const endByte = Math.min(startByte + chunkSize, meta.endByte);
 
@@ -228,7 +228,7 @@ export class File {
     return Promise.resolve({
       ...meta,
       buffer: slice,
-      md5: md5(slice),
+      md5: md5Enabled ? md5(slice) : undefined,
       size: slice.byteLength,
       startByte,
       endByte,

diff --git a/src/lib/api/upload/types.ts b/src/lib/api/upload/types.ts
@@ -66,6 +66,12 @@ export interface UploadOptions {
    * Set the default intiial chunk size for Intelligent Ingestion. Defaults to 8MB on desktop and 1MB on mobile.
    */
   intelligentChunkSize?: number;
+
+  /**
+   * Disable checking integrity of uploaded files.
+   * On slower devices it can boost upload performance (disable counting md5 from file parts)
+   */
+  disableIntegrityCheck?: boolean;
 }
 
 export type StoreUploadOptions = StoreBaseParams & {

diff --git a/src/lib/api/upload/upload.spec.ts b/src/lib/api/upload/upload.spec.ts
@@ -98,6 +98,11 @@ describe('Api/Upload/upload', () => {
       expect(S3Uploader.prototype.setUploadMode).toHaveBeenCalledWith(UploadMode.INTELLIGENT);
     });
 
+    it('should set respect disableIntegrityCheck param', () => {
+      const u = new Upload({ disableIntegrityCheck: true });
+      expect(S3Uploader.prototype.setIntegrityCheck).toHaveBeenCalledWith(false);
+    });
+
     it('should fallback upload mode', () => {
       const u = new Upload({ intelligent: 'fallback' });
 

diff --git a/src/lib/api/upload/upload.ts b/src/lib/api/upload/upload.ts
@@ -107,6 +107,10 @@ export class Upload extends EventEmitter {
       this.uploader.setIntelligentChunkSize(options.intelligentChunkSize);
     }
 
+    if (options.disableIntegrityCheck) {
+      this.uploader.setIntegrityCheck(false);
+    }
+
     if (options.intelligent) {
       this.uploader.setUploadMode(options.intelligent === 'fallback' ? UploadMode.FALLBACK : UploadMode.INTELLIGENT);
     }

diff --git a/src/lib/api/upload/uploaders/abstract.ts b/src/lib/api/upload/uploaders/abstract.ts
@@ -67,6 +67,7 @@ export abstract class UploaderAbstract extends EventEmitter {
 
   protected isModeLocked: boolean = false; // if account does not support ii in fallback mode we should abort
   protected retryConfig: RetryConfig;
+  protected integrityCheck: boolean = true;
 
   constructor(protected readonly storeOptions: StoreUploadOptions, protected readonly concurrency: number = 3) {
     super();
@@ -95,6 +96,14 @@ export abstract class UploaderAbstract extends EventEmitter {
     this.url = url;
   }
 
+  /**
+   * Set state of checking file integrity
+   * @param state
+   */
+  public setIntegrityCheck(state) {
+    this.integrityCheck = state;
+  }
+
   /**
    * Sets upload mode
    *