From 0f91e3ba8a6d0aa540ef55e80d5aded569e399a8 Mon Sep 17 00:00:00 2001 From: Athithyaa Selvam <33496652+athithyaaselvam@users.noreply.github.com> Date: Tue, 16 Apr 2024 12:01:18 -0700 Subject: [PATCH] Importer supports chunked file uploads (#3686) Change-Id: Icf0bdaac63599e947820b2fc5dd5ca8453038292 Co-authored-by: Athithyaa Selvam --- .../js/jquery/plugins/jquery.filechooser.js | 168 +++++++++++++----- 1 file changed, 124 insertions(+), 44 deletions(-) diff --git a/desktop/core/src/desktop/js/jquery/plugins/jquery.filechooser.js b/desktop/core/src/desktop/js/jquery/plugins/jquery.filechooser.js index 8508ab22493..d63fbb529f0 100644 --- a/desktop/core/src/desktop/js/jquery/plugins/jquery.filechooser.js +++ b/desktop/core/src/desktop/js/jquery/plugins/jquery.filechooser.js @@ -16,8 +16,8 @@ import $ from 'jquery'; import * as ko from 'knockout'; -import qq from 'ext/fileuploader.custom'; - +import fileuploader from 'ext/fileuploader.custom'; +import qq from 'ext/fileuploader.custom.new.js'; import { GLOBAL_ERROR_TOPIC, GLOBAL_INFO_TOPIC } from 'reactComponents/AlertComponent/events'; import huePubSub from 'utils/huePubSub'; import I18n from 'utils/i18n'; @@ -806,48 +806,128 @@ Plugin.prototype.navigateTo = function (path) { let num_of_pending_uploads = 0; function initUploader(path, _parent, el, labels) { - new qq.FileUploader({ - element: el[0], - action: '/filebrowser/upload/file', - params: { - dest: path, - fileFieldLabel: 'hdfs_file' - }, - onComplete: function (id, fileName, responseJSON) { - num_of_pending_uploads--; - if (responseJSON.status == -1) { - huePubSub.publish(GLOBAL_ERROR_TOPIC, { message: responseJSON.data }); - } else if (!num_of_pending_uploads) { - _parent.navigateTo(path); - huePubSub.publish('assist.' + getFs(getScheme(path)) + '.refresh'); - } - }, - onSubmit: function (id, fileName) { - num_of_pending_uploads++; - window.hueAnalytics.log('filechooser', 'uploading-file'); - }, - template: - '
' + - '
' + - '
' + - labels.UPLOAD_FILE + - '

' + - '' + - '
', - fileTemplate: - '
  • ' + - '' + - '' + - '' + - '' + - labels.CANCEL + - '' + - '' + - labels.FAILED + - '' + - '
    • ', - debug: false - }); + let uploader; + if (window.getLastKnownConfig().hue_config.enable_chunked_file_uploader) { + const action = '/filebrowser/upload/chunks/'; + const qqTemplate = document.createElement('div'); + qqTemplate.id = 'qq-template'; + qqTemplate.innerHTML = ` +
    +
    + ${'Drop the files here to upload'} +
    +
    +
    ${'Upload file'}
    +
    +
      +
    • + + + + + + ${'Failed'} +
      • +
    +
    +
    +
    + `; + document.body.appendChild(qqTemplate); + uploader = new qq.FileUploader({ + element: el[0], + request: { + endpoint: action, + paramsInBody: false, + params: { + dest: path, + inputName: 'hdfs_file' + } + }, + maxConnections: window.CONCURRENT_MAX_CONNECTIONS || 5, + chunking: { + enabled: true, + concurrent: { + enabled: true + }, + partSize: window.FILE_UPLOAD_CHUNK_SIZE || 5242880, + success: { + endpoint: '/filebrowser/upload/complete/' + }, + paramNames: { + partIndex: 'qqpartindex', + partByteOffset: 'qqpartbyteoffset', + chunkSize: 'qqchunksize', + totalFileSize: 'qqtotalfilesize', + totalParts: 'qqtotalparts' + } + }, + template: 'qq-template', + callbacks: { + onComplete: function (id, fileName, response) { + num_of_pending_uploads--; + if (response.status != 0) { + huePubSub.publish('hue.global.error', { message: response.data }); + } else if (!num_of_pending_uploads) { + _parent.navigateTo(path); + huePubSub.publish('assist.' + getFs(getScheme(path)) + '.refresh'); + } + }, + onSubmit: function (id, fileName) { + let newPath = + '/filebrowser/upload/chunks/file?dest=' + encodeURIComponent(path.normalize('NFC')); + this.setEndpoint(newPath); + num_of_pending_uploads++; + window.hueAnalytics.log('filechooser', 'uploading-file'); + } + }, + debug: false + }); + } else { + // eslint-disable-next-line no-unused-vars + uploader = new fileuploader.FileUploader({ + element: el[0], + action: '/filebrowser/upload/file', + params: { + dest: path, + fileFieldLabel: 'hdfs_file' + }, + onComplete: function (id, fileName, responseJSON) { + num_of_pending_uploads--; + if (responseJSON.status == -1) { + huePubSub.publish('hue.global.error', { message: responseJSON.data }); + } else if (!num_of_pending_uploads) { + _parent.navigateTo(path); + huePubSub.publish('assist.' + getFs(getScheme(path)) + '.refresh'); + } + }, + onSubmit: function (id, fileName) { + num_of_pending_uploads++; + window.hueAnalytics.log('filechooser', 'uploading-file'); + }, + template: + '
    ' + + '
    ' + + '
    ' + + labels.UPLOAD_FILE + + '

    ' + + '' + + '
    ', + fileTemplate: + '
  • ' + + '' + + '' + + '' + + '' + + labels.CANCEL + + '' + + '' + + labels.FAILED + + '' + + '
    • ', + debug: false + }); + } } Plugin.prototype.init = function () {