From a36f7f2fbda43f27d38a736ea7d2b4be9dee3d4e Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Thu, 4 May 2023 10:30:03 +0200 Subject: [PATCH 01/24] Update zip4j to 2.11.5 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 28ad78c0..0b23fcb2 100644 --- a/pom.xml +++ b/pom.xml @@ -280,7 +280,7 @@ net.lingala.zip4j zip4j - 2.9.1 + 2.11.5 From 50acf41231cd5d50fcbdd1b5dd98c72971bb641a Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Thu, 4 May 2023 10:31:40 +0200 Subject: [PATCH 02/24] Update apache Ivy to 2.5.1 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0b23fcb2..0a70a2b6 100644 --- a/pom.xml +++ b/pom.xml @@ -227,7 +227,7 @@ org.apache.ivy ivy - 2.5.0 + 2.5.1 From 330d0ed1d5d0f0da61a6cdb03aab86dc6a86d4da Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Thu, 4 May 2023 10:34:19 +0200 Subject: [PATCH 03/24] Update spring-security-crypto 5.8.3 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0a70a2b6..729285d9 100644 --- a/pom.xml +++ b/pom.xml @@ -302,7 +302,7 @@ org.springframework.security spring-security-crypto - 5.6.3 + 5.8.3 From 27d25cfced9247d7a3f0ce4f3d9311fbdf4e99a7 Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Thu, 4 May 2023 10:36:45 +0200 Subject: [PATCH 04/24] Set image to Ubuntu 20.04 for workflow execution --- .github/workflows/create-release.yml | 2 +- .github/workflows/maven-test.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 9f24c968..089b5fcc 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -7,7 +7,7 @@ on: jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/maven-test.yml b/.github/workflows/maven-test.yml index bc8376b5..32f53a03 100644 --- a/.github/workflows/maven-test.yml +++ b/.github/workflows/maven-test.yml @@ -8,7 +8,7 @@ on: [push, pull_request] jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 From ec088bb32b12c335b31215d56e98947ea54230c9 Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Thu, 4 May 2023 10:37:39 +0200 Subject: [PATCH 05/24] Switch to alternative Restlet repository --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 729285d9..33690186 100644 --- a/pom.xml +++ b/pom.xml @@ -69,7 +69,7 @@ maven-restlet Restlet Framework repository - https://maven.restlet.org + https://maven.restlet.talend.com From 0103838d11b4a86f23575a7d36fbe6a5fc978c9c Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Thu, 4 May 2023 12:50:13 +0200 Subject: [PATCH 06/24] Remove url-based uploads --- pom.xml | 11 - .../job/submit/controls/folder-pattern.stache | 13 +- .../core/job/submit/controls/folder.stache | 23 -- .../core/job/submit/dialogs/http.stache | 6 - .../core/job/submit/dialogs/s3.stache | 6 - .../core/job/submit/dialogs/sftp.stache | 17 -- .../components/core/job/submit/submit.js | 204 +----------------- .../server/controller/ImportController.java | 56 +---- .../mapred/api/v2/jobs/SubmitJobTest.java | 43 ---- .../cloudgene/mapred/util/TestSFTPServer.java | 64 ------ 10 files changed, 8 insertions(+), 435 deletions(-) delete mode 100644 src/main/html/webapp/components/core/job/submit/dialogs/http.stache delete mode 100644 src/main/html/webapp/components/core/job/submit/dialogs/s3.stache delete mode 100644 src/main/html/webapp/components/core/job/submit/dialogs/sftp.stache delete mode 100644 src/test/java/cloudgene/mapred/util/TestSFTPServer.java diff --git a/pom.xml b/pom.xml index 33690186..1e8aef3d 100644 --- a/pom.xml +++ b/pom.xml @@ -271,11 +271,6 @@ - - com.jcraft - jsch - 0.1.55 - net.lingala.zip4j @@ -380,12 +375,6 @@ - - org.apache.sshd - sshd-sftp - 0.9.0 - test - diff --git a/src/main/html/webapp/components/core/job/submit/controls/folder-pattern.stache b/src/main/html/webapp/components/core/job/submit/controls/folder-pattern.stache index 6441b20b..727d8755 100644 --- a/src/main/html/webapp/components/core/job/submit/controls/folder-pattern.stache +++ b/src/main/html/webapp/components/core/job/submit/controls/folder-pattern.stache @@ -6,16 +6,11 @@
- Please define a glob pattern to filter or pair your files:
- {{#is(source,'upload')}} @@ -25,12 +20,8 @@ style="display:none;" multiple > Multiple files can be selected by using the ctrl / cmd or shift keys. - {{/is}} - {{#is(source,'s3')}} - - - {{/is}} - {{#if(details)}} + + {{#if(details)}} {{details}} {{/if}}
diff --git a/src/main/html/webapp/components/core/job/submit/controls/folder.stache b/src/main/html/webapp/components/core/job/submit/controls/folder.stache index d2a6dfa8..82b853da 100644 --- a/src/main/html/webapp/components/core/job/submit/controls/folder.stache +++ b/src/main/html/webapp/components/core/job/submit/controls/folder.stache @@ -6,17 +6,10 @@
-
- {{#is(source,'upload')}} @@ -26,22 +19,6 @@ style="display:none;" multiple > Multiple files can be selected by using the ctrl / cmd or shift keys. - {{/is}} - {{#is(source,'http')}} - - - {{/is}} - {{#is(source,'sftp')}} - - - {{/is}} - {{#is(source,'s3')}} - - - {{/is}} - {{#if(details)}} - {{details}} - {{/if}}
Please choose one or more files.
diff --git a/src/main/html/webapp/components/core/job/submit/dialogs/http.stache b/src/main/html/webapp/components/core/job/submit/dialogs/http.stache deleted file mode 100644 index 70d23f8e..00000000 --- a/src/main/html/webapp/components/core/job/submit/dialogs/http.stache +++ /dev/null @@ -1,6 +0,0 @@ -

Import data from public URLs

-

Please enter your URLs.

-
- - To specify more than one url, separate the urls with a space or new line. -
diff --git a/src/main/html/webapp/components/core/job/submit/dialogs/s3.stache b/src/main/html/webapp/components/core/job/submit/dialogs/s3.stache deleted file mode 100644 index 570774a8..00000000 --- a/src/main/html/webapp/components/core/job/submit/dialogs/s3.stache +++ /dev/null @@ -1,6 +0,0 @@ -

Import data from S3 bucket

-

Please enter the S3 bucket of your input folder.

-
- - Enter a the location of a s3 folder or file. To specify more than one file, separate the files with a space or new line. -
diff --git a/src/main/html/webapp/components/core/job/submit/dialogs/sftp.stache b/src/main/html/webapp/components/core/job/submit/dialogs/sftp.stache deleted file mode 100644 index 8d1e5556..00000000 --- a/src/main/html/webapp/components/core/job/submit/dialogs/sftp.stache +++ /dev/null @@ -1,17 +0,0 @@ -

Secure File Transfer Protocol (SFTP)

-

Please enter your SFTP connection informations.

-
-
- - -
-
- - -
-
- - - To specify more than one file, separate the files with a space or new line. -
-
diff --git a/src/main/html/webapp/components/core/job/submit/submit.js b/src/main/html/webapp/components/core/job/submit/submit.js index 50a954be..596f2421 100644 --- a/src/main/html/webapp/components/core/job/submit/submit.js +++ b/src/main/html/webapp/components/core/job/submit/submit.js @@ -9,9 +9,6 @@ import ErrorPage from 'helpers/error-page'; import Application from 'models/application'; import template from './submit.stache'; -import templateS3Dialog from './dialogs/s3.stache'; -import templateHttpDialog from './dialogs/http.stache'; -import templateSftpDialog from './dialogs/sftp.stache'; import templateUploadingDialog from './dialogs/uploading.stache'; import templateLabel from './controls/label.stache'; import templateSelect from './controls/select.stache'; @@ -146,9 +143,9 @@ export default Control.extend({ // custom file upload controls for single files -'.select-control change': function(){ - this.application.updateBinding(); -}, + '.select-control change': function(){ + this.application.updateBinding(); + }, '#select-single-file-btn click': function(button) { // trigger click to open file dialog @@ -210,200 +207,7 @@ export default Control.extend({ fileUpload.parent().find("#select-files").show(); fileUpload.parent().find("#change-files").hide(); fileUpload.parent().find("#remove-all-files").hide(); - }, - - // custom handler for import urls - - '.folder-source change': function(source) { - - //delete filelist - var parent = $(source).parent(); - - var fileList = $(parent).find(".file-list"); - fileList.empty(); - - //update parameter source - var param = domData.get.call($(parent)[0], 'param'); - param.attr('source', $(source).val()); - }, - - '#add-urls-btn click': function(button) { - - var parent = $(button).parent(); - - var fileList = $(parent).find(".file-list"); - //fileList.empty(); - - var paramInputField = $(parent).find(".hidden-parameter"); - - - var urlDialog = bootbox.confirm( - templateHttpDialog({ - value: paramInputField.val() - }), - function(result) { - if (result) { - var urls = $('#urls').val(); - $.ajax({ - url: "api/v2/importer/files", - type: "POST", - data: { - input: urls - }, - success: function(data) { - fileList.empty(); - $.each(data, function(index, value) { - fileList.append('
  • ' + value["text"].toString() + '
    • '); - }); - - //update value - if (data.length > 0) { - paramInputField.val(urls); - urlDialog.modal('hide'); - } else { - paramInputField.val(""); - bootbox.alert("Error: No valid files found on the provided urls."); - } - - }, - error: function(message) { - bootbox.alert("Error: " + message.responseText); - } - }); - - return false; - } - }); - }, - - '#add-s3-btn click': function(button) { - - var parent = $(button).parent(); - - var fileList = $(parent).find(".file-list"); - //fileList.empty(); - - var paramInputField = $(parent).find(".hidden-parameter"); - - var urlDialog = bootbox.confirm( - templateS3Dialog(), - function(result) { - if (result) { - var buckets = $('#buckets').val(); - - var waitingDialog = bootbox.dialog({ - close: false, - message: '

    Connecting...

    ', - show: false - }); - - waitingDialog.on('shown.bs.modal', function() { - - $.ajax({ - url: "api/v2/importer/files", - type: "POST", - data: { - input: buckets - }, - - success: function(data) { - - waitingDialog.modal('hide'); - - fileList.empty(); - $.each(data, function(index, value) { - fileList.append('
  • ' + value["text"].toString() + '
    • '); - }); - - //update value - if (data.length > 0) { - paramInputField.val(buckets); - urlDialog.modal('hide'); - } else { - paramInputField.val(""); - bootbox.alert('

    Error: No valid files found on the provided urls. Please check your credentials and your file path.'); - } - - }, - error: function(message) { - waitingDialog.modal('hide'); - bootbox.alert('

    Error: ' + message.responseText + '

    '); - } - }); - - }); - - waitingDialog.modal('show'); - - return false; - } - }); - }, - - '#add-sftp-files-btn click': function(button) { - - var parent = $(button).parent(); - - var fileList = $(parent).find(".file-list"); - //fileList.empty(); - - var paramInputField = $(parent).find(".hidden-parameter"); - - var urlDialog = bootbox.confirm( - templateSftpDialog(), - function(result) { - if (result) { - var path = $('#path').val(); - var username = $('#username').val(); - var password = $('#password').val(); - - var waitingDialog = bootbox.dialog({ - close: false, - message: '

    Connecting...

    ', - show: false - }); - - waitingDialog.on('shown.bs.modal', function() { - - $.ajax({ - url: "api/v2/importer/files", - type: "POST", - data: { - input: path + ';' + username + ';' + password - }, - - success: function(data) { - - waitingDialog.modal('hide'); - - fileList.empty(); - $.each(data, function(index, value) { - fileList.append('
  • ' + value["text"].toString() + '
    • '); - }); - - //update value - if (data.length > 0) { - paramInputField.val(path + ';' + username + ';' + password); - urlDialog.modal('hide'); - } else { - paramInputField.val(""); - bootbox.alert('

    Error: No valid files found on the provided urls. Please check your credentials and your file path.'); - } - - }, - error: function(message) { - waitingDialog.modal('hide'); - bootbox.alert('

    Error: ' + message.responseText + '

    '); - } - }); - - }); - - waitingDialog.modal('show'); - - return false; - } - }); + } }); diff --git a/src/main/java/cloudgene/mapred/server/controller/ImportController.java b/src/main/java/cloudgene/mapred/server/controller/ImportController.java index 42a00593..3dc057d9 100644 --- a/src/main/java/cloudgene/mapred/server/controller/ImportController.java +++ b/src/main/java/cloudgene/mapred/server/controller/ImportController.java @@ -1,12 +1,6 @@ package cloudgene.mapred.server.controller; -import java.util.List; -import java.util.Vector; - import cloudgene.mapred.server.exceptions.JsonHttpStatusException; -import genepi.hadoop.importer.FileItem; -import genepi.hadoop.importer.IImporter; -import genepi.hadoop.importer.ImporterFactory; import io.micronaut.http.HttpStatus; import io.micronaut.http.MediaType; import io.micronaut.http.annotation.Consumes; @@ -17,59 +11,13 @@ @Controller public class ImportController { - @Post("/api/v2/importer/files") @Consumes(MediaType.APPLICATION_FORM_URLENCODED) public JSONArray validateImport(String input) { - List results = new Vector(); - - try { - - if (ImporterFactory.needsImport(input)) { - - for (String url : ImporterFactory.parseImportString(input)) { - - try { - - IImporter importer = ImporterFactory.createImporter(url, null); - - if (importer != null) { - - List items = importer.getFiles(); - - if (items != null) { - - // add files to list - results.addAll(items); - - } else { - - - } - - } else { - throw new JsonHttpStatusException(HttpStatus.NOT_FOUND, "Protocol not supported"); - - } - - } catch (Exception e) { - throw new JsonHttpStatusException(HttpStatus.NOT_FOUND,e.toString()); - - } - - } - - } - - JSONArray jsonArray = JSONArray.fromObject(results); - System.out.println(jsonArray); - return jsonArray; - - } catch (Exception e) { - throw new JsonHttpStatusException(HttpStatus.INTERNAL_SERVER_ERROR,e.toString()); - } + throw new JsonHttpStatusException(HttpStatus.BAD_GATEWAY, + "URL-based uploads are no longer supported. Please use direct file uploads instead."); } diff --git a/src/test/java/cloudgene/mapred/api/v2/jobs/SubmitJobTest.java b/src/test/java/cloudgene/mapred/api/v2/jobs/SubmitJobTest.java index 7457b267..577e9cf3 100644 --- a/src/test/java/cloudgene/mapred/api/v2/jobs/SubmitJobTest.java +++ b/src/test/java/cloudgene/mapred/api/v2/jobs/SubmitJobTest.java @@ -23,7 +23,6 @@ import cloudgene.mapred.util.CloudgeneClient; import cloudgene.mapred.util.LoginToken; import cloudgene.mapred.util.TestCluster; -import cloudgene.mapred.util.TestSFTPServer; import cloudgene.sdk.internal.WorkflowContext; import genepi.io.FileUtil; import io.micronaut.test.extensions.junit5.annotation.MicronautTest; @@ -384,46 +383,4 @@ public void testSubmitWithHiddenInputs() throws IOException, JSONException, Inte } - @Test - public void testSubmitSftpUpload() throws IOException, JSONException, InterruptedException { - - TestSFTPServer sftp = new TestSFTPServer("test-data"); - - String url = "sftp://localhost:8001/" + new File("test-data/sftp-import.yaml").getAbsolutePath() + ";" - + TestSFTPServer.USERNAME + ";" + TestSFTPServer.PASSWORD; - - // form data - - FormDataSet form = new FormDataSet(); - form.setMultipart(true); - form.getEntries().add(new FormData("input-input", url)); - - // submit job - String id = client.submitJobPublic("sftp-import", form); - - // get details to check *** bug - client.getJobDetails(id); - - // check feedback - client.waitForJob(id); - - JSONObject result = client.getJobDetails(id); - - // check if no sftp url is in json - assertFalse(result.toString().contains(url)); - - // get log file - - assertEquals(AbstractJob.STATE_SUCCESS, result.get("state")); - - sftp.stop(); - - // check results! - - } - - // TODO: wrong permissions - - // TODO: wrong id - } diff --git a/src/test/java/cloudgene/mapred/util/TestSFTPServer.java b/src/test/java/cloudgene/mapred/util/TestSFTPServer.java deleted file mode 100644 index dbc49222..00000000 --- a/src/test/java/cloudgene/mapred/util/TestSFTPServer.java +++ /dev/null @@ -1,64 +0,0 @@ -package cloudgene.mapred.util; - -import java.io.File; -import java.io.IOException; -import java.util.Arrays; - -import org.apache.commons.lang.StringUtils; -import org.apache.sshd.SshServer; -import org.apache.sshd.common.NamedFactory; -import org.apache.sshd.common.Session; -import org.apache.sshd.common.file.FileSystemView; -import org.apache.sshd.common.file.nativefs.NativeFileSystemFactory; -import org.apache.sshd.common.file.nativefs.NativeFileSystemView; -import org.apache.sshd.server.Command; -import org.apache.sshd.server.PasswordAuthenticator; -import org.apache.sshd.server.command.ScpCommandFactory; -import org.apache.sshd.server.keyprovider.SimpleGeneratorHostKeyProvider; -import org.apache.sshd.server.session.ServerSession; -import org.apache.sshd.sftp.subsystem.SftpSubsystem; - -public class TestSFTPServer { - - private SshServer sshd; - - public static final String USERNAME = "username"; - - public static final String PASSWORD = "password"; - - public TestSFTPServer(final String rootFolder) throws IOException { - - sshd = SshServer.setUpDefaultServer(); - sshd.setFileSystemFactory(new NativeFileSystemFactory() { - @Override - public FileSystemView createFileSystemView(final Session session) { - return new NativeFileSystemView(session.getUsername(), false) { - @Override - public String getVirtualUserDir() { - System.out.println("Virtual Root: " + new File(rootFolder).getAbsolutePath()); - return new File(rootFolder).getAbsolutePath(); - } - - }; - }; - }); - sshd.setPort(8001); - sshd.setSubsystemFactories(Arrays.>asList(new SftpSubsystem.Factory())); - sshd.setCommandFactory(new ScpCommandFactory()); - sshd.setKeyPairProvider( - new SimpleGeneratorHostKeyProvider(new File(rootFolder + "/hostkey.ser").getAbsolutePath())); - sshd.setPasswordAuthenticator(new PasswordAuthenticator() { - @Override - public boolean authenticate(final String username, final String password, final ServerSession session) { - return StringUtils.equals(username, USERNAME) && StringUtils.equals(password, PASSWORD); - } - }); - sshd.start(); - - } - - public void stop() throws InterruptedException { - sshd.stop(); - } - -} From 8aa77c7d848bd25ca80a50902b851d6de7de71ad Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Thu, 4 May 2023 12:59:15 +0200 Subject: [PATCH 07/24] Improve input params handling --- .../mapred/server/services/JobService.java | 6 +++-- .../mapred/api/v2/jobs/SubmitJobTest.java | 26 +++++++++++++++++++ test-data/print-hidden-inputs.yaml | 2 +- 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/src/main/java/cloudgene/mapred/server/services/JobService.java b/src/main/java/cloudgene/mapred/server/services/JobService.java index 02ce242f..806c5db6 100644 --- a/src/main/java/cloudgene/mapred/server/services/JobService.java +++ b/src/main/java/cloudgene/mapred/server/services/JobService.java @@ -8,6 +8,8 @@ import java.util.Map; import java.util.Vector; +import org.apache.commons.lang.StringEscapeUtils; + import cloudgene.mapred.apps.ApplicationRepository; import cloudgene.mapred.core.User; import cloudgene.mapred.database.DownloadDao; @@ -458,13 +460,13 @@ private Map parseAndUpdateInputParams(List form, WdlA } else { - String key = name; + String key = StringEscapeUtils.escapeHtml(name); if (key.startsWith("input-")) { key = key.replace("input-", ""); } if (!props.containsKey(key)) { // don't override uploaded files - props.put(key, value.toString()); + props.put(key, StringEscapeUtils.escapeHtml(value.toString())); } } diff --git a/src/test/java/cloudgene/mapred/api/v2/jobs/SubmitJobTest.java b/src/test/java/cloudgene/mapred/api/v2/jobs/SubmitJobTest.java index 577e9cf3..2f3702ef 100644 --- a/src/test/java/cloudgene/mapred/api/v2/jobs/SubmitJobTest.java +++ b/src/test/java/cloudgene/mapred/api/v2/jobs/SubmitJobTest.java @@ -383,4 +383,30 @@ public void testSubmitWithHiddenInputs() throws IOException, JSONException, Inte } + @Test + public void testSubmitHtmlInParams() throws IOException, JSONException, InterruptedException { + + // form data + + String html = "