From 3a06506c58f814c0d9d04975713de7a76635a35c Mon Sep 17 00:00:00 2001 From: Christian Freitas Date: Tue, 28 Feb 2023 20:13:09 -0500 Subject: [PATCH 01/34] Initial Blob Storage Read Test --- .../azure_blob_storage_read.inputs | 3 +++ .../azure_blob_storage_read.wdl | 11 +++++++++++ .../azure_blob_storage_read_write.test | 14 ++++++++++++++ 3 files changed, 28 insertions(+) create mode 100644 centaur/src/main/resources/standardTestCases/azure_blob_storage_read/azure_blob_storage_read.inputs create mode 100644 centaur/src/main/resources/standardTestCases/azure_blob_storage_read/azure_blob_storage_read.wdl create mode 100644 centaur/src/main/resources/standardTestCases/azure_blob_storage_read_write.test diff --git a/centaur/src/main/resources/standardTestCases/azure_blob_storage_read/azure_blob_storage_read.inputs b/centaur/src/main/resources/standardTestCases/azure_blob_storage_read/azure_blob_storage_read.inputs new file mode 100644 index 00000000000..c81e166493f --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/azure_blob_storage_read/azure_blob_storage_read.inputs @@ -0,0 +1,3 @@ +{ + "azure_blob_storage_read.file1": "https://centaurtesting.blob.core.windows.net/test-blob/testRead.txt" +} diff --git a/centaur/src/main/resources/standardTestCases/azure_blob_storage_read/azure_blob_storage_read.wdl b/centaur/src/main/resources/standardTestCases/azure_blob_storage_read/azure_blob_storage_read.wdl new file mode 100644 index 00000000000..4ebe1bd6599 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/azure_blob_storage_read/azure_blob_storage_read.wdl @@ -0,0 +1,11 @@ +workflow azure_blob_storage_read { + + input { + File file1 + } + + output { + String s1 = read_string(file1) + } + +} diff --git a/centaur/src/main/resources/standardTestCases/azure_blob_storage_read_write.test b/centaur/src/main/resources/standardTestCases/azure_blob_storage_read_write.test new file mode 100644 index 00000000000..26b6f57590a --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/azure_blob_storage_read_write.test @@ -0,0 +1,14 @@ +name: azure_blob_storage_read_write +testFormat: workflowsuccess +backends: [Local] +tags: ["blob", "azure"] +retryTestFailures: false + +files { + workflow: azure_blob_storage_read_write/azure_blob_storage_read_write.wdl +} + +metadata { + status: Succeeded + "outputs.s1": "This is my test file! Did it work??" +} From 0abca4b449adc44ea24a21290e732186aa872c72 Mon Sep 17 00:00:00 2001 From: Christian Freitas Date: Wed, 1 Mar 2023 16:00:22 -0500 Subject: [PATCH 02/34] Fix read test for azure --- .../azure_blob_storage_read.test | 15 +++++++++++++++ .../azure_blob_storage_read.wdl | 15 ++++++++------- .../azure_blob_storage_read_write.test | 14 -------------- 3 files changed, 23 insertions(+), 21 deletions(-) create mode 100644 centaur/src/main/resources/standardTestCases/azure_blob_storage_read.test delete mode 100644 centaur/src/main/resources/standardTestCases/azure_blob_storage_read_write.test diff --git a/centaur/src/main/resources/standardTestCases/azure_blob_storage_read.test b/centaur/src/main/resources/standardTestCases/azure_blob_storage_read.test new file mode 100644 index 00000000000..bd8bae37229 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/azure_blob_storage_read.test @@ -0,0 +1,15 @@ +name: azure_blob_storage_read +testFormat: workflowsuccess +backends: [Local] +tags: ["blob", "azure"] +retryTestFailures: false + +files { + workflow: azure_blob_storage_read/azure_blob_storage_read.wdl + inputs: azure_blob_storage_read/azure_blob_storage_read.inputs +} + +metadata { + status: Succeeded + "outputs.azure_blob_storage_read.s1": "This is my test file! Did it work??" +} diff --git a/centaur/src/main/resources/standardTestCases/azure_blob_storage_read/azure_blob_storage_read.wdl b/centaur/src/main/resources/standardTestCases/azure_blob_storage_read/azure_blob_storage_read.wdl index 4ebe1bd6599..d19d417cdd5 100644 --- a/centaur/src/main/resources/standardTestCases/azure_blob_storage_read/azure_blob_storage_read.wdl +++ b/centaur/src/main/resources/standardTestCases/azure_blob_storage_read/azure_blob_storage_read.wdl @@ -1,11 +1,12 @@ -workflow azure_blob_storage_read { +version 1.0 - input { - File file1 - } +workflow azure_blob_storage_read { - output { - String s1 = read_string(file1) - } + input { + File file1 + } + output { + String s1 = read_string(file1) + } } diff --git a/centaur/src/main/resources/standardTestCases/azure_blob_storage_read_write.test b/centaur/src/main/resources/standardTestCases/azure_blob_storage_read_write.test deleted file mode 100644 index 26b6f57590a..00000000000 --- a/centaur/src/main/resources/standardTestCases/azure_blob_storage_read_write.test +++ /dev/null @@ -1,14 +0,0 @@ -name: azure_blob_storage_read_write -testFormat: workflowsuccess -backends: [Local] -tags: ["blob", "azure"] -retryTestFailures: false - -files { - workflow: azure_blob_storage_read_write/azure_blob_storage_read_write.wdl -} - -metadata { - status: Succeeded - "outputs.s1": "This is my test file! Did it work??" -} From 7a20b313eeffd038c9b04ca688f45e333b179ad0 Mon Sep 17 00:00:00 2001 From: Christian Freitas Date: Mon, 6 Mar 2023 15:50:22 -0500 Subject: [PATCH 03/34] Add log workflow option to test --- .../resources/standardTestCases/azure_blob_storage_read.test | 1 + .../azure_blob_storage_read/azure_blob_storage_read.options | 3 +++ 2 files changed, 4 insertions(+) create mode 100644 centaur/src/main/resources/standardTestCases/azure_blob_storage_read/azure_blob_storage_read.options diff --git a/centaur/src/main/resources/standardTestCases/azure_blob_storage_read.test b/centaur/src/main/resources/standardTestCases/azure_blob_storage_read.test index bd8bae37229..123a2745e6d 100644 --- a/centaur/src/main/resources/standardTestCases/azure_blob_storage_read.test +++ b/centaur/src/main/resources/standardTestCases/azure_blob_storage_read.test @@ -7,6 +7,7 @@ retryTestFailures: false files { workflow: azure_blob_storage_read/azure_blob_storage_read.wdl inputs: azure_blob_storage_read/azure_blob_storage_read.inputs + options: azure_blob_storage_read/azure_blob_storage_read.options } metadata { diff --git a/centaur/src/main/resources/standardTestCases/azure_blob_storage_read/azure_blob_storage_read.options b/centaur/src/main/resources/standardTestCases/azure_blob_storage_read/azure_blob_storage_read.options new file mode 100644 index 00000000000..8d68fcdd6bf --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/azure_blob_storage_read/azure_blob_storage_read.options @@ -0,0 +1,3 @@ +{ + "final_workflow_log_dir": "https://centaurtesting.blob.core.windows.net/test-blob/test-cromwell-workflow-logs" +} From 0e107ba3cdcc0cda65083ed25fb5e2696a6d92b6 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Fri, 10 Mar 2023 10:50:19 -0500 Subject: [PATCH 04/34] added environment variables to CI --- .github/workflows/integration_tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 3923d7f37d1..b33225c0bfe 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -72,6 +72,9 @@ jobs: BUILD_MARIADB: ${{ matrix.build_mariadb }} VAULT_ROLE_ID: ${{ secrets.VAULT_ROLE_ID_CI }} VAULT_SECRET_ID: ${{ secrets.VAULT_SECRET_ID_CI }} + AZURE_CLIENT_ID: ${{ secrets.VAULT_AZURE_CENTAUR_CLIENT_ID }} + AZURE_PASSWORD: ${{ secrets.VAULT_AZURE_CENTAUR_CLIENT_SECRET }} + AZURE_TENANT_ID: ${{ secrets.VAULT_AZURE_CENTAUR_TENANT_ID }} runs-on: ubuntu-latest timeout-minutes: 120 steps: From 28a3507c22940bee3ffa475f600b7aad2a66c8b4 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Fri, 10 Mar 2023 13:51:47 -0500 Subject: [PATCH 05/34] new script --- .github/workflows/integration_tests.yml | 3 ++ src/ci/bin/test.inc.sh | 3 ++ src/ci/bin/testCentaurBlob.sh | 0 src/ci/resources/centaur_blob_test.conf | 72 +++++++++++++++++++++++++ 4 files changed, 78 insertions(+) create mode 100644 src/ci/bin/testCentaurBlob.sh create mode 100644 src/ci/resources/centaur_blob_test.conf diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index b33225c0bfe..304b6e92aa7 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -63,6 +63,9 @@ jobs: - build_type: centaurSlurm build_mysql: 5.7 friendly_name: "Centaur Slurm with MySQL 5.7" + - build_type: centaurBlob + build_mysql: 5.7 + friendly_name: Centaur Blob name: ${{ matrix.friendly_name }} env: BUILD_NAME: ${{ matrix.build_type }} diff --git a/src/ci/bin/test.inc.sh b/src/ci/bin/test.inc.sh index 017fbd95b54..5d7e9a98837 100644 --- a/src/ci/bin/test.inc.sh +++ b/src/ci/bin/test.inc.sh @@ -390,6 +390,9 @@ cromwell::private::create_build_variables() { centaurHoricromtalEngineUpgrade*) CROMWELL_BUILD_CROMWELL_CONFIG="${CROMWELL_BUILD_RESOURCES_DIRECTORY}/papi_v2alpha1_horicromtal_application.conf" ;; + centaurBlob*) + CROMWELL_BUILD_CROMWELL_CONFIG="${CROMWELL_BUILD_RESOURCES_DIRECTORY}/centaur_blob_test.conf" + ;; *) CROMWELL_BUILD_CROMWELL_CONFIG="${CROMWELL_BUILD_RESOURCES_DIRECTORY}/${CROMWELL_BUILD_BACKEND_TYPE}_application.conf" ;; diff --git a/src/ci/bin/testCentaurBlob.sh b/src/ci/bin/testCentaurBlob.sh new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/ci/resources/centaur_blob_test.conf b/src/ci/resources/centaur_blob_test.conf new file mode 100644 index 00000000000..fdc9775466d --- /dev/null +++ b/src/ci/resources/centaur_blob_test.conf @@ -0,0 +1,72 @@ +include required(classpath("application.conf")) +include "build_application.inc.conf" + +filesystems { + blob { + class = "cromwell.filesystems.blob.BlobPathBuilderFactory" + global { + # One BFSM is shared across all BlobPathBuilders + class = "cromwell.filesystems.blob.BlobFileSystemManager" + config { + container: "test-blob" + endpoint: "https://centaurtesting.blob.core.windows.net" + subscription: "62b22893-6bc1-46d9-8a90-806bb3cce3c9" + # WSM config is needed for accessing WSM-managed blob containers + # created in Terra workspaces. + workspace-manager { + url: "https://workspace.dsde-dev.broadinstitute.org" + workspace-id: "00001111-2222-3333-aaaa-bbbbccccdddd" + container-resource-id: "00001111-2222-3333-aaaa-bbbbccccdddd" + b2cToken: "Zardoz" + } + } + } + } +} + +engine { + filesystems { + local { + enabled: false + } + http { + enabled: false + } + blob { + enabled: true + } + } +} + + +backend { + default = "TES" + providers { + TES { + actor-factory = "cromwell.backend.impl.tes.TesBackendLifecycleActorFactory" + config { + # Use for running on blob storage + #root = "https://.blob.core.windows.net/cromwell/cromwell-executions" + root = "cromwell-executions" + dockerRoot = "/cromwell-executions" + endpoint = "http://127.0.0.1:9000/v1/tasks" + concurrent-job-limit = 1000 + transform-blob-to-local-path = false + filesystems { + blob { + enabled: false + } + local { + enabled: true + } + http { + enabled: true + } + } + } + + # Have the engine (maybe) authenticate to docker.io. See BT-141 for more info. + include "dockerhub_provider_config_v1.inc.conf" + } + } +} From b7dd69e928631e2423fbc3f0cc91a15353effa49 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Fri, 10 Mar 2023 13:58:48 -0500 Subject: [PATCH 06/34] denied --- src/ci/bin/testCentaurBlob.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/ci/bin/testCentaurBlob.sh b/src/ci/bin/testCentaurBlob.sh index e69de29bb2d..d1f3932e047 100644 --- a/src/ci/bin/testCentaurBlob.sh +++ b/src/ci/bin/testCentaurBlob.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -o errexit -o nounset -o pipefail +# import in shellcheck / CI / IntelliJ compatible ways +# shellcheck source=/dev/null +source "${BASH_SOURCE%/*}/test.inc.sh" || source test.inc.sh + +cromwell::build::setup_common_environment + +cromwell::build::setup_centaur_environment + +cromwell::build::assemble_jars + +cromwell::build::run_centaur + +cromwell::build::generate_code_coverage \ No newline at end of file From 21a4e93768dd4046865bc6e535494c2749b88f7e Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Fri, 10 Mar 2023 14:07:48 -0500 Subject: [PATCH 07/34] permissions --- src/ci/bin/test.inc.sh | 0 src/ci/bin/testCentaurBlob.sh | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 src/ci/bin/test.inc.sh mode change 100644 => 100755 src/ci/bin/testCentaurBlob.sh diff --git a/src/ci/bin/test.inc.sh b/src/ci/bin/test.inc.sh old mode 100644 new mode 100755 diff --git a/src/ci/bin/testCentaurBlob.sh b/src/ci/bin/testCentaurBlob.sh old mode 100644 new mode 100755 From 1b35ceb04349181cea407cb69393b1205f5a2428 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Fri, 10 Mar 2023 14:53:01 -0500 Subject: [PATCH 08/34] testing --- src/ci/bin/test.inc.sh | 0 src/ci/bin/testCentaurBlob.sh | 6 ++++-- 2 files changed, 4 insertions(+), 2 deletions(-) mode change 100755 => 100644 src/ci/bin/test.inc.sh mode change 100755 => 100644 src/ci/bin/testCentaurBlob.sh diff --git a/src/ci/bin/test.inc.sh b/src/ci/bin/test.inc.sh old mode 100755 new mode 100644 diff --git a/src/ci/bin/testCentaurBlob.sh b/src/ci/bin/testCentaurBlob.sh old mode 100755 new mode 100644 index d1f3932e047..a707f5ee5de --- a/src/ci/bin/testCentaurBlob.sh +++ b/src/ci/bin/testCentaurBlob.sh @@ -11,6 +11,8 @@ cromwell::build::setup_centaur_environment cromwell::build::assemble_jars -cromwell::build::run_centaur +cromwell::build::run_centaur \ + -p 2 \ + -e azure_blob_storage_read \ -cromwell::build::generate_code_coverage \ No newline at end of file +cromwell::build::generate_code_coverage From 578f3a7e3705d450e7b69a96fc40626475dbb7aa Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Fri, 10 Mar 2023 14:56:24 -0500 Subject: [PATCH 09/34] testing --- src/ci/bin/test.inc.sh | 0 src/ci/bin/testCentaurBlob.sh | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 src/ci/bin/test.inc.sh mode change 100644 => 100755 src/ci/bin/testCentaurBlob.sh diff --git a/src/ci/bin/test.inc.sh b/src/ci/bin/test.inc.sh old mode 100644 new mode 100755 diff --git a/src/ci/bin/testCentaurBlob.sh b/src/ci/bin/testCentaurBlob.sh old mode 100644 new mode 100755 From ccde773119cb330dc51b8bed790eda6f6ef18258 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Fri, 10 Mar 2023 15:18:24 -0500 Subject: [PATCH 10/34] i --- src/ci/bin/test.inc.sh | 0 src/ci/bin/testCentaurBlob.sh | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) mode change 100755 => 100644 src/ci/bin/test.inc.sh mode change 100755 => 100644 src/ci/bin/testCentaurBlob.sh diff --git a/src/ci/bin/test.inc.sh b/src/ci/bin/test.inc.sh old mode 100755 new mode 100644 diff --git a/src/ci/bin/testCentaurBlob.sh b/src/ci/bin/testCentaurBlob.sh old mode 100755 new mode 100644 index a707f5ee5de..70d30a1fe9a --- a/src/ci/bin/testCentaurBlob.sh +++ b/src/ci/bin/testCentaurBlob.sh @@ -13,6 +13,6 @@ cromwell::build::assemble_jars cromwell::build::run_centaur \ -p 2 \ - -e azure_blob_storage_read \ + -i azure_blob_storage_read \ cromwell::build::generate_code_coverage From 91f7f72adbfde11f997b226c2e8753ae66b61475 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Fri, 10 Mar 2023 15:26:51 -0500 Subject: [PATCH 11/34] per --- src/ci/bin/test.inc.sh | 0 src/ci/bin/testCentaurBlob.sh | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 src/ci/bin/test.inc.sh mode change 100644 => 100755 src/ci/bin/testCentaurBlob.sh diff --git a/src/ci/bin/test.inc.sh b/src/ci/bin/test.inc.sh old mode 100644 new mode 100755 diff --git a/src/ci/bin/testCentaurBlob.sh b/src/ci/bin/testCentaurBlob.sh old mode 100644 new mode 100755 From 33aad95d84efa327059588f0d9821ea459b152be Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Mon, 13 Mar 2023 10:35:52 -0400 Subject: [PATCH 12/34] testing --- src/ci/bin/test.inc.sh | 0 src/ci/bin/testCentaurBlob.sh | 0 src/ci/resources/centaur_blob_test.conf | 8 -------- 3 files changed, 8 deletions(-) mode change 100755 => 100644 src/ci/bin/test.inc.sh mode change 100755 => 100644 src/ci/bin/testCentaurBlob.sh diff --git a/src/ci/bin/test.inc.sh b/src/ci/bin/test.inc.sh old mode 100755 new mode 100644 diff --git a/src/ci/bin/testCentaurBlob.sh b/src/ci/bin/testCentaurBlob.sh old mode 100755 new mode 100644 diff --git a/src/ci/resources/centaur_blob_test.conf b/src/ci/resources/centaur_blob_test.conf index fdc9775466d..5bd7c449822 100644 --- a/src/ci/resources/centaur_blob_test.conf +++ b/src/ci/resources/centaur_blob_test.conf @@ -11,14 +11,6 @@ filesystems { container: "test-blob" endpoint: "https://centaurtesting.blob.core.windows.net" subscription: "62b22893-6bc1-46d9-8a90-806bb3cce3c9" - # WSM config is needed for accessing WSM-managed blob containers - # created in Terra workspaces. - workspace-manager { - url: "https://workspace.dsde-dev.broadinstitute.org" - workspace-id: "00001111-2222-3333-aaaa-bbbbccccdddd" - container-resource-id: "00001111-2222-3333-aaaa-bbbbccccdddd" - b2cToken: "Zardoz" - } } } } From 196925fe4bab1f72e492efea7f6bce19ef3ceb11 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Mon, 13 Mar 2023 10:41:24 -0400 Subject: [PATCH 13/34] testing --- src/ci/bin/test.inc.sh | 0 src/ci/bin/testCentaurBlob.sh | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 src/ci/bin/test.inc.sh mode change 100644 => 100755 src/ci/bin/testCentaurBlob.sh diff --git a/src/ci/bin/test.inc.sh b/src/ci/bin/test.inc.sh old mode 100644 new mode 100755 diff --git a/src/ci/bin/testCentaurBlob.sh b/src/ci/bin/testCentaurBlob.sh old mode 100644 new mode 100755 From cab62aaa8f0ba3b54979c58edb514f09c4b45473 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Mon, 13 Mar 2023 14:07:08 -0400 Subject: [PATCH 14/34] change --- src/ci/resources/centaur_blob_test.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ci/resources/centaur_blob_test.conf b/src/ci/resources/centaur_blob_test.conf index 5bd7c449822..7a29e2a67b0 100644 --- a/src/ci/resources/centaur_blob_test.conf +++ b/src/ci/resources/centaur_blob_test.conf @@ -38,8 +38,8 @@ backend { actor-factory = "cromwell.backend.impl.tes.TesBackendLifecycleActorFactory" config { # Use for running on blob storage - #root = "https://.blob.core.windows.net/cromwell/cromwell-executions" - root = "cromwell-executions" + root = "https://.blob.core.windows.net/cromwell/cromwell-executions" + #root = "cromwell-executions" dockerRoot = "/cromwell-executions" endpoint = "http://127.0.0.1:9000/v1/tasks" concurrent-job-limit = 1000 From d5da50bcdb1ea65905b74d477f7851100d5612ab Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Mon, 13 Mar 2023 14:35:04 -0400 Subject: [PATCH 15/34] testing --- .github/workflows/integration_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 304b6e92aa7..7cf7bc8719f 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -76,7 +76,7 @@ jobs: VAULT_ROLE_ID: ${{ secrets.VAULT_ROLE_ID_CI }} VAULT_SECRET_ID: ${{ secrets.VAULT_SECRET_ID_CI }} AZURE_CLIENT_ID: ${{ secrets.VAULT_AZURE_CENTAUR_CLIENT_ID }} - AZURE_PASSWORD: ${{ secrets.VAULT_AZURE_CENTAUR_CLIENT_SECRET }} + AZURE_CLIENT_SECRET: ${{ secrets.VAULT_AZURE_CENTAUR_CLIENT_SECRET }} AZURE_TENANT_ID: ${{ secrets.VAULT_AZURE_CENTAUR_TENANT_ID }} runs-on: ubuntu-latest timeout-minutes: 120 From 1717090f822dbcdac06b998fbd0bbbc3805744fb Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Mon, 13 Mar 2023 14:57:38 -0400 Subject: [PATCH 16/34] testy' --- src/ci/resources/centaur_blob_test.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ci/resources/centaur_blob_test.conf b/src/ci/resources/centaur_blob_test.conf index 7a29e2a67b0..5bd7c449822 100644 --- a/src/ci/resources/centaur_blob_test.conf +++ b/src/ci/resources/centaur_blob_test.conf @@ -38,8 +38,8 @@ backend { actor-factory = "cromwell.backend.impl.tes.TesBackendLifecycleActorFactory" config { # Use for running on blob storage - root = "https://.blob.core.windows.net/cromwell/cromwell-executions" - #root = "cromwell-executions" + #root = "https://.blob.core.windows.net/cromwell/cromwell-executions" + root = "cromwell-executions" dockerRoot = "/cromwell-executions" endpoint = "http://127.0.0.1:9000/v1/tasks" concurrent-job-limit = 1000 From e709eacd87bc9f5e44af3c83457d4d5afe04fa19 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Mon, 13 Mar 2023 15:34:27 -0400 Subject: [PATCH 17/34] testi --- src/ci/resources/centaur_blob_test.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ci/resources/centaur_blob_test.conf b/src/ci/resources/centaur_blob_test.conf index 5bd7c449822..3fdea1d89ad 100644 --- a/src/ci/resources/centaur_blob_test.conf +++ b/src/ci/resources/centaur_blob_test.conf @@ -46,7 +46,7 @@ backend { transform-blob-to-local-path = false filesystems { blob { - enabled: false + enabled: true } local { enabled: true From 3d536605a0dbedee970aea20b0d81ff367de2d84 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 14 Mar 2023 10:06:48 -0400 Subject: [PATCH 18/34] testing --- .../scala/cromwell/filesystems/blob/BlobFileSystemManager.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala b/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala index 7b198f09f6f..a43f29a4bb3 100644 --- a/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala +++ b/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala @@ -232,7 +232,7 @@ case class NativeBlobSasTokenGenerator(container: BlobContainerName, endpoint: E private def azure = subscription.map(authenticateWithSubscription(_)).getOrElse(authenticateWithDefaultSubscription) private def findAzureStorageAccount(name: StorageAccountName) = azure.storageAccounts.list.asScala.find(_.name.equals(name.value)) - .map(Success(_)).getOrElse(Failure(new Exception("Azure Storage Account not found"))) + .map(Success(_)).getOrElse(Failure(new Exception("Azure Storage Account not found" + azure.storageAccounts.list.toString))) private def buildBlobContainerClient(credential: StorageSharedKeyCredential, endpoint: EndpointURL, container: BlobContainerName): BlobContainerClient = { new BlobContainerClientBuilder() .credential(credential) From fd8f65b5a377c8862f72161c38b30b748813761a Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 14 Mar 2023 10:24:29 -0400 Subject: [PATCH 19/34] stomething --- .../cromwell/filesystems/blob/BlobFileSystemManager.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala b/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala index a43f29a4bb3..dccca26fded 100644 --- a/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala +++ b/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala @@ -231,8 +231,11 @@ case class NativeBlobSasTokenGenerator(container: BlobContainerName, endpoint: E private def authenticateWithDefaultSubscription = AzureResourceManager.authenticate(azureCredentialBuilder, azureProfile).withDefaultSubscription() private def azure = subscription.map(authenticateWithSubscription(_)).getOrElse(authenticateWithDefaultSubscription) +private def printList(args: TraversableOnce[_]): Unit = { + args.foreach(println) +} private def findAzureStorageAccount(name: StorageAccountName) = azure.storageAccounts.list.asScala.find(_.name.equals(name.value)) - .map(Success(_)).getOrElse(Failure(new Exception("Azure Storage Account not found" + azure.storageAccounts.list.toString))) + .map(Success(_)).getOrElse(Failure(new Exception("Azure Storage Account not found " + printList(azure.storageAccounts.list.toString)))) private def buildBlobContainerClient(credential: StorageSharedKeyCredential, endpoint: EndpointURL, container: BlobContainerName): BlobContainerClient = { new BlobContainerClientBuilder() .credential(credential) From 44534da88e7918767011200271cb57fccaa0a32c Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 14 Mar 2023 10:26:04 -0400 Subject: [PATCH 20/34] something --- .../cromwell/filesystems/blob/BlobFileSystemManager.scala | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala b/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala index dccca26fded..86e0dd4de15 100644 --- a/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala +++ b/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala @@ -231,11 +231,8 @@ case class NativeBlobSasTokenGenerator(container: BlobContainerName, endpoint: E private def authenticateWithDefaultSubscription = AzureResourceManager.authenticate(azureCredentialBuilder, azureProfile).withDefaultSubscription() private def azure = subscription.map(authenticateWithSubscription(_)).getOrElse(authenticateWithDefaultSubscription) -private def printList(args: TraversableOnce[_]): Unit = { - args.foreach(println) -} private def findAzureStorageAccount(name: StorageAccountName) = azure.storageAccounts.list.asScala.find(_.name.equals(name.value)) - .map(Success(_)).getOrElse(Failure(new Exception("Azure Storage Account not found " + printList(azure.storageAccounts.list.toString)))) + .map(Success(_)).getOrElse(Failure(new Exception("Azure Storage Account not found " + printList(azure.storageAccounts.list.mkString(" "))))) private def buildBlobContainerClient(credential: StorageSharedKeyCredential, endpoint: EndpointURL, container: BlobContainerName): BlobContainerClient = { new BlobContainerClientBuilder() .credential(credential) From 8b7c4f3fcf3f8a03053fe59c83af592405d115fd Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 14 Mar 2023 10:48:14 -0400 Subject: [PATCH 21/34] testing --- .../scala/cromwell/filesystems/blob/BlobFileSystemManager.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala b/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala index 86e0dd4de15..aded8ee1c01 100644 --- a/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala +++ b/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala @@ -232,7 +232,7 @@ case class NativeBlobSasTokenGenerator(container: BlobContainerName, endpoint: E private def azure = subscription.map(authenticateWithSubscription(_)).getOrElse(authenticateWithDefaultSubscription) private def findAzureStorageAccount(name: StorageAccountName) = azure.storageAccounts.list.asScala.find(_.name.equals(name.value)) - .map(Success(_)).getOrElse(Failure(new Exception("Azure Storage Account not found " + printList(azure.storageAccounts.list.mkString(" "))))) + .map(Success(_)).getOrElse(Failure(new Exception("Azure Storage Account not found " + azure.storageAccounts.list.mkString(" ")))) private def buildBlobContainerClient(credential: StorageSharedKeyCredential, endpoint: EndpointURL, container: BlobContainerName): BlobContainerClient = { new BlobContainerClientBuilder() .credential(credential) From ccea5a45f18aca7624af62f1a9f546e4a494cdf0 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 14 Mar 2023 11:37:52 -0400 Subject: [PATCH 22/34] testing --- .../cromwell/filesystems/blob/BlobFileSystemManager.scala | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala b/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala index aded8ee1c01..c7efe58e5c0 100644 --- a/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala +++ b/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala @@ -254,7 +254,11 @@ case class NativeBlobSasTokenGenerator(container: BlobContainerName, endpoint: E */ def generateBlobSasToken: Try[AzureSasCredential] = for { uri <- BlobPathBuilder.parseURI(endpoint.value) + println("URI:") + println(uri) configuredAccount <- BlobPathBuilder.parseStorageAccount(uri) + println("Configured account:") + println(configuredAccount) azureAccount <- findAzureStorageAccount(configuredAccount) keys = azureAccount.getKeys.asScala key <- keys.headOption.fold[Try[StorageAccountKey]](Failure(new Exception("Storage account has no keys")))(Success(_)) From d58500568285c59e8f27cd93e096e218e204ce78 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 14 Mar 2023 12:43:09 -0400 Subject: [PATCH 23/34] testing --- .../cromwell/filesystems/blob/BlobFileSystemManager.scala | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala b/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala index c7efe58e5c0..e52a56bd828 100644 --- a/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala +++ b/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala @@ -232,7 +232,7 @@ case class NativeBlobSasTokenGenerator(container: BlobContainerName, endpoint: E private def azure = subscription.map(authenticateWithSubscription(_)).getOrElse(authenticateWithDefaultSubscription) private def findAzureStorageAccount(name: StorageAccountName) = azure.storageAccounts.list.asScala.find(_.name.equals(name.value)) - .map(Success(_)).getOrElse(Failure(new Exception("Azure Storage Account not found " + azure.storageAccounts.list.mkString(" ")))) + .map(Success(_)).getOrElse(Failure(new Exception("Azure Storage Account not found " + azure.storageAccounts.list.asScala.mkString(" ")))) private def buildBlobContainerClient(credential: StorageSharedKeyCredential, endpoint: EndpointURL, container: BlobContainerName): BlobContainerClient = { new BlobContainerClientBuilder() .credential(credential) @@ -254,11 +254,7 @@ case class NativeBlobSasTokenGenerator(container: BlobContainerName, endpoint: E */ def generateBlobSasToken: Try[AzureSasCredential] = for { uri <- BlobPathBuilder.parseURI(endpoint.value) - println("URI:") - println(uri) configuredAccount <- BlobPathBuilder.parseStorageAccount(uri) - println("Configured account:") - println(configuredAccount) azureAccount <- findAzureStorageAccount(configuredAccount) keys = azureAccount.getKeys.asScala key <- keys.headOption.fold[Try[StorageAccountKey]](Failure(new Exception("Storage account has no keys")))(Success(_)) From a1ba566283fd636b90625dce81237ad7752269a7 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 14 Mar 2023 13:11:49 -0400 Subject: [PATCH 24/34] test --- .../scala/cromwell/filesystems/blob/BlobFileSystemManager.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala b/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala index e52a56bd828..f0dfed3e215 100644 --- a/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala +++ b/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala @@ -232,7 +232,7 @@ case class NativeBlobSasTokenGenerator(container: BlobContainerName, endpoint: E private def azure = subscription.map(authenticateWithSubscription(_)).getOrElse(authenticateWithDefaultSubscription) private def findAzureStorageAccount(name: StorageAccountName) = azure.storageAccounts.list.asScala.find(_.name.equals(name.value)) - .map(Success(_)).getOrElse(Failure(new Exception("Azure Storage Account not found " + azure.storageAccounts.list.asScala.mkString(" ")))) + .map(Success(_)).getOrElse(Failure(new Exception("Azure Storage Account not found " + azure.storageAccounts.list.asScala.map(b => println(b.toString()))))) private def buildBlobContainerClient(credential: StorageSharedKeyCredential, endpoint: EndpointURL, container: BlobContainerName): BlobContainerClient = { new BlobContainerClientBuilder() .credential(credential) From 0f0c481c48570652d96c4be5329a7c117ea40b7c Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 14 Mar 2023 13:34:06 -0400 Subject: [PATCH 25/34] emtpy --- .../scala/cromwell/filesystems/blob/BlobFileSystemManager.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala b/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala index f0dfed3e215..21b2471b3e8 100644 --- a/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala +++ b/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala @@ -232,7 +232,7 @@ case class NativeBlobSasTokenGenerator(container: BlobContainerName, endpoint: E private def azure = subscription.map(authenticateWithSubscription(_)).getOrElse(authenticateWithDefaultSubscription) private def findAzureStorageAccount(name: StorageAccountName) = azure.storageAccounts.list.asScala.find(_.name.equals(name.value)) - .map(Success(_)).getOrElse(Failure(new Exception("Azure Storage Account not found " + azure.storageAccounts.list.asScala.map(b => println(b.toString()))))) + .map(Success(_)).getOrElse(Failure(new Exception("Azure Storage Account not found " + azure.storageAccounts.list.asScala.isEmpty))) private def buildBlobContainerClient(credential: StorageSharedKeyCredential, endpoint: EndpointURL, container: BlobContainerName): BlobContainerClient = { new BlobContainerClientBuilder() .credential(credential) From 1ccf70547208e1186a256b968548bdc848f4e74c Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Tue, 14 Mar 2023 15:42:18 -0400 Subject: [PATCH 26/34] testing --- src/ci/bin/test.inc.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 src/ci/bin/test.inc.sh diff --git a/src/ci/bin/test.inc.sh b/src/ci/bin/test.inc.sh old mode 100755 new mode 100644 From 05d516810f2a6e627e0ec2b9160507a21c13a1c0 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Wed, 15 Mar 2023 13:00:35 -0400 Subject: [PATCH 27/34] cleanup --- .gitignore | 1 + .../cromwell/filesystems/blob/BlobFileSystemManager.scala | 3 +-- src/ci/bin/testCentaurBlob.sh | 3 +++ 3 files changed, 5 insertions(+), 2 deletions(-) mode change 100755 => 100644 src/ci/bin/testCentaurBlob.sh diff --git a/.gitignore b/.gitignore index a5b72f6b263..8a2771c240b 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ actual.json console_output.txt expected.json run_mode_metadata.json +https:/** # custom config cromwell-executions diff --git a/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala b/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala index 21b2471b3e8..1672bd5bd88 100644 --- a/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala +++ b/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala @@ -230,9 +230,8 @@ case class NativeBlobSasTokenGenerator(container: BlobContainerName, endpoint: E private def authenticateWithSubscription(sub: SubscriptionId) = AzureResourceManager.authenticate(azureCredentialBuilder, azureProfile).withSubscription(sub.toString) private def authenticateWithDefaultSubscription = AzureResourceManager.authenticate(azureCredentialBuilder, azureProfile).withDefaultSubscription() private def azure = subscription.map(authenticateWithSubscription(_)).getOrElse(authenticateWithDefaultSubscription) - private def findAzureStorageAccount(name: StorageAccountName) = azure.storageAccounts.list.asScala.find(_.name.equals(name.value)) - .map(Success(_)).getOrElse(Failure(new Exception("Azure Storage Account not found " + azure.storageAccounts.list.asScala.isEmpty))) + .map(Success(_)).getOrElse(Failure(new Exception("Azure Storage Account not found"))) private def buildBlobContainerClient(credential: StorageSharedKeyCredential, endpoint: EndpointURL, container: BlobContainerName): BlobContainerClient = { new BlobContainerClientBuilder() .credential(credential) diff --git a/src/ci/bin/testCentaurBlob.sh b/src/ci/bin/testCentaurBlob.sh old mode 100755 new mode 100644 index 70d30a1fe9a..1ac6783446e --- a/src/ci/bin/testCentaurBlob.sh +++ b/src/ci/bin/testCentaurBlob.sh @@ -5,12 +5,15 @@ set -o errexit -o nounset -o pipefail # shellcheck source=/dev/null source "${BASH_SOURCE%/*}/test.inc.sh" || source test.inc.sh +#NB: This function ensures that the correct .conf file is being used by cromwell (among other things). +#Blob storage requires a configuration file tailored for Azure. cromwell::build::setup_common_environment cromwell::build::setup_centaur_environment cromwell::build::assemble_jars +#Run only the azure_blob_storage_read test cromwell::build::run_centaur \ -p 2 \ -i azure_blob_storage_read \ From 8c9b37fe36b2e6072b77c99a6fc243171b04f488 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Wed, 15 Mar 2023 13:23:01 -0400 Subject: [PATCH 28/34] h --- src/ci/bin/test.inc.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 src/ci/bin/test.inc.sh diff --git a/src/ci/bin/test.inc.sh b/src/ci/bin/test.inc.sh old mode 100644 new mode 100755 From f1e09e4e4bdcee19a76fdc0640e125ace6a2acb3 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Wed, 15 Mar 2023 13:30:07 -0400 Subject: [PATCH 29/34] h --- src/ci/bin/test.inc.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 src/ci/bin/test.inc.sh diff --git a/src/ci/bin/test.inc.sh b/src/ci/bin/test.inc.sh old mode 100755 new mode 100644 From de2bf9b8fad5c972cddae83f3101de1e4c08a01e Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Wed, 15 Mar 2023 13:38:17 -0400 Subject: [PATCH 30/34] h --- src/ci/bin/test.inc.sh | 0 src/ci/bin/testCentaurBlob.sh | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 src/ci/bin/test.inc.sh mode change 100644 => 100755 src/ci/bin/testCentaurBlob.sh diff --git a/src/ci/bin/test.inc.sh b/src/ci/bin/test.inc.sh old mode 100644 new mode 100755 diff --git a/src/ci/bin/testCentaurBlob.sh b/src/ci/bin/testCentaurBlob.sh old mode 100644 new mode 100755 From 958db3c60dda5dc5c792a06edeaaa71f4dda84ed Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Wed, 15 Mar 2023 14:13:11 -0400 Subject: [PATCH 31/34] newdir --- .../azure_blob_storage_read.test | 0 .../azure_blob_storage_read/azure_blob_storage_read.inputs | 0 .../azure_blob_storage_read/azure_blob_storage_read.options | 0 .../azure_blob_storage_read/azure_blob_storage_read.wdl | 0 src/ci/bin/test.inc.sh | 4 ++++ src/ci/bin/testCentaurBlob.sh | 5 +---- 6 files changed, 5 insertions(+), 4 deletions(-) rename centaur/src/main/resources/{standardTestCases => azureBlobTestCases}/azure_blob_storage_read.test (100%) rename centaur/src/main/resources/{standardTestCases => azureBlobTestCases}/azure_blob_storage_read/azure_blob_storage_read.inputs (100%) rename centaur/src/main/resources/{standardTestCases => azureBlobTestCases}/azure_blob_storage_read/azure_blob_storage_read.options (100%) rename centaur/src/main/resources/{standardTestCases => azureBlobTestCases}/azure_blob_storage_read/azure_blob_storage_read.wdl (100%) mode change 100755 => 100644 src/ci/bin/test.inc.sh mode change 100755 => 100644 src/ci/bin/testCentaurBlob.sh diff --git a/centaur/src/main/resources/standardTestCases/azure_blob_storage_read.test b/centaur/src/main/resources/azureBlobTestCases/azure_blob_storage_read.test similarity index 100% rename from centaur/src/main/resources/standardTestCases/azure_blob_storage_read.test rename to centaur/src/main/resources/azureBlobTestCases/azure_blob_storage_read.test diff --git a/centaur/src/main/resources/standardTestCases/azure_blob_storage_read/azure_blob_storage_read.inputs b/centaur/src/main/resources/azureBlobTestCases/azure_blob_storage_read/azure_blob_storage_read.inputs similarity index 100% rename from centaur/src/main/resources/standardTestCases/azure_blob_storage_read/azure_blob_storage_read.inputs rename to centaur/src/main/resources/azureBlobTestCases/azure_blob_storage_read/azure_blob_storage_read.inputs diff --git a/centaur/src/main/resources/standardTestCases/azure_blob_storage_read/azure_blob_storage_read.options b/centaur/src/main/resources/azureBlobTestCases/azure_blob_storage_read/azure_blob_storage_read.options similarity index 100% rename from centaur/src/main/resources/standardTestCases/azure_blob_storage_read/azure_blob_storage_read.options rename to centaur/src/main/resources/azureBlobTestCases/azure_blob_storage_read/azure_blob_storage_read.options diff --git a/centaur/src/main/resources/standardTestCases/azure_blob_storage_read/azure_blob_storage_read.wdl b/centaur/src/main/resources/azureBlobTestCases/azure_blob_storage_read/azure_blob_storage_read.wdl similarity index 100% rename from centaur/src/main/resources/standardTestCases/azure_blob_storage_read/azure_blob_storage_read.wdl rename to centaur/src/main/resources/azureBlobTestCases/azure_blob_storage_read/azure_blob_storage_read.wdl diff --git a/src/ci/bin/test.inc.sh b/src/ci/bin/test.inc.sh old mode 100755 new mode 100644 index 5d7e9a98837..eef3a9fab6b --- a/src/ci/bin/test.inc.sh +++ b/src/ci/bin/test.inc.sh @@ -598,6 +598,7 @@ cromwell::private::create_centaur_variables() { CROMWELL_BUILD_CENTAUR_TYPE_PAPI_UPGRADE_NEW_WORKFLOWS="papiUpgradeNewWorkflows" CROMWELL_BUILD_CENTAUR_TYPE_HORICROMTAL_ENGINE_UPGRADE="horicromtalEngineUpgrade" CROMWELL_BUILD_CENTAUR_TYPE_HORICROMTAL="horicromtal" + CROMWELL_BUILD_CENTAUR_TYPE_AZURE_BLOB="azureBlob" case "${CROMWELL_BUILD_TYPE}" in centaurEngineUpgrade*) @@ -615,6 +616,9 @@ cromwell::private::create_centaur_variables() { centaurHoricromtal*) CROMWELL_BUILD_CENTAUR_TYPE="${CROMWELL_BUILD_CENTAUR_TYPE_HORICROMTAL}" ;; + centaurBlob*) + CROMWELL_BUILD_CENTAUR_TYPE="${CROMWELL_BUILD_CENTAUR_TYPE_AZURE_BLOB}" + ;; *) # Only set the type if Jenkins, etc. has not already set the centaur type if [[ -z "${CROMWELL_BUILD_CENTAUR_TYPE-}" ]]; then diff --git a/src/ci/bin/testCentaurBlob.sh b/src/ci/bin/testCentaurBlob.sh old mode 100755 new mode 100644 index 1ac6783446e..a0075441b44 --- a/src/ci/bin/testCentaurBlob.sh +++ b/src/ci/bin/testCentaurBlob.sh @@ -13,9 +13,6 @@ cromwell::build::setup_centaur_environment cromwell::build::assemble_jars -#Run only the azure_blob_storage_read test -cromwell::build::run_centaur \ - -p 2 \ - -i azure_blob_storage_read \ +cromwell::build::run_centaur cromwell::build::generate_code_coverage From 0b80d198e8ae76b47e55523b3520029f84af2d4a Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Wed, 15 Mar 2023 14:14:00 -0400 Subject: [PATCH 32/34] h --- src/ci/bin/test.inc.sh | 0 src/ci/bin/testCentaurBlob.sh | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 src/ci/bin/test.inc.sh mode change 100644 => 100755 src/ci/bin/testCentaurBlob.sh diff --git a/src/ci/bin/test.inc.sh b/src/ci/bin/test.inc.sh old mode 100644 new mode 100755 diff --git a/src/ci/bin/testCentaurBlob.sh b/src/ci/bin/testCentaurBlob.sh old mode 100644 new mode 100755 From bcd11a5611346ff7941b1f58489759ee21f059e9 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Wed, 15 Mar 2023 15:24:52 -0400 Subject: [PATCH 33/34] remove --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index 8a2771c240b..a5b72f6b263 100644 --- a/.gitignore +++ b/.gitignore @@ -22,7 +22,6 @@ actual.json console_output.txt expected.json run_mode_metadata.json -https:/** # custom config cromwell-executions From 70913a0efc99c472838affaa5df4d8be3b27177e Mon Sep 17 00:00:00 2001 From: Christian Freitas Date: Thu, 23 Mar 2023 14:50:53 -0400 Subject: [PATCH 34/34] Add comment to clarify the config --- src/ci/resources/centaur_blob_test.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ci/resources/centaur_blob_test.conf b/src/ci/resources/centaur_blob_test.conf index 3fdea1d89ad..b97940b856d 100644 --- a/src/ci/resources/centaur_blob_test.conf +++ b/src/ci/resources/centaur_blob_test.conf @@ -32,6 +32,8 @@ engine { backend { + # Note that the backend is not selected purposefully here, + # the existing tests at the time of writing do not submit tasks to a backend. default = "TES" providers { TES {