diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index aacd46ffe78..94d3a6611a3 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -83,6 +83,9 @@ jobs: - build_type: centaurSlurm build_mysql: 5.7 friendly_name: "Centaur Slurm with MySQL 5.7" + - build_type: centaurBlob + build_mysql: 5.7 + friendly_name: Centaur Blob name: ${{ matrix.friendly_name }} env: BUILD_NAME: ${{ matrix.build_type }} @@ -92,6 +95,9 @@ jobs: BUILD_MARIADB: ${{ matrix.build_mariadb }} VAULT_ROLE_ID: ${{ secrets.VAULT_ROLE_ID_CI }} VAULT_SECRET_ID: ${{ secrets.VAULT_SECRET_ID_CI }} + AZURE_CLIENT_ID: ${{ secrets.VAULT_AZURE_CENTAUR_CLIENT_ID }} + AZURE_CLIENT_SECRET: ${{ secrets.VAULT_AZURE_CENTAUR_CLIENT_SECRET }} + AZURE_TENANT_ID: ${{ secrets.VAULT_AZURE_CENTAUR_TENANT_ID }} runs-on: ubuntu-latest timeout-minutes: 120 steps: diff --git a/centaur/src/main/resources/azureBlobTestCases/azure_blob_storage_read.test b/centaur/src/main/resources/azureBlobTestCases/azure_blob_storage_read.test new file mode 100644 index 00000000000..123a2745e6d --- /dev/null +++ b/centaur/src/main/resources/azureBlobTestCases/azure_blob_storage_read.test @@ -0,0 +1,16 @@ +name: azure_blob_storage_read +testFormat: workflowsuccess +backends: [Local] +tags: ["blob", "azure"] +retryTestFailures: false + +files { + workflow: azure_blob_storage_read/azure_blob_storage_read.wdl + inputs: azure_blob_storage_read/azure_blob_storage_read.inputs + options: azure_blob_storage_read/azure_blob_storage_read.options +} + +metadata { + status: Succeeded + "outputs.azure_blob_storage_read.s1": "This is my test file! Did it work??" +} diff --git a/centaur/src/main/resources/azureBlobTestCases/azure_blob_storage_read/azure_blob_storage_read.inputs b/centaur/src/main/resources/azureBlobTestCases/azure_blob_storage_read/azure_blob_storage_read.inputs new file mode 100644 index 00000000000..c81e166493f --- /dev/null +++ b/centaur/src/main/resources/azureBlobTestCases/azure_blob_storage_read/azure_blob_storage_read.inputs @@ -0,0 +1,3 @@ +{ + "azure_blob_storage_read.file1": "https://centaurtesting.blob.core.windows.net/test-blob/testRead.txt" +} diff --git a/centaur/src/main/resources/azureBlobTestCases/azure_blob_storage_read/azure_blob_storage_read.options b/centaur/src/main/resources/azureBlobTestCases/azure_blob_storage_read/azure_blob_storage_read.options new file mode 100644 index 00000000000..8d68fcdd6bf --- /dev/null +++ b/centaur/src/main/resources/azureBlobTestCases/azure_blob_storage_read/azure_blob_storage_read.options @@ -0,0 +1,3 @@ +{ + "final_workflow_log_dir": "https://centaurtesting.blob.core.windows.net/test-blob/test-cromwell-workflow-logs" +} diff --git a/centaur/src/main/resources/azureBlobTestCases/azure_blob_storage_read/azure_blob_storage_read.wdl b/centaur/src/main/resources/azureBlobTestCases/azure_blob_storage_read/azure_blob_storage_read.wdl new file mode 100644 index 00000000000..d19d417cdd5 --- /dev/null +++ b/centaur/src/main/resources/azureBlobTestCases/azure_blob_storage_read/azure_blob_storage_read.wdl @@ -0,0 +1,12 @@ +version 1.0 + +workflow azure_blob_storage_read { + + input { + File file1 + } + + output { + String s1 = read_string(file1) + } +} diff --git a/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala b/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala index 7b198f09f6f..1672bd5bd88 100644 --- a/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala +++ b/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala @@ -230,7 +230,6 @@ case class NativeBlobSasTokenGenerator(container: BlobContainerName, endpoint: E private def authenticateWithSubscription(sub: SubscriptionId) = AzureResourceManager.authenticate(azureCredentialBuilder, azureProfile).withSubscription(sub.toString) private def authenticateWithDefaultSubscription = AzureResourceManager.authenticate(azureCredentialBuilder, azureProfile).withDefaultSubscription() private def azure = subscription.map(authenticateWithSubscription(_)).getOrElse(authenticateWithDefaultSubscription) - private def findAzureStorageAccount(name: StorageAccountName) = azure.storageAccounts.list.asScala.find(_.name.equals(name.value)) .map(Success(_)).getOrElse(Failure(new Exception("Azure Storage Account not found"))) private def buildBlobContainerClient(credential: StorageSharedKeyCredential, endpoint: EndpointURL, container: BlobContainerName): BlobContainerClient = { diff --git a/src/ci/bin/test.inc.sh b/src/ci/bin/test.inc.sh old mode 100644 new mode 100755 index 017fbd95b54..eef3a9fab6b --- a/src/ci/bin/test.inc.sh +++ b/src/ci/bin/test.inc.sh @@ -390,6 +390,9 @@ cromwell::private::create_build_variables() { centaurHoricromtalEngineUpgrade*) CROMWELL_BUILD_CROMWELL_CONFIG="${CROMWELL_BUILD_RESOURCES_DIRECTORY}/papi_v2alpha1_horicromtal_application.conf" ;; + centaurBlob*) + CROMWELL_BUILD_CROMWELL_CONFIG="${CROMWELL_BUILD_RESOURCES_DIRECTORY}/centaur_blob_test.conf" + ;; *) CROMWELL_BUILD_CROMWELL_CONFIG="${CROMWELL_BUILD_RESOURCES_DIRECTORY}/${CROMWELL_BUILD_BACKEND_TYPE}_application.conf" ;; @@ -595,6 +598,7 @@ cromwell::private::create_centaur_variables() { CROMWELL_BUILD_CENTAUR_TYPE_PAPI_UPGRADE_NEW_WORKFLOWS="papiUpgradeNewWorkflows" CROMWELL_BUILD_CENTAUR_TYPE_HORICROMTAL_ENGINE_UPGRADE="horicromtalEngineUpgrade" CROMWELL_BUILD_CENTAUR_TYPE_HORICROMTAL="horicromtal" + CROMWELL_BUILD_CENTAUR_TYPE_AZURE_BLOB="azureBlob" case "${CROMWELL_BUILD_TYPE}" in centaurEngineUpgrade*) @@ -612,6 +616,9 @@ cromwell::private::create_centaur_variables() { centaurHoricromtal*) CROMWELL_BUILD_CENTAUR_TYPE="${CROMWELL_BUILD_CENTAUR_TYPE_HORICROMTAL}" ;; + centaurBlob*) + CROMWELL_BUILD_CENTAUR_TYPE="${CROMWELL_BUILD_CENTAUR_TYPE_AZURE_BLOB}" + ;; *) # Only set the type if Jenkins, etc. has not already set the centaur type if [[ -z "${CROMWELL_BUILD_CENTAUR_TYPE-}" ]]; then diff --git a/src/ci/bin/testCentaurBlob.sh b/src/ci/bin/testCentaurBlob.sh new file mode 100755 index 00000000000..a0075441b44 --- /dev/null +++ b/src/ci/bin/testCentaurBlob.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +set -o errexit -o nounset -o pipefail +# import in shellcheck / CI / IntelliJ compatible ways +# shellcheck source=/dev/null +source "${BASH_SOURCE%/*}/test.inc.sh" || source test.inc.sh + +#NB: This function ensures that the correct .conf file is being used by cromwell (among other things). +#Blob storage requires a configuration file tailored for Azure. +cromwell::build::setup_common_environment + +cromwell::build::setup_centaur_environment + +cromwell::build::assemble_jars + +cromwell::build::run_centaur + +cromwell::build::generate_code_coverage diff --git a/src/ci/resources/centaur_blob_test.conf b/src/ci/resources/centaur_blob_test.conf new file mode 100644 index 00000000000..b97940b856d --- /dev/null +++ b/src/ci/resources/centaur_blob_test.conf @@ -0,0 +1,66 @@ +include required(classpath("application.conf")) +include "build_application.inc.conf" + +filesystems { + blob { + class = "cromwell.filesystems.blob.BlobPathBuilderFactory" + global { + # One BFSM is shared across all BlobPathBuilders + class = "cromwell.filesystems.blob.BlobFileSystemManager" + config { + container: "test-blob" + endpoint: "https://centaurtesting.blob.core.windows.net" + subscription: "62b22893-6bc1-46d9-8a90-806bb3cce3c9" + } + } + } +} + +engine { + filesystems { + local { + enabled: false + } + http { + enabled: false + } + blob { + enabled: true + } + } +} + + +backend { + # Note that the backend is not selected purposefully here, + # the existing tests at the time of writing do not submit tasks to a backend. + default = "TES" + providers { + TES { + actor-factory = "cromwell.backend.impl.tes.TesBackendLifecycleActorFactory" + config { + # Use for running on blob storage + #root = "https://.blob.core.windows.net/cromwell/cromwell-executions" + root = "cromwell-executions" + dockerRoot = "/cromwell-executions" + endpoint = "http://127.0.0.1:9000/v1/tasks" + concurrent-job-limit = 1000 + transform-blob-to-local-path = false + filesystems { + blob { + enabled: true + } + local { + enabled: true + } + http { + enabled: true + } + } + } + + # Have the engine (maybe) authenticate to docker.io. See BT-141 for more info. + include "dockerhub_provider_config_v1.inc.conf" + } + } +}