Allow identity based authentication on Azure Batch (#3132)

Signed-off-by: Abhinav Sharma <[email protected]> Signed-off-by: Abhinav Sharma <[email protected]> Signed-off-by: Paolo Di Tommaso <[email protected]> Co-authored-by: Paolo Di Tommaso <[email protected]>
nextflow-io · Nov 4, 2022 · a08611b · a08611b
1 parent 9ec80d4
commit a08611b
Show file tree

Hide file tree

Showing 10 changed files with 273 additions and 43 deletions.
diff --git a/docs/azure.rst b/docs/azure.rst
@@ -308,13 +308,13 @@ Together, these settings determine the Operating System and version installed on
 By default, Nextflow creates CentOS 8-based pool nodes, but this behavior can be customised in the pool configuration.
 Below the configurations for image reference/SKU combinations to select two popular systems.
 
-* Ubuntu 20.04::
+* Ubuntu 20.04 (default)::
 
 	azure.batch.pools.<name>.sku = "batch.node.ubuntu 20.04"
 	azure.batch.pools.<name>.offer = "ubuntu-server-container"
 	azure.batch.pools.<name>.publisher = "microsoft-azure-batch"
 
-* CentOS 8 (default)::
+* CentOS 8::
 
 	azure.batch.pools.<name>.sku = "batch.node.centos 8"
 	azure.batch.pools.<name>.offer = "centos-container"
@@ -345,6 +345,42 @@ Public images from other registries are still pulled (if requested by a Task) wh
   specified via the :ref:`container <process-container>` directive using the format: ``[server]/[your-organization]/[your-image]:[tag]``.
   Read more about image fully qualified image names in the `Docker documentation <https://docs.docker.com/engine/reference/commandline/pull/#pull-from-a-different-registry>`_.
 
+Active Directory Authentication
+===============================
+
+As of version ``22.11.0-edge``, `Service Principal <https://learn.microsoft.com/en-us/azure/active-directory/develop/howto-create-service-principal-portal>`_ credentials can optionally be used instead of Shared Keys for Azure Batch and Storage accounts. 
+
+The Service Principal should have the at least the following role assignments :
+
+1. Contributor
+
+2. Storage Blob Data Reader
+
+3. Storage Blob Data Contributor
+
+.. note::
+  To assign the necessary roles to the Service Principal refer to the `official Azure documentation <https://learn.microsoft.com/en-us/azure/role-based-access-control/role-assignments-portal?tabs=current>`_.
+
+The credentials for Service Principal can be specified as follows::
+
+    azure {
+        activeDirectory {
+            servicePrincipalId = '<YOUR SERVICE PRINCIPAL CLIENT ID>'
+            servicePrincipalSecret = '<YOUR SERVICE PRINCIPAL CLIENT SECRET>'
+            tenantId = '<YOUR TENANT ID>'
+          }
+
+        storage {
+            accountName = '<YOUR STORAGE ACCOUNT NAME>'
+          }
+
+        batch {
+            accountName = '<YOUR BATCH ACCOUNT NAME>'
+            location = '<YOUR BATCH ACCOUNT LOCATION>'
+          }
+    }
+
+
 Advanced settings
 ==================
 
@@ -353,10 +389,13 @@ The following configuration options are available:
 ============================================== =================
 Name                                           Description
 ============================================== =================
+azure.activeDirectory.servicePrincipalId        The service principal client ID
+azure.activeDirectory.servicePrincipalSecret    The service principal client secret
+azure.activeDirectory.tenantId                  The Azure tenant ID
 azure.storage.accountName                       The blob storage account name
 azure.storage.accountKey                        The blob storage account key
 azure.storage.sasToken                          The blob storage shared access signature token. This can be provided as an alternative to the ``accountKey`` setting.
-azure.storage.tokenDuration                     The duration of the shared access signature token created by Nextflow when the ``sasToken`` option is *not* specified (default: ``12h``).
+azure.storage.tokenDuration                     The duration of the shared access signature token created by Nextflow when the ``sasToken`` option is *not* specified (default: ``48h``).
 azure.batch.accountName                         The batch service account name.
 azure.batch.accountKey                          The batch service account key.
 azure.batch.endpoint                            The batch service endpoint e.g. ``https://nfbatch1.westeurope.batch.azure.com``.

diff --git a/plugins/nf-azure/build.gradle b/plugins/nf-azure/build.gradle
@@ -36,13 +36,20 @@ dependencies {
     compileOnly project(':nextflow')
     compileOnly 'org.slf4j:slf4j-api:1.7.10'
     compileOnly 'org.pf4j:pf4j:3.4.1'
-    api('com.azure:azure-storage-blob:12.9.0') {
+
+    api('com.azure:azure-storage-blob:12.19.0') {
         exclude group: 'org.slf4j', module: 'slf4j-api'
     }
-    api('com.microsoft.azure:azure-batch:9.0.0') {
+    api('com.microsoft.azure:azure-batch:10.0.0') {
         exclude group: 'org.slf4j', module: 'slf4j-api'
         exclude group: 'com.google.guava', module: 'guava'
     }
+    api('com.azure:azure-identity:1.5.5') {
+        exclude group: 'org.slf4j', module: 'slf4j-api'
+        exclude group: 'com.nimbusds', module: 'oauth2-oidc-sdk'
+    }
+
+    compileOnly(group: 'com.nimbusds', name: 'oauth2-oidc-sdk', version: '9.43')
 
     testImplementation(testFixtures(project(":nextflow")))
     testImplementation project(':nextflow')

diff --git a/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzBatchExecutor.groovy b/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzBatchExecutor.groovy
@@ -65,7 +65,7 @@ class AzBatchExecutor extends Executor implements ExtensionPoint {
 
     protected void validateWorkDir() {
         /*
-         * make sure the work dir is a S3 bucket
+         * make sure the work dir is an Azure bucket
          */
         if( !(workDir instanceof AzPath) ) {
             session.abort()
@@ -94,11 +94,15 @@ class AzBatchExecutor extends Executor implements ExtensionPoint {
     protected void initBatchService() {
         config = AzConfig.getConfig(session)
         batchService = new AzBatchService(this)
-        // generate an account SAS token if missing
-        if( !config.storage().sasToken )
-            config.storage().sasToken = AzHelper.generateAccountSas(workDir, config.storage().tokenDuration)
 
-        Global.onCleanup((it)->batchService.close())
+        // Generate an account SAS token using either activeDirectory configs or storage account keys
+        if (!config.storage().sasToken) {
+            config.storage().sasToken = config.activeDirectory().isConfigured()
+                    ? AzHelper.generateContainerSasWithActiveDirectory(workDir, config.storage().tokenDuration)
+                    : AzHelper.generateAccountSasWithAccountKey(workDir, config.storage().tokenDuration)
+        }
+
+        Global.onCleanup((it) -> batchService.close())
     }
 
     /**

diff --git a/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzBatchService.groovy b/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzBatchService.groovy
@@ -16,6 +16,9 @@
 
 package nextflow.cloud.azure.batch
 
+import com.microsoft.azure.batch.auth.BatchApplicationTokenCredentials
+import com.microsoft.azure.batch.auth.BatchCredentials
+
 import java.math.RoundingMode
 import java.nio.file.Path
 import java.time.Instant
@@ -254,19 +257,54 @@ class AzBatchService implements Closeable {
         result.setScale(0, RoundingMode.UP).intValue()
     }
 
+
+    protected createBatchCredentialsWithKey() {
+        log.debug "[AZURE BATCH] Creating Azure Batch client using shared key creddentials"
+
+        if (config.batch().endpoint || config.batch().accountKey || config.batch().accountName) {
+            // Create batch client
+            if (!config.batch().endpoint)
+                throw new IllegalArgumentException("Missing Azure Batch endpoint -- Specify it in the nextflow.config file using the setting 'azure.batch.endpoint'")
+            if (!config.batch().accountName)
+                throw new IllegalArgumentException("Missing Azure Batch account name -- Specify it in the nextflow.config file using the setting 'azure.batch.accountName'")
+            if (!config.batch().accountKey)
+                throw new IllegalArgumentException("Missing Azure Batch account key -- Specify it in the nextflow.config file using the setting 'azure.batch.accountKet'")
+
+            return new BatchSharedKeyCredentials(config.batch().endpoint, config.batch().accountName, config.batch().accountKey)
+
+        }
+    }
+
+    protected createBatchCredentialsWithServicePrincipal() {
+        log.debug "[AZURE BATCH] Creating Azure Batch client using Service Principal credentials"
+
+        final batchEndpoint = "https://batch.core.windows.net/";
+        final authenticationEndpoint = "https://login.microsoftonline.com/";
+
+        def servicePrincipalBasedCred = new BatchApplicationTokenCredentials(
+                config.batch().endpoint,
+                config.activeDirectory().servicePrincipalId,
+                config.activeDirectory().servicePrincipalSecret,
+                config.activeDirectory().tenantId,
+                batchEndpoint,
+                authenticationEndpoint
+        )
+
+        return servicePrincipalBasedCred
+    }
+
     protected BatchClient createBatchClient() {
         log.debug "[AZURE BATCH] Executor options=${config.batch()}"
+
+        def cred = config.activeDirectory().isConfigured()
+                ? createBatchCredentialsWithServicePrincipal()
+                : createBatchCredentialsWithKey()
+
         // Create batch client
-        if( !config.batch().endpoint )
-            throw new IllegalArgumentException("Missing Azure Batch endpoint -- Specify it in the nextflow.config file using the setting 'azure.batch.endpoint'")
-        if( !config.batch().accountName )
-            throw new IllegalArgumentException("Missing Azure Batch account name -- Specify it in the nextflow.config file using the setting 'azure.batch.accountName'")
-        if( !config.batch().accountKey )
-            throw new IllegalArgumentException("Missing Azure Batch account key -- Specify it in the nextflow.config file using the setting 'azure.batch.accountKet'")
-
-        final cred = new BatchSharedKeyCredentials(config.batch().endpoint, config.batch().accountName, config.batch().accountKey)
-        final client = BatchClient.open(cred)
+        def client = BatchClient.open(cred as BatchCredentials)
+
         Global.onCleanup((it)->client.protocolLayer().restClient().close())
+
         return client
     }
 

diff --git a/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzHelper.groovy b/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzHelper.groovy
@@ -14,7 +14,9 @@
  * limitations under the License.
  */
 package nextflow.cloud.azure.batch
+
 import com.azure.storage.blob.BlobServiceClient
+import com.azure.storage.blob.models.UserDelegationKey
 import com.azure.storage.common.sas.AccountSasPermission
 import com.azure.storage.common.sas.AccountSasResourceType
 import com.azure.storage.common.sas.AccountSasService
@@ -56,14 +58,6 @@ class AzHelper {
         return !sas ? url : "${url}?${sas}"
     }
 
-    static String generateContainerSas(Path path, Duration duration) {
-        generateSas(az0(path).containerClient(), duration)
-    }
-
-    static String generateAccountSas(Path path, Duration duration) {
-        generateAccountSas(az0(path).getFileSystem().getBlobServiceClient(), duration)
-    }
-
     static BlobContainerSasPermission CONTAINER_PERMS = new BlobContainerSasPermission()
             .setAddPermission(true)
             .setCreatePermission(true)
@@ -103,16 +97,55 @@ class AzHelper {
             .setObject(true)
             .setService(true)
 
-    static String generateSas(BlobContainerClient client, Duration duration) {
-        final now = OffsetDateTime .now()
+
+    static String generateContainerSasWithActiveDirectory(Path path, Duration duration) {
+        final key = generateUserDelegationKey(az0(path), duration)
+
+        return generateContainerUserDelegationSas(az0(path).containerClient(), duration, key)
+    }
+
+    static String generateAccountSasWithAccountKey(Path path, Duration duration) {
+        generateAccountSas(az0(path).getFileSystem().getBlobServiceClient(), duration)
+    }
+
+    static UserDelegationKey generateUserDelegationKey(Path path, Duration duration) {
+
+        final client = az0(path).getFileSystem().getBlobServiceClient()
+
+        final startTime = OffsetDateTime.now()
+        final indicatedExpiryTime = startTime.plusHours(duration.toHours())
+
+        // The maximum lifetime for user delegation key (and therefore delegation SAS) is 7 days
+        // Reference https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blob-user-delegation-sas-create-cli
+        final maxExpiryTime = startTime.plusDays(7)
+
+        final expiryTime = (indicatedExpiryTime.toEpochSecond() <= maxExpiryTime.toEpochSecond()) ? indicatedExpiryTime : maxExpiryTime
+
+        final delegationKey = client.getUserDelegationKey(startTime, expiryTime)
+
+        return delegationKey
+    }
+
+    static String generateContainerUserDelegationSas(BlobContainerClient client, Duration duration, UserDelegationKey key) {
+
+        final startTime = OffsetDateTime.now()
+        final indicatedExpiryTime = startTime.plusHours(duration.toHours())
+
+        // The maximum lifetime for user delegation key (and therefore delegation SAS) is 7 days
+        // Reference https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blob-user-delegation-sas-create-cli
+        final maxExpiryTime = startTime.plusDays(7)
+
+        final expiryTime = (indicatedExpiryTime.toEpochSecond() <= maxExpiryTime.toEpochSecond()) ? indicatedExpiryTime : maxExpiryTime
 
         final signature = new BlobServiceSasSignatureValues()
                 .setPermissions(BLOB_PERMS)
                 .setPermissions(CONTAINER_PERMS)
-                .setStartTime(now)
-                .setExpiryTime( now.plusSeconds(duration.toSeconds()) )
+                .setStartTime(startTime)
+                .setExpiryTime(expiryTime)
+
+        final generatedSas = client.generateUserDelegationSas(signature, key)
 
-        return client .generateSas(signature)
+        return generatedSas
     }
 
     static String generateAccountSas(BlobServiceClient client, Duration duration) {

diff --git a/plugins/nf-azure/src/main/nextflow/cloud/azure/config/AzActiveDirectoryOpts.groovy b/plugins/nf-azure/src/main/nextflow/cloud/azure/config/AzActiveDirectoryOpts.groovy
@@ -0,0 +1,59 @@
+/*
+ * Copyright 2022, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package nextflow.cloud.azure.config
+
+import groovy.transform.CompileStatic
+import nextflow.cloud.azure.nio.AzFileSystemProvider
+
+/**
+ * Model Azure identity options from nextflow config file
+ *
+ * @author Abhinav Sharma <[email protected]>
+ */
+@CompileStatic
+class AzActiveDirectoryOpts {
+
+    private Map<String, String> sysEnv
+
+    String servicePrincipalId
+    String servicePrincipalSecret
+    String tenantId
+
+    AzActiveDirectoryOpts(Map config, Map<String, String> env = null) {
+        assert config != null
+        this.sysEnv = env == null ? new HashMap<String, String>(System.getenv()) : env
+        this.servicePrincipalId = config.servicePrincipalId ?: sysEnv.get('AZURE_CLIENT_ID')
+        this.servicePrincipalSecret = config.servicePrincipalSecret ?: sysEnv.get('AZURE_CLIENT_SECRET')
+        this.tenantId = config.tenantId ?: sysEnv.get('AZURE_TENANT_ID')
+    }
+
+    Map<String, Object> getEnv() {
+        Map<String, Object> props = new HashMap<>();
+        props.put(AzFileSystemProvider.AZURE_CLIENT_ID, servicePrincipalId)
+        props.put(AzFileSystemProvider.AZURE_CLIENT_SECRET, servicePrincipalSecret)
+        props.put(AzFileSystemProvider.AZURE_TENANT_ID, tenantId)
+        return props
+    }
+
+    boolean isConfigured() {
+        if (servicePrincipalId && servicePrincipalSecret && tenantId)
+            return true
+        if (!servicePrincipalId && !servicePrincipalSecret && !tenantId)
+            return false
+        throw new IllegalArgumentException("Invalid Service Principal configuration - Make sure servicePrincipalId and servicePrincipalClient are set in nextflow.config or configured via environment variables")
+    }
+
+}
diff --git a/plugins/nf-azure/src/main/nextflow/cloud/azure/config/AzConfig.groovy b/plugins/nf-azure/src/main/nextflow/cloud/azure/config/AzConfig.groovy
@@ -38,12 +38,15 @@ class AzConfig {
 
     private AzRetryConfig retryConfig
 
+    private AzActiveDirectoryOpts activeDirectoryOpts
+
     AzConfig(Map azure) {
         this.batchOpts = new AzBatchOpts( (Map)azure.batch ?: Collections.emptyMap() )
         this.storageOpts = new AzStorageOpts( (Map)azure.storage ?: Collections.emptyMap() )
         this.registryOpts = new AzRegistryOpts( (Map)azure.registry ?: Collections.emptyMap() )
         this.azcopyOpts = new AzCopyOpts( (Map)azure.azcopy ?: Collections.emptyMap() )
         this.retryConfig = new AzRetryConfig( (Map)azure.retryPolicy ?: Collections.emptyMap() )
+        this.activeDirectoryOpts = new AzActiveDirectoryOpts((Map) azure.activeDirectory ?: Collections.emptyMap())
     }
 
     AzCopyOpts azcopy() { azcopyOpts }
@@ -56,6 +59,8 @@ class AzConfig {
 
     AzRetryConfig retryConfig() { retryConfig }
 
+    AzActiveDirectoryOpts activeDirectory() { activeDirectoryOpts }
+
     static AzConfig getConfig(Session session) {
         if( !session )
             throw new IllegalStateException("Missing Nextflow session")

diff --git a/plugins/nf-azure/src/main/nextflow/cloud/azure/config/AzStorageOpts.groovy b/plugins/nf-azure/src/main/nextflow/cloud/azure/config/AzStorageOpts.groovy
@@ -43,9 +43,9 @@ class AzStorageOpts {
         this.accountKey = config.accountKey ?: sysEnv.get('AZURE_STORAGE_ACCOUNT_KEY')
         this.accountName = config.accountName ?: sysEnv.get('AZURE_STORAGE_ACCOUNT_NAME')
         this.sasToken = config.sasToken
-        this.tokenDuration = (config.tokenDuration as Duration) ?: Duration.of('12h')
-        this.fileShares = parseFileShares(config.fileShares instanceof Map ? config.fileShares as Map<String,Map>
-                : Collections.<String,Map>emptyMap())
+        this.tokenDuration = (config.tokenDuration as Duration) ?: Duration.of('48h')
+        this.fileShares = parseFileShares(config.fileShares instanceof Map ? config.fileShares as Map<String, Map>
+                : Collections.<String,Map> emptyMap())
 
     }