Refactor Fusion config

Signed-off-by: Paolo Di Tommaso <[email protected]>

modules/nextflow/src/main/groovy/nextflow/fusion/FusionConfig.groovy

@@ -30,17 +30,21 @@ class FusionConfig {
     final static public String DEFAULT_FUSION_AMD64_URL = 'https://fusionfs.seqera.io/releases/v0.6-amd64.json'
     final static public String DEFAULT_FUSION_ARM64_URL = 'https://fusionfs.seqera.io/releases/v0.6-arm64.json'
 
-    final private enabled
+    final private Boolean enabled
     final private String containerConfigUrl
+    final private Boolean exportAwsAccessKeys
 
     boolean enabled() { enabled }
 
+    boolean exportAwsAccessKeys() { exportAwsAccessKeys }
+
     URL containerConfigUrl() {
         this.containerConfigUrl ? new URL(this.containerConfigUrl) : null
     }
 
     FusionConfig(Map opts, Map<String,String> env=System.getenv()) {
         this.enabled = opts.enabled
+        this.exportAwsAccessKeys = opts.exportAwsAccessKeys
         this.containerConfigUrl = opts.containerConfigUrl?.toString() ?: env.get('FUSION_CONTAINER_CONFIG_URL')
         if( containerConfigUrl && (!containerConfigUrl.startsWith('http://') && !containerConfigUrl.startsWith('https://')))
             throw new IllegalArgumentException("Fusion container config URL should start with 'http:' or 'https:' protocol prefix - offending value: $containerConfigUrl")

modules/nextflow/src/main/groovy/nextflow/fusion/FusionEnv.groovy

@@ -0,0 +1,28 @@
+/*
+ * Copyright 2020-2022, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package nextflow.fusion
+
+/**
+ * Allow importing platform specific env variables in the Fusion context
+ *
+ * @author Paolo Di Tommaso <[email protected]>
+ */
+interface FusionEnv {
+
+    Map<String,String> getEnvironment(String scheme, FusionConfig config)
+}

modules/nextflow/src/main/groovy/nextflow/fusion/FusionEnvProvider.groovy

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2020-2022, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package nextflow.fusion
+
+import nextflow.Global
+import nextflow.SysEnv
+import nextflow.plugin.Plugins
+/**
+ * Provider strategy for {@link FusionEnv}
+ *
+ * @author Paolo Di Tommaso <[email protected]>
+ */
+class FusionEnvProvider {
+
+    Map<String,String> getEnvironment(String scheme) {
+        final config = new FusionConfig(Global.config.fusion as Map ?: Collections.emptyMap(), SysEnv.get())
+        final list = Plugins.getExtensions(FusionEnv)
+        final result = new HashMap<String,String>()
+        for( FusionEnv it : list ) {
+            final env = it.getEnvironment(scheme,config)
+            if( env ) result.putAll(env)
+        }
+        return result
+    }
+}

modules/nextflow/src/main/groovy/nextflow/fusion/FusionScriptLauncher.groovy

@@ -17,14 +17,13 @@
 
 package nextflow.fusion
 
-import static FusionHelper.*
+
+import static nextflow.fusion.FusionHelper.*
 
 import java.nio.file.Path
 
 import groovy.transform.CompileStatic
-import groovy.transform.Memoized
 import groovy.util.logging.Slf4j
-import nextflow.Global
 import nextflow.executor.BashWrapperBuilder
 import nextflow.processor.TaskBean
 import nextflow.processor.TaskRun
@@ -99,14 +98,9 @@ class FusionScriptLauncher extends BashWrapperBuilder {
             final result = new LinkedHashMap(10)
             result.NXF_FUSION_WORK = work
             result.NXF_FUSION_BUCKETS = buckets
-            final endpoint = Global.getAwsS3Endpoint()
-            final creds = exportAwsAccessKeys() ? Global.getAwsCredentials() : Collections.<String>emptyList()
-            if( creds ) {
-                result.AWS_ACCESS_KEY_ID = creds[0]
-                result.AWS_SECRET_ACCESS_KEY = creds[1]
-            }
-            if( endpoint )
-                result.AWS_S3_ENDPOINT = endpoint
+            // foreign env
+            final provider = new FusionEnvProvider()
+            result.putAll(provider.getEnvironment(scheme))
             env = result
         }
         return env
@@ -127,12 +121,4 @@ class FusionScriptLauncher extends BashWrapperBuilder {
         return remoteWorkDir.resolve(TaskRun.CMD_INFILE)
     }
 
-    boolean exportAwsAccessKeys() {
-        exportAwsAccessKeys0()
-    }
-
-    @Memoized
-    protected boolean exportAwsAccessKeys0() {
-        return Global.config?.navigate('fusion.exportAwsAccessKeys', false)
-    }
 }

modules/nextflow/src/test/groovy/nextflow/executor/fusion/FusionScriptLauncherTest.groovy

@@ -20,7 +20,7 @@ package nextflow.executor.fusion
 import java.nio.file.Path
 
 import nextflow.Global
-import nextflow.SysEnv
+import nextflow.Session
 import nextflow.file.http.XPath
 import nextflow.fusion.FusionScriptLauncher
 import nextflow.processor.TaskBean
@@ -33,6 +33,8 @@ class FusionScriptLauncherTest extends Specification {
 
     def 'should get container mount' () {
         given:
+        Global.session = Mock(Session) { getConfig() >> [:] }
+        and:
         def fusion = new FusionScriptLauncher(scheme: 'http')
 
         when:
@@ -57,77 +59,16 @@ class FusionScriptLauncherTest extends Specification {
 
     def 'should get fusion env' () {
         given:
-        def fusion = new FusionScriptLauncher(
-                scheme: 'http',
-                buckets: ['foo'] as Set,
-                remoteWorkDir: XPath.get('http://foo/work'))
-
-        expect:
-        fusion.fusionEnv() == [NXF_FUSION_BUCKETS: 'http://foo',
-                               NXF_FUSION_WORK: '/fusion/http/foo/work']
-    }
-
-    def 'should get fusion env with s3 endpoint' () {
-        given:
-        SysEnv.push([AWS_S3_ENDPOINT: 'http://foo.com'])
+        Global.config = [:]
         and:
         def fusion = new FusionScriptLauncher(
                 scheme: 'http',
                 buckets: ['foo'] as Set,
                 remoteWorkDir: XPath.get('http://foo/work'))
 
         expect:
-        fusion.fusionEnv() == [AWS_S3_ENDPOINT: 'http://foo.com',
-                               NXF_FUSION_BUCKETS: 'http://foo',
-                               NXF_FUSION_WORK: '/fusion/http/foo/work']
-
-        cleanup:
-        SysEnv.pop()
-    }
-
-    def 'should get fusion env with aws credentials' () {
-        given:
-        SysEnv.push([AWS_ACCESS_KEY_ID: 'xxx', AWS_SECRET_ACCESS_KEY: 'zzz'])
-        Global.config = [fusion: [exportAwsAccessKeys: true]]
-        and:
-        def fusion = new FusionScriptLauncher(
-                scheme: 'http',
-                buckets: ['foo'] as Set,
-                remoteWorkDir: XPath.get('http://foo/work'))
-
-        expect:
-        fusion.fusionEnv() == [AWS_ACCESS_KEY_ID: 'xxx',
-                               AWS_SECRET_ACCESS_KEY: 'zzz',
-                               NXF_FUSION_BUCKETS: 'http://foo',
-                               NXF_FUSION_WORK: '/fusion/http/foo/work']
-
-        cleanup:
-        Global.config = null
-        SysEnv.pop()
-    }
-
-    def 'should get fusion env with aws credentials in nextflow config' () {
-        given:
-        SysEnv.push([:])
-        and:
-        def CONFIG = [fusion: [exportAwsAccessKeys: true], aws: [accessKey: 'k1', secretKey: 's1', client: [endpoint: 'http://minio.com']]]
-        Global.config = CONFIG
-        and:
-        def fusion = new FusionScriptLauncher(
-                scheme: 'http',
-                buckets: ['foo'] as Set,
-                remoteWorkDir: XPath.get('http://foo/work'))
-
-        expect:
-        fusion.fusionEnv() == [AWS_ACCESS_KEY_ID: 'k1',
-                               AWS_SECRET_ACCESS_KEY: 's1',
-                               AWS_S3_ENDPOINT: 'http://minio.com',
-                               NXF_FUSION_BUCKETS: 'http://foo',
+        fusion.fusionEnv() == [NXF_FUSION_BUCKETS: 'http://foo',
                                NXF_FUSION_WORK: '/fusion/http/foo/work']
-
-        cleanup:
-        Global.config = null
-        SysEnv.pop()
     }
 
     def 'should get header script' () {

modules/nextflow/src/test/groovy/nextflow/fusion/FusionConfigTest.groovy

@@ -56,5 +56,16 @@ class FusionConfigTest extends Specification {
 
     }
 
+    @Unroll
+    def 'should get export aws key' () {
+        expect:
+        new FusionConfig(OPTS).exportAwsAccessKeys() == EXPECTED
+
+        where:
+        OPTS                            | EXPECTED
+        [:]                             | false
+        [exportAwsAccessKeys: false]    | false
+        [exportAwsAccessKeys: true]     | true
+    }
 
 }

plugins/nf-amazon/src/main/nextflow/cloud/aws/fusion/AwsFusionEnv.groovy

@@ -0,0 +1,50 @@
+/*
+ * Copyright 2020-2022, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package nextflow.cloud.aws.fusion
+
+import groovy.transform.CompileStatic
+import nextflow.Global
+import nextflow.fusion.FusionConfig
+import nextflow.fusion.FusionEnv
+import org.pf4j.Extension
+/**
+ * Implements {@link FusionEnv} for AWS cloud
+ *
+ * @author Paolo Di Tommaso <[email protected]>
+ */
+@Extension
+@CompileStatic
+class AwsFusionEnv implements FusionEnv {
+
+    @Override
+    Map<String, String> getEnvironment(String scheme, FusionConfig config) {
+        if( scheme!='s3' )
+            return Collections.<String,String>emptyMap()
+
+        final result = new HashMap<String,String>()
+        final endpoint = Global.getAwsS3Endpoint()
+        final creds = config.exportAwsAccessKeys() ? Global.getAwsCredentials() : Collections.<String>emptyList()
+        if( creds ) {
+            result.AWS_ACCESS_KEY_ID = creds[0]
+            result.AWS_SECRET_ACCESS_KEY = creds[1]
+        }
+        if( endpoint )
+            result.AWS_S3_ENDPOINT = endpoint
+        return result
+    }
+}

plugins/nf-amazon/src/resources/META-INF/extensions.idx

@@ -18,3 +18,4 @@
 nextflow.cloud.aws.batch.AwsBatchExecutor
 nextflow.cloud.aws.util.S3PathSerializer
 nextflow.cloud.aws.util.S3PathFactory
+nextflow.cloud.aws.fusion.AwsFusionEnv

plugins/nf-amazon/src/test/nextflow/cloud/aws/fusion/AwsFusionEnvTest.groovy

@@ -0,0 +1,60 @@
+/*
+ * Copyright 2020-2022, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package nextflow.cloud.aws.fusion
+
+import nextflow.SysEnv
+import nextflow.fusion.FusionConfig
+import spock.lang.Specification
+/**
+ *
+ * @author Paolo Di Tommaso <[email protected]>
+ */
+class AwsFusionEnvTest extends Specification {
+
+    def 'should return empty env' () {
+        given:
+        def provider = new AwsFusionEnv()
+        when:
+        def env = provider.getEnvironment('az', Mock(FusionConfig))
+        then:
+        env == Collections.emptyMap()
+    }
+
+    def 'should return env environment' () {
+        given:
+        SysEnv.push([AWS_ACCESS_KEY_ID: 'x1', AWS_SECRET_ACCESS_KEY: 'y1', AWS_S3_ENDPOINT: 'http://my-host.com'])
+        and:
+
+        when:
+        def config = Mock(FusionConfig)
+        def env = new AwsFusionEnv().getEnvironment('s3', Mock(FusionConfig))
+        then:
+        env == [AWS_S3_ENDPOINT:'http://my-host.com']
+
+        when:
+        config = Mock(FusionConfig) { exportAwsAccessKeys() >> true }
+        env = new AwsFusionEnv().getEnvironment('s3', config)
+        then:
+        env == [AWS_ACCESS_KEY_ID: 'x1',
+                AWS_SECRET_ACCESS_KEY: 'y1',
+                AWS_S3_ENDPOINT:'http://my-host.com']
+
+        cleanup:
+        SysEnv.pop()
+    }
+}