From 902e5b347f749ee8ec86f6e5c38e86919673455b Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Fri, 16 Dec 2022 20:40:52 +0100 Subject: [PATCH] Refactor Fusion config Signed-off-by: Paolo Di Tommaso --- .../nextflow/fusion/FusionConfig.groovy | 6 +- .../groovy/nextflow/fusion/FusionEnv.groovy | 28 ++++++++ .../nextflow/fusion/FusionEnvProvider.groovy | 40 +++++++++++ .../fusion/FusionScriptLauncher.groovy | 24 ++----- .../fusion/FusionScriptLauncherTest.groovy | 69 ++----------------- .../nextflow/fusion/FusionConfigTest.groovy | 11 +++ .../cloud/aws/fusion/AwsFusionEnv.groovy | 50 ++++++++++++++ .../src/resources/META-INF/extensions.idx | 1 + .../cloud/aws/fusion/AwsFusionEnvTest.groovy | 60 ++++++++++++++++ .../FusionScriptLauncherS3Test.groovy | 69 +++++++++++++++++++ 10 files changed, 274 insertions(+), 84 deletions(-) create mode 100644 modules/nextflow/src/main/groovy/nextflow/fusion/FusionEnv.groovy create mode 100644 modules/nextflow/src/main/groovy/nextflow/fusion/FusionEnvProvider.groovy create mode 100644 plugins/nf-amazon/src/main/nextflow/cloud/aws/fusion/AwsFusionEnv.groovy create mode 100644 plugins/nf-amazon/src/test/nextflow/cloud/aws/fusion/AwsFusionEnvTest.groovy diff --git a/modules/nextflow/src/main/groovy/nextflow/fusion/FusionConfig.groovy b/modules/nextflow/src/main/groovy/nextflow/fusion/FusionConfig.groovy index 20a48e4afc..3a3d7c0b0b 100644 --- a/modules/nextflow/src/main/groovy/nextflow/fusion/FusionConfig.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/fusion/FusionConfig.groovy @@ -30,17 +30,21 @@ class FusionConfig { final static public String DEFAULT_FUSION_AMD64_URL = 'https://fusionfs.seqera.io/releases/v0.6-amd64.json' final static public String DEFAULT_FUSION_ARM64_URL = 'https://fusionfs.seqera.io/releases/v0.6-arm64.json' - final private enabled + final private Boolean enabled final private String containerConfigUrl + final private Boolean exportAwsAccessKeys boolean enabled() { enabled } + boolean exportAwsAccessKeys() { exportAwsAccessKeys } + URL containerConfigUrl() { this.containerConfigUrl ? new URL(this.containerConfigUrl) : null } FusionConfig(Map opts, Map env=System.getenv()) { this.enabled = opts.enabled + this.exportAwsAccessKeys = opts.exportAwsAccessKeys this.containerConfigUrl = opts.containerConfigUrl?.toString() ?: env.get('FUSION_CONTAINER_CONFIG_URL') if( containerConfigUrl && (!containerConfigUrl.startsWith('http://') && !containerConfigUrl.startsWith('https://'))) throw new IllegalArgumentException("Fusion container config URL should start with 'http:' or 'https:' protocol prefix - offending value: $containerConfigUrl") diff --git a/modules/nextflow/src/main/groovy/nextflow/fusion/FusionEnv.groovy b/modules/nextflow/src/main/groovy/nextflow/fusion/FusionEnv.groovy new file mode 100644 index 0000000000..1b6dc22b11 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/fusion/FusionEnv.groovy @@ -0,0 +1,28 @@ +/* + * Copyright 2020-2022, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.fusion + +/** + * Allow importing platform specific env variables in the Fusion context + * + * @author Paolo Di Tommaso + */ +interface FusionEnv { + + Map getEnvironment(String scheme, FusionConfig config) +} diff --git a/modules/nextflow/src/main/groovy/nextflow/fusion/FusionEnvProvider.groovy b/modules/nextflow/src/main/groovy/nextflow/fusion/FusionEnvProvider.groovy new file mode 100644 index 0000000000..cc03721cbb --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/fusion/FusionEnvProvider.groovy @@ -0,0 +1,40 @@ +/* + * Copyright 2020-2022, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.fusion + +import nextflow.Global +import nextflow.SysEnv +import nextflow.plugin.Plugins +/** + * Provider strategy for {@link FusionEnv} + * + * @author Paolo Di Tommaso + */ +class FusionEnvProvider { + + Map getEnvironment(String scheme) { + final config = new FusionConfig(Global.config.fusion as Map ?: Collections.emptyMap(), SysEnv.get()) + final list = Plugins.getExtensions(FusionEnv) + final result = new HashMap() + for( FusionEnv it : list ) { + final env = it.getEnvironment(scheme,config) + if( env ) result.putAll(env) + } + return result + } +} diff --git a/modules/nextflow/src/main/groovy/nextflow/fusion/FusionScriptLauncher.groovy b/modules/nextflow/src/main/groovy/nextflow/fusion/FusionScriptLauncher.groovy index 38cd62c009..16d852ea51 100644 --- a/modules/nextflow/src/main/groovy/nextflow/fusion/FusionScriptLauncher.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/fusion/FusionScriptLauncher.groovy @@ -17,14 +17,13 @@ package nextflow.fusion -import static FusionHelper.* + +import static nextflow.fusion.FusionHelper.* import java.nio.file.Path import groovy.transform.CompileStatic -import groovy.transform.Memoized import groovy.util.logging.Slf4j -import nextflow.Global import nextflow.executor.BashWrapperBuilder import nextflow.processor.TaskBean import nextflow.processor.TaskRun @@ -99,14 +98,9 @@ class FusionScriptLauncher extends BashWrapperBuilder { final result = new LinkedHashMap(10) result.NXF_FUSION_WORK = work result.NXF_FUSION_BUCKETS = buckets - final endpoint = Global.getAwsS3Endpoint() - final creds = exportAwsAccessKeys() ? Global.getAwsCredentials() : Collections.emptyList() - if( creds ) { - result.AWS_ACCESS_KEY_ID = creds[0] - result.AWS_SECRET_ACCESS_KEY = creds[1] - } - if( endpoint ) - result.AWS_S3_ENDPOINT = endpoint + // foreign env + final provider = new FusionEnvProvider() + result.putAll(provider.getEnvironment(scheme)) env = result } return env @@ -127,12 +121,4 @@ class FusionScriptLauncher extends BashWrapperBuilder { return remoteWorkDir.resolve(TaskRun.CMD_INFILE) } - boolean exportAwsAccessKeys() { - exportAwsAccessKeys0() - } - - @Memoized - protected boolean exportAwsAccessKeys0() { - return Global.config?.navigate('fusion.exportAwsAccessKeys', false) - } } diff --git a/modules/nextflow/src/test/groovy/nextflow/executor/fusion/FusionScriptLauncherTest.groovy b/modules/nextflow/src/test/groovy/nextflow/executor/fusion/FusionScriptLauncherTest.groovy index 5f17f9d7c7..d2521f4ae7 100644 --- a/modules/nextflow/src/test/groovy/nextflow/executor/fusion/FusionScriptLauncherTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/executor/fusion/FusionScriptLauncherTest.groovy @@ -20,7 +20,7 @@ package nextflow.executor.fusion import java.nio.file.Path import nextflow.Global -import nextflow.SysEnv +import nextflow.Session import nextflow.file.http.XPath import nextflow.fusion.FusionScriptLauncher import nextflow.processor.TaskBean @@ -33,6 +33,8 @@ class FusionScriptLauncherTest extends Specification { def 'should get container mount' () { given: + Global.session = Mock(Session) { getConfig() >> [:] } + and: def fusion = new FusionScriptLauncher(scheme: 'http') when: @@ -57,19 +59,7 @@ class FusionScriptLauncherTest extends Specification { def 'should get fusion env' () { given: - def fusion = new FusionScriptLauncher( - scheme: 'http', - buckets: ['foo'] as Set, - remoteWorkDir: XPath.get('http://foo/work')) - - expect: - fusion.fusionEnv() == [NXF_FUSION_BUCKETS: 'http://foo', - NXF_FUSION_WORK: '/fusion/http/foo/work'] - } - - def 'should get fusion env with s3 endpoint' () { - given: - SysEnv.push([AWS_S3_ENDPOINT: 'http://foo.com']) + Global.config = [:] and: def fusion = new FusionScriptLauncher( scheme: 'http', @@ -77,57 +67,8 @@ class FusionScriptLauncherTest extends Specification { remoteWorkDir: XPath.get('http://foo/work')) expect: - fusion.fusionEnv() == [AWS_S3_ENDPOINT: 'http://foo.com', - NXF_FUSION_BUCKETS: 'http://foo', - NXF_FUSION_WORK: '/fusion/http/foo/work'] - - cleanup: - SysEnv.pop() - } - - def 'should get fusion env with aws credentials' () { - given: - SysEnv.push([AWS_ACCESS_KEY_ID: 'xxx', AWS_SECRET_ACCESS_KEY: 'zzz']) - Global.config = [fusion: [exportAwsAccessKeys: true]] - and: - def fusion = new FusionScriptLauncher( - scheme: 'http', - buckets: ['foo'] as Set, - remoteWorkDir: XPath.get('http://foo/work')) - - expect: - fusion.fusionEnv() == [AWS_ACCESS_KEY_ID: 'xxx', - AWS_SECRET_ACCESS_KEY: 'zzz', - NXF_FUSION_BUCKETS: 'http://foo', - NXF_FUSION_WORK: '/fusion/http/foo/work'] - - cleanup: - Global.config = null - SysEnv.pop() - } - - def 'should get fusion env with aws credentials in nextflow config' () { - given: - SysEnv.push([:]) - and: - def CONFIG = [fusion: [exportAwsAccessKeys: true], aws: [accessKey: 'k1', secretKey: 's1', client: [endpoint: 'http://minio.com']]] - Global.config = CONFIG - and: - def fusion = new FusionScriptLauncher( - scheme: 'http', - buckets: ['foo'] as Set, - remoteWorkDir: XPath.get('http://foo/work')) - - expect: - fusion.fusionEnv() == [AWS_ACCESS_KEY_ID: 'k1', - AWS_SECRET_ACCESS_KEY: 's1', - AWS_S3_ENDPOINT: 'http://minio.com', - NXF_FUSION_BUCKETS: 'http://foo', + fusion.fusionEnv() == [NXF_FUSION_BUCKETS: 'http://foo', NXF_FUSION_WORK: '/fusion/http/foo/work'] - - cleanup: - Global.config = null - SysEnv.pop() } def 'should get header script' () { diff --git a/modules/nextflow/src/test/groovy/nextflow/fusion/FusionConfigTest.groovy b/modules/nextflow/src/test/groovy/nextflow/fusion/FusionConfigTest.groovy index c25756b7f6..7b5bbd5225 100644 --- a/modules/nextflow/src/test/groovy/nextflow/fusion/FusionConfigTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/fusion/FusionConfigTest.groovy @@ -56,5 +56,16 @@ class FusionConfigTest extends Specification { } + @Unroll + def 'should get export aws key' () { + expect: + new FusionConfig(OPTS).exportAwsAccessKeys() == EXPECTED + + where: + OPTS | EXPECTED + [:] | false + [exportAwsAccessKeys: false] | false + [exportAwsAccessKeys: true] | true + } } diff --git a/plugins/nf-amazon/src/main/nextflow/cloud/aws/fusion/AwsFusionEnv.groovy b/plugins/nf-amazon/src/main/nextflow/cloud/aws/fusion/AwsFusionEnv.groovy new file mode 100644 index 0000000000..b9c6e9da7e --- /dev/null +++ b/plugins/nf-amazon/src/main/nextflow/cloud/aws/fusion/AwsFusionEnv.groovy @@ -0,0 +1,50 @@ +/* + * Copyright 2020-2022, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.cloud.aws.fusion + +import groovy.transform.CompileStatic +import nextflow.Global +import nextflow.fusion.FusionConfig +import nextflow.fusion.FusionEnv +import org.pf4j.Extension +/** + * Implements {@link FusionEnv} for AWS cloud + * + * @author Paolo Di Tommaso + */ +@Extension +@CompileStatic +class AwsFusionEnv implements FusionEnv { + + @Override + Map getEnvironment(String scheme, FusionConfig config) { + if( scheme!='s3' ) + return Collections.emptyMap() + + final result = new HashMap() + final endpoint = Global.getAwsS3Endpoint() + final creds = config.exportAwsAccessKeys() ? Global.getAwsCredentials() : Collections.emptyList() + if( creds ) { + result.AWS_ACCESS_KEY_ID = creds[0] + result.AWS_SECRET_ACCESS_KEY = creds[1] + } + if( endpoint ) + result.AWS_S3_ENDPOINT = endpoint + return result + } +} diff --git a/plugins/nf-amazon/src/resources/META-INF/extensions.idx b/plugins/nf-amazon/src/resources/META-INF/extensions.idx index b8e38da893..c99fc5b567 100644 --- a/plugins/nf-amazon/src/resources/META-INF/extensions.idx +++ b/plugins/nf-amazon/src/resources/META-INF/extensions.idx @@ -18,3 +18,4 @@ nextflow.cloud.aws.batch.AwsBatchExecutor nextflow.cloud.aws.util.S3PathSerializer nextflow.cloud.aws.util.S3PathFactory +nextflow.cloud.aws.fusion.AwsFusionEnv diff --git a/plugins/nf-amazon/src/test/nextflow/cloud/aws/fusion/AwsFusionEnvTest.groovy b/plugins/nf-amazon/src/test/nextflow/cloud/aws/fusion/AwsFusionEnvTest.groovy new file mode 100644 index 0000000000..fab93189bd --- /dev/null +++ b/plugins/nf-amazon/src/test/nextflow/cloud/aws/fusion/AwsFusionEnvTest.groovy @@ -0,0 +1,60 @@ +/* + * Copyright 2020-2022, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.cloud.aws.fusion + +import nextflow.SysEnv +import nextflow.fusion.FusionConfig +import spock.lang.Specification +/** + * + * @author Paolo Di Tommaso + */ +class AwsFusionEnvTest extends Specification { + + def 'should return empty env' () { + given: + def provider = new AwsFusionEnv() + when: + def env = provider.getEnvironment('az', Mock(FusionConfig)) + then: + env == Collections.emptyMap() + } + + def 'should return env environment' () { + given: + SysEnv.push([AWS_ACCESS_KEY_ID: 'x1', AWS_SECRET_ACCESS_KEY: 'y1', AWS_S3_ENDPOINT: 'http://my-host.com']) + and: + + when: + def config = Mock(FusionConfig) + def env = new AwsFusionEnv().getEnvironment('s3', Mock(FusionConfig)) + then: + env == [AWS_S3_ENDPOINT:'http://my-host.com'] + + when: + config = Mock(FusionConfig) { exportAwsAccessKeys() >> true } + env = new AwsFusionEnv().getEnvironment('s3', config) + then: + env == [AWS_ACCESS_KEY_ID: 'x1', + AWS_SECRET_ACCESS_KEY: 'y1', + AWS_S3_ENDPOINT:'http://my-host.com'] + + cleanup: + SysEnv.pop() + } +} diff --git a/plugins/nf-amazon/src/test/nextflow/executor/FusionScriptLauncherS3Test.groovy b/plugins/nf-amazon/src/test/nextflow/executor/FusionScriptLauncherS3Test.groovy index 47b5aff8fd..a49873fd1f 100644 --- a/plugins/nf-amazon/src/test/nextflow/executor/FusionScriptLauncherS3Test.groovy +++ b/plugins/nf-amazon/src/test/nextflow/executor/FusionScriptLauncherS3Test.groovy @@ -9,6 +9,8 @@ package nextflow.executor import java.nio.file.Path +import nextflow.Global +import nextflow.SysEnv import nextflow.cloud.aws.util.S3PathFactory import nextflow.fusion.FusionScriptLauncher import spock.lang.Specification @@ -43,4 +45,71 @@ class FusionScriptLauncherS3Test extends Specification { } + + def 'should get fusion env with s3 endpoint' () { + given: + Global.config = [:] + and: + SysEnv.push([AWS_S3_ENDPOINT: 'http://foo.com']) + and: + def fusion = new FusionScriptLauncher( + scheme: 's3', + buckets: ['foo'] as Set, + remoteWorkDir: S3PathFactory.parse('s3://foo/work')) + + expect: + fusion.fusionEnv() == [AWS_S3_ENDPOINT: 'http://foo.com', + NXF_FUSION_BUCKETS: 's3://foo', + NXF_FUSION_WORK: '/fusion/s3/foo/work'] + + cleanup: + SysEnv.pop() + } + + def 'should get fusion env with aws credentials' () { + given: + SysEnv.push([AWS_ACCESS_KEY_ID: 'xxx', AWS_SECRET_ACCESS_KEY: 'zzz']) + and: + Global.config = [fusion: [exportAwsAccessKeys: true]] + and: + def fusion = new FusionScriptLauncher( + scheme: 's3', + buckets: ['foo'] as Set, + remoteWorkDir: S3PathFactory.parse('s3://foo/work')) + + expect: + fusion.fusionEnv() == [AWS_ACCESS_KEY_ID: 'xxx', + AWS_SECRET_ACCESS_KEY: 'zzz', + NXF_FUSION_BUCKETS: 's3://foo', + NXF_FUSION_WORK: '/fusion/s3/foo/work'] + + cleanup: + Global.config = null + SysEnv.pop() + } + + def 'should get fusion env with aws credentials in nextflow config' () { + given: + SysEnv.push([:]) + and: + def CONFIG = [fusion: [exportAwsAccessKeys: true], aws: [accessKey: 'k1', secretKey: 's1', client: [endpoint: 'http://minio.com']]] + Global.config = CONFIG + and: + def fusion = new FusionScriptLauncher( + scheme: 's3', + buckets: ['foo'] as Set, + remoteWorkDir: S3PathFactory.parse('s3://foo/work')) + + expect: + fusion.fusionEnv() == [AWS_ACCESS_KEY_ID: 'k1', + AWS_SECRET_ACCESS_KEY: 's1', + AWS_S3_ENDPOINT: 'http://minio.com', + NXF_FUSION_BUCKETS: 's3://foo', + NXF_FUSION_WORK: '/fusion/s3/foo/work'] + + cleanup: + Global.config = null + SysEnv.pop() + } + }