From e9168a5e3764821b63ce3611a318e914508f2222 Mon Sep 17 00:00:00 2001 From: James Carnegie Date: Fri, 7 Oct 2016 12:08:55 +0100 Subject: [PATCH] Add support for pluggable S3CredentialProviders --- .../PropertyBasedS3CredentialsProvider.java | 47 +++++++++++++++ .../exhibitor/standalone/ExhibitorCLI.java | 2 + .../standalone/ExhibitorCreator.java | 48 ++++++++++++--- .../standalone/TestExhibitorCreator.java | 59 +++++++++++++++++++ 4 files changed, 148 insertions(+), 8 deletions(-) create mode 100644 exhibitor-core/src/main/java/com/netflix/exhibitor/core/s3/PropertyBasedS3CredentialsProvider.java create mode 100644 exhibitor-standalone/src/test/java/com/netflix/exhibitor/standalone/TestExhibitorCreator.java diff --git a/exhibitor-core/src/main/java/com/netflix/exhibitor/core/s3/PropertyBasedS3CredentialsProvider.java b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/s3/PropertyBasedS3CredentialsProvider.java new file mode 100644 index 00000000..4f65d6d0 --- /dev/null +++ b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/s3/PropertyBasedS3CredentialsProvider.java @@ -0,0 +1,47 @@ +/* + * + * Copyright 2016 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.exhibitor.core.s3; + +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; + +public class PropertyBasedS3CredentialsProvider implements S3CredentialsProvider { + + private final AWSCredentialsProvider credentialsProvider; + + public PropertyBasedS3CredentialsProvider(final PropertyBasedS3Credential creds){ + + credentialsProvider = new AWSCredentialsProvider() { + @Override + public AWSCredentials getCredentials() { + return new BasicAWSCredentials(creds.getAccessKeyId(), creds.getSecretAccessKey()); + } + + @Override + public void refresh() { + //do nothing + } + }; + } + @Override + public AWSCredentialsProvider getAWSCredentialProvider() { + return credentialsProvider; + } +} diff --git a/exhibitor-standalone/src/main/java/com/netflix/exhibitor/standalone/ExhibitorCLI.java b/exhibitor-standalone/src/main/java/com/netflix/exhibitor/standalone/ExhibitorCLI.java index 3ee5febb..21cc263a 100644 --- a/exhibitor-standalone/src/main/java/com/netflix/exhibitor/standalone/ExhibitorCLI.java +++ b/exhibitor-standalone/src/main/java/com/netflix/exhibitor/standalone/ExhibitorCLI.java @@ -70,6 +70,7 @@ private OptionSection(String sectionName, Options options) public static final String S3_BACKUP = "s3backup"; public static final String S3_CONFIG = "s3config"; public static final String S3_CONFIG_PREFIX = "s3configprefix"; + public static final String S3_CREDENTIAL_PROVIDER_CLASS = "s3credsproviderclass"; public static final String S3_REGION = "s3region"; public static final String ZOOKEEPER_CONFIG_INITIAL_CONNECT_STRING = "zkconfigconnect"; public static final String ZOOKEEPER_CONFIG_EXHIBITOR_PORT = "zkconfigexhibitorport"; @@ -155,6 +156,7 @@ public ExhibitorCLI() Options s3Options = new Options(); s3Options.addOption(null, S3_CREDENTIALS, true, "Optional credentials to use for s3backup or s3config. Argument is the path to an AWS credential properties file with two properties: " + PropertyBasedS3Credential.PROPERTY_S3_KEY_ID + " and " + PropertyBasedS3Credential.PROPERTY_S3_SECRET_KEY); + s3Options.addOption(null,S3_CREDENTIAL_PROVIDER_CLASS, true, "Optional S3 Credentials Provider class name, which should implement com.netflix.exhibitor.core.s3.S3CredentialsProvider and should have a null constructor"); s3Options.addOption(null, S3_REGION, true, "Optional region for S3 calls (e.g. \"eu-west-1\"). Will be used to set the S3 client's endpoint."); s3Options.addOption(null, S3_PROXY, true, "Optional configuration used when when connecting to S3 via a proxy. Argument is the path to an AWS credential properties file with four properties (only host, port and protocol are required if using a proxy): " + PropertyBasedS3ClientConfig.PROPERTY_S3_PROXY_HOST + ", " + PropertyBasedS3ClientConfig.PROPERTY_S3_PROXY_PORT + ", " + PropertyBasedS3ClientConfig.PROPERTY_S3_PROXY_USERNAME + ", " + PropertyBasedS3ClientConfig.PROPERTY_S3_PROXY_PASSWORD); diff --git a/exhibitor-standalone/src/main/java/com/netflix/exhibitor/standalone/ExhibitorCreator.java b/exhibitor-standalone/src/main/java/com/netflix/exhibitor/standalone/ExhibitorCreator.java index da90e1da..7920482a 100644 --- a/exhibitor-standalone/src/main/java/com/netflix/exhibitor/standalone/ExhibitorCreator.java +++ b/exhibitor-standalone/src/main/java/com/netflix/exhibitor/standalone/ExhibitorCreator.java @@ -38,7 +38,9 @@ import com.netflix.exhibitor.core.config.zookeeper.ZookeeperConfigProvider; import com.netflix.exhibitor.core.s3.PropertyBasedS3ClientConfig; import com.netflix.exhibitor.core.s3.PropertyBasedS3Credential; +import com.netflix.exhibitor.core.s3.PropertyBasedS3CredentialsProvider; import com.netflix.exhibitor.core.s3.S3ClientFactoryImpl; +import com.netflix.exhibitor.core.s3.S3CredentialsProvider; import com.netflix.exhibitor.core.servo.ServoRegistration; import com.netflix.servo.jmx.JmxMonitorRegistry; import org.apache.commons.cli.CommandLine; @@ -117,13 +119,27 @@ public ExhibitorCreator(String[] args) throws Exception } checkMutuallyExclusive(cli, commandLine, S3_BACKUP, FILESYSTEMBACKUP); + checkMutuallyExclusive(cli, commandLine, S3_CREDENTIAL_PROVIDER_CLASS, S3_CREDENTIALS); + + if (!commandLine.hasOption( S3_CREDENTIAL_PROVIDER_CLASS) && !commandLine.hasOption(S3_CREDENTIALS)) + { + throw new MissingConfigurationTypeException("S3 credentials via (--" + S3_CREDENTIAL_PROVIDER_CLASS + " or --" + S3_CREDENTIALS + ") must be specified", cli); + } + String s3Region = commandLine.getOptionValue(S3_REGION, null); - PropertyBasedS3Credential awsCredentials = null; + S3CredentialsProvider s3CredentialsProvider = null; PropertyBasedS3ClientConfig awsClientConfig = null; + if ( commandLine.hasOption(S3_CREDENTIALS) ) { - awsCredentials = new PropertyBasedS3Credential(new File(commandLine.getOptionValue(S3_CREDENTIALS))); + PropertyBasedS3Credential awsCredentials = new PropertyBasedS3Credential(new File(commandLine.getOptionValue(S3_CREDENTIALS))); + s3CredentialsProvider = new PropertyBasedS3CredentialsProvider(awsCredentials); + } + + if (commandLine.hasOption(S3_CREDENTIAL_PROVIDER_CLASS)) + { + s3CredentialsProvider = makeCredentialsProvider(commandLine.getOptionValue(S3_CREDENTIAL_PROVIDER_CLASS)); } if ( commandLine.hasOption(S3_PROXY) ) @@ -134,7 +150,7 @@ public ExhibitorCreator(String[] args) throws Exception BackupProvider backupProvider = null; if ( "true".equalsIgnoreCase(commandLine.getOptionValue(S3_BACKUP)) ) { - backupProvider = new S3BackupProvider(new S3ClientFactoryImpl(), awsCredentials, awsClientConfig, s3Region); + backupProvider = new S3BackupProvider(new S3ClientFactoryImpl(), s3CredentialsProvider, awsClientConfig, s3Region); } else if ( "true".equalsIgnoreCase(commandLine.getOptionValue(FILESYSTEMBACKUP)) ) { @@ -155,7 +171,7 @@ else if ( "true".equalsIgnoreCase(commandLine.getOptionValue(FILESYSTEMBACKUP)) throw new MissingConfigurationTypeException("Configuration type (-" + SHORT_CONFIG_TYPE + " or --" + CONFIG_TYPE + ") must be specified", cli); } - ConfigProvider configProvider = makeConfigProvider(configType, cli, commandLine, awsCredentials, awsClientConfig, backupProvider, useHostname, s3Region); + ConfigProvider configProvider = makeConfigProvider(configType, cli, commandLine, s3CredentialsProvider, awsClientConfig, backupProvider, useHostname, s3Region); if ( configProvider == null ) { throw new ExhibitorCreatorExit(cli); @@ -233,6 +249,21 @@ else if ( "true".equalsIgnoreCase(commandLine.getOptionValue(FILESYSTEMBACKUP)) this.httpPort = httpPort; } + /** + * Use thread classloader to create an instance of a custom S3CredentialsProvider + * @param classname name of class implementing S3CredentialsProvider + * @param commandLine all cli options passed in + * @return + */ + protected static S3CredentialsProvider makeCredentialsProvider(String classname) { + try{ + return (S3CredentialsProvider)Thread.currentThread().getContextClassLoader().loadClass(classname).newInstance(); + } catch (Throwable t){ + throw new RuntimeException("Unable to create credentials provider: " + classname, t ); + } + + } + public ExhibitorArguments.Builder getBuilder() { return builder; @@ -278,14 +309,14 @@ public String getRemoteAuthSpec() return remoteAuthSpec; } - private ConfigProvider makeConfigProvider(String configType, ExhibitorCLI cli, CommandLine commandLine, PropertyBasedS3Credential awsCredentials, PropertyBasedS3ClientConfig awsClientConfig, BackupProvider backupProvider, String useHostname, String s3Region) throws Exception + private ConfigProvider makeConfigProvider(String configType, ExhibitorCLI cli, CommandLine commandLine, S3CredentialsProvider awsCredentialProvider, PropertyBasedS3ClientConfig awsClientConfig, BackupProvider backupProvider, String useHostname, String s3Region) throws Exception { Properties defaultProperties = makeDefaultProperties(commandLine, backupProvider); ConfigProvider configProvider; if ( configType.equals("s3") ) { - configProvider = getS3Provider(cli, commandLine, awsCredentials, awsClientConfig, useHostname, defaultProperties, s3Region); + configProvider = getS3Provider(cli, commandLine, awsCredentialProvider, awsClientConfig, useHostname, defaultProperties, s3Region); } else if ( configType.equals("file") ) { @@ -522,12 +553,13 @@ private ConfigProvider getFileSystemProvider(CommandLine commandLine, Properties return new FileSystemConfigProvider(directory, name, defaultProperties, new AutoManageLockArguments(lockPrefix)); } - private ConfigProvider getS3Provider(ExhibitorCLI cli, CommandLine commandLine, PropertyBasedS3Credential awsCredentials, PropertyBasedS3ClientConfig awsClientConfig, String hostname, Properties defaultProperties, String s3Region) throws Exception + private ConfigProvider getS3Provider(ExhibitorCLI cli, CommandLine commandLine, S3CredentialsProvider awsCredentialsProvider, PropertyBasedS3ClientConfig awsClientConfig, String hostname, Properties defaultProperties, String s3Region) throws Exception { String prefix = cli.getOptions().hasOption(S3_CONFIG_PREFIX) ? commandLine.getOptionValue(S3_CONFIG_PREFIX) : DEFAULT_PREFIX; - return new S3ConfigProvider(new S3ClientFactoryImpl(), awsCredentials, awsClientConfig, getS3Arguments(cli, commandLine.getOptionValue(S3_CONFIG), prefix), hostname, defaultProperties, s3Region); + return new S3ConfigProvider(new S3ClientFactoryImpl(), awsCredentialsProvider, awsClientConfig, getS3Arguments(cli, commandLine.getOptionValue(S3_CONFIG), prefix), hostname, defaultProperties, s3Region); } + private void checkMutuallyExclusive(ExhibitorCLI cli, CommandLine commandLine, String option1, String option2) throws ExhibitorCreatorExit { if ( commandLine.hasOption(option1) && commandLine.hasOption(option2) ) diff --git a/exhibitor-standalone/src/test/java/com/netflix/exhibitor/standalone/TestExhibitorCreator.java b/exhibitor-standalone/src/test/java/com/netflix/exhibitor/standalone/TestExhibitorCreator.java new file mode 100644 index 00000000..604f1918 --- /dev/null +++ b/exhibitor-standalone/src/test/java/com/netflix/exhibitor/standalone/TestExhibitorCreator.java @@ -0,0 +1,59 @@ +/* + * + * Copyright 2016 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.exhibitor.standalone; + +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.netflix.exhibitor.core.s3.S3CredentialsProvider; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.PosixParser; +import org.testng.annotations.Test; + +import static org.testng.AssertJUnit.assertNotNull; + +public class TestExhibitorCreator { + + @Test + public void makeCredentialsProviderTest() throws Exception{ + + S3CredentialsProvider provider = ExhibitorCreator.makeCredentialsProvider("com.netflix.exhibitor.standalone.TestS3CredsProvider"); + assertNotNull(provider); + } +} + +class TestS3CredsProvider implements S3CredentialsProvider { + + @Override + public AWSCredentialsProvider getAWSCredentialProvider() { + return new AWSCredentialsProvider() { + @Override + public AWSCredentials getCredentials() { + return new BasicAWSCredentials("access","secret"); + } + + @Override + public void refresh() { + + } + }; + } +} \ No newline at end of file