From 7269c84a94b5bf5b74104303eac1f586f768fb4f Mon Sep 17 00:00:00 2001 From: liketic Date: Sun, 10 Sep 2017 17:47:24 +0800 Subject: [PATCH 1/5] Allow specify azure storage endpoint suffix explicitly --- .../storage/AzureStorageServiceImpl.java | 8 ++++-- .../azure/storage/AzureStorageSettings.java | 20 +++++++++++++-- .../azure/AzureRepositoryPlugin.java | 1 + .../storage/AzureStorageServiceTests.java | 25 +++++++++++++++++++ 4 files changed, 50 insertions(+), 4 deletions(-) diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java index c928d79c0c242..2b1bc0898a772 100644 --- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java @@ -102,14 +102,18 @@ public AzureStorageServiceImpl(Settings settings, Map SecureSetting.secureString(key, null)); + /** + * Azure endpoint suffix. Default to core.windows.net (CloudStorageAccount.DEFAULT_DNS). + */ + public static final AffixSetting ENDPOINT_SUFFIX_SETTING = Setting.affixKeySetting(PREFIX, "endpoint_suffix", + key -> SecureSetting.secureString(key, null)); + /** * max_retries: Number of retries in case of Azure errors. Defaults to 3 (RetryPolicy.DEFAULT_CLIENT_RETRY_COUNT). */ @@ -85,15 +91,17 @@ public final class AzureStorageSettings { private final String name; private final String account; private final String key; + private final String endpointSuffix; private final TimeValue timeout; @Deprecated private final boolean activeByDefault; private final int maxRetries; - public AzureStorageSettings(String account, String key, TimeValue timeout, int maxRetries) { + public AzureStorageSettings(String account, String key, String endpointSuffix, TimeValue timeout, int maxRetries) { this.name = null; this.account = account; this.key = key; + this.endpointSuffix = endpointSuffix; this.timeout = timeout; this.activeByDefault = false; this.maxRetries = maxRetries; @@ -104,6 +112,7 @@ public AzureStorageSettings(String name, String account, String key, TimeValue t this.name = name; this.account = account; this.key = key; + this.endpointSuffix = null; this.timeout = timeout; this.activeByDefault = activeByDefault; this.maxRetries = maxRetries; @@ -118,6 +127,10 @@ public String getKey() { return key; } + public String getEndpointSuffix() { + return endpointSuffix; + } + public String getAccount() { return account; } @@ -141,6 +154,7 @@ public String toString() { sb.append("name='").append(name).append('\''); sb.append(", account='").append(account).append('\''); sb.append(", key='").append(key).append('\''); + sb.append(", endpointSuffix='").append(endpointSuffix).append('\''); sb.append(", activeByDefault='").append(activeByDefault).append('\''); sb.append(", timeout=").append(timeout); sb.append(", maxRetries=").append(maxRetries); @@ -185,8 +199,10 @@ public static Map load(Settings settings) { /** Parse settings for a single client. */ static AzureStorageSettings getClientSettings(Settings settings, String clientName) { try (SecureString account = getConfigValue(settings, clientName, ACCOUNT_SETTING); - SecureString key = getConfigValue(settings, clientName, KEY_SETTING)) { + SecureString key = getConfigValue(settings, clientName, KEY_SETTING); + SecureString endpointSuffix = getConfigValue(settings, clientName, ENDPOINT_SUFFIX_SETTING)) { return new AzureStorageSettings(account.toString(), key.toString(), + endpointSuffix.toString(), getValue(settings, clientName, TIMEOUT_SETTING), getValue(settings, clientName, MAX_RETRIES_SETTING)); } diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/plugin/repository/azure/AzureRepositoryPlugin.java b/plugins/repository-azure/src/main/java/org/elasticsearch/plugin/repository/azure/AzureRepositoryPlugin.java index b90d44264ecf7..61f5b06161b04 100644 --- a/plugins/repository-azure/src/main/java/org/elasticsearch/plugin/repository/azure/AzureRepositoryPlugin.java +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/plugin/repository/azure/AzureRepositoryPlugin.java @@ -65,6 +65,7 @@ public List> getSettings() { AzureStorageService.Storage.STORAGE_ACCOUNTS, AzureStorageSettings.ACCOUNT_SETTING, AzureStorageSettings.KEY_SETTING, + AzureStorageSettings.ENDPOINT_SUFFIX_SETTING, AzureStorageSettings.TIMEOUT_SETTING ); } diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceTests.java index b232ee12e05c4..1d3a7f836e49c 100644 --- a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceTests.java +++ b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceTests.java @@ -22,6 +22,8 @@ import com.microsoft.azure.storage.LocationMode; import com.microsoft.azure.storage.RetryExponentialRetry; import com.microsoft.azure.storage.blob.CloudBlobClient; +import com.microsoft.azure.storage.core.Base64; + import org.elasticsearch.common.settings.MockSecureSettings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; @@ -38,8 +40,10 @@ import static org.elasticsearch.cloud.azure.storage.AzureStorageSettings.DEPRECATED_TIMEOUT_SETTING; import static org.elasticsearch.repositories.azure.AzureSettingsParserTests.getConcreteSetting; import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.isEmptyString; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; @@ -82,10 +86,31 @@ public void testReadSecuredSettings() { secureSettings.setString("azure.client.azure2.key", "mykey2"); secureSettings.setString("azure.client.azure3.account", "myaccount3"); secureSettings.setString("azure.client.azure3.key", "mykey3"); + secureSettings.setString("azure.client.azure3.endpoint_suffix", "my_endpoint_suffix"); Settings settings = Settings.builder().setSecureSettings(secureSettings).build(); Map loadedSettings = AzureStorageSettings.load(settings); assertThat(loadedSettings.keySet(), containsInAnyOrder("azure1","azure2","azure3","default")); + + assertThat(loadedSettings.get("azure1").getEndpointSuffix(), isEmptyString()); + assertThat(loadedSettings.get("azure2").getEndpointSuffix(), isEmptyString()); + assertThat(loadedSettings.get("azure3").getEndpointSuffix(), equalTo("my_endpoint_suffix")); + } + + public void testCreateClientWithEndpointSuffix() { + MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("azure.client.azure1.account", "myaccount1"); + secureSettings.setString("azure.client.azure1.key", Base64.encode("mykey1".getBytes())); + secureSettings.setString("azure.client.azure1.endpoint_suffix", "my_endpoint_suffix"); + secureSettings.setString("azure.client.azure2.account", "myaccount2"); + secureSettings.setString("azure.client.azure2.key", Base64.encode("mykey2".getBytes())); + Settings settings = Settings.builder().setSecureSettings(secureSettings).build(); + AzureStorageServiceImpl azureStorageService = new AzureStorageServiceImpl(settings, AzureStorageSettings.load(settings)); + CloudBlobClient client1 = azureStorageService.getSelectedClient("azure1", LocationMode.PRIMARY_ONLY); + assertThat(client1.getEndpoint().toString(), equalTo("https://myaccount1.blob.my_endpoint_suffix")); + + CloudBlobClient client2 = azureStorageService.getSelectedClient("azure2", LocationMode.PRIMARY_ONLY); + assertThat(client2.getEndpoint().toString(), equalTo("https://myaccount2.blob.core.windows.net")); } public void testGetSelectedClientWithNoPrimaryAndSecondary() { From 46b5da070eed5c9a7b75e6ddd7606ee7c4595725 Mon Sep 17 00:00:00 2001 From: liketic Date: Sun, 10 Sep 2017 20:46:11 +0800 Subject: [PATCH 2/5] Update documentation for Azure storage endpoint suffix --- docs/plugins/repository-azure.asciidoc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/plugins/repository-azure.asciidoc b/docs/plugins/repository-azure.asciidoc index a7b4dbaa7fbdf..3801dabdc85e5 100644 --- a/docs/plugins/repository-azure.asciidoc +++ b/docs/plugins/repository-azure.asciidoc @@ -21,6 +21,15 @@ bin/elasticsearch-keystore add azure.client.default.key Where `account` is the azure account name and `key` the azure secret key. +You can specify an `endpoint_suffix` for storage account explicitly. Defaults to `core.windows.net`. + +[source,sh] +------------------------------------------------------------------- +bin/elasticsearch-keystore add azure.client.default.account +bin/elasticsearch-keystore add azure.client.default.key +bin/elasticsearch-keystore add azure.client.default.endpoint_suffix +------------------------------------------------------------------- + Note that you can also define more than one account: [source,sh] From 61fe37270ed69419dae28079d2ee30a284c2c5a9 Mon Sep 17 00:00:00 2001 From: liketic Date: Thu, 14 Sep 2017 10:24:23 +0800 Subject: [PATCH 3/5] Make endpoint_suffix to be a simple string setting for Azure plugin --- docs/plugins/repository-azure.asciidoc | 14 ++++---------- .../cloud/azure/storage/AzureStorageSettings.java | 9 ++++----- .../azure/storage/AzureStorageServiceTests.java | 8 ++++---- 3 files changed, 12 insertions(+), 19 deletions(-) diff --git a/docs/plugins/repository-azure.asciidoc b/docs/plugins/repository-azure.asciidoc index 3801dabdc85e5..e4411c2fd20f2 100644 --- a/docs/plugins/repository-azure.asciidoc +++ b/docs/plugins/repository-azure.asciidoc @@ -21,15 +21,6 @@ bin/elasticsearch-keystore add azure.client.default.key Where `account` is the azure account name and `key` the azure secret key. -You can specify an `endpoint_suffix` for storage account explicitly. Defaults to `core.windows.net`. - -[source,sh] -------------------------------------------------------------------- -bin/elasticsearch-keystore add azure.client.default.account -bin/elasticsearch-keystore add azure.client.default.key -bin/elasticsearch-keystore add azure.client.default.endpoint_suffix -------------------------------------------------------------------- - Note that you can also define more than one account: [source,sh] @@ -53,15 +44,18 @@ The initial backoff period is defined by Azure SDK as `30s`. Which means `30s` o before retrying after a first timeout or failure. The maximum backoff period is defined by Azure SDK as `90s`. +`endpoint_suffix` can be used to specify Azure endpoint suffix explicitly. Defaults to `core.windows.net`. + [source,yaml] ---- cloud.azure.storage.timeout: 10s azure.client.default.max_retries: 7 +azure.client.default.endpoint_suffix: core.chinacloudapi.cn azure.client.secondary.timeout: 30s ---- In this example, timeout will be `10s` per try for `default` with `7` retries before failing -and `30s` per try for `secondary` with `3` retries. +and endpoint suffix will be `core.chinacloudapi.cn` and `30s` per try for `secondary` with `3` retries. [IMPORTANT] .Supported Azure Storage Account types diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettings.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettings.java index 7197e066cf763..1b9f230b85a00 100644 --- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettings.java +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettings.java @@ -53,8 +53,8 @@ public final class AzureStorageSettings { /** * Azure endpoint suffix. Default to core.windows.net (CloudStorageAccount.DEFAULT_DNS). */ - public static final AffixSetting ENDPOINT_SUFFIX_SETTING = Setting.affixKeySetting(PREFIX, "endpoint_suffix", - key -> SecureSetting.secureString(key, null)); + public static final Setting ENDPOINT_SUFFIX_SETTING = Setting.affixKeySetting(PREFIX, "endpoint_suffix", + key -> SecureSetting.simpleString(key)); /** * max_retries: Number of retries in case of Azure errors. Defaults to 3 (RetryPolicy.DEFAULT_CLIENT_RETRY_COUNT). @@ -199,10 +199,9 @@ public static Map load(Settings settings) { /** Parse settings for a single client. */ static AzureStorageSettings getClientSettings(Settings settings, String clientName) { try (SecureString account = getConfigValue(settings, clientName, ACCOUNT_SETTING); - SecureString key = getConfigValue(settings, clientName, KEY_SETTING); - SecureString endpointSuffix = getConfigValue(settings, clientName, ENDPOINT_SUFFIX_SETTING)) { + SecureString key = getConfigValue(settings, clientName, KEY_SETTING)) { return new AzureStorageSettings(account.toString(), key.toString(), - endpointSuffix.toString(), + getValue(settings, clientName, ENDPOINT_SUFFIX_SETTING), getValue(settings, clientName, TIMEOUT_SETTING), getValue(settings, clientName, MAX_RETRIES_SETTING)); } diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceTests.java index 1d3a7f836e49c..2395de579f7dd 100644 --- a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceTests.java +++ b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceTests.java @@ -86,8 +86,8 @@ public void testReadSecuredSettings() { secureSettings.setString("azure.client.azure2.key", "mykey2"); secureSettings.setString("azure.client.azure3.account", "myaccount3"); secureSettings.setString("azure.client.azure3.key", "mykey3"); - secureSettings.setString("azure.client.azure3.endpoint_suffix", "my_endpoint_suffix"); - Settings settings = Settings.builder().setSecureSettings(secureSettings).build(); + Settings settings = Settings.builder().setSecureSettings(secureSettings) + .put("azure.client.azure3.endpoint_suffix", "my_endpoint_suffix").build(); Map loadedSettings = AzureStorageSettings.load(settings); assertThat(loadedSettings.keySet(), containsInAnyOrder("azure1","azure2","azure3","default")); @@ -101,10 +101,10 @@ public void testCreateClientWithEndpointSuffix() { MockSecureSettings secureSettings = new MockSecureSettings(); secureSettings.setString("azure.client.azure1.account", "myaccount1"); secureSettings.setString("azure.client.azure1.key", Base64.encode("mykey1".getBytes())); - secureSettings.setString("azure.client.azure1.endpoint_suffix", "my_endpoint_suffix"); secureSettings.setString("azure.client.azure2.account", "myaccount2"); secureSettings.setString("azure.client.azure2.key", Base64.encode("mykey2".getBytes())); - Settings settings = Settings.builder().setSecureSettings(secureSettings).build(); + Settings settings = Settings.builder().setSecureSettings(secureSettings) + .put("azure.client.azure1.endpoint_suffix", "my_endpoint_suffix").build(); AzureStorageServiceImpl azureStorageService = new AzureStorageServiceImpl(settings, AzureStorageSettings.load(settings)); CloudBlobClient client1 = azureStorageService.getSelectedClient("azure1", LocationMode.PRIMARY_ONLY); assertThat(client1.getEndpoint().toString(), equalTo("https://myaccount1.blob.my_endpoint_suffix")); From 78ec96b5b1d64340874623d56ae285a3f9b0b90a Mon Sep 17 00:00:00 2001 From: liketic Date: Thu, 14 Sep 2017 13:11:27 +0800 Subject: [PATCH 4/5] Do not use secure string for endpoint suffix --- .../elasticsearch/cloud/azure/storage/AzureStorageSettings.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettings.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettings.java index 1b9f230b85a00..82ca5b10e15e2 100644 --- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettings.java +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettings.java @@ -54,7 +54,7 @@ public final class AzureStorageSettings { * Azure endpoint suffix. Default to core.windows.net (CloudStorageAccount.DEFAULT_DNS). */ public static final Setting ENDPOINT_SUFFIX_SETTING = Setting.affixKeySetting(PREFIX, "endpoint_suffix", - key -> SecureSetting.simpleString(key)); + key -> Setting.simpleString(key, Property.NodeScope)); /** * max_retries: Number of retries in case of Azure errors. Defaults to 3 (RetryPolicy.DEFAULT_CLIENT_RETRY_COUNT). From c6ce182b0375aa32d493c186b704f8857568cf4a Mon Sep 17 00:00:00 2001 From: liketic Date: Sat, 16 Sep 2017 11:58:18 +0800 Subject: [PATCH 5/5] Convert string to bytes with encoding --- .../repositories/azure/AzureStorageServiceTests.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceTests.java index d628988dcc447..72cd015f14847 100644 --- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceTests.java +++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceTests.java @@ -35,6 +35,7 @@ import java.net.URI; import java.net.URISyntaxException; import java.net.UnknownHostException; +import java.nio.charset.StandardCharsets; import java.util.Map; import static org.elasticsearch.repositories.azure.AzureStorageServiceImpl.blobNameFromUri; @@ -87,9 +88,9 @@ public void testReadSecuredSettings() { public void testCreateClientWithEndpointSuffix() { MockSecureSettings secureSettings = new MockSecureSettings(); secureSettings.setString("azure.client.azure1.account", "myaccount1"); - secureSettings.setString("azure.client.azure1.key", Base64.encode("mykey1".getBytes())); + secureSettings.setString("azure.client.azure1.key", Base64.encode("mykey1".getBytes(StandardCharsets.UTF_8))); secureSettings.setString("azure.client.azure2.account", "myaccount2"); - secureSettings.setString("azure.client.azure2.key", Base64.encode("mykey2".getBytes())); + secureSettings.setString("azure.client.azure2.key", Base64.encode("mykey2".getBytes(StandardCharsets.UTF_8))); Settings settings = Settings.builder().setSecureSettings(secureSettings) .put("azure.client.azure1.endpoint_suffix", "my_endpoint_suffix").build(); AzureStorageServiceImpl azureStorageService = new AzureStorageServiceImpl(settings, AzureStorageSettings.load(settings));