Add azure storage endpoint suffix #26432

docs/plugins/repository-azure.asciidoc

@@ -44,15 +44,18 @@ The initial backoff period is defined by Azure SDK as `30s`. Which means `30s` o
 before retrying after a first timeout or failure. The maximum backoff period is defined by Azure SDK as
 `90s`.
 
+`endpoint_suffix` can be used to specify Azure endpoint suffix explicitly. Defaults to `core.windows.net`.
+
 [source,yaml]
 ----
 cloud.azure.storage.timeout: 10s
 azure.client.default.max_retries: 7
+azure.client.default.endpoint_suffix: core.chinacloudapi.cn
 azure.client.secondary.timeout: 30s
 ----
 
 In this example, timeout will be `10s` per try for `default` with `7` retries before failing
-and `30s` per try for `secondary` with `3` retries.
+and endpoint suffix will be `core.chinacloudapi.cn` and `30s` per try for `secondary` with `3` retries.
 
 [IMPORTANT]
 .Supported Azure Storage Account types

plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java

@@ -102,14 +102,18 @@ public AzureStorageServiceImpl(Settings settings, Map<String, AzureStorageSettin
 
     void createClient(AzureStorageSettings azureStorageSettings) {
         try {
-            logger.trace("creating new Azure storage client using account [{}], key [{}]",
-                azureStorageSettings.getAccount(), azureStorageSettings.getKey());
+            logger.trace("creating new Azure storage client using account [{}], key [{}], endpoint suffix [{}]",
+                azureStorageSettings.getAccount(), azureStorageSettings.getKey(), azureStorageSettings.getEndpointSuffix());
 
             String storageConnectionString =
                 "DefaultEndpointsProtocol=https;"
                     + "AccountName=" + azureStorageSettings.getAccount() + ";"
                     + "AccountKey=" + azureStorageSettings.getKey();
 
+            String endpointSuffix = azureStorageSettings.getEndpointSuffix();
+            if (endpointSuffix != null && !endpointSuffix.isEmpty()) {
+                storageConnectionString += ";EndpointSuffix=" + endpointSuffix;
+            }
             // Retrieve storage account from connection-string.
             CloudStorageAccount storageAccount = CloudStorageAccount.parse(storageConnectionString);
 

plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettings.java

@@ -50,6 +50,12 @@ public final class AzureStorageSettings {
     public static final AffixSetting<SecureString> ACCOUNT_SETTING = Setting.affixKeySetting(PREFIX, "account",
         key -> SecureSetting.secureString(key, null));
 
+    /**
+     * Azure endpoint suffix. Default to core.windows.net (CloudStorageAccount.DEFAULT_DNS).
+     */
+    public static final Setting<String> ENDPOINT_SUFFIX_SETTING = Setting.affixKeySetting(PREFIX, "endpoint_suffix",
+        key -> SecureSetting.simpleString(key));
+
     /**
      * max_retries: Number of retries in case of Azure errors. Defaults to 3 (RetryPolicy.DEFAULT_CLIENT_RETRY_COUNT).
      */
@@ -85,15 +91,17 @@ public final class AzureStorageSettings {
     private final String name;
     private final String account;
     private final String key;
+    private final String endpointSuffix;
     private final TimeValue timeout;
     @Deprecated
     private final boolean activeByDefault;
     private final int maxRetries;
 
-    public AzureStorageSettings(String account, String key, TimeValue timeout, int maxRetries) {
+    public AzureStorageSettings(String account, String key, String endpointSuffix, TimeValue timeout, int maxRetries) {
         this.name = null;
         this.account = account;
         this.key = key;
+        this.endpointSuffix = endpointSuffix;
         this.timeout = timeout;
         this.activeByDefault = false;
         this.maxRetries = maxRetries;
@@ -104,6 +112,7 @@ public AzureStorageSettings(String name, String account, String key, TimeValue t
         this.name = name;
         this.account = account;
         this.key = key;
+        this.endpointSuffix = null;
         this.timeout = timeout;
         this.activeByDefault = activeByDefault;
         this.maxRetries = maxRetries;
@@ -118,6 +127,10 @@ public String getKey() {
         return key;
     }
 
+    public String getEndpointSuffix() {
+        return endpointSuffix;
+    }
+
     public String getAccount() {
         return account;
     }
@@ -141,6 +154,7 @@ public String toString() {
         sb.append("name='").append(name).append('\'');
         sb.append(", account='").append(account).append('\'');
         sb.append(", key='").append(key).append('\'');
+        sb.append(", endpointSuffix='").append(endpointSuffix).append('\'');
         sb.append(", activeByDefault='").append(activeByDefault).append('\'');
         sb.append(", timeout=").append(timeout);
         sb.append(", maxRetries=").append(maxRetries);
@@ -187,6 +201,7 @@ static AzureStorageSettings getClientSettings(Settings settings, String clientNa
         try (SecureString account = getConfigValue(settings, clientName, ACCOUNT_SETTING);
              SecureString key = getConfigValue(settings, clientName, KEY_SETTING)) {
             return new AzureStorageSettings(account.toString(), key.toString(),
+                getValue(settings, clientName, ENDPOINT_SUFFIX_SETTING),
                 getValue(settings, clientName, TIMEOUT_SETTING),
                 getValue(settings, clientName, MAX_RETRIES_SETTING));
         }

plugins/repository-azure/src/main/java/org/elasticsearch/plugin/repository/azure/AzureRepositoryPlugin.java

@@ -65,6 +65,7 @@ public List<Setting<?>> getSettings() {
             AzureStorageService.Storage.STORAGE_ACCOUNTS,
             AzureStorageSettings.ACCOUNT_SETTING,
             AzureStorageSettings.KEY_SETTING,
+            AzureStorageSettings.ENDPOINT_SUFFIX_SETTING,
             AzureStorageSettings.TIMEOUT_SETTING
         );
     }

plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceTests.java

@@ -22,6 +22,8 @@
 import com.microsoft.azure.storage.LocationMode;
 import com.microsoft.azure.storage.RetryExponentialRetry;
 import com.microsoft.azure.storage.blob.CloudBlobClient;
+import com.microsoft.azure.storage.core.Base64;
+
 import org.elasticsearch.common.settings.MockSecureSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
@@ -38,8 +40,10 @@
 import static org.elasticsearch.cloud.azure.storage.AzureStorageSettings.DEPRECATED_TIMEOUT_SETTING;
 import static org.elasticsearch.repositories.azure.AzureSettingsParserTests.getConcreteSetting;
 import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.isEmptyString;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
@@ -82,10 +86,31 @@ public void testReadSecuredSettings() {
         secureSettings.setString("azure.client.azure2.key", "mykey2");
         secureSettings.setString("azure.client.azure3.account", "myaccount3");
         secureSettings.setString("azure.client.azure3.key", "mykey3");
-        Settings settings = Settings.builder().setSecureSettings(secureSettings).build();
+        Settings settings = Settings.builder().setSecureSettings(secureSettings)
+            .put("azure.client.azure3.endpoint_suffix", "my_endpoint_suffix").build();
 
         Map<String, AzureStorageSettings> loadedSettings = AzureStorageSettings.load(settings);
         assertThat(loadedSettings.keySet(), containsInAnyOrder("azure1","azure2","azure3","default"));
+
+        assertThat(loadedSettings.get("azure1").getEndpointSuffix(), isEmptyString());
+        assertThat(loadedSettings.get("azure2").getEndpointSuffix(), isEmptyString());
+        assertThat(loadedSettings.get("azure3").getEndpointSuffix(), equalTo("my_endpoint_suffix"));
+    }
+
+    public void testCreateClientWithEndpointSuffix() {
+        MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("azure.client.azure1.account", "myaccount1");
+        secureSettings.setString("azure.client.azure1.key", Base64.encode("mykey1".getBytes()));
+        secureSettings.setString("azure.client.azure2.account", "myaccount2");
+        secureSettings.setString("azure.client.azure2.key", Base64.encode("mykey2".getBytes()));
+        Settings settings = Settings.builder().setSecureSettings(secureSettings)
+            .put("azure.client.azure1.endpoint_suffix", "my_endpoint_suffix").build();
+        AzureStorageServiceImpl azureStorageService = new AzureStorageServiceImpl(settings, AzureStorageSettings.load(settings));
+        CloudBlobClient client1 = azureStorageService.getSelectedClient("azure1", LocationMode.PRIMARY_ONLY);
+        assertThat(client1.getEndpoint().toString(), equalTo("https://myaccount1.blob.my_endpoint_suffix"));
+
+        CloudBlobClient client2 = azureStorageService.getSelectedClient("azure2", LocationMode.PRIMARY_ONLY);
+        assertThat(client2.getEndpoint().toString(), equalTo("https://myaccount2.blob.core.windows.net"));
     }
 
     public void testGetSelectedClientWithNoPrimaryAndSecondary() {