[Backport 2.x] Allow test clusters to run with TLS #8900

CHANGELOG.md

@@ -18,6 +18,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Remote Store] Add Segment download stats to remotestore stats API ([#8718](https://github.com/opensearch-project/OpenSearch/pull/8718))
 - [Remote Store] Add remote segment transfer stats on NodesStats API ([#9168](https://github.com/opensearch-project/OpenSearch/pull/9168))
 - [Segment Replication] Support realtime reads for GET requests ([#9212](https://github.com/opensearch-project/OpenSearch/pull/9212))
+- Allow test clusters to run with TLS ([#8900](https://github.com/opensearch-project/OpenSearch/pull/8900))
 
 ### Dependencies
 - Bump `org.apache.logging.log4j:log4j-core` from 2.17.1 to 2.20.0 ([#8307](https://github.com/opensearch-project/OpenSearch/pull/8307))

buildSrc/src/main/java/org/opensearch/gradle/http/WaitForHttpResource.java

@@ -83,6 +83,24 @@ public WaitForHttpResource(String protocol, String host, int numberOfNodes) thro
         this(new URL(protocol + "://" + host + "/_cluster/health?wait_for_nodes=>=" + numberOfNodes + "&wait_for_status=yellow"));
     }
 
+    public WaitForHttpResource(String protocol, String host, String username, String password, int numberOfNodes)
+        throws MalformedURLException {
+        this(
+            new URL(
+                protocol
+                    + "://"
+                    + username
+                    + ":"
+                    + password
+                    + "@"
+                    + host
+                    + "/_cluster/health?wait_for_nodes=>="
+                    + numberOfNodes
+                    + "&wait_for_status=yellow"
+            )
+        );
+    }
+
     public WaitForHttpResource(URL url) {
         this.url = url;
     }

buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchCluster.java

@@ -240,6 +240,11 @@ public void keystorePassword(String password) {
         nodes.all(each -> each.keystorePassword(password));
     }
 
+    @Override
+    public void setSecure(boolean secure) {
+        nodes.all(each -> each.setSecure(secure));
+    }
+
     @Override
     public void cliSetup(String binTool, CharSequence... args) {
         nodes.all(each -> each.cliSetup(binTool, args));
@@ -529,12 +534,25 @@ public OpenSearchNode singleNode() {
     private void addWaitForClusterHealth() {
         waitConditions.put("cluster health yellow", (node) -> {
             try {
-                WaitForHttpResource wait = new WaitForHttpResource("http", getFirstNode().getHttpSocketURI(), nodes.size());
-
-                List<Map<String, String>> credentials = getFirstNode().getCredentials();
-                if (getFirstNode().getCredentials().isEmpty() == false) {
-                    wait.setUsername(credentials.get(0).get("useradd"));
-                    wait.setPassword(credentials.get(0).get("-p"));
+                WaitForHttpResource wait;
+                if (!getFirstNode().isSecure()) {
+                    wait = new WaitForHttpResource("http", getFirstNode().getHttpSocketURI(), nodes.size());
+                    List<Map<String, String>> credentials = getFirstNode().getCredentials();
+                    if (getFirstNode().getCredentials().isEmpty() == false) {
+                        wait.setUsername(credentials.get(0).get("useradd"));
+                        wait.setPassword(credentials.get(0).get("-p"));
+                    }
+                } else {
+                    wait = new WaitForHttpResource(
+                        "https",
+                        getFirstNode().getHttpSocketURI(),
+                        getFirstNode().getCredentials().get(0).get("username"),
+                        getFirstNode().getCredentials().get(0).get("password"),
+                        nodes.size()
+                    );
+                    wait.setUsername(getFirstNode().getCredentials().get(0).get("username"));
+                    wait.setPassword(getFirstNode().getCredentials().get(0).get("password"));
+                    wait.setCertificateAuthorities(getFirstNode().getExtraConfigFilesMap().get("root-ca.pem"));
                 }
                 return wait.wait(500);
             } catch (IOException e) {

buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchNode.java

@@ -161,6 +161,7 @@ public class OpenSearchNode implements TestClusterConfiguration {
     private final Path httpPortsFile;
     private final Path tmpDir;
 
+    private boolean secure = false;
     private int currentDistro = 0;
     private TestDistribution testDistribution;
     private List<OpenSearchDistribution> distributions = new ArrayList<>();
@@ -206,6 +207,7 @@ public class OpenSearchNode implements TestClusterConfiguration {
         opensearchConfig = Config.getOpenSearchConfig(workingDir);
         legacyESConfig = Config.getLegacyESConfig(workingDir);
         currentConfig = opensearchConfig;
+        this.credentials.add(new HashMap<>());
     }
 
     /*
@@ -306,6 +308,11 @@ public String getName() {
         return nameCustomization.apply(name);
     }
 
+    @Internal
+    public boolean isSecure() {
+        return secure;
+    }
+
     @Internal
     public Version getVersion() {
         return Version.fromString(distributions.get(currentDistro).getVersion());
@@ -543,6 +550,11 @@ public void setPreserveDataDir(boolean preserveDataDir) {
         this.preserveDataDir = preserveDataDir;
     }
 
+    @Override
+    public void setSecure(boolean secure) {
+        this.secure = secure;
+    }
+
     @Override
     public void freeze() {
         requireNonNull(testDistribution, "null testDistribution passed when configuring test cluster `" + this + "`");
@@ -562,6 +574,18 @@ public Stream<String> logLines() throws IOException {
     @Override
     public synchronized void start() {
         LOGGER.info("Starting `{}`", this);
+        if (System.getProperty("tests.opensearch.secure") != null
+            && System.getProperty("tests.opensearch.secure").equalsIgnoreCase("true")) {
+            secure = true;
+        }
+        if (System.getProperty("tests.opensearch.username") != null) {
+            this.credentials.get(0).put("username", System.getProperty("tests.opensearch.username"));
+            LOGGER.info("Overwriting username to: " + this.getCredentials().get(0).get("username"));
+        }
+        if (System.getProperty("tests.opensearch.password") != null) {
+            this.credentials.get(0).put("password", System.getProperty("tests.opensearch.password"));
+            LOGGER.info("Overwriting password to: " + this.getCredentials().get(0).get("password"));
+        }
         if (Files.exists(getExtractedDistributionDir()) == false) {
             throw new TestClustersException("Can not start " + this + ", missing: " + getExtractedDistributionDir());
         }
@@ -1478,6 +1502,11 @@ public List<?> getExtraConfigFiles() {
         return extraConfigFiles.getNormalizedCollection();
     }
 
+    @Internal
+    public Map<String, File> getExtraConfigFilesMap() {
+        return extraConfigFiles;
+    }
+
     @Override
     @Internal
     public boolean isProcessAlive() {

buildSrc/src/main/java/org/opensearch/gradle/testclusters/TestClusterConfiguration.java

@@ -108,6 +108,8 @@ public interface TestClusterConfiguration {
 
     void setPreserveDataDir(boolean preserveDataDir);
 
+    void setSecure(boolean secure);
+
     void freeze();
 
     void start();