diff --git a/x-pack/plugin/ccr/qa/chain/build.gradle b/x-pack/plugin/ccr/qa/chain/build.gradle deleted file mode 100644 index b9bf933d10fe1..0000000000000 --- a/x-pack/plugin/ccr/qa/chain/build.gradle +++ /dev/null @@ -1,63 +0,0 @@ -import org.elasticsearch.gradle.test.RestIntegTestTask - -apply plugin: 'elasticsearch.standalone-test' - -dependencies { - testCompile project(path: xpackModule('core'), configuration: 'testArtifacts') - testCompile project(path: xpackModule('ccr'), configuration: 'runtime') - testCompile project(':x-pack:plugin:ccr:qa') -} - -task leaderClusterTest(type: RestIntegTestTask) { - mustRunAfter(precommit) -} - -leaderClusterTestCluster { - numNodes = 1 - clusterName = 'leader-cluster' - setting 'xpack.license.self_generated.type', 'trial' - setting 'node.name', 'leader' -} - -leaderClusterTestRunner { - systemProperty 'tests.target_cluster', 'leader' -} - -task middleClusterTest(type: RestIntegTestTask) {} - -middleClusterTestCluster { - dependsOn leaderClusterTestRunner - numNodes = 1 - clusterName = 'middle-cluster' - setting 'xpack.license.self_generated.type', 'trial' - setting 'cluster.remote.leader_cluster.seeds', "\"${-> leaderClusterTest.nodes.get(0).transportUri()}\"" - setting 'node.name', 'middle' -} - -middleClusterTestRunner { - systemProperty 'tests.target_cluster', 'middle' - systemProperty 'tests.leader_host', "${-> leaderClusterTest.nodes.get(0).httpUri()}" -} - -task followClusterTest(type: RestIntegTestTask) {} - -followClusterTestCluster { - dependsOn middleClusterTestRunner - numNodes = 1 - clusterName = 'follow-cluster' - setting 'xpack.license.self_generated.type', 'trial' - setting 'cluster.remote.leader_cluster.seeds', "\"${-> leaderClusterTest.nodes.get(0).transportUri()}\"" - setting 'cluster.remote.middle_cluster.seeds', "\"${-> middleClusterTest.nodes.get(0).transportUri()}\"" - setting 'node.name', 'follow' -} - -followClusterTestRunner { - systemProperty 'tests.target_cluster', 'follow' - systemProperty 'tests.leader_host', "${-> leaderClusterTest.nodes.get(0).httpUri()}" - systemProperty 'tests.middle_host', "${-> middleClusterTest.nodes.get(0).httpUri()}" - finalizedBy 'leaderClusterTestCluster#stop' - finalizedBy 'middleClusterTestCluster#stop' -} - -check.dependsOn followClusterTest -test.enabled = false diff --git a/x-pack/plugin/ccr/qa/multi-cluster-downgraded-to-basic-license/build.gradle b/x-pack/plugin/ccr/qa/downgraded-to-basic-license/build.gradle similarity index 100% rename from x-pack/plugin/ccr/qa/multi-cluster-downgraded-to-basic-license/build.gradle rename to x-pack/plugin/ccr/qa/downgraded-to-basic-license/build.gradle diff --git a/x-pack/plugin/ccr/qa/multi-cluster-downgraded-to-basic-license/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java b/x-pack/plugin/ccr/qa/downgraded-to-basic-license/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java similarity index 100% rename from x-pack/plugin/ccr/qa/multi-cluster-downgraded-to-basic-license/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java rename to x-pack/plugin/ccr/qa/downgraded-to-basic-license/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java diff --git a/x-pack/plugin/ccr/qa/multi-cluster/build.gradle b/x-pack/plugin/ccr/qa/multi-cluster/build.gradle index 3e3661aae1a23..f1ea63269b058 100644 --- a/x-pack/plugin/ccr/qa/multi-cluster/build.gradle +++ b/x-pack/plugin/ccr/qa/multi-cluster/build.gradle @@ -16,27 +16,48 @@ leaderClusterTestCluster { numNodes = 1 clusterName = 'leader-cluster' setting 'xpack.license.self_generated.type', 'trial' + setting 'node.name', 'leader' } leaderClusterTestRunner { systemProperty 'tests.target_cluster', 'leader' } +task middleClusterTest(type: RestIntegTestTask) {} + +middleClusterTestCluster { + dependsOn leaderClusterTestRunner + numNodes = 1 + clusterName = 'middle-cluster' + setting 'xpack.license.self_generated.type', 'trial' + setting 'cluster.remote.leader_cluster.seeds', "\"${-> leaderClusterTest.nodes.get(0).transportUri()}\"" + setting 'node.name', 'middle' +} + +middleClusterTestRunner { + systemProperty 'tests.target_cluster', 'middle' + systemProperty 'tests.leader_host', "${-> leaderClusterTest.nodes.get(0).httpUri()}" +} + task followClusterTest(type: RestIntegTestTask) {} followClusterTestCluster { - dependsOn leaderClusterTestRunner + dependsOn middleClusterTestRunner numNodes = 1 clusterName = 'follow-cluster' setting 'xpack.monitoring.collection.enabled', 'true' setting 'xpack.license.self_generated.type', 'trial' setting 'cluster.remote.leader_cluster.seeds', "\"${-> leaderClusterTest.nodes.get(0).transportUri()}\"" + setting 'cluster.remote.middle_cluster.seeds', "\"${-> middleClusterTest.nodes.get(0).transportUri()}\"" + setting 'node.name', 'follow' } followClusterTestRunner { systemProperty 'tests.target_cluster', 'follow' systemProperty 'tests.leader_host', "${-> leaderClusterTest.nodes.get(0).httpUri()}" + systemProperty 'tests.middle_host', "${-> middleClusterTest.nodes.get(0).httpUri()}" finalizedBy 'leaderClusterTestCluster#stop' + finalizedBy 'middleClusterTestCluster#stop' } check.dependsOn followClusterTest diff --git a/x-pack/plugin/ccr/qa/chain/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java b/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java similarity index 54% rename from x-pack/plugin/ccr/qa/chain/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java rename to x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java index 4096202c92acc..d1f28fa0da918 100644 --- a/x-pack/plugin/ccr/qa/chain/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java +++ b/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java @@ -11,16 +11,21 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; +import java.io.IOException; import java.util.Map; +import java.util.concurrent.TimeUnit; import static org.hamcrest.Matchers.equalTo; public class AutoFollowIT extends ESCCRRestTestCase { - public void testAutoFollowPatterns() throws Exception { + public void testMultipleAutoFollowPatternsDifferentClusters() throws Exception { if ("follow".equals(targetCluster) == false) { + logger.info("skipping test, waiting for target cluster [follow]" ); return; } + + int initialNumberOfSuccessfulFollowedIndices = getNumberOfSuccessfulFollowedIndices(); Request putPatternRequest = new Request("PUT", "/_ccr/auto_follow/leader_cluster_pattern"); putPatternRequest.setJsonEntity("{\"leader_index_patterns\": [\"logs-*\"], \"remote_cluster\": \"leader_cluster\"}"); assertOK(client().performRequest(putPatternRequest)); @@ -54,10 +59,7 @@ public void testAutoFollowPatterns() throws Exception { } } assertBusy(() -> { - Request statsRequest = new Request("GET", "/_ccr/stats"); - Map response = toMap(client().performRequest(statsRequest)); - Map autoFollowStats = (Map) response.get("auto_follow_stats"); - assertThat(autoFollowStats.get("number_of_successful_follow_indices"), equalTo(2)); + assertThat(getNumberOfSuccessfulFollowedIndices(), equalTo(initialNumberOfSuccessfulFollowedIndices + 2)); ensureYellow("logs-20190101"); ensureYellow("logs-20200101"); @@ -66,4 +68,49 @@ public void testAutoFollowPatterns() throws Exception { }); } + public void testAutoFollowPatterns() throws Exception { + if ("follow".equals(targetCluster) == false) { + logger.info("skipping test, waiting for target cluster [follow]" ); + return; + } + + int initialNumberOfSuccessfulFollowedIndices = getNumberOfSuccessfulFollowedIndices(); + Request request = new Request("PUT", "/_ccr/auto_follow/test_pattern"); + request.setJsonEntity("{\"leader_index_patterns\": [\"metrics-*\"], \"remote_cluster\": \"leader_cluster\"}"); + assertOK(client().performRequest(request)); + + try (RestClient leaderClient = buildLeaderClient()) { + Settings settings = Settings.builder() + .put("index.soft_deletes.enabled", true) + .build(); + request = new Request("PUT", "/metrics-20210101"); + request.setJsonEntity("{\"settings\": " + Strings.toString(settings) + + ", \"mappings\": {\"_doc\": {\"properties\": {\"field\": {\"type\": \"keyword\"}}}} }"); + assertOK(leaderClient.performRequest(request)); + + for (int i = 0; i < 5; i++) { + String id = Integer.toString(i); + index(leaderClient, "metrics-20210101", id, "field", i, "filtered_field", "true"); + } + } + + assertBusy(() -> { + assertThat(getNumberOfSuccessfulFollowedIndices(), equalTo(initialNumberOfSuccessfulFollowedIndices + 1)); + ensureYellow("metrics-20210101"); + verifyDocuments("metrics-20210101", 5, "filtered_field:true"); + }); + assertBusy(() -> { + verifyCcrMonitoring("metrics-20210101", "metrics-20210101"); + verifyAutoFollowMonitoring(); + }, 30, TimeUnit.SECONDS); + } + + private int getNumberOfSuccessfulFollowedIndices() throws IOException { + Request statsRequest = new Request("GET", "/_ccr/stats"); + Map response = toMap(client().performRequest(statsRequest)); + response = (Map) response.get("auto_follow_stats"); + return (Integer) response.get("number_of_successful_follow_indices"); + } + + } diff --git a/x-pack/plugin/ccr/qa/chain/src/test/java/org/elasticsearch/xpack/ccr/ChainIT.java b/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/ChainIT.java similarity index 100% rename from x-pack/plugin/ccr/qa/chain/src/test/java/org/elasticsearch/xpack/ccr/ChainIT.java rename to x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/ChainIT.java diff --git a/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java b/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java index 17cba1844462e..b44abb15fad15 100644 --- a/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java +++ b/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java @@ -8,10 +8,8 @@ import org.elasticsearch.client.Request; import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; -import java.util.Map; import java.util.concurrent.TimeUnit; import static org.hamcrest.Matchers.containsString; @@ -44,7 +42,7 @@ public void testFollowIndex() throws Exception { } refresh(leaderIndexName); verifyDocuments(leaderIndexName, numDocs, "filtered_field:true"); - } else { + } else if ("follow".equals(targetCluster)) { logger.info("Running against follow cluster"); final String followIndexName = "test_index2"; followIndex(leaderIndexName, followIndexName); @@ -70,7 +68,10 @@ public void testFollowIndex() throws Exception { } public void testFollowNonExistingLeaderIndex() throws Exception { - assumeFalse("Test should only run when both clusters are running", "leader".equals(targetCluster)); + if ("follow".equals(targetCluster) == false) { + logger.info("skipping test, waiting for target cluster [follow]" ); + return; + } ResponseException e = expectThrows(ResponseException.class, () -> resumeFollow("non-existing-index")); assertThat(e.getMessage(), containsString("no such index [non-existing-index]")); assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(404)); @@ -80,41 +81,4 @@ public void testFollowNonExistingLeaderIndex() throws Exception { assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(404)); } - public void testAutoFollowPatterns() throws Exception { - assumeFalse("Test should only run when both clusters are running", "leader".equals(targetCluster)); - - Request request = new Request("PUT", "/_ccr/auto_follow/test_pattern"); - request.setJsonEntity("{\"leader_index_patterns\": [\"logs-*\"], \"remote_cluster\": \"leader_cluster\"}"); - assertOK(client().performRequest(request)); - - try (RestClient leaderClient = buildLeaderClient()) { - Settings settings = Settings.builder() - .put("index.soft_deletes.enabled", true) - .build(); - request = new Request("PUT", "/logs-20190101"); - request.setJsonEntity("{\"settings\": " + Strings.toString(settings) + - ", \"mappings\": {\"_doc\": {\"properties\": {\"field\": {\"type\": \"keyword\"}}}} }"); - assertOK(leaderClient.performRequest(request)); - - for (int i = 0; i < 5; i++) { - String id = Integer.toString(i); - index(leaderClient, "logs-20190101", id, "field", i, "filtered_field", "true"); - } - } - - assertBusy(() -> { - Request statsRequest = new Request("GET", "/_ccr/stats"); - Map response = toMap(client().performRequest(statsRequest)); - response = (Map) response.get("auto_follow_stats"); - assertThat(response.get("number_of_successful_follow_indices"), equalTo(1)); - - ensureYellow("logs-20190101"); - verifyDocuments("logs-20190101", 5, "filtered_field:true"); - }); - assertBusy(() -> { - verifyCcrMonitoring("logs-20190101", "logs-20190101"); - verifyAutoFollowMonitoring(); - }, 30, TimeUnit.SECONDS); - } - } diff --git a/x-pack/plugin/ccr/qa/multi-cluster-with-non-compliant-license/build.gradle b/x-pack/plugin/ccr/qa/non-compliant-license/build.gradle similarity index 100% rename from x-pack/plugin/ccr/qa/multi-cluster-with-non-compliant-license/build.gradle rename to x-pack/plugin/ccr/qa/non-compliant-license/build.gradle diff --git a/x-pack/plugin/ccr/qa/multi-cluster-with-non-compliant-license/src/test/java/org/elasticsearch/xpack/ccr/CcrMultiClusterLicenseIT.java b/x-pack/plugin/ccr/qa/non-compliant-license/src/test/java/org/elasticsearch/xpack/ccr/CcrMultiClusterLicenseIT.java similarity index 100% rename from x-pack/plugin/ccr/qa/multi-cluster-with-non-compliant-license/src/test/java/org/elasticsearch/xpack/ccr/CcrMultiClusterLicenseIT.java rename to x-pack/plugin/ccr/qa/non-compliant-license/src/test/java/org/elasticsearch/xpack/ccr/CcrMultiClusterLicenseIT.java diff --git a/x-pack/plugin/ccr/qa/multi-cluster-with-security/build.gradle b/x-pack/plugin/ccr/qa/security/build.gradle similarity index 100% rename from x-pack/plugin/ccr/qa/multi-cluster-with-security/build.gradle rename to x-pack/plugin/ccr/qa/security/build.gradle diff --git a/x-pack/plugin/ccr/qa/multi-cluster-with-security/follower-roles.yml b/x-pack/plugin/ccr/qa/security/follower-roles.yml similarity index 100% rename from x-pack/plugin/ccr/qa/multi-cluster-with-security/follower-roles.yml rename to x-pack/plugin/ccr/qa/security/follower-roles.yml diff --git a/x-pack/plugin/ccr/qa/multi-cluster-with-security/leader-roles.yml b/x-pack/plugin/ccr/qa/security/leader-roles.yml similarity index 100% rename from x-pack/plugin/ccr/qa/multi-cluster-with-security/leader-roles.yml rename to x-pack/plugin/ccr/qa/security/leader-roles.yml diff --git a/x-pack/plugin/ccr/qa/multi-cluster-with-security/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexSecurityIT.java b/x-pack/plugin/ccr/qa/security/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexSecurityIT.java similarity index 100% rename from x-pack/plugin/ccr/qa/multi-cluster-with-security/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexSecurityIT.java rename to x-pack/plugin/ccr/qa/security/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexSecurityIT.java