From cf9a419f4add89c104f367befd19b34a2b66b83e Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 25 Jan 2019 15:05:23 +0000 Subject: [PATCH 01/18] Forbid minimum_master_nodes in node settings Today the `discovery.zen.minimum_master_nodes` setting is ignored in 7.0 but must still be permitted to live as a cluster-level setting for cases where 6.x nodes leave and rejoin the cluster. However there is no good reason for it to exist as a node-level setting on 7.0 nodes. This change prevents a 7.0 node from starting up with the `discovery.zen.minimum_master_nodes` setting set. --- .../migration/migrate_7_0/discovery.asciidoc | 12 ++- .../rest/discovery/Zen2RestApiIT.java | 8 +- .../ec2/Ec2DiscoveryUpdateSettingsTests.java | 14 ++-- .../cluster/coordination/Coordinator.java | 7 ++ .../elasticsearch/gateway/GatewayService.java | 3 +- .../admin/indices/exists/IndicesExistsIT.java | 4 +- .../cluster/MinimumMasterNodesIT.java | 7 +- .../cluster/SpecificMasterNodesIT.java | 7 -- .../coordination/CoordinatorTests.java | 9 +++ .../coordination/UnsafeBootstrapMasterIT.java | 26 ++----- .../cluster/coordination/Zen1IT.java | 5 +- .../discovery/MasterDisruptionIT.java | 6 -- .../MinimumMasterNodesInClusterStateIT.java | 66 ---------------- .../gateway/RecoverAfterNodesIT.java | 31 ++++---- .../elasticsearch/test/ESIntegTestCase.java | 11 --- .../test/InternalTestCluster.java | 68 +++++++++++----- .../test/discovery/TestZenDiscovery.java | 8 +- .../test/test/InternalTestClusterTests.java | 77 +------------------ .../elasticsearch/license/LicensingTests.java | 15 ++-- ...ServerTransportFilterIntegrationTests.java | 13 +++- 20 files changed, 139 insertions(+), 258 deletions(-) delete mode 100644 server/src/test/java/org/elasticsearch/discovery/zen/MinimumMasterNodesInClusterStateIT.java diff --git a/docs/reference/migration/migrate_7_0/discovery.asciidoc b/docs/reference/migration/migrate_7_0/discovery.asciidoc index d568e7fe32c25..378f4c56b74f8 100644 --- a/docs/reference/migration/migrate_7_0/discovery.asciidoc +++ b/docs/reference/migration/migrate_7_0/discovery.asciidoc @@ -13,9 +13,6 @@ settings summary>> for an example, and the <> describes this setting in more detail. -The `discovery.zen.minimum_master_nodes` setting is required during a rolling -upgrade from 6.x, but can be removed in all other circumstances. - [float] ==== Removing master-eligible nodes sometimes requires voting exclusions @@ -38,3 +35,12 @@ file: - `discovery.zen.ping.unicast.hosts` - `discovery.zen.hosts_provider` - `cluster.initial_master_nodes` + +[float] +==== Minimum master nodes must not be configured + +The `discovery.zen.minimum_master_nodes` setting must be removed from the +`elasticsearch.yml` configuration file. This setting is, however, required to +be set on master-eligible nodes running Elasticsearch 6 or earlier, so if you +are upgrading a node from version 6 to version 7 you must remove this setting +after shutting the version 6 node down for the last time before the upgrade. diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/rest/discovery/Zen2RestApiIT.java b/modules/transport-netty4/src/test/java/org/elasticsearch/rest/discovery/Zen2RestApiIT.java index cfa5a3f6d79c8..88afa57e83e23 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/rest/discovery/Zen2RestApiIT.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/rest/discovery/Zen2RestApiIT.java @@ -33,9 +33,7 @@ import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.common.Priority; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.settings.Settings.Builder; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.discovery.zen.ElectMasterService; import org.elasticsearch.http.HttpServerTransport; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.InternalTestCluster; @@ -58,11 +56,7 @@ public class Zen2RestApiIT extends ESNetty4IntegTestCase { @Override protected Settings nodeSettings(int nodeOrdinal) { - final Builder builder = Settings.builder().put(super.nodeSettings(nodeOrdinal)) - .put(TestZenDiscovery.USE_ZEN2.getKey(), true) - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE); - - return builder.build(); + return Settings.builder().put(super.nodeSettings(nodeOrdinal)).put(TestZenDiscovery.USE_ZEN2.getKey(), true).build(); } @Override diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryUpdateSettingsTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryUpdateSettingsTests.java index e91dff713b4f0..f11bd539fba7d 100644 --- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryUpdateSettingsTests.java +++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryUpdateSettingsTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse; +import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.discovery.DiscoveryModule; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; @@ -41,13 +42,14 @@ public void testMinimumMasterNodesStart() { .build(); internalCluster().startNode(nodeSettings); - // We try to update minimum_master_nodes now - ClusterUpdateSettingsResponse response = client().admin().cluster().prepareUpdateSettings() - .setPersistentSettings(Settings.builder().put("discovery.zen.minimum_master_nodes", 1)) - .setTransientSettings(Settings.builder().put("discovery.zen.minimum_master_nodes", 1)) + // We try to update a setting now + final String expectedValue = UUIDs.randomBase64UUID(random()); + final String settingName = "cluster.routing.allocation.exclude.any_attribute"; + final ClusterUpdateSettingsResponse response = client().admin().cluster().prepareUpdateSettings() + .setPersistentSettings(Settings.builder().put(settingName, expectedValue)) .get(); - Integer min = response.getPersistentSettings().getAsInt("discovery.zen.minimum_master_nodes", null); - assertThat(min, is(1)); + final String value = response.getPersistentSettings().get(settingName); + assertThat(value, is(expectedValue)); } } diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java index aabe5466d69a9..7fb1197793c7a 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java @@ -63,6 +63,7 @@ import org.elasticsearch.discovery.HandshakingTransportAddressConnector; import org.elasticsearch.discovery.PeerFinder; import org.elasticsearch.discovery.UnicastConfiguredHostsResolver; +import org.elasticsearch.discovery.zen.ElectMasterService; import org.elasticsearch.discovery.zen.PendingClusterStateStats; import org.elasticsearch.discovery.zen.UnicastHostsProvider; import org.elasticsearch.threadpool.ThreadPool.Names; @@ -141,6 +142,12 @@ public Coordinator(String nodeName, Settings settings, ClusterSettings clusterSe NamedWriteableRegistry namedWriteableRegistry, AllocationService allocationService, MasterService masterService, Supplier persistedStateSupplier, UnicastHostsProvider unicastHostsProvider, ClusterApplier clusterApplier, Collection> onJoinValidators, Random random) { + + if (ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.exists(settings)) { + throw new IllegalArgumentException("node setting [" + ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey() + + "] is not permitted and must be removed before starting this node"); + } + this.settings = settings; this.transportService = transportService; this.masterService = masterService; diff --git a/server/src/main/java/org/elasticsearch/gateway/GatewayService.java b/server/src/main/java/org/elasticsearch/gateway/GatewayService.java index c71ae40235056..9fc169a2d0d69 100644 --- a/server/src/main/java/org/elasticsearch/gateway/GatewayService.java +++ b/server/src/main/java/org/elasticsearch/gateway/GatewayService.java @@ -40,6 +40,7 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.discovery.Discovery; +import org.elasticsearch.discovery.zen.ElectMasterService; import org.elasticsearch.discovery.zen.ZenDiscovery; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.rest.RestStatus; @@ -116,7 +117,7 @@ public GatewayService(final Settings settings, final AllocationService allocatio if (RECOVER_AFTER_MASTER_NODES_SETTING.exists(settings)) { recoverAfterMasterNodes = RECOVER_AFTER_MASTER_NODES_SETTING.get(settings); } else if (discovery instanceof ZenDiscovery) { - recoverAfterMasterNodes = settings.getAsInt("discovery.zen.minimum_master_nodes", -1); + recoverAfterMasterNodes = settings.getAsInt(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), -1); } else { recoverAfterMasterNodes = -1; } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/exists/IndicesExistsIT.java b/server/src/test/java/org/elasticsearch/action/admin/indices/exists/IndicesExistsIT.java index cd90cda2ba286..33c0d22473c65 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/exists/IndicesExistsIT.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/exists/IndicesExistsIT.java @@ -23,7 +23,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.discovery.MasterNotDiscoveredException; -import org.elasticsearch.discovery.zen.ElectMasterService; import org.elasticsearch.gateway.GatewayService; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; @@ -51,8 +50,7 @@ protected List addExtraClusterBootstrapSettings(List allNode public void testIndexExistsWithBlocksInPlace() throws IOException { Settings settings = Settings.builder() - .put(GatewayService.RECOVER_AFTER_NODES_SETTING.getKey(), 99) - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE).build(); + .put(GatewayService.RECOVER_AFTER_NODES_SETTING.getKey(), 99).build(); String node = internalCluster().startNode(settings); assertThrows(client(node).admin().indices().prepareExists("test").setMasterNodeTimeout(TimeValue.timeValueSeconds(0)), diff --git a/server/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java b/server/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java index c16d9b11e0d91..26b8ae88d266d 100644 --- a/server/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java +++ b/server/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java @@ -26,14 +26,13 @@ import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.coordination.ClusterBootstrapService; +import org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Priority; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.discovery.DiscoverySettings; -import org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException; -import org.elasticsearch.discovery.zen.ElectMasterService; import org.elasticsearch.discovery.zen.ZenDiscovery; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.node.Node; @@ -102,7 +101,6 @@ public void testTwoNodesNoMasterBlock() throws Exception { bootstrapNodeId = 2; Settings settings = Settings.builder() - .put("discovery.zen.minimum_master_nodes", 2) .put(ZenDiscovery.PING_TIMEOUT_SETTING.getKey(), "200ms") .put("discovery.initial_state_timeout", "500ms") .build(); @@ -237,7 +235,6 @@ public void testThreeNodesNoMasterBlock() throws Exception { bootstrapNodeId = 3; Settings settings = Settings.builder() - .put("discovery.zen.minimum_master_nodes", 3) .put(ZenDiscovery.PING_TIMEOUT_SETTING.getKey(), "1s") .put("discovery.initial_state_timeout", "500ms") .build(); @@ -316,11 +313,9 @@ public void testCannotCommitStateThreeNodes() throws Exception { Settings settings = Settings.builder() .put(ZenDiscovery.PING_TIMEOUT_SETTING.getKey(), "200ms") .put("discovery.initial_state_timeout", "500ms") - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), 2) .put(DiscoverySettings.COMMIT_TIMEOUT_SETTING.getKey(), "100ms") // speed things up .build(); - internalCluster().startNodes(3, settings); ensureGreen(); // ensure cluster state is recovered before we disrupt things diff --git a/server/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java b/server/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java index aaef1e58fb50e..63e72d91e0b8e 100644 --- a/server/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java +++ b/server/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java @@ -38,7 +38,6 @@ import java.util.List; import java.util.concurrent.ExecutionException; -import static org.elasticsearch.discovery.zen.ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.nullValue; @@ -47,12 +46,6 @@ @TestLogging("_root:DEBUG,org.elasticsearch.action.admin.cluster.state:TRACE") public class SpecificMasterNodesIT extends ESIntegTestCase { - @Override - protected Settings nodeSettings(int nodeOrdinal) { - return Settings.builder().put(super.nodeSettings(nodeOrdinal)) - .put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), 1).build(); - } - @Override protected List addExtraClusterBootstrapSettings(List allNodesSettings) { // if it's the first master in the cluster bootstrap the cluster with this node name diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java index be40f0c888362..50842201557b6 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java @@ -56,6 +56,7 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor; import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.discovery.zen.ElectMasterService; import org.elasticsearch.discovery.zen.PublishClusterStateStats; import org.elasticsearch.discovery.zen.UnicastHostsProvider.HostsResolver; import org.elasticsearch.env.NodeEnvironment; @@ -1000,6 +1001,14 @@ public void testClusterCannotFormWithFailingJoinValidation() { assertTrue(cluster.clusterNodes.stream().allMatch(cn -> cn.getLastAppliedClusterState().version() == 0)); } + public void testRejectsSettingsWithMinimumMasterNodes() { + final IllegalArgumentException illegalArgumentException = expectThrows(IllegalArgumentException.class, () -> new Coordinator("node", + Settings.builder().put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), 1).build(), + null, null, null, null, null, null, null, null, null, null)); + assertThat(illegalArgumentException.getMessage(), equalTo( + "node setting [discovery.zen.minimum_master_nodes] is not permitted and must be removed before starting this node")); + } + private static long defaultMillis(Setting setting) { return setting.get(Settings.EMPTY).millis() + Cluster.DEFAULT_DELAY_VARIABILITY; } diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterIT.java b/server/src/test/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterIT.java index 73add5ba83520..9741cc6b355cd 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterIT.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterIT.java @@ -28,7 +28,6 @@ import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.discovery.DiscoverySettings; -import org.elasticsearch.discovery.zen.ElectMasterService; import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.env.NodeMetaData; @@ -155,7 +154,6 @@ public void testNoNodeMetaData() throws IOException { public void testNotBootstrappedCluster() throws Exception { internalCluster().startNode( Settings.builder() - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) .put(DiscoverySettings.INITIAL_STATE_TIMEOUT_SETTING.getKey(), "0s") // to ensure quick node startup .build()); assertBusy(() -> { @@ -172,9 +170,7 @@ public void testNotBootstrappedCluster() throws Exception { public void testNoManifestFile() throws IOException { bootstrapNodeId = 1; - internalCluster().startNode(Settings.builder() - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) - .build()); + internalCluster().startNode(Settings.EMPTY); ensureStableCluster(1); NodeEnvironment nodeEnvironment = internalCluster().getMasterNodeInstance(NodeEnvironment.class); internalCluster().stopRandomDataNode(); @@ -186,9 +182,7 @@ public void testNoManifestFile() throws IOException { public void testNoMetaData() throws IOException { bootstrapNodeId = 1; - internalCluster().startNode(Settings.builder() - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) - .build()); + internalCluster().startNode(Settings.EMPTY); ensureStableCluster(1); NodeEnvironment nodeEnvironment = internalCluster().getMasterNodeInstance(NodeEnvironment.class); internalCluster().stopRandomDataNode(); @@ -201,9 +195,7 @@ public void testNoMetaData() throws IOException { public void testAbortedByUser() throws IOException { bootstrapNodeId = 1; - internalCluster().startNode(Settings.builder() - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) - .build()); + internalCluster().startNode(Settings.EMPTY); ensureStableCluster(1); internalCluster().stopRandomDataNode(); @@ -213,13 +205,9 @@ public void testAbortedByUser() throws IOException { public void test3MasterNodes2Failed() throws Exception { bootstrapNodeId = 3; - List masterNodes = internalCluster().startMasterOnlyNodes(3, Settings.builder() - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) - .build()); + List masterNodes = internalCluster().startMasterOnlyNodes(3, Settings.EMPTY); - String dataNode = internalCluster().startDataOnlyNode(Settings.builder() - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) - .build()); + String dataNode = internalCluster().startDataOnlyNode(Settings.EMPTY); createIndex("test"); Client dataNodeClient = internalCluster().client(dataNode); @@ -246,9 +234,7 @@ public void test3MasterNodes2Failed() throws Exception { String.format(Locale.ROOT, UnsafeBootstrapMasterCommand.CLUSTER_STATE_TERM_VERSION_MSG_FORMAT, metaData.coordinationMetaData().term(), metaData.version()))); - internalCluster().startMasterOnlyNode(Settings.builder() - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) - .build()); + internalCluster().startMasterOnlyNode(Settings.EMPTY); assertBusy(() -> { ClusterState state = dataNodeClient.admin().cluster().prepareState().setLocal(true) diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/Zen1IT.java b/server/src/test/java/org/elasticsearch/cluster/coordination/Zen1IT.java index eb4b2d75c73f9..1940bd0fbe2ce 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/Zen1IT.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/Zen1IT.java @@ -34,6 +34,7 @@ import org.elasticsearch.common.Priority; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.discovery.zen.ElectMasterService; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.gateway.MetaStateService; import org.elasticsearch.test.ESIntegTestCase; @@ -194,7 +195,8 @@ public Settings onNodeStopped(String nodeName) { } ClusterHealthResponse clusterHealthResponse = clusterHealthRequestBuilder.get(); assertFalse(nodeName, clusterHealthResponse.isTimedOut()); - return Coordinator.addZen1Attribute(false, Settings.builder().put(ZEN2_SETTINGS)).build(); + return Coordinator.addZen1Attribute(false, Settings.builder().put(ZEN2_SETTINGS) + .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE)).build(); } }); @@ -289,6 +291,7 @@ public Settings onNodeStopped(String nodeName) throws Exception { return Coordinator.addZen1Attribute(false, Settings.builder()) .put(ZEN2_SETTINGS) .putList(INITIAL_MASTER_NODES_SETTING.getKey(), nodeNames) + .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) .build(); } }); diff --git a/server/src/test/java/org/elasticsearch/discovery/MasterDisruptionIT.java b/server/src/test/java/org/elasticsearch/discovery/MasterDisruptionIT.java index 652ce1fca1e14..a14c06fd49fb9 100644 --- a/server/src/test/java/org/elasticsearch/discovery/MasterDisruptionIT.java +++ b/server/src/test/java/org/elasticsearch/discovery/MasterDisruptionIT.java @@ -35,7 +35,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.discovery.zen.ElectMasterService; import org.elasticsearch.discovery.zen.ZenDiscovery; import org.elasticsearch.monitor.jvm.HotThreads; import org.elasticsearch.test.ESIntegTestCase; @@ -125,11 +124,6 @@ public void testNodesFDAfterMasterReelection() throws Exception { ensureStableCluster(3); - logger.info("--> reducing min master nodes to 2"); - assertAcked(client().admin().cluster().prepareUpdateSettings() - .setTransientSettings(Settings.builder().put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), 2)) - .get()); - String master = internalCluster().getMasterName(); String nonMaster = null; for (String node : internalCluster().getNodeNames()) { diff --git a/server/src/test/java/org/elasticsearch/discovery/zen/MinimumMasterNodesInClusterStateIT.java b/server/src/test/java/org/elasticsearch/discovery/zen/MinimumMasterNodesInClusterStateIT.java deleted file mode 100644 index f60a313a5d41e..0000000000000 --- a/server/src/test/java/org/elasticsearch/discovery/zen/MinimumMasterNodesInClusterStateIT.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.discovery.zen; - -import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.test.ESIntegTestCase; - -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static org.elasticsearch.discovery.zen.ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING; -import static org.elasticsearch.test.InternalTestCluster.nameFilter; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.isIn; - -@ESIntegTestCase.ClusterScope(numDataNodes = 0, numClientNodes = 0) -public class MinimumMasterNodesInClusterStateIT extends ESIntegTestCase { - - public void testMasterPublishes() throws Exception { - final String firstNode = internalCluster().startNode(); - - { - final ClusterState localState - = client(firstNode).admin().cluster().state(new ClusterStateRequest().local(true)).get().getState(); - assertThat(localState.getMinimumMasterNodesOnPublishingMaster(), equalTo(1)); - assertFalse(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.exists(localState.metaData().settings())); - } - - final List secondThirdNodes = internalCluster().startNodes(2); - assertThat(internalCluster().getMasterName(), equalTo(firstNode)); - - final List allNodes = Stream.concat(Stream.of(firstNode), secondThirdNodes.stream()).collect(Collectors.toList()); - for (final String node : allNodes) { - final ClusterState localState = client(node).admin().cluster().state(new ClusterStateRequest().local(true)).get().getState(); - assertThat(localState.getMinimumMasterNodesOnPublishingMaster(), equalTo(1)); - assertThat(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.get(localState.metaData().settings()), equalTo(2)); - } - - internalCluster().stopRandomNode(nameFilter(firstNode)); - assertThat(internalCluster().getMasterName(), isIn(secondThirdNodes)); - - for (final String node : secondThirdNodes) { - final ClusterState localState = client(node).admin().cluster().state(new ClusterStateRequest().local(true)).get().getState(); - assertThat(localState.getMinimumMasterNodesOnPublishingMaster(), equalTo(2)); - assertThat(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.get(localState.metaData().settings()), equalTo(2)); - } - } -} diff --git a/server/src/test/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java b/server/src/test/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java index e97f69b6d4965..e6fc2ed975fbb 100644 --- a/server/src/test/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java +++ b/server/src/test/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java @@ -25,7 +25,6 @@ import org.elasticsearch.cluster.coordination.ClusterBootstrapService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.discovery.zen.ElectMasterService; import org.elasticsearch.node.Node; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; @@ -59,7 +58,7 @@ protected List addExtraClusterBootstrapSettings(List allNode return super.addExtraClusterBootstrapSettings(allNodesSettings); } - public Set waitForNoBlocksOnNode(TimeValue timeout, Client nodeClient) throws InterruptedException { + public Set waitForNoBlocksOnNode(TimeValue timeout, Client nodeClient) { long start = System.currentTimeMillis(); Set blocks; do { @@ -70,22 +69,20 @@ public Set waitForNoBlocksOnNode(TimeValue timeout, Client nodeCli return blocks; } - public Client startNode(Settings.Builder settings, int minMasterNodes) { - String name = internalCluster().startNode( - Settings.builder().put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), minMasterNodes) - .put(settings.build())); + public Client startNode(Settings.Builder settings) { + String name = internalCluster().startNode(Settings.builder().put(settings.build())); return internalCluster().client(name); } public void testRecoverAfterNodes() throws Exception { logger.info("--> start node (1)"); - Client clientNode1 = startNode(Settings.builder().put("gateway.recover_after_nodes", 3), 1); + Client clientNode1 = startNode(Settings.builder().put("gateway.recover_after_nodes", 3)); assertThat(clientNode1.admin().cluster().prepareState().setLocal(true).execute().actionGet() .getState().blocks().global(ClusterBlockLevel.METADATA_WRITE), hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK)); logger.info("--> start node (2)"); - Client clientNode2 = startNode(Settings.builder().put("gateway.recover_after_nodes", 3), 1); + Client clientNode2 = startNode(Settings.builder().put("gateway.recover_after_nodes", 3)); Thread.sleep(BLOCK_WAIT_TIMEOUT.millis()); assertThat(clientNode1.admin().cluster().prepareState().setLocal(true).execute().actionGet() .getState().blocks().global(ClusterBlockLevel.METADATA_WRITE), @@ -95,7 +92,7 @@ public void testRecoverAfterNodes() throws Exception { hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK)); logger.info("--> start node (3)"); - Client clientNode3 = startNode(Settings.builder().put("gateway.recover_after_nodes", 3), 1); + Client clientNode3 = startNode(Settings.builder().put("gateway.recover_after_nodes", 3)); assertThat(waitForNoBlocksOnNode(BLOCK_WAIT_TIMEOUT, clientNode1).isEmpty(), equalTo(true)); assertThat(waitForNoBlocksOnNode(BLOCK_WAIT_TIMEOUT, clientNode2).isEmpty(), equalTo(true)); @@ -106,7 +103,7 @@ public void testRecoverAfterMasterNodes() throws Exception { logger.info("--> start master_node (1)"); Client master1 = startNode(Settings.builder() .put("gateway.recover_after_master_nodes", 2).put(Node.NODE_DATA_SETTING.getKey(), false) - .put(Node.NODE_MASTER_SETTING.getKey(), true), 1); + .put(Node.NODE_MASTER_SETTING.getKey(), true)); assertThat(master1.admin().cluster().prepareState().setLocal(true).execute().actionGet() .getState().blocks().global(ClusterBlockLevel.METADATA_WRITE), hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK)); @@ -114,7 +111,7 @@ public void testRecoverAfterMasterNodes() throws Exception { logger.info("--> start data_node (1)"); Client data1 = startNode(Settings.builder() .put("gateway.recover_after_master_nodes", 2) - .put(Node.NODE_DATA_SETTING.getKey(), true).put(Node.NODE_MASTER_SETTING.getKey(), false), 1); + .put(Node.NODE_DATA_SETTING.getKey(), true).put(Node.NODE_MASTER_SETTING.getKey(), false)); assertThat(master1.admin().cluster().prepareState().setLocal(true).execute().actionGet() .getState().blocks().global(ClusterBlockLevel.METADATA_WRITE), hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK)); @@ -125,7 +122,7 @@ public void testRecoverAfterMasterNodes() throws Exception { logger.info("--> start data_node (2)"); Client data2 = startNode(Settings.builder() .put("gateway.recover_after_master_nodes", 2).put(Node.NODE_DATA_SETTING.getKey(), true) - .put(Node.NODE_MASTER_SETTING.getKey(), false), 1); + .put(Node.NODE_MASTER_SETTING.getKey(), false)); assertThat(master1.admin().cluster().prepareState().setLocal(true).execute().actionGet() .getState().blocks().global(ClusterBlockLevel.METADATA_WRITE), hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK)); @@ -140,7 +137,7 @@ public void testRecoverAfterMasterNodes() throws Exception { Client master2 = startNode(Settings.builder() .put("gateway.recover_after_master_nodes", 2) .put(Node.NODE_DATA_SETTING.getKey(), false) - .put(Node.NODE_MASTER_SETTING.getKey(), true), 1); + .put(Node.NODE_MASTER_SETTING.getKey(), true)); assertThat(waitForNoBlocksOnNode(BLOCK_WAIT_TIMEOUT, master1).isEmpty(), equalTo(true)); assertThat(waitForNoBlocksOnNode(BLOCK_WAIT_TIMEOUT, master2).isEmpty(), equalTo(true)); assertThat(waitForNoBlocksOnNode(BLOCK_WAIT_TIMEOUT, data1).isEmpty(), equalTo(true)); @@ -152,7 +149,7 @@ public void testRecoverAfterDataNodes() throws Exception { Client master1 = startNode(Settings.builder() .put("gateway.recover_after_data_nodes", 2) .put(Node.NODE_DATA_SETTING.getKey(), false) - .put(Node.NODE_MASTER_SETTING.getKey(), true), 1); + .put(Node.NODE_MASTER_SETTING.getKey(), true)); assertThat(master1.admin().cluster().prepareState().setLocal(true).execute().actionGet() .getState().blocks().global(ClusterBlockLevel.METADATA_WRITE), hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK)); @@ -161,7 +158,7 @@ public void testRecoverAfterDataNodes() throws Exception { Client data1 = startNode(Settings.builder() .put("gateway.recover_after_data_nodes", 2) .put(Node.NODE_DATA_SETTING.getKey(), true) - .put(Node.NODE_MASTER_SETTING.getKey(), false), 1); + .put(Node.NODE_MASTER_SETTING.getKey(), false)); assertThat(master1.admin().cluster().prepareState().setLocal(true).execute().actionGet() .getState().blocks().global(ClusterBlockLevel.METADATA_WRITE), hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK)); @@ -173,7 +170,7 @@ public void testRecoverAfterDataNodes() throws Exception { Client master2 = startNode(Settings.builder() .put("gateway.recover_after_data_nodes", 2) .put(Node.NODE_DATA_SETTING.getKey(), false) - .put(Node.NODE_MASTER_SETTING.getKey(), true), 1); + .put(Node.NODE_MASTER_SETTING.getKey(), true)); assertThat(master2.admin().cluster().prepareState().setLocal(true).execute().actionGet() .getState().blocks().global(ClusterBlockLevel.METADATA_WRITE), hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK)); @@ -188,7 +185,7 @@ public void testRecoverAfterDataNodes() throws Exception { Client data2 = startNode(Settings.builder() .put("gateway.recover_after_data_nodes", 2) .put(Node.NODE_DATA_SETTING.getKey(), true) - .put(Node.NODE_MASTER_SETTING.getKey(), false), 1); + .put(Node.NODE_MASTER_SETTING.getKey(), false)); assertThat(waitForNoBlocksOnNode(BLOCK_WAIT_TIMEOUT, master1).isEmpty(), equalTo(true)); assertThat(waitForNoBlocksOnNode(BLOCK_WAIT_TIMEOUT, master2).isEmpty(), equalTo(true)); assertThat(waitForNoBlocksOnNode(BLOCK_WAIT_TIMEOUT, data1).isEmpty(), equalTo(true)); diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index 6e3089a6d63e1..45f8682dc5e61 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -1087,17 +1087,6 @@ public long waitForDocs(final long numDocs, int maxWaitTime, TimeUnit maxWaitTim return lastKnownCount.get(); } - - /** - * Sets the cluster's minimum master node and make sure the response is acknowledge. - * Note: this doesn't guarantee that the new setting has taken effect, just that it has been received by all nodes. - */ - public void setMinimumMasterNodes(int n) { - assertTrue(client().admin().cluster().prepareUpdateSettings().setTransientSettings( - Settings.builder().put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), n)) - .get().isAcknowledged()); - } - /** * Prints the current cluster state as debug logging. */ diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java index 55e356d093e6e..c9b652e69f5f0 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java @@ -33,8 +33,8 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.admin.cluster.configuration.AddVotingConfigExclusionsAction; import org.elasticsearch.action.admin.cluster.configuration.AddVotingConfigExclusionsRequest; -import org.elasticsearch.action.admin.cluster.configuration.ClearVotingConfigExclusionsRequest; import org.elasticsearch.action.admin.cluster.configuration.ClearVotingConfigExclusionsAction; +import org.elasticsearch.action.admin.cluster.configuration.ClearVotingConfigExclusionsRequest; import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags.Flag; @@ -157,11 +157,15 @@ import static org.elasticsearch.test.ESTestCase.awaitBusy; import static org.elasticsearch.test.ESTestCase.getTestTransportType; import static org.elasticsearch.test.ESTestCase.randomFrom; +import static org.elasticsearch.test.discovery.TestZenDiscovery.USE_ZEN2; +import static org.elasticsearch.test.discovery.TestZenDiscovery.usingZen1; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.nullValue; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -615,19 +619,28 @@ private Settings getNodeSettings(final int nodeId, final long seed, final Settin .put("node.name", name) .put(NodeEnvironment.NODE_ID_SEED_SETTING.getKey(), seed); - final boolean usingSingleNodeDiscovery = DiscoveryModule.DISCOVERY_TYPE_SETTING.get(updatedSettings.build()).equals("single-node"); - if (!usingSingleNodeDiscovery && autoManageMinMasterNodes) { - assert updatedSettings.get(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey()) == null : - "min master nodes may not be set when auto managed"; - assert updatedSettings.get(INITIAL_STATE_TIMEOUT_SETTING.getKey()) == null : - "automatically managing min master nodes require nodes to complete a join cycle" + - " when starting"; - updatedSettings - // don't wait too long not to slow down tests - .put(ZenDiscovery.MASTER_ELECTION_WAIT_FOR_JOINS_TIMEOUT_SETTING.getKey(), "5s") - .put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), defaultMinMasterNodes); - } else if (!usingSingleNodeDiscovery && updatedSettings.get(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey()) == null) { - throw new IllegalArgumentException(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey() + " must be configured"); + final String discoveryType = DiscoveryModule.DISCOVERY_TYPE_SETTING.get(updatedSettings.build()); + final boolean usingSingleNodeDiscovery = discoveryType.equals("single-node"); + final boolean usingZen1 = TestZenDiscovery.usingZen1(updatedSettings.build()); + if (usingSingleNodeDiscovery == false) { + if (autoManageMinMasterNodes) { + assertThat("min master nodes may not be set when auto managed", + updatedSettings.get(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey()), nullValue()); + assertThat("automatically managing min master nodes require nodes to complete a join cycle when starting", + updatedSettings.get(INITIAL_STATE_TIMEOUT_SETTING.getKey()), nullValue()); + + if (usingZen1) { + updatedSettings + // don't wait too long not to slow down tests + .put(ZenDiscovery.MASTER_ELECTION_WAIT_FOR_JOINS_TIMEOUT_SETTING.getKey(), "5s") + .put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), defaultMinMasterNodes); + } + } else { + if (usingZen1) { + assertThat(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey() + " must be configured", + updatedSettings.get(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey()), not(nullValue())); + } + } } return updatedSettings.build(); @@ -933,8 +946,10 @@ Settings closeForRestart(RestartCallback callback, int minMasterNodes) throws Ex Settings.Builder newSettings = Settings.builder(); newSettings.put(callbackSettings); if (minMasterNodes >= 0) { - assert DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.exists(newSettings.build()) == false : "min master nodes is auto managed"; - newSettings.put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), minMasterNodes); + if (usingZen1(newSettings.build())) { + assertFalse("min master nodes is auto managed", DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.exists(newSettings.build())); + newSettings.put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), minMasterNodes); + } if (INITIAL_MASTER_NODES_SETTING.exists(callbackSettings) == false) { newSettings.putList(INITIAL_MASTER_NODES_SETTING.getKey()); } @@ -966,10 +981,23 @@ private void recreateNode(final Settings newSettings, final Runnable onTransport .put(newSettings) .put(NodeEnvironment.NODE_ID_SEED_SETTING.getKey(), newIdSeed) .build(); - final boolean usingSingleNodeDiscovery = DiscoveryModule.DISCOVERY_TYPE_SETTING.get(finalSettings).equals("single-node"); - if (usingSingleNodeDiscovery == false && DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.exists(finalSettings) == false) { - throw new IllegalStateException(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey() + - " is not configured after restart of [" + name + "]"); + if (usingZen1(finalSettings)) { + if (DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.exists(finalSettings) == false) { + throw new IllegalStateException(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey() + + " is not configured after restart of [" + name + "]"); + } + } else { + if (DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.exists(finalSettings)) { + // simulating an upgrade from Zen1 to Zen2, but there's no way to remove a setting when restarting a node, so + // you have to set it to Integer.MAX_VALUE to indicate its removal: + assertTrue(USE_ZEN2.exists(finalSettings)); + assertTrue(USE_ZEN2.get(finalSettings)); + assertThat(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.get(finalSettings), equalTo(Integer.MAX_VALUE)); + + final Builder builder = Settings.builder().put(finalSettings); + builder.remove(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey()); + finalSettings = builder.build(); + } } Collection> plugins = node.getClasspathPlugins(); node = new MockNode(finalSettings, plugins); diff --git a/test/framework/src/main/java/org/elasticsearch/test/discovery/TestZenDiscovery.java b/test/framework/src/main/java/org/elasticsearch/test/discovery/TestZenDiscovery.java index 53be34c0b40c8..56e6c24571715 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/discovery/TestZenDiscovery.java +++ b/test/framework/src/main/java/org/elasticsearch/test/discovery/TestZenDiscovery.java @@ -60,6 +60,7 @@ public class TestZenDiscovery extends ZenDiscovery { public static final Setting USE_ZEN2 = Setting.boolSetting("discovery.zen.use_zen2", true, Setting.Property.NodeScope); + private static final String TEST_ZEN_DISCOVERY_TYPE = "test-zen"; /** A plugin which installs mock discovery and configures it to be used. */ public static class TestPlugin extends Plugin implements DiscoveryPlugin { @@ -97,7 +98,7 @@ public List> getSettings() { @Override public Settings additionalSettings() { return Settings.builder() - .put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), "test-zen") + .put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), TEST_ZEN_DISCOVERY_TYPE) .putList(DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING.getKey()) .build(); } @@ -124,4 +125,9 @@ protected ZenPing newZenPing(Settings settings, ThreadPool threadPool, Transport public ZenPing getZenPing() { return zenPing; } + + public static boolean usingZen1(Settings settings) { + return DiscoveryModule.ZEN_DISCOVERY_TYPE.equals(DiscoveryModule.DISCOVERY_TYPE_SETTING.get(settings)) + || USE_ZEN2.get(settings) == false; + } } diff --git a/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java b/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java index 38d76423eea9c..1a1e036e7c7ee 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java @@ -101,7 +101,6 @@ public void testInitializiationIsConsistent() { nodePrefix, Collections.emptyList(), Function.identity()); // TODO: this is not ideal - we should have a way to make sure ports are initialized in the same way assertClusters(cluster0, cluster1, false); - } /** @@ -140,36 +139,6 @@ public static void assertSettings(Settings left, Settings right, boolean checkCl } } - private void assertMMNinNodeSetting(InternalTestCluster cluster, int masterNodes) { - for (final String node : cluster.getNodeNames()) { - assertMMNinNodeSetting(node, cluster, masterNodes); - } - } - - private void assertMMNinNodeSetting(String node, InternalTestCluster cluster, int masterNodes) { - final int minMasterNodes = masterNodes / 2 + 1; - Settings nodeSettings = cluster.client(node).admin().cluster().prepareNodesInfo(node).get().getNodes().get(0).getSettings(); - assertEquals("node setting of node [" + node + "] has the wrong min_master_node setting: [" - + nodeSettings.get(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey()) + "]", - DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.get(nodeSettings).intValue(), minMasterNodes); - } - - private void assertMMNinClusterSetting(InternalTestCluster cluster, int masterNodes) { - for (final String node : cluster.getNodeNames()) { - assertMMNinClusterSetting(node, cluster, masterNodes); - } - } - - private void assertMMNinClusterSetting(String node, InternalTestCluster cluster, int masterNodes) { - final int minMasterNodes = masterNodes / 2 + 1; - Settings stateSettings = cluster.client(node).admin().cluster().prepareState().setLocal(true) - .get().getState().getMetaData().settings(); - - assertEquals("dynamic setting for node [" + node + "] has the wrong min_master_node setting : [" - + stateSettings.get(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey()) + "]", - DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.get(stateSettings).intValue(), minMasterNodes); - } - public void testBeforeTest() throws Exception { final boolean autoManageMinMasterNodes = randomBoolean(); long clusterSeed = randomLong(); @@ -257,8 +226,6 @@ public Settings transportClientSettings() { Client other = iterator1.next(); assertSettings(client.settings(), other.settings(), false); } - assertMMNinNodeSetting(cluster0, cluster0.numMasterNodes()); - assertMMNinNodeSetting(cluster1, cluster0.numMasterNodes()); cluster0.afterTest(); cluster1.afterTest(); } finally { @@ -320,7 +287,6 @@ public Settings transportClientSettings() { try { cluster.beforeTest(random(), 0.0); final int originalMasterCount = cluster.numMasterNodes(); - assertMMNinNodeSetting(cluster, originalMasterCount); final Map shardNodePaths = new HashMap<>(); for (String name: cluster.getNodeNames()) { shardNodePaths.put(name, getNodePaths(cluster, name)); @@ -335,10 +301,6 @@ public Settings transportClientSettings() { expectedMasterCount--; } cluster.stopRandomNode(InternalTestCluster.nameFilter(poorNode)); - if (expectedMasterCount != originalMasterCount) { - // check for updated - assertMMNinClusterSetting(cluster, expectedMasterCount); - } assertFileExists(testMarker); // stopping a node half way shouldn't clean data final String stableNode = randomFrom(cluster.getNodeNames()); @@ -351,14 +313,8 @@ public Settings transportClientSettings() { expectedMasterCount++; assertThat(getNodePaths(cluster, newNode1)[0], equalTo(dataPath)); assertFileExists(testMarker); // starting a node should re-use data folders and not clean it - if (expectedMasterCount > 1) { // this is the first master, it's in cluster state settings won't be updated - assertMMNinClusterSetting(cluster, expectedMasterCount); - } - assertMMNinNodeSetting(newNode1, cluster, expectedMasterCount); - final String newNode2 = cluster.startNode(); expectedMasterCount++; - assertMMNinClusterSetting(cluster, expectedMasterCount); final Path newDataPath = getNodePaths(cluster, newNode2)[0]; final Path newTestMarker = newDataPath.resolve("newTestMarker"); assertThat(newDataPath, not(dataPath)); @@ -377,8 +333,6 @@ public Settings transportClientSettings() { assertThat("data paths for " + name + " changed", getNodePaths(cluster, name), equalTo(shardNodePaths.get(name))); } - assertMMNinNodeSetting(cluster, originalMasterCount); - } finally { cluster.close(); } @@ -393,7 +347,6 @@ private Path[] getNodePaths(InternalTestCluster cluster, String name) { } } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/37462") public void testDifferentRolesMaintainPathOnRestart() throws Exception { final Path baseDir = createTempDir(); final int numNodes = 5; @@ -448,11 +401,6 @@ public Settings transportClientSettings() { roles.add(role); } - final long masterCount = roles.stream().filter(role -> role == MASTER).count(); - final Settings minMasterNodes = Settings.builder() - .put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), masterCount / 2 + 1) - .build(); - try { Map> pathsPerRole = new HashMap<>(); for (int i = 0; i < numNodes; i++) { @@ -460,13 +408,13 @@ public Settings transportClientSettings() { final String node; switch (role) { case MASTER: - node = cluster.startMasterOnlyNode(minMasterNodes); + node = cluster.startMasterOnlyNode(); break; case DATA: - node = cluster.startDataOnlyNode(minMasterNodes); + node = cluster.startDataOnlyNode(); break; case INGEST: - node = cluster.startCoordinatingOnlyNode(minMasterNodes); + node = cluster.startCoordinatingOnlyNode(Settings.EMPTY); break; default: throw new IllegalStateException("get your story straight"); @@ -533,34 +481,18 @@ public Settings transportClientSettings() { plugins, Function.identity()); try { cluster.beforeTest(random(), 0.0); - assertMMNinNodeSetting(cluster, 2); switch (randomInt(2)) { case 0: cluster.stopRandomDataNode(); - assertMMNinClusterSetting(cluster, 1); cluster.startNode(); - assertMMNinClusterSetting(cluster, 2); - assertMMNinNodeSetting(cluster, 2); break; case 1: - cluster.rollingRestart(new InternalTestCluster.RestartCallback() { - @Override - public Settings onNodeStopped(String nodeName) throws Exception { - for (String name : cluster.getNodeNames()) { - if (name.equals(nodeName) == false) { - assertMMNinClusterSetting(name, cluster, 1); - } - } - return super.onNodeStopped(nodeName); - } - }); - assertMMNinClusterSetting(cluster, 2); + cluster.rollingRestart(InternalTestCluster.EMPTY_CALLBACK); break; case 2: cluster.fullRestart(); break; } - assertMMNinNodeSetting(cluster, 2); } finally { cluster.close(); } @@ -585,6 +517,5 @@ public Settings additionalSettings() { } return Settings.builder().put("node.attr.dummy", true).build(); } - } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/license/LicensingTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/license/LicensingTests.java index e79ec2e2fc059..f146f12245e1f 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/license/LicensingTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/license/LicensingTests.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.discovery.DiscoveryModule; +import org.elasticsearch.discovery.zen.ElectMasterService; import org.elasticsearch.node.MockNode; import org.elasticsearch.node.Node; import org.elasticsearch.plugins.Plugin; @@ -290,23 +291,25 @@ public void testNodeJoinWithoutSecurityExplicitlyEnabled() throws Exception { Path home = createTempDir(); Path conf = home.resolve("config"); Files.createDirectories(conf); - Settings nodeSettings = Settings.builder() + Settings.Builder nodeSettings = Settings.builder() .put(nodeSettings(maxNumberOfNodes() - 1).filter(s -> "xpack.security.enabled".equals(s) == false)) .put("node.name", "my-test-node") .put("network.host", "localhost") .put("cluster.name", internalCluster().getClusterName()) - .put("discovery.zen.minimum_master_nodes", - internalCluster().getInstance(Settings.class).get("discovery.zen.minimum_master_nodes")) .put("path.home", home) .put(TestZenDiscovery.USE_MOCK_PINGS.getKey(), false) .put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), "test-zen") .put(TestZenDiscovery.USE_ZEN2.getKey(), getUseZen2()) .putList(DiscoveryModule.DISCOVERY_HOSTS_PROVIDER_SETTING.getKey()) - .putList(DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING.getKey(), unicastHostsList) - .build(); + .putList(DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING.getKey(), unicastHostsList); + if (getUseZen2() == false) { + nodeSettings.put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), + ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.get(internalCluster().getInstance(Settings.class))); + } + Collection> mockPlugins = Arrays.asList(LocalStateSecurity.class, TestZenDiscovery.TestPlugin.class, MockHttpTransport.TestPlugin.class); - try (Node node = new MockNode(nodeSettings, mockPlugins)) { + try (Node node = new MockNode(nodeSettings.build(), mockPlugins)) { node.start(); ensureStableCluster(cluster().size() + 1); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ServerTransportFilterIntegrationTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ServerTransportFilterIntegrationTests.java index 2383f3b3ac739..83640d9e931b5 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ServerTransportFilterIntegrationTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ServerTransportFilterIntegrationTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.common.network.NetworkAddress; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.discovery.zen.ElectMasterService; import org.elasticsearch.node.MockNode; import org.elasticsearch.node.Node; import org.elasticsearch.node.NodeValidationException; @@ -101,8 +102,6 @@ public void testThatConnectionToServerTypeConnectionWorks() throws IOException, .put("network.host", "localhost") .put("cluster.name", internalCluster().getClusterName()) .put("discovery.zen.ping.unicast.hosts", unicastHost) - .put("discovery.zen.minimum_master_nodes", - internalCluster().getInstance(Settings.class).get("discovery.zen.minimum_master_nodes")) .put("xpack.security.enabled", true) .put("xpack.security.audit.enabled", false) .put("xpack.security.transport.ssl.enabled", true) @@ -111,6 +110,10 @@ public void testThatConnectionToServerTypeConnectionWorks() throws IOException, .put(Node.NODE_MASTER_SETTING.getKey(), false) .put(TestZenDiscovery.USE_ZEN2.getKey(), getUseZen2()) .put(TestZenDiscovery.USE_MOCK_PINGS.getKey(), false); + if (getUseZen2() == false) { + nodeSettings.put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), + ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.get(internalCluster().getInstance(Settings.class))); + } Collection> mockPlugins = Arrays.asList( LocalStateSecurity.class, TestZenDiscovery.TestPlugin.class, MockHttpTransport.TestPlugin.class); addSSLSettingsForPEMFiles( @@ -144,8 +147,6 @@ public void testThatConnectionToClientTypeConnectionIsRejected() throws IOExcept .put(SecurityField.USER_SETTING.getKey(), "test_user:" + SecuritySettingsSourceField.TEST_PASSWORD) .put("cluster.name", internalCluster().getClusterName()) .put("discovery.zen.ping.unicast.hosts", unicastHost) - .put("discovery.zen.minimum_master_nodes", - internalCluster().getInstance(Settings.class).get("discovery.zen.minimum_master_nodes")) .put("xpack.security.enabled", true) .put("xpack.security.audit.enabled", false) .put("xpack.security.transport.ssl.enabled", true) @@ -155,6 +156,10 @@ public void testThatConnectionToClientTypeConnectionIsRejected() throws IOExcept .put(Node.NODE_MASTER_SETTING.getKey(), false) .put(TestZenDiscovery.USE_ZEN2.getKey(), getUseZen2()) .put(TestZenDiscovery.USE_MOCK_PINGS.getKey(), false); + if (getUseZen2() == false) { + nodeSettings.put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), + ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.get(internalCluster().getInstance(Settings.class))); + } Collection> mockPlugins = Arrays.asList( LocalStateSecurity.class, TestZenDiscovery.TestPlugin.class, MockHttpTransport.TestPlugin.class); addSSLSettingsForPEMFiles( From 7aed19d886993ae59c944304bbb39b33d9b0714c Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 25 Jan 2019 15:10:43 +0000 Subject: [PATCH 02/18] startNode(Settings.EMPTY) is just startNode() --- .../cluster/coordination/UnsafeBootstrapMasterIT.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterIT.java b/server/src/test/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterIT.java index 9741cc6b355cd..2ae5c31a28896 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterIT.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterIT.java @@ -170,7 +170,7 @@ public void testNotBootstrappedCluster() throws Exception { public void testNoManifestFile() throws IOException { bootstrapNodeId = 1; - internalCluster().startNode(Settings.EMPTY); + internalCluster().startNode(); ensureStableCluster(1); NodeEnvironment nodeEnvironment = internalCluster().getMasterNodeInstance(NodeEnvironment.class); internalCluster().stopRandomDataNode(); @@ -182,7 +182,7 @@ public void testNoManifestFile() throws IOException { public void testNoMetaData() throws IOException { bootstrapNodeId = 1; - internalCluster().startNode(Settings.EMPTY); + internalCluster().startNode(); ensureStableCluster(1); NodeEnvironment nodeEnvironment = internalCluster().getMasterNodeInstance(NodeEnvironment.class); internalCluster().stopRandomDataNode(); @@ -195,7 +195,7 @@ public void testNoMetaData() throws IOException { public void testAbortedByUser() throws IOException { bootstrapNodeId = 1; - internalCluster().startNode(Settings.EMPTY); + internalCluster().startNode(); ensureStableCluster(1); internalCluster().stopRandomDataNode(); From 9a952da71601dfb96323e82b8d8b8abff596bea4 Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 25 Jan 2019 15:11:30 +0000 Subject: [PATCH 03/18] startNode(Settings.EMPTY) is just startNode() --- .../cluster/coordination/UnsafeBootstrapMasterIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterIT.java b/server/src/test/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterIT.java index 2ae5c31a28896..334d392b1793d 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterIT.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterIT.java @@ -207,7 +207,7 @@ public void test3MasterNodes2Failed() throws Exception { bootstrapNodeId = 3; List masterNodes = internalCluster().startMasterOnlyNodes(3, Settings.EMPTY); - String dataNode = internalCluster().startDataOnlyNode(Settings.EMPTY); + String dataNode = internalCluster().startDataOnlyNode(); createIndex("test"); Client dataNodeClient = internalCluster().client(dataNode); @@ -234,7 +234,7 @@ public void test3MasterNodes2Failed() throws Exception { String.format(Locale.ROOT, UnsafeBootstrapMasterCommand.CLUSTER_STATE_TERM_VERSION_MSG_FORMAT, metaData.coordinationMetaData().term(), metaData.version()))); - internalCluster().startMasterOnlyNode(Settings.EMPTY); + internalCluster().startMasterOnlyNode(); assertBusy(() -> { ClusterState state = dataNodeClient.admin().cluster().prepareState().setLocal(true) From 6dc78b16bf948323b2ce44831c7544c43350ab46 Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 25 Jan 2019 16:53:29 +0000 Subject: [PATCH 04/18] Fix REST tests too --- .../elasticsearch/gradle/test/ClusterFormationTasks.groovy | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy index e38cb854a10b0..b2d8012afcf2c 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy @@ -356,12 +356,10 @@ class ClusterFormationTasks { 'node.attr.testattr' : 'test' ] int minimumMasterNodes = node.config.minimumMasterNodes.call() - if (minimumMasterNodes > 0) { + if (node.nodeVersion.before("7.0.0") && minimumMasterNodes > 0) { esConfig['discovery.zen.minimum_master_nodes'] = minimumMasterNodes } - if (minimumMasterNodes > 1) { - // don't wait for state.. just start up quickly - // this will also allow new and old nodes in the BWC case to become the master + if (node.config.numNodes > 1) { esConfig['discovery.initial_state_timeout'] = '0s' } if (esConfig.containsKey('discovery.zen.master_election.wait_for_joins_timeout') == false) { From 186a16d1186a1e6c4fdd78b3983ab23b08a6c7ce Mon Sep 17 00:00:00 2001 From: David Turner Date: Sat, 26 Jan 2019 12:10:12 +0000 Subject: [PATCH 05/18] Add NOCOMMIT because we don't want to be so strict --- .../elasticsearch/cluster/coordination/Coordinator.java | 2 ++ .../cluster/coordination/CoordinatorTests.java | 9 --------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java index 7fb1197793c7a..3a577eb7865f1 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java @@ -144,6 +144,8 @@ public Coordinator(String nodeName, Settings settings, ClusterSettings clusterSe ClusterApplier clusterApplier, Collection> onJoinValidators, Random random) { if (ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.exists(settings)) { + // NOCOMMIT we are not going to do this in the finished PR, but we want to be sure that we've caught all places that still + // set this setting first throw new IllegalArgumentException("node setting [" + ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey() + "] is not permitted and must be removed before starting this node"); } diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java index 50842201557b6..be40f0c888362 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java @@ -56,7 +56,6 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor; import org.elasticsearch.common.util.set.Sets; -import org.elasticsearch.discovery.zen.ElectMasterService; import org.elasticsearch.discovery.zen.PublishClusterStateStats; import org.elasticsearch.discovery.zen.UnicastHostsProvider.HostsResolver; import org.elasticsearch.env.NodeEnvironment; @@ -1001,14 +1000,6 @@ public void testClusterCannotFormWithFailingJoinValidation() { assertTrue(cluster.clusterNodes.stream().allMatch(cn -> cn.getLastAppliedClusterState().version() == 0)); } - public void testRejectsSettingsWithMinimumMasterNodes() { - final IllegalArgumentException illegalArgumentException = expectThrows(IllegalArgumentException.class, () -> new Coordinator("node", - Settings.builder().put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), 1).build(), - null, null, null, null, null, null, null, null, null, null)); - assertThat(illegalArgumentException.getMessage(), equalTo( - "node setting [discovery.zen.minimum_master_nodes] is not permitted and must be removed before starting this node")); - } - private static long defaultMillis(Setting setting) { return setting.get(Settings.EMPTY).millis() + Cluster.DEFAULT_DELAY_VARIABILITY; } From 05af9a7851dd038ff5c08702675811eeb75b41f8 Mon Sep 17 00:00:00 2001 From: David Turner Date: Sun, 27 Jan 2019 16:26:32 +0000 Subject: [PATCH 06/18] Gotta catch them all --- .../org/elasticsearch/test/test/InternalTestClusterTests.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java b/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java index 1a1e036e7c7ee..312baef1b72bb 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java @@ -63,7 +63,6 @@ import static org.elasticsearch.cluster.node.DiscoveryNode.Role.INGEST; import static org.elasticsearch.cluster.node.DiscoveryNode.Role.MASTER; import static org.elasticsearch.discovery.DiscoveryModule.DISCOVERY_HOSTS_PROVIDER_SETTING; -import static org.elasticsearch.discovery.zen.ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING; import static org.elasticsearch.node.Node.NODE_MASTER_SETTING; import static org.elasticsearch.node.Node.NODE_NAME_SETTING; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFileExists; @@ -173,7 +172,6 @@ public Settings nodeSettings(int nodeOrdinal) { if (autoManageMinMasterNodes == false) { assert minNumDataNodes == maxNumDataNodes; assert masterNodes == false; - settings.put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), minNumDataNodes / 2 + 1); } return settings.build(); } From b8f9e4543439755878a76d68ba855192e0b283d2 Mon Sep 17 00:00:00 2001 From: David Turner Date: Sun, 27 Jan 2019 17:05:12 +0000 Subject: [PATCH 07/18] Fix docs --- .../migration/migrate_7_0/discovery.asciidoc | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/docs/reference/migration/migrate_7_0/discovery.asciidoc b/docs/reference/migration/migrate_7_0/discovery.asciidoc index 378f4c56b74f8..193f6bdd86a6f 100644 --- a/docs/reference/migration/migrate_7_0/discovery.asciidoc +++ b/docs/reference/migration/migrate_7_0/discovery.asciidoc @@ -13,6 +13,9 @@ settings summary>> for an example, and the <> describes this setting in more detail. +The `discovery.zen.minimum_master_nodes` setting is permitted, but ignored, on +7.x nodes. + [float] ==== Removing master-eligible nodes sometimes requires voting exclusions @@ -35,12 +38,3 @@ file: - `discovery.zen.ping.unicast.hosts` - `discovery.zen.hosts_provider` - `cluster.initial_master_nodes` - -[float] -==== Minimum master nodes must not be configured - -The `discovery.zen.minimum_master_nodes` setting must be removed from the -`elasticsearch.yml` configuration file. This setting is, however, required to -be set on master-eligible nodes running Elasticsearch 6 or earlier, so if you -are upgrading a node from version 6 to version 7 you must remove this setting -after shutting the version 6 node down for the last time before the upgrade. From 8585d63ced72f8895f4aa213a65f39b37d3c811d Mon Sep 17 00:00:00 2001 From: David Turner Date: Sun, 27 Jan 2019 18:50:19 +0000 Subject: [PATCH 08/18] Always set discovery.initial_state_timeout: 0s --- .../elasticsearch/gradle/test/ClusterFormationTasks.groovy | 4 +--- qa/rolling-upgrade/build.gradle | 1 - 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy index b2d8012afcf2c..8ec13f0eebe10 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy @@ -359,9 +359,7 @@ class ClusterFormationTasks { if (node.nodeVersion.before("7.0.0") && minimumMasterNodes > 0) { esConfig['discovery.zen.minimum_master_nodes'] = minimumMasterNodes } - if (node.config.numNodes > 1) { - esConfig['discovery.initial_state_timeout'] = '0s' - } + esConfig['discovery.initial_state_timeout'] = '0s' if (esConfig.containsKey('discovery.zen.master_election.wait_for_joins_timeout') == false) { // If a node decides to become master based on partial information from the pinging, don't let it hang for 30 seconds to correct // its mistake. Instead, only wait 5s to do another round of pinging. diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle index 6894463dd6fa6..160edea6a7898 100644 --- a/qa/rolling-upgrade/build.gradle +++ b/qa/rolling-upgrade/build.gradle @@ -77,7 +77,6 @@ for (Version version : bwcVersions.wireCompatible) { dependsOn lastRunner, "${baseName}#oldClusterTestCluster#node${stopNode}.stop" clusterName = 'rolling-upgrade' otherUnicastHostAddresses = { getOtherUnicastHostAddresses() } - minimumMasterNodes = { 2 } autoSetInitialMasterNodes = false /* Override the data directory so the new node always gets the node we * just stopped's data directory. */ From 38332f4c038bc568eec08baf711f77ca14101e55 Mon Sep 17 00:00:00 2001 From: David Turner Date: Sun, 27 Jan 2019 19:34:24 +0000 Subject: [PATCH 09/18] Revert no-op changes to GatewayService --- .../main/java/org/elasticsearch/gateway/GatewayService.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/gateway/GatewayService.java b/server/src/main/java/org/elasticsearch/gateway/GatewayService.java index 9fc169a2d0d69..c71ae40235056 100644 --- a/server/src/main/java/org/elasticsearch/gateway/GatewayService.java +++ b/server/src/main/java/org/elasticsearch/gateway/GatewayService.java @@ -40,7 +40,6 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.discovery.Discovery; -import org.elasticsearch.discovery.zen.ElectMasterService; import org.elasticsearch.discovery.zen.ZenDiscovery; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.rest.RestStatus; @@ -117,7 +116,7 @@ public GatewayService(final Settings settings, final AllocationService allocatio if (RECOVER_AFTER_MASTER_NODES_SETTING.exists(settings)) { recoverAfterMasterNodes = RECOVER_AFTER_MASTER_NODES_SETTING.get(settings); } else if (discovery instanceof ZenDiscovery) { - recoverAfterMasterNodes = settings.getAsInt(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), -1); + recoverAfterMasterNodes = settings.getAsInt("discovery.zen.minimum_master_nodes", -1); } else { recoverAfterMasterNodes = -1; } From 4871b7e9ddec3784538e1c4e73477d4c2ee0de55 Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 28 Jan 2019 07:29:18 +0000 Subject: [PATCH 10/18] Remove exception and mark property as deprecated --- .../elasticsearch/cluster/coordination/Coordinator.java | 9 --------- .../elasticsearch/discovery/zen/ElectMasterService.java | 2 +- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java index 3a577eb7865f1..aabe5466d69a9 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java @@ -63,7 +63,6 @@ import org.elasticsearch.discovery.HandshakingTransportAddressConnector; import org.elasticsearch.discovery.PeerFinder; import org.elasticsearch.discovery.UnicastConfiguredHostsResolver; -import org.elasticsearch.discovery.zen.ElectMasterService; import org.elasticsearch.discovery.zen.PendingClusterStateStats; import org.elasticsearch.discovery.zen.UnicastHostsProvider; import org.elasticsearch.threadpool.ThreadPool.Names; @@ -142,14 +141,6 @@ public Coordinator(String nodeName, Settings settings, ClusterSettings clusterSe NamedWriteableRegistry namedWriteableRegistry, AllocationService allocationService, MasterService masterService, Supplier persistedStateSupplier, UnicastHostsProvider unicastHostsProvider, ClusterApplier clusterApplier, Collection> onJoinValidators, Random random) { - - if (ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.exists(settings)) { - // NOCOMMIT we are not going to do this in the finished PR, but we want to be sure that we've caught all places that still - // set this setting first - throw new IllegalArgumentException("node setting [" + ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey() - + "] is not permitted and must be removed before starting this node"); - } - this.settings = settings; this.transportService = transportService; this.masterService = masterService; diff --git a/server/src/main/java/org/elasticsearch/discovery/zen/ElectMasterService.java b/server/src/main/java/org/elasticsearch/discovery/zen/ElectMasterService.java index 87ad0a396ca76..7e0f0cfca2a99 100644 --- a/server/src/main/java/org/elasticsearch/discovery/zen/ElectMasterService.java +++ b/server/src/main/java/org/elasticsearch/discovery/zen/ElectMasterService.java @@ -42,7 +42,7 @@ public class ElectMasterService { private static final Logger logger = LogManager.getLogger(ElectMasterService.class); public static final Setting DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING = - Setting.intSetting("discovery.zen.minimum_master_nodes", -1, Property.Dynamic, Property.NodeScope); + Setting.intSetting("discovery.zen.minimum_master_nodes", -1, Property.Dynamic, Property.NodeScope, Property.Deprecated); private volatile int minimumMasterNodes; From dd30a2e9a3cfca1c629b918a1d94067aaa212dbf Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 28 Jan 2019 08:48:54 +0000 Subject: [PATCH 11/18] Validate cluster fully formed before restart, preventing re-bootstrapping --- .../org/elasticsearch/test/test/InternalTestClusterTests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java b/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java index 312baef1b72bb..ca2fe8c753e44 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java @@ -422,6 +422,7 @@ public Settings transportClientSettings() { assertTrue(rolePaths.add(path.toString())); } } + cluster.validateClusterFormed(); cluster.fullRestart(); Map> result = new HashMap<>(); From b9a64f5c9c08f0071a76536aaeea925e3f84cd81 Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 28 Jan 2019 08:55:12 +0000 Subject: [PATCH 12/18] Don't use mmn in REST test regarding settings --- .../test/cluster.put_settings/10_basic.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml index d801f3aeac89f..825bac9f91649 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml @@ -4,22 +4,22 @@ cluster.put_settings: body: transient: - discovery.zen.minimum_master_nodes: 1 + cluster.routing.allocation.enable: "none" flat_settings: true - - match: {transient: {discovery.zen.minimum_master_nodes: "1"}} + - match: {transient: {cluster.routing.allocation.enable: "none"}} - do: cluster.get_settings: flat_settings: true - - match: {transient: {discovery.zen.minimum_master_nodes: "1"}} + - match: {transient: {cluster.routing.allocation.enable: "none"}} - do: cluster.put_settings: body: transient: - discovery.zen.minimum_master_nodes: null + cluster.routing.allocation.enable: null flat_settings: true - match: {transient: {}} @@ -35,22 +35,22 @@ cluster.put_settings: body: persistent: - cluster.routing.allocation.disk.threshold_enabled: false + cluster.routing.allocation.enable: "none" flat_settings: true - - match: {persistent: {cluster.routing.allocation.disk.threshold_enabled: "false"}} + - match: {persistent: {cluster.routing.allocation.enable: "none"}} - do: cluster.get_settings: flat_settings: true - - match: {persistent: {cluster.routing.allocation.disk.threshold_enabled: "false"}} + - match: {persistent: {cluster.routing.allocation.enable: "none"}} - do: cluster.put_settings: body: persistent: - cluster.routing.allocation.disk.threshold_enabled: null + cluster.routing.allocation.enable: null flat_settings: true - match: {persistent: {}} From 79f7f4166ac0a047aaa87986bf61a38b15b1d558 Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 28 Jan 2019 08:56:53 +0000 Subject: [PATCH 13/18] Remove MMN from comment --- .../java/org/elasticsearch/discovery/MasterDisruptionIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/discovery/MasterDisruptionIT.java b/server/src/test/java/org/elasticsearch/discovery/MasterDisruptionIT.java index a14c06fd49fb9..718904eecb5bb 100644 --- a/server/src/test/java/org/elasticsearch/discovery/MasterDisruptionIT.java +++ b/server/src/test/java/org/elasticsearch/discovery/MasterDisruptionIT.java @@ -148,7 +148,7 @@ public void testNodesFDAfterMasterReelection() throws Exception { */ @TestLogging("_root:DEBUG,org.elasticsearch.cluster.service:TRACE,org.elasticsearch.test.disruption:TRACE") public void testStaleMasterNotHijackingMajority() throws Exception { - // 3 node cluster with unicast discovery and minimum_master_nodes set to the default of 2: + // 3 node cluster with unicast discovery: final List nodes = startCluster(3); // Save the current master node as old master node, because that node will get frozen From c2674ab34d18d5908f7e0f12b7dbf50fd145b32a Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 28 Jan 2019 12:05:43 +0000 Subject: [PATCH 14/18] Expect warnings in tests --- .../elasticsearch/discovery/zen/ZenDiscoveryUnitTests.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/server/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryUnitTests.java b/server/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryUnitTests.java index c9a2f7dc58388..084ba62c4792d 100644 --- a/server/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryUnitTests.java +++ b/server/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryUnitTests.java @@ -268,6 +268,9 @@ public void testNodesUpdatedAfterClusterStatePublished() throws Exception { IOUtils.close(toClose); terminate(threadPool); } + + assertWarnings("[discovery.zen.minimum_master_nodes] setting was deprecated in Elasticsearch and will be removed in a future " + + "release! See the breaking changes documentation for the next major version."); } public void testPendingCSQueueIsClearedWhenClusterStatePublished() throws Exception { @@ -318,6 +321,9 @@ public void testPendingCSQueueIsClearedWhenClusterStatePublished() throws Except IOUtils.close(toClose); terminate(threadPool); } + + assertWarnings("[discovery.zen.minimum_master_nodes] setting was deprecated in Elasticsearch and will be removed in a future " + + "release! See the breaking changes documentation for the next major version."); } private class AwaitingPublishListener implements ActionListener { From 6f5bb4345a2f2da5ebfbda4410697d223a3023a2 Mon Sep 17 00:00:00 2001 From: David Turner Date: Tue, 29 Jan 2019 15:18:58 +0000 Subject: [PATCH 15/18] Bring comment back --- .../elasticsearch/gradle/test/ClusterFormationTasks.groovy | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy index 8ec13f0eebe10..16808c3bb983d 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy @@ -353,13 +353,14 @@ class ClusterFormationTasks { 'path.repo' : "${node.sharedDir}/repo", 'path.shared_data' : "${node.sharedDir}/", // Define a node attribute so we can test that it exists - 'node.attr.testattr' : 'test' + 'node.attr.testattr' : 'test', + // Don't wait for state, just start up quickly. This will also allow new and old nodes in the BWC case to become the master + 'discovery.initial_state_timeout' : '0s' ] int minimumMasterNodes = node.config.minimumMasterNodes.call() if (node.nodeVersion.before("7.0.0") && minimumMasterNodes > 0) { esConfig['discovery.zen.minimum_master_nodes'] = minimumMasterNodes } - esConfig['discovery.initial_state_timeout'] = '0s' if (esConfig.containsKey('discovery.zen.master_election.wait_for_joins_timeout') == false) { // If a node decides to become master based on partial information from the pinging, don't let it hang for 30 seconds to correct // its mistake. Instead, only wait 5s to do another round of pinging. From 7c8a540335a79cc8bad59c9621e37d904b1f336e Mon Sep 17 00:00:00 2001 From: David Turner Date: Tue, 29 Jan 2019 15:19:52 +0000 Subject: [PATCH 16/18] Do not set wait_for_joins_timeout on 7.x nodes --- .../org/elasticsearch/gradle/test/ClusterFormationTasks.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy index 16808c3bb983d..0091053271a55 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy @@ -361,7 +361,7 @@ class ClusterFormationTasks { if (node.nodeVersion.before("7.0.0") && minimumMasterNodes > 0) { esConfig['discovery.zen.minimum_master_nodes'] = minimumMasterNodes } - if (esConfig.containsKey('discovery.zen.master_election.wait_for_joins_timeout') == false) { + if (node.nodeVersion.before("7.0.0") && esConfig.containsKey('discovery.zen.master_election.wait_for_joins_timeout') == false) { // If a node decides to become master based on partial information from the pinging, don't let it hang for 30 seconds to correct // its mistake. Instead, only wait 5s to do another round of pinging. // This is necessary since we use 30s as the default timeout in REST requests waiting for cluster formation From 9137ad2869a04795f1e40240f2164acdcf1163cd Mon Sep 17 00:00:00 2001 From: David Turner Date: Tue, 29 Jan 2019 15:23:36 +0000 Subject: [PATCH 17/18] Use a more specific constant instead of Integer.MAX_VALUE --- .../java/org/elasticsearch/cluster/coordination/Zen1IT.java | 5 +++-- .../java/org/elasticsearch/test/InternalTestCluster.java | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/Zen1IT.java b/server/src/test/java/org/elasticsearch/cluster/coordination/Zen1IT.java index 1940bd0fbe2ce..e8cd691129745 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/Zen1IT.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/Zen1IT.java @@ -51,6 +51,7 @@ import static org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING; import static org.elasticsearch.cluster.routing.allocation.decider.FilterAllocationDecider.CLUSTER_ROUTING_EXCLUDE_GROUP_SETTING; import static org.elasticsearch.node.Node.NODE_NAME_SETTING; +import static org.elasticsearch.test.InternalTestCluster.REMOVED_MINIMUM_MASTER_NODES; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; @@ -196,7 +197,7 @@ public Settings onNodeStopped(String nodeName) { ClusterHealthResponse clusterHealthResponse = clusterHealthRequestBuilder.get(); assertFalse(nodeName, clusterHealthResponse.isTimedOut()); return Coordinator.addZen1Attribute(false, Settings.builder().put(ZEN2_SETTINGS) - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE)).build(); + .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), REMOVED_MINIMUM_MASTER_NODES)).build(); } }); @@ -291,7 +292,7 @@ public Settings onNodeStopped(String nodeName) throws Exception { return Coordinator.addZen1Attribute(false, Settings.builder()) .put(ZEN2_SETTINGS) .putList(INITIAL_MASTER_NODES_SETTING.getKey(), nodeNames) - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) + .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), REMOVED_MINIMUM_MASTER_NODES) .build(); } }); diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java index c9b652e69f5f0..9313d9389d49c 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java @@ -835,6 +835,8 @@ public synchronized void close() { } } + public static final int REMOVED_MINIMUM_MASTER_NODES = Integer.MAX_VALUE; + private final class NodeAndClient implements Closeable { private MockNode node; private final Settings originalNodeSettings; @@ -989,10 +991,10 @@ private void recreateNode(final Settings newSettings, final Runnable onTransport } else { if (DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.exists(finalSettings)) { // simulating an upgrade from Zen1 to Zen2, but there's no way to remove a setting when restarting a node, so - // you have to set it to Integer.MAX_VALUE to indicate its removal: + // you have to set it to REMOVED_MINIMUM_MASTER_NODES (== Integer.MAX_VALUE) to indicate its removal: assertTrue(USE_ZEN2.exists(finalSettings)); assertTrue(USE_ZEN2.get(finalSettings)); - assertThat(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.get(finalSettings), equalTo(Integer.MAX_VALUE)); + assertThat(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.get(finalSettings), equalTo(REMOVED_MINIMUM_MASTER_NODES)); final Builder builder = Settings.builder().put(finalSettings); builder.remove(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey()); From 140dfcefea07a10a0fc0f9571161c66bd52e2052 Mon Sep 17 00:00:00 2001 From: David Turner Date: Wed, 30 Jan 2019 08:20:32 +0000 Subject: [PATCH 18/18] Reducing the recovery timeout changes the expected log message --- .../elasticsearch/qa/custom_logging/CustomLoggingConfigIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/logging-config/src/test/java/org/elasticsearch/qa/custom_logging/CustomLoggingConfigIT.java b/qa/logging-config/src/test/java/org/elasticsearch/qa/custom_logging/CustomLoggingConfigIT.java index 407d23de99769..2e6b32d79fc41 100644 --- a/qa/logging-config/src/test/java/org/elasticsearch/qa/custom_logging/CustomLoggingConfigIT.java +++ b/qa/logging-config/src/test/java/org/elasticsearch/qa/custom_logging/CustomLoggingConfigIT.java @@ -40,7 +40,7 @@ * The intention is to confirm that users can still run their Elasticsearch instances with previous configurations. */ public class CustomLoggingConfigIT extends ESRestTestCase { - private static final String NODE_STARTED = ".*node-0.*cluster.uuid.*node.id.*started.*"; + private static final String NODE_STARTED = ".*node-0.*cluster.uuid.*node.id.*recovered.*cluster_state.*"; public void testSuccessfulStartupWithCustomConfig() throws Exception { assertBusy(() -> {