From 91127415ea665febf0484e3e98764501e949de32 Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Tue, 22 Jan 2019 16:38:06 +0100 Subject: [PATCH 01/14] Enforce cluster uuid --- .../coordination/CoordinationState.java | 16 +++++- .../cluster/coordination/Coordinator.java | 14 ++++- .../cluster/coordination/JoinHelper.java | 10 +++- .../cluster/metadata/MetaData.java | 43 ++++++++++++++-- .../coordination/CoordinatorTests.java | 51 +++++++++++++++++-- .../cluster/coordination/JoinHelperTests.java | 2 +- 6 files changed, 122 insertions(+), 14 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationState.java b/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationState.java index 4d542566ccd70..c6891821dc99e 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationState.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationState.java @@ -475,12 +475,24 @@ public interface PersistedState { */ default void markLastAcceptedConfigAsCommitted() { final ClusterState lastAcceptedState = getLastAcceptedState(); + MetaData.Builder metaDataBuilder = null; if (lastAcceptedState.getLastAcceptedConfiguration().equals(lastAcceptedState.getLastCommittedConfiguration()) == false) { final CoordinationMetaData coordinationMetaData = CoordinationMetaData.builder(lastAcceptedState.coordinationMetaData()) .lastCommittedConfiguration(lastAcceptedState.getLastAcceptedConfiguration()) .build(); - final MetaData metaData = MetaData.builder(lastAcceptedState.metaData()).coordinationMetaData(coordinationMetaData).build(); - setLastAcceptedState(ClusterState.builder(lastAcceptedState).metaData(metaData).build()); + if (metaDataBuilder == null) { + metaDataBuilder = MetaData.builder(lastAcceptedState.metaData()); + } + metaDataBuilder.coordinationMetaData(coordinationMetaData); + } + if (lastAcceptedState.metaData().clusterUUIDCommitted() == false) { + if (metaDataBuilder == null) { + metaDataBuilder = MetaData.builder(lastAcceptedState.metaData()); + } + metaDataBuilder.clusterUUIDCommitted(true); + } + if (metaDataBuilder != null) { + setLastAcceptedState(ClusterState.builder(lastAcceptedState).metaData(metaDataBuilder).build()); } } } diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java index 4a018c1f78f91..72ae51d90c8a8 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java @@ -146,7 +146,7 @@ public Coordinator(String nodeName, Settings settings, ClusterSettings clusterSe this.masterService = masterService; this.onJoinValidators = JoinTaskExecutor.addBuiltInJoinValidators(onJoinValidators); this.joinHelper = new JoinHelper(settings, allocationService, masterService, transportService, - this::getCurrentTerm, this::handleJoinRequest, this::joinLeaderInTerm, this.onJoinValidators); + this::getCurrentTerm, this::getStateForMasterService, this::handleJoinRequest, this::joinLeaderInTerm, this.onJoinValidators); this.persistedStateSupplier = persistedStateSupplier; this.discoverySettings = new DiscoverySettings(settings, clusterSettings); this.lastKnownLeader = Optional.empty(); @@ -279,7 +279,16 @@ PublishWithJoinResponse handlePublishRequest(PublishRequest publishRequest) { + lastKnownLeader + ", rejecting"); } - if (publishRequest.getAcceptedState().term() > coordinationState.get().getLastAcceptedState().term()) { + final ClusterState localState = coordinationState.get().getLastAcceptedState(); + + if (localState.metaData().clusterUUIDCommitted() && + localState.metaData().clusterUUID().equals(publishRequest.getAcceptedState().metaData().clusterUUID()) == false) { + throw new CoordinationStateRejectedException("received cluster state from " + sourceNode + + " with a different cluster uuid " + publishRequest.getAcceptedState().metaData().clusterUUID() + + " than local cluster uuid " + localState.metaData().clusterUUID() + ", rejecting"); + } + + if (publishRequest.getAcceptedState().term() > localState.term()) { // only do join validation if we have not accepted state from this master yet onJoinValidators.forEach(a -> a.accept(getLocalNode(), publishRequest.getAcceptedState())); } @@ -621,6 +630,7 @@ public void invariant() { assert followersChecker.getFastResponseState().term == getCurrentTerm() : followersChecker.getFastResponseState(); assert followersChecker.getFastResponseState().mode == getMode() : followersChecker.getFastResponseState(); assert (applierState.nodes().getMasterNodeId() == null) == applierState.blocks().hasGlobalBlockWithId(NO_MASTER_BLOCK_ID); + assert applierState.nodes().getMasterNodeId() == null || applierState.metaData().clusterUUIDCommitted(); assert preVoteCollector.getPreVoteResponse().equals(getPreVoteResponse()) : preVoteCollector + " vs " + getPreVoteResponse(); diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java index 8c41d7b2eaa52..6dfbdf2baeca3 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java @@ -62,6 +62,7 @@ import java.util.function.BiConsumer; import java.util.function.Function; import java.util.function.LongSupplier; +import java.util.function.Supplier; public class JoinHelper { @@ -84,7 +85,7 @@ public class JoinHelper { final Set> pendingOutgoingJoins = ConcurrentCollections.newConcurrentSet(); public JoinHelper(Settings settings, AllocationService allocationService, MasterService masterService, - TransportService transportService, LongSupplier currentTermSupplier, + TransportService transportService, LongSupplier currentTermSupplier, Supplier currentStateSupplier, BiConsumer joinHandler, Function joinLeaderInTerm, Collection> joinValidators) { this.masterService = masterService; @@ -132,6 +133,13 @@ public ClusterTasksResult execute(ClusterState currentSta transportService.registerRequestHandler(VALIDATE_JOIN_ACTION_NAME, MembershipAction.ValidateJoinRequest::new, ThreadPool.Names.GENERIC, (request, channel, task) -> { + final ClusterState localState = currentStateSupplier.get(); + if (localState.metaData().clusterUUIDCommitted() && + localState.metaData().clusterUUID().equals(request.getState().metaData().clusterUUID()) == false) { + throw new CoordinationStateRejectedException("join validation on cluster state" + + " with a different cluster uuid " + request.getState().metaData().clusterUUID() + + " than local cluster uuid " + localState.metaData().clusterUUID() + ", rejecting"); + } joinValidators.forEach(action -> action.accept(transportService.getLocalNode(), request.getState())); channel.sendResponse(Empty.INSTANCE); }); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java index 3cce3f791d2b8..61c886e0194ca 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java @@ -88,6 +88,7 @@ public class MetaData implements Iterable, Diffable, To private static final Logger logger = LogManager.getLogger(MetaData.class); public static final String ALL = "_all"; + public static final String UNKNOWN_CLUSTER_UUID = "_na_"; public enum XContentContext { /* Custom metadata should be returns as part of API call */ @@ -159,6 +160,7 @@ public interface Custom extends NamedDiffable, ToXContentFragment, Clust private static final NamedDiffableValueSerializer CUSTOM_VALUE_SERIALIZER = new NamedDiffableValueSerializer<>(Custom.class); private final String clusterUUID; + private final boolean clusterUUIDCommitted; private final long version; private final CoordinationMetaData coordinationMetaData; @@ -179,12 +181,13 @@ public interface Custom extends NamedDiffable, ToXContentFragment, Clust private final SortedMap aliasAndIndexLookup; - MetaData(String clusterUUID, long version, CoordinationMetaData coordinationMetaData, + MetaData(String clusterUUID, boolean clusterUUIDCommitted, long version, CoordinationMetaData coordinationMetaData, Settings transientSettings, Settings persistentSettings, ImmutableOpenMap indices, ImmutableOpenMap templates, ImmutableOpenMap customs, String[] allIndices, String[] allOpenIndices, String[] allClosedIndices, SortedMap aliasAndIndexLookup) { this.clusterUUID = clusterUUID; + this.clusterUUIDCommitted = clusterUUIDCommitted; this.version = version; this.coordinationMetaData = coordinationMetaData; this.transientSettings = transientSettings; @@ -218,6 +221,10 @@ public String clusterUUID() { return this.clusterUUID; } + public boolean clusterUUIDCommitted() { + return this.clusterUUIDCommitted; + } + /** * Returns the merged transient and persistent settings. */ @@ -798,6 +805,7 @@ private static class MetaDataDiff implements Diff { private long version; private String clusterUUID; + private boolean clusterUUIDCommitted; private CoordinationMetaData coordinationMetaData; private Settings transientSettings; private Settings persistentSettings; @@ -807,6 +815,7 @@ private static class MetaDataDiff implements Diff { MetaDataDiff(MetaData before, MetaData after) { clusterUUID = after.clusterUUID; + clusterUUIDCommitted = after.clusterUUIDCommitted; version = after.version; coordinationMetaData = after.coordinationMetaData; transientSettings = after.transientSettings; @@ -818,6 +827,9 @@ private static class MetaDataDiff implements Diff { MetaDataDiff(StreamInput in) throws IOException { clusterUUID = in.readString(); + if (in.getVersion().onOrAfter(Version.V_7_0_0)) { + clusterUUIDCommitted = in.readBoolean(); + } version = in.readLong(); if (in.getVersion().onOrAfter(Version.V_7_0_0)) { //TODO revisit after Zen2 BWC is implemented coordinationMetaData = new CoordinationMetaData(in); @@ -836,6 +848,9 @@ private static class MetaDataDiff implements Diff { @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(clusterUUID); + if (out.getVersion().onOrAfter(Version.V_7_0_0)) { + out.writeBoolean(clusterUUIDCommitted); + } out.writeLong(version); if (out.getVersion().onOrAfter(Version.V_7_0_0)) { coordinationMetaData.writeTo(out); @@ -851,6 +866,7 @@ public void writeTo(StreamOutput out) throws IOException { public MetaData apply(MetaData part) { Builder builder = builder(); builder.clusterUUID(clusterUUID); + builder.clusterUUIDCommitted(clusterUUIDCommitted); builder.version(version); builder.coordinationMetaData(coordinationMetaData); builder.transientSettings(transientSettings); @@ -866,6 +882,9 @@ public static MetaData readFrom(StreamInput in) throws IOException { Builder builder = new Builder(); builder.version = in.readLong(); builder.clusterUUID = in.readString(); + if (in.getVersion().onOrAfter(Version.V_7_0_0)) { + builder.clusterUUIDCommitted = in.readBoolean(); + } if (in.getVersion().onOrAfter(Version.V_7_0_0)) { builder.coordinationMetaData(new CoordinationMetaData(in)); } @@ -891,6 +910,9 @@ public static MetaData readFrom(StreamInput in) throws IOException { public void writeTo(StreamOutput out) throws IOException { out.writeLong(version); out.writeString(clusterUUID); + if (out.getVersion().onOrAfter(Version.V_7_0_0)) { + out.writeBoolean(clusterUUIDCommitted); + } if (out.getVersion().onOrAfter(Version.V_7_0_0)) { coordinationMetaData.writeTo(out); } @@ -930,6 +952,7 @@ public static Builder builder(MetaData metaData) { public static class Builder { private String clusterUUID; + private boolean clusterUUIDCommitted; private long version; private CoordinationMetaData coordinationMetaData = CoordinationMetaData.EMPTY_META_DATA; @@ -941,7 +964,7 @@ public static class Builder { private final ImmutableOpenMap.Builder customs; public Builder() { - clusterUUID = "_na_"; + clusterUUID = UNKNOWN_CLUSTER_UUID; indices = ImmutableOpenMap.builder(); templates = ImmutableOpenMap.builder(); customs = ImmutableOpenMap.builder(); @@ -950,6 +973,7 @@ public Builder() { public Builder(MetaData metaData) { this.clusterUUID = metaData.clusterUUID; + this.clusterUUIDCommitted = metaData.clusterUUIDCommitted; this.coordinationMetaData = metaData.coordinationMetaData; this.transientSettings = metaData.transientSettings; this.persistentSettings = metaData.persistentSettings; @@ -1125,8 +1149,13 @@ public Builder clusterUUID(String clusterUUID) { return this; } + public Builder clusterUUIDCommitted(boolean clusterUUIDCommitted) { + this.clusterUUIDCommitted = clusterUUIDCommitted; + return this; + } + public Builder generateClusterUuidIfNeeded() { - if (clusterUUID.equals("_na_")) { + if (clusterUUID.equals(UNKNOWN_CLUSTER_UUID)) { clusterUUID = UUIDs.randomBase64UUID(); } return this; @@ -1182,8 +1211,9 @@ public MetaData build() { String[] allOpenIndicesArray = allOpenIndices.toArray(new String[allOpenIndices.size()]); String[] allClosedIndicesArray = allClosedIndices.toArray(new String[allClosedIndices.size()]); - return new MetaData(clusterUUID, version, coordinationMetaData, transientSettings, persistentSettings, indices.build(), - templates.build(), customs.build(), allIndicesArray, allOpenIndicesArray, allClosedIndicesArray, aliasAndIndexLookup); + return new MetaData(clusterUUID, clusterUUIDCommitted, version, coordinationMetaData, transientSettings, persistentSettings, + indices.build(), templates.build(), customs.build(), allIndicesArray, allOpenIndicesArray, allClosedIndicesArray, + aliasAndIndexLookup); } private SortedMap buildAliasAndIndexLookup() { @@ -1226,6 +1256,7 @@ public static void toXContent(MetaData metaData, XContentBuilder builder, ToXCon builder.field("version", metaData.version()); builder.field("cluster_uuid", metaData.clusterUUID); + builder.field("cluster_uuid_committed", metaData.clusterUUIDCommitted); builder.startObject("cluster_coordination"); metaData.coordinationMetaData().toXContent(builder, params); @@ -1324,6 +1355,8 @@ public static MetaData fromXContent(XContentParser parser) throws IOException { builder.version = parser.longValue(); } else if ("cluster_uuid".equals(currentFieldName) || "uuid".equals(currentFieldName)) { builder.clusterUUID = parser.text(); + } else if ("cluster_uuid_committed".equals(currentFieldName)) { + builder.clusterUUIDCommitted = parser.booleanValue(); } else { throw new IllegalArgumentException("Unexpected field [" + currentFieldName + "]"); } diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java index 7db63ab120e91..9a04eb228c356 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java @@ -20,6 +20,7 @@ import com.carrotsearch.randomizedtesting.RandomizedContext; import org.apache.logging.log4j.CloseableThreadContext; +import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; @@ -46,6 +47,7 @@ import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; @@ -59,6 +61,7 @@ import org.elasticsearch.gateway.MockGatewayMetaState; import org.elasticsearch.indices.cluster.FakeThreadPoolMasterService; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockLogAppender; import org.elasticsearch.test.disruption.DisruptableMockTransport; import org.elasticsearch.test.disruption.DisruptableMockTransport.ConnectionStatus; import org.elasticsearch.transport.TransportService; @@ -80,6 +83,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.Callable; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiConsumer; import java.util.function.Consumer; import java.util.function.Function; @@ -133,6 +137,13 @@ public class CoordinatorTests extends ESTestCase { private final List nodeEnvironments = new ArrayList<>(); + private final AtomicInteger nextNodeIndex = new AtomicInteger(); + + @Before + public void resetNodeIndexBeforeEachTest() { + nextNodeIndex.set(0); + } + @After public void closeNodeEnvironmentsAfterEachTest() { for (NodeEnvironment nodeEnvironment : nodeEnvironments) { @@ -997,6 +1008,40 @@ public void testClusterCannotFormWithFailingJoinValidation() { assertTrue(cluster.clusterNodes.stream().allMatch(cn -> cn.getLastAppliedClusterState().version() == 0)); } + public void testCannotJoinClusterWithDifferentUUID() throws IllegalAccessException { + final Cluster cluster1 = new Cluster(randomIntBetween(1, 3)); + cluster1.runRandomly(); + cluster1.stabilise(); + + final Cluster cluster2 = new Cluster(3); + cluster2.runRandomly(); + cluster2.stabilise(); + + final ClusterNode shiftedNode = randomFrom(cluster2.clusterNodes).restartedNode(); + final ClusterNode newNode = cluster1.new ClusterNode(nextNodeIndex.getAndIncrement(), + shiftedNode.getLocalNode(), n -> shiftedNode.persistedState); + cluster1.clusterNodes.add(newNode); + + MockLogAppender mockAppender = new MockLogAppender(); + mockAppender.start(); + mockAppender.addExpectation( + new MockLogAppender.SeenEventExpectation( + "test1", + JoinHelper.class.getCanonicalName(), + Level.INFO, + "*failed to join*")); + Logger joinLogger = LogManager.getLogger(JoinHelper.class); + Loggers.addAppender(joinLogger, mockAppender); + cluster1.runFor(10000, "failing join validation"); + try { + mockAppender.assertAllExpectationsMatched(); + } finally { + Loggers.removeAppender(joinLogger, mockAppender); + mockAppender.stop(); + } + assertTrue(newNode.getLastAppliedClusterState().version() == 0); + } + private static long defaultMillis(Setting setting) { return setting.get(Settings.EMPTY).millis() + Cluster.DEFAULT_DELAY_VARIABILITY; } @@ -1073,7 +1118,8 @@ class Cluster { final Set masterEligibleNodeIds = new HashSet<>(initialNodeCount); clusterNodes = new ArrayList<>(initialNodeCount); for (int i = 0; i < initialNodeCount; i++) { - final ClusterNode clusterNode = new ClusterNode(i, allNodesMasterEligible || i == 0 || randomBoolean()); + final ClusterNode clusterNode = new ClusterNode(nextNodeIndex.getAndIncrement(), + allNodesMasterEligible || i == 0 || randomBoolean()); clusterNodes.add(clusterNode); if (clusterNode.getLocalNode().isMasterNode()) { masterEligibleNodeIds.add(clusterNode.getId()); @@ -1104,10 +1150,9 @@ List addNodesAndStabilise(int newNodesCount) { List addNodes(int newNodesCount) { logger.info("--> adding {} nodes", newNodesCount); - final int nodeSizeAtStart = clusterNodes.size(); final List addedNodes = new ArrayList<>(); for (int i = 0; i < newNodesCount; i++) { - final ClusterNode clusterNode = new ClusterNode(nodeSizeAtStart + i, true); + final ClusterNode clusterNode = new ClusterNode(nextNodeIndex.getAndIncrement(), true); addedNodes.add(clusterNode); } clusterNodes.addAll(addedNodes); diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/JoinHelperTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/JoinHelperTests.java index ef843717fb469..4361660876c7a 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/JoinHelperTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/JoinHelperTests.java @@ -43,7 +43,7 @@ public void testJoinDeduplication() { TransportService transportService = capturingTransport.createTransportService(Settings.EMPTY, deterministicTaskQueue.getThreadPool(), TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> localNode, null, Collections.emptySet()); - JoinHelper joinHelper = new JoinHelper(Settings.EMPTY, null, null, transportService, () -> 0L, + JoinHelper joinHelper = new JoinHelper(Settings.EMPTY, null, null, transportService, () -> 0L, () -> null, (joinRequest, joinCallback) -> { throw new AssertionError(); }, startJoinRequest -> { throw new AssertionError(); }, Collections.emptyList()); transportService.start(); From afa6c1b2d468b52d759276e1e916ed424ba8485a Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Thu, 24 Jan 2019 14:05:22 +0100 Subject: [PATCH 02/14] more tests --- .../cluster/metadata/MetaData.java | 10 +++++ .../cluster/metadata/MetaDataTests.java | 37 +++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java index 61c886e0194ca..e58cc3d144e49 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java @@ -221,6 +221,10 @@ public String clusterUUID() { return this.clusterUUID; } + /** + * Whether the current node with the given cluster state is locked into the cluster with the UUID returned by {@link #clusterUUID()}, + * meaning that it will not accept any cluster state with a different clusterUUID. + */ public boolean clusterUUIDCommitted() { return this.clusterUUIDCommitted; } @@ -764,6 +768,12 @@ public static boolean isGlobalStateEquals(MetaData metaData1, MetaData metaData2 if (!metaData1.templates.equals(metaData2.templates())) { return false; } + if (!metaData1.clusterUUID.equals(metaData2.clusterUUID)) { + return false; + } + if (metaData1.clusterUUIDCommitted != metaData2.clusterUUIDCommitted) { + return false; + } // Check if any persistent metadata needs to be saved int customCount1 = 0; for (ObjectObjectCursor cursor : metaData1.customs) { diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java index 5dcfccaea5874..685b7cca98a94 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java @@ -411,6 +411,43 @@ public void testXContentWithIndexGraveyard() throws IOException { } } + public void testXContentClusterUUID() throws IOException { + final MetaData originalMeta = MetaData.builder().clusterUUID(UUIDs.randomBase64UUID()) + .clusterUUIDCommitted(randomBoolean()).build(); + final XContentBuilder builder = JsonXContent.contentBuilder(); + builder.startObject(); + originalMeta.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) { + final MetaData fromXContentMeta = MetaData.fromXContent(parser); + assertThat(fromXContentMeta.clusterUUID(), equalTo(originalMeta.clusterUUID())); + assertThat(fromXContentMeta.clusterUUIDCommitted(), equalTo(originalMeta.clusterUUIDCommitted())); + } + } + + public void testSerializationClusterUUID() throws IOException { + final MetaData originalMeta = MetaData.builder().clusterUUID(UUIDs.randomBase64UUID()) + .clusterUUIDCommitted(randomBoolean()).build(); + final BytesStreamOutput out = new BytesStreamOutput(); + originalMeta.writeTo(out); + NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(ClusterModule.getNamedWriteables()); + final MetaData fromStreamMeta = MetaData.readFrom( + new NamedWriteableAwareStreamInput(out.bytes().streamInput(), namedWriteableRegistry) + ); + assertThat(fromStreamMeta.clusterUUID(), equalTo(originalMeta.clusterUUID())); + assertThat(fromStreamMeta.clusterUUIDCommitted(), equalTo(originalMeta.clusterUUIDCommitted())); + } + + public void testMetaDataGlobalStateChangesOnClusterUUIDChanges() { + final MetaData metaData1 = MetaData.builder().clusterUUID(UUIDs.randomBase64UUID()).clusterUUIDCommitted(randomBoolean()).build(); + final MetaData metaData2 = MetaData.builder(metaData1).clusterUUID(UUIDs.randomBase64UUID()).build(); + final MetaData metaData3 = MetaData.builder(metaData1).clusterUUIDCommitted(!metaData1.clusterUUIDCommitted()).build(); + assertFalse(MetaData.isGlobalStateEquals(metaData1, metaData2)); + assertFalse(MetaData.isGlobalStateEquals(metaData1, metaData3)); + final MetaData metaData4 = MetaData.builder(metaData2).clusterUUID(metaData1.clusterUUID()).build(); + assertTrue(MetaData.isGlobalStateEquals(metaData1, metaData4)); + } + private static CoordinationMetaData.VotingConfiguration randomVotingConfig() { return new CoordinationMetaData.VotingConfiguration(Sets.newHashSet(generateRandomStringArray(randomInt(10), 20, false))); } From 265eb44562cc5b3ed04af24e4f82b4215d060afd Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Thu, 24 Jan 2019 14:39:44 +0100 Subject: [PATCH 03/14] wip --- .../elasticsearch/cluster/metadata/MetaData.java | 15 ++++++++------- .../cluster/coordination/CoordinatorTests.java | 7 +++++++ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java index e58cc3d144e49..48a01b8498bbc 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java @@ -39,6 +39,7 @@ import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.coordination.CoordinationMetaData; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.Randomness; import org.elasticsearch.common.Strings; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.collect.HppcMaps; @@ -768,12 +769,12 @@ public static boolean isGlobalStateEquals(MetaData metaData1, MetaData metaData2 if (!metaData1.templates.equals(metaData2.templates())) { return false; } - if (!metaData1.clusterUUID.equals(metaData2.clusterUUID)) { - return false; - } - if (metaData1.clusterUUIDCommitted != metaData2.clusterUUIDCommitted) { - return false; - } +// if (!metaData1.clusterUUID.equals(metaData2.clusterUUID)) { +// return false; +// } +// if (metaData1.clusterUUIDCommitted != metaData2.clusterUUIDCommitted) { +// return false; +// } // Check if any persistent metadata needs to be saved int customCount1 = 0; for (ObjectObjectCursor cursor : metaData1.customs) { @@ -1166,7 +1167,7 @@ public Builder clusterUUIDCommitted(boolean clusterUUIDCommitted) { public Builder generateClusterUuidIfNeeded() { if (clusterUUID.equals(UNKNOWN_CLUSTER_UUID)) { - clusterUUID = UUIDs.randomBase64UUID(); + clusterUUID = UUIDs.randomBase64UUID(Randomness.createSecure()); } return this; } diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java index 9a04eb228c356..4fbc656aa0e21 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java @@ -1040,6 +1040,9 @@ public void testCannotJoinClusterWithDifferentUUID() throws IllegalAccessExcepti mockAppender.stop(); } assertTrue(newNode.getLastAppliedClusterState().version() == 0); + + // reset clusterUUIDCommitted to let node join again + //clusterNodes.replaceAll(cn -> cn == clusterNode ? cn.restartedNode() : cn); } private static long defaultMillis(Setting setting) { @@ -1644,6 +1647,10 @@ void close() { } ClusterNode restartedNode() { + return restartedNode(Function.identity()); + } + + ClusterNode restartedNode(Function adaptClusterState) { final TransportAddress address = randomBoolean() ? buildNewFakeTransportAddress() : localNode.getAddress(); final DiscoveryNode newLocalNode = new DiscoveryNode(localNode.getName(), localNode.getId(), UUIDs.randomBase64UUID(random()), // generated deterministically for repeatable tests From 7f8f97f247e953a077ad6e6625d4e23acfa3bb37 Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Thu, 24 Jan 2019 15:24:05 +0100 Subject: [PATCH 04/14] seems to work now --- .../cluster/metadata/MetaData.java | 15 +++--- .../coordination/CoordinatorTests.java | 50 +++++++++++++++---- 2 files changed, 48 insertions(+), 17 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java index 48a01b8498bbc..e58cc3d144e49 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java @@ -39,7 +39,6 @@ import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.coordination.CoordinationMetaData; import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.Randomness; import org.elasticsearch.common.Strings; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.collect.HppcMaps; @@ -769,12 +768,12 @@ public static boolean isGlobalStateEquals(MetaData metaData1, MetaData metaData2 if (!metaData1.templates.equals(metaData2.templates())) { return false; } -// if (!metaData1.clusterUUID.equals(metaData2.clusterUUID)) { -// return false; -// } -// if (metaData1.clusterUUIDCommitted != metaData2.clusterUUIDCommitted) { -// return false; -// } + if (!metaData1.clusterUUID.equals(metaData2.clusterUUID)) { + return false; + } + if (metaData1.clusterUUIDCommitted != metaData2.clusterUUIDCommitted) { + return false; + } // Check if any persistent metadata needs to be saved int customCount1 = 0; for (ObjectObjectCursor cursor : metaData1.customs) { @@ -1167,7 +1166,7 @@ public Builder clusterUUIDCommitted(boolean clusterUUIDCommitted) { public Builder generateClusterUuidIfNeeded() { if (clusterUUID.equals(UNKNOWN_CLUSTER_UUID)) { - clusterUUID = UUIDs.randomBase64UUID(Randomness.createSecure()); + clusterUUID = UUIDs.randomBase64UUID(); } return this; } diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java index 4fbc656aa0e21..fe14d71e23cfd 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java @@ -37,6 +37,7 @@ import org.elasticsearch.cluster.coordination.CoordinationState.PersistedState; import org.elasticsearch.cluster.coordination.Coordinator.Mode; import org.elasticsearch.cluster.coordination.CoordinatorTests.Cluster.ClusterNode; +import org.elasticsearch.cluster.metadata.Manifest; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNode.Role; @@ -58,6 +59,7 @@ import org.elasticsearch.discovery.zen.PublishClusterStateStats; import org.elasticsearch.discovery.zen.UnicastHostsProvider.HostsResolver; import org.elasticsearch.env.NodeEnvironment; +import org.elasticsearch.gateway.MetaStateService; import org.elasticsearch.gateway.MockGatewayMetaState; import org.elasticsearch.indices.cluster.FakeThreadPoolMasterService; import org.elasticsearch.test.ESTestCase; @@ -160,6 +162,7 @@ public void resetPortCounterBeforeEachTest() { // check that runRandomly leads to reproducible results public void testRepeatableTests() throws Exception { final Callable test = () -> { + resetNodeIndexBeforeEachTest(); final Cluster cluster = new Cluster(randomIntBetween(1, 5)); cluster.runRandomly(); final long afterRunRandomly = value(cluster.getAnyNode().getLastAppliedClusterState()); @@ -1041,8 +1044,16 @@ public void testCannotJoinClusterWithDifferentUUID() throws IllegalAccessExcepti } assertTrue(newNode.getLastAppliedClusterState().version() == 0); - // reset clusterUUIDCommitted to let node join again - //clusterNodes.replaceAll(cn -> cn == clusterNode ? cn.restartedNode() : cn); + // reset clusterUUIDCommitted (and node / cluster state term) to let node join again + final ClusterNode detachedNode = newNode.restartedNode( + metaData -> MetaData.builder(metaData) + .clusterUUIDCommitted(false) + .coordinationMetaData(CoordinationMetaData.builder(metaData.coordinationMetaData()) + .term(0L).build()) + .build(), + term -> 0L); + cluster1.clusterNodes.replaceAll(cn -> cn == newNode ? detachedNode : cn); + cluster1.stabilise(); } private static long defaultMillis(Setting setting) { @@ -1513,21 +1524,41 @@ class MockPersistedState implements PersistedState { } } - MockPersistedState(DiscoveryNode newLocalNode, MockPersistedState oldState) { + MockPersistedState(DiscoveryNode newLocalNode, MockPersistedState oldState, + Function adaptGlobalMetaData, Function adaptCurrentTerm) { try { if (oldState.nodeEnvironment != null) { nodeEnvironment = oldState.nodeEnvironment; + final MetaStateService metaStateService = new MetaStateService(nodeEnvironment, xContentRegistry()); + final MetaData updatedMetaData = adaptGlobalMetaData.apply(oldState.getLastAcceptedState().metaData()); + if (updatedMetaData != oldState.getLastAcceptedState().metaData()) { + metaStateService.writeGlobalStateAndUpdateManifest("update global state", updatedMetaData); + } + final long updatedTerm = adaptCurrentTerm.apply(oldState.getCurrentTerm()); + if (updatedTerm != oldState.getCurrentTerm()) { + final Manifest manifest = metaStateService.loadManifestOrEmpty(); + metaStateService.writeManifestAndCleanup("update term", + new Manifest(updatedTerm, manifest.getClusterStateVersion(), manifest.getGlobalGeneration(), + manifest.getIndexGenerations())); + } delegate = new MockGatewayMetaState(Settings.EMPTY, nodeEnvironment, xContentRegistry(), newLocalNode) .getPersistedState(Settings.EMPTY, null); } else { nodeEnvironment = null; BytesStreamOutput outStream = new BytesStreamOutput(); outStream.setVersion(Version.CURRENT); - oldState.getLastAcceptedState().writeTo(outStream); + final MetaData updatedMetaData = adaptGlobalMetaData.apply(oldState.getLastAcceptedState().metaData()); + final ClusterState clusterState; + if (updatedMetaData != oldState.getLastAcceptedState().metaData()) { + clusterState = ClusterState.builder(oldState.getLastAcceptedState()).metaData(updatedMetaData).build(); + } else { + clusterState = oldState.getLastAcceptedState(); + } + clusterState.writeTo(outStream); StreamInput inStream = new NamedWriteableAwareStreamInput(outStream.bytes().streamInput(), new NamedWriteableRegistry(ClusterModule.getNamedWriteables())); - delegate = new InMemoryPersistedState(oldState.getCurrentTerm(), ClusterState.readFrom(inStream, - newLocalNode)); // adapts it to new localNode instance + delegate = new InMemoryPersistedState(adaptCurrentTerm.apply(oldState.getCurrentTerm()), + ClusterState.readFrom(inStream, newLocalNode)); // adapts it to new localNode instance } } catch (IOException e) { throw new UncheckedIOException("Unable to create MockPersistedState", e); @@ -1647,16 +1678,17 @@ void close() { } ClusterNode restartedNode() { - return restartedNode(Function.identity()); + return restartedNode(Function.identity(), Function.identity()); } - ClusterNode restartedNode(Function adaptClusterState) { + ClusterNode restartedNode(Function adaptGlobalMetaData, Function adaptCurrentTerm) { final TransportAddress address = randomBoolean() ? buildNewFakeTransportAddress() : localNode.getAddress(); final DiscoveryNode newLocalNode = new DiscoveryNode(localNode.getName(), localNode.getId(), UUIDs.randomBase64UUID(random()), // generated deterministically for repeatable tests address.address().getHostString(), address.getAddress(), address, Collections.emptyMap(), localNode.isMasterNode() ? EnumSet.allOf(Role.class) : emptySet(), Version.CURRENT); - return new ClusterNode(nodeIndex, newLocalNode, node -> new MockPersistedState(newLocalNode, persistedState)); + return new ClusterNode(nodeIndex, newLocalNode, + node -> new MockPersistedState(newLocalNode, persistedState, adaptGlobalMetaData, adaptCurrentTerm)); } private PersistedState getPersistedState() { From f93e0b49a21c10d36480be6b8aff5e498db60773 Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Thu, 24 Jan 2019 15:44:25 +0100 Subject: [PATCH 05/14] add warning log --- .../org/elasticsearch/cluster/coordination/Coordinator.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java index 72ae51d90c8a8..3264e9f95ed9d 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java @@ -283,6 +283,8 @@ PublishWithJoinResponse handlePublishRequest(PublishRequest publishRequest) { if (localState.metaData().clusterUUIDCommitted() && localState.metaData().clusterUUID().equals(publishRequest.getAcceptedState().metaData().clusterUUID()) == false) { + logger.warn("received cluster state from {} with a different cluster uuid {} than local cluster uuid {}, rejecting", + sourceNode, publishRequest.getAcceptedState().metaData().clusterUUID(), localState.metaData().clusterUUID()); throw new CoordinationStateRejectedException("received cluster state from " + sourceNode + " with a different cluster uuid " + publishRequest.getAcceptedState().metaData().clusterUUID() + " than local cluster uuid " + localState.metaData().clusterUUID() + ", rejecting"); From c890ad4e9a374a64e6c33ad8e913126a6cdbdd6e Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Thu, 24 Jan 2019 15:50:09 +0100 Subject: [PATCH 06/14] fix testMixedClusterFormation --- .../elasticsearch/cluster/coordination/CoordinationState.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationState.java b/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationState.java index c6891821dc99e..9731e13a977de 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationState.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationState.java @@ -485,7 +485,8 @@ default void markLastAcceptedConfigAsCommitted() { } metaDataBuilder.coordinationMetaData(coordinationMetaData); } - if (lastAcceptedState.metaData().clusterUUIDCommitted() == false) { + if (lastAcceptedState.metaData().clusterUUID().equals(MetaData.UNKNOWN_CLUSTER_UUID) == false && + lastAcceptedState.metaData().clusterUUIDCommitted() == false) { if (metaDataBuilder == null) { metaDataBuilder = MetaData.builder(lastAcceptedState.metaData()); } From 861ad05b250e41930fdae783f1699bb1998b5e41 Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Fri, 25 Jan 2019 00:19:11 +0100 Subject: [PATCH 07/14] fix test --- .../org/elasticsearch/gateway/ClusterStateUpdatersTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/gateway/ClusterStateUpdatersTests.java b/server/src/test/java/org/elasticsearch/gateway/ClusterStateUpdatersTests.java index b34bcf87bdbd8..8affc1520b633 100644 --- a/server/src/test/java/org/elasticsearch/gateway/ClusterStateUpdatersTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/ClusterStateUpdatersTests.java @@ -268,7 +268,7 @@ public void testMixCurrentAndRecoveredState() { final ClusterState updatedState = mixCurrentStateAndRecoveredState(currentState, recoveredState); assertThat(updatedState.metaData().clusterUUID(), not(equalTo("_na_"))); - assertTrue(MetaData.isGlobalStateEquals(metaData, updatedState.metaData())); + assertFalse(MetaData.isGlobalStateEquals(metaData, updatedState.metaData())); assertThat(updatedState.metaData().index("test"), equalTo(indexMetaData)); assertTrue(updatedState.blocks().hasGlobalBlock(STATE_NOT_RECOVERED_BLOCK)); assertTrue(updatedState.blocks().hasGlobalBlock(CLUSTER_READ_ONLY_BLOCK)); From a0f3f796b01c6d9f2114e022c2d1656a81ae9f2d Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Fri, 25 Jan 2019 10:29:42 +0100 Subject: [PATCH 08/14] feedback --- .../cluster/coordination/CoordinationState.java | 11 +++++------ .../org/elasticsearch/cluster/metadata/MetaData.java | 2 +- .../gateway/GatewayMetaStatePersistedStateTests.java | 4 ++-- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationState.java b/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationState.java index 9731e13a977de..f6452bf0742aa 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationState.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationState.java @@ -422,7 +422,7 @@ public void handleCommit(ApplyCommitRequest applyCommit) { logger.trace("handleCommit: applying commit request for term [{}] and version [{}]", applyCommit.getTerm(), applyCommit.getVersion()); - persistedState.markLastAcceptedConfigAsCommitted(); + persistedState.markLastAcceptedStateAsCommitted(); assert getLastCommittedConfiguration().equals(getLastAcceptedConfiguration()); } @@ -471,18 +471,17 @@ public interface PersistedState { /** * Marks the last accepted cluster state as committed. * After a successful call to this method, {@link #getLastAcceptedState()} should return the last cluster state that was set, - * with the last committed configuration now corresponding to the last accepted configuration. + * with the last committed configuration now corresponding to the last accepted configuration, and the cluster uuid, if set, + * marked as committed. */ - default void markLastAcceptedConfigAsCommitted() { + default void markLastAcceptedStateAsCommitted() { final ClusterState lastAcceptedState = getLastAcceptedState(); MetaData.Builder metaDataBuilder = null; if (lastAcceptedState.getLastAcceptedConfiguration().equals(lastAcceptedState.getLastCommittedConfiguration()) == false) { final CoordinationMetaData coordinationMetaData = CoordinationMetaData.builder(lastAcceptedState.coordinationMetaData()) .lastCommittedConfiguration(lastAcceptedState.getLastAcceptedConfiguration()) .build(); - if (metaDataBuilder == null) { - metaDataBuilder = MetaData.builder(lastAcceptedState.metaData()); - } + metaDataBuilder = MetaData.builder(lastAcceptedState.metaData()); metaDataBuilder.coordinationMetaData(coordinationMetaData); } if (lastAcceptedState.metaData().clusterUUID().equals(MetaData.UNKNOWN_CLUSTER_UUID) == false && diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java index e58cc3d144e49..54c3001d9036f 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java @@ -841,7 +841,7 @@ private static class MetaDataDiff implements Diff { clusterUUIDCommitted = in.readBoolean(); } version = in.readLong(); - if (in.getVersion().onOrAfter(Version.V_7_0_0)) { //TODO revisit after Zen2 BWC is implemented + if (in.getVersion().onOrAfter(Version.V_7_0_0)) { coordinationMetaData = new CoordinationMetaData(in); } else { coordinationMetaData = CoordinationMetaData.EMPTY_META_DATA; diff --git a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStatePersistedStateTests.java b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStatePersistedStateTests.java index 921bcac3d4c65..d9496e2cfda46 100644 --- a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStatePersistedStateTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStatePersistedStateTests.java @@ -219,7 +219,7 @@ public void testMarkAcceptedConfigAsCommitted() throws IOException { gateway = maybeNew(gateway); assertThat(gateway.getLastAcceptedState().getLastAcceptedConfiguration(), not(equalTo(gateway.getLastAcceptedState().getLastCommittedConfiguration()))); - gateway.markLastAcceptedConfigAsCommitted(); + gateway.markLastAcceptedStateAsCommitted(); CoordinationMetaData expectedCoordinationMetaData = CoordinationMetaData.builder(coordinationMetaData) .lastCommittedConfiguration(coordinationMetaData.getLastAcceptedConfiguration()).build(); @@ -228,7 +228,7 @@ public void testMarkAcceptedConfigAsCommitted() throws IOException { gateway = maybeNew(gateway); assertClusterStateEqual(expectedClusterState, gateway.getLastAcceptedState()); - gateway.markLastAcceptedConfigAsCommitted(); + gateway.markLastAcceptedStateAsCommitted(); gateway = maybeNew(gateway); assertClusterStateEqual(expectedClusterState, gateway.getLastAcceptedState()); From 1280e22d909ac2ec2704b4688bae2dcc6687f585 Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Fri, 25 Jan 2019 10:40:55 +0100 Subject: [PATCH 09/14] add test --- .../discovery/ClusterDisruptionIT.java | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/server/src/test/java/org/elasticsearch/discovery/ClusterDisruptionIT.java b/server/src/test/java/org/elasticsearch/discovery/ClusterDisruptionIT.java index f1e78fd3c6ae6..cef2f4b88ee68 100644 --- a/server/src/test/java/org/elasticsearch/discovery/ClusterDisruptionIT.java +++ b/server/src/test/java/org/elasticsearch/discovery/ClusterDisruptionIT.java @@ -28,6 +28,7 @@ import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.action.shard.ShardStateAction; +import org.elasticsearch.cluster.coordination.ClusterBootstrapService; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.routing.Murmur3HashFunction; import org.elasticsearch.cluster.routing.ShardRouting; @@ -377,6 +378,33 @@ public boolean clearData(String nodeName) { assertTrue(client().prepareGet("index", "_doc", "1").get().isExists()); } + public void testCannotJoinIfMasterLostDataFolder() throws Exception { + String masterNode = internalCluster().startMasterOnlyNode(); + String dataNode = internalCluster().startDataOnlyNode(); + + internalCluster().restartNode(masterNode, new InternalTestCluster.RestartCallback() { + @Override + public boolean clearData(String nodeName) { + return true; + } + + @Override + public Settings onNodeStopped(String nodeName) { + return Settings.builder().put(ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING.getKey(), nodeName).build(); + } + + @Override + public boolean validateClusterForming() { + return false; + } + }); + + assertFalse(internalCluster().client(masterNode).admin().cluster().prepareHealth().get().isTimedOut()); + assertTrue(internalCluster().client(masterNode).admin().cluster().prepareHealth().setWaitForNodes("2").setTimeout("2s").get() + .isTimedOut()); + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(dataNode)); // otherwise we will fail during clean-up + } + /** * Tests that indices are properly deleted even if there is a master transition in between. * Test for https://github.com/elastic/elasticsearch/issues/11665 From 3dc2c625e53548ca689c0f69c133bde05e673be7 Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Mon, 28 Jan 2019 11:25:46 +0100 Subject: [PATCH 10/14] use constant --- .../org/elasticsearch/gateway/ClusterStateUpdatersTests.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/gateway/ClusterStateUpdatersTests.java b/server/src/test/java/org/elasticsearch/gateway/ClusterStateUpdatersTests.java index 8affc1520b633..cae33db90a6bc 100644 --- a/server/src/test/java/org/elasticsearch/gateway/ClusterStateUpdatersTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/ClusterStateUpdatersTests.java @@ -263,11 +263,11 @@ public void testMixCurrentAndRecoveredState() { .blocks(ClusterBlocks.builder().addGlobalBlock(CLUSTER_READ_ONLY_BLOCK).build()) .metaData(metaData) .build(); - assertThat(recoveredState.metaData().clusterUUID(), equalTo("_na_")); + assertThat(recoveredState.metaData().clusterUUID(), equalTo(MetaData.UNKNOWN_CLUSTER_UUID)); final ClusterState updatedState = mixCurrentStateAndRecoveredState(currentState, recoveredState); - assertThat(updatedState.metaData().clusterUUID(), not(equalTo("_na_"))); + assertThat(updatedState.metaData().clusterUUID(), not(equalTo(MetaData.UNKNOWN_CLUSTER_UUID))); assertFalse(MetaData.isGlobalStateEquals(metaData, updatedState.metaData())); assertThat(updatedState.metaData().index("test"), equalTo(indexMetaData)); assertTrue(updatedState.blocks().hasGlobalBlock(STATE_NOT_RECOVERED_BLOCK)); From 23558dff23bd47250cb62321a9029656ac327249 Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Mon, 28 Jan 2019 11:34:11 +0100 Subject: [PATCH 11/14] add assertion --- .../elasticsearch/cluster/coordination/CoordinationState.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationState.java b/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationState.java index f6452bf0742aa..c65904608abd8 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationState.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationState.java @@ -484,6 +484,10 @@ default void markLastAcceptedStateAsCommitted() { metaDataBuilder = MetaData.builder(lastAcceptedState.metaData()); metaDataBuilder.coordinationMetaData(coordinationMetaData); } + // if we receive a commit from a Zen1 master that has not recovered its state yet, the cluster uuid might not been known yet. + assert lastAcceptedState.metaData().clusterUUID().equals(MetaData.UNKNOWN_CLUSTER_UUID) == false || + lastAcceptedState.coordinationMetaData().getLastCommittedConfiguration().isEmpty() : + "received cluster state with empty cluster uuid but voting configuration: " + lastAcceptedState; if (lastAcceptedState.metaData().clusterUUID().equals(MetaData.UNKNOWN_CLUSTER_UUID) == false && lastAcceptedState.metaData().clusterUUIDCommitted() == false) { if (metaDataBuilder == null) { From b7be853c73d88b459e53eda1355fc0ec6fb2a4ac Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Mon, 28 Jan 2019 11:36:07 +0100 Subject: [PATCH 12/14] longer fail time --- .../elasticsearch/cluster/coordination/CoordinatorTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java index 8b59ebd51234c..dd637dc845d9d 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java @@ -1038,7 +1038,7 @@ public void testCannotJoinClusterWithDifferentUUID() throws IllegalAccessExcepti "*failed to join*")); Logger joinLogger = LogManager.getLogger(JoinHelper.class); Loggers.addAppender(joinLogger, mockAppender); - cluster1.runFor(10000, "failing join validation"); + cluster1.runFor(DEFAULT_STABILISATION_TIME, "failing join validation"); try { mockAppender.assertAllExpectationsMatched(); } finally { From 2697b4403c479a614357712d1e5446712e4c5718 Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Mon, 28 Jan 2019 12:58:54 +0100 Subject: [PATCH 13/14] rewrite assertion using Zen1 BWC term --- .../cluster/coordination/CoordinationState.java | 4 ++-- .../gateway/GatewayMetaStatePersistedStateTests.java | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationState.java b/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationState.java index c65904608abd8..dff6b5add0b09 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationState.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationState.java @@ -486,8 +486,8 @@ default void markLastAcceptedStateAsCommitted() { } // if we receive a commit from a Zen1 master that has not recovered its state yet, the cluster uuid might not been known yet. assert lastAcceptedState.metaData().clusterUUID().equals(MetaData.UNKNOWN_CLUSTER_UUID) == false || - lastAcceptedState.coordinationMetaData().getLastCommittedConfiguration().isEmpty() : - "received cluster state with empty cluster uuid but voting configuration: " + lastAcceptedState; + lastAcceptedState.term() == ZEN1_BWC_TERM : + "received cluster state with empty cluster uuid but not Zen1 BWC term: " + lastAcceptedState; if (lastAcceptedState.metaData().clusterUUID().equals(MetaData.UNKNOWN_CLUSTER_UUID) == false && lastAcceptedState.metaData().clusterUUIDCommitted() == false) { if (metaDataBuilder == null) { diff --git a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStatePersistedStateTests.java b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStatePersistedStateTests.java index d9496e2cfda46..8ccfa5e406ae2 100644 --- a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStatePersistedStateTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStatePersistedStateTests.java @@ -213,7 +213,8 @@ public void testMarkAcceptedConfigAsCommitted() throws IOException { } while (coordinationMetaData.getLastAcceptedConfiguration().equals(coordinationMetaData.getLastCommittedConfiguration())); ClusterState state = createClusterState(randomNonNegativeLong(), - MetaData.builder().coordinationMetaData(coordinationMetaData).build()); + MetaData.builder().coordinationMetaData(coordinationMetaData) + .clusterUUID(randomAlphaOfLength(10)).build()); gateway.setLastAcceptedState(state); gateway = maybeNew(gateway); @@ -224,7 +225,8 @@ public void testMarkAcceptedConfigAsCommitted() throws IOException { CoordinationMetaData expectedCoordinationMetaData = CoordinationMetaData.builder(coordinationMetaData) .lastCommittedConfiguration(coordinationMetaData.getLastAcceptedConfiguration()).build(); ClusterState expectedClusterState = - ClusterState.builder(state).metaData(MetaData.builder().coordinationMetaData(expectedCoordinationMetaData).build()).build(); + ClusterState.builder(state).metaData(MetaData.builder().coordinationMetaData(expectedCoordinationMetaData) + .clusterUUID(state.metaData().clusterUUID()).clusterUUIDCommitted(true).build()).build(); gateway = maybeNew(gateway); assertClusterStateEqual(expectedClusterState, gateway.getLastAcceptedState()); From c3e0aa2572b21cae9785025ef73e72934972a15a Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Tue, 29 Jan 2019 10:20:44 +0100 Subject: [PATCH 14/14] add TODO --- .../org/elasticsearch/cluster/coordination/CoordinatorTests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java index dd637dc845d9d..fa964bbe67930 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java @@ -1048,6 +1048,7 @@ public void testCannotJoinClusterWithDifferentUUID() throws IllegalAccessExcepti assertTrue(newNode.getLastAppliedClusterState().version() == 0); // reset clusterUUIDCommitted (and node / cluster state term) to let node join again + // TODO: use elasticsearch-node detach-cluster tool once it's implemented final ClusterNode detachedNode = newNode.restartedNode( metaData -> MetaData.builder(metaData) .clusterUUIDCommitted(false)