Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Backport 2.x] Fix cluster chaining during bootstrap #10115

Merged
merged 1 commit into from
Sep 19, 2023
Merged

[Backport 2.x] Fix cluster chaining during bootstrap #10115

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 37 additions & 6 deletions server/src/main/java/org/opensearch/gateway/GatewayMetaState.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.BiConsumer;
Expand Down Expand Up @@ -172,11 +173,12 @@ public void start(
// If the cluster UUID loaded from local is unknown (_na_) then fetch the best state from remote
// If there is no valid state on remote, continue with initial empty state
// If there is a valid state, then restore index metadata using this state
String lastKnownClusterUUID = ClusterState.UNKNOWN_UUID;
if (ClusterState.UNKNOWN_UUID.equals(clusterState.metadata().clusterUUID())) {
String lastKnownClusterUUID = remoteClusterStateService.getLastKnownUUIDFromRemote(
lastKnownClusterUUID = remoteClusterStateService.getLastKnownUUIDFromRemote(
clusterState.getClusterName().value()
);
if (!ClusterState.UNKNOWN_UUID.equals(lastKnownClusterUUID)) {
if (ClusterState.UNKNOWN_UUID.equals(lastKnownClusterUUID) == false) {
// Load state from remote
final RemoteRestoreResult remoteRestoreResult = remoteStoreRestoreService.restore(
clusterState,
Expand All @@ -187,7 +189,7 @@ public void start(
clusterState = remoteRestoreResult.getClusterState();
}
}
remotePersistedState = new RemotePersistedState(remoteClusterStateService);
remotePersistedState = new RemotePersistedState(remoteClusterStateService, lastKnownClusterUUID);
}
persistedState = new LucenePersistedState(persistedClusterStateService, currentTerm, clusterState);
} else {
Expand Down Expand Up @@ -648,9 +650,11 @@ public static class RemotePersistedState implements PersistedState {
private ClusterState lastAcceptedState;
private ClusterMetadataManifest lastAcceptedManifest;
private final RemoteClusterStateService remoteClusterStateService;
private String previousClusterUUID;

public RemotePersistedState(final RemoteClusterStateService remoteClusterStateService) {
public RemotePersistedState(final RemoteClusterStateService remoteClusterStateService, final String previousClusterUUID) {
this.remoteClusterStateService = remoteClusterStateService;
this.previousClusterUUID = previousClusterUUID;
}

@Override
Expand All @@ -675,7 +679,26 @@ public void setLastAcceptedState(ClusterState clusterState) {
try {
final ClusterMetadataManifest manifest;
if (shouldWriteFullClusterState(clusterState)) {
manifest = remoteClusterStateService.writeFullMetadata(clusterState);
if (clusterState.metadata().clusterUUIDCommitted() == true) {
final Optional<ClusterMetadataManifest> latestManifest = remoteClusterStateService.getLatestClusterMetadataManifest(
clusterState.getClusterName().value(),
clusterState.metadata().clusterUUID()
);
if (latestManifest.isPresent()) {
// The previous UUID should not change for the current UUID. So fetching the latest manifest
// from remote store and getting the previous UUID.
previousClusterUUID = latestManifest.get().getPreviousClusterUUID();
} else {
// When the user starts the cluster with remote state disabled but later enables the remote state,
// there will not be any manifest for the current cluster UUID.
logger.error(
"Latest manifest is not present in remote store for cluster UUID: {}",
clusterState.metadata().clusterUUID()
);
previousClusterUUID = ClusterState.UNKNOWN_UUID;
}
}
manifest = remoteClusterStateService.writeFullMetadata(clusterState, previousClusterUUID);
} else {
assert verifyManifestAndClusterState(lastAcceptedManifest, lastAcceptedState) == true
: "Previous manifest and previous ClusterState are not in sync";
Expand Down Expand Up @@ -724,11 +747,19 @@ public void markLastAcceptedStateAsCommitted() {
try {
assert lastAcceptedState != null : "Last accepted state is not present";
assert lastAcceptedManifest != null : "Last accepted manifest is not present";
ClusterState clusterState = lastAcceptedState;
if (lastAcceptedState.metadata().clusterUUID().equals(Metadata.UNKNOWN_CLUSTER_UUID) == false
&& lastAcceptedState.metadata().clusterUUIDCommitted() == false) {
Metadata.Builder metadataBuilder = Metadata.builder(lastAcceptedState.metadata());
metadataBuilder.clusterUUIDCommitted(true);
clusterState = ClusterState.builder(lastAcceptedState).metadata(metadataBuilder).build();
}
final ClusterMetadataManifest committedManifest = remoteClusterStateService.markLastStateAsCommitted(
lastAcceptedState,
clusterState,
lastAcceptedManifest
);
lastAcceptedManifest = committedManifest;
lastAcceptedState = clusterState;
} catch (Exception e) {
handleExceptionOnWrite(e);
}
Expand Down
37 changes: 32 additions & 5 deletions server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ public class ClusterMetadataManifest implements Writeable, ToXContentFragment {
private static final ParseField COMMITTED_FIELD = new ParseField("committed");
private static final ParseField INDICES_FIELD = new ParseField("indices");
private static final ParseField PREVIOUS_CLUSTER_UUID = new ParseField("previous_cluster_uuid");
private static final ParseField CLUSTER_UUID_COMMITTED = new ParseField("cluster_uuid_committed");

private static long term(Object[] fields) {
return (long) fields[0];
Expand Down Expand Up @@ -79,6 +80,10 @@ private static String previousClusterUUID(Object[] fields) {
return (String) fields[8];
}

private static boolean clusterUUIDCommitted(Object[] fields) {
return (boolean) fields[9];
}

private static final ConstructingObjectParser<ClusterMetadataManifest, Void> PARSER = new ConstructingObjectParser<>(
"cluster_metadata_manifest",
fields -> new ClusterMetadataManifest(
Expand All @@ -90,7 +95,8 @@ private static String previousClusterUUID(Object[] fields) {
nodeId(fields),
committed(fields),
indices(fields),
previousClusterUUID(fields)
previousClusterUUID(fields),
clusterUUIDCommitted(fields)
)
);

Expand All @@ -108,6 +114,7 @@ private static String previousClusterUUID(Object[] fields) {
INDICES_FIELD
);
PARSER.declareString(ConstructingObjectParser.constructorArg(), PREVIOUS_CLUSTER_UUID);
PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), CLUSTER_UUID_COMMITTED);
}

private final List<UploadedIndexMetadata> indices;
Expand All @@ -119,6 +126,7 @@ private static String previousClusterUUID(Object[] fields) {
private final String nodeId;
private final boolean committed;
private final String previousClusterUUID;
private final boolean clusterUUIDCommitted;

public List<UploadedIndexMetadata> getIndices() {
return indices;
Expand Down Expand Up @@ -156,6 +164,10 @@ public String getPreviousClusterUUID() {
return previousClusterUUID;
}

public boolean isClusterUUIDCommitted() {
return clusterUUIDCommitted;
}

public ClusterMetadataManifest(
long clusterTerm,
long version,
Expand All @@ -165,7 +177,8 @@ public ClusterMetadataManifest(
String nodeId,
boolean committed,
List<UploadedIndexMetadata> indices,
String previousClusterUUID
String previousClusterUUID,
boolean clusterUUIDCommitted
) {
this.clusterTerm = clusterTerm;
this.stateVersion = version;
Expand All @@ -176,6 +189,7 @@ public ClusterMetadataManifest(
this.committed = committed;
this.indices = Collections.unmodifiableList(indices);
this.previousClusterUUID = previousClusterUUID;
this.clusterUUIDCommitted = clusterUUIDCommitted;
}

public ClusterMetadataManifest(StreamInput in) throws IOException {
Expand All @@ -188,6 +202,7 @@ public ClusterMetadataManifest(StreamInput in) throws IOException {
this.committed = in.readBoolean();
this.indices = Collections.unmodifiableList(in.readList(UploadedIndexMetadata::new));
this.previousClusterUUID = in.readString();
this.clusterUUIDCommitted = in.readBoolean();
}

public static Builder builder() {
Expand Down Expand Up @@ -215,6 +230,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
}
builder.endArray();
builder.field(PREVIOUS_CLUSTER_UUID.getPreferredName(), getPreviousClusterUUID());
builder.field(CLUSTER_UUID_COMMITTED.getPreferredName(), isClusterUUIDCommitted());
return builder;
}

Expand All @@ -229,6 +245,7 @@ public void writeTo(StreamOutput out) throws IOException {
out.writeBoolean(committed);
out.writeCollection(indices);
out.writeString(previousClusterUUID);
out.writeBoolean(clusterUUIDCommitted);
}

@Override
Expand All @@ -248,7 +265,8 @@ public boolean equals(Object o) {
&& Objects.equals(opensearchVersion, that.opensearchVersion)
&& Objects.equals(nodeId, that.nodeId)
&& Objects.equals(committed, that.committed)
&& Objects.equals(previousClusterUUID, that.previousClusterUUID);
&& Objects.equals(previousClusterUUID, that.previousClusterUUID)
&& Objects.equals(clusterUUIDCommitted, that.clusterUUIDCommitted);
}

@Override
Expand All @@ -262,7 +280,8 @@ public int hashCode() {
opensearchVersion,
nodeId,
committed,
previousClusterUUID
previousClusterUUID,
clusterUUIDCommitted
);
}

Expand Down Expand Up @@ -291,6 +310,7 @@ public static class Builder {
private String nodeId;
private String previousClusterUUID;
private boolean committed;
private boolean clusterUUIDCommitted;

public Builder indices(List<UploadedIndexMetadata> indices) {
this.indices = indices;
Expand Down Expand Up @@ -341,6 +361,11 @@ public Builder previousClusterUUID(String previousClusterUUID) {
return this;
}

public Builder clusterUUIDCommitted(boolean clusterUUIDCommitted) {
this.clusterUUIDCommitted = clusterUUIDCommitted;
return this;
}

public Builder() {
indices = new ArrayList<>();
}
Expand All @@ -355,6 +380,7 @@ public Builder(ClusterMetadataManifest manifest) {
this.committed = manifest.committed;
this.indices = new ArrayList<>(manifest.indices);
this.previousClusterUUID = manifest.previousClusterUUID;
this.clusterUUIDCommitted = manifest.clusterUUIDCommitted;
}

public ClusterMetadataManifest build() {
Expand All @@ -367,7 +393,8 @@ public ClusterMetadataManifest build() {
nodeId,
committed,
indices,
previousClusterUUID
previousClusterUUID,
clusterUUIDCommitted
);
}

Expand Down
Loading