-
Notifications
You must be signed in to change notification settings - Fork 25k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Allow Deleting Multiple Snapshots at Once #55474
Changes from all commits
540ff1d
e8bfdaa
32fb757
3067bc5
add63fd
703e727
ce32074
7ff1851
5cf930b
e7428e5
7588a93
1984cc0
e91a873
116f95f
e881459
2cc0c74
609daad
e72a50b
ffe5adb
488f4e7
92ab9d5
f192cb5
821c36a
522b6fa
d9bd055
0e07404
4d1b3c8
b4123ba
75d2eef
4f5ff30
bf170bf
8c0db75
21bc756
841e016
5cd9706
36dd5d5
3772426
4b7f76b
a85f127
e0775d4
b894d58
e6d5189
44cf432
7002a15
bda3e6e
b841528
e02710e
9d6bfb5
4cb1dd5
b9171d5
f4ca634
bf48d12
eada6f2
8440aef
151ae50
8bfb7b2
a12e520
2ebf708
0a392ec
704d984
ebf4599
b41bab3
c413dd9
d2da9bc
0efa76f
53bd7a7
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -29,10 +29,13 @@ | |
import org.elasticsearch.repositories.RepositoryData; | ||
import org.elasticsearch.repositories.RepositoryOperation; | ||
import org.elasticsearch.snapshots.Snapshot; | ||
import org.elasticsearch.snapshots.SnapshotId; | ||
import org.elasticsearch.snapshots.SnapshotsService; | ||
|
||
import java.io.IOException; | ||
import java.util.ArrayList; | ||
import java.util.Collections; | ||
import java.util.HashSet; | ||
import java.util.List; | ||
import java.util.Objects; | ||
|
||
|
@@ -140,8 +143,12 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws | |
for (Entry entry : entries) { | ||
builder.startObject(); | ||
{ | ||
builder.field("repository", entry.snapshot.getRepository()); | ||
builder.field("snapshot", entry.snapshot.getSnapshotId().getName()); | ||
builder.field("repository", entry.repository()); | ||
builder.startArray("snapshots"); | ||
for (SnapshotId snapshot : entry.snapshots) { | ||
builder.value(snapshot.getName()); | ||
} | ||
builder.endArray(); | ||
builder.humanReadableField("start_time_millis", "start_time", new TimeValue(entry.startTime)); | ||
builder.field("repository_state_id", entry.repositoryStateId); | ||
} | ||
|
@@ -155,7 +162,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws | |
public String toString() { | ||
StringBuilder builder = new StringBuilder("SnapshotDeletionsInProgress["); | ||
for (int i = 0; i < entries.size(); i++) { | ||
builder.append(entries.get(i).getSnapshot().getSnapshotId().getName()); | ||
builder.append(entries.get(i).getSnapshots()); | ||
if (i + 1 < entries.size()) { | ||
builder.append(","); | ||
} | ||
|
@@ -167,29 +174,36 @@ public String toString() { | |
* A class representing a snapshot deletion request entry in the cluster state. | ||
*/ | ||
public static final class Entry implements Writeable, RepositoryOperation { | ||
private final Snapshot snapshot; | ||
private final List<SnapshotId> snapshots; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. why did you opt for having one entry for multiple snapshots? Should we rather have one entry per snapshot that is to be deleted? If we build it that way, could it allow us to queue up multiple deletes in the future? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
I thought about that actually. What I didn't like about it, was that it introduced quite a bit of duplication in what is stored in the cluster state. We'd be adding a bunch of deletes with the same start time, repository and generation to then batch them up again when it comes to running the actual delete.
I don't think that makes things easier for queuing up deletes. My thinking is this: A delete will always move us from a set of snapshots to another set of snapshots and move the repository generation ahead by So if we were to add deletes by just adding entries, then we'll have to queue up new deletes at a generation higher than the lowest generation of existing deletes (the lowest generation is already processing in the repository so we can't add to it). There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. ok, makes sense |
||
private final String repoName; | ||
private final long startTime; | ||
private final long repositoryStateId; | ||
|
||
public Entry(Snapshot snapshot, long startTime, long repositoryStateId) { | ||
this.snapshot = snapshot; | ||
public Entry(List<SnapshotId> snapshots, String repoName, long startTime, long repositoryStateId) { | ||
this.snapshots = snapshots; | ||
assert snapshots.size() == new HashSet<>(snapshots).size() : "Duplicate snapshot ids in " + snapshots; | ||
this.repoName = repoName; | ||
this.startTime = startTime; | ||
this.repositoryStateId = repositoryStateId; | ||
assert repositoryStateId > RepositoryData.EMPTY_REPO_GEN : | ||
"Can't delete based on an empty or unknown repository generation but saw [" + repositoryStateId + "]"; | ||
} | ||
|
||
public Entry(StreamInput in) throws IOException { | ||
this.snapshot = new Snapshot(in); | ||
if (in.getVersion().onOrAfter(SnapshotsService.MULTI_DELETE_VERSION)) { | ||
this.repoName = in.readString(); | ||
this.snapshots = in.readList(SnapshotId::new); | ||
} else { | ||
final Snapshot snapshot = new Snapshot(in); | ||
this.snapshots = Collections.singletonList(snapshot.getSnapshotId()); | ||
this.repoName = snapshot.getRepository(); | ||
} | ||
this.startTime = in.readVLong(); | ||
this.repositoryStateId = in.readLong(); | ||
} | ||
|
||
/** | ||
* The snapshot to delete. | ||
*/ | ||
public Snapshot getSnapshot() { | ||
return snapshot; | ||
public List<SnapshotId> getSnapshots() { | ||
return snapshots; | ||
} | ||
|
||
/** | ||
|
@@ -208,26 +222,34 @@ public boolean equals(Object o) { | |
return false; | ||
} | ||
Entry that = (Entry) o; | ||
return snapshot.equals(that.snapshot) | ||
return repoName.equals(that.repoName) | ||
&& snapshots.equals(that.snapshots) | ||
&& startTime == that.startTime | ||
&& repositoryStateId == that.repositoryStateId; | ||
} | ||
|
||
@Override | ||
public int hashCode() { | ||
return Objects.hash(snapshot, startTime, repositoryStateId); | ||
return Objects.hash(snapshots, repoName, startTime, repositoryStateId); | ||
} | ||
|
||
@Override | ||
public void writeTo(StreamOutput out) throws IOException { | ||
snapshot.writeTo(out); | ||
if (out.getVersion().onOrAfter(SnapshotsService.MULTI_DELETE_VERSION)) { | ||
out.writeString(repoName); | ||
out.writeCollection(snapshots); | ||
} else { | ||
assert snapshots.size() == 1 : "Only single deletion allowed in mixed version cluster containing [" + out.getVersion() + | ||
"] but saw " + snapshots; | ||
new Snapshot(repoName, snapshots.get(0)).writeTo(out); | ||
} | ||
out.writeVLong(startTime); | ||
out.writeLong(repositoryStateId); | ||
} | ||
|
||
@Override | ||
public String repository() { | ||
return snapshot.getRepository(); | ||
return repoName; | ||
} | ||
|
||
@Override | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
String, String...
is really dirty I know, but it's a massive change-set to adjust this API since we use it in so many places in tests so I figured I'd keep it this way for now.