Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow Deleting Multiple Snapshots at Once #55474

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
66 commits
Select commit Hold shift + click to select a range
540ff1d
bck
original-brownbear Nov 28, 2019
e8bfdaa
Merge remote-tracking branch 'elastic/master' into multi-delete-snapshot
original-brownbear Nov 28, 2019
32fb757
Merge remote-tracking branch 'elastic/master' into multi-delete-snapshot
original-brownbear Dec 2, 2019
3067bc5
works
original-brownbear Dec 2, 2019
add63fd
Merge remote-tracking branch 'elastic/master' into multi-delete-snapshot
original-brownbear Dec 2, 2019
703e727
Merge remote-tracking branch 'elastic/master' into multi-delete-snapshot
original-brownbear Dec 3, 2019
ce32074
bck
original-brownbear Dec 3, 2019
7ff1851
bck
original-brownbear Dec 3, 2019
5cf930b
Merge remote-tracking branch 'elastic/master' into multi-delete-snapshot
original-brownbear Dec 3, 2019
e7428e5
bck
original-brownbear Dec 3, 2019
7588a93
bck
original-brownbear Dec 3, 2019
1984cc0
more multi
original-brownbear Dec 3, 2019
e91a873
works
original-brownbear Dec 3, 2019
116f95f
Merge remote-tracking branch 'elastic/master' into multi-delete-snapshot
original-brownbear Dec 3, 2019
e881459
bck
original-brownbear Dec 3, 2019
2cc0c74
Merge remote-tracking branch 'elastic/master' into multi-delete-snapshot
original-brownbear Dec 10, 2019
609daad
Merge remote-tracking branch 'elastic/master' into multi-delete-snapshot
original-brownbear Dec 10, 2019
e72a50b
Merge remote-tracking branch 'elastic/master' into multi-delete-snapshot
original-brownbear Dec 10, 2019
ffe5adb
Merge remote-tracking branch 'elastic/master' into multi-delete-snapshot
original-brownbear Dec 10, 2019
488f4e7
Merge remote-tracking branch 'elastic/master' into multi-delete-snapshot
original-brownbear Dec 12, 2019
92ab9d5
bck
original-brownbear Dec 12, 2019
f192cb5
Merge remote-tracking branch 'elastic/master' into multi-delete-snapshot
original-brownbear Dec 12, 2019
821c36a
bck
original-brownbear Dec 12, 2019
522b6fa
drop it for now
original-brownbear Dec 12, 2019
d9bd055
Merge branch 'multi-delete-snapshot' into multi-delete-snapshot-v2
original-brownbear Apr 20, 2020
0e07404
fixed
original-brownbear Apr 20, 2020
4d1b3c8
Merge remote-tracking branch 'elastic/master' into multi-delete-snaps…
original-brownbear Apr 20, 2020
b4123ba
dead code
original-brownbear Apr 20, 2020
75d2eef
mroe stuff
original-brownbear Apr 20, 2020
4f5ff30
fixes
original-brownbear Apr 20, 2020
bf170bf
nicer
original-brownbear Apr 20, 2020
8c0db75
nicer
original-brownbear Apr 20, 2020
21bc756
works better
original-brownbear Apr 20, 2020
841e016
simpler
original-brownbear Apr 20, 2020
5cd9706
nicer
original-brownbear Apr 20, 2020
36dd5d5
fixes
original-brownbear Apr 20, 2020
3772426
Merge remote-tracking branch 'elastic/master' into multi-delete-snaps…
original-brownbear Apr 20, 2020
4b7f76b
shorter diff
original-brownbear Apr 20, 2020
a85f127
revert naming stuff
original-brownbear Apr 20, 2020
e0775d4
assert
original-brownbear Apr 20, 2020
b894d58
shorter diff
original-brownbear Apr 20, 2020
e6d5189
shorter diff
original-brownbear Apr 20, 2020
44cf432
shorter diff
original-brownbear Apr 20, 2020
7002a15
shorter diff
original-brownbear Apr 20, 2020
bda3e6e
shorter diff
original-brownbear Apr 20, 2020
b841528
shorter diff
original-brownbear Apr 20, 2020
e02710e
shorter diff
original-brownbear Apr 20, 2020
9d6bfb5
shorter diff
original-brownbear Apr 20, 2020
4cb1dd5
shorter diff
original-brownbear Apr 20, 2020
b9171d5
shorter diff
original-brownbear Apr 20, 2020
f4ca634
Merge remote-tracking branch 'elastic/master' into multi-delete-snaps…
original-brownbear Apr 21, 2020
bf48d12
shorter diff
original-brownbear Apr 21, 2020
eada6f2
shorter diff
original-brownbear Apr 21, 2020
8440aef
fix docs
original-brownbear Apr 21, 2020
151ae50
docs
original-brownbear Apr 21, 2020
8bfb7b2
rearrange stuff
original-brownbear Apr 21, 2020
a12e520
shorter
original-brownbear Apr 21, 2020
2ebf708
some improvemnts
original-brownbear Apr 21, 2020
0a392ec
some improvemnts
original-brownbear Apr 21, 2020
704d984
lets keep it simple
original-brownbear Apr 21, 2020
ebf4599
Merge remote-tracking branch 'elastic/master' into multi-delete-snaps…
original-brownbear Apr 21, 2020
b41bab3
update javadoc wording
original-brownbear Apr 21, 2020
c413dd9
Merge remote-tracking branch 'elastic/master' into multi-delete-snaps…
original-brownbear Apr 21, 2020
d2da9bc
Merge remote-tracking branch 'elastic/master' into multi-delete-snaps…
original-brownbear Apr 22, 2020
0efa76f
CR comments
original-brownbear Apr 22, 2020
53bd7a7
Merge remote-tracking branch 'elastic/master' into multi-delete-snaps…
original-brownbear Apr 23, 2020
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,7 @@ static Request restoreSnapshot(RestoreSnapshotRequest restoreSnapshotRequest) th
static Request deleteSnapshot(DeleteSnapshotRequest deleteSnapshotRequest) {
String endpoint = new RequestConverters.EndpointBuilder().addPathPartAsIs("_snapshot")
.addPathPart(deleteSnapshotRequest.repository())
.addPathPart(deleteSnapshotRequest.snapshot())
.addCommaSeparatedPathParts(deleteSnapshotRequest.snapshots())
.build();
Request request = new Request(HttpDelete.METHOD_NAME, endpoint);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -269,7 +269,7 @@ public void testDeleteSnapshot() {

DeleteSnapshotRequest deleteSnapshotRequest = new DeleteSnapshotRequest();
deleteSnapshotRequest.repository(repository);
deleteSnapshotRequest.snapshot(snapshot);
deleteSnapshotRequest.snapshots(snapshot);
RequestConvertersTests.setRandomMasterTimeout(deleteSnapshotRequest, expectedParams);

Request request = SnapshotRequestConverters.deleteSnapshot(deleteSnapshotRequest);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -753,7 +753,7 @@ public void testSnapshotDeleteSnapshot() throws IOException {

// tag::delete-snapshot-request
DeleteSnapshotRequest request = new DeleteSnapshotRequest(repositoryName);
request.snapshot(snapshotName);
request.snapshots(snapshotName);
// end::delete-snapshot-request

// tag::delete-snapshot-request-masterTimeout
Expand Down
9 changes: 9 additions & 0 deletions docs/reference/snapshot-restore/take-snapshot.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,15 @@ created the snapshotting process will be aborted and all files created as part o
cleaned. Therefore, the delete snapshot operation can be used to cancel long running snapshot operations that were
started by mistake.

It is also possible to delete multiple snapshots from a repository in one go, for example:

[source,console]
-----------------------------------
DELETE /_snapshot/my_backup/my_backup,my_fs_backup
DELETE /_snapshot/my_backup/snap*
-----------------------------------
// TEST[skip:no my_fs_backup]

A repository can be unregistered using the following command:

[source,console]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@
import org.elasticsearch.threadpool.Scheduler;
import org.elasticsearch.threadpool.ThreadPool;

import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
Expand Down Expand Up @@ -250,12 +251,12 @@ public void finalizeSnapshot(SnapshotId snapshotId, ShardGenerations shardGenera
}

@Override
public void deleteSnapshot(SnapshotId snapshotId, long repositoryStateId, Version repositoryMetaVersion,
ActionListener<Void> listener) {
public void deleteSnapshots(Collection<SnapshotId> snapshotIds, long repositoryStateId, Version repositoryMetaVersion,
ActionListener<Void> listener) {
if (SnapshotsService.useShardGenerations(repositoryMetaVersion) == false) {
listener = delayedListener(listener);
}
super.deleteSnapshot(snapshotId, repositoryStateId, repositoryMetaVersion, listener);
super.deleteSnapshots(snapshotIds, repositoryStateId, repositoryMetaVersion, listener);
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import org.elasticsearch.action.support.master.MasterNodeRequest;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.snapshots.SnapshotsService;

import java.io.IOException;

Expand All @@ -31,15 +32,14 @@
/**
* Delete snapshot request
* <p>
* Delete snapshot request removes the snapshot record from the repository and cleans up all
* files that are associated with this particular snapshot. All files that are shared with
* at least one other existing snapshot are left intact.
* Delete snapshot request removes snapshots from the repository and cleans up all files that are associated with the snapshots.
* All files that are shared with at least one other existing snapshot are left intact.
*/
public class DeleteSnapshotRequest extends MasterNodeRequest<DeleteSnapshotRequest> {

private String repository;

private String snapshot;
private String[] snapshots;

/**
* Constructs a new delete snapshots request
Expand All @@ -48,14 +48,14 @@ public DeleteSnapshotRequest() {
}

/**
* Constructs a new delete snapshots request with repository and snapshot name
* Constructs a new delete snapshots request with repository and snapshot names
*
* @param repository repository name
* @param snapshot snapshot name
* @param snapshots snapshot names
*/
public DeleteSnapshotRequest(String repository, String snapshot) {
public DeleteSnapshotRequest(String repository, String... snapshots) {
this.repository = repository;
this.snapshot = snapshot;
this.snapshots = snapshots;
}

/**
Expand All @@ -70,14 +70,26 @@ public DeleteSnapshotRequest(String repository) {
public DeleteSnapshotRequest(StreamInput in) throws IOException {
super(in);
repository = in.readString();
snapshot = in.readString();
if (in.getVersion().onOrAfter(SnapshotsService.MULTI_DELETE_VERSION)) {
snapshots = in.readStringArray();
} else {
snapshots = new String[] {in.readString()};
}
}

@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
out.writeString(repository);
out.writeString(snapshot);
if (out.getVersion().onOrAfter(SnapshotsService.MULTI_DELETE_VERSION)) {
out.writeStringArray(snapshots);
} else {
if (snapshots.length != 1) {
throw new IllegalArgumentException(
"Can't write snapshot delete with more than one snapshot to version [" + out.getVersion() + "]");
}
out.writeString(snapshots[0]);
}
}

@Override
Expand All @@ -86,8 +98,8 @@ public ActionRequestValidationException validate() {
if (repository == null) {
validationException = addValidationError("repository is missing", validationException);
}
if (snapshot == null) {
validationException = addValidationError("snapshot is missing", validationException);
if (snapshots == null || snapshots.length == 0) {
validationException = addValidationError("snapshots are missing", validationException);
}
return validationException;
}
Expand All @@ -108,21 +120,21 @@ public String repository() {
}

/**
* Returns repository name
* Returns snapshot names
*
* @return repository name
* @return snapshot names
*/
public String snapshot() {
return this.snapshot;
public String[] snapshots() {
return this.snapshots;
}

/**
* Sets snapshot name
* Sets snapshot names
*
* @return this request
*/
public DeleteSnapshotRequest snapshot(String snapshot) {
this.snapshot = snapshot;
public DeleteSnapshotRequest snapshots(String... snapshots) {
this.snapshots = snapshots;
return this;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,8 @@ public DeleteSnapshotRequestBuilder(ElasticsearchClient client, DeleteSnapshotAc
/**
* Constructs delete snapshot request builder with specified repository and snapshot names
*/
public DeleteSnapshotRequestBuilder(ElasticsearchClient client, DeleteSnapshotAction action, String repository, String snapshot) {
super(client, action, new DeleteSnapshotRequest(repository, snapshot));
public DeleteSnapshotRequestBuilder(ElasticsearchClient client, DeleteSnapshotAction action, String repository, String... snapshots) {
super(client, action, new DeleteSnapshotRequest(repository, snapshots));
}

/**
Expand All @@ -57,11 +57,11 @@ public DeleteSnapshotRequestBuilder setRepository(String repository) {
/**
* Sets the snapshot name
*
* @param snapshot snapshot name
* @param snapshots snapshot names
* @return this builder
*/
public DeleteSnapshotRequestBuilder setSnapshot(String snapshot) {
request.snapshot(snapshot);
public DeleteSnapshotRequestBuilder setSnapshots(String... snapshots) {
request.snapshots(snapshots);
return this;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import org.elasticsearch.transport.TransportService;

import java.io.IOException;
import java.util.Arrays;

/**
* Transport action for delete snapshot operation
Expand Down Expand Up @@ -71,7 +72,7 @@ protected ClusterBlockException checkBlock(DeleteSnapshotRequest request, Cluste
@Override
protected void masterOperation(Task task, final DeleteSnapshotRequest request, ClusterState state,
final ActionListener<AcknowledgedResponse> listener) {
snapshotsService.deleteSnapshot(request.repository(), request.snapshot(),
snapshotsService.deleteSnapshots(request.repository(), Arrays.asList(request.snapshots()),
ActionListener.map(listener, v -> new AcknowledgedResponse(true)));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -529,7 +529,7 @@ public interface ClusterAdminClient extends ElasticsearchClient {
/**
* Delete snapshot.
*/
DeleteSnapshotRequestBuilder prepareDeleteSnapshot(String repository, String snapshot);
DeleteSnapshotRequestBuilder prepareDeleteSnapshot(String repository, String... snapshot);
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

String, String... is really dirty I know, but it's a massive change-set to adjust this API since we use it in so many places in tests so I figured I'd keep it this way for now.


/**
* Restores a snapshot.
Expand Down
8 changes: 4 additions & 4 deletions server/src/main/java/org/elasticsearch/client/Requests.java
Original file line number Diff line number Diff line change
Expand Up @@ -502,14 +502,14 @@ public static RestoreSnapshotRequest restoreSnapshotRequest(String repository, S
}

/**
* Deletes a snapshot
* Deletes snapshots
*
* @param snapshot snapshot name
* @param snapshots snapshot names
* @param repository repository name
* @return delete snapshot request
*/
public static DeleteSnapshotRequest deleteSnapshotRequest(String repository, String snapshot) {
return new DeleteSnapshotRequest(repository, snapshot);
public static DeleteSnapshotRequest deleteSnapshotRequest(String repository, String... snapshots) {
return new DeleteSnapshotRequest(repository, snapshots);
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -952,8 +952,8 @@ public void deleteSnapshot(DeleteSnapshotRequest request, ActionListener<Acknowl
}

@Override
public DeleteSnapshotRequestBuilder prepareDeleteSnapshot(String repository, String name) {
return new DeleteSnapshotRequestBuilder(this, DeleteSnapshotAction.INSTANCE, repository, name);
public DeleteSnapshotRequestBuilder prepareDeleteSnapshot(String repository, String... names) {
return new DeleteSnapshotRequestBuilder(this, DeleteSnapshotAction.INSTANCE, repository, names);
}


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,13 @@
import org.elasticsearch.repositories.RepositoryData;
import org.elasticsearch.repositories.RepositoryOperation;
import org.elasticsearch.snapshots.Snapshot;
import org.elasticsearch.snapshots.SnapshotId;
import org.elasticsearch.snapshots.SnapshotsService;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;

Expand Down Expand Up @@ -140,8 +143,12 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
for (Entry entry : entries) {
builder.startObject();
{
builder.field("repository", entry.snapshot.getRepository());
builder.field("snapshot", entry.snapshot.getSnapshotId().getName());
builder.field("repository", entry.repository());
builder.startArray("snapshots");
for (SnapshotId snapshot : entry.snapshots) {
builder.value(snapshot.getName());
}
builder.endArray();
builder.humanReadableField("start_time_millis", "start_time", new TimeValue(entry.startTime));
builder.field("repository_state_id", entry.repositoryStateId);
}
Expand All @@ -155,7 +162,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
public String toString() {
StringBuilder builder = new StringBuilder("SnapshotDeletionsInProgress[");
for (int i = 0; i < entries.size(); i++) {
builder.append(entries.get(i).getSnapshot().getSnapshotId().getName());
builder.append(entries.get(i).getSnapshots());
if (i + 1 < entries.size()) {
builder.append(",");
}
Expand All @@ -167,29 +174,36 @@ public String toString() {
* A class representing a snapshot deletion request entry in the cluster state.
*/
public static final class Entry implements Writeable, RepositoryOperation {
private final Snapshot snapshot;
private final List<SnapshotId> snapshots;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why did you opt for having one entry for multiple snapshots? Should we rather have one entry per snapshot that is to be deleted?

If we build it that way, could it allow us to queue up multiple deletes in the future?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why did you opt for having one entry for multiple snapshots?

I thought about that actually. What I didn't like about it, was that it introduced quite a bit of duplication in what is stored in the cluster state. We'd be adding a bunch of deletes with the same start time, repository and generation to then batch them up again when it comes to running the actual delete.

If we build it that way, could it allow us to queue up multiple deletes in the future?

I don't think that makes things easier for queuing up deletes. My thinking is this:

A delete will always move us from a set of snapshots to another set of snapshots and move the repository generation ahead by 1. It has a defined point in time where it starts (delete entry is processed in the cluster state) and a defined repository state from which it starts. We cannot add to the delete while it is running against the repo as that's an atomic operation also.

So if we were to add deletes by just adding entries, then we'll have to queue up new deletes at a generation higher than the lowest generation of existing deletes (the lowest generation is already processing in the repository so we can't add to it).
=> I figured I'd rather be explicit about how deletes get batched together than do some implicit magic around the repository generations here.
With the way it is now, we can queue up deletes very nicely IMO: If there is only a single delete entry in the CS, we add a new entry because that one is already processing on the repo. If there are multiple delete entries we can add the additional snapshots we want to delete to the second one (that we know isn't processing yet).

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok, makes sense

private final String repoName;
private final long startTime;
private final long repositoryStateId;

public Entry(Snapshot snapshot, long startTime, long repositoryStateId) {
this.snapshot = snapshot;
public Entry(List<SnapshotId> snapshots, String repoName, long startTime, long repositoryStateId) {
this.snapshots = snapshots;
assert snapshots.size() == new HashSet<>(snapshots).size() : "Duplicate snapshot ids in " + snapshots;
this.repoName = repoName;
this.startTime = startTime;
this.repositoryStateId = repositoryStateId;
assert repositoryStateId > RepositoryData.EMPTY_REPO_GEN :
"Can't delete based on an empty or unknown repository generation but saw [" + repositoryStateId + "]";
}

public Entry(StreamInput in) throws IOException {
this.snapshot = new Snapshot(in);
if (in.getVersion().onOrAfter(SnapshotsService.MULTI_DELETE_VERSION)) {
this.repoName = in.readString();
this.snapshots = in.readList(SnapshotId::new);
} else {
final Snapshot snapshot = new Snapshot(in);
this.snapshots = Collections.singletonList(snapshot.getSnapshotId());
this.repoName = snapshot.getRepository();
}
this.startTime = in.readVLong();
this.repositoryStateId = in.readLong();
}

/**
* The snapshot to delete.
*/
public Snapshot getSnapshot() {
return snapshot;
public List<SnapshotId> getSnapshots() {
return snapshots;
}

/**
Expand All @@ -208,26 +222,34 @@ public boolean equals(Object o) {
return false;
}
Entry that = (Entry) o;
return snapshot.equals(that.snapshot)
return repoName.equals(that.repoName)
&& snapshots.equals(that.snapshots)
&& startTime == that.startTime
&& repositoryStateId == that.repositoryStateId;
}

@Override
public int hashCode() {
return Objects.hash(snapshot, startTime, repositoryStateId);
return Objects.hash(snapshots, repoName, startTime, repositoryStateId);
}

@Override
public void writeTo(StreamOutput out) throws IOException {
snapshot.writeTo(out);
if (out.getVersion().onOrAfter(SnapshotsService.MULTI_DELETE_VERSION)) {
out.writeString(repoName);
out.writeCollection(snapshots);
} else {
assert snapshots.size() == 1 : "Only single deletion allowed in mixed version cluster containing [" + out.getVersion() +
"] but saw " + snapshots;
new Snapshot(repoName, snapshots.get(0)).writeTo(out);
}
out.writeVLong(startTime);
out.writeLong(repositoryStateId);
}

@Override
public String repository() {
return snapshot.getRepository();
return repoName;
}

@Override
Expand Down
Loading