-
Notifications
You must be signed in to change notification settings - Fork 25k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
This is related to #35975. It implements a file based restore in the CcrRepository. The restore transfers files from the leader cluster to the follower cluster. It does not implement any advanced resiliency features at the moment. Any request failure will end the restore.
- Loading branch information
1 parent
c801b89
commit 5c68338
Showing
18 changed files
with
934 additions
and
310 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
240 changes: 23 additions & 217 deletions
240
server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
Large diffs are not rendered by default.
Oops, something went wrong.
304 changes: 304 additions & 0 deletions
304
server/src/main/java/org/elasticsearch/repositories/blobstore/FileRestoreContext.java
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
129 changes: 129 additions & 0 deletions
129
...in/java/org/elasticsearch/xpack/ccr/action/repositories/GetCcrRestoreFileChunkAction.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,129 @@ | ||
| /* | ||
| * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| * or more contributor license agreements. Licensed under the Elastic License; | ||
| * you may not use this file except in compliance with the Elastic License. | ||
| */ | ||
|
|
||
| package org.elasticsearch.xpack.ccr.action.repositories; | ||
|
|
||
| import org.elasticsearch.action.Action; | ||
| import org.elasticsearch.action.ActionListener; | ||
| import org.elasticsearch.action.ActionResponse; | ||
| import org.elasticsearch.action.support.ActionFilters; | ||
| import org.elasticsearch.action.support.HandledTransportAction; | ||
| import org.elasticsearch.common.bytes.BytesReference; | ||
| import org.elasticsearch.common.bytes.ReleasablePagedBytesReference; | ||
| import org.elasticsearch.common.inject.Inject; | ||
| import org.elasticsearch.common.io.stream.StreamInput; | ||
| import org.elasticsearch.common.io.stream.StreamOutput; | ||
| import org.elasticsearch.common.io.stream.Writeable; | ||
| import org.elasticsearch.common.util.BigArrays; | ||
| import org.elasticsearch.common.util.ByteArray; | ||
| import org.elasticsearch.common.util.concurrent.AbstractRunnable; | ||
| import org.elasticsearch.tasks.Task; | ||
| import org.elasticsearch.threadpool.ThreadPool; | ||
| import org.elasticsearch.transport.TransportActionProxy; | ||
| import org.elasticsearch.transport.TransportService; | ||
| import org.elasticsearch.xpack.ccr.repository.CcrRestoreSourceService; | ||
|
|
||
| import java.io.IOException; | ||
|
|
||
| public class GetCcrRestoreFileChunkAction extends Action<GetCcrRestoreFileChunkAction.GetCcrRestoreFileChunkResponse> { | ||
|
|
||
| public static final GetCcrRestoreFileChunkAction INSTANCE = new GetCcrRestoreFileChunkAction(); | ||
| public static final String NAME = "internal:admin/ccr/restore/file_chunk/get"; | ||
|
|
||
| private GetCcrRestoreFileChunkAction() { | ||
| super(NAME); | ||
| } | ||
|
|
||
| @Override | ||
| public GetCcrRestoreFileChunkResponse newResponse() { | ||
| throw new UnsupportedOperationException(); | ||
| } | ||
|
|
||
| @Override | ||
| public Writeable.Reader<GetCcrRestoreFileChunkResponse> getResponseReader() { | ||
| return GetCcrRestoreFileChunkResponse::new; | ||
| } | ||
|
|
||
|
|
||
| public static class TransportGetCcrRestoreFileChunkAction | ||
| extends HandledTransportAction<GetCcrRestoreFileChunkRequest, GetCcrRestoreFileChunkAction.GetCcrRestoreFileChunkResponse> { | ||
|
|
||
| private final CcrRestoreSourceService restoreSourceService; | ||
| private final ThreadPool threadPool; | ||
| private final BigArrays bigArrays; | ||
|
|
||
| @Inject | ||
| public TransportGetCcrRestoreFileChunkAction(BigArrays bigArrays, TransportService transportService, ActionFilters actionFilters, | ||
| CcrRestoreSourceService restoreSourceService) { | ||
| super(NAME, transportService, actionFilters, GetCcrRestoreFileChunkRequest::new); | ||
| TransportActionProxy.registerProxyAction(transportService, NAME, GetCcrRestoreFileChunkResponse::new); | ||
| this.threadPool = transportService.getThreadPool(); | ||
| this.restoreSourceService = restoreSourceService; | ||
| this.bigArrays = bigArrays; | ||
| } | ||
|
|
||
| @Override | ||
| protected void doExecute(Task task, GetCcrRestoreFileChunkRequest request, | ||
| ActionListener<GetCcrRestoreFileChunkResponse> listener) { | ||
| threadPool.generic().execute(new AbstractRunnable() { | ||
| @Override | ||
| public void onFailure(Exception e) { | ||
| listener.onFailure(e); | ||
| } | ||
|
|
||
| @Override | ||
| protected void doRun() throws Exception { | ||
| int bytesRequested = request.getSize(); | ||
| ByteArray array = bigArrays.newByteArray(bytesRequested, false); | ||
| String fileName = request.getFileName(); | ||
| String sessionUUID = request.getSessionUUID(); | ||
| // This is currently safe to do because calling `onResponse` will serialize the bytes to the network layer data | ||
| // structure on the same thread. So the bytes will be copied before the reference is released. | ||
| try (ReleasablePagedBytesReference reference = new ReleasablePagedBytesReference(array, bytesRequested, array)) { | ||
| try (CcrRestoreSourceService.SessionReader sessionReader = restoreSourceService.getSessionReader(sessionUUID)) { | ||
| long offsetAfterRead = sessionReader.readFileBytes(fileName, reference); | ||
| long offsetBeforeRead = offsetAfterRead - reference.length(); | ||
| listener.onResponse(new GetCcrRestoreFileChunkResponse(offsetBeforeRead, reference)); | ||
| } | ||
| } | ||
| } | ||
| }); | ||
| } | ||
| } | ||
|
|
||
| public static class GetCcrRestoreFileChunkResponse extends ActionResponse { | ||
|
|
||
| private final long offset; | ||
| private final BytesReference chunk; | ||
|
|
||
| GetCcrRestoreFileChunkResponse(StreamInput streamInput) throws IOException { | ||
| super(streamInput); | ||
| offset = streamInput.readVLong(); | ||
| chunk = streamInput.readBytesReference(); | ||
| } | ||
|
|
||
| GetCcrRestoreFileChunkResponse(long offset, BytesReference chunk) { | ||
| this.offset = offset; | ||
| this.chunk = chunk; | ||
| } | ||
|
|
||
| public long getOffset() { | ||
| return offset; | ||
| } | ||
|
|
||
| public BytesReference getChunk() { | ||
| return chunk; | ||
| } | ||
|
|
||
| @Override | ||
| public void writeTo(StreamOutput out) throws IOException { | ||
| super.writeTo(out); | ||
| out.writeVLong(offset); | ||
| out.writeBytesReference(chunk); | ||
| } | ||
|
|
||
| } | ||
| } |
76 changes: 76 additions & 0 deletions
76
...n/java/org/elasticsearch/xpack/ccr/action/repositories/GetCcrRestoreFileChunkRequest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,76 @@ | ||
| /* | ||
| * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| * or more contributor license agreements. Licensed under the Elastic License; | ||
| * you may not use this file except in compliance with the Elastic License. | ||
| */ | ||
|
|
||
| package org.elasticsearch.xpack.ccr.action.repositories; | ||
|
|
||
| import org.elasticsearch.action.ActionRequest; | ||
| import org.elasticsearch.action.ActionRequestValidationException; | ||
| import org.elasticsearch.cluster.node.DiscoveryNode; | ||
| import org.elasticsearch.common.io.stream.StreamInput; | ||
| import org.elasticsearch.common.io.stream.StreamOutput; | ||
| import org.elasticsearch.transport.RemoteClusterAwareRequest; | ||
|
|
||
| import java.io.IOException; | ||
|
|
||
| public class GetCcrRestoreFileChunkRequest extends ActionRequest implements RemoteClusterAwareRequest { | ||
|
|
||
| private final DiscoveryNode node; | ||
| private final String sessionUUID; | ||
| private final String fileName; | ||
| private final int size; | ||
|
|
||
| @Override | ||
| public ActionRequestValidationException validate() { | ||
| return null; | ||
| } | ||
|
|
||
| public GetCcrRestoreFileChunkRequest(DiscoveryNode node, String sessionUUID, String fileName, int size) { | ||
| this.node = node; | ||
| this.sessionUUID = sessionUUID; | ||
| this.fileName = fileName; | ||
| this.size = size; | ||
| assert size > -1 : "The file chunk request size must be positive. Found: [" + size + "]."; | ||
| } | ||
|
|
||
| GetCcrRestoreFileChunkRequest(StreamInput in) throws IOException { | ||
| super(in); | ||
| node = null; | ||
| sessionUUID = in.readString(); | ||
| fileName = in.readString(); | ||
| size = in.readVInt(); | ||
| } | ||
|
|
||
| @Override | ||
| public void writeTo(StreamOutput out) throws IOException { | ||
| super.writeTo(out); | ||
| out.writeString(sessionUUID); | ||
| out.writeString(fileName); | ||
| out.writeVInt(size); | ||
| } | ||
|
|
||
| @Override | ||
| public void readFrom(StreamInput in) throws IOException { | ||
| throw new UnsupportedOperationException(); | ||
| } | ||
|
|
||
| String getSessionUUID() { | ||
| return sessionUUID; | ||
| } | ||
|
|
||
| String getFileName() { | ||
| return fileName; | ||
| } | ||
|
|
||
| int getSize() { | ||
| return size; | ||
| } | ||
|
|
||
| @Override | ||
| public DiscoveryNode getPreferredTargetNode() { | ||
| assert node != null : "Target node is null"; | ||
| return node; | ||
| } | ||
| } |
Oops, something went wrong.