From 30c83baf19d4eed0a253b4127ad4663ae0adfdb1 Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Mon, 4 May 2020 23:48:09 +0200 Subject: [PATCH 1/5] prefer_v2_templates for reindex --- .../index/reindex/Reindexer.java | 2 ++ .../index/reindex/RestReindexAction.java | 5 ++++ .../resources/rest-api-spec/api/reindex.json | 4 ++++ .../index/reindex/ReindexRequest.java | 24 +++++++++++++++++++ 4 files changed, 35 insertions(+) diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/Reindexer.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/Reindexer.java index 39879845bdca1..8b4c20116a2d1 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/Reindexer.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/Reindexer.java @@ -221,6 +221,8 @@ public BiFunction, ScrollableHitSource.Hit, RequestWrapper> protected RequestWrapper buildRequest(ScrollableHitSource.Hit doc) { IndexRequest index = new IndexRequest(); + index.preferV2Templates(mainRequest.preferV2Templates()); + // Copy the index from the request so we always write where it asked to write index.index(mainRequest.getDestination().index()); diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java index 63a67b904efaa..54ef411a60469 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java @@ -68,6 +68,11 @@ protected ReindexRequest buildRequest(RestRequest request) throws IOException { if (request.hasParam("scroll")) { internal.setScroll(parseTimeValue(request.param("scroll"), "scroll")); } + + if (request.hasParam("prefer_v2_templates")) { + internal.preferV2Templates(Boolean.parseBoolean(request.param("prefer_v2_templates"))); + } + return internal; } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/reindex.json b/rest-api-spec/src/main/resources/rest-api-spec/api/reindex.json index 2fbaf86cab616..101eed5ec0bfd 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/reindex.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/reindex.json @@ -52,6 +52,10 @@ "max_docs":{ "type":"number", "description":"Maximum number of documents to process (default: all documents)" + }, + "prefer_v2_templates": { + "type": "boolean", + "description": "favor V2 templates instead of V1 templates during index creation" } }, "body":{ diff --git a/server/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java b/server/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java index 9ed9396942a7b..bf310004fa65d 100644 --- a/server/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java +++ b/server/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java @@ -19,10 +19,13 @@ package org.elasticsearch.index.reindex; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.CompositeIndicesRequest; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; @@ -69,6 +72,8 @@ public class ReindexRequest extends AbstractBulkIndexByScrollRequest Date: Mon, 4 May 2020 23:55:11 +0200 Subject: [PATCH 2/5] hlrc --- .../java/org/elasticsearch/client/RequestConverters.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java index ccbe33f4b2250..374bf14cbf0b9 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java @@ -577,6 +577,11 @@ private static Request prepareReindexRequest(ReindexRequest reindexRequest, bool if (reindexRequest.getScrollTime() != null) { params.putParam("scroll", reindexRequest.getScrollTime()); } + + if (reindexRequest.preferV2Templates() != null) { + params.putParam("prefer_v2_templates", reindexRequest.preferV2Templates().toString()); + } + request.addParameters(params.asMap()); request.setEntity(createEntity(reindexRequest, REQUEST_BODY_CONTENT_TYPE)); return request; From f3d11049e296243a1a1b4f5e3854fed891795ae9 Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Wed, 6 May 2020 00:07:36 +0200 Subject: [PATCH 3/5] tests --- .../org/elasticsearch/client/ReindexIT.java | 47 +++++++++++++++++++ .../AbstractAsyncBulkByScrollAction.java | 2 +- .../index/reindex/Reindexer.java | 6 +++ .../index/reindex/ReindexRequest.java | 3 -- 4 files changed, 54 insertions(+), 4 deletions(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ReindexIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ReindexIT.java index 1c33a7e183e5a..bbbb91c232414 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ReindexIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ReindexIT.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; import org.elasticsearch.action.admin.cluster.node.tasks.list.TaskGroup; +import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.get.GetRequest; @@ -30,7 +31,12 @@ import org.elasticsearch.action.ingest.PutPipelineRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.client.indices.PutIndexTemplateRequest; +import org.elasticsearch.client.indices.PutIndexTemplateV2Request; import org.elasticsearch.client.tasks.TaskSubmissionResponse; +import org.elasticsearch.cluster.metadata.AliasMetadata; +import org.elasticsearch.cluster.metadata.IndexTemplateV2; +import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.CheckedRunnable; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; @@ -50,6 +56,9 @@ import java.io.IOException; import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -113,6 +122,44 @@ public void testReindex() throws IOException { } } + public void testReindexPreferV2Templates() throws Exception { + //we don't care about warnings here + RequestOptions options = RequestOptions.DEFAULT.toBuilder().setWarningsHandler(warnings -> false).build(); + + IndexRequest indexRequest = new IndexRequest("sourcev2").source(Collections.singletonMap("foo", "bar"), XContentType.JSON); + indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + RestHighLevelClient client = highLevelClient(); + assertEquals(RestStatus.CREATED, client.index(indexRequest, options).status()); + + PutIndexTemplateRequest putIndexTemplateRequest = new PutIndexTemplateRequest("v1", Collections.singletonList("target*")); + assertTrue(client.indices().putTemplate(putIndexTemplateRequest, options).isAcknowledged()); + + AliasMetadata alias = AliasMetadata.builder("alias").build(); + Template template = new Template(null, null, Map.of("alias", alias)); + List pattern = List.of("target*"); + IndexTemplateV2 indexTemplate = new IndexTemplateV2(pattern, template, Collections.emptyList(), 1L, 1L, new HashMap<>()); + PutIndexTemplateV2Request putTemplateRequest = new PutIndexTemplateV2Request().name("v2").indexTemplate(indexTemplate); + assertTrue(client.indices().putIndexTemplate(putTemplateRequest, options).isAcknowledged()); + + ReindexRequest reindexRequest = new ReindexRequest() + .preferV2Templates(true) + .setSourceIndices("sourcev2") + .setDestIndex("target1") + .setRefresh(true); + assertEquals(1, client.reindex(reindexRequest, options).getStatus().getSuccessfullyProcessed()); + + GetAliasesResponse aliases = client.indices().getAlias(new GetAliasesRequest().indices("target1"), options); + assertEquals(RestStatus.OK, aliases.status()); + assertEquals(Collections.singletonMap("target1", Collections.singleton(alias)), aliases.getAliases()); + + reindexRequest.setDestIndex("target2").preferV2Templates(false); + assertEquals(1, client.reindex(reindexRequest, options).getStatus().getSuccessfullyProcessed()); + + aliases = client.indices().getAlias(new GetAliasesRequest().indices("target2"), options); + assertEquals(RestStatus.OK, aliases.status()); + assertEquals(Collections.singletonMap("target2", Collections.emptySet()), aliases.getAliases()); + } + public void testReindexTask() throws Exception { final String sourceIndex = "source123"; final String destinationIndex = "dest2"; diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java index d5afc38e702d5..cebb909dbdfac 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java @@ -199,7 +199,7 @@ protected boolean accept(ScrollableHitSource.Hit doc) { return true; } - private BulkRequest buildBulk(Iterable docs) { + protected BulkRequest buildBulk(Iterable docs) { BulkRequest bulkRequest = new BulkRequest(); for (ScrollableHitSource.Hit doc : docs) { if (accept(doc)) { diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/Reindexer.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/Reindexer.java index 8b4c20116a2d1..edf37b6853eee 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/Reindexer.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/Reindexer.java @@ -32,6 +32,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.bulk.BackoffPolicy; import org.elasticsearch.action.bulk.BulkItemResponse; +import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.client.Client; import org.elasticsearch.client.ParentTaskAssigningClient; @@ -217,6 +218,11 @@ public BiFunction, ScrollableHitSource.Hit, RequestWrapper> return super.buildScriptApplier(); } + @Override + protected BulkRequest buildBulk(Iterable docs) { + return super.buildBulk(docs).preferV2Templates(mainRequest.preferV2Templates()); + } + @Override protected RequestWrapper buildRequest(ScrollableHitSource.Hit doc) { IndexRequest index = new IndexRequest(); diff --git a/server/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java b/server/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java index bf310004fa65d..8d9f72529e1f5 100644 --- a/server/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java +++ b/server/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java @@ -336,9 +336,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (isAbortOnVersionConflict() == false) { builder.field("conflicts", "proceed"); } - if(preferV2Templates() != null){ - builder.field("prefer_v2_templates", preferV2Templates()); - } } builder.endObject(); return builder; From 44009a13cebc19618b8b6d2944d909a4b0f28158 Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Wed, 6 May 2020 00:08:35 +0200 Subject: [PATCH 4/5] fix for backport --- .../src/test/java/org/elasticsearch/client/ReindexIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ReindexIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ReindexIT.java index bbbb91c232414..2d76234c05b0d 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ReindexIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ReindexIT.java @@ -136,7 +136,7 @@ public void testReindexPreferV2Templates() throws Exception { AliasMetadata alias = AliasMetadata.builder("alias").build(); Template template = new Template(null, null, Map.of("alias", alias)); - List pattern = List.of("target*"); + List pattern = Collections.singletonList("target*"); IndexTemplateV2 indexTemplate = new IndexTemplateV2(pattern, template, Collections.emptyList(), 1L, 1L, new HashMap<>()); PutIndexTemplateV2Request putTemplateRequest = new PutIndexTemplateV2Request().name("v2").indexTemplate(indexTemplate); assertTrue(client.indices().putIndexTemplate(putTemplateRequest, options).isAcknowledged()); From 026d0b23bd39dc963cbe05249aba1f280c1b43cf Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Wed, 6 May 2020 17:57:36 +0200 Subject: [PATCH 5/5] bwc version fix --- .../main/java/org/elasticsearch/index/reindex/Reindexer.java | 2 -- .../java/org/elasticsearch/index/reindex/ReindexRequest.java | 5 ++--- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/Reindexer.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/Reindexer.java index edf37b6853eee..976922188f4be 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/Reindexer.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/Reindexer.java @@ -227,8 +227,6 @@ protected BulkRequest buildBulk(Iterable docs protected RequestWrapper buildRequest(ScrollableHitSource.Hit doc) { IndexRequest index = new IndexRequest(); - index.preferV2Templates(mainRequest.preferV2Templates()); - // Copy the index from the request so we always write where it asked to write index.index(mainRequest.getDestination().index()); diff --git a/server/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java b/server/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java index 8d9f72529e1f5..4f333474bcdf4 100644 --- a/server/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java +++ b/server/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java @@ -22,7 +22,6 @@ import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.CompositeIndicesRequest; -import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.common.Nullable; @@ -91,7 +90,7 @@ public ReindexRequest(StreamInput in) throws IOException { super(in); destination = new IndexRequest(in); remoteInfo = in.readOptionalWriteable(RemoteInfo::new); - if (in.getVersion().onOrAfter(Version.V_7_8_0)) { + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { preferV2Templates = in.readOptionalBoolean(); } } @@ -280,7 +279,7 @@ public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); destination.writeTo(out); out.writeOptionalWriteable(remoteInfo); - if (out.getVersion().onOrAfter(Version.V_7_8_0)) { + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { out.writeOptionalBoolean(preferV2Templates); } }