Add prefer_v2_templates flag and index setting (#55411)

This commit adds a new querystring parameter on the following APIs:
- Index
- Update
- Bulk
- Create Index
- Rollover

These APIs now support a `?prefer_v2_templates=true|false` flag. This flag changes the preference
creation to use either V2 index templates or V1 templates. This flag defaults to `false` and will be
changed to `true` for 8.0+ in subsequent work.

Additionally, setting this flag internally sets the `index.prefer_v2_templates` index-level setting.
This setting is used so that actions that automatically create a new index (things like rollover
initiated by ILM) will inherit the preference from the original index. This setting is dynamic so
that a transition from v1 to v2 templates can occur for long-running indices grouped by an alias
performing periodic rollover.

This also adds support for sending this parameter to the High Level Rest Client.

Relates to #53101

client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesRequestConverters.java

@@ -113,6 +113,9 @@ static Request createIndex(CreateIndexRequest createIndexRequest) throws IOExcep
         parameters.withTimeout(createIndexRequest.timeout());
         parameters.withMasterTimeout(createIndexRequest.masterNodeTimeout());
         parameters.withWaitForActiveShards(createIndexRequest.waitForActiveShards());
+        if (createIndexRequest.preferV2Templates() != null) {
+            parameters.putParam(IndexMetadata.PREFER_V2_TEMPLATES_FLAG, Boolean.toString(createIndexRequest.preferV2Templates()));
+        }
         request.addParameters(parameters.asMap());
         request.setEntity(RequestConverters.createEntity(createIndexRequest, RequestConverters.REQUEST_BODY_CONTENT_TYPE));
         return request;

client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java

@@ -56,6 +56,7 @@
 import org.elasticsearch.client.security.RefreshPolicy;
 import org.elasticsearch.client.tasks.TaskId;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.Strings;
@@ -131,6 +132,9 @@ static Request bulk(BulkRequest bulkRequest) throws IOException {
         parameters.withRefreshPolicy(bulkRequest.getRefreshPolicy());
         parameters.withPipeline(bulkRequest.pipeline());
         parameters.withRouting(bulkRequest.routing());
+        if (bulkRequest.preferV2Templates() != null) {
+            parameters.putParam(IndexMetadata.PREFER_V2_TEMPLATES_FLAG, Boolean.toString(bulkRequest.preferV2Templates()));
+        }
         // Bulk API only supports newline delimited JSON or Smile. Before executing
         // the bulk, we need to check that all requests have the same content-type
         // and this content-type is supported by the Bulk API.
@@ -331,6 +335,9 @@ static Request index(IndexRequest indexRequest) {
         parameters.withPipeline(indexRequest.getPipeline());
         parameters.withRefreshPolicy(indexRequest.getRefreshPolicy());
         parameters.withWaitForActiveShards(indexRequest.waitForActiveShards());
+        if (indexRequest.preferV2Templates() != null) {
+            parameters.putParam(IndexMetadata.PREFER_V2_TEMPLATES_FLAG, Boolean.toString(indexRequest.preferV2Templates()));
+        }
 
         BytesRef source = indexRequest.source().toBytesRef();
         ContentType contentType = createContentType(indexRequest.getContentType());
@@ -357,6 +364,9 @@ static Request update(UpdateRequest updateRequest) throws IOException {
         parameters.withRetryOnConflict(updateRequest.retryOnConflict());
         parameters.withVersion(updateRequest.version());
         parameters.withVersionType(updateRequest.versionType());
+        if (updateRequest.preferV2Templates() != null) {
+            parameters.putParam(IndexMetadata.PREFER_V2_TEMPLATES_FLAG, Boolean.toString(updateRequest.preferV2Templates()));
+        }
 
         // The Java API allows update requests with different content types
         // set for the partial document and the upsert document. This client

client/rest-high-level/src/main/java/org/elasticsearch/client/indices/CreateIndexRequest.java

@@ -25,6 +25,7 @@
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.client.TimedRequest;
 import org.elasticsearch.client.Validatable;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
@@ -62,6 +63,7 @@ public class CreateIndexRequest extends TimedRequest implements Validatable, ToX
 
     private BytesReference mappings;
     private XContentType mappingsXContentType;
+    private Boolean preferV2Templates;
 
     private final Set<Alias> aliases = new HashSet<>();
 
@@ -265,6 +267,16 @@ public CreateIndexRequest aliases(Collection<Alias> aliases) {
         return this;
     }
 
+    public CreateIndexRequest preferV2Templates(Boolean preferV2Templates) {
+        this.preferV2Templates = preferV2Templates;
+        return this;
+    }
+
+    @Nullable
+    public Boolean preferV2Templates() {
+        return this.preferV2Templates;
+    }
+
     /**
      * Sets the settings and mappings as a single source.
      *

docs/reference/api-conventions.asciidoc

@@ -385,6 +385,7 @@ Returns:
     "settings": {
       "index.number_of_replicas": "1",
       "index.number_of_shards": "1",
+      "index.prefer_v2_templates": "false",
       "index.creation_date": "1474389951325",
       "index.uuid": "n6gzFZTgS664GUfx0Xrpjw",
       "index.version.created": ...,
@@ -421,7 +422,8 @@ Returns:
         "version": {
           "created": ...
         },
-        "provided_name" : "twitter"
+        "provided_name" : "twitter",
+        "prefer_v2_templates": "false"
       }
     }
   }

rest-api-spec/src/main/resources/rest-api-spec/api/bulk.json

@@ -87,6 +87,10 @@
       "pipeline":{
         "type":"string",
         "description":"The pipeline id to preprocess incoming documents with"
+      },
+      "prefer_v2_templates": {
+        "type": "boolean",
+        "description": "favor V2 templates instead of V1 templates during automatic index creation"
       }
     },
     "body":{

rest-api-spec/src/main/resources/rest-api-spec/api/create.json

@@ -90,6 +90,10 @@
       "pipeline":{
         "type":"string",
         "description":"The pipeline id to preprocess incoming documents with"
+      },
+      "prefer_v2_templates": {
+        "type": "boolean",
+        "description": "favor V2 templates instead of V1 templates during automatic index creation"
       }
     },
     "body":{

rest-api-spec/src/main/resources/rest-api-spec/api/index.json

@@ -92,6 +92,10 @@
       "pipeline":{
         "type":"string",
         "description":"The pipeline id to preprocess incoming documents with"
+      },
+      "prefer_v2_templates": {
+        "type": "boolean",
+        "description": "favor V2 templates instead of V1 templates during automatic index creation"
       }
     },
     "body":{

rest-api-spec/src/main/resources/rest-api-spec/api/indices.create.json

@@ -33,6 +33,10 @@
       "master_timeout":{
         "type":"time",
         "description":"Specify timeout for connection to master"
+      },
+      "prefer_v2_templates": {
+        "type": "boolean",
+        "description": "favor V2 templates instead of V1 templates during index creation"
       }
     },
     "body":{

rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json

@@ -53,6 +53,10 @@
       "wait_for_active_shards":{
         "type":"string",
         "description":"Set the number of active shards to wait for on the newly created rollover index before the operation returns."
+      },
+      "prefer_v2_templates": {
+        "type": "boolean",
+        "description": "favor V2 templates instead of V1 templates during automatic index creation"
       }
     },
     "body":{

rest-api-spec/src/main/resources/rest-api-spec/api/update.json

@@ -99,6 +99,10 @@
       "if_primary_term":{
         "type":"number",
         "description":"only perform the update operation if the last operation that has changed the document has the specified primary term"
+      },
+      "prefer_v2_templates": {
+        "type": "boolean",
+        "description": "favor V2 templates instead of V1 templates during automatic index creation"
       }
     },
     "body":{

rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_index_template/15_composition.yml

@@ -1,7 +1,7 @@
 ---
 "Component and index template composition":
   - skip:
-      version: " - 7.7.99"
+      version: " - 7.9.99"
       reason: "index template v2 API unavailable before 7.8"
       features: allowed_warnings
 
@@ -61,6 +61,7 @@
   - do:
       indices.create:
         index: bar-baz
+        prefer_v2_templates: true
         body:
           settings:
             index.priority: 17
@@ -88,7 +89,7 @@
 ---
 "Index template priority":
   - skip:
-      version: " - 7.7.99"
+      version: " - 7.9.99"
       reason: "index template v2 API unavailable before 7.8"
       features: allowed_warnings
 
@@ -120,6 +121,7 @@
 
   - do:
       indices.create:
+        prefer_v2_templates: true
         index: bar-baz
 
   - do:
@@ -131,7 +133,7 @@
 ---
 "Component template only composition":
   - skip:
-      version: " - 7.7.99"
+      version: " - 7.9.99"
       reason: "index template v2 API unavailable before 7.8"
       features: allowed_warnings
 
@@ -164,6 +166,7 @@
 
   - do:
       indices.create:
+        prefer_v2_templates: true
         index: bazfoo
 
   - do:
@@ -176,7 +179,7 @@
 ---
 "Index template without component templates":
   - skip:
-      version: " - 7.7.99"
+      version: " - 7.9.99"
       reason: "index template v2 API unavailable before 7.8"
       features: allowed_warnings
 
@@ -193,10 +196,40 @@
 
   - do:
       indices.create:
+        prefer_v2_templates: true
         index: eggplant
 
   - do:
       indices.get:
         index: eggplant
 
   - match: {eggplant.settings.index.number_of_shards: "3"}
+
+---
+"Version 1 templates are preferred if the flag is set":
+  - skip:
+      version: " - 7.9.99"
+      reason: "not backported yet"
+      features: allowed_warnings
+
+  - do:
+      allowed_warnings:
+        - "index template [my-template] has index patterns [eggplant] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template] will take precedence during new index creation"
+      indices.put_index_template:
+        name: my-template
+        body:
+          index_patterns: ["eggplant"]
+          template:
+            settings:
+              number_of_replicas: 2
+
+  - do:
+      indices.create:
+        prefer_v2_templates: false
+        index: eggplant
+
+  - do:
+      indices.get:
+        index: eggplant
+
+  - match: {eggplant.settings.index.number_of_replicas: "1"}

server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java

@@ -25,6 +25,7 @@
 import org.elasticsearch.cluster.ack.ClusterStateUpdateRequest;
 import org.elasticsearch.cluster.block.ClusterBlock;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
 
@@ -42,6 +43,7 @@ public class CreateIndexClusterStateUpdateRequest extends ClusterStateUpdateRequ
     private Index recoverFrom;
     private ResizeType resizeType;
     private boolean copySettings;
+    private Boolean preferV2Templates;
 
     private Settings settings = Settings.Builder.EMPTY_SETTINGS;
 
@@ -94,6 +96,11 @@ public CreateIndexClusterStateUpdateRequest copySettings(final boolean copySetti
         return this;
     }
 
+    public CreateIndexClusterStateUpdateRequest preferV2Templates(@Nullable Boolean preferV2Templates) {
+        this.preferV2Templates = preferV2Templates;
+        return this;
+    }
+
     public String cause() {
         return cause;
     }
@@ -145,6 +152,11 @@ public boolean copySettings() {
         return copySettings;
     }
 
+    @Nullable
+    public Boolean preferV2Templates() {
+        return preferV2Templates;
+    }
+
     @Override
     public String toString() {
         return "CreateIndexClusterStateUpdateRequest{" +
@@ -158,6 +170,7 @@ public String toString() {
             ", aliases=" + aliases +
             ", blocks=" + blocks +
             ", waitForActiveShards=" + waitForActiveShards +
+            ", preferV2Templates=" + preferV2Templates +
             '}';
     }
 }

server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java

@@ -29,6 +29,7 @@
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
@@ -80,6 +81,8 @@ public class CreateIndexRequest extends AcknowledgedRequest<CreateIndexRequest>
 
     private String mappings = "{}";
 
+    private Boolean preferV2Templates;
+
     private final Set<Alias> aliases = new HashSet<>();
 
     private ActiveShardCount waitForActiveShards = ActiveShardCount.DEFAULT;
@@ -107,6 +110,9 @@ public CreateIndexRequest(StreamInput in) throws IOException {
             aliases.add(new Alias(in));
         }
         waitForActiveShards = ActiveShardCount.readFrom(in);
+        if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
+            preferV2Templates = in.readOptionalBoolean();
+        }
     }
 
     public CreateIndexRequest() {
@@ -158,6 +164,16 @@ public CreateIndexRequest index(String index) {
         return this;
     }
 
+    public CreateIndexRequest preferV2Templates(@Nullable Boolean preferV2Templates) {
+        this.preferV2Templates = preferV2Templates;
+        return this;
+    }
+
+    @Nullable
+    public Boolean preferV2Templates() {
+        return this.preferV2Templates;
+    }
+
     /**
      * The settings to create the index with.
      */
@@ -468,6 +484,9 @@ public void writeTo(StreamOutput out) throws IOException {
             alias.writeTo(out);
         }
         waitForActiveShards.writeTo(out);
+        if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+            out.writeOptionalBoolean(preferV2Templates);
+        }
     }
 
 }

server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java

@@ -82,6 +82,7 @@ protected void masterOperation(Task task, final CreateIndexRequest request, fina
                 .ackTimeout(request.timeout()).masterNodeTimeout(request.masterNodeTimeout())
                 .settings(request.settings()).mappings(request.mappings())
                 .aliases(request.aliases())
+                .preferV2Templates(request.preferV2Templates())
                 .waitForActiveShards(request.waitForActiveShards());
 
         createIndexService.createIndex(updateRequest, ActionListener.map(listener, response ->

server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java

@@ -138,7 +138,8 @@ static CreateIndexClusterStateUpdateRequest prepareCreateIndexRequest(final Stri
             .settings(createIndexRequest.settings())
             .aliases(createIndexRequest.aliases())
             .waitForActiveShards(ActiveShardCount.NONE) // not waiting for shards here, will wait on the alias switch operation
-            .mappings(createIndexRequest.mappings());
+            .mappings(createIndexRequest.mappings())
+            .preferV2Templates(createIndexRequest.preferV2Templates());
     }
 
     /**