Add prefer_v2_templates flag and index setting (elastic#55411)

This commit adds a new querystring parameter on the following APIs:
- Index
- Update
- Bulk
- Create Index
- Rollover

These APIs now support a `?prefer_v2_templates=true|false` flag. This flag changes the preference
creation to use either V2 index templates or V1 templates. This flag defaults to `false` and will be
changed to `true` for 8.0+ in subsequent work.

Additionally, setting this flag internally sets the `index.prefer_v2_templates` index-level setting.
This setting is used so that actions that automatically create a new index (things like rollover
initiated by ILM) will inherit the preference from the original index. This setting is dynamic so
that a transition from v1 to v2 templates can occur for long-running indices grouped by an alias
performing periodic rollover.

This also adds support for sending this parameter to the High Level Rest Client.

Relates to elastic#53101

client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesRequestConverters.java

@@ -116,6 +116,9 @@ static Request createIndex(CreateIndexRequest createIndexRequest) throws IOExcep
         parameters.withTimeout(createIndexRequest.timeout());
         parameters.withMasterTimeout(createIndexRequest.masterNodeTimeout());
         parameters.withWaitForActiveShards(createIndexRequest.waitForActiveShards());
+        if (createIndexRequest.preferV2Templates() != null) {
+            parameters.putParam(IndexMetadata.PREFER_V2_TEMPLATES_FLAG, Boolean.toString(createIndexRequest.preferV2Templates()));
+        }
         request.addParameters(parameters.asMap());
         request.setEntity(RequestConverters.createEntity(createIndexRequest, RequestConverters.REQUEST_BODY_CONTENT_TYPE));
         return request;

client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java

@@ -56,6 +56,7 @@
 import org.elasticsearch.client.security.RefreshPolicy;
 import org.elasticsearch.client.tasks.TaskId;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.Strings;
@@ -132,6 +133,9 @@ static Request bulk(BulkRequest bulkRequest) throws IOException {
         parameters.withRefreshPolicy(bulkRequest.getRefreshPolicy());
         parameters.withPipeline(bulkRequest.pipeline());
         parameters.withRouting(bulkRequest.routing());
+        if (bulkRequest.preferV2Templates() != null) {
+            parameters.putParam(IndexMetadata.PREFER_V2_TEMPLATES_FLAG, Boolean.toString(bulkRequest.preferV2Templates()));
+        }
         // Bulk API only supports newline delimited JSON or Smile. Before executing
         // the bulk, we need to check that all requests have the same content-type
         // and this content-type is supported by the Bulk API.
@@ -345,6 +349,9 @@ static Request index(IndexRequest indexRequest) {
         parameters.withPipeline(indexRequest.getPipeline());
         parameters.withRefreshPolicy(indexRequest.getRefreshPolicy());
         parameters.withWaitForActiveShards(indexRequest.waitForActiveShards());
+        if (indexRequest.preferV2Templates() != null) {
+            parameters.putParam(IndexMetadata.PREFER_V2_TEMPLATES_FLAG, Boolean.toString(indexRequest.preferV2Templates()));
+        }
 
         BytesRef source = indexRequest.source().toBytesRef();
         ContentType contentType = createContentType(indexRequest.getContentType());
@@ -373,6 +380,9 @@ static Request update(UpdateRequest updateRequest) throws IOException {
         parameters.withRetryOnConflict(updateRequest.retryOnConflict());
         parameters.withVersion(updateRequest.version());
         parameters.withVersionType(updateRequest.versionType());
+        if (updateRequest.preferV2Templates() != null) {
+            parameters.putParam(IndexMetadata.PREFER_V2_TEMPLATES_FLAG, Boolean.toString(updateRequest.preferV2Templates()));
+        }
 
         // The Java API allows update requests with different content types
         // set for the partial document and the upsert document. This client

client/rest-high-level/src/main/java/org/elasticsearch/client/indices/CreateIndexRequest.java

@@ -25,6 +25,7 @@
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.client.TimedRequest;
 import org.elasticsearch.client.Validatable;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
@@ -62,6 +63,7 @@ public class CreateIndexRequest extends TimedRequest implements Validatable, ToX
 
     private BytesReference mappings;
     private XContentType mappingsXContentType;
+    private Boolean preferV2Templates;
 
     private final Set<Alias> aliases = new HashSet<>();
 
@@ -265,6 +267,16 @@ public CreateIndexRequest aliases(Collection<Alias> aliases) {
         return this;
     }
 
+    public CreateIndexRequest preferV2Templates(Boolean preferV2Templates) {
+        this.preferV2Templates = preferV2Templates;
+        return this;
+    }
+
+    @Nullable
+    public Boolean preferV2Templates() {
+        return this.preferV2Templates;
+    }
+
     /**
      * Sets the settings and mappings as a single source.
      *

docs/reference/api-conventions.asciidoc

@@ -385,6 +385,7 @@ Returns:
     "settings": {
       "index.number_of_replicas": "1",
       "index.number_of_shards": "1",
+      "index.prefer_v2_templates": "false",
       "index.creation_date": "1474389951325",
       "index.uuid": "n6gzFZTgS664GUfx0Xrpjw",
       "index.version.created": ...,
@@ -421,7 +422,8 @@ Returns:
         "version": {
           "created": ...
         },
-        "provided_name" : "twitter"
+        "provided_name" : "twitter",
+        "prefer_v2_templates": "false"
       }
     }
   }

rest-api-spec/src/main/resources/rest-api-spec/api/bulk.json

@@ -87,6 +87,10 @@
       "pipeline":{
         "type":"string",
         "description":"The pipeline id to preprocess incoming documents with"
+      },
+      "prefer_v2_templates": {
+        "type": "boolean",
+        "description": "favor V2 templates instead of V1 templates during automatic index creation"
       }
     },
     "body":{

rest-api-spec/src/main/resources/rest-api-spec/api/create.json

@@ -90,6 +90,10 @@
       "pipeline":{
         "type":"string",
         "description":"The pipeline id to preprocess incoming documents with"
+      },
+      "prefer_v2_templates": {
+        "type": "boolean",
+        "description": "favor V2 templates instead of V1 templates during automatic index creation"
       }
     },
     "body":{

rest-api-spec/src/main/resources/rest-api-spec/api/index.json

@@ -139,6 +139,10 @@
       "pipeline":{
         "type":"string",
         "description":"The pipeline id to preprocess incoming documents with"
+      },
+      "prefer_v2_templates": {
+        "type": "boolean",
+        "description": "favor V2 templates instead of V1 templates during automatic index creation"
       }
     },
     "body":{

rest-api-spec/src/main/resources/rest-api-spec/api/indices.create.json

@@ -37,6 +37,10 @@
       "master_timeout":{
         "type":"time",
         "description":"Specify timeout for connection to master"
+      },
+      "prefer_v2_templates": {
+        "type": "boolean",
+        "description": "favor V2 templates instead of V1 templates during index creation"
       }
     },
     "body":{

rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json

@@ -57,6 +57,10 @@
       "wait_for_active_shards":{
         "type":"string",
         "description":"Set the number of active shards to wait for on the newly created rollover index before the operation returns."
+      },
+      "prefer_v2_templates": {
+        "type": "boolean",
+        "description": "favor V2 templates instead of V1 templates during automatic index creation"
       }
     },
     "body":{

rest-api-spec/src/main/resources/rest-api-spec/api/update.json

@@ -99,6 +99,10 @@
       "if_primary_term":{
         "type":"number",
         "description":"only perform the update operation if the last operation that has changed the document has the specified primary term"
+      },
+      "prefer_v2_templates": {
+        "type": "boolean",
+        "description": "favor V2 templates instead of V1 templates during automatic index creation"
       }
     },
     "body":{

rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_index_template/15_composition.yml

@@ -61,6 +61,7 @@
   - do:
       indices.create:
         index: bar-baz
+        prefer_v2_templates: true
         body:
           settings:
             index.priority: 17
@@ -120,6 +121,7 @@
 
   - do:
       indices.create:
+        prefer_v2_templates: true
         index: bar-baz
 
   - do:
@@ -164,6 +166,7 @@
 
   - do:
       indices.create:
+        prefer_v2_templates: true
         index: bazfoo
 
   - do:
@@ -193,10 +196,40 @@
 
   - do:
       indices.create:
+        prefer_v2_templates: true
         index: eggplant
 
   - do:
       indices.get:
         index: eggplant
 
   - match: {eggplant.settings.index.number_of_shards: "3"}
+
+---
+"Version 1 templates are preferred if the flag is set":
+  - skip:
+      version: " - 7.7.99"
+      reason: "index template v2 API unavailable before 7.8"
+      features: allowed_warnings
+
+  - do:
+      allowed_warnings:
+        - "index template [my-template] has index patterns [eggplant] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template] will take precedence during new index creation"
+      indices.put_index_template:
+        name: my-template
+        body:
+          index_patterns: ["eggplant"]
+          template:
+            settings:
+              number_of_replicas: 2
+
+  - do:
+      indices.create:
+        prefer_v2_templates: false
+        index: eggplant
+
+  - do:
+      indices.get:
+        index: eggplant
+
+  - match: {eggplant.settings.index.number_of_replicas: "1"}

server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java

@@ -25,6 +25,7 @@
 import org.elasticsearch.cluster.ack.ClusterStateUpdateRequest;
 import org.elasticsearch.cluster.block.ClusterBlock;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
 
@@ -44,6 +45,7 @@ public class CreateIndexClusterStateUpdateRequest extends ClusterStateUpdateRequ
     private Index recoverFrom;
     private ResizeType resizeType;
     private boolean copySettings;
+    private Boolean preferV2Templates;
 
     private Settings settings = Settings.Builder.EMPTY_SETTINGS;
 
@@ -96,6 +98,11 @@ public CreateIndexClusterStateUpdateRequest copySettings(final boolean copySetti
         return this;
     }
 
+    public CreateIndexClusterStateUpdateRequest preferV2Templates(@Nullable Boolean preferV2Templates) {
+        this.preferV2Templates = preferV2Templates;
+        return this;
+    }
+
     public String cause() {
         return cause;
     }
@@ -147,4 +154,8 @@ public boolean copySettings() {
         return copySettings;
     }
 
+    @Nullable
+    public Boolean preferV2Templates() {
+        return preferV2Templates;
+    }
 }

server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java

@@ -29,6 +29,7 @@
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
@@ -83,6 +84,8 @@ public class CreateIndexRequest extends AcknowledgedRequest<CreateIndexRequest>
 
     private final Map<String, String> mappings = new HashMap<>();
 
+    private Boolean preferV2Templates;
+
     private final Set<Alias> aliases = new HashSet<>();
 
     private ActiveShardCount waitForActiveShards = ActiveShardCount.DEFAULT;
@@ -118,6 +121,9 @@ public CreateIndexRequest(StreamInput in) throws IOException {
             in.readBoolean(); // updateAllTypes
         }
         waitForActiveShards = ActiveShardCount.readFrom(in);
+        if (in.getVersion().onOrAfter(Version.V_7_8_0)) {
+            preferV2Templates = in.readOptionalBoolean();
+        }
     }
 
     public CreateIndexRequest() {
@@ -169,6 +175,16 @@ public CreateIndexRequest index(String index) {
         return this;
     }
 
+    public CreateIndexRequest preferV2Templates(@Nullable Boolean preferV2Templates) {
+        this.preferV2Templates = preferV2Templates;
+        return this;
+    }
+
+    @Nullable
+    public Boolean preferV2Templates() {
+        return this.preferV2Templates;
+    }
+
     /**
      * The settings to create the index with.
      */
@@ -459,7 +475,7 @@ public CreateIndexRequest waitForActiveShards(ActiveShardCount waitForActiveShar
     public CreateIndexRequest waitForActiveShards(final int waitForActiveShards) {
         return waitForActiveShards(ActiveShardCount.from(waitForActiveShards));
     }
-    
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
@@ -483,6 +499,9 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeBoolean(true); // updateAllTypes
         }
         waitForActiveShards.writeTo(out);
+        if (out.getVersion().onOrAfter(Version.V_7_8_0)) {
+            out.writeOptionalBoolean(preferV2Templates);
+        }
     }
 
     @Override

server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java

@@ -81,6 +81,7 @@ protected void masterOperation(final CreateIndexRequest request, final ClusterSt
                 .ackTimeout(request.timeout()).masterNodeTimeout(request.masterNodeTimeout())
                 .settings(request.settings()).mappings(request.mappings())
                 .aliases(request.aliases())
+                .preferV2Templates(request.preferV2Templates())
                 .waitForActiveShards(request.waitForActiveShards());
 
         createIndexService.createIndex(updateRequest, ActionListener.map(listener, response ->

server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java

@@ -140,7 +140,8 @@ static CreateIndexClusterStateUpdateRequest prepareCreateIndexRequest(final Stri
             .settings(createIndexRequest.settings())
             .aliases(createIndexRequest.aliases())
             .waitForActiveShards(ActiveShardCount.NONE) // not waiting for shards here, will wait on the alias switch operation
-            .mappings(createIndexRequest.mappings());
+            .mappings(createIndexRequest.mappings())
+            .preferV2Templates(createIndexRequest.preferV2Templates());
     }
 
     /**

server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java

@@ -40,6 +40,7 @@
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.index.shard.DocsStats;
 import org.elasticsearch.tasks.Task;
@@ -133,6 +134,13 @@ public void onResponse(IndicesStatsResponse statsResponse) {
                             + rolloverIndexName + "]", new ClusterStateUpdateTask() {
                             @Override
                             public ClusterState execute(ClusterState currentState) throws Exception {
+                                // If they haven't explicitly specified whether to use V2 or V1 templates, inherit their preference
+                                // from the existing index (the source index) settings.
+                                if (rolloverRequest.getCreateIndexRequest().preferV2Templates() == null) {
+                                    Settings originalIndexSettings = currentState.metadata().index(sourceIndexName).getSettings();
+                                    rolloverRequest.getCreateIndexRequest()
+                                        .preferV2Templates(IndexMetadata.PREFER_V2_TEMPLATES_SETTING.get(originalIndexSettings));
+                                }
                                 MetadataRolloverService.RolloverResult rolloverResult = rolloverService.rolloverClusterState(currentState,
                                     rolloverRequest.getAlias(), rolloverRequest.getNewIndexName(), rolloverRequest.getCreateIndexRequest(),
                                     metConditions, false);