Use V2 index templates during index creation (#54669)

* Use V2 index templates during index creation

This commit changes our index creation code to use (and favor!) V2 index templates during index
creation. The creation precedence goes like so, in order of precedence:

- Existing source `IndexMetadata` - for example, when recovering from a peer or a shrink/split/clone
  where index templates should not be applied
- A matching V2 index template, if one is found
  - When a V2 template is found, all component templates (in the `composed_of` field) are applied
    in the order that they appear, with the index template having the 2nd highest precedence (the
    create index request always has the top priority when it comes to index settings)
- All matching V1 templates (the old style)

This also adds index template validation when `PUT`-ing a new v2 index template (because this was
required) and ensures that all index and component templates specify *no* top-level mapping type (it
is automatically added when the template is added to the cluster state).

This does not yet implement fine-grained component template merging of mappings, where we favor
merging only a single field's configuration, that will be done in subsequent work.

This also keeps the existing hidden index behavior present for v1 templates, where a hidden index
will match v2 index templates unless they are global (`*`) templates.

Relates to #53101

rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_index_template/15_composition.yml

@@ -0,0 +1,188 @@
+---
+"Component and index template composition":
+  - skip:
+      version: " - 7.99.99"
+      reason: "not yet backported"
+
+  - do:
+      cluster.put_component_template:
+        name: ct_low
+        body:
+          template:
+            settings:
+              number_of_replicas: 1
+            mappings:
+              properties:
+                field2:
+                  type: text
+            aliases:
+              aliasname:
+                is_write_index: false
+
+  - do:
+      cluster.put_component_template:
+        name: ct_high
+        body:
+          template:
+            settings:
+              index.number_of_replicas: 0
+            mappings:
+              properties:
+                field2:
+                  type: keyword
+            aliases:
+              aliasname:
+                is_write_index: true
+
+  - do:
+      indices.put_index_template:
+        name: my-template
+        body:
+          index_patterns: ["foo", "bar-*"]
+          template:
+            settings:
+              index.number_of_shards: 2
+            mappings:
+              properties:
+                field:
+                  type: keyword
+                  ignore_above: 255
+            aliases:
+              my_alias: {}
+              aliasname:
+                filter:
+                  match_all: {}
+          composed_of: ["ct_low", "ct_high"]
+          priority: 400
+
+  - do:
+      indices.create:
+        index: bar-baz
+        body:
+          settings:
+            index.priority: 17
+          mappings:
+            properties:
+              foo:
+                type: keyword
+          aliases:
+            other: {}
+
+  - do:
+      indices.get:
+        index: bar-baz
+
+  - match: {bar-baz.settings.index.number_of_shards: "2"}
+  - match: {bar-baz.settings.index.number_of_replicas: "0"}
+  - match: {bar-baz.settings.index.priority: "17"}
+  - match: {bar-baz.mappings.properties.field: {type: keyword, ignore_above: 255}}
+  - match: {bar-baz.mappings.properties.field2: {type: keyword}}
+  - match: {bar-baz.mappings.properties.foo: {type: keyword}}
+  - match: {bar-baz.aliases.aliasname: {filter: {match_all: {}}}}
+  - match: {bar-baz.aliases.my_alias: {}}
+  - match: {bar-baz.aliases.other: {}}
+
+---
+"Index template priority":
+  - skip:
+      version: " - 7.99.99"
+      reason: "not yet backported"
+
+  - do:
+      indices.put_index_template:
+        name: my-template
+        body:
+          index_patterns: ["foo", "bar-*"]
+          template:
+            settings:
+              index.number_of_shards: 2
+          composed_of: []
+          priority: 400
+
+  - do:
+      indices.put_index_template:
+        name: another-template
+        body:
+          index_patterns: ["bar-*"]
+          template:
+            settings:
+              index.number_of_shards: 3
+          composed_of: []
+          priority: 405
+
+  - do:
+      indices.create:
+        index: bar-baz
+
+  - do:
+      indices.get:
+        index: bar-baz
+
+  - match: {bar-baz.settings.index.number_of_shards: "3"}
+
+---
+"Component template only composition":
+  - skip:
+      version: " - 7.99.99"
+      reason: "not yet backported"
+
+  - do:
+      cluster.put_component_template:
+        name: ct_low
+        body:
+          template:
+            aliases:
+              alias1: {}
+
+  - do:
+      cluster.put_component_template:
+        name: ct_high
+        body:
+          template:
+            mappings:
+              properties:
+                field:
+                  type: keyword
+
+  - do:
+      indices.put_index_template:
+        name: my-template
+        body:
+          index_patterns: ["baz*"]
+          composed_of: ["ct_low", "ct_high"]
+
+  - do:
+      indices.create:
+        index: bazfoo
+
+  - do:
+      indices.get:
+        index: bazfoo
+
+  - match: {bazfoo.mappings.properties.field: {type: keyword}}
+  - match: {bazfoo.aliases.alias1: {}}
+
+---
+"Index template without component templates":
+  - skip:
+      version: " - 7.99.99"
+      reason: "not yet backported"
+
+  - do:
+      indices.put_index_template:
+        name: my-template
+        body:
+          index_patterns: ["eggplant"]
+          template:
+            settings:
+              number_of_shards: 3
+
+  - do:
+      indices.create:
+        index: eggplant
+
+  - do:
+      indices.get:
+        index: eggplant
+
+  - match: {eggplant.settings.index.number_of_shards: "3"}

server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java

@@ -40,7 +40,7 @@
 import java.util.Locale;
 import java.util.regex.Pattern;
 
-import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findTemplates;
+import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findV1Templates;
 
 public class MetadataRolloverService {
     private static final Pattern INDEX_NAME_PATTERN = Pattern.compile("^.*-\\d+$");
@@ -161,10 +161,9 @@ static List<AliasAction> rolloverAliasToNewIndex(String oldIndex, String newInde
      * the rollover alias will point to multiple indices. This causes indexing requests to be rejected.
      * To avoid this, we make sure that there is no duplicated alias in index templates before creating a new index.
      */
-    static void checkNoDuplicatedAliasInIndexTemplate(
-            Metadata metadata, String rolloverIndexName, String rolloverRequestAlias,
-            @Nullable Boolean isHidden) {
-        final List<IndexTemplateMetadata> matchedTemplates = findTemplates(metadata, rolloverIndexName, isHidden);
+    static void checkNoDuplicatedAliasInIndexTemplate(Metadata metadata, String rolloverIndexName, String rolloverRequestAlias,
+                                                      @Nullable Boolean isHidden) {
+        final List<IndexTemplateMetadata> matchedTemplates = findV1Templates(metadata, rolloverIndexName, isHidden);
         for (IndexTemplateMetadata template : matchedTemplates) {
             if (template.aliases().containsKey(rolloverRequestAlias)) {
                 throw new IllegalArgumentException(String.format(Locale.ROOT,

server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateV2Action.java

@@ -26,15 +26,12 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
-import org.elasticsearch.cluster.metadata.IndexTemplateV2;
-import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.IndexTemplateV2;
 import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService;
-import org.elasticsearch.cluster.metadata.Template;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -75,14 +72,6 @@ protected ClusterBlockException checkBlock(PutIndexTemplateV2Action.Request requ
     protected void masterOperation(Task task, final PutIndexTemplateV2Action.Request request, final ClusterState state,
                                    final ActionListener<AcknowledgedResponse> listener) {
         IndexTemplateV2 indexTemplate = request.indexTemplate();
-        Template template = indexTemplate.template();
-        // Normalize the index settings if necessary
-        if (template.settings() != null) {
-            Settings.Builder settings = Settings.builder().put(template.settings()).normalizePrefix(IndexMetadata.INDEX_SETTING_PREFIX);
-            template = new Template(settings.build(), template.mappings(), template.aliases());
-            indexTemplate = new IndexTemplateV2(indexTemplate.indexPatterns(), template, indexTemplate.composedOf(),
-                indexTemplate.priority(), indexTemplate.version(), indexTemplate.metadata());
-        }
         indexTemplateService.putIndexTemplateV2(request.cause(), request.create(), request.name(), request.masterNodeTimeout(),
             indexTemplate, listener);
     }

server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java

@@ -307,7 +307,7 @@ static boolean resolvePipelines(final DocWriteRequest<?> originalRequest, final
                 }
             } else if (indexRequest.index() != null) {
                 // the index does not exist yet (and this is a valid request), so match index templates to look for pipelines
-                List<IndexTemplateMetadata> templates = MetadataIndexTemplateService.findTemplates(metadata, indexRequest.index(), null);
+                List<IndexTemplateMetadata> templates = MetadataIndexTemplateService.findV1Templates(metadata, indexRequest.index(), null);
                 assert (templates != null);
                 // order of templates are highest order first
                 for (final IndexTemplateMetadata template : templates) {

server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateV2.java

@@ -114,11 +114,15 @@ public List<String> indexPatterns() {
         return indexPatterns;
     }
 
+    @Nullable
     public Template template() {
         return template;
     }
 
     public List<String> composedOf() {
+        if (componentTemplates == null) {
+            return List.of();
+        }
         return componentTemplates;
     }