Use V2 index templates during index creation (elastic#54669)

* Use V2 index templates during index creation

This commit changes our index creation code to use (and favor!) V2 index templates during index
creation. The creation precedence goes like so, in order of precedence:

- Existing source `IndexMetadata` - for example, when recovering from a peer or a shrink/split/clone
  where index templates should not be applied
- A matching V2 index template, if one is found
  - When a V2 template is found, all component templates (in the `composed_of` field) are applied
    in the order that they appear, with the index template having the 2nd highest precedence (the
    create index request always has the top priority when it comes to index settings)
- All matching V1 templates (the old style)

This also adds index template validation when `PUT`-ing a new v2 index template (because this was
required) and ensures that all index and component templates specify *no* top-level mapping type (it
is automatically added when the template is added to the cluster state).

This does not yet implement fine-grained component template merging of mappings, where we favor
merging only a single field's configuration, that will be done in subsequent work.

This also keeps the existing hidden index behavior present for v1 templates, where a hidden index
will match v2 index templates unless they are global (`*`) templates.

Relates to elastic#53101

rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_index_template/15_composition.yml

@@ -0,0 +1,188 @@
+---
+"Component and index template composition":
+  - skip:
+      version: " - 7.7.99"
+      reason: "itv2 is available in 7.8+"
+
+  - do:
+      cluster.put_component_template:
+        name: ct_low
+        body:
+          template:
+            settings:
+              number_of_replicas: 1
+            mappings:
+              properties:
+                field2:
+                  type: text
+            aliases:
+              aliasname:
+                is_write_index: false
+
+  - do:
+      cluster.put_component_template:
+        name: ct_high
+        body:
+          template:
+            settings:
+              index.number_of_replicas: 0
+            mappings:
+              properties:
+                field2:
+                  type: keyword
+            aliases:
+              aliasname:
+                is_write_index: true
+
+  - do:
+      indices.put_index_template:
+        name: my-template
+        body:
+          index_patterns: ["foo", "bar-*"]
+          template:
+            settings:
+              index.number_of_shards: 2
+            mappings:
+              properties:
+                field:
+                  type: keyword
+                  ignore_above: 255
+            aliases:
+              my_alias: {}
+              aliasname:
+                filter:
+                  match_all: {}
+          composed_of: ["ct_low", "ct_high"]
+          priority: 400
+
+  - do:
+      indices.create:
+        index: bar-baz
+        body:
+          settings:
+            index.priority: 17
+          mappings:
+            properties:
+              foo:
+                type: keyword
+          aliases:
+            other: {}
+
+  - do:
+      indices.get:
+        index: bar-baz
+
+  - match: {bar-baz.settings.index.number_of_shards: "2"}
+  - match: {bar-baz.settings.index.number_of_replicas: "0"}
+  - match: {bar-baz.settings.index.priority: "17"}
+  - match: {bar-baz.mappings.properties.field: {type: keyword, ignore_above: 255}}
+  - match: {bar-baz.mappings.properties.field2: {type: keyword}}
+  - match: {bar-baz.mappings.properties.foo: {type: keyword}}
+  - match: {bar-baz.aliases.aliasname: {filter: {match_all: {}}}}
+  - match: {bar-baz.aliases.my_alias: {}}
+  - match: {bar-baz.aliases.other: {}}
+
+---
+"Index template priority":
+  - skip:
+      version: " - 7.7.99"
+      reason: "itv2 is available in 7.8+"
+
+  - do:
+      indices.put_index_template:
+        name: my-template
+        body:
+          index_patterns: ["foo", "bar-*"]
+          template:
+            settings:
+              index.number_of_shards: 2
+          composed_of: []
+          priority: 400
+
+  - do:
+      indices.put_index_template:
+        name: another-template
+        body:
+          index_patterns: ["bar-*"]
+          template:
+            settings:
+              index.number_of_shards: 3
+          composed_of: []
+          priority: 405
+
+  - do:
+      indices.create:
+        index: bar-baz
+
+  - do:
+      indices.get:
+        index: bar-baz
+
+  - match: {bar-baz.settings.index.number_of_shards: "3"}
+
+---
+"Component template only composition":
+  - skip:
+      version: " - 7.7.99"
+      reason: "itv2 is available in 7.8+"
+
+  - do:
+      cluster.put_component_template:
+        name: ct_low
+        body:
+          template:
+            aliases:
+              alias1: {}
+
+  - do:
+      cluster.put_component_template:
+        name: ct_high
+        body:
+          template:
+            mappings:
+              properties:
+                field:
+                  type: keyword
+
+  - do:
+      indices.put_index_template:
+        name: my-template
+        body:
+          index_patterns: ["baz*"]
+          composed_of: ["ct_low", "ct_high"]
+
+  - do:
+      indices.create:
+        index: bazfoo
+
+  - do:
+      indices.get:
+        index: bazfoo
+
+  - match: {bazfoo.mappings.properties.field: {type: keyword}}
+  - match: {bazfoo.aliases.alias1: {}}
+
+---
+"Index template without component templates":
+  - skip:
+      version: " - 7.7.99"
+      reason: "itv2 is available in 7.8+"
+
+  - do:
+      indices.put_index_template:
+        name: my-template
+        body:
+          index_patterns: ["eggplant"]
+          template:
+            settings:
+              number_of_shards: 3
+
+  - do:
+      indices.create:
+        index: eggplant
+
+  - do:
+      indices.get:
+        index: eggplant
+
+  - match: {eggplant.settings.index.number_of_shards: "3"}

server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java

@@ -42,7 +42,7 @@
 import java.util.Locale;
 import java.util.regex.Pattern;
 
-import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findTemplates;
+import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findV1Templates;
 
 public class MetadataRolloverService {
     private static final Pattern INDEX_NAME_PATTERN = Pattern.compile("^.*-\\d+$");
@@ -165,7 +165,7 @@ static List<AliasAction> rolloverAliasToNewIndex(String oldIndex, String newInde
      */
     static void checkNoDuplicatedAliasInIndexTemplate(Metadata metadata, String rolloverIndexName, String rolloverRequestAlias,
                                                       @Nullable Boolean isHidden) {
-        final List<IndexTemplateMetadata> matchedTemplates = findTemplates(metadata, rolloverIndexName, isHidden);
+        final List<IndexTemplateMetadata> matchedTemplates = findV1Templates(metadata, rolloverIndexName, isHidden);
         for (IndexTemplateMetadata template : matchedTemplates) {
             if (template.aliases().containsKey(rolloverRequestAlias)) {
                 throw new IllegalArgumentException(String.format(Locale.ROOT,

server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateV2Action.java

@@ -26,15 +26,12 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
-import org.elasticsearch.cluster.metadata.IndexTemplateV2;
-import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.IndexTemplateV2;
 import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService;
-import org.elasticsearch.cluster.metadata.Template;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -74,14 +71,6 @@ protected ClusterBlockException checkBlock(PutIndexTemplateV2Action.Request requ
     protected void masterOperation(final PutIndexTemplateV2Action.Request request, final ClusterState state,
                                    final ActionListener<AcknowledgedResponse> listener) {
         IndexTemplateV2 indexTemplate = request.indexTemplate();
-        Template template = indexTemplate.template();
-        // Normalize the index settings if necessary
-        if (template.settings() != null) {
-            Settings.Builder settings = Settings.builder().put(template.settings()).normalizePrefix(IndexMetadata.INDEX_SETTING_PREFIX);
-            template = new Template(settings.build(), template.mappings(), template.aliases());
-            indexTemplate = new IndexTemplateV2(indexTemplate.indexPatterns(), template, indexTemplate.composedOf(),
-                indexTemplate.priority(), indexTemplate.version(), indexTemplate.metadata());
-        }
         indexTemplateService.putIndexTemplateV2(request.cause(), request.create(), request.name(), request.masterNodeTimeout(),
             indexTemplate, listener);
     }

server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java

@@ -311,7 +311,7 @@ static boolean resolvePipelines(final DocWriteRequest<?> originalRequest, final
                 }
             } else if (indexRequest.index() != null) {
                 // the index does not exist yet (and this is a valid request), so match index templates to look for pipelines
-                List<IndexTemplateMetadata> templates = MetadataIndexTemplateService.findTemplates(metadata, indexRequest.index(), null);
+                List<IndexTemplateMetadata> templates = MetadataIndexTemplateService.findV1Templates(metadata, indexRequest.index(), null);
                 assert (templates != null);
                 // order of templates are highest order first
                 for (final IndexTemplateMetadata template : templates) {

server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateV2.java

@@ -32,6 +32,7 @@
 import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -114,11 +115,15 @@ public List<String> indexPatterns() {
         return indexPatterns;
     }
 
+    @Nullable
     public Template template() {
         return template;
     }
 
     public List<String> composedOf() {
+        if (componentTemplates == null) {
+            return Collections.emptyList();
+        }
         return componentTemplates;
     }