From 9f9ade7dcb04c1ffc66e92357f6cd32721dcda38 Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Fri, 3 Apr 2020 09:34:50 -0600 Subject: [PATCH] Use V2 index templates during index creation (#54669) * Use V2 index templates during index creation This commit changes our index creation code to use (and favor!) V2 index templates during index creation. The creation precedence goes like so, in order of precedence: - Existing source `IndexMetadata` - for example, when recovering from a peer or a shrink/split/clone where index templates should not be applied - A matching V2 index template, if one is found - When a V2 template is found, all component templates (in the `composed_of` field) are applied in the order that they appear, with the index template having the 2nd highest precedence (the create index request always has the top priority when it comes to index settings) - All matching V1 templates (the old style) This also adds index template validation when `PUT`-ing a new v2 index template (because this was required) and ensures that all index and component templates specify *no* top-level mapping type (it is automatically added when the template is added to the cluster state). This does not yet implement fine-grained component template merging of mappings, where we favor merging only a single field's configuration, that will be done in subsequent work. This also keeps the existing hidden index behavior present for v1 templates, where a hidden index will match v2 index templates unless they are global (`*`) templates. Relates to #53101 --- .../15_composition.yml | 188 ++++++++++++ .../rollover/MetadataRolloverService.java | 9 +- .../TransportPutIndexTemplateV2Action.java | 13 +- .../action/bulk/TransportBulkAction.java | 2 +- .../cluster/metadata/IndexTemplateV2.java | 4 + .../metadata/MetadataCreateIndexService.java | 226 ++++++++++---- .../MetadataIndexTemplateService.java | 237 ++++++++++++++- .../cluster/metadata/Template.java | 12 +- .../metadata/ComponentTemplateTests.java | 9 +- .../metadata/IndexTemplateV2Tests.java | 9 +- .../MetadataCreateIndexServiceTests.java | 39 +-- .../MetadataIndexTemplateServiceTests.java | 287 ++++++++++++++++-- 12 files changed, 895 insertions(+), 140 deletions(-) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_index_template/15_composition.yml diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_index_template/15_composition.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_index_template/15_composition.yml new file mode 100644 index 0000000000000..8c2c7d84dc461 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_index_template/15_composition.yml @@ -0,0 +1,188 @@ +--- +"Component and index template composition": + - skip: + version: " - 7.99.99" + reason: "not yet backported" + + - do: + cluster.put_component_template: + name: ct_low + body: + template: + settings: + number_of_replicas: 1 + mappings: + properties: + field2: + type: text + aliases: + aliasname: + is_write_index: false + + - do: + cluster.put_component_template: + name: ct_high + body: + template: + settings: + index.number_of_replicas: 0 + mappings: + properties: + field2: + type: keyword + aliases: + aliasname: + is_write_index: true + + - do: + indices.put_index_template: + name: my-template + body: + index_patterns: ["foo", "bar-*"] + template: + settings: + index.number_of_shards: 2 + mappings: + properties: + field: + type: keyword + ignore_above: 255 + aliases: + my_alias: {} + aliasname: + filter: + match_all: {} + composed_of: ["ct_low", "ct_high"] + priority: 400 + + - do: + indices.create: + index: bar-baz + body: + settings: + index.priority: 17 + mappings: + properties: + foo: + type: keyword + aliases: + other: {} + + - do: + indices.get: + index: bar-baz + + - match: {bar-baz.settings.index.number_of_shards: "2"} + - match: {bar-baz.settings.index.number_of_replicas: "0"} + - match: {bar-baz.settings.index.priority: "17"} + - match: {bar-baz.mappings.properties.field: {type: keyword, ignore_above: 255}} + - match: {bar-baz.mappings.properties.field2: {type: keyword}} + - match: {bar-baz.mappings.properties.foo: {type: keyword}} + - match: {bar-baz.aliases.aliasname: {filter: {match_all: {}}}} + - match: {bar-baz.aliases.my_alias: {}} + - match: {bar-baz.aliases.other: {}} + +--- +"Index template priority": + - skip: + version: " - 7.99.99" + reason: "not yet backported" + + - do: + indices.put_index_template: + name: my-template + body: + index_patterns: ["foo", "bar-*"] + template: + settings: + index.number_of_shards: 2 + composed_of: [] + priority: 400 + + - do: + indices.put_index_template: + name: another-template + body: + index_patterns: ["bar-*"] + template: + settings: + index.number_of_shards: 3 + composed_of: [] + priority: 405 + + - do: + indices.create: + index: bar-baz + + - do: + indices.get: + index: bar-baz + + - match: {bar-baz.settings.index.number_of_shards: "3"} + +--- +"Component template only composition": + - skip: + version: " - 7.99.99" + reason: "not yet backported" + + - do: + cluster.put_component_template: + name: ct_low + body: + template: + aliases: + alias1: {} + + - do: + cluster.put_component_template: + name: ct_high + body: + template: + mappings: + properties: + field: + type: keyword + + - do: + indices.put_index_template: + name: my-template + body: + index_patterns: ["baz*"] + composed_of: ["ct_low", "ct_high"] + + - do: + indices.create: + index: bazfoo + + - do: + indices.get: + index: bazfoo + + - match: {bazfoo.mappings.properties.field: {type: keyword}} + - match: {bazfoo.aliases.alias1: {}} + +--- +"Index template without component templates": + - skip: + version: " - 7.99.99" + reason: "not yet backported" + + - do: + indices.put_index_template: + name: my-template + body: + index_patterns: ["eggplant"] + template: + settings: + number_of_shards: 3 + + - do: + indices.create: + index: eggplant + + - do: + indices.get: + index: eggplant + + - match: {eggplant.settings.index.number_of_shards: "3"} diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java index 4def9e85aa76d..15f990ad435ba 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java @@ -40,7 +40,7 @@ import java.util.Locale; import java.util.regex.Pattern; -import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findTemplates; +import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findV1Templates; public class MetadataRolloverService { private static final Pattern INDEX_NAME_PATTERN = Pattern.compile("^.*-\\d+$"); @@ -161,10 +161,9 @@ static List rolloverAliasToNewIndex(String oldIndex, String newInde * the rollover alias will point to multiple indices. This causes indexing requests to be rejected. * To avoid this, we make sure that there is no duplicated alias in index templates before creating a new index. */ - static void checkNoDuplicatedAliasInIndexTemplate( - Metadata metadata, String rolloverIndexName, String rolloverRequestAlias, - @Nullable Boolean isHidden) { - final List matchedTemplates = findTemplates(metadata, rolloverIndexName, isHidden); + static void checkNoDuplicatedAliasInIndexTemplate(Metadata metadata, String rolloverIndexName, String rolloverRequestAlias, + @Nullable Boolean isHidden) { + final List matchedTemplates = findV1Templates(metadata, rolloverIndexName, isHidden); for (IndexTemplateMetadata template : matchedTemplates) { if (template.aliases().containsKey(rolloverRequestAlias)) { throw new IllegalArgumentException(String.format(Locale.ROOT, diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateV2Action.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateV2Action.java index 7dd42893ee9ea..8a594bb5cdce5 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateV2Action.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateV2Action.java @@ -26,15 +26,12 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; -import org.elasticsearch.cluster.metadata.IndexTemplateV2; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.IndexTemplateV2; import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; -import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -75,14 +72,6 @@ protected ClusterBlockException checkBlock(PutIndexTemplateV2Action.Request requ protected void masterOperation(Task task, final PutIndexTemplateV2Action.Request request, final ClusterState state, final ActionListener listener) { IndexTemplateV2 indexTemplate = request.indexTemplate(); - Template template = indexTemplate.template(); - // Normalize the index settings if necessary - if (template.settings() != null) { - Settings.Builder settings = Settings.builder().put(template.settings()).normalizePrefix(IndexMetadata.INDEX_SETTING_PREFIX); - template = new Template(settings.build(), template.mappings(), template.aliases()); - indexTemplate = new IndexTemplateV2(indexTemplate.indexPatterns(), template, indexTemplate.composedOf(), - indexTemplate.priority(), indexTemplate.version(), indexTemplate.metadata()); - } indexTemplateService.putIndexTemplateV2(request.cause(), request.create(), request.name(), request.masterNodeTimeout(), indexTemplate, listener); } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java index af71f10a078e8..a53acdf59cad2 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java @@ -307,7 +307,7 @@ static boolean resolvePipelines(final DocWriteRequest originalRequest, final } } else if (indexRequest.index() != null) { // the index does not exist yet (and this is a valid request), so match index templates to look for pipelines - List templates = MetadataIndexTemplateService.findTemplates(metadata, indexRequest.index(), null); + List templates = MetadataIndexTemplateService.findV1Templates(metadata, indexRequest.index(), null); assert (templates != null); // order of templates are highest order first for (final IndexTemplateMetadata template : templates) { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateV2.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateV2.java index e707b84488171..06394a3a1c4d4 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateV2.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateV2.java @@ -114,11 +114,15 @@ public List indexPatterns() { return indexPatterns; } + @Nullable public Template template() { return template; } public List composedOf() { + if (componentTemplates == null) { + return List.of(); + } return componentTemplates; } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index ef8539aadf90e..ee221ede892da 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -19,7 +19,6 @@ package org.elasticsearch.cluster.metadata; -import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -90,6 +89,7 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiFunction; +import java.util.function.Function; import java.util.function.Predicate; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -313,39 +313,60 @@ public ClusterState applyCreateIndexRequest(ClusterState currentState, CreateInd final Index recoverFromIndex = request.recoverFrom(); final IndexMetadata sourceMetadata = recoverFromIndex == null ? null : currentState.metadata().getIndexSafe(recoverFromIndex); - // we only find a template when its an API call (a new index) - // find templates, highest order are better matching - final Boolean isHiddenFromRequest = IndexMetadata.INDEX_HIDDEN_SETTING.exists(request.settings()) ? - IndexMetadata.INDEX_HIDDEN_SETTING.get(request.settings()) : null; - final List templates = sourceMetadata == null ? - Collections.unmodifiableList(MetadataIndexTemplateService.findTemplates(currentState.metadata(), - request.index(), - isHiddenFromRequest)) : - List.of(); - - final Map mappings = Collections.unmodifiableMap(parseMappings(request.mappings(), templates, xContentRegistry)); - - final Settings aggregatedIndexSettings = - aggregateIndexSettings(currentState, request, templates, mappings, sourceMetadata, settings, indexScopedSettings); - int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, sourceMetadata); - - final boolean isHiddenAfterTemplates = IndexMetadata.INDEX_HIDDEN_SETTING.get(aggregatedIndexSettings); - validateDotIndex(request.index(), currentState, isHiddenAfterTemplates); - - // remove the setting it's temporary and is only relevant once we create the index - final Settings.Builder settingsBuilder = Settings.builder().put(aggregatedIndexSettings); - settingsBuilder.remove(IndexMetadata.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.getKey()); - final Settings indexSettings = settingsBuilder.build(); + if (sourceMetadata != null) { + // If source metadata was provided, it means we're recovering from an existing index, + // in which case templates don't apply, so create the index from the source metadata + return applyCreateIndexRequestWithExistingMetadata(currentState, request, silent, sourceMetadata); + } else { + // Hidden indices apply templates slightly differently (ignoring wildcard '*' + // templates), so we need to check to see if the request is creating a hidden index + // prior to resolving which templates it matches + final Boolean isHiddenFromRequest = IndexMetadata.INDEX_HIDDEN_SETTING.exists(request.settings()) ? + IndexMetadata.INDEX_HIDDEN_SETTING.get(request.settings()) : null; + + // Check to see if a v2 template matched + final String v2Template = MetadataIndexTemplateService.findV2Template(currentState.metadata(), + request.index(), isHiddenFromRequest); + + if (v2Template != null) { + // If a v2 template was found, it takes precedence over all v1 templates, so create + // the index using that template and the request's specified settings + return applyCreateIndexRequestWithV2Template(currentState, request, silent, v2Template); + } else { + // A v2 template wasn't found, check the v1 templates, in the event no templates are + // found creation still works using the request's specified index settings + final List v1Templates = MetadataIndexTemplateService.findV1Templates(currentState.metadata(), + request.index(), isHiddenFromRequest); - final IndexMetadata.Builder tmpImdBuilder = IndexMetadata.builder(request.index()); - tmpImdBuilder.setRoutingNumShards(routingNumShards); - tmpImdBuilder.settings(indexSettings); + return applyCreateIndexRequestWithV1Templates(currentState, request, silent, v1Templates); + } + } + } - // Set up everything, now locally create the index to see that things are ok, and apply - IndexMetadata tmpImd = tmpImdBuilder.build(); - validateActiveShardCount(request.waitForActiveShards(), tmpImd); + /** + * Given the state and a request as well as the metadata necessary to build a new index, + * validate the configuration with an actual index service as return a new cluster state with + * the index added (and rerouted) + * @param currentState the current state to base the new state off of + * @param request the create index request + * @param silent a boolean for whether logging should be at a lower or higher level + * @param sourceMetadata when recovering from an existing index, metadata that should be copied to the new index + * @param temporaryIndexMeta metadata for the new index built from templates, source metadata, and request settings + * @param mappings a map of mappings for the new index + * @param aliasSupplier a function that takes the real {@link IndexService} and returns a list of {@link AliasMetadata} aliases + * @param templatesApplied a list of the names of the templates applied, for logging + * @return a new cluster state with the index added + */ + private ClusterState applyCreateIndexWithTemporaryService(final ClusterState currentState, + final CreateIndexClusterStateUpdateRequest request, + final boolean silent, + final IndexMetadata sourceMetadata, + final IndexMetadata temporaryIndexMeta, + final Map mappings, + final Function> aliasSupplier, + final List templatesApplied) throws Exception { // create the index here (on the master) to validate it can be created, as well as adding the mapping - return indicesService.withTempIndexService(tmpImd, indexService -> { + return indicesService.withTempIndexService(temporaryIndexMeta, indexService -> { try { updateIndexMappingsAndBuildSortOrder(indexService, mappings, sourceMetadata); } catch (Exception e) { @@ -353,25 +374,20 @@ public ClusterState applyCreateIndexRequest(ClusterState currentState, CreateInd throw e; } - // the context is only used for validation so it's fine to pass fake values for the shard id and the current - // timestamp - final List aliases = Collections.unmodifiableList( - resolveAndValidateAliases(request.index(), request.aliases(), templates, currentState.metadata(), aliasValidator, - xContentRegistry, indexService.newQueryShardContext(0, null, () -> 0L, null)) - ); + final List aliases = aliasSupplier.apply(indexService); final IndexMetadata indexMetadata; try { - indexMetadata = buildIndexMetadata(request.index(), aliases, indexService.mapperService()::documentMapper, indexSettings, - routingNumShards, sourceMetadata); + indexMetadata = buildIndexMetadata(request.index(), aliases, indexService.mapperService()::documentMapper, + temporaryIndexMeta.getSettings(), temporaryIndexMeta.getRoutingNumShards(), sourceMetadata); } catch (Exception e) { logger.info("failed to build index metadata [{}]", request.index()); throw e; } logger.log(silent ? Level.DEBUG : Level.INFO, "[{}] creating index, cause [{}], templates {}, shards [{}]/[{}], mappings {}", - request.index(), request.cause(), templates.stream().map(IndexTemplateMetadata::getName).collect(toList()), - indexMetadata.getNumberOfShards(), indexMetadata.getNumberOfReplicas(), mappings.keySet()); + request.index(), request.cause(), templatesApplied, indexMetadata.getNumberOfShards(), + indexMetadata.getNumberOfReplicas(), mappings.keySet()); indexService.getIndexEventListener().beforeIndexAddedToCluster(indexMetadata.getIndex(), indexMetadata.getSettings()); @@ -379,21 +395,124 @@ public ClusterState applyCreateIndexRequest(ClusterState currentState, CreateInd }); } + /** + * Given a state and index settings calculated after applying templates, validate metadata for + * the new index, returning an {@link IndexMetadata} for the new index + */ + private IndexMetadata buildAndValidateTemporaryIndexMetadata(final ClusterState currentState, + final Settings aggregatedIndexSettings, + final CreateIndexClusterStateUpdateRequest request, + final int routingNumShards) { + + final boolean isHiddenAfterTemplates = IndexMetadata.INDEX_HIDDEN_SETTING.get(aggregatedIndexSettings); + validateDotIndex(request.index(), currentState, isHiddenAfterTemplates); + + // remove the setting it's temporary and is only relevant once we create the index + final Settings.Builder settingsBuilder = Settings.builder().put(aggregatedIndexSettings); + settingsBuilder.remove(IndexMetadata.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.getKey()); + final Settings indexSettings = settingsBuilder.build(); + + final IndexMetadata.Builder tmpImdBuilder = IndexMetadata.builder(request.index()); + tmpImdBuilder.setRoutingNumShards(routingNumShards); + tmpImdBuilder.settings(indexSettings); + + // Set up everything, now locally create the index to see that things are ok, and apply + IndexMetadata tempMetadata = tmpImdBuilder.build(); + validateActiveShardCount(request.waitForActiveShards(), tempMetadata); + + return tempMetadata; + } + + // TODO: this method can be removed in 9.0 because we will no longer use v1 templates to create indices (only v2 templates) + private ClusterState applyCreateIndexRequestWithV1Templates(final ClusterState currentState, + final CreateIndexClusterStateUpdateRequest request, + final boolean silent, + final List templates) throws Exception { + logger.info("applying create index request using v1 templates {}", templates); + + final Map mappings = Collections.unmodifiableMap(parseMappings(request.mappings(), + templates.stream().map(IndexTemplateMetadata::getMappings).collect(toList()), xContentRegistry)); + + final Settings aggregatedIndexSettings = + aggregateIndexSettings(currentState, request, MetadataIndexTemplateService.resolveSettings(templates), mappings, + null, settings, indexScopedSettings); + int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, null); + IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(currentState, aggregatedIndexSettings, request, routingNumShards); + + return applyCreateIndexWithTemporaryService(currentState, request, silent, null, tmpImd, mappings, + indexService -> resolveAndValidateAliases(request.index(), request.aliases(), + MetadataIndexTemplateService.resolveAliases(templates), currentState.metadata(), aliasValidator, + // the context is only used for validation so it's fine to pass fake values for the + // shard id and the current timestamp + xContentRegistry, indexService.newQueryShardContext(0, null, () -> 0L, null)), + templates.stream().map(IndexTemplateMetadata::getName).collect(toList())); + } + + private ClusterState applyCreateIndexRequestWithV2Template(final ClusterState currentState, + final CreateIndexClusterStateUpdateRequest request, + final boolean silent, + final String templateName) throws Exception { + logger.info("applying create index request using v2 template [{}]", templateName); + + final Map mappings = Collections.unmodifiableMap(parseMappings(request.mappings(), + MetadataIndexTemplateService.resolveMappings(currentState, templateName), xContentRegistry)); + + final Settings aggregatedIndexSettings = + aggregateIndexSettings(currentState, request, MetadataIndexTemplateService.resolveSettings(currentState, templateName), + mappings, null, settings, indexScopedSettings); + int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, null); + IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(currentState, aggregatedIndexSettings, request, routingNumShards); + + return applyCreateIndexWithTemporaryService(currentState, request, silent, null, tmpImd, mappings, + indexService -> resolveAndValidateAliases(request.index(), request.aliases(), + MetadataIndexTemplateService.resolveAliases(currentState, templateName), currentState.metadata(), aliasValidator, + // the context is only used for validation so it's fine to pass fake values for the + // shard id and the current timestamp + xContentRegistry, indexService.newQueryShardContext(0, null, () -> 0L, null)), + Collections.singletonList(templateName)); + } + + private ClusterState applyCreateIndexRequestWithExistingMetadata(final ClusterState currentState, + final CreateIndexClusterStateUpdateRequest request, + final boolean silent, + final IndexMetadata sourceMetadata) throws Exception { + logger.info("applying create index request using existing index [{}] metadata", sourceMetadata.getIndex().getName()); + + final Map mappings = Collections.unmodifiableMap(MapperService.parseMapping(xContentRegistry, request.mappings())); + + final Settings aggregatedIndexSettings = + aggregateIndexSettings(currentState, request, Settings.EMPTY, mappings, sourceMetadata, settings, indexScopedSettings); + final int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, sourceMetadata); + IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(currentState, aggregatedIndexSettings, request, routingNumShards); + + return applyCreateIndexWithTemporaryService(currentState, request, silent, sourceMetadata, tmpImd, mappings, + indexService -> resolveAndValidateAliases(request.index(), request.aliases(), Collections.emptyList(), + currentState.metadata(), aliasValidator, xContentRegistry, + // the context is only used for validation so it's fine to pass fake values for the + // shard id and the current timestamp + indexService.newQueryShardContext(0, null, () -> 0L, null)), + Collections.emptyList()); + } + /** * Parses the provided mappings json and the inheritable mappings from the templates (if any) into a map. * * The template mappings are applied in the order they are encountered in the list (clients should make sure the lower index, closer * to the head of the list, templates have the highest {@link IndexTemplateMetadata#order()}) */ - static Map parseMappings(String mappingsJson, List templates, + static Map parseMappings(String mappingsJson, List templateMappings, NamedXContentRegistry xContentRegistry) throws Exception { Map mappings = MapperService.parseMapping(xContentRegistry, mappingsJson); // apply templates, merging the mappings into the request mapping if exists - for (IndexTemplateMetadata template : templates) { - CompressedXContent mapping = template.mappings(); + for (CompressedXContent mapping : templateMappings) { if (mapping != null) { Map templateMapping = MapperService.parseMapping(xContentRegistry, mapping.string()); - assert templateMapping.size() == 1 : templateMapping; + if (templateMapping.isEmpty()) { + // Someone provided an empty '{}' for mappings, which is okay, but to avoid + // tripping the below assertion, we can safely ignore it + continue; + } + assert templateMapping.size() == 1 : "expected exactly one mapping value, got: " + templateMapping; // pre-8x templates may have a wrapper type other than _doc, so we re-wrap things here templateMapping = Collections.singletonMap(MapperService.SINGLE_MAPPING_NAME, templateMapping.values().iterator().next()); @@ -418,15 +537,12 @@ static Map parseMappings(String mappingsJson, List templates, Map mappings, + Settings templateSettings, Map mappings, @Nullable IndexMetadata sourceMetadata, Settings settings, IndexScopedSettings indexScopedSettings) { Settings.Builder indexSettingsBuilder = Settings.builder(); if (sourceMetadata == null) { - // apply templates, here, in reverse order, since first ones are better matching - for (int i = templates.size() - 1; i >= 0; i--) { - indexSettingsBuilder.put(templates.get(i).settings()); - } + indexSettingsBuilder.put(templateSettings); } // now, put the request settings, so they override templates indexSettingsBuilder.put(request.settings()); @@ -514,7 +630,7 @@ static int getIndexNumberOfRoutingShards(Settings indexSettings, @Nullable Index * @return the list of resolved aliases, with the explicitly provided aliases occurring first (having a higher priority) followed by * the ones inherited from the templates */ - static List resolveAndValidateAliases(String index, Set aliases, List templates, + static List resolveAndValidateAliases(String index, Set aliases, List> templateAliases, Metadata metadata, AliasValidator aliasValidator, NamedXContentRegistry xContentRegistry, QueryShardContext queryShardContext) { List resolvedAliases = new ArrayList<>(); @@ -530,17 +646,17 @@ static List resolveAndValidateAliases(String index, Set al } Map templatesAliases = new HashMap<>(); - for (IndexTemplateMetadata template : templates) { + for (Map templateAliasConfig : templateAliases) { // handle aliases - for (ObjectObjectCursor cursor : template.aliases()) { - AliasMetadata aliasMetadata = cursor.value; + for (Map.Entry entry : templateAliasConfig.entrySet()) { + AliasMetadata aliasMetadata = entry.getValue(); // if an alias with same name came with the create index request itself, // ignore this one taken from the index template if (aliases.contains(new Alias(aliasMetadata.alias()))) { continue; } // if an alias with same name was already processed, ignore this one - if (templatesAliases.containsKey(cursor.key)) { + if (templatesAliases.containsKey(entry.getKey())) { continue; } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 6823fa8c56f5c..c396000d863f2 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -46,6 +46,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.MapperParsingException; @@ -63,6 +64,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.function.Predicate; @@ -183,11 +185,34 @@ ClusterState addComponentTemplate(final ClusterState currentState, final boolean CompressedXContent mappings = template.template().mappings(); String stringMappings = mappings == null ? null : mappings.string(); - validateTemplate(template.template().settings(), stringMappings, indicesService, xContentRegistry); + // We may need to normalize index settings, so do that also + Settings finalSettings = template.template().settings(); + if (finalSettings != null) { + finalSettings = Settings.builder() + .put(finalSettings).normalizePrefix(IndexMetadata.INDEX_SETTING_PREFIX) + .build(); + } + + validateTemplate(finalSettings, stringMappings, indicesService, xContentRegistry); + + // Mappings in component templates don't include _doc, so update the mappings to include this single type + if (stringMappings != null) { + Map parsedMappings = MapperService.parseMapping(xContentRegistry, stringMappings); + if (parsedMappings.size() > 0) { + stringMappings = Strings.toString(XContentFactory.jsonBuilder() + .startObject() + .field(MapperService.SINGLE_MAPPING_NAME, parsedMappings) + .endObject()); + } + } + + final Template finalTemplate = new Template(finalSettings, + stringMappings == null ? null : new CompressedXContent(stringMappings), template.template().aliases()); + final ComponentTemplate finalComponentTemplate = new ComponentTemplate(finalTemplate, template.version(), template.metadata()); logger.info("adding component template [{}]", name); return ClusterState.builder(currentState) - .metadata(Metadata.builder(currentState.metadata()).put(name, template)) + .metadata(Metadata.builder(currentState.metadata()).put(name, finalComponentTemplate)) .build(); } @@ -262,7 +287,7 @@ public void onFailure(String source, Exception e) { } @Override - public ClusterState execute(ClusterState currentState) { + public ClusterState execute(ClusterState currentState) throws Exception { return addIndexTemplateV2(currentState, create, name, template); } @@ -274,8 +299,8 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS } // Package visible for testing - static ClusterState addIndexTemplateV2(final ClusterState currentState, final boolean create, - final String name, final IndexTemplateV2 template) { + ClusterState addIndexTemplateV2(final ClusterState currentState, final boolean create, + final String name, final IndexTemplateV2 template) throws Exception { if (create && currentState.metadata().templatesV2().containsKey(name)) { throw new IllegalArgumentException("index template [" + name + "] already exists"); } @@ -295,12 +320,41 @@ static ClusterState addIndexTemplateV2(final ClusterState currentState, final bo deprecationLogger.deprecated(warning); } - // TODO: validation of index template - // validateAndAddTemplate(request, templateBuilder, indicesService, xContentRegistry); + IndexTemplateV2 finalIndexTemplate = template; + Template innerTemplate = template.template(); + if (innerTemplate != null) { + // We may need to normalize index settings, so do that also + Settings finalSettings = innerTemplate.settings(); + if (finalSettings != null) { + finalSettings = Settings.builder() + .put(finalSettings).normalizePrefix(IndexMetadata.INDEX_SETTING_PREFIX) + .build(); + } + // If an inner template was specified, its mappings may need to be + // adjusted (to add _doc) and it should be validated + CompressedXContent mappings = innerTemplate.mappings(); + String stringMappings = mappings == null ? null : mappings.string(); + validateTemplate(finalSettings, stringMappings, indicesService, xContentRegistry); + + // Mappings in index templates don't include _doc, so update the mappings to include this single type + if (stringMappings != null) { + Map parsedMappings = MapperService.parseMapping(xContentRegistry, stringMappings); + if (parsedMappings.size() > 0) { + stringMappings = Strings.toString(XContentFactory.jsonBuilder() + .startObject() + .field(MapperService.SINGLE_MAPPING_NAME, parsedMappings) + .endObject()); + } + } + final Template finalTemplate = new Template(finalSettings, + stringMappings == null ? null : new CompressedXContent(stringMappings), innerTemplate.aliases()); + finalIndexTemplate = new IndexTemplateV2(template.indexPatterns(), finalTemplate, template.composedOf(), + template.priority(), template.version(), template.metadata()); + } logger.info("adding index template [{}]", name); return ClusterState.builder(currentState) - .metadata(Metadata.builder(currentState.metadata()).put(name, template)) + .metadata(Metadata.builder(currentState.metadata()).put(name, finalIndexTemplate)) .build(); } @@ -532,7 +586,7 @@ static ClusterState innerPutTemplate(final ClusterState currentState, PutRequest * @return a list of templates sorted by {@link IndexTemplateMetadata#order()} descending. * */ - public static List findTemplates(Metadata metadata, String indexName, @Nullable Boolean isHidden) { + public static List findV1Templates(Metadata metadata, String indexName, @Nullable Boolean isHidden) { final Predicate patternMatchPredicate = pattern -> Regex.simpleMatch(pattern, indexName); final List matchedTemplates = new ArrayList<>(); for (ObjectCursor cursor : metadata.templates().values()) { @@ -575,10 +629,165 @@ public static List findTemplates(Metadata metadata, Strin } } } - return matchedTemplates; + return Collections.unmodifiableList(matchedTemplates); + } + + /** + * Return the name (id) of the highest matching index template for the given index name. In + * the event that no templates are matched, {@code null} is returned. + */ + @Nullable + public static String findV2Template(Metadata metadata, String indexName, @Nullable Boolean isHidden) { + final Predicate patternMatchPredicate = pattern -> Regex.simpleMatch(pattern, indexName); + final Map matchedTemplates = new HashMap<>(); + for (Map.Entry entry : metadata.templatesV2().entrySet()) { + final String name = entry.getKey(); + final IndexTemplateV2 template = entry.getValue(); + if (isHidden == null || isHidden == Boolean.FALSE) { + final boolean matched = template.indexPatterns().stream().anyMatch(patternMatchPredicate); + if (matched) { + matchedTemplates.put(template, name); + } + } else { + assert isHidden == Boolean.TRUE; + final boolean isNotMatchAllTemplate = template.indexPatterns().stream().noneMatch(Regex::isMatchAllPattern); + if (isNotMatchAllTemplate) { + if (template.indexPatterns().stream().anyMatch(patternMatchPredicate)) { + matchedTemplates.put(template, name); + } + } + } + } + + if (matchedTemplates.size() == 0) { + return null; + } + + final List candidates = new ArrayList<>(matchedTemplates.keySet()); + CollectionUtil.timSort(candidates, Comparator.comparingLong(IndexTemplateV2::priority).reversed()); + + assert candidates.size() > 0 : "we should have returned early with no candidates"; + IndexTemplateV2 winner = candidates.get(0); + return matchedTemplates.get(winner); + } + + /** + * Resolve the given v2 template into an ordered list of mappings + */ + public static List resolveMappings(final ClusterState state, final String templateName) { + final IndexTemplateV2 template = state.metadata().templatesV2().get(templateName); + assert template != null : "attempted to resolve mappings for a template [" + templateName + + "] that did not exist in the cluster state"; + if (template == null) { + return List.of(); + } + final Map componentTemplates = state.metadata().componentTemplates(); + // TODO: more fine-grained merging of component template mappings, ie, merge fields as distint entities + List mappings = template.composedOf().stream() + .map(componentTemplates::get) + .filter(Objects::nonNull) + .map(ComponentTemplate::template) + .map(Template::mappings) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + // Add the actual index template's mappings, since it takes the highest precedence + Optional.ofNullable(template.template()) + .map(Template::mappings) + .ifPresent(mappings::add); + // When actually merging mappings, the highest precedence ones should go first, so reverse the list + Collections.reverse(mappings); + return Collections.unmodifiableList(mappings); } - private static void validateTemplate(Settings settings, String mappings, + /** + * Resolve index settings for the given list of v1 templates, templates are apply in reverse + * order since they should be provided in order of priority/order + */ + public static Settings resolveSettings(final List templates) { + Settings.Builder templateSettings = Settings.builder(); + // apply templates, here, in reverse order, since first ones are better matching + for (int i = templates.size() - 1; i >= 0; i--) { + templateSettings.put(templates.get(i).settings()); + } + return templateSettings.build(); + } + + /** + * Resolve the given v2 template into a collected {@link Settings} object + */ + public static Settings resolveSettings(final ClusterState state, final String templateName) { + final IndexTemplateV2 template = state.metadata().templatesV2().get(templateName); + assert template != null : "attempted to resolve settings for a template [" + templateName + + "] that did not exist in the cluster state"; + if (template == null) { + return Settings.EMPTY; + } + final Map componentTemplates = state.metadata().componentTemplates(); + List componentSettings = template.composedOf().stream() + .map(componentTemplates::get) + .filter(Objects::nonNull) + .map(ComponentTemplate::template) + .map(Template::settings) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + + Settings.Builder templateSettings = Settings.builder(); + componentSettings.forEach(templateSettings::put); + // Add the actual index template's settings to the end, since it takes the highest precedence. + Optional.ofNullable(template.template()) + .map(Template::settings) + .ifPresent(templateSettings::put); + return templateSettings.build(); + } + + /** + * Resolve the given v1 templates into an ordered list of aliases + */ + public static List> resolveAliases(final List templates) { + final List> resolvedAliases = new ArrayList<>(); + templates.forEach(template -> { + if (template.aliases() != null) { + Map aliasMeta = new HashMap<>(); + for (ObjectObjectCursor cursor : template.aliases()) { + aliasMeta.put(cursor.key, cursor.value); + } + resolvedAliases.add(aliasMeta); + } + }); + return Collections.unmodifiableList(resolvedAliases); + } + + /** + * Resolve the given v2 template into an ordered list of aliases + */ + public static List> resolveAliases(final ClusterState state, final String templateName) { + final IndexTemplateV2 template = state.metadata().templatesV2().get(templateName); + assert template != null : "attempted to resolve aliases for a template [" + templateName + + "] that did not exist in the cluster state"; + if (template == null) { + return List.of(); + } + final Map componentTemplates = state.metadata().componentTemplates(); + List> aliases = template.composedOf().stream() + .map(componentTemplates::get) + .filter(Objects::nonNull) + .map(ComponentTemplate::template) + .map(Template::aliases) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + + // Add the actual index template's aliases to the end if they exist + Optional.ofNullable(template.template()) + .map(Template::aliases) + .ifPresent(aliases::add); + // Aliases are applied in order, but subsequent alias configuration from the same name is + // ignored, so in order for the order to be correct, alias configuration should be in order + // of precedence (with the index template first) + Collections.reverse(aliases); + return Collections.unmodifiableList(aliases); + } + + private static void validateTemplate(Settings validateSettings, String mappings, IndicesService indicesService, NamedXContentRegistry xContentRegistry) throws Exception { // First check to see if mappings are valid XContent if (mappings != null) { @@ -589,6 +798,12 @@ private static void validateTemplate(Settings settings, String mappings, } } + // Hard to validate settings if they're non-existent, so used empty ones if none were provided + Settings settings = validateSettings; + if (settings == null) { + settings = Settings.EMPTY; + } + Index createdIndex = null; final String temporaryIndexName = UUIDs.randomBase64UUID(); try { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java index eb819bb8f3993..f86952c65fde9 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java @@ -35,6 +35,7 @@ import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.mapper.MapperService; import java.io.IOException; import java.util.HashMap; @@ -171,7 +172,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws XContentHelper.convertToMap(new BytesArray(this.mappings.uncompressed()), true, XContentType.JSON).v2(); if (uncompressedMapping.size() > 0) { builder.field(MAPPINGS.getPreferredName()); - builder.map(uncompressedMapping); + builder.map(reduceMapping(uncompressedMapping)); } } if (this.aliases != null) { @@ -184,4 +185,13 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.endObject(); return builder; } + + @SuppressWarnings("unchecked") + private static Map reduceMapping(Map mapping) { + if (mapping.size() == 1 && MapperService.SINGLE_MAPPING_NAME.equals(mapping.keySet().iterator().next())) { + return (Map) mapping.values().iterator().next(); + } else { + return mapping; + } + } } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java index af48738753a5f..2709c59863c77 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java @@ -98,7 +98,7 @@ public static Map randomAliases() { private static CompressedXContent randomMappings() { try { - return new CompressedXContent("{\"" + randomAlphaOfLength(3) + "\":\"" + randomAlphaOfLength(7) + "\"}"); + return new CompressedXContent("{\"properties\":{\"" + randomAlphaOfLength(5) + "\":{\"type\":\"keyword\"}}}"); } catch (IOException e) { fail("got an IO exception creating fake mappings: " + e); return null; @@ -107,7 +107,12 @@ private static CompressedXContent randomMappings() { private static Settings randomSettings() { return Settings.builder() - .put(randomAlphaOfLength(4), randomAlphaOfLength(10)) + .put(IndexMetadata.SETTING_BLOCKS_READ, randomBoolean()) + .put(IndexMetadata.SETTING_BLOCKS_WRITE, randomBoolean()) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, randomIntBetween(1, 10)) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, randomIntBetween(0, 5)) + .put(IndexMetadata.SETTING_BLOCKS_WRITE, randomBoolean()) + .put(IndexMetadata.SETTING_PRIORITY, randomIntBetween(0, 100000)) .build(); } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateV2Tests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateV2Tests.java index 1940442c59adb..121c10a5338bc 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateV2Tests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateV2Tests.java @@ -110,7 +110,7 @@ private static Map randomAliases() { private static CompressedXContent randomMappings() { try { - return new CompressedXContent("{\"" + randomAlphaOfLength(3) + "\":\"" + randomAlphaOfLength(7) + "\"}"); + return new CompressedXContent("{\"properties\":{\"" + randomAlphaOfLength(5) + "\":{\"type\":\"keyword\"}}}"); } catch (IOException e) { fail("got an IO exception creating fake mappings: " + e); return null; @@ -119,7 +119,12 @@ private static CompressedXContent randomMappings() { private static Settings randomSettings() { return Settings.builder() - .put(randomAlphaOfLength(4), randomAlphaOfLength(10)) + .put(IndexMetadata.SETTING_BLOCKS_READ, randomBoolean()) + .put(IndexMetadata.SETTING_BLOCKS_WRITE, randomBoolean()) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, randomIntBetween(1, 10)) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, randomIntBetween(0, 5)) + .put(IndexMetadata.SETTING_BLOCKS_WRITE, randomBoolean()) + .put(IndexMetadata.SETTING_PRIORITY, randomIntBetween(0, 100000)) .build(); } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java index 0304cfaabd2f7..0108315240f59 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -621,8 +621,8 @@ public void testParseMappingsAppliesDataFromTemplateAndRequest() throws Exceptio }); request.mappings(createMapping("mapping_from_request", "text").string()); - Map parsedMappings = MetadataCreateIndexService.parseMappings(request.mappings(), List.of(templateMetadata), - NamedXContentRegistry.EMPTY); + Map parsedMappings = MetadataCreateIndexService.parseMappings(request.mappings(), + List.of(templateMetadata.getMappings()), NamedXContentRegistry.EMPTY); assertThat(parsedMappings, hasKey("_doc")); Map doc = (Map) parsedMappings.get("_doc"); @@ -645,7 +645,7 @@ public void testAggregateSettingsAppliesSettingsFromTemplatesAndRequest() { .build(); request.settings(Settings.builder().put("request_setting", "value2").build()); - Settings aggregatedIndexSettings = aggregateIndexSettings(clusterState, request, List.of(templateMetadata), Map.of(), + Settings aggregatedIndexSettings = aggregateIndexSettings(clusterState, request, templateMetadata.settings(), Map.of(), null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS); assertThat(aggregatedIndexSettings.get("template_setting"), equalTo("value1")); @@ -677,11 +677,12 @@ public void testRequestDataHavePriorityOverTemplateData() throws Exception { request.aliases(Set.of(new Alias("alias").searchRouting("fromRequest"))); request.settings(Settings.builder().put("key1", "requestValue").build()); - Map parsedMappings = MetadataCreateIndexService.parseMappings(request.mappings(), List.of(templateMetadata), - xContentRegistry()); - List resolvedAliases = resolveAndValidateAliases(request.index(), request.aliases(), List.of(templateMetadata), + Map parsedMappings = MetadataCreateIndexService.parseMappings(request.mappings(), + List.of(templateMetadata.mappings()), xContentRegistry()); + List resolvedAliases = resolveAndValidateAliases(request.index(), request.aliases(), + MetadataIndexTemplateService.resolveAliases(List.of(templateMetadata)), Metadata.builder().build(), aliasValidator, xContentRegistry(), queryShardContext); - Settings aggregatedIndexSettings = aggregateIndexSettings(ClusterState.EMPTY_STATE, request, List.of(templateMetadata), Map.of(), + Settings aggregatedIndexSettings = aggregateIndexSettings(ClusterState.EMPTY_STATE, request, templateMetadata.settings(), Map.of(), null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS); assertThat(resolvedAliases.get(0).getSearchRouting(), equalTo("fromRequest")); @@ -695,14 +696,14 @@ public void testRequestDataHavePriorityOverTemplateData() throws Exception { } public void testDefaultSettings() { - Settings aggregatedIndexSettings = aggregateIndexSettings(ClusterState.EMPTY_STATE, request, List.of(), Map.of(), + Settings aggregatedIndexSettings = aggregateIndexSettings(ClusterState.EMPTY_STATE, request, Settings.EMPTY, Map.of(), null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS); assertThat(aggregatedIndexSettings.get(SETTING_NUMBER_OF_SHARDS), equalTo("1")); } public void testSettingsFromClusterState() { - Settings aggregatedIndexSettings = aggregateIndexSettings(ClusterState.EMPTY_STATE, request, List.of(), Map.of(), + Settings aggregatedIndexSettings = aggregateIndexSettings(ClusterState.EMPTY_STATE, request, Settings.EMPTY, Map.of(), null, Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 15).build(), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS); assertThat(aggregatedIndexSettings.get(SETTING_NUMBER_OF_SHARDS), equalTo("15")); @@ -725,9 +726,11 @@ public void testTemplateOrder() throws Exception { .settings(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 10)) .putAlias(AliasMetadata.builder("alias1").searchRouting("1").build()) )); - Settings aggregatedIndexSettings = aggregateIndexSettings(ClusterState.EMPTY_STATE, request, templates, Map.of(), + Settings aggregatedIndexSettings = aggregateIndexSettings(ClusterState.EMPTY_STATE, request, + MetadataIndexTemplateService.resolveSettings(templates), Map.of(), null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS); - List resolvedAliases = resolveAndValidateAliases(request.index(), request.aliases(), templates, + List resolvedAliases = resolveAndValidateAliases(request.index(), request.aliases(), + MetadataIndexTemplateService.resolveAliases(templates), Metadata.builder().build(), aliasValidator, xContentRegistry(), queryShardContext); assertThat(aggregatedIndexSettings.get(SETTING_NUMBER_OF_SHARDS), equalTo("12")); AliasMetadata alias = resolvedAliases.get(0); @@ -751,7 +754,7 @@ public void testAggregateIndexSettingsIgnoresTemplatesOnCreateFromSourceIndex() createClusterState("sourceIndex", 1, 0, Settings.builder().put("index.blocks.write", true).build()); - Settings aggregatedIndexSettings = aggregateIndexSettings(clusterState, request, List.of(templateMetadata), Map.of(), + Settings aggregatedIndexSettings = aggregateIndexSettings(clusterState, request, templateMetadata.settings(), Map.of(), clusterState.metadata().index("sourceIndex"), Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS); assertThat(aggregatedIndexSettings.get("templateSetting"), is(nullValue())); @@ -813,7 +816,7 @@ public void testParseMappingsWithTypedTemplateAndTypelessIndexMapping() throws E } }); - Map mappings = parseMappings("{\"_doc\":{}}", List.of(templateMetadata), xContentRegistry()); + Map mappings = parseMappings("{\"_doc\":{}}", List.of(templateMetadata.mappings()), xContentRegistry()); assertThat(mappings, Matchers.hasKey(MapperService.SINGLE_MAPPING_NAME)); } @@ -826,7 +829,7 @@ public void testParseMappingsWithTypedTemplate() throws Exception { ExceptionsHelper.reThrowIfNotNull(e); } }); - Map mappings = parseMappings("", List.of(templateMetadata), xContentRegistry()); + Map mappings = parseMappings("", List.of(templateMetadata.mappings()), xContentRegistry()); assertThat(mappings, Matchers.hasKey(MapperService.SINGLE_MAPPING_NAME)); } @@ -838,7 +841,7 @@ public void testParseMappingsWithTypelessTemplate() throws Exception { ExceptionsHelper.reThrowIfNotNull(e); } }); - Map mappings = parseMappings("", List.of(templateMetadata), xContentRegistry()); + Map mappings = parseMappings("", List.of(templateMetadata.mappings()), xContentRegistry()); assertThat(mappings, Matchers.hasKey(MapperService.SINGLE_MAPPING_NAME)); } @@ -905,7 +908,7 @@ public void testRejectWithSoftDeletesDisabled() { final IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> { request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test"); request.settings(Settings.builder().put(INDEX_SOFT_DELETES_SETTING.getKey(), false).build()); - aggregateIndexSettings(ClusterState.EMPTY_STATE, request, List.of(), Map.of(), + aggregateIndexSettings(ClusterState.EMPTY_STATE, request, Settings.EMPTY, Map.of(), null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS); }); assertThat(error.getMessage(), equalTo("Creating indices with soft-deletes disabled is no longer supported. " @@ -926,7 +929,7 @@ public void testRejectTranslogRetentionSettings() { } request.settings(settings.build()); IllegalArgumentException error = expectThrows(IllegalArgumentException.class, - () -> aggregateIndexSettings(ClusterState.EMPTY_STATE, request, List.of(), Map.of(), + () -> aggregateIndexSettings(ClusterState.EMPTY_STATE, request, Settings.EMPTY, Map.of(), null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS)); assertThat(error.getMessage(), equalTo("Translog retention settings [index.translog.retention.age] " + "and [index.translog.retention.size] are no longer supported. Please do not specify values for these settings")); @@ -942,7 +945,7 @@ public void testDeprecateTranslogRetentionSettings() { } settings.put(SETTING_INDEX_VERSION_CREATED.getKey(), VersionUtils.randomPreviousCompatibleVersion(random(), Version.V_8_0_0)); request.settings(settings.build()); - aggregateIndexSettings(ClusterState.EMPTY_STATE, request, List.of(), Map.of(), + aggregateIndexSettings(ClusterState.EMPTY_STATE, request, Settings.EMPTY, Map.of(), null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS); assertWarnings("Translog retention settings [index.translog.retention.age] " + "and [index.translog.retention.size] are deprecated and effectively ignored. They will be removed in a future version."); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java index 1006822595485..e67268b61ec41 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java @@ -28,21 +28,28 @@ import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.env.Environment; import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.indices.IndexTemplateMissingException; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.InvalidIndexTemplateException; import org.elasticsearch.test.ESSingleNodeTestCase; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.stream.Collectors; @@ -193,11 +200,11 @@ public void testFindTemplates() throws Exception { putTemplateDetail(new PutRequest("test", "foo-2").patterns(singletonList("foo-*")).order(2)); putTemplateDetail(new PutRequest("test", "bar").patterns(singletonList("bar-*")).order(between(0, 100))); final ClusterState state = client().admin().cluster().prepareState().get().getState(); - assertThat(MetadataIndexTemplateService.findTemplates(state.metadata(), "foo-1234", randomBoolean()).stream() + assertThat(MetadataIndexTemplateService.findV1Templates(state.metadata(), "foo-1234", randomBoolean()).stream() .map(IndexTemplateMetadata::name).collect(Collectors.toList()), contains("foo-2", "foo-1")); - assertThat(MetadataIndexTemplateService.findTemplates(state.metadata(), "bar-xyz", randomBoolean()).stream() + assertThat(MetadataIndexTemplateService.findV1Templates(state.metadata(), "bar-xyz", randomBoolean()).stream() .map(IndexTemplateMetadata::name).collect(Collectors.toList()), contains("bar")); - assertThat(MetadataIndexTemplateService.findTemplates(state.metadata(), "baz", randomBoolean()), empty()); + assertThat(MetadataIndexTemplateService.findV1Templates(state.metadata(), "baz", randomBoolean()), empty()); } public void testFindTemplatesWithHiddenIndices() throws Exception { @@ -212,32 +219,32 @@ public void testFindTemplatesWithHiddenIndices() throws Exception { final ClusterState state = client().admin().cluster().prepareState().get().getState(); // hidden - assertThat(MetadataIndexTemplateService.findTemplates(state.metadata(), "foo-1234", true).stream() + assertThat(MetadataIndexTemplateService.findV1Templates(state.metadata(), "foo-1234", true).stream() .map(IndexTemplateMetadata::name).collect(Collectors.toList()), containsInAnyOrder("foo-2", "foo-1")); - assertThat(MetadataIndexTemplateService.findTemplates(state.metadata(), "bar-xyz", true).stream() + assertThat(MetadataIndexTemplateService.findV1Templates(state.metadata(), "bar-xyz", true).stream() .map(IndexTemplateMetadata::name).collect(Collectors.toList()), contains("bar")); - assertThat(MetadataIndexTemplateService.findTemplates(state.metadata(), "baz", true), empty()); - assertThat(MetadataIndexTemplateService.findTemplates(state.metadata(), "sneaky1", true).stream() + assertThat(MetadataIndexTemplateService.findV1Templates(state.metadata(), "baz", true), empty()); + assertThat(MetadataIndexTemplateService.findV1Templates(state.metadata(), "sneaky1", true).stream() .map(IndexTemplateMetadata::name).collect(Collectors.toList()), contains("sneaky-hidden")); // not hidden - assertThat(MetadataIndexTemplateService.findTemplates(state.metadata(), "foo-1234", false).stream() + assertThat(MetadataIndexTemplateService.findV1Templates(state.metadata(), "foo-1234", false).stream() .map(IndexTemplateMetadata::name).collect(Collectors.toList()), containsInAnyOrder("foo-2", "foo-1", "global")); - assertThat(MetadataIndexTemplateService.findTemplates(state.metadata(), "bar-xyz", false).stream() + assertThat(MetadataIndexTemplateService.findV1Templates(state.metadata(), "bar-xyz", false).stream() .map(IndexTemplateMetadata::name).collect(Collectors.toList()), containsInAnyOrder("bar", "global")); - assertThat(MetadataIndexTemplateService.findTemplates(state.metadata(), "baz", false).stream() + assertThat(MetadataIndexTemplateService.findV1Templates(state.metadata(), "baz", false).stream() .map(IndexTemplateMetadata::name).collect(Collectors.toList()), contains("global")); - assertThat(MetadataIndexTemplateService.findTemplates(state.metadata(), "sneaky1", false).stream() + assertThat(MetadataIndexTemplateService.findV1Templates(state.metadata(), "sneaky1", false).stream() .map(IndexTemplateMetadata::name).collect(Collectors.toList()), containsInAnyOrder("global", "sneaky-hidden")); // unknown - assertThat(MetadataIndexTemplateService.findTemplates(state.metadata(), "foo-1234", null).stream() + assertThat(MetadataIndexTemplateService.findV1Templates(state.metadata(), "foo-1234", null).stream() .map(IndexTemplateMetadata::name).collect(Collectors.toList()), containsInAnyOrder("foo-2", "foo-1", "global")); - assertThat(MetadataIndexTemplateService.findTemplates(state.metadata(), "bar-xyz", null).stream() + assertThat(MetadataIndexTemplateService.findV1Templates(state.metadata(), "bar-xyz", null).stream() .map(IndexTemplateMetadata::name).collect(Collectors.toList()), containsInAnyOrder("bar", "global")); - assertThat(MetadataIndexTemplateService.findTemplates(state.metadata(), "baz", null).stream() + assertThat(MetadataIndexTemplateService.findV1Templates(state.metadata(), "baz", null).stream() .map(IndexTemplateMetadata::name).collect(Collectors.toList()), contains("global")); - assertThat(MetadataIndexTemplateService.findTemplates(state.metadata(), "sneaky1", null).stream() + assertThat(MetadataIndexTemplateService.findV1Templates(state.metadata(), "sneaky1", null).stream() .map(IndexTemplateMetadata::name).collect(Collectors.toList()), contains("sneaky-hidden")); } @@ -285,32 +292,34 @@ public void testAddComponentTemplate() throws Exception{ () -> metadataIndexTemplateService.addComponentTemplate(throwState, true, "foo2", componentTemplate4)); } - public void testAddIndexTemplateV2() { + public void testAddIndexTemplateV2() throws Exception { ClusterState state = ClusterState.EMPTY_STATE; + final MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService(); IndexTemplateV2 template = IndexTemplateV2Tests.randomInstance(); - state = MetadataIndexTemplateService.addIndexTemplateV2(state, false, "foo", template); + state = metadataIndexTemplateService.addIndexTemplateV2(state, false, "foo", template); assertNotNull(state.metadata().templatesV2().get("foo")); - assertThat(state.metadata().templatesV2().get("foo"), equalTo(template)); + assertTemplatesEqual(state.metadata().templatesV2().get("foo"), template); final ClusterState throwState = ClusterState.builder(state).build(); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, - () -> MetadataIndexTemplateService.addIndexTemplateV2(throwState, true, "foo", template)); + () -> metadataIndexTemplateService.addIndexTemplateV2(throwState, true, "foo", template)); assertThat(e.getMessage(), containsString("index template [foo] already exists")); - state = MetadataIndexTemplateService.addIndexTemplateV2(state, randomBoolean(), "bar", template); + state = metadataIndexTemplateService.addIndexTemplateV2(state, randomBoolean(), "bar", template); assertNotNull(state.metadata().templatesV2().get("bar")); } - public void testRemoveIndexTemplateV2() { + public void testRemoveIndexTemplateV2() throws Exception { IndexTemplateV2 template = IndexTemplateV2Tests.randomInstance(); + final MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService(); IndexTemplateMissingException e = expectThrows(IndexTemplateMissingException.class, () -> MetadataIndexTemplateService.innerRemoveIndexTemplateV2(ClusterState.EMPTY_STATE, "foo")); assertThat(e.getMessage(), equalTo("index_template [foo] missing")); - final ClusterState state = MetadataIndexTemplateService.addIndexTemplateV2(ClusterState.EMPTY_STATE, false, "foo", template); + final ClusterState state = metadataIndexTemplateService.addIndexTemplateV2(ClusterState.EMPTY_STATE, false, "foo", template); assertNotNull(state.metadata().templatesV2().get("foo")); - assertThat(state.metadata().templatesV2().get("foo"), equalTo(template)); + assertTemplatesEqual(state.metadata().templatesV2().get("foo"), template); ClusterState updatedState = MetadataIndexTemplateService.innerRemoveIndexTemplateV2(state, "foo"); assertNull(updatedState.metadata().templatesV2().get("foo")); @@ -319,10 +328,11 @@ public void testRemoveIndexTemplateV2() { /** * Test that if we have a pre-existing v1 template and put a v2 template that would match the same indices, we generate a warning */ - public void testPuttingV2TemplateGeneratesWarning() { + public void testPuttingV2TemplateGeneratesWarning() throws Exception { IndexTemplateMetadata v1Template = IndexTemplateMetadata.builder("v1-template") .patterns(Arrays.asList("fo*", "baz")) .build(); + final MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService(); ClusterState state = ClusterState.builder(ClusterState.EMPTY_STATE) .metadata(Metadata.builder(Metadata.EMPTY_METADATA) @@ -331,7 +341,7 @@ public void testPuttingV2TemplateGeneratesWarning() { .build(); IndexTemplateV2 v2Template = new IndexTemplateV2(Arrays.asList("foo-bar-*", "eggplant"), null, null, null, null, null); - state = MetadataIndexTemplateService.addIndexTemplateV2(state, false, "v2-template", v2Template); + state = metadataIndexTemplateService.addIndexTemplateV2(state, false, "v2-template", v2Template); assertWarnings("index template [v2-template] has index patterns [foo-bar-*, eggplant] matching patterns " + "from existing older templates [v1-template] with patterns (v1-template => [fo*, baz]); this template [v2-template] will " + @@ -344,9 +354,10 @@ public void testPuttingV2TemplateGeneratesWarning() { /** * Test that if we have a pre-existing v2 template and put a "*" v1 template, we generate a warning */ - public void testPuttingV1StarTemplateGeneratesWarning() { + public void testPuttingV1StarTemplateGeneratesWarning() throws Exception { + final MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService(); IndexTemplateV2 v2Template = new IndexTemplateV2(Arrays.asList("foo-bar-*", "eggplant"), null, null, null, null, null); - ClusterState state = MetadataIndexTemplateService.addIndexTemplateV2(ClusterState.EMPTY_STATE, false, "v2-template", v2Template); + ClusterState state = metadataIndexTemplateService.addIndexTemplateV2(ClusterState.EMPTY_STATE, false, "v2-template", v2Template); MetadataIndexTemplateService.PutRequest req = new MetadataIndexTemplateService.PutRequest("cause", "v1-template"); req.patterns(Arrays.asList("*", "baz")); @@ -363,9 +374,10 @@ public void testPuttingV1StarTemplateGeneratesWarning() { /** * Test that if we have a pre-existing v2 template and put a v1 template that would match the same indices, we generate a hard error */ - public void testPuttingV1NonStarTemplateGeneratesError() { + public void testPuttingV1NonStarTemplateGeneratesError() throws Exception { + final MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService(); IndexTemplateV2 v2Template = new IndexTemplateV2(Arrays.asList("foo-bar-*", "eggplant"), null, null, null, null, null); - ClusterState state = MetadataIndexTemplateService.addIndexTemplateV2(ClusterState.EMPTY_STATE, false, "v2-template", v2Template); + ClusterState state = metadataIndexTemplateService.addIndexTemplateV2(ClusterState.EMPTY_STATE, false, "v2-template", v2Template); MetadataIndexTemplateService.PutRequest req = new MetadataIndexTemplateService.PutRequest("cause", "v1-template"); req.patterns(Arrays.asList("egg*", "baz")); @@ -384,7 +396,9 @@ public void testPuttingV1NonStarTemplateGeneratesError() { * Test that if we have a pre-existing v1 and v2 template, and we update the existing v1 * template without changing its index patterns, a warning is generated */ - public void testUpdatingV1NonStarTemplateWithUnchangedPatternsGeneratesWarning() { + public void testUpdatingV1NonStarTemplateWithUnchangedPatternsGeneratesWarning() throws Exception { + final MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService(); + IndexTemplateMetadata v1Template = IndexTemplateMetadata.builder("v1-template") .patterns(Arrays.asList("fo*", "baz")) .build(); @@ -396,7 +410,7 @@ public void testUpdatingV1NonStarTemplateWithUnchangedPatternsGeneratesWarning() .build(); IndexTemplateV2 v2Template = new IndexTemplateV2(Arrays.asList("foo-bar-*", "eggplant"), null, null, null, null, null); - state = MetadataIndexTemplateService.addIndexTemplateV2(state, false, "v2-template", v2Template); + state = metadataIndexTemplateService.addIndexTemplateV2(state, false, "v2-template", v2Template); assertWarnings("index template [v2-template] has index patterns [foo-bar-*, eggplant] matching patterns " + "from existing older templates [v1-template] with patterns (v1-template => [fo*, baz]); this template [v2-template] will " + @@ -423,7 +437,8 @@ public void testUpdatingV1NonStarTemplateWithUnchangedPatternsGeneratesWarning() * Test that if we have a pre-existing v1 and v2 template, and we update the existing v1 * template *AND* change the index patterns that an error is generated */ - public void testUpdatingV1NonStarWithChangedPatternsTemplateGeneratesError() { + public void testUpdatingV1NonStarWithChangedPatternsTemplateGeneratesError() throws Exception { + final MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService(); IndexTemplateMetadata v1Template = IndexTemplateMetadata.builder("v1-template") .patterns(Arrays.asList("fo*", "baz")) .build(); @@ -435,7 +450,7 @@ public void testUpdatingV1NonStarWithChangedPatternsTemplateGeneratesError() { .build(); IndexTemplateV2 v2Template = new IndexTemplateV2(Arrays.asList("foo-bar-*", "eggplant"), null, null, null, null, null); - state = MetadataIndexTemplateService.addIndexTemplateV2(state, false, "v2-template", v2Template); + state = metadataIndexTemplateService.addIndexTemplateV2(state, false, "v2-template", v2Template); assertWarnings("index template [v2-template] has index patterns [foo-bar-*, eggplant] matching patterns " + "from existing older templates [v1-template] with patterns (v1-template => [fo*, baz]); this template [v2-template] will " + @@ -457,6 +472,161 @@ public void testUpdatingV1NonStarWithChangedPatternsTemplateGeneratesError() { "templates (/_index_template) instead")); } + public void testFindV2Templates() throws Exception { + final MetadataIndexTemplateService service = getMetadataIndexTemplateService(); + ClusterState state = ClusterState.EMPTY_STATE; + assertNull(MetadataIndexTemplateService.findV2Template(state.metadata(), "index", randomBoolean() ? null : randomBoolean())); + + ComponentTemplate ct = ComponentTemplateTests.randomInstance(); + state = service.addComponentTemplate(state, true, "ct", ct); + IndexTemplateV2 it = new IndexTemplateV2(List.of("i*"), null, List.of("ct"), 0L, 1L, null); + state = service.addIndexTemplateV2(state, true, "my-template", it); + IndexTemplateV2 it2 = new IndexTemplateV2(List.of("in*"), null, List.of("ct"), 10L, 2L, null); + state = service.addIndexTemplateV2(state, true, "my-template2", it2); + + String result = MetadataIndexTemplateService.findV2Template(state.metadata(), "index", randomBoolean() ? null : randomBoolean()); + + assertThat(result, equalTo("my-template2")); + } + + public void testFindV2TemplatesForHiddenIndex() throws Exception { + final MetadataIndexTemplateService service = getMetadataIndexTemplateService(); + ClusterState state = ClusterState.EMPTY_STATE; + assertNull(MetadataIndexTemplateService.findV2Template(state.metadata(), "index", true)); + + ComponentTemplate ct = ComponentTemplateTests.randomInstance(); + state = service.addComponentTemplate(state, true, "ct", ct); + IndexTemplateV2 it = new IndexTemplateV2(List.of("i*"), null, List.of("ct"), 0L, 1L, null); + state = service.addIndexTemplateV2(state, true, "my-template", it); + IndexTemplateV2 it2 = new IndexTemplateV2(List.of("*"), null, List.of("ct"), 10L, 2L, null); + state = service.addIndexTemplateV2(state, true, "my-template2", it2); + + String result = MetadataIndexTemplateService.findV2Template(state.metadata(), "index", true); + + assertThat(result, equalTo("my-template")); + } + + public void testResolveMappings() throws Exception { + final MetadataIndexTemplateService service = getMetadataIndexTemplateService(); + ClusterState state = ClusterState.EMPTY_STATE; + + ComponentTemplate ct1 = new ComponentTemplate(new Template(null, + new CompressedXContent("{\n" + + " \"properties\": {\n" + + " \"field2\": {\n" + + " \"type\": \"keyword\"\n" + + " }\n" + + " }\n" + + " }"), null), null, null); + ComponentTemplate ct2 = new ComponentTemplate(new Template(null, + new CompressedXContent("{\n" + + " \"properties\": {\n" + + " \"field2\": {\n" + + " \"type\": \"text\"\n" + + " }\n" + + " }\n" + + " }"), null), null, null); + state = service.addComponentTemplate(state, true, "ct_high", ct1); + state = service.addComponentTemplate(state, true, "ct_low", ct2); + IndexTemplateV2 it = new IndexTemplateV2(List.of("i*"), + new Template(null, + new CompressedXContent("{\n" + + " \"properties\": {\n" + + " \"field\": {\n" + + " \"type\": \"keyword\"\n" + + " }\n" + + " }\n" + + " }"), null), + List.of("ct_low", "ct_high"), 0L, 1L, null); + state = service.addIndexTemplateV2(state, true, "my-template", it); + + List mappings = MetadataIndexTemplateService.resolveMappings(state, "my-template"); + + assertNotNull(mappings); + assertThat(mappings.size(), equalTo(3)); + List> parsedMappings = mappings.stream() + .map(m -> { + try { + return MapperService.parseMapping(new NamedXContentRegistry(List.of()), m.string()); + } catch (Exception e) { + logger.error(e); + fail("failed to parse mappings: " + m.string()); + return null; + } + }) + .collect(Collectors.toList()); + + // The order of mappings should be: + // - index template + // - ct_high + // - ct_low + // Because the first elements when merging mappings have the highest precedence + assertThat(parsedMappings.get(0), + equalTo(Map.of("_doc", Map.of("properties", Map.of("field", Map.of("type", "keyword")))))); + assertThat(parsedMappings.get(1), + equalTo(Map.of("_doc", Map.of("properties", Map.of("field2", Map.of("type", "keyword")))))); + assertThat(parsedMappings.get(2), + equalTo(Map.of("_doc", Map.of("properties", Map.of("field2", Map.of("type", "text")))))); + } + + public void testResolveSettings() throws Exception { + final MetadataIndexTemplateService service = getMetadataIndexTemplateService(); + ClusterState state = ClusterState.EMPTY_STATE; + + ComponentTemplate ct1 = new ComponentTemplate(new Template(Settings.builder() + .put("number_of_replicas", 2) + .put("index.blocks.write", true) + .build(), + null, null), null, null); + ComponentTemplate ct2 = new ComponentTemplate(new Template(Settings.builder() + .put("index.number_of_replicas", 1) + .put("index.blocks.read", true) + .build(), + null, null), null, null); + state = service.addComponentTemplate(state, true, "ct_high", ct1); + state = service.addComponentTemplate(state, true, "ct_low", ct2); + IndexTemplateV2 it = new IndexTemplateV2(List.of("i*"), + new Template(Settings.builder() + .put("index.blocks.write", false) + .put("index.number_of_shards", 3) + .build(), null, null), + List.of("ct_low", "ct_high"), 0L, 1L, null); + state = service.addIndexTemplateV2(state, true, "my-template", it); + + Settings settings = MetadataIndexTemplateService.resolveSettings(state, "my-template"); + assertThat(settings.get("index.number_of_replicas"), equalTo("2")); + assertThat(settings.get("index.blocks.write"), equalTo("false")); + assertThat(settings.get("index.blocks.read"), equalTo("true")); + assertThat(settings.get("index.number_of_shards"), equalTo("3")); + } + + public void testResolveAliases() throws Exception { + final MetadataIndexTemplateService service = getMetadataIndexTemplateService(); + ClusterState state = ClusterState.EMPTY_STATE; + + Map a1 = new HashMap<>(); + a1.put("foo", AliasMetadata.newAliasMetadataBuilder("foo").build()); + Map a2 = new HashMap<>(); + a2.put("bar", AliasMetadata.newAliasMetadataBuilder("bar").build()); + Map a3 = new HashMap<>(); + a3.put("eggplant", AliasMetadata.newAliasMetadataBuilder("eggplant").build()); + a3.put("baz", AliasMetadata.newAliasMetadataBuilder("baz").build()); + + ComponentTemplate ct1 = new ComponentTemplate(new Template(null, null, a1), null, null); + ComponentTemplate ct2 = new ComponentTemplate(new Template(null, null, a2), null, null); + state = service.addComponentTemplate(state, true, "ct_high", ct1); + state = service.addComponentTemplate(state, true, "ct_low", ct2); + IndexTemplateV2 it = new IndexTemplateV2(List.of("i*"), + new Template(null, null, a3), + List.of("ct_low", "ct_high"), 0L, 1L, null); + state = service.addIndexTemplateV2(state, true, "my-template", it); + + List> resolvedAliases = MetadataIndexTemplateService.resolveAliases(state, "my-template"); + + // These should be order of precedence, so the index template (a3), then ct_high (a1), then ct_low (a2) + assertThat(resolvedAliases, equalTo(List.of(a3, a1, a2))); + } + private static List putTemplate(NamedXContentRegistry xContentRegistry, PutRequest request) { MetadataCreateIndexService createIndexService = new MetadataCreateIndexService( Settings.EMPTY, @@ -529,4 +699,55 @@ private MetadataIndexTemplateService getMetadataIndexTemplateService() { clusterService, createIndexService, new AliasValidator(), indicesService, new IndexScopedSettings(Settings.EMPTY, IndexScopedSettings.BUILT_IN_INDEX_SETTINGS), xContentRegistry()); } + + @SuppressWarnings("unchecked") + public static void assertTemplatesEqual(IndexTemplateV2 actual, IndexTemplateV2 expected) { + IndexTemplateV2 actualNoTemplate = new IndexTemplateV2(actual.indexPatterns(), null, + actual.composedOf(), actual.priority(), actual.version(), actual.metadata()); + IndexTemplateV2 expectedNoTemplate = new IndexTemplateV2(expected.indexPatterns(), null, + expected.composedOf(), expected.priority(), expected.version(), expected.metadata()); + + assertThat(actualNoTemplate, equalTo(expectedNoTemplate)); + Template actualTemplate = actual.template(); + Template expectedTemplate = expected.template(); + + assertThat("expected both templates to have either a template or no template", + Objects.nonNull(actualTemplate), equalTo(Objects.nonNull(expectedTemplate))); + + if (actualTemplate != null) { + assertThat(actualTemplate.settings(), equalTo(expectedTemplate.settings())); + assertThat(actualTemplate.aliases(), equalTo(expectedTemplate.aliases())); + assertThat("expected both templates to have either mappings or no mappings", + Objects.nonNull(actualTemplate.mappings()), equalTo(Objects.nonNull(expectedTemplate.mappings()))); + + if (actualTemplate.mappings() != null) { + Map actualMappings; + Map expectedMappings; + try (XContentParser parser = XContentType.JSON.xContent() + .createParser(new NamedXContentRegistry(List.of()), LoggingDeprecationHandler.INSTANCE, + actualTemplate.mappings().string())) { + actualMappings = parser.map(); + } catch (IOException e) { + throw new AssertionError(e); + } + try (XContentParser parser = XContentType.JSON.xContent() + .createParser(new NamedXContentRegistry(List.of()), LoggingDeprecationHandler.INSTANCE, + expectedTemplate.mappings().string())) { + expectedMappings = parser.map(); + } catch (IOException e) { + throw new AssertionError(e); + } + + if (actualMappings.size() == 1 && actualMappings.containsKey(MapperService.SINGLE_MAPPING_NAME)) { + actualMappings = (Map) actualMappings.get(MapperService.SINGLE_MAPPING_NAME); + } + + if (expectedMappings.size() == 1 && expectedMappings.containsKey(MapperService.SINGLE_MAPPING_NAME)) { + expectedMappings = (Map) expectedMappings.get(MapperService.SINGLE_MAPPING_NAME); + } + + assertThat(actualMappings, equalTo(expectedMappings)); + } + } + } }