Skip to content

Commit

Permalink
Add warnings/errors when V2 templates would match same indices… (#54367)
Browse files Browse the repository at this point in the history 
* Add warnings/errors when V2 templates would match same indices as V1

With the introduction of V2 index templates, we want to warn users that templates they put in place
might not take precedence (because v2 templates are going to "win"). This adds this validation at
`PUT` time for both V1 and V2 templates with the following rules:

** When creating or updating a V2 template
- If the v2 template would match indices for an existing v1 template or templates, provide a
warning (through the deprecation logging so it shows up to the client) as well as logging the
warning

The v2 warning looks like:

```
index template [my-v2-template] has index patterns [foo-*] matching patterns from existing older
templates [old-v1-template,match-all-template] with patterns (old-v1-template =>
[foo*],match-all-template => [*]); this template [my-v2-template] will take
precedence during new index creation
```

** When creating a V1 template
- If the v1 template is for index patterns of `"*"` and a v2 template exists, warn that the v2
template may take precedence
- If the v1 template is for index patterns other than all indices, and a v2 template exists that
would match, throw an error preventing creation of the v1 template

** When updating a V1 template (without changing its existing `index_patterns`!)
- If the v1 template is for index patterns that would match an existing v2 template, warn that the
v2 template may take precedence.

The v1 warning looks like:

```
template [my-v1-template] has index patterns [*] matching patterns from existing index templates
[existing-v2-template] with patterns (existing-v2-template => [foo*]); this template [my-v1-template] may be ignored in favor of an index template at index creation time
```

And the v1 error looks like:

```
template [my-v1-template] has index patterns [foo*] matching patterns from existing index templates
[existing-v2-template] with patterns (existing-v2-template => [f*]), use index templates (/_index_template) instead
```

Relates to #53101

* Remove v2 index and component templates when cleaning up tests

* Finish half-finished comment sentence

* Guard template removal and ignore for earlier versions of ES

Co-authored-by: Elastic Machine <[email protected]>
  • Loading branch information
@dakrone @elasticmachine
dakrone and elasticmachine authored Mar 30, 2020
1 parent ef45e06 commit e502b89
Show file tree
Hide file tree
Showing 3 changed files with 305 additions and 29 deletions.
173 changes: 144 additions & 29 deletions server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,12 @@
package org.elasticsearch.cluster.metadata;

import com.carrotsearch.hppc.cursors.ObjectCursor;
import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.lucene.util.CollectionUtil;
import org.apache.lucene.util.automaton.Automaton;
import org.apache.lucene.util.automaton.Operations;
import org.elasticsearch.Version;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.indices.alias.Alias;
Expand All @@ -37,6 +40,7 @@
import org.elasticsearch.common.ValidationException;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.logging.DeprecationLogger;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.settings.IndexScopedSettings;
import org.elasticsearch.common.settings.Settings;
Expand All @@ -54,12 +58,15 @@
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.function.Predicate;
import java.util.stream.Collectors;

import static org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason.NO_LONGER_ASSIGNED;

Expand All @@ -69,6 +76,7 @@
public class MetaDataIndexTemplateService {

private static final Logger logger = LogManager.getLogger(MetaDataIndexTemplateService.class);
private static final DeprecationLogger deprecationLogger = new DeprecationLogger(logger);

private final ClusterService clusterService;
private final AliasValidator aliasValidator;
Expand Down Expand Up @@ -272,6 +280,21 @@ static ClusterState addIndexTemplateV2(final ClusterState currentState, final bo
throw new IllegalArgumentException("index template [" + name + "] already exists");
}

Map<String, List<String>> overlaps = findConflictingV1Templates(currentState, name, template.indexPatterns());
if (overlaps.size() > 0) {
String warning = String.format(Locale.ROOT, "index template [%s] has index patterns %s matching patterns from " +
"existing older templates [%s] with patterns (%s); this template [%s] will take precedence during new index creation",
name,
template.indexPatterns(),
Strings.collectionToCommaDelimitedString(overlaps.keySet()),
overlaps.entrySet().stream()
.map(e -> e.getKey() + " => " + e.getValue())
.collect(Collectors.joining(",")),
name);
logger.warn(warning);
deprecationLogger.deprecated(warning);
}

// TODO: validation of index template
// validateAndAddTemplate(request, templateBuilder, indicesService, xContentRegistry);

Expand All @@ -281,6 +304,48 @@ static ClusterState addIndexTemplateV2(final ClusterState currentState, final bo
.build();
}

/**
* Return a map of v1 template names to their index patterns for v1 templates that would overlap
* with the given v2 template's index patterns.
*/
static Map<String, List<String>> findConflictingV1Templates(final ClusterState state, final String candidateName,
final List<String> indexPatterns) {
Automaton v2automaton = Regex.simpleMatchToAutomaton(indexPatterns.toArray(Strings.EMPTY_ARRAY));
Map<String, List<String>> overlappingTemplates = new HashMap<>();
for (ObjectObjectCursor<String, IndexTemplateMetaData> cursor : state.metaData().templates()) {
String name = cursor.key;
IndexTemplateMetaData template = cursor.value;
Automaton v1automaton = Regex.simpleMatchToAutomaton(template.patterns().toArray(Strings.EMPTY_ARRAY));
if (Operations.isEmpty(Operations.intersection(v2automaton, v1automaton)) == false) {
logger.debug("index template {} and old template {} would overlap: {} <=> {}",
candidateName, name, indexPatterns, template.patterns());
overlappingTemplates.put(name, template.patterns());
}
}
return overlappingTemplates;
}

/**
* Return a map of v2 template names to their index patterns for v2 templates that would overlap
* with the given v1 template's index patterns.
*/
static Map<String, List<String>> findConflictingV2Templates(final ClusterState state, final String candidateName,
final List<String> indexPatterns) {
Automaton v1automaton = Regex.simpleMatchToAutomaton(indexPatterns.toArray(Strings.EMPTY_ARRAY));
Map<String, List<String>> overlappingTemplates = new HashMap<>();
for (Map.Entry<String, IndexTemplateV2> entry : state.metaData().templatesV2().entrySet()) {
String name = entry.getKey();
IndexTemplateV2 template = entry.getValue();
Automaton v2automaton = Regex.simpleMatchToAutomaton(template.indexPatterns().toArray(Strings.EMPTY_ARRAY));
if (Operations.isEmpty(Operations.intersection(v1automaton, v2automaton)) == false) {
logger.debug("old template {} and index template {} would overlap: {} <=> {}",
candidateName, name, indexPatterns, template.indexPatterns());
overlappingTemplates.put(name, template.indexPatterns());
}
}
return overlappingTemplates;
}

/**
* Remove the given index template from the cluster state. The index template name
* supports simple regex wildcards for removing multiple index templates at a time.
Expand Down Expand Up @@ -374,36 +439,8 @@ public void onFailure(String source, Exception e) {

@Override
public ClusterState execute(ClusterState currentState) throws Exception {
if (request.create && currentState.metaData().templates().containsKey(request.name)) {
throw new IllegalArgumentException("index_template [" + request.name + "] already exists");
}

templateBuilder.order(request.order);
templateBuilder.version(request.version);
templateBuilder.patterns(request.indexPatterns);
templateBuilder.settings(request.settings);

if (request.mappings != null) {
try {
templateBuilder.putMapping(MapperService.SINGLE_MAPPING_NAME, request.mappings);
} catch (Exception e) {
throw new MapperParsingException("Failed to parse mapping: {}", e, request.mappings);
}
}

validateTemplate(request.settings, request.mappings, indicesService, xContentRegistry);

for (Alias alias : request.aliases) {
AliasMetaData aliasMetaData = AliasMetaData.builder(alias.name()).filter(alias.filter())
.indexRouting(alias.indexRouting()).searchRouting(alias.searchRouting()).build();
templateBuilder.putAlias(aliasMetaData);
}
IndexTemplateMetaData template = templateBuilder.build();

MetaData.Builder builder = MetaData.builder(currentState.metaData()).put(template);

logger.info("adding template [{}] for index patterns {}", request.name, request.indexPatterns);
return ClusterState.builder(currentState).metaData(builder).build();
return innerPutTemplate(currentState, request, templateBuilder);
}

@Override
Expand All @@ -413,6 +450,75 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS
});
}

// Package visible for testing
static ClusterState innerPutTemplate(final ClusterState currentState, PutRequest request,
IndexTemplateMetaData.Builder templateBuilder) {
// Flag for whether this is updating an existing template or adding a new one
// TODO: in 8.0+, only allow updating index templates, not adding new ones
boolean isUpdate = currentState.metaData().templates().containsKey(request.name);
if (request.create && isUpdate) {
throw new IllegalArgumentException("index_template [" + request.name + "] already exists");
}
boolean isUpdateAndPatternsAreUnchanged = isUpdate &&
currentState.metaData().templates().get(request.name).patterns().equals(request.indexPatterns);

Map<String, List<String>> overlaps = findConflictingV2Templates(currentState, request.name, request.indexPatterns);
if (overlaps.size() > 0) {
// Be less strict (just a warning) if we're updating an existing template or this is a match-all template
if (isUpdateAndPatternsAreUnchanged || request.indexPatterns.stream().anyMatch(Regex::isMatchAllPattern)) {
String warning = String.format(Locale.ROOT, "template [%s] has index patterns %s matching patterns" +
" from existing index templates [%s] with patterns (%s); this template [%s] may be ignored in favor of " +
"an index template at index creation time",
request.name,
request.indexPatterns,
Strings.collectionToCommaDelimitedString(overlaps.keySet()),
overlaps.entrySet().stream()
.map(e -> e.getKey() + " => " + e.getValue())
.collect(Collectors.joining(",")),
request.name);
logger.warn(warning);
deprecationLogger.deprecated(warning);
} else {
// Otherwise, this is a hard error, the user should use V2 index templates instead
String error = String.format(Locale.ROOT, "template [%s] has index patterns %s matching patterns" +
" from existing index templates [%s] with patterns (%s), use index templates (/_index_template) instead",
request.name,
request.indexPatterns,
Strings.collectionToCommaDelimitedString(overlaps.keySet()),
overlaps.entrySet().stream()
.map(e -> e.getKey() + " => " + e.getValue())
.collect(Collectors.joining(",")));
logger.error(error);
throw new IllegalArgumentException(error);
}
}

templateBuilder.order(request.order);
templateBuilder.version(request.version);
templateBuilder.patterns(request.indexPatterns);
templateBuilder.settings(request.settings);

if (request.mappings != null) {
try {
templateBuilder.putMapping(MapperService.SINGLE_MAPPING_NAME, request.mappings);
} catch (Exception e) {
throw new MapperParsingException("Failed to parse mapping: {}", e, request.mappings);
}
}

for (Alias alias : request.aliases) {
AliasMetaData aliasMetaData = AliasMetaData.builder(alias.name()).filter(alias.filter())
.indexRouting(alias.indexRouting()).searchRouting(alias.searchRouting()).build();
templateBuilder.putAlias(aliasMetaData);
}
IndexTemplateMetaData template = templateBuilder.build();

MetaData.Builder builder = MetaData.builder(currentState.metaData()).put(template);

logger.info("adding template [{}] for index patterns {}", request.name, request.indexPatterns);
return ClusterState.builder(currentState).metaData(builder).build();
}

/**
* Finds index templates whose index pattern matched with the given index name. In the case of
* hidden indices, a template with a match all pattern or global template will not be returned.
Expand Down Expand Up @@ -474,6 +580,15 @@ public static List<IndexTemplateMetaData> findTemplates(MetaData metaData, Strin

private static void validateTemplate(Settings settings, String mappings,
IndicesService indicesService, NamedXContentRegistry xContentRegistry) throws Exception {
// First check to see if mappings are valid XContent
if (mappings != null) {
try {
new CompressedXContent(mappings);
} catch (Exception e) {
throw new MapperParsingException("Failed to parse mapping: {}", e, mappings);
}
}

Index createdIndex = null;
final String temporaryIndexName = UUIDs.randomBase64UUID();
try {
Expand Down
Loading

0 comments on commit e502b89

Please sign in to comment.