Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use V2 templates when reading duplicate aliases and ingest pipelines #54902

Merged
merged 1 commit into from
Apr 8, 2020
Merged

Use V2 templates when reading duplicate aliases and ingest pipelines #54902

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 16 additions & 1 deletion ...rc/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,22 +25,25 @@
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.AliasAction;
import org.elasticsearch.cluster.metadata.AliasMetadata;
import org.elasticsearch.cluster.metadata.IndexAbstraction;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.metadata.IndexAbstraction;
import org.elasticsearch.cluster.metadata.IndexTemplateMetadata;
import org.elasticsearch.cluster.metadata.Metadata;
import org.elasticsearch.cluster.metadata.MetadataCreateIndexService;
import org.elasticsearch.cluster.metadata.MetadataIndexAliasesService;
import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.threadpool.ThreadPool;

import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.regex.Pattern;

import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findV1Templates;
import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findV2Template;

public class MetadataRolloverService {
private static final Pattern INDEX_NAME_PATTERN = Pattern.compile("^.*-\\d+$");
Expand Down Expand Up @@ -171,6 +174,18 @@ static void checkNoDuplicatedAliasInIndexTemplate(Metadata metadata, String roll
rolloverRequestAlias, template.aliases().keys(), template.name()));
}
}

final String matchedV2Template = findV2Template(metadata, rolloverIndexName, isHidden);
if (matchedV2Template != null) {
List<Map<String, AliasMetadata>> aliases = MetadataIndexTemplateService.resolveAliases(metadata, matchedV2Template);
for (Map<String, AliasMetadata> aliasConfig : aliases) {
if (aliasConfig.containsKey(rolloverRequestAlias)) {
throw new IllegalArgumentException(String.format(Locale.ROOT,
"Rollover alias [%s] can point to multiple indices, found duplicated alias [%s] in index template [%s]",
rolloverRequestAlias, aliasConfig.keySet(), matchedV2Template));
}
}
}
}

static void validate(Metadata metadata, String aliasName) {
Expand Down
40 changes: 29 additions & 11 deletions server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -306,12 +306,12 @@ static boolean resolvePipelines(final DocWriteRequest<?> originalRequest, final
indexRequest.setFinalPipeline(finalPipeline);
}
} else if (indexRequest.index() != null) {
// the index does not exist yet (and this is a valid request), so match index templates to look for pipelines
List<IndexTemplateMetadata> templates = MetadataIndexTemplateService.findV1Templates(metadata, indexRequest.index(), null);
assert (templates != null);
// order of templates are highest order first
for (final IndexTemplateMetadata template : templates) {
final Settings settings = template.settings();
// the index does not exist yet (and this is a valid request), so match index
// templates to look for pipelines in either a matching V2 template (which takes
// precedence), or if a V2 template does not match, any V1 templates
String v2Template = MetadataIndexTemplateService.findV2Template(metadata, indexRequest.index(), null);
if (v2Template != null) {
Settings settings = MetadataIndexTemplateService.resolveSettings(metadata, v2Template);
if (defaultPipeline == null && IndexSettings.DEFAULT_PIPELINE.exists(settings)) {
defaultPipeline = IndexSettings.DEFAULT_PIPELINE.get(settings);
// we can not break in case a lower-order template has a final pipeline that we need to collect
Expand All @@ -320,13 +320,31 @@ static boolean resolvePipelines(final DocWriteRequest<?> originalRequest, final
finalPipeline = IndexSettings.FINAL_PIPELINE.get(settings);
// we can not break in case a lower-order template has a default pipeline that we need to collect
}
if (defaultPipeline != null && finalPipeline != null) {
// we can break if we have already collected a default and final pipeline
break;
indexRequest.setPipeline(Objects.requireNonNullElse(defaultPipeline, IngestService.NOOP_PIPELINE_NAME));
indexRequest.setFinalPipeline(Objects.requireNonNullElse(finalPipeline, IngestService.NOOP_PIPELINE_NAME));
} else {
List<IndexTemplateMetadata> templates =
MetadataIndexTemplateService.findV1Templates(metadata, indexRequest.index(), null);
assert (templates != null);
// order of templates are highest order first
for (final IndexTemplateMetadata template : templates) {
final Settings settings = template.settings();
if (defaultPipeline == null && IndexSettings.DEFAULT_PIPELINE.exists(settings)) {
defaultPipeline = IndexSettings.DEFAULT_PIPELINE.get(settings);
// we can not break in case a lower-order template has a final pipeline that we need to collect
}
if (finalPipeline == null && IndexSettings.FINAL_PIPELINE.exists(settings)) {
finalPipeline = IndexSettings.FINAL_PIPELINE.get(settings);
// we can not break in case a lower-order template has a default pipeline that we need to collect
}
if (defaultPipeline != null && finalPipeline != null) {
// we can break if we have already collected a default and final pipeline
break;
}
}
indexRequest.setPipeline(Objects.requireNonNullElse(defaultPipeline, IngestService.NOOP_PIPELINE_NAME));
indexRequest.setFinalPipeline(Objects.requireNonNullElse(finalPipeline, IngestService.NOOP_PIPELINE_NAME));
}
indexRequest.setPipeline(Objects.requireNonNullElse(defaultPipeline, IngestService.NOOP_PIPELINE_NAME));
indexRequest.setFinalPipeline(Objects.requireNonNullElse(finalPipeline, IngestService.NOOP_PIPELINE_NAME));
}

if (requestPipeline != null) {
Expand Down
5 changes: 3 additions & 2 deletions server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -458,14 +458,15 @@ private ClusterState applyCreateIndexRequestWithV2Template(final ClusterState cu
MetadataIndexTemplateService.resolveMappings(currentState, templateName), xContentRegistry));

final Settings aggregatedIndexSettings =
aggregateIndexSettings(currentState, request, MetadataIndexTemplateService.resolveSettings(currentState, templateName),
aggregateIndexSettings(currentState, request,
MetadataIndexTemplateService.resolveSettings(currentState.metadata(), templateName),
mappings, null, settings, indexScopedSettings);
int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, null);
IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(currentState, aggregatedIndexSettings, request, routingNumShards);

return applyCreateIndexWithTemporaryService(currentState, request, silent, null, tmpImd, mappings,
indexService -> resolveAndValidateAliases(request.index(), request.aliases(),
MetadataIndexTemplateService.resolveAliases(currentState, templateName), currentState.metadata(), aliasValidator,
MetadataIndexTemplateService.resolveAliases(currentState.metadata(), templateName), currentState.metadata(), aliasValidator,
// the context is only used for validation so it's fine to pass fake values for the
// shard id and the current timestamp
xContentRegistry, indexService.newQueryShardContext(0, null, () -> 0L, null)),
Expand Down
12 changes: 6 additions & 6 deletions server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -715,14 +715,14 @@ public static Settings resolveSettings(final List<IndexTemplateMetadata> templat
/**
* Resolve the given v2 template into a collected {@link Settings} object
*/
public static Settings resolveSettings(final ClusterState state, final String templateName) {
final IndexTemplateV2 template = state.metadata().templatesV2().get(templateName);
public static Settings resolveSettings(final Metadata metadata, final String templateName) {
final IndexTemplateV2 template = metadata.templatesV2().get(templateName);
assert template != null : "attempted to resolve settings for a template [" + templateName +
"] that did not exist in the cluster state";
if (template == null) {
return Settings.EMPTY;
}
final Map<String, ComponentTemplate> componentTemplates = state.metadata().componentTemplates();
final Map<String, ComponentTemplate> componentTemplates = metadata.componentTemplates();
List<Settings> componentSettings = template.composedOf().stream()
.map(componentTemplates::get)
.filter(Objects::nonNull)
Expand Down Expand Up @@ -760,14 +760,14 @@ public static List<Map<String, AliasMetadata>> resolveAliases(final List<IndexTe
/**
* Resolve the given v2 template into an ordered list of aliases
*/
public static List<Map<String, AliasMetadata>> resolveAliases(final ClusterState state, final String templateName) {
final IndexTemplateV2 template = state.metadata().templatesV2().get(templateName);
public static List<Map<String, AliasMetadata>> resolveAliases(final Metadata metadata, final String templateName) {
final IndexTemplateV2 template = metadata.templatesV2().get(templateName);
assert template != null : "attempted to resolve aliases for a template [" + templateName +
"] that did not exist in the cluster state";
if (template == null) {
return List.of();
}
final Map<String, ComponentTemplate> componentTemplates = state.metadata().componentTemplates();
final Map<String, ComponentTemplate> componentTemplates = metadata.componentTemplates();
List<Map<String, AliasMetadata>> aliases = template.composedOf().stream()
.map(componentTemplates::get)
.filter(Objects::nonNull)
Expand Down
85 changes: 84 additions & 1 deletion ...st/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,16 @@
import org.elasticsearch.cluster.metadata.AliasAction;
import org.elasticsearch.cluster.metadata.AliasMetadata;
import org.elasticsearch.cluster.metadata.AliasValidator;
import org.elasticsearch.cluster.metadata.ComponentTemplate;
import org.elasticsearch.cluster.metadata.IndexAbstraction;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.metadata.IndexAbstraction;
import org.elasticsearch.cluster.metadata.IndexTemplateMetadata;
import org.elasticsearch.cluster.metadata.IndexTemplateV2;
import org.elasticsearch.cluster.metadata.Metadata;
import org.elasticsearch.cluster.metadata.MetadataCreateIndexService;
import org.elasticsearch.cluster.metadata.MetadataIndexAliasesService;
import org.elasticsearch.cluster.metadata.Template;
import org.elasticsearch.cluster.routing.allocation.AllocationService;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.CheckedFunction;
Expand All @@ -53,8 +56,10 @@

import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;

import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
Expand Down Expand Up @@ -271,6 +276,41 @@ public void testRejectDuplicateAlias() {
assertThat(ex.getMessage(), containsString("index template [test-template]"));
}

public void testRejectDuplicateAliasV2() {
Map<String, AliasMetadata> aliases = new HashMap<>();
aliases.put("foo-write", AliasMetadata.builder("foo-write").build());
aliases.put("bar-write", AliasMetadata.builder("bar-write").writeIndex(randomBoolean()).build());
final IndexTemplateV2 template = new IndexTemplateV2(Arrays.asList("foo-*", "bar-*"), new Template(null, null, aliases),
null, null, null, null);

final Metadata metadata = Metadata.builder().put(createMetadata(randomAlphaOfLengthBetween(5, 7)), false)
.put("test-template", template).build();
String indexName = randomFrom("foo-123", "bar-xyz");
String aliasName = randomFrom("foo-write", "bar-write");
final IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
() -> MetadataRolloverService.checkNoDuplicatedAliasInIndexTemplate(metadata, indexName, aliasName, randomBoolean()));
assertThat(ex.getMessage(), containsString("index template [test-template]"));
}

public void testRejectDuplicateAliasV2UsingComponentTemplates() {
Map<String, AliasMetadata> aliases = new HashMap<>();
aliases.put("foo-write", AliasMetadata.builder("foo-write").build());
aliases.put("bar-write", AliasMetadata.builder("bar-write").writeIndex(randomBoolean()).build());
final ComponentTemplate ct = new ComponentTemplate(new Template(null, null, aliases), null, null);
final IndexTemplateV2 template = new IndexTemplateV2(Arrays.asList("foo-*", "bar-*"), null,
Collections.singletonList("ct"), null, null, null);

final Metadata metadata = Metadata.builder().put(createMetadata(randomAlphaOfLengthBetween(5, 7)), false)
.put("ct", ct)
.put("test-template", template)
.build();
String indexName = randomFrom("foo-123", "bar-xyz");
String aliasName = randomFrom("foo-write", "bar-write");
final IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
() -> MetadataRolloverService.checkNoDuplicatedAliasInIndexTemplate(metadata, indexName, aliasName, randomBoolean()));
assertThat(ex.getMessage(), containsString("index template [test-template]"));
}

public void testHiddenAffectsResolvedTemplates() {
final IndexTemplateMetadata template = IndexTemplateMetadata.builder("test-template")
.patterns(Collections.singletonList("*"))
Expand All @@ -288,6 +328,49 @@ public void testHiddenAffectsResolvedTemplates() {
assertThat(ex.getMessage(), containsString("index template [test-template]"));
}

public void testHiddenAffectsResolvedV2Templates() {
Map<String, AliasMetadata> aliases = new HashMap<>();
aliases.put("foo-write", AliasMetadata.builder("foo-write").build());
aliases.put("bar-write", AliasMetadata.builder("bar-write").writeIndex(randomBoolean()).build());
final IndexTemplateV2 template = new IndexTemplateV2(Collections.singletonList("*"), new Template(null, null, aliases),
null, null, null, null);

final Metadata metadata = Metadata.builder().put(createMetadata(randomAlphaOfLengthBetween(5, 7)), false)
.put("test-template", template).build();
String indexName = randomFrom("foo-123", "bar-xyz");
String aliasName = randomFrom("foo-write", "bar-write");

// hidden shouldn't throw
MetadataRolloverService.checkNoDuplicatedAliasInIndexTemplate(metadata, indexName, aliasName, Boolean.TRUE);
// not hidden will throw
final IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () ->
MetadataRolloverService.checkNoDuplicatedAliasInIndexTemplate(metadata, indexName, aliasName, randomFrom(Boolean.FALSE, null)));
assertThat(ex.getMessage(), containsString("index template [test-template]"));
}

public void testHiddenAffectsResolvedV2ComponentTemplates() {
Map<String, AliasMetadata> aliases = new HashMap<>();
aliases.put("foo-write", AliasMetadata.builder("foo-write").build());
aliases.put("bar-write", AliasMetadata.builder("bar-write").writeIndex(randomBoolean()).build());
final ComponentTemplate ct = new ComponentTemplate(new Template(null, null, aliases), null, null);
final IndexTemplateV2 template = new IndexTemplateV2(Collections.singletonList("*"), null,
Collections.singletonList("ct"), null, null, null);

final Metadata metadata = Metadata.builder().put(createMetadata(randomAlphaOfLengthBetween(5, 7)), false)
.put("ct", ct)
.put("test-template", template)
.build();
String indexName = randomFrom("foo-123", "bar-xyz");
String aliasName = randomFrom("foo-write", "bar-write");

// hidden shouldn't throw
MetadataRolloverService.checkNoDuplicatedAliasInIndexTemplate(metadata, indexName, aliasName, Boolean.TRUE);
// not hidden will throw
final IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () ->
MetadataRolloverService.checkNoDuplicatedAliasInIndexTemplate(metadata, indexName, aliasName, randomFrom(Boolean.FALSE, null)));
assertThat(ex.getMessage(), containsString("index template [test-template]"));
}

/**
* Test the main rolloverClusterState method. This does not validate every detail to depth, rather focuses on observing that each
* parameter is used for the purpose intended.
Expand Down
32 changes: 32 additions & 0 deletions server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,9 @@
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.metadata.IndexTemplateMetadata;
import org.elasticsearch.cluster.metadata.IndexTemplateV2;
import org.elasticsearch.cluster.metadata.Metadata;
import org.elasticsearch.cluster.metadata.Template;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.node.DiscoveryNodes;
import org.elasticsearch.cluster.service.ClusterService;
Expand Down Expand Up @@ -573,6 +575,36 @@ public void testFindDefaultPipelineFromTemplateMatch(){
completionHandler.capture(), any());
}

public void testFindDefaultPipelineFromV2TemplateMatch() {
Exception exception = new Exception("fake exception");

IndexTemplateV2 t1 = new IndexTemplateV2(Collections.singletonList("missing_*"),
new Template(Settings.builder().put(IndexSettings.DEFAULT_PIPELINE.getKey(), "pipeline2").build(), null, null),
null, null, null, null);

ClusterState state = clusterService.state();
Metadata metadata = Metadata.builder()
.put("my-template", t1)
.build();
when(state.metadata()).thenReturn(metadata);
when(state.getMetadata()).thenReturn(metadata);

IndexRequest indexRequest = new IndexRequest("missing_index").id("id");
indexRequest.source(Collections.emptyMap());
AtomicBoolean responseCalled = new AtomicBoolean(false);
AtomicBoolean failureCalled = new AtomicBoolean(false);
ActionTestUtils.execute(singleItemBulkWriteAction, null, indexRequest, ActionListener.wrap(
response -> responseCalled.set(true),
e -> {
assertThat(e, sameInstance(exception));
failureCalled.set(true);
}));

assertEquals("pipeline2", indexRequest.getPipeline());
verify(ingestService).executeBulkRequest(eq(1), bulkDocsItr.capture(), failureHandler.capture(),
completionHandler.capture(), any());
}

private void validateDefaultPipeline(IndexRequest indexRequest) {
Exception exception = new Exception("fake exception");
indexRequest.source(Collections.emptyMap());
Expand Down
Loading