Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use V2 index templates during index creation #54669

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,188 @@
---
"Component and index template composition":
- skip:
version: " - 7.99.99"
reason: "not yet backported"

- do:
cluster.put_component_template:
name: ct_low
body:
template:
settings:
number_of_replicas: 1
mappings:
properties:
field2:
type: text
aliases:
aliasname:
is_write_index: false

- do:
cluster.put_component_template:
name: ct_high
body:
template:
settings:
index.number_of_replicas: 0
mappings:
properties:
field2:
type: keyword
aliases:
aliasname:
is_write_index: true

- do:
indices.put_index_template:
name: my-template
body:
index_patterns: ["foo", "bar-*"]
template:
settings:
index.number_of_shards: 2
mappings:
properties:
field:
type: keyword
ignore_above: 255
aliases:
my_alias: {}
aliasname:
filter:
match_all: {}
composed_of: ["ct_low", "ct_high"]
priority: 400

- do:
indices.create:
index: bar-baz
body:
settings:
index.priority: 17
mappings:
properties:
foo:
type: keyword
aliases:
other: {}

- do:
indices.get:
index: bar-baz

- match: {bar-baz.settings.index.number_of_shards: "2"}
- match: {bar-baz.settings.index.number_of_replicas: "0"}
- match: {bar-baz.settings.index.priority: "17"}
- match: {bar-baz.mappings.properties.field: {type: keyword, ignore_above: 255}}
- match: {bar-baz.mappings.properties.field2: {type: keyword}}
- match: {bar-baz.mappings.properties.foo: {type: keyword}}
- match: {bar-baz.aliases.aliasname: {filter: {match_all: {}}}}
- match: {bar-baz.aliases.my_alias: {}}
- match: {bar-baz.aliases.other: {}}

---
"Index template priority":
- skip:
version: " - 7.99.99"
reason: "not yet backported"

- do:
indices.put_index_template:
name: my-template
body:
index_patterns: ["foo", "bar-*"]
template:
settings:
index.number_of_shards: 2
composed_of: []
priority: 400

- do:
indices.put_index_template:
name: another-template
body:
index_patterns: ["bar-*"]
template:
settings:
index.number_of_shards: 3
composed_of: []
priority: 405

- do:
indices.create:
index: bar-baz

- do:
indices.get:
index: bar-baz

- match: {bar-baz.settings.index.number_of_shards: "3"}

---
"Component template only composition":
- skip:
version: " - 7.99.99"
reason: "not yet backported"

- do:
cluster.put_component_template:
name: ct_low
body:
template:
aliases:
alias1: {}

- do:
cluster.put_component_template:
name: ct_high
body:
template:
mappings:
properties:
field:
type: keyword

- do:
indices.put_index_template:
name: my-template
body:
index_patterns: ["baz*"]
composed_of: ["ct_low", "ct_high"]

- do:
indices.create:
index: bazfoo

- do:
indices.get:
index: bazfoo

- match: {bazfoo.mappings.properties.field: {type: keyword}}
- match: {bazfoo.aliases.alias1: {}}

---
"Index template without component templates":
- skip:
version: " - 7.99.99"
reason: "not yet backported"

- do:
indices.put_index_template:
name: my-template
body:
index_patterns: ["eggplant"]
template:
settings:
number_of_shards: 3

- do:
indices.create:
index: eggplant

- do:
indices.get:
index: eggplant

- match: {eggplant.settings.index.number_of_shards: "3"}
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@
import java.util.Locale;
import java.util.regex.Pattern;

import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findTemplates;
import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findV1Templates;

public class MetadataRolloverService {
private static final Pattern INDEX_NAME_PATTERN = Pattern.compile("^.*-\\d+$");
Expand Down Expand Up @@ -161,10 +161,9 @@ static List<AliasAction> rolloverAliasToNewIndex(String oldIndex, String newInde
* the rollover alias will point to multiple indices. This causes indexing requests to be rejected.
* To avoid this, we make sure that there is no duplicated alias in index templates before creating a new index.
*/
static void checkNoDuplicatedAliasInIndexTemplate(
Metadata metadata, String rolloverIndexName, String rolloverRequestAlias,
@Nullable Boolean isHidden) {
final List<IndexTemplateMetadata> matchedTemplates = findTemplates(metadata, rolloverIndexName, isHidden);
static void checkNoDuplicatedAliasInIndexTemplate(Metadata metadata, String rolloverIndexName, String rolloverRequestAlias,
@Nullable Boolean isHidden) {
final List<IndexTemplateMetadata> matchedTemplates = findV1Templates(metadata, rolloverIndexName, isHidden);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we check v2 templates here too? If so, then I think it is ok to do this in a followup change as well.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Definitely, I'll be addressing these in followups!

for (IndexTemplateMetadata template : matchedTemplates) {
if (template.aliases().containsKey(rolloverRequestAlias)) {
throw new IllegalArgumentException(String.format(Locale.ROOT,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,15 +26,12 @@
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.block.ClusterBlockException;
import org.elasticsearch.cluster.block.ClusterBlockLevel;
import org.elasticsearch.cluster.metadata.IndexTemplateV2;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.metadata.IndexTemplateV2;
import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService;
import org.elasticsearch.cluster.metadata.Template;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.tasks.Task;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportService;
Expand Down Expand Up @@ -75,14 +72,6 @@ protected ClusterBlockException checkBlock(PutIndexTemplateV2Action.Request requ
protected void masterOperation(Task task, final PutIndexTemplateV2Action.Request request, final ClusterState state,
final ActionListener<AcknowledgedResponse> listener) {
IndexTemplateV2 indexTemplate = request.indexTemplate();
Template template = indexTemplate.template();
// Normalize the index settings if necessary
if (template.settings() != null) {
Settings.Builder settings = Settings.builder().put(template.settings()).normalizePrefix(IndexMetadata.INDEX_SETTING_PREFIX);
template = new Template(settings.build(), template.mappings(), template.aliases());
indexTemplate = new IndexTemplateV2(indexTemplate.indexPatterns(), template, indexTemplate.composedOf(),
indexTemplate.priority(), indexTemplate.version(), indexTemplate.metadata());
}
indexTemplateService.putIndexTemplateV2(request.cause(), request.create(), request.name(), request.masterNodeTimeout(),
indexTemplate, listener);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -307,7 +307,7 @@ static boolean resolvePipelines(final DocWriteRequest<?> originalRequest, final
}
} else if (indexRequest.index() != null) {
// the index does not exist yet (and this is a valid request), so match index templates to look for pipelines
List<IndexTemplateMetadata> templates = MetadataIndexTemplateService.findTemplates(metadata, indexRequest.index(), null);
List<IndexTemplateMetadata> templates = MetadataIndexTemplateService.findV1Templates(metadata, indexRequest.index(), null);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we also check v2 templates here, if default/final pipeline has been specified? If so then I think it is fine to do this in a follow up pr.

assert (templates != null);
// order of templates are highest order first
for (final IndexTemplateMetadata template : templates) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -114,11 +114,15 @@ public List<String> indexPatterns() {
return indexPatterns;
}

@Nullable
public Template template() {
return template;
}

public List<String> composedOf() {
if (componentTemplates == null) {
return List.of();
}
return componentTemplates;
}

Expand Down
Loading