From 5393480e3c77924e1b58541a8df0a16381c26573 Mon Sep 17 00:00:00 2001 From: Michael Peterson Date: Tue, 25 Apr 2023 14:56:21 -0400 Subject: [PATCH 01/11] Allow multiple field names/patterns for (path_)(un)match (#66364) Arrays of patterns are now allowed for dynamic_templates in the match, unmatch, path_match and path_unmatch fields. DynamicTemplate has been modified to support List for these fields. The patterns can be either simple wildcards or regex. As with previous functionality, mixing of wildcards and regex will not throw an error, but will not work as expected at mapping time. One new error pathway was added: if a user specifies a list of non-strings for one of these pattern fields (e.g., "match": [10, false]) a MapperParserException will be thrown. Closes #66364. --- .../mapping/dynamic/templates.asciidoc | 95 +++- .../DataStreamIndexSettingsProvider.java | 10 +- .../DataStreamIndexSettingsProviderTests.java | 42 ++ .../index/mapper/DynamicTemplate.java | 175 ++++-- .../index/mapper/DynamicTemplatesTests.java | 510 +++++++++++++++++- 5 files changed, 774 insertions(+), 58 deletions(-) diff --git a/docs/reference/mapping/dynamic/templates.asciidoc b/docs/reference/mapping/dynamic/templates.asciidoc index 2b4491cac5435..89029cadedd0e 100644 --- a/docs/reference/mapping/dynamic/templates.asciidoc +++ b/docs/reference/mapping/dynamic/templates.asciidoc @@ -183,8 +183,8 @@ PUT my-index-000001/_doc/1 [[match-unmatch]] ==== `match` and `unmatch` -The `match` parameter uses a pattern to match on the field name, while -`unmatch` uses a pattern to exclude fields matched by `match`. +The `match` parameter uses one or more patterns to match on the field name, while +`unmatch` uses one or more patterns to exclude fields matched by `match`. The `match_pattern` parameter adjusts the behavior of the `match` parameter to support full Java regular expressions matching on the field name @@ -231,6 +231,48 @@ PUT my-index-000001/_doc/1 <1> The `long_num` field is mapped as a `long`. <2> The `long_text` field uses the default `string` mapping. + +You can specify a list of patterns using a JSON array for either the +`match` or `unmatch` fields. + +The next example matches all fields whose name starts with +`one` or has `two` in the name, except for those which end with `ip` or ``) +and maps them as `keyword` fields: + +[source,console] +-------------------------------------------------- +PUT my-index-000001 +{ + "mappings": { + "dynamic_templates": [ + { + "ip_fields": { + "match": ["ip_*", "*_ip"], + "unmatch": ["one*", "*two"], + "mapping": { + "type": "ip" + } + } + } + ] + } +} + +PUT my-index/_doc/1 +{ + "one_ip": "will not match", <1> + "ip_two": "will not match", <2> + "three_ip": "12.12.12.12", <3> + "ip_four": "13.13.13.13" <4> +} +-------------------------------------------------- + +<1> The `one_ip` field is unmatched, so uses the default mapping of `text`. +<1> The `ip_two` field is unmatched, so uses the default mapping of `text`. +<3> The `three_ip` field is mapped as `ip`. +<4> The `ip_four` field is mapped as `ip`. + + [[path-match-unmatch]] ==== `path_match` and `path_unmatch` @@ -271,6 +313,55 @@ PUT my-index-000001/_doc/1 } -------------------------------------------------- +And the following example uses an array of patterns for both `path_match` +and `path_unmatch`. + +The values of any fields in the `name` object or the `user.name` object +are copied to the top-level `full_name` field, except for the `middle` +and `midinitial` fields: + +[source,console] +-------------------------------------------------- +PUT my-index-000001 +{ + "mappings": { + "dynamic_templates": [ + { + "full_name": { + "path_match": ["name.*", "user.name.*"], + "path_unmatch": ["*.middle", "*.midinitial"], + "mapping": { + "type": "text", + "copy_to": "full_name" + } + } + } + ] + } +} + +PUT my-index-000001/_doc/1 +{ + "name": { + "first": "John", + "middle": "Winston", + "last": "Lennon" + } +} + +PUT my-index-000001/_doc/2 +{ + "user": { + "name": { + "first": "Jane", + "midinitial": "M", + "last": "Salazar" + } + } +} +-------------------------------------------------- + + Note that the `path_match` and `path_unmatch` parameters match on object paths in addition to leaf fields. As an example, indexing the following document will result in an error because the `path_match` setting also matches the object diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java index 92833d48df4e3..31e59619f6163 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java @@ -178,10 +178,12 @@ private List findRoutingPaths(String indexName, Settings allSettings, Li } MappingParserContext parserContext = mapperService.parserContext(); - var mapper = parserContext.typeParser(mappingSnippetType) - .parse(template.pathMatch(), mappingSnippet, parserContext) - .build(MapperBuilderContext.root(false)); - extractPath(routingPaths, mapper); + for (String pathMatch : template.pathMatch()) { + var mapper = parserContext.typeParser(mappingSnippetType) + .parse(pathMatch, mappingSnippet, parserContext) + .build(MapperBuilderContext.root(false)); + extractPath(routingPaths, mapper); + } } return routingPaths; } catch (IOException e) { diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java index 833dfab9100a6..ea8dce722fd90 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java @@ -429,6 +429,48 @@ public void testGenerateRoutingPathFromDynamicTemplate() throws Exception { assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("host.id", "prometheus.labels.*")); } + public void testGenerateRoutingPathFromDynamicTemplateWithMultiplePathMatchEntries() throws Exception { + Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS); + TimeValue lookAheadTime = TimeValue.timeValueHours(2); // default + String mapping = """ + { + "_doc": { + "dynamic_templates": [ + { + "labels": { + "path_match": ["xprometheus.labels.*", "yprometheus.labels.*"], + "mapping": { + "type": "keyword", + "time_series_dimension": true + } + } + } + ], + "properties": { + "host": { + "properties": { + "id": { + "type": "keyword", + "time_series_dimension": true + } + } + }, + "another_field": { + "type": "keyword" + } + } + } + } + """; + Settings result = generateTsdbSettings(mapping, now); + assertThat(result.size(), equalTo(3)); + assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(lookAheadTime.getMillis()))); + assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(lookAheadTime.getMillis()))); + assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("host.id", "xprometheus.labels.*")); + List routingPathList = IndexMetadata.INDEX_ROUTING_PATH.get(result); + assertEquals(2, routingPathList.size()); + } + public void testGenerateRoutingPathFromDynamicTemplate_templateWithNoPathMatch() throws Exception { Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS); TimeValue lookAheadTime = TimeValue.timeValueHours(2); // default diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java b/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java index 34d5cc92e1161..9db9630d1619e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java @@ -8,6 +8,7 @@ package org.elasticsearch.index.mapper; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; @@ -15,6 +16,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Locale; @@ -141,10 +143,10 @@ public final String toString() { @SuppressWarnings("unchecked") static DynamicTemplate parse(String name, Map conf) throws MapperParsingException { - String match = null; - String pathMatch = null; - String unmatch = null; - String pathUnmatch = null; + List match = null; + List pathMatch = null; + List unmatch = null; + List pathUnmatch = null; Map mapping = null; boolean runtime = false; String matchMappingType = null; @@ -153,13 +155,17 @@ static DynamicTemplate parse(String name, Map conf) throws Mappe for (Map.Entry entry : conf.entrySet()) { String propName = entry.getKey(); if ("match".equals(propName)) { - match = entry.getValue().toString(); + match = new ArrayList<>(); + addEntriesToPatternList(match, propName, entry); } else if ("path_match".equals(propName)) { - pathMatch = entry.getValue().toString(); + pathMatch = new ArrayList<>(); + addEntriesToPatternList(pathMatch, propName, entry); } else if ("unmatch".equals(propName)) { - unmatch = entry.getValue().toString(); + unmatch = new ArrayList<>(); + addEntriesToPatternList(unmatch, propName, entry); } else if ("path_unmatch".equals(propName)) { - pathUnmatch = entry.getValue().toString(); + pathUnmatch = new ArrayList<>(); + addEntriesToPatternList(pathUnmatch, propName, entry); } else if ("match_mapping_type".equals(propName)) { matchMappingType = entry.getValue().toString(); } else if ("match_pattern".equals(propName)) { @@ -217,30 +223,65 @@ static DynamicTemplate parse(String name, Map conf) throws Mappe } final MatchType matchType = MatchType.fromString(matchPattern); + ensurePatternsDoNotMatchEmptyString( + name, + matchType, + List.of( + match != null ? match : Collections.emptyList(), + unmatch != null ? unmatch : Collections.emptyList(), + pathMatch != null ? pathMatch : Collections.emptyList(), + pathUnmatch != null ? pathUnmatch : Collections.emptyList() + ) + ); - // Validate that the pattern - for (String regex : new String[] { pathMatch, match, pathUnmatch, unmatch }) { - if (regex == null) { - continue; - } - try { - matchType.matches(regex, ""); - } catch (IllegalArgumentException e) { - throw new IllegalArgumentException( - "Pattern [" + regex + "] of type [" + matchType + "] is invalid. Cannot create dynamic template [" + name + "].", - e - ); + return new DynamicTemplate(name, pathMatch, pathUnmatch, match, unmatch, xContentFieldTypes, matchType, mapping, runtime); + } + + private static void addEntriesToPatternList(List matchList, String propName, Map.Entry entry) { + if (entry.getValue() instanceof String s) { + matchList.add(s); + } else if (entry.getValue() instanceof List ls) { + for (Object o : ls) { + if (o instanceof String s) { + matchList.add(s); + } else { + throw new MapperParsingException( + Strings.format("[%s] values must either be a string or list of strings, but was [%s]", propName, entry.getValue()) + ); + } } + } else { + throw new MapperParsingException( + Strings.format("[%s] values must either be a string or list of strings, but was [%s]", propName, entry.getValue()) + ); } + } - return new DynamicTemplate(name, pathMatch, pathUnmatch, match, unmatch, xContentFieldTypes, matchType, mapping, runtime); + private static void ensurePatternsDoNotMatchEmptyString(String templateName, MatchType matchType, List> patterns) { + for (List patternList : patterns) { + for (String regex : patternList) { + try { + matchType.matches(regex, ""); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException( + Strings.format( + "Pattern [%s] of type [%s] is invalid. Cannot create dynamic template [%s].", + regex, + matchType, + templateName + ), + e + ); + } + } + } } private final String name; - private final String pathMatch; - private final String pathUnmatch; - private final String match; - private final String unmatch; + private final List pathMatch; + private final List pathUnmatch; + private final List match; + private final List unmatch; private final MatchType matchType; private final XContentFieldType[] xContentFieldTypes; private final Map mapping; @@ -248,10 +289,10 @@ static DynamicTemplate parse(String name, Map conf) throws Mappe private DynamicTemplate( String name, - String pathMatch, - String pathUnmatch, - String match, - String unmatch, + List pathMatch, + List pathUnmatch, + List match, + List unmatch, XContentFieldType[] xContentFieldTypes, MatchType matchType, Map mapping, @@ -272,11 +313,11 @@ public String name() { return this.name; } - public String pathMatch() { + public List pathMatch() { return pathMatch; } - public String match() { + public List match() { return match; } @@ -285,17 +326,41 @@ public boolean match(String templateName, String path, String fieldName, XConten if (templateName != null) { return templateName.equals(name); } - if (pathMatch != null && matchType.matches(pathMatch, path) == false) { - return false; + if (pathMatch != null) { + boolean anyMatch = false; + for (String m : pathMatch) { + if (matchType.matches(m, path)) { + anyMatch = true; + } + } + if (anyMatch == false) { + return false; // if none match, this template does not apply to the path + } } - if (match != null && matchType.matches(match, fieldName) == false) { - return false; + if (match != null) { + boolean anyMatch = false; + for (String m : match) { + if (matchType.matches(m, fieldName)) { + anyMatch = true; + } + } + if (anyMatch == false) { + return false; // if none match, this template does not apply to the fieldName + } } - if (pathUnmatch != null && matchType.matches(pathUnmatch, path)) { - return false; + if (pathUnmatch != null) { + for (String um : pathUnmatch) { + if (matchType.matches(um, path)) { + return false; + } + } } - if (unmatch != null && matchType.matches(unmatch, fieldName)) { - return false; + if (unmatch != null) { + for (String um : unmatch) { + if (matchType.matches(um, fieldName)) { + return false; + } + } } if (Arrays.stream(xContentFieldTypes).noneMatch(xcontentFieldType::equals)) { return false; @@ -387,17 +452,33 @@ Map getMapping() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - if (match != null) { - builder.field("match", match); + if (match != null && match.isEmpty() == false) { + if (match.size() == 1) { + builder.field("match", match.get(0)); + } else { + builder.field("match", match); + } } - if (pathMatch != null) { - builder.field("path_match", pathMatch); + if (pathMatch != null && pathMatch.isEmpty() == false) { + if (pathMatch.size() == 1) { + builder.field("path_match", pathMatch.get(0)); + } else { + builder.field("path_match", pathMatch); + } } - if (unmatch != null) { - builder.field("unmatch", unmatch); + if (unmatch != null && unmatch.isEmpty() == false) { + if (unmatch.size() == 1) { + builder.field("unmatch", unmatch.get(0)); + } else { + builder.field("unmatch", unmatch); + } } - if (pathUnmatch != null) { - builder.field("path_unmatch", pathUnmatch); + if (pathUnmatch != null && pathUnmatch.isEmpty() == false) { + if (pathUnmatch.size() == 1) { + builder.field("path_unmatch", pathUnmatch.get(0)); + } else { + builder.field("path_unmatch", pathUnmatch); + } } // We have more than one types when (1) `match_mapping_type` is "*", and (2) match and/or path_match are defined but // not `match_mapping_type`. In the latter the template implicitly accepts all types and we don't need to serialize diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java index c125fc4e8e664..113437f0a1a11 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java @@ -8,6 +8,9 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.document.InetAddressPoint; +import org.apache.lucene.document.IntField; +import org.apache.lucene.document.LongField; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; import org.apache.lucene.util.BytesRef; @@ -18,9 +21,13 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.test.VersionUtils; +import org.elasticsearch.test.XContentTestUtils; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xcontent.json.JsonXContent; import java.io.IOException; import java.util.Collections; @@ -29,9 +36,11 @@ import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; import static org.elasticsearch.test.VersionUtils.randomVersionBetween; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; public class DynamicTemplatesTests extends MapperServiceTestCase { @@ -342,9 +351,9 @@ public void testDynamicTemplatesForIndexTemplate() throws IOException { DynamicTemplate[] templates = mapper.mapping().getRoot().dynamicTemplates(); assertEquals(2, templates.length); assertEquals("first_template", templates[0].name()); - assertEquals("first", templates[0].pathMatch()); + assertEquals("first", templates[0].pathMatch().get(0)); assertEquals("second_template", templates[1].name()); - assertEquals("second", templates[1].pathMatch()); + assertEquals("second", templates[1].pathMatch().get(0)); // Dynamic templates should be appended and deduplicated. mapping = Strings.toString( @@ -379,11 +388,11 @@ public void testDynamicTemplatesForIndexTemplate() throws IOException { templates = mapper.mapping().getRoot().dynamicTemplates(); assertEquals(3, templates.length); assertEquals("first_template", templates[0].name()); - assertEquals("first", templates[0].pathMatch()); + assertEquals("first", templates[0].pathMatch().get(0)); assertEquals("second_template", templates[1].name()); - assertEquals("second_updated", templates[1].pathMatch()); + assertEquals("second_updated", templates[1].pathMatch().get(0)); assertEquals("third_template", templates[2].name()); - assertEquals("third", templates[2].pathMatch()); + assertEquals("third", templates[2].pathMatch().get(0)); } public void testIllegalDynamicTemplates() throws Exception { @@ -1752,4 +1761,495 @@ public void testSubobjectsFalseDocWithEmptyObject() throws IOException { ObjectMapper leaf = (ObjectMapper) artifacts.getMapper("leaf"); assertFalse(leaf.subobjects()); } + + public void testMatchWithArrayOfFieldNames() throws IOException { + String mapping = """ + { + "_mappings": { + "dynamic_templates": [ + { + "test": { + "match": ["long_*", "int_*"], + "mapping": { + "type": "integer" + } + } + } + ] + } + } + """; + String docJson = """ + { + "long_one": 10, + "int_text": "12", + "mynum": 13 + } + """; + + MapperService mapperService = createMapperService(mapping); + ParsedDocument parsedDoc = mapperService.documentMapper().parse(source(docJson)); + merge(mapperService, dynamicMapping(parsedDoc.dynamicMappingsUpdate())); + LuceneDocument doc = parsedDoc.rootDoc(); + + assertEquals(IntField.class, doc.getField("long_one").getClass()); + Mapper fieldMapper = mapperService.documentMapper().mappers().getMapper("long_one"); + assertNotNull(fieldMapper); + assertEquals("integer", fieldMapper.typeName()); + + assertEquals(IntField.class, doc.getField("int_text").getClass()); + fieldMapper = mapperService.documentMapper().mappers().getMapper("int_text"); + assertNotNull(fieldMapper); + assertEquals("integer", fieldMapper.typeName()); + + assertEquals(LongField.class, doc.getField("mynum").getClass()); + fieldMapper = mapperService.documentMapper().mappers().getMapper("mynum"); + assertNotNull(fieldMapper); + assertEquals("long", fieldMapper.typeName()); + } + + public void testMatchAndUnmatchWithArrayOfFieldNamesMapToIpType() throws IOException { + String mapping = """ + { + "_mappings": { + "dynamic_templates": [ + { + "test": { + "match": ["ip_*", "*_ip"], + "unmatch": ["one*", "*two"], + "mapping": { + "type": "ip" + } + } + } + ] + } + } + """; + String docJson = """ + { + "one_ip": "will not match", + "ip_two": "will not match", + "three_ip": "12.12.12.12", + "ip_four": "13.13.13.13" + } + """; + + MapperService mapperService = createMapperService(mapping); + ParsedDocument parsedDoc = mapperService.documentMapper().parse(source(docJson)); + merge(mapperService, dynamicMapping(parsedDoc.dynamicMappingsUpdate())); + LuceneDocument doc = parsedDoc.rootDoc(); + + assertNotEquals(InetAddressPoint.class, doc.getField("one_ip").getClass()); + Mapper fieldMapper = mapperService.documentMapper().mappers().getMapper("one_ip"); + assertNotNull(fieldMapper); + assertEquals("text", fieldMapper.typeName()); + + assertNotEquals(InetAddressPoint.class, doc.getField("ip_two").getClass()); + fieldMapper = mapperService.documentMapper().mappers().getMapper("ip_two"); + assertNotNull(fieldMapper); + assertEquals("text", fieldMapper.typeName()); + + assertEquals(InetAddressPoint.class, doc.getField("three_ip").getClass()); + fieldMapper = mapperService.documentMapper().mappers().getMapper("three_ip"); + assertNotNull(fieldMapper); + assertEquals("ip", fieldMapper.typeName()); + + assertEquals(InetAddressPoint.class, doc.getField("ip_four").getClass()); + fieldMapper = mapperService.documentMapper().mappers().getMapper("ip_four"); + assertNotNull(fieldMapper); + assertEquals("ip", fieldMapper.typeName()); + } + + public void testMatchWithArrayOfFieldNamesUsingRegex() throws IOException { + String mapping = """ + { + "_mappings": { + "dynamic_templates": [ + { + "test": { + "match_pattern": "regex", + "match": ["^one\\\\d.*$", "^.*two", ".*xyz.*"], + "mapping": { + "type": "ip" + } + } + } + ] + } + } + """; + String docJson = """ + { + "one100_ip": "11.11.11.120", + "iptwo": "10.10.10.10", + "threeip": "12.12.12.12" + } + """; + + MapperService mapperService = createMapperService(mapping); + ParsedDocument parsedDoc = mapperService.documentMapper().parse(source(docJson)); + merge(mapperService, dynamicMapping(parsedDoc.dynamicMappingsUpdate())); + LuceneDocument doc = parsedDoc.rootDoc(); + + assertEquals(InetAddressPoint.class, doc.getField("one100_ip").getClass()); + Mapper fieldMapper = mapperService.documentMapper().mappers().getMapper("one100_ip"); + assertNotNull(fieldMapper); + assertEquals("ip", fieldMapper.typeName()); + + assertEquals(InetAddressPoint.class, doc.getField("iptwo").getClass()); + fieldMapper = mapperService.documentMapper().mappers().getMapper("iptwo"); + assertNotNull(fieldMapper); + assertEquals("ip", fieldMapper.typeName()); + + assertNotEquals(InetAddressPoint.class, doc.getField("threeip").getClass()); + fieldMapper = mapperService.documentMapper().mappers().getMapper("threeip"); + assertNotNull(fieldMapper); + assertEquals("text", fieldMapper.typeName()); + } + + public void testMatchWithArrayOfFieldNamesMixingGlobsAndRegex() throws IOException { + String mapping = """ + { + "_mappings": { + "dynamic_templates": [ + { + "test": { + "match": ["one*", ".*two$", "^xyz.*"], + "mapping": { + "type": "ip" + } + } + } + ] + } + } + """; + String docJson = """ + { + "oneip": "11.11.11.120", + "iptwo": "10.10.10.10", + "threeip": "12.12.12.12" + } + """; + + MapperService mapperService = createMapperService(mapping); + ParsedDocument parsedDoc = mapperService.documentMapper().parse(source(docJson)); + merge(mapperService, dynamicMapping(parsedDoc.dynamicMappingsUpdate())); + LuceneDocument doc = parsedDoc.rootDoc(); + + assertEquals(InetAddressPoint.class, doc.getField("oneip").getClass()); + Mapper fieldMapper = mapperService.documentMapper().mappers().getMapper("oneip"); + assertNotNull(fieldMapper); + assertEquals("ip", fieldMapper.typeName()); + + // this one will not match and be an IP field because it was specified with a regex but match_pattern is implicit "simple" + assertNotEquals(InetAddressPoint.class, doc.getField("iptwo").getClass()); + fieldMapper = mapperService.documentMapper().mappers().getMapper("iptwo"); + assertNotNull(fieldMapper); + assertEquals("text", fieldMapper.typeName()); + + assertNotEquals(InetAddressPoint.class, doc.getField("threeip").getClass()); + fieldMapper = mapperService.documentMapper().mappers().getMapper("threeip"); + assertNotNull(fieldMapper); + assertEquals("text", fieldMapper.typeName()); + } + + public void testMatchAndUnmatchWithArrayOfFieldNamesAsRuntimeFields() throws IOException { + String mapping = """ + { + "_mappings": { + "dynamic_templates": [ + { + "test": { + "match": ["*one*", "two*"], + "unmatch": ["*_xyz", "*foo"], + "runtime": {} + } + } + ] + } + } + """; + // twothing should map to runtime field of type 'keyword' (default for runtime strings) + // one_xyz should be excluded because of the unmatch, so be multi-field of text/keyword + String docJson = """ + { + "twothing": "ipsum", + "one_xyz": "13" + } + """; + + MapperService mapperService = createMapperService(mapping); + ParsedDocument parsedDoc = mapperService.documentMapper().parse(source(docJson)); + merge(mapperService, dynamicMapping(parsedDoc.dynamicMappingsUpdate())); + + Map actualDynamicMappings = XContentTestUtils.convertToMap(parsedDoc.dynamicMappingsUpdate()); + + String expected = """ + { + "_doc": { + "runtime": { + "twothing": { + "type": "keyword" + } + }, + "properties": { + "one_xyz": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } + }"""; + + try (XContentParser xparser = JsonXContent.jsonXContent.createParser(XContentParserConfiguration.EMPTY, expected)) { + Map expectedDynamicMappings = xparser.map(); + String diff = XContentTestUtils.differenceBetweenMapsIgnoringArrayOrder(actualDynamicMappings, expectedDynamicMappings); + assertNull("difference between expected and actual Mappings", diff); + } + } + + public void testMatchAndUnmatchWithArrayOfFieldNamesWithMatchMappingType() throws IOException { + String mapping = """ + { + "_mappings": { + "dynamic_templates": [ + { + "test": { + "match_mapping_type": "string", + "match": ["*one*", "two*"], + "mapping": { + "type": "keyword" + } + } + } + ] + } + } + """; + String docJson = """ + { + "one_bool": "true", + "two_bool": false + } + """; + + MapperService mapperService = createMapperService(mapping); + ParsedDocument parsedDoc = mapperService.documentMapper().parse(source(docJson)); + merge(mapperService, dynamicMapping(parsedDoc.dynamicMappingsUpdate())); + + Mapper fieldMapper = mapperService.documentMapper().mappers().getMapper("one_bool"); + assertNotNull(fieldMapper); + assertEquals("keyword", fieldMapper.typeName()); + + fieldMapper = mapperService.documentMapper().mappers().getMapper("two_bool"); + assertNotNull(fieldMapper); + // this would be keyword if we hadn't specified match_mapping_type = string + assertEquals("boolean", fieldMapper.typeName()); + } + + public void testPathMatchWithArrayOfFieldNames() throws IOException { + String mapping = """ + { + "_mappings": { + "dynamic_templates": [ + { + "full_name": { + "path_match": ["name.*", "user.name.*"], + "mapping": { + "type": "text", + "copy_to": "full_name" + } + } + } + ] + } + } + """; + + String docJson1 = """ + { + "name": { + "first": "John", + "middle": "Winston", + "last": "Lennon" + } + } + """; + + String docJson2 = """ + { + "user": { + "name": { + "first": "Jane", + "midinitial": "M", + "last": "Salazar" + } + } + } + """; + + MapperService mapperService = createMapperService(mapping); + ParsedDocument parsedDoc = mapperService.documentMapper().parse(source(docJson1)); + merge(mapperService, dynamicMapping(parsedDoc.dynamicMappingsUpdate())); + + Mapper fieldMapper = mapperService.documentMapper().mappers().getMapper("name.first"); + assertThat(fieldMapper, instanceOf(TextFieldMapper.class)); + String copyToField = ((TextFieldMapper) fieldMapper).copyTo().copyToFields().get(0); + assertEquals("full_name", copyToField); + assertEquals("text", fieldMapper.typeName()); + + fieldMapper = mapperService.documentMapper().mappers().getMapper("name.middle"); + assertThat(fieldMapper, instanceOf(TextFieldMapper.class)); + copyToField = ((TextFieldMapper) fieldMapper).copyTo().copyToFields().get(0); + assertEquals("full_name", copyToField); + assertEquals("text", fieldMapper.typeName()); + + fieldMapper = mapperService.documentMapper().mappers().getMapper("name.last"); + assertThat(fieldMapper, instanceOf(TextFieldMapper.class)); + copyToField = ((TextFieldMapper) fieldMapper).copyTo().copyToFields().get(0); + assertEquals("full_name", copyToField); + assertEquals("text", fieldMapper.typeName()); + + // test second doc with user.name.xxx + parsedDoc = mapperService.documentMapper().parse(source(docJson2)); + merge(mapperService, dynamicMapping(parsedDoc.dynamicMappingsUpdate())); + + fieldMapper = mapperService.documentMapper().mappers().getMapper("user.name.first"); + assertThat(fieldMapper, instanceOf(TextFieldMapper.class)); + copyToField = ((TextFieldMapper) fieldMapper).copyTo().copyToFields().get(0); + assertEquals("full_name", copyToField); + assertEquals("text", fieldMapper.typeName()); + + fieldMapper = mapperService.documentMapper().mappers().getMapper("user.name.midinitial"); + assertThat(fieldMapper, instanceOf(TextFieldMapper.class)); + copyToField = ((TextFieldMapper) fieldMapper).copyTo().copyToFields().get(0); + assertEquals("full_name", copyToField); + assertEquals("text", fieldMapper.typeName()); + + fieldMapper = mapperService.documentMapper().mappers().getMapper("user.name.last"); + assertThat(fieldMapper, instanceOf(TextFieldMapper.class)); + copyToField = ((TextFieldMapper) fieldMapper).copyTo().copyToFields().get(0); + assertEquals("full_name", copyToField); + assertEquals("text", fieldMapper.typeName()); + } + + public void testPathMatchAndPathUnmatchWithArrayOfFieldNames() throws IOException { + String mapping = """ + { + "_mappings": { + "dynamic_templates": [ + { + "full_name": { + "path_match": ["name.*", "user.name.*"], + "path_unmatch": ["*.middle", "*.midinitial"], + "mapping": { + "type": "text", + "copy_to": "full_name" + } + } + } + ] + } + } + """; + + String docJson1 = """ + { + "name": { + "first": "John", + "middle": "Winston", + "last": "Lennon" + } + } + """; + + String docJson2 = """ + { + "user": { + "name": { + "first": "Jane", + "midinitial": "M", + "last": "Salazar" + } + } + } + """; + + MapperService mapperService = createMapperService(mapping); + ParsedDocument parsedDoc = mapperService.documentMapper().parse(source(docJson1)); + merge(mapperService, dynamicMapping(parsedDoc.dynamicMappingsUpdate())); + + Mapper fieldMapper = mapperService.documentMapper().mappers().getMapper("name.first"); + assertThat(fieldMapper, instanceOf(TextFieldMapper.class)); + String copyToField = ((TextFieldMapper) fieldMapper).copyTo().copyToFields().get(0); + assertEquals("full_name", copyToField); + assertEquals("text", fieldMapper.typeName()); + + fieldMapper = mapperService.documentMapper().mappers().getMapper("name.middle"); + assertThat(fieldMapper, instanceOf(TextFieldMapper.class)); + assertThat(((TextFieldMapper) fieldMapper).copyTo().copyToFields(), is(empty())); + assertEquals("text", fieldMapper.typeName()); + + fieldMapper = mapperService.documentMapper().mappers().getMapper("name.last"); + assertThat(fieldMapper, instanceOf(TextFieldMapper.class)); + copyToField = ((TextFieldMapper) fieldMapper).copyTo().copyToFields().get(0); + assertEquals("full_name", copyToField); + assertEquals("text", fieldMapper.typeName()); + + // test second doc with user.name.xxx + parsedDoc = mapperService.documentMapper().parse(source(docJson2)); + merge(mapperService, dynamicMapping(parsedDoc.dynamicMappingsUpdate())); + + fieldMapper = mapperService.documentMapper().mappers().getMapper("user.name.first"); + assertThat(fieldMapper, instanceOf(TextFieldMapper.class)); + copyToField = ((TextFieldMapper) fieldMapper).copyTo().copyToFields().get(0); + assertEquals("full_name", copyToField); + assertEquals("text", fieldMapper.typeName()); + + fieldMapper = mapperService.documentMapper().mappers().getMapper("user.name.midinitial"); + assertThat(fieldMapper, instanceOf(TextFieldMapper.class)); + assertThat(((TextFieldMapper) fieldMapper).copyTo().copyToFields(), is(empty())); + assertEquals("text", fieldMapper.typeName()); + + fieldMapper = mapperService.documentMapper().mappers().getMapper("user.name.last"); + assertThat(fieldMapper, instanceOf(TextFieldMapper.class)); + copyToField = ((TextFieldMapper) fieldMapper).copyTo().copyToFields().get(0); + assertEquals("full_name", copyToField); + assertEquals("text", fieldMapper.typeName()); + } + + public void testInvalidMatchWithArrayOfFieldNamesUsingNonStringEntries() throws IOException { + String mapping = """ + { + "_mappings": { + "dynamic_templates": [ + { + "test": { + "match": [23.45, false], + "mapping": { + "type": "integer" + } + } + } + ] + } + } + """; + + // throws MapperParsingException Failed to parse mapping: [match] values must either be a string or list of strings, but was + // [[23.45, false]] + Exception e = expectThrows(MapperParsingException.class, () -> createMapperService(mapping)); + assertThat(e.getMessage(), containsString("Failed to parse mapping")); + assertThat( + e.getCause().getMessage(), + containsString("[match] values must either be a string or list of strings, but was [[23.45, false]]") + ); + } } From e831212a8dbb2b7ba1ebe0003052e2e5868fc459 Mon Sep 17 00:00:00 2001 From: Michael Peterson Date: Wed, 26 Apr 2023 09:00:04 -0400 Subject: [PATCH 02/11] First PR changes --- .../mapping/dynamic/templates.asciidoc | 8 +++---- .../DataStreamIndexSettingsProvider.java | 6 +++++- .../DataStreamIndexSettingsProviderTests.java | 21 ++++++++++--------- .../index/mapper/DynamicTemplate.java | 4 ++-- 4 files changed, 22 insertions(+), 17 deletions(-) diff --git a/docs/reference/mapping/dynamic/templates.asciidoc b/docs/reference/mapping/dynamic/templates.asciidoc index 89029cadedd0e..20a29880eaad7 100644 --- a/docs/reference/mapping/dynamic/templates.asciidoc +++ b/docs/reference/mapping/dynamic/templates.asciidoc @@ -235,9 +235,9 @@ PUT my-index-000001/_doc/1 You can specify a list of patterns using a JSON array for either the `match` or `unmatch` fields. -The next example matches all fields whose name starts with -`one` or has `two` in the name, except for those which end with `ip` or ``) -and maps them as `keyword` fields: +The next example matches all fields whose name starts with `ip_` or ends with `_ip` +in the name, except for those which start with "one" or end with "two" and maps them +as `keyword` fields: [source,console] -------------------------------------------------- @@ -268,7 +268,7 @@ PUT my-index/_doc/1 -------------------------------------------------- <1> The `one_ip` field is unmatched, so uses the default mapping of `text`. -<1> The `ip_two` field is unmatched, so uses the default mapping of `text`. +<2> The `ip_two` field is unmatched, so uses the default mapping of `text`. <3> The `three_ip` field is mapped as `ip`. <4> The `ip_four` field is mapped as `ip`. diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java index 31e59619f6163..b645de41bd202 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java @@ -31,6 +31,7 @@ import java.io.UncheckedIOException; import java.time.Instant; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Locale; @@ -180,7 +181,10 @@ private List findRoutingPaths(String indexName, Settings allSettings, Li MappingParserContext parserContext = mapperService.parserContext(); for (String pathMatch : template.pathMatch()) { var mapper = parserContext.typeParser(mappingSnippetType) - .parse(pathMatch, mappingSnippet, parserContext) + // Since FieldMapper.parse modifies the Map passed in (removing entries for "type"), that means + // that only the first pathMatch passed in gets recognized as a time_series_dimension. To counteract + // that, we wrap the mappingSnippet in a new HashMap for each pathMatch instance. + .parse(pathMatch, new HashMap<>(mappingSnippet), parserContext) .build(MapperBuilderContext.root(false)); extractPath(routingPaths, mapper); } diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java index ea8dce722fd90..203d9bdde5d6a 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java @@ -466,9 +466,9 @@ public void testGenerateRoutingPathFromDynamicTemplateWithMultiplePathMatchEntri assertThat(result.size(), equalTo(3)); assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(lookAheadTime.getMillis()))); assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(lookAheadTime.getMillis()))); - assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("host.id", "xprometheus.labels.*")); + assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("host.id", "xprometheus.labels.*", "yprometheus.labels.*")); List routingPathList = IndexMetadata.INDEX_ROUTING_PATH.get(result); - assertEquals(2, routingPathList.size()); + assertEquals(3, routingPathList.size()); } public void testGenerateRoutingPathFromDynamicTemplate_templateWithNoPathMatch() throws Exception { @@ -528,20 +528,20 @@ public void testGenerateRoutingPathFromDynamicTemplate_nonKeywordTemplate() thro "_doc": { "dynamic_templates": [ { - "labels": { - "path_match": "prometheus.labels.*", + "docker.cpu.core.*.pct": { + "path_match": "docker.cpu.core.*.pct", "mapping": { - "type": "keyword", - "time_series_dimension": true + "coerce": true, + "type": "float" } } }, { - "docker.cpu.core.*.pct": { - "path_match": "docker.cpu.core.*.pct", + "labels": { + "path_match": "prometheus.labels.*", "mapping": { - "coerce": true, - "type": "float" + "type": "keyword", + "time_series_dimension": true } } } @@ -566,6 +566,7 @@ public void testGenerateRoutingPathFromDynamicTemplate_nonKeywordTemplate() thro assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(lookAheadTime.getMillis()))); assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(lookAheadTime.getMillis()))); assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("host.id", "prometheus.labels.*")); + assertEquals(2, IndexMetadata.INDEX_ROUTING_PATH.get(result).size()); } private Settings generateTsdbSettings(String mapping, Instant now) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java b/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java index 9db9630d1619e..509b65ea51d67 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java @@ -223,7 +223,7 @@ static DynamicTemplate parse(String name, Map conf) throws Mappe } final MatchType matchType = MatchType.fromString(matchPattern); - ensurePatternsDoNotMatchEmptyString( + validatePatterns( name, matchType, List.of( @@ -257,7 +257,7 @@ private static void addEntriesToPatternList(List matchList, String propN } } - private static void ensurePatternsDoNotMatchEmptyString(String templateName, MatchType matchType, List> patterns) { + private static void validatePatterns(String templateName, MatchType matchType, List> patterns) { for (List patternList : patterns) { for (String regex : patternList) { try { From 7fd9431802b705c5e2560b124fcc2d70367c3c25 Mon Sep 17 00:00:00 2001 From: Michael Peterson Date: Wed, 26 Apr 2023 10:13:06 -0400 Subject: [PATCH 03/11] Second PR change: _mappings to _doc in test class --- .../index/mapper/DynamicTemplatesTests.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java index 113437f0a1a11..db51575560f7b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java @@ -1765,7 +1765,7 @@ public void testSubobjectsFalseDocWithEmptyObject() throws IOException { public void testMatchWithArrayOfFieldNames() throws IOException { String mapping = """ { - "_mappings": { + "_doc": { "dynamic_templates": [ { "test": { @@ -1811,7 +1811,7 @@ public void testMatchWithArrayOfFieldNames() throws IOException { public void testMatchAndUnmatchWithArrayOfFieldNamesMapToIpType() throws IOException { String mapping = """ { - "_mappings": { + "_doc": { "dynamic_templates": [ { "test": { @@ -1864,7 +1864,7 @@ public void testMatchAndUnmatchWithArrayOfFieldNamesMapToIpType() throws IOExcep public void testMatchWithArrayOfFieldNamesUsingRegex() throws IOException { String mapping = """ { - "_mappings": { + "_doc": { "dynamic_templates": [ { "test": { @@ -1911,7 +1911,7 @@ public void testMatchWithArrayOfFieldNamesUsingRegex() throws IOException { public void testMatchWithArrayOfFieldNamesMixingGlobsAndRegex() throws IOException { String mapping = """ { - "_mappings": { + "_doc": { "dynamic_templates": [ { "test": { @@ -1958,7 +1958,7 @@ public void testMatchWithArrayOfFieldNamesMixingGlobsAndRegex() throws IOExcepti public void testMatchAndUnmatchWithArrayOfFieldNamesAsRuntimeFields() throws IOException { String mapping = """ { - "_mappings": { + "_doc": { "dynamic_templates": [ { "test": { @@ -2018,7 +2018,7 @@ public void testMatchAndUnmatchWithArrayOfFieldNamesAsRuntimeFields() throws IOE public void testMatchAndUnmatchWithArrayOfFieldNamesWithMatchMappingType() throws IOException { String mapping = """ { - "_mappings": { + "_doc": { "dynamic_templates": [ { "test": { @@ -2057,7 +2057,7 @@ public void testMatchAndUnmatchWithArrayOfFieldNamesWithMatchMappingType() throw public void testPathMatchWithArrayOfFieldNames() throws IOException { String mapping = """ { - "_mappings": { + "_doc": { "dynamic_templates": [ { "full_name": { @@ -2143,7 +2143,7 @@ public void testPathMatchWithArrayOfFieldNames() throws IOException { public void testPathMatchAndPathUnmatchWithArrayOfFieldNames() throws IOException { String mapping = """ { - "_mappings": { + "_doc": { "dynamic_templates": [ { "full_name": { @@ -2228,7 +2228,7 @@ public void testPathMatchAndPathUnmatchWithArrayOfFieldNames() throws IOExceptio public void testInvalidMatchWithArrayOfFieldNamesUsingNonStringEntries() throws IOException { String mapping = """ { - "_mappings": { + "_doc": { "dynamic_templates": [ { "test": { From f5ae738ae2276fde35408190c8437335733119bb Mon Sep 17 00:00:00 2001 From: Michael Peterson Date: Wed, 26 Apr 2023 12:30:27 -0400 Subject: [PATCH 04/11] PR commit 3 - list of matches/unmatches defaults to empty list not null --- .../DataStreamIndexSettingsProvider.java | 2 +- .../DataStreamIndexSettingsProviderTests.java | 5 +- .../index/mapper/DynamicTemplate.java | 121 ++++++++---------- 3 files changed, 57 insertions(+), 71 deletions(-) diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java index b645de41bd202..5a4759385076e 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java @@ -167,7 +167,7 @@ private List findRoutingPaths(String indexName, Settings allSettings, Li extractPath(routingPaths, fieldMapper); } for (var template : mapperService.getAllDynamicTemplates()) { - if (template.pathMatch() == null) { + if (template.pathMatch().isEmpty()) { continue; } diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java index 203d9bdde5d6a..27fe65ba309d3 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java @@ -466,7 +466,10 @@ public void testGenerateRoutingPathFromDynamicTemplateWithMultiplePathMatchEntri assertThat(result.size(), equalTo(3)); assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(lookAheadTime.getMillis()))); assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(lookAheadTime.getMillis()))); - assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("host.id", "xprometheus.labels.*", "yprometheus.labels.*")); + assertThat( + IndexMetadata.INDEX_ROUTING_PATH.get(result), + containsInAnyOrder("host.id", "xprometheus.labels.*", "yprometheus.labels.*") + ); List routingPathList = IndexMetadata.INDEX_ROUTING_PATH.get(result); assertEquals(3, routingPathList.size()); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java b/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java index 509b65ea51d67..cfb0052f89484 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java @@ -16,12 +16,13 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.TreeMap; +import java.util.stream.Collectors; +import java.util.stream.Stream; public class DynamicTemplate implements ToXContentObject { @@ -143,10 +144,10 @@ public final String toString() { @SuppressWarnings("unchecked") static DynamicTemplate parse(String name, Map conf) throws MapperParsingException { - List match = null; - List pathMatch = null; - List unmatch = null; - List pathUnmatch = null; + List match = new ArrayList<>(4); // these pattern lists will typically be very small + List pathMatch = new ArrayList<>(4); + List unmatch = new ArrayList<>(4); + List pathUnmatch = new ArrayList<>(4); Map mapping = null; boolean runtime = false; String matchMappingType = null; @@ -155,16 +156,12 @@ static DynamicTemplate parse(String name, Map conf) throws Mappe for (Map.Entry entry : conf.entrySet()) { String propName = entry.getKey(); if ("match".equals(propName)) { - match = new ArrayList<>(); addEntriesToPatternList(match, propName, entry); } else if ("path_match".equals(propName)) { - pathMatch = new ArrayList<>(); addEntriesToPatternList(pathMatch, propName, entry); } else if ("unmatch".equals(propName)) { - unmatch = new ArrayList<>(); addEntriesToPatternList(unmatch, propName, entry); } else if ("path_unmatch".equals(propName)) { - pathUnmatch = new ArrayList<>(); addEntriesToPatternList(pathUnmatch, propName, entry); } else if ("match_mapping_type".equals(propName)) { matchMappingType = entry.getValue().toString(); @@ -198,7 +195,7 @@ static DynamicTemplate parse(String name, Map conf) throws Mappe } final XContentFieldType[] xContentFieldTypes; - if ("*".equals(matchMappingType) || (matchMappingType == null && (match != null || pathMatch != null))) { + if ("*".equals(matchMappingType) || (matchMappingType == null && matchPatternsAreDefined(match, pathMatch))) { if (runtime) { xContentFieldTypes = Arrays.stream(XContentFieldType.values()) .filter(XContentFieldType::supportsRuntimeField) @@ -223,20 +220,24 @@ static DynamicTemplate parse(String name, Map conf) throws Mappe } final MatchType matchType = MatchType.fromString(matchPattern); - validatePatterns( - name, - matchType, - List.of( - match != null ? match : Collections.emptyList(), - unmatch != null ? unmatch : Collections.emptyList(), - pathMatch != null ? pathMatch : Collections.emptyList(), - pathUnmatch != null ? pathUnmatch : Collections.emptyList() - ) - ); + List allPatterns = Stream.concat( + Stream.concat(match.stream(), unmatch.stream()), + Stream.concat(pathMatch.stream(), pathUnmatch.stream()) + ).collect(Collectors.toList()); + validatePatterns(name, matchType, allPatterns); return new DynamicTemplate(name, pathMatch, pathUnmatch, match, unmatch, xContentFieldTypes, matchType, mapping, runtime); } + /** + * @param match list of match patterns (can be empty but not null) + * @param pathMatch list of pathMatch patterns (can be empty but not null) + * @return return true if there is at least 1 match or pathMatch pattern defined + */ + private static boolean matchPatternsAreDefined(List match, List pathMatch) { + return match.size() + pathMatch.size() > 0; + } + private static void addEntriesToPatternList(List matchList, String propName, Map.Entry entry) { if (entry.getValue() instanceof String s) { matchList.add(s); @@ -257,22 +258,20 @@ private static void addEntriesToPatternList(List matchList, String propN } } - private static void validatePatterns(String templateName, MatchType matchType, List> patterns) { - for (List patternList : patterns) { - for (String regex : patternList) { - try { - matchType.matches(regex, ""); - } catch (IllegalArgumentException e) { - throw new IllegalArgumentException( - Strings.format( - "Pattern [%s] of type [%s] is invalid. Cannot create dynamic template [%s].", - regex, - matchType, - templateName - ), - e - ); - } + private static void validatePatterns(String templateName, MatchType matchType, List patterns) { + for (String regex : patterns) { + try { + matchType.matches(regex, ""); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException( + Strings.format( + "Pattern [%s] of type [%s] is invalid. Cannot create dynamic template [%s].", + regex, + matchType, + templateName + ), + e + ); } } } @@ -326,40 +325,24 @@ public boolean match(String templateName, String path, String fieldName, XConten if (templateName != null) { return templateName.equals(name); } - if (pathMatch != null) { - boolean anyMatch = false; - for (String m : pathMatch) { - if (matchType.matches(m, path)) { - anyMatch = true; - } - } - if (anyMatch == false) { - return false; // if none match, this template does not apply to the path + if (pathMatch.isEmpty() == false) { + if (pathMatch.stream().anyMatch(m -> matchType.matches(m, path)) == false) { + return false; } } - if (match != null) { - boolean anyMatch = false; - for (String m : match) { - if (matchType.matches(m, fieldName)) { - anyMatch = true; - } - } - if (anyMatch == false) { - return false; // if none match, this template does not apply to the fieldName + if (match.isEmpty() == false) { + if (match.stream().anyMatch(m -> matchType.matches(m, fieldName)) == false) { + return false; } } - if (pathUnmatch != null) { - for (String um : pathUnmatch) { - if (matchType.matches(um, path)) { - return false; - } + for (String um : pathUnmatch) { + if (matchType.matches(um, path)) { + return false; } } - if (unmatch != null) { - for (String um : unmatch) { - if (matchType.matches(um, fieldName)) { - return false; - } + for (String um : unmatch) { + if (matchType.matches(um, fieldName)) { + return false; } } if (Arrays.stream(xContentFieldTypes).noneMatch(xcontentFieldType::equals)) { @@ -452,28 +435,28 @@ Map getMapping() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - if (match != null && match.isEmpty() == false) { + if (match.isEmpty() == false) { if (match.size() == 1) { builder.field("match", match.get(0)); } else { builder.field("match", match); } } - if (pathMatch != null && pathMatch.isEmpty() == false) { + if (pathMatch.isEmpty() == false) { if (pathMatch.size() == 1) { builder.field("path_match", pathMatch.get(0)); } else { builder.field("path_match", pathMatch); } } - if (unmatch != null && unmatch.isEmpty() == false) { + if (unmatch.isEmpty() == false) { if (unmatch.size() == 1) { builder.field("unmatch", unmatch.get(0)); } else { builder.field("unmatch", unmatch); } } - if (pathUnmatch != null && pathUnmatch.isEmpty() == false) { + if (pathUnmatch.isEmpty() == false) { if (pathUnmatch.size() == 1) { builder.field("path_unmatch", pathUnmatch.get(0)); } else { @@ -483,7 +466,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws // We have more than one types when (1) `match_mapping_type` is "*", and (2) match and/or path_match are defined but // not `match_mapping_type`. In the latter the template implicitly accepts all types and we don't need to serialize // the `match_mapping_type` values. - if (xContentFieldTypes.length > 1 && match == null && pathMatch == null) { + if (xContentFieldTypes.length > 1 && match.isEmpty() && pathMatch.isEmpty()) { builder.field("match_mapping_type", "*"); } else if (xContentFieldTypes.length == 1) { builder.field("match_mapping_type", xContentFieldTypes[0]); From e103dad649492f29d531c1c1b2539eef00cef1b0 Mon Sep 17 00:00:00 2001 From: Michael Peterson Date: Wed, 26 Apr 2023 12:45:05 -0400 Subject: [PATCH 05/11] PR4 - clean up templates.asciidoc further --- .../mapping/dynamic/templates.asciidoc | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/reference/mapping/dynamic/templates.asciidoc b/docs/reference/mapping/dynamic/templates.asciidoc index 20a29880eaad7..6f2cec356edb4 100644 --- a/docs/reference/mapping/dynamic/templates.asciidoc +++ b/docs/reference/mapping/dynamic/templates.asciidoc @@ -235,9 +235,9 @@ PUT my-index-000001/_doc/1 You can specify a list of patterns using a JSON array for either the `match` or `unmatch` fields. -The next example matches all fields whose name starts with `ip_` or ends with `_ip` -in the name, except for those which start with "one" or end with "two" and maps them -as `keyword` fields: +The next example matches all fields whose name starts with `ip_` or ends with `_ip`, +except for fields which start with `one` or end with `two` and maps them +as `ip` fields: [source,console] -------------------------------------------------- @@ -247,7 +247,7 @@ PUT my-index-000001 "dynamic_templates": [ { "ip_fields": { - "match": ["ip_*", "*_ip"], + "match": ["ip_*", "*_ip"], "unmatch": ["one*", "*two"], "mapping": { "type": "ip" @@ -260,17 +260,17 @@ PUT my-index-000001 PUT my-index/_doc/1 { - "one_ip": "will not match", <1> - "ip_two": "will not match", <2> + "one_ip": "will not match", <1> + "ip_two": "will not match", <2> "three_ip": "12.12.12.12", <3> - "ip_four": "13.13.13.13" <4> + "ip_four": "13.13.13.13" <4> } -------------------------------------------------- <1> The `one_ip` field is unmatched, so uses the default mapping of `text`. <2> The `ip_two` field is unmatched, so uses the default mapping of `text`. -<3> The `three_ip` field is mapped as `ip`. -<4> The `ip_four` field is mapped as `ip`. +<3> The `three_ip` field is mapped as type `ip`. +<4> The `ip_four` field is mapped as type `ip`. [[path-match-unmatch]] @@ -353,9 +353,9 @@ PUT my-index-000001/_doc/2 { "user": { "name": { - "first": "Jane", + "first": "Jane", "midinitial": "M", - "last": "Salazar" + "last": "Salazar" } } } From b6f4dd9780a61db64f62cd7fa680f8290b38471a Mon Sep 17 00:00:00 2001 From: Michael Peterson Date: Wed, 26 Apr 2023 14:39:55 -0400 Subject: [PATCH 06/11] PR5 - added tests to DynamicTemplateParseTests --- .../mapper/DynamicTemplateParseTests.java | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateParseTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateParseTests.java index 73aaf316a7bdc..9b3a37e113174 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateParseTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateParseTests.java @@ -207,6 +207,29 @@ public void testMatchAllTypesTemplateRuntime() { assertFalse(template.match(null, "a.b", "b", XContentFieldType.BINARY)); } + public void testMatchAllTypesTemplateRuntimeWithListOfMatches() { + Map templateDef = new HashMap<>(); + templateDef.put("match", List.of("b", "c*")); + templateDef.put("runtime", Collections.emptyMap()); + DynamicTemplate template = DynamicTemplate.parse("my_template", templateDef); + assertTrue(template.isRuntimeMapping()); + assertTrue(template.match(null, "a.b", "b", XContentFieldType.BOOLEAN)); + assertTrue(template.match(null, "a.b", "b", XContentFieldType.DATE)); + assertTrue(template.match(null, "a.b", "b", XContentFieldType.STRING)); + assertTrue(template.match(null, "a.b", "b", XContentFieldType.DOUBLE)); + assertTrue(template.match(null, "a.b", "b", XContentFieldType.LONG)); + assertFalse(template.match(null, "a.b", "b", XContentFieldType.OBJECT)); + assertFalse(template.match(null, "a.b", "b", XContentFieldType.BINARY)); + + assertTrue(template.match(null, null, "cx", XContentFieldType.BOOLEAN)); + assertTrue(template.match(null, null, "cx", XContentFieldType.DATE)); + assertTrue(template.match(null, null, "cx", XContentFieldType.STRING)); + assertTrue(template.match(null, null, "cx", XContentFieldType.DOUBLE)); + assertTrue(template.match(null, null, "cx", XContentFieldType.LONG)); + assertFalse(template.match(null, null, "cx", XContentFieldType.OBJECT)); + assertFalse(template.match(null, null, "cx", XContentFieldType.BINARY)); + } + public void testMatchTypeTemplate() { Map templateDef = new HashMap<>(); templateDef.put("match_mapping_type", "string"); @@ -303,6 +326,20 @@ public void testSerialization() throws Exception { assertEquals(""" {"match":"*name","unmatch":"first_name","mapping":{"store":true}}""", Strings.toString(builder)); + // name-based template with array of match patterns + templateDef = new HashMap<>(); + if (randomBoolean()) { + templateDef.put("match_mapping_type", "*"); + } + templateDef.put("match", List.of("*name", "user*")); + templateDef.put("unmatch", "first_name"); + templateDef.put("mapping", Collections.singletonMap("store", true)); + template = DynamicTemplate.parse("my_template", templateDef); + builder = JsonXContent.contentBuilder(); + template.toXContent(builder, ToXContent.EMPTY_PARAMS); + assertEquals(""" + {"match":["*name","user*"],"unmatch":"first_name","mapping":{"store":true}}""", Strings.toString(builder)); + // path-based template templateDef = new HashMap<>(); templateDef.put("path_match", "*name"); @@ -317,6 +354,34 @@ public void testSerialization() throws Exception { assertEquals(""" {"path_match":"*name","path_unmatch":"first_name","mapping":{"store":true}}""", Strings.toString(builder)); + // path-based template with single-entry array - still serializes as single string, rather than list + templateDef = new HashMap<>(); + templateDef.put("path_match", List.of("*name")); + templateDef.put("path_unmatch", List.of("first_name")); + if (randomBoolean()) { + templateDef.put("match_mapping_type", "*"); + } + templateDef.put("mapping", Collections.singletonMap("store", true)); + template = DynamicTemplate.parse("my_template", templateDef); + builder = JsonXContent.contentBuilder(); + template.toXContent(builder, ToXContent.EMPTY_PARAMS); + assertEquals(""" + {"path_match":"*name","path_unmatch":"first_name","mapping":{"store":true}}""", Strings.toString(builder)); + + // path-based template with multi-entry array - now serializes as list + templateDef = new HashMap<>(); + templateDef.put("path_match", List.of("*name", "user*")); + templateDef.put("path_unmatch", List.of("first_name", "username")); + if (randomBoolean()) { + templateDef.put("match_mapping_type", "*"); + } + templateDef.put("mapping", Collections.singletonMap("store", true)); + template = DynamicTemplate.parse("my_template", templateDef); + builder = JsonXContent.contentBuilder(); + template.toXContent(builder, ToXContent.EMPTY_PARAMS); + assertEquals(""" + {"path_match":["*name","user*"],"path_unmatch":["first_name","username"],"mapping":{"store":true}}""", Strings.toString(builder)); + // regex matching templateDef = new HashMap<>(); templateDef.put("match", "^a$"); @@ -424,6 +489,22 @@ public void testMatchTemplateName() throws Exception { assertTrue(template.match("my_template", "foo.bar", "not_match_name", randomFrom(XContentFieldType.values()))); assertFalse(template.match(null, "foo.bar", "not_match_name", randomFrom(XContentFieldType.values()))); } + // match name with array of patterns + { + Map templateDef = new HashMap<>(); + templateDef.put("match", List.of("baz*", "*quux", "*wibble*")); + templateDef.put("mapping", Map.of()); + DynamicTemplate template = DynamicTemplate.parse("my_template", templateDef); + assertTrue(template.match("my_template", "foo.bar", "foo", randomFrom(XContentFieldType.values()))); + // won't match because fieldName doesn't match + assertFalse(template.match(null, "foo.bar", "foo", randomFrom(XContentFieldType.values()))); + assertTrue(template.match(null, null, "bazzy", randomFrom(XContentFieldType.values()))); + assertTrue(template.match(null, null, "myquux", randomFrom(XContentFieldType.values()))); + assertTrue(template.match(null, null, "foo.wibble_bar", randomFrom(XContentFieldType.values()))); + assertFalse(template.match("not_template_name", "foo.bar", "foo", randomFrom(XContentFieldType.values()))); + assertTrue(template.match("my_template", "foo.bar", "not_match_name", randomFrom(XContentFieldType.values()))); + assertFalse(template.match(null, "foo.bar", "not_match_name", randomFrom(XContentFieldType.values()))); + } // no match condition { Map templateDef = new HashMap<>(); From bde9b596b4d694315a0145ea0acb69218cd75244 Mon Sep 17 00:00:00 2001 From: Michael Peterson Date: Wed, 26 Apr 2023 14:44:16 -0400 Subject: [PATCH 07/11] PR6 - changes to MapperParsingException in validatePatterns --- .../elasticsearch/index/mapper/DynamicTemplate.java | 2 +- .../index/mapper/DynamicTemplateParseTests.java | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java b/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java index cfb0052f89484..66ff2c31ed077 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java @@ -263,7 +263,7 @@ private static void validatePatterns(String templateName, MatchType matchType, L try { matchType.matches(regex, ""); } catch (IllegalArgumentException e) { - throw new IllegalArgumentException( + throw new MapperParsingException( Strings.format( "Pattern [%s] of type [%s] is invalid. Cannot create dynamic template [%s].", regex, diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateParseTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateParseTests.java index 9b3a37e113174..bf1683538b570 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateParseTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateParseTests.java @@ -136,10 +136,7 @@ public void testParseInvalidRegex() { templateDef.put(param, "*a"); templateDef.put("match_pattern", "regex"); templateDef.put("mapping", Collections.singletonMap("store", true)); - IllegalArgumentException e = expectThrows( - IllegalArgumentException.class, - () -> DynamicTemplate.parse("my_template", templateDef) - ); + MapperParsingException e = expectThrows(MapperParsingException.class, () -> DynamicTemplate.parse("my_template", templateDef)); assertEquals("Pattern [*a] of type [regex] is invalid. Cannot create dynamic template [my_template].", e.getMessage()); } } @@ -379,8 +376,11 @@ public void testSerialization() throws Exception { template = DynamicTemplate.parse("my_template", templateDef); builder = JsonXContent.contentBuilder(); template.toXContent(builder, ToXContent.EMPTY_PARAMS); - assertEquals(""" - {"path_match":["*name","user*"],"path_unmatch":["first_name","username"],"mapping":{"store":true}}""", Strings.toString(builder)); + assertEquals( + """ + {"path_match":["*name","user*"],"path_unmatch":["first_name","username"],"mapping":{"store":true}}""", + Strings.toString(builder) + ); // regex matching templateDef = new HashMap<>(); From 3b9f9d49f287c333fbc698554039f0154dcddf6c Mon Sep 17 00:00:00 2001 From: Michael Peterson Date: Wed, 26 Apr 2023 16:41:31 -0400 Subject: [PATCH 08/11] Added yamlRestTest indices.create/30_dynamic_template.yml --- .../indices.create/30_dynamic_template.yml | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/30_dynamic_template.yml diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/30_dynamic_template.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/30_dynamic_template.yml new file mode 100644 index 0000000000000..dbd3fc4a112a7 --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/30_dynamic_template.yml @@ -0,0 +1,59 @@ +--- +"Create index with dynamic_mappings, match is single-valued": + + - do: + indices.create: + index: test_index + body: + mappings: + dynamic_templates: + - mytemplate: + match: "*name" + mapping: + type: keyword + + - do: + indices.get_mapping: + index: test_index + + - is_true: test_index.mappings + + - match: { test_index.mappings.dynamic_templates.0.mytemplate.match: "*name"} + - match: { test_index.mappings.dynamic_templates.0.mytemplate.mapping.type: "keyword"} + +--- +"Create index with dynamic_mappings, using lists for some match/unmatch sections": + + - do: + indices.create: + index: test_index + body: + mappings: + dynamic_templates: + - mytemplate: + match: + - "*name" + - "user*" + unmatch: + - "*one" + - "two*" + path_match: + - "name.*" + - "user.name.*" + path_unmatch: "*.middle" + mapping: + type: keyword + + - do: + indices.get_mapping: + index: test_index + + - is_true: test_index.mappings + - match: { test_index.mappings.dynamic_templates.0.mytemplate.match.0: "*name"} + - match: { test_index.mappings.dynamic_templates.0.mytemplate.match.1: "user*"} + - match: { test_index.mappings.dynamic_templates.0.mytemplate.unmatch.0: "*one"} + - match: { test_index.mappings.dynamic_templates.0.mytemplate.unmatch.1: "two*"} + - match: { test_index.mappings.dynamic_templates.0.mytemplate.path_match.0: "name.*"} + - match: { test_index.mappings.dynamic_templates.0.mytemplate.path_match.1: "user.name.*"} + - match: { test_index.mappings.dynamic_templates.0.mytemplate.path_unmatch: "*.middle"} + - match: { test_index.mappings.dynamic_templates.0.mytemplate.mapping.type: "keyword" } From 4b2ddbd85acd17184b18c72127472035e336843b Mon Sep 17 00:00:00 2001 From: Michael Peterson Date: Thu, 27 Apr 2023 08:51:27 -0400 Subject: [PATCH 09/11] PR7 - Changed to using flatMap instead of concat for accumulating all patterns for validation --- .../org/elasticsearch/index/mapper/DynamicTemplate.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java b/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java index 66ff2c31ed077..33971cbb14405 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java @@ -21,7 +21,6 @@ import java.util.Locale; import java.util.Map; import java.util.TreeMap; -import java.util.stream.Collectors; import java.util.stream.Stream; public class DynamicTemplate implements ToXContentObject { @@ -220,10 +219,9 @@ static DynamicTemplate parse(String name, Map conf) throws Mappe } final MatchType matchType = MatchType.fromString(matchPattern); - List allPatterns = Stream.concat( - Stream.concat(match.stream(), unmatch.stream()), - Stream.concat(pathMatch.stream(), pathUnmatch.stream()) - ).collect(Collectors.toList()); + List allPatterns = Stream.of(match.stream(), unmatch.stream(), pathMatch.stream(), pathUnmatch.stream()) + .flatMap(s -> s) + .toList(); validatePatterns(name, matchType, allPatterns); return new DynamicTemplate(name, pathMatch, pathUnmatch, match, unmatch, xContentFieldTypes, matchType, mapping, runtime); From a0c7c391de2b7956ef2c8d6b1570bc77494b4ecd Mon Sep 17 00:00:00 2001 From: Michael Peterson Date: Thu, 27 Apr 2023 10:41:37 -0400 Subject: [PATCH 10/11] Add skip notes to new 30_dynamic_template.yml yamlRestTest to not run in 8.8.99 and earlier --- .../rest-api-spec/test/indices.create/30_dynamic_template.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/30_dynamic_template.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/30_dynamic_template.yml index dbd3fc4a112a7..a0fc4ad28eb4f 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/30_dynamic_template.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/30_dynamic_template.yml @@ -23,7 +23,9 @@ --- "Create index with dynamic_mappings, using lists for some match/unmatch sections": - + - skip: + version: " - 8.8.99" + reason: introduced in 8.9 - do: indices.create: index: test_index From 82f79ecd964e85e7ccde4fdeb61cb4a958d43aed Mon Sep 17 00:00:00 2001 From: Michael Peterson Date: Thu, 27 Apr 2023 10:43:17 -0400 Subject: [PATCH 11/11] Improved reason in skip notes in 30_dynamic_template.yml yamlRestTest --- .../rest-api-spec/test/indices.create/30_dynamic_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/30_dynamic_template.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/30_dynamic_template.yml index a0fc4ad28eb4f..648ebb1705123 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/30_dynamic_template.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/30_dynamic_template.yml @@ -25,7 +25,7 @@ "Create index with dynamic_mappings, using lists for some match/unmatch sections": - skip: version: " - 8.8.99" - reason: introduced in 8.9 + reason: Arrays in dynamic templates added in 8.9 - do: indices.create: index: test_index