From 0fe9c6cf0b8c29b8ed4025e07dde6c95d3fa30c4 Mon Sep 17 00:00:00 2001 From: jimczi Date: Thu, 4 Mar 2021 16:38:08 +0100 Subject: [PATCH 01/12] Expose if a field is a metadata field in the field capabilities response This change exposes for each field in the _field_caps response if the field is a metadata field. This is needed for consumers of this API that want to filter these fields. Currently ML keeps a static list and QL checks that the family type starts with `_`. In order to ease the addition of new metadata fields, this change reworks the strategy in this solution and now only checks for the new flag. Note that the new flag is also applied at the coordinator level in a best-effort to apply the logic on older nodes in a mixed-version cluster. --- .../org/elasticsearch/client/SearchIT.java | 6 +- .../search/fieldcaps/FieldCapabilitiesIT.java | 82 +++++++++++++++---- .../action/fieldcaps/FieldCapabilities.java | 45 +++++++--- .../FieldCapabilitiesIndexResponse.java | 7 ++ .../fieldcaps/FieldCapabilitiesResponse.java | 11 +++ .../fieldcaps/IndexFieldCapabilities.java | 20 ++++- .../TransportFieldCapabilitiesAction.java | 28 +++++-- ...TransportFieldCapabilitiesIndexAction.java | 19 +++-- .../elasticsearch/indices/IndicesService.java | 4 + .../FieldCapabilitiesResponseTests.java | 2 +- .../fieldcaps/FieldCapabilitiesTests.java | 27 +++--- .../MergedFieldCapabilitiesResponseTests.java | 7 +- .../analyses/ClassificationTests.java | 2 +- .../xpack/eql/analysis/CancellationTests.java | 6 +- .../extractor/ExtractedFieldsDetector.java | 11 ++- .../ml/dataframe/DestinationIndexTests.java | 2 +- .../ExtractedFieldsDetectorTests.java | 2 +- .../xpack/ql/index/IndexResolver.java | 54 ++++++------ .../analysis/index/IndexResolverTests.java | 17 ++-- .../common/DocumentConversionUtilsTests.java | 2 +- .../AggregationSchemaAndResultTests.java | 2 +- .../transforms/pivot/SchemaUtilTests.java | 1 + 22 files changed, 248 insertions(+), 109 deletions(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java index 80cb84f04ce41..c3c2b377b7a37 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java @@ -1247,11 +1247,11 @@ public void testFieldCaps() throws IOException { assertEquals(2, ratingResponse.size()); FieldCapabilities expectedKeywordCapabilities = new FieldCapabilities( - "rating", "keyword", true, true, new String[]{"index2"}, null, null, Collections.emptyMap()); + "rating", "keyword", false, true, true, new String[]{"index2"}, null, null, Collections.emptyMap()); assertEquals(expectedKeywordCapabilities, ratingResponse.get("keyword")); FieldCapabilities expectedLongCapabilities = new FieldCapabilities( - "rating", "long", true, true, new String[]{"index1"}, null, null, Collections.emptyMap()); + "rating", "long", false, true, true, new String[]{"index1"}, null, null, Collections.emptyMap()); assertEquals(expectedLongCapabilities, ratingResponse.get("long")); // Check the capabilities for the 'field' field. @@ -1260,7 +1260,7 @@ public void testFieldCaps() throws IOException { assertEquals(1, fieldResponse.size()); FieldCapabilities expectedTextCapabilities = new FieldCapabilities( - "field", "text", true, false, null, null, null, Collections.emptyMap()); + "field", "text", false, true, false, null, null, null, Collections.emptyMap()); assertEquals(expectedTextCapabilities, fieldResponse.get("text")); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java index 1b36abd97303b..dc8a3ff9d545d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java @@ -13,12 +13,16 @@ import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.mapper.KeywordFieldMapper; +import org.elasticsearch.index.mapper.MetadataFieldMapper; +import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.junit.Before; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -86,16 +90,9 @@ public void setUp() throws Exception { assertAcked(client().admin().indices().prepareAliases().addAlias("new_index", "current")); } - public static class FieldFilterPlugin extends Plugin implements MapperPlugin { - @Override - public Function> getFieldFilter() { - return index -> field -> field.equals("playlist") == false; - } - } - @Override protected Collection> nodePlugins() { - return Collections.singleton(FieldFilterPlugin.class); + return List.of(TestMapperPlugin.class); } public void testFieldAlias() { @@ -112,13 +109,13 @@ public void testFieldAlias() { assertTrue(distance.containsKey("double")); assertEquals( - new FieldCapabilities("distance", "double", true, true, new String[] {"old_index"}, null, null, + new FieldCapabilities("distance", "double", false, true, true, new String[] {"old_index"}, null, null, Collections.emptyMap()), distance.get("double")); assertTrue(distance.containsKey("text")); assertEquals( - new FieldCapabilities("distance", "text", true, false, new String[] {"new_index"}, null, null, + new FieldCapabilities("distance", "text", false, true, false, new String[] {"new_index"}, null, null, Collections.emptyMap()), distance.get("text")); @@ -128,7 +125,7 @@ public void testFieldAlias() { assertTrue(routeLength.containsKey("double")); assertEquals( - new FieldCapabilities("route_length_miles", "double", true, true, null, null, null, Collections.emptyMap()), + new FieldCapabilities("route_length_miles", "double", false, true, true, null, null, null, Collections.emptyMap()), routeLength.get("double")); } @@ -169,13 +166,13 @@ public void testWithUnmapped() { assertTrue(oldField.containsKey("long")); assertEquals( - new FieldCapabilities("old_field", "long", true, true, new String[] {"old_index"}, null, null, + new FieldCapabilities("old_field", "long", false, true, true, new String[] {"old_index"}, null, null, Collections.emptyMap()), oldField.get("long")); assertTrue(oldField.containsKey("unmapped")); assertEquals( - new FieldCapabilities("old_field", "unmapped", false, false, new String[] {"new_index"}, null, null, + new FieldCapabilities("old_field", "unmapped", false, false, false, new String[] {"new_index"}, null, null, Collections.emptyMap()), oldField.get("unmapped")); @@ -184,7 +181,7 @@ public void testWithUnmapped() { assertTrue(newField.containsKey("long")); assertEquals( - new FieldCapabilities("new_field", "long", true, true, null, null, null, Collections.emptyMap()), + new FieldCapabilities("new_field", "long", false, true, true, null, null, null, Collections.emptyMap()), newField.get("long")); } @@ -235,10 +232,67 @@ public void testWithIndexFilter() throws InterruptedException { assertTrue(newField.containsKey("keyword")); } + public void testMetaFields() { + for (int i = 0; i < 2; i++) { + String[] fields = i == 0 ? new String[] { "*" } : new String[] { "_id", "_test" }; + FieldCapabilitiesResponse response = client() + .prepareFieldCaps() + .setFields(fields) + .get(); + + Map idField = response.getField("_id"); + assertEquals(1, idField.size()); + + assertTrue(idField.containsKey("_id")); + assertEquals( + new FieldCapabilities("_id", "_id", true, true, false, null, null, null, Collections.emptyMap()), + idField.get("_id")); + + Map testField = response.getField("_test"); + assertEquals(1, testField.size()); + + assertTrue(testField.containsKey("keyword")); + assertEquals( + new FieldCapabilities("_test", "keyword", true, true, true, null, null, null, Collections.emptyMap()), + testField.get("keyword")); + } + } + private void assertIndices(FieldCapabilitiesResponse response, String... indices) { assertNotNull(response.getIndices()); Arrays.sort(indices); Arrays.sort(response.getIndices()); assertArrayEquals(indices, response.getIndices()); } + + public static final class TestMapperPlugin extends Plugin implements MapperPlugin { + @Override + public Map getMetadataMappers() { + return Collections.singletonMap(TestMetadataMapper.CONTENT_TYPE, TestMetadataMapper.PARSER); + } + + @Override + public Function> getFieldFilter() { + return index -> field -> field.equals("playlist") == false; + } + } + + private static final class TestMetadataMapper extends MetadataFieldMapper { + private static final String CONTENT_TYPE = "_test"; + private static final String FIELD_NAME = "_test"; + + protected TestMetadataMapper() { + super(new KeywordFieldMapper.KeywordFieldType(FIELD_NAME)); + } + + @Override + protected void parseCreateField(ParseContext context) throws IOException {} + + @Override + protected String contentType() { + return CONTENT_TYPE; + } + + private static final TypeParser PARSER = new FixedTypeParser(c -> new TestMetadataMapper()); + } } diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java index c88ab60b6da86..7c5e75b45d1d0 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java @@ -39,6 +39,7 @@ public class FieldCapabilities implements Writeable, ToXContentObject { private static final ParseField TYPE_FIELD = new ParseField("type"); + private static final ParseField METADATA_FIELD = new ParseField("metadata_field"); private static final ParseField SEARCHABLE_FIELD = new ParseField("searchable"); private static final ParseField AGGREGATABLE_FIELD = new ParseField("aggregatable"); private static final ParseField INDICES_FIELD = new ParseField("indices"); @@ -48,6 +49,7 @@ public class FieldCapabilities implements Writeable, ToXContentObject { private final String name; private final String type; + private final boolean isMetadataField; private final boolean isSearchable; private final boolean isAggregatable; @@ -61,6 +63,7 @@ public class FieldCapabilities implements Writeable, ToXContentObject { * Constructor for a set of indices. * @param name The name of the field * @param type The type associated with the field. + * @param isMetadataField Whether this field is a metadata field. * @param isSearchable Whether this field is indexed for search. * @param isAggregatable Whether this field can be aggregated on. * @param indices The list of indices where this field name is defined as {@code type}, @@ -72,13 +75,16 @@ public class FieldCapabilities implements Writeable, ToXContentObject { * @param meta Merged metadata across indices. */ public FieldCapabilities(String name, String type, - boolean isSearchable, boolean isAggregatable, + boolean isMetadataField, + boolean isSearchable, + boolean isAggregatable, String[] indices, String[] nonSearchableIndices, String[] nonAggregatableIndices, Map> meta) { this.name = name; this.type = type; + this.isMetadataField = isMetadataField; this.isSearchable = isSearchable; this.isAggregatable = isAggregatable; this.indices = indices; @@ -90,6 +96,7 @@ public FieldCapabilities(String name, String type, FieldCapabilities(StreamInput in) throws IOException { this.name = in.readString(); this.type = in.readString(); + this.isMetadataField = in.getVersion().onOrAfter(Version.V_8_0_0) ? in.readBoolean() : false; this.isSearchable = in.readBoolean(); this.isAggregatable = in.readBoolean(); this.indices = in.readOptionalStringArray(); @@ -106,6 +113,9 @@ public FieldCapabilities(String name, String type, public void writeTo(StreamOutput out) throws IOException { out.writeString(name); out.writeString(type); + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + out.writeBoolean(isMetadataField); + } out.writeBoolean(isSearchable); out.writeBoolean(isAggregatable); out.writeOptionalStringArray(indices); @@ -120,6 +130,9 @@ public void writeTo(StreamOutput out) throws IOException { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field(TYPE_FIELD.getPreferredName(), type); + if (isMetadataField) { + builder.field(METADATA_FIELD.getPreferredName(), isMetadataField); + } builder.field(SEARCHABLE_FIELD.getPreferredName(), isSearchable); builder.field(AGGREGATABLE_FIELD.getPreferredName(), isAggregatable); if (indices != null) { @@ -156,17 +169,19 @@ public static FieldCapabilities fromXContent(String name, XContentParser parser) true, (a, name) -> new FieldCapabilities(name, (String) a[0], + a[3] == null ? false : (boolean) a[3], (boolean) a[1], (boolean) a[2], - a[3] != null ? ((List) a[3]).toArray(new String[0]) : null, a[4] != null ? ((List) a[4]).toArray(new String[0]) : null, a[5] != null ? ((List) a[5]).toArray(new String[0]) : null, - a[6] != null ? ((Map>) a[6]) : Collections.emptyMap())); + a[6] != null ? ((List) a[6]).toArray(new String[0]) : null, + a[7] != null ? ((Map>) a[7]) : Collections.emptyMap())); static { PARSER.declareString(ConstructingObjectParser.constructorArg(), TYPE_FIELD); PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), SEARCHABLE_FIELD); PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), AGGREGATABLE_FIELD); + PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), METADATA_FIELD); PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), INDICES_FIELD); PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), NON_SEARCHABLE_INDICES_FIELD); PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), NON_AGGREGATABLE_INDICES_FIELD); @@ -181,6 +196,13 @@ public String getName() { return name; } + /** + * Whether this field is a metadata field. + */ + public boolean isMetaField() { + return isMetadataField; + } + /** * Whether this field can be aggregated on all indices. */ @@ -238,7 +260,8 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; FieldCapabilities that = (FieldCapabilities) o; - return isSearchable == that.isSearchable && + return isMetadataField == that.isMetadataField && + isSearchable == that.isSearchable && isAggregatable == that.isAggregatable && Objects.equals(name, that.name) && Objects.equals(type, that.type) && @@ -250,7 +273,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - int result = Objects.hash(name, type, isSearchable, isAggregatable, meta); + int result = Objects.hash(name, type, isMetadataField, isSearchable, isAggregatable, meta); result = 31 * result + Arrays.hashCode(indices); result = 31 * result + Arrays.hashCode(nonSearchableIndices); result = 31 * result + Arrays.hashCode(nonAggregatableIndices); @@ -263,8 +286,9 @@ public String toString() { } static class Builder { - private String name; - private String type; + private final String name; + private final String type; + private boolean isMetadataField; private boolean isSearchable; private boolean isAggregatable; private List indiceList; @@ -282,11 +306,12 @@ static class Builder { /** * Collect the field capabilities for an index. */ - void add(String index, boolean search, boolean agg, Map meta) { + void add(String index, boolean isMetaField, boolean search, boolean agg, Map meta) { IndexCaps indexCaps = new IndexCaps(index, search, agg); indiceList.add(indexCaps); this.isSearchable &= search; this.isAggregatable &= agg; + this.isMetadataField |= isMetaField; for (Map.Entry entry : meta.entrySet()) { this.meta.computeIfAbsent(entry.getKey(), key -> new HashSet<>()) .add(entry.getValue()); @@ -310,7 +335,7 @@ FieldCapabilities build(boolean withIndices) { final String[] nonSearchableIndices; if (isSearchable == false && - indiceList.stream().anyMatch((caps) -> caps.isSearchable)) { + indiceList.stream().anyMatch((caps) -> caps.isSearchable)) { // Iff this field is searchable in some indices AND non-searchable in others // we record the list of non-searchable indices nonSearchableIndices = indiceList.stream() @@ -337,7 +362,7 @@ FieldCapabilities build(boolean withIndices) { Map> immutableMeta = meta.entrySet().stream() .collect(Collectors.toUnmodifiableMap( Map.Entry::getKey, entryValueFunction.andThen(Set::copyOf))); - return new FieldCapabilities(name, type, isSearchable, isAggregatable, + return new FieldCapabilities(name, type, isMetadataField, isSearchable, isAggregatable, indices, nonSearchableIndices, nonAggregatableIndices, immutableMeta); } } diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexResponse.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexResponse.java index 28db4a9044073..919f131cd80cf 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexResponse.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexResponse.java @@ -25,11 +25,13 @@ public class FieldCapabilitiesIndexResponse extends ActionResponse implements Wr private final String indexName; private final Map responseMap; private final boolean canMatch; + private final transient Version originVersion; FieldCapabilitiesIndexResponse(String indexName, Map responseMap, boolean canMatch) { this.indexName = indexName; this.responseMap = responseMap; this.canMatch = canMatch; + this.originVersion = Version.CURRENT; } FieldCapabilitiesIndexResponse(StreamInput in) throws IOException { @@ -37,6 +39,7 @@ public class FieldCapabilitiesIndexResponse extends ActionResponse implements Wr this.indexName = in.readString(); this.responseMap = in.readMap(StreamInput::readString, IndexFieldCapabilities::new); this.canMatch = in.getVersion().onOrAfter(Version.V_7_9_0) ? in.readBoolean() : true; + this.originVersion = in.getVersion(); } /** @@ -65,6 +68,10 @@ public IndexFieldCapabilities getField(String field) { return responseMap.get(field); } + Version getOriginVersion() { + return originVersion; + } + @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(indexName); diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java index 6bf83a5802db6..b6e61fd8d4c7b 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java @@ -105,6 +105,17 @@ public Map getField(String field) { return responseMap.get(field); } + /** + * Returns true if the provided field is a metadata field. + */ + public boolean isMetadataField(String field) { + Map caps = getField(field); + if (caps == null) { + return false; + } + return caps.values().stream().anyMatch(FieldCapabilities::isMetaField); + } + private static Map readField(StreamInput in) throws IOException { return in.readMap(StreamInput::readString, FieldCapabilities::new); } diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/IndexFieldCapabilities.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/IndexFieldCapabilities.java index 480c50383e422..ec2eabc86b522 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/IndexFieldCapabilities.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/IndexFieldCapabilities.java @@ -26,6 +26,7 @@ public class IndexFieldCapabilities implements Writeable { private final String name; private final String type; + private final boolean isMetadatafield; private final boolean isSearchable; private final boolean isAggregatable; private final Map meta; @@ -38,11 +39,13 @@ public class IndexFieldCapabilities implements Writeable { * @param meta Metadata about the field. */ IndexFieldCapabilities(String name, String type, + boolean isMetadatafield, boolean isSearchable, boolean isAggregatable, Map meta) { this.name = name; this.type = type; + this.isMetadatafield = isMetadatafield; this.isSearchable = isSearchable; this.isAggregatable = isAggregatable; this.meta = meta; @@ -52,6 +55,7 @@ public class IndexFieldCapabilities implements Writeable { if (in.getVersion().onOrAfter(Version.V_7_7_0)) { this.name = in.readString(); this.type = in.readString(); + this.isMetadatafield = in.getVersion().onOrAfter(Version.V_8_0_0) ? in.readBoolean() : false; this.isSearchable = in.readBoolean(); this.isAggregatable = in.readBoolean(); this.meta = in.readMap(StreamInput::readString, StreamInput::readString); @@ -60,6 +64,7 @@ public class IndexFieldCapabilities implements Writeable { FieldCapabilities fieldCaps = new FieldCapabilities(in); this.name = fieldCaps.getName(); this.type = fieldCaps.getType(); + this.isMetadatafield = fieldCaps.isMetaField(); this.isSearchable = fieldCaps.isSearchable(); this.isAggregatable = fieldCaps.isAggregatable(); this.meta = fieldCaps.meta().entrySet().stream().collect(Collectors.toMap( @@ -73,6 +78,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_7_7_0)) { out.writeString(name); out.writeString(type); + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + out.writeBoolean(isMetadatafield); + } out.writeBoolean(isSearchable); out.writeBoolean(isAggregatable); out.writeMap(meta, StreamOutput::writeString, StreamOutput::writeString); @@ -81,7 +89,8 @@ public void writeTo(StreamOutput out) throws IOException { Map> wrappedMeta = meta.entrySet().stream().collect(Collectors.toMap( Map.Entry::getKey, entry -> Set.of(entry.getValue()))); - FieldCapabilities fieldCaps = new FieldCapabilities(name, type, isSearchable, isAggregatable, null, null, null, wrappedMeta); + FieldCapabilities fieldCaps = new FieldCapabilities(name, type, isMetadatafield, + isSearchable, isAggregatable, null, null, null, wrappedMeta); fieldCaps.writeTo(out); } } @@ -94,6 +103,10 @@ public String getType() { return type; } + public boolean isMetadatafield() { + return isMetadatafield; + } + public boolean isAggregatable() { return isAggregatable; } @@ -111,7 +124,8 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; IndexFieldCapabilities that = (IndexFieldCapabilities) o; - return isSearchable == that.isSearchable && + return isMetadatafield == that.isMetadatafield && + isSearchable == that.isSearchable && isAggregatable == that.isAggregatable && Objects.equals(name, that.name) && Objects.equals(type, that.type) && @@ -120,6 +134,6 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(name, type, isSearchable, isAggregatable, meta); + return Objects.hash(name, type, isMetadatafield, isSearchable, isAggregatable, meta); } } diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java index 3bcaaa23f4f38..3f6e63f28592a 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java @@ -8,6 +8,7 @@ package org.elasticsearch.action.fieldcaps; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.action.support.ActionFilters; @@ -20,6 +21,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.util.concurrent.CountDown; +import org.elasticsearch.indices.IndicesService; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.RemoteClusterAware; @@ -34,6 +36,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Predicate; public class TransportFieldCapabilitiesAction extends HandledTransportAction { private final ThreadPool threadPool; @@ -41,10 +44,15 @@ public class TransportFieldCapabilitiesAction extends HandledTransportAction metadataFieldPred; @Inject - public TransportFieldCapabilitiesAction(TransportService transportService, ClusterService clusterService, ThreadPool threadPool, - NodeClient client, ActionFilters actionFilters, + public TransportFieldCapabilitiesAction(TransportService transportService, + ClusterService clusterService, + ThreadPool threadPool, + NodeClient client, + ActionFilters actionFilters, + IndicesService indicesService, IndexNameExpressionResolver indexNameExpressionResolver) { super(FieldCapabilitiesAction.NAME, transportService, actionFilters, FieldCapabilitiesRequest::new); this.threadPool = threadPool; @@ -52,6 +60,9 @@ public TransportFieldCapabilitiesAction(TransportService transportService, Clust this.clusterService = clusterService; this.remoteClusterService = transportService.getRemoteClusterService(); this.indexNameExpressionResolver = indexNameExpressionResolver; + final Set metadataFields = indicesService.getMapperRegistry() + .getMetadataMapperParsers(Version.CURRENT).keySet(); + this.metadataFieldPred = metadataFields::contains; } @Override @@ -136,7 +147,7 @@ private FieldCapabilitiesResponse merge(List ind .toArray(String[]::new); final Map> responseMapBuilder = new HashMap<> (); for (FieldCapabilitiesIndexResponse response : indexResponses) { - innerMerge(responseMapBuilder, response.getIndexName(), response.get()); + innerMerge(responseMapBuilder, response); } final Map> responseMap = new HashMap<>(); for (Map.Entry> entry : responseMapBuilder.entrySet()) { @@ -163,20 +174,23 @@ private void addUnmappedFields(String[] indices, String field, Map> responseMapBuilder, - String indexName, Map map) { - for (Map.Entry entry : map.entrySet()) { + FieldCapabilitiesIndexResponse response) { + for (Map.Entry entry : response.get().entrySet()) { final String field = entry.getKey(); + // best effort to detect metadata field coming from older nodes + final boolean isMetadataField = response.getOriginVersion().onOrAfter(Version.V_8_0_0) ? + entry.getValue().isMetadatafield() : metadataFieldPred.test(field); final IndexFieldCapabilities fieldCap = entry.getValue(); Map typeMap = responseMapBuilder.computeIfAbsent(field, f -> new HashMap<>()); FieldCapabilities.Builder builder = typeMap.computeIfAbsent(fieldCap.getType(), key -> new FieldCapabilities.Builder(field, key)); - builder.add(indexName, fieldCap.isSearchable(), fieldCap.isAggregatable(), fieldCap.meta()); + builder.add(response.getIndexName(), isMetadataField, fieldCap.isSearchable(), fieldCap.isAggregatable(), fieldCap.meta()); } } } diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java index 0cd788fc71abf..9b434c1a8328e 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java @@ -82,7 +82,8 @@ public class TransportFieldCapabilitiesIndexAction private final Executor executor; @Inject - public TransportFieldCapabilitiesIndexAction(ClusterService clusterService, TransportService transportService, + public TransportFieldCapabilitiesIndexAction(ClusterService clusterService, + TransportService transportService, IndicesService indicesService, SearchService searchService, ThreadPool threadPool, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) { super(ACTION_NAME, transportService, actionFilters, FieldCapabilitiesIndexRequest::new); @@ -114,8 +115,10 @@ private FieldCapabilitiesIndexResponse shardOperation(final FieldCapabilitiesInd } Set fieldNames = new HashSet<>(); - for (String field : request.fields()) { - fieldNames.addAll(searchExecutionContext.simpleMatchToIndexNames(field)); + for (String pattern : request.fields()) { + for (String field : searchExecutionContext.simpleMatchToIndexNames(pattern)) { + fieldNames.add(field); + } } Predicate fieldPredicate = indicesService.getFieldFilter().apply(shardId.getIndexName()); @@ -123,10 +126,10 @@ private FieldCapabilitiesIndexResponse shardOperation(final FieldCapabilitiesInd for (String field : fieldNames) { MappedFieldType ft = searchExecutionContext.getFieldType(field); if (ft != null) { - if (searchExecutionContext.isMetadataField(field) - || fieldPredicate.test(ft.name())) { - IndexFieldCapabilities fieldCap = new IndexFieldCapabilities(field, ft.familyTypeName(), - ft.isSearchable(), ft.isAggregatable(), ft.meta()); + boolean isMetadataField = searchExecutionContext.isMetadataField(field); + if (isMetadataField || fieldPredicate.test(ft.name())) { + IndexFieldCapabilities fieldCap = new IndexFieldCapabilities(field, + ft.familyTypeName(), isMetadataField, ft.isSearchable(), ft.isAggregatable(), ft.meta()); responseMap.put(field, fieldCap); } else { continue; @@ -148,7 +151,7 @@ private FieldCapabilitiesIndexResponse shardOperation(final FieldCapabilitiesInd ObjectMapper mapper = searchExecutionContext.getObjectMapper(parentField); String type = mapper.nested().isNested() ? "nested" : "object"; IndexFieldCapabilities fieldCap = new IndexFieldCapabilities(parentField, type, - false, false, Collections.emptyMap()); + false, false, false, Collections.emptyMap()); responseMap.put(parentField, fieldCap); } dotIndex = parentField.lastIndexOf('.'); diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java index 82efbe8b5bf1c..b7d56564e711c 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -829,6 +829,10 @@ public IndicesQueryCache getIndicesQueryCache() { return indicesQueryCache; } + public MapperRegistry getMapperRegistry() { + return mapperRegistry; + } + static class OldShardsStats implements IndexEventListener { final SearchStats searchStats = new SearchStats(); diff --git a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java index 0bd51b823a46e..02cdb7f22f87b 100644 --- a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java @@ -64,7 +64,7 @@ private static IndexFieldCapabilities randomFieldCaps(String fieldName) { } return new IndexFieldCapabilities(fieldName, randomAlphaOfLengthBetween(5, 20), - randomBoolean(), randomBoolean(), meta); + randomBoolean(), randomBoolean(), randomBoolean(), meta); } @Override diff --git a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesTests.java b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesTests.java index 6f64bf017442b..dcae8389f0288 100644 --- a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesTests.java +++ b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesTests.java @@ -40,9 +40,9 @@ protected Writeable.Reader instanceReader() { public void testBuilder() { FieldCapabilities.Builder builder = new FieldCapabilities.Builder("field", "type"); - builder.add("index1", true, false, Collections.emptyMap()); - builder.add("index2", true, false, Collections.emptyMap()); - builder.add("index3", true, false, Collections.emptyMap()); + builder.add("index1", false, true, false, Collections.emptyMap()); + builder.add("index2", false, true, false, Collections.emptyMap()); + builder.add("index3", false, true, false, Collections.emptyMap()); { FieldCapabilities cap1 = builder.build(false); @@ -64,9 +64,9 @@ public void testBuilder() { } builder = new FieldCapabilities.Builder("field", "type"); - builder.add("index1", false, true, Collections.emptyMap()); - builder.add("index2", true, false, Collections.emptyMap()); - builder.add("index3", false, false, Collections.emptyMap()); + builder.add("index1", false, false, true, Collections.emptyMap()); + builder.add("index2", false, true, false, Collections.emptyMap()); + builder.add("index3", false, false, false, Collections.emptyMap()); { FieldCapabilities cap1 = builder.build(false); assertThat(cap1.isSearchable(), equalTo(false)); @@ -87,9 +87,9 @@ public void testBuilder() { } builder = new FieldCapabilities.Builder("field", "type"); - builder.add("index1", true, true, Collections.emptyMap()); - builder.add("index2", true, true, Map.of("foo", "bar")); - builder.add("index3", true, true, Map.of("foo", "quux")); + builder.add("index1", false, true, true, Collections.emptyMap()); + builder.add("index2", false, true, true, Map.of("foo", "bar")); + builder.add("index3", false, true, true, Map.of("foo", "quux")); { FieldCapabilities cap1 = builder.build(false); assertThat(cap1.isSearchable(), equalTo(true)); @@ -147,7 +147,7 @@ static FieldCapabilities randomFieldCaps(String fieldName) { } return new FieldCapabilities(fieldName, - randomAlphaOfLengthBetween(5, 20), randomBoolean(), randomBoolean(), + randomAlphaOfLengthBetween(5, 20), randomBoolean(), randomBoolean(), randomBoolean(), indices, nonSearchableIndices, nonAggregatableIndices, meta); } @@ -155,6 +155,7 @@ static FieldCapabilities randomFieldCaps(String fieldName) { protected FieldCapabilities mutateInstance(FieldCapabilities instance) { String name = instance.getName(); String type = instance.getType(); + boolean isMetaField = instance.isMetaField(); boolean isSearchable = instance.isSearchable(); boolean isAggregatable = instance.isAggregatable(); String[] indices = instance.indices(); @@ -225,9 +226,13 @@ protected FieldCapabilities mutateInstance(FieldCapabilities instance) { } meta = newMeta; break; + case 8: + isMetaField = isMetaField == false; + break; default: throw new AssertionError(); } - return new FieldCapabilities(name, type, isSearchable, isAggregatable, indices, nonSearchableIndices, nonAggregatableIndices, meta); + return new FieldCapabilities(name, type, isMetaField, isSearchable, isAggregatable, + indices, nonSearchableIndices, nonAggregatableIndices, meta); } } diff --git a/server/src/test/java/org/elasticsearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java b/server/src/test/java/org/elasticsearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java index bd16539f93ac6..781f67cd20e03 100644 --- a/server/src/test/java/org/elasticsearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java @@ -141,16 +141,17 @@ public void testEmptyResponse() throws IOException { private static FieldCapabilitiesResponse createSimpleResponse() { Map titleCapabilities = new HashMap<>(); - titleCapabilities.put("text", new FieldCapabilities("title", "text", true, false, null, null, null, Collections.emptyMap())); + titleCapabilities.put("text", new FieldCapabilities("title", "text", false, true, false, + null, null, null, Collections.emptyMap())); Map ratingCapabilities = new HashMap<>(); ratingCapabilities.put("long", new FieldCapabilities("rating", "long", - true, false, + false, true, false, new String[]{"index1", "index2"}, null, new String[]{"index1"}, Collections.emptyMap())); ratingCapabilities.put("keyword", new FieldCapabilities("rating", "keyword", - false, true, + false, false, true, new String[]{"index3", "index4"}, new String[]{"index4"}, null, Collections.emptyMap())); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java index 8dba910aabba3..08b439dc8f928 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java @@ -518,6 +518,6 @@ public Long getCardinality(String field) { } private static FieldCapabilities createFieldCapabilities(String field, String type) { - return new FieldCapabilities(field, type, true, true, null, null, null, Collections.emptyMap()); + return new FieldCapabilities(field, type, false, true, true, null, null, null, Collections.emptyMap()); } } diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/CancellationTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/CancellationTests.java index d69f522f8bf47..7d074ea626a0d 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/CancellationTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/CancellationTests.java @@ -84,11 +84,11 @@ public void onFailure(Exception e) { private Map> fields(String[] indices) { FieldCapabilities fooField = - new FieldCapabilities("foo", "integer", true, true, indices, null, null, emptyMap()); + new FieldCapabilities("foo", "integer", false, true, true, indices, null, null, emptyMap()); FieldCapabilities categoryField = - new FieldCapabilities("event.category", "keyword", true, true, indices, null, null, emptyMap()); + new FieldCapabilities("event.category", "keyword", false, true, true, indices, null, null, emptyMap()); FieldCapabilities timestampField = - new FieldCapabilities("@timestamp", "date", true, true, indices, null, null, emptyMap()); + new FieldCapabilities("@timestamp", "date", false, true, true, indices, null, null, emptyMap()); Map> fields = new HashMap<>(); fields.put(fooField.getName(), singletonMap(fooField.getName(), fooField)); fields.put(categoryField.getName(), singletonMap(categoryField.getName(), categoryField)); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetector.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetector.java index bb8a1e071fc86..53ef1ff5d69be 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetector.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetector.java @@ -48,6 +48,7 @@ import java.util.Objects; import java.util.Set; import java.util.TreeSet; +import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -58,9 +59,7 @@ public class ExtractedFieldsDetector { /** * Fields to ignore. These are mostly internal meta fields. */ - private static final List IGNORE_FIELDS = Arrays.asList("_id", "_field_names", "_index", "_parent", "_routing", "_seq_no", - "_source", "_type", "_uid", "_version", "_feature", "_ignored", "_nested_path", DestinationIndex.INCREMENTAL_ID, - "_data_stream_timestamp", "_doc_count"); + private static final List IGNORE_FIELDS = Collections.singletonList(DestinationIndex.INCREMENTAL_ID); private final DataFrameAnalyticsConfig config; private final int docValueFieldsLimit; @@ -100,7 +99,11 @@ public Tuple> detect() { } private Set getIncludedFields(Set fieldSelection, Set requiredFieldsForProcessors) { - Set fields = new TreeSet<>(fieldCapabilitiesResponse.get().keySet()); + Set fields = new TreeSet<>(); + // filter metadata field + fieldCapabilitiesResponse.get().keySet().stream() + .filter(Predicate.not(fieldCapabilitiesResponse::isMetadataField)) + .forEach(fields::add); validateFieldsRequireForProcessors(requiredFieldsForProcessors); fields.removeAll(IGNORE_FIELDS); removeFieldsUnderResultsField(fields); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndexTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndexTests.java index 4e284306840be..0fe48f85c18f2 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndexTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndexTests.java @@ -470,6 +470,6 @@ private static DataFrameAnalyticsConfig createConfig(DataFrameAnalysis analysis) } private static FieldCapabilities createFieldCapabilities(String field, String type) { - return new FieldCapabilities(field, type, true, true, null, null, null, Collections.emptyMap()); + return new FieldCapabilities(field, type, false, true, true, null, null, null, Collections.emptyMap()); } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java index a8a52673f1627..b045a1a6b2406 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java @@ -1228,7 +1228,7 @@ private MockFieldCapsResponseBuilder addNonAggregatableField(String field, Strin private MockFieldCapsResponseBuilder addField(String field, boolean isAggregatable, String... types) { Map caps = new HashMap<>(); for (String type : types) { - caps.put(type, new FieldCapabilities(field, type, true, isAggregatable, null, null, null, Collections.emptyMap())); + caps.put(type, new FieldCapabilities(field, type, false, true, isAggregatable, null, null, null, Collections.emptyMap())); } fieldCaps.put(field, caps); return this; diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java index 98ae62e660145..c5a82b7d97db3 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java @@ -25,7 +25,6 @@ import org.elasticsearch.cluster.metadata.AliasMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.ImmutableOpenMap; -import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.xpack.ql.QlIllegalArgumentException; import org.elasticsearch.xpack.ql.type.DataType; @@ -158,7 +157,6 @@ public boolean equals(Object obj) { EnumSet.of(Option.ALLOW_NO_INDICES, Option.IGNORE_UNAVAILABLE), EnumSet.of(WildcardStates.OPEN)); - private static final Set FIELD_NAMES_BLACKLIST = Sets.newHashSet("_size", "_doc_count"); private static final String UNMAPPED = "unmapped"; private final Client client; @@ -284,7 +282,7 @@ public void resolveAsMergedMapping(String indexWildcard, String javaRegex, Indic FieldCapabilitiesRequest fieldRequest = createFieldCapsRequest(indexWildcard, indicesOptions); client.fieldCaps(fieldRequest, ActionListener.wrap( - response -> listener.onResponse(mergedMappings(typeRegistry, indexWildcard, response.getIndices(), response.get())), + response -> listener.onResponse(mergedMappings(typeRegistry, indexWildcard, response)), listener::onFailure)); } @@ -296,19 +294,19 @@ public void resolveAsMergedMapping(String indexWildcard, String javaRegex, boole FieldCapabilitiesRequest fieldRequest = createFieldCapsRequest(indexWildcard, includeFrozen); client.fieldCaps(fieldRequest, ActionListener.wrap( - response -> listener.onResponse(mergedMappings(typeRegistry, indexWildcard, response.getIndices(), response.get())), + response -> listener.onResponse(mergedMappings(typeRegistry, indexWildcard, response)), listener::onFailure)); } - public static IndexResolution mergedMappings(DataTypeRegistry typeRegistry, String indexPattern, String[] indexNames, - Map> fieldCaps) { + public static IndexResolution mergedMappings(DataTypeRegistry typeRegistry, String indexPattern, + FieldCapabilitiesResponse fieldCapsResponse) { - if (indexNames.length == 0) { + if (fieldCapsResponse.getIndices().length == 0) { return IndexResolution.notFound(indexPattern); } // merge all indices onto the same one - List indices = buildIndices(typeRegistry, indexNames, null, fieldCaps, null, i -> indexPattern, (n, types) -> { + List indices = buildIndices(typeRegistry, null, fieldCapsResponse, null, i -> indexPattern, (n, types) -> { StringBuilder errorMessage = new StringBuilder(); boolean hasUnmapped = types.containsKey(UNMAPPED); @@ -368,7 +366,8 @@ public static IndexResolution mergedMappings(DataTypeRegistry typeRegistry, Stri indices.size()); } - return IndexResolution.valid(indices.isEmpty() ? new EsIndex(indexNames[0], emptyMap()) : indices.get(0)); + final String indexName= fieldCapsResponse.getIndices()[0]; + return IndexResolution.valid(indices.isEmpty() ? new EsIndex(indexName, emptyMap()) : indices.get(0)); } private static EsField createField(DataTypeRegistry typeRegistry, String fieldName, @@ -479,10 +478,10 @@ public void resolveAsSeparateMappings(String indexWildcard, String javaRegex, bo FieldCapabilitiesRequest fieldRequest = createFieldCapsRequest(indexWildcard, includeFrozen); client.fieldCaps(fieldRequest, wrap(response -> { client.admin().indices().getAliases(createGetAliasesRequest(response, includeFrozen), wrap(aliases -> - listener.onResponse(separateMappings(typeRegistry, javaRegex, response.getIndices(), response.get(), aliases.getAliases())), + listener.onResponse(separateMappings(typeRegistry, javaRegex, response, aliases.getAliases())), ex -> { if (ex instanceof IndexNotFoundException || ex instanceof ElasticsearchSecurityException) { - listener.onResponse(separateMappings(typeRegistry, javaRegex, response.getIndices(), response.get(), null)); + listener.onResponse(separateMappings(typeRegistry, javaRegex, response, null)); } else { listener.onFailure(ex); } @@ -500,9 +499,11 @@ private GetAliasesRequest createGetAliasesRequest(FieldCapabilitiesResponse resp .indicesOptions(includeFrozen ? FIELD_CAPS_FROZEN_INDICES_OPTIONS : FIELD_CAPS_INDICES_OPTIONS); } - public static List separateMappings(DataTypeRegistry typeRegistry, String javaRegex, String[] indexNames, - Map> fieldCaps, ImmutableOpenMap> aliases) { - return buildIndices(typeRegistry, indexNames, javaRegex, fieldCaps, aliases, Function.identity(), (s, cap) -> null); + public static List separateMappings(DataTypeRegistry typeRegistry, + String javaRegex, + FieldCapabilitiesResponse fieldCaps, + ImmutableOpenMap> aliases) { + return buildIndices(typeRegistry, javaRegex, fieldCaps, aliases, Function.identity(), (s, cap) -> null); } private static class Fields { @@ -514,12 +515,13 @@ private static class Fields { * Assemble an index-based mapping from the field caps (which is field based) by looking at the indices associated with * each field. */ - private static List buildIndices(DataTypeRegistry typeRegistry, String[] indexNames, String javaRegex, - Map> fieldCaps, ImmutableOpenMap> aliases, + private static List buildIndices(DataTypeRegistry typeRegistry, String javaRegex, + FieldCapabilitiesResponse fieldCapsResponse, ImmutableOpenMap> aliases, Function indexNameProcessor, BiFunction, InvalidMappedField> validityVerifier) { - if ((indexNames == null || indexNames.length == 0) && (aliases == null || aliases.isEmpty())) { + if ((fieldCapsResponse.getIndices() == null || fieldCapsResponse.getIndices().length == 0) + && (aliases == null || aliases.isEmpty())) { return emptyList(); } @@ -533,7 +535,7 @@ private static List buildIndices(DataTypeRegistry typeRegistry, String[ } } - List resolvedIndices = new ArrayList<>(asList(indexNames)); + List resolvedIndices = new ArrayList<>(asList(fieldCapsResponse.getIndices())); int mapSize = CollectionUtils.mapSize(resolvedIndices.size() + resolvedAliases.size()); Map indices = new LinkedHashMap<>(mapSize); Pattern pattern = javaRegex != null ? Pattern.compile(javaRegex) : null; @@ -541,23 +543,21 @@ private static List buildIndices(DataTypeRegistry typeRegistry, String[ // sort fields in reverse order to build the field hierarchy Set>> sortedFields = new TreeSet<>( Collections.reverseOrder(Comparator.comparing(Entry::getKey))); - + final Map> fieldCaps = fieldCapsResponse.get(); sortedFields.addAll(fieldCaps.entrySet()); for (Entry> entry : sortedFields) { String fieldName = entry.getKey(); - - // ignore size added by the mapper plugin - if (FIELD_NAMES_BLACKLIST.contains(fieldName)) { + if (fieldCapsResponse.isMetadataField(fieldName)) { + // skip metadata field! continue; } - Map types = new LinkedHashMap<>(entry.getValue()); final InvalidMappedField invalidField = validityVerifier.apply(fieldName, types); // apply verification for fields belonging to index aliases Map invalidFieldsForAliases = getInvalidFieldsForAliases(fieldName, types, aliases); - // filter meta fields and unmapped + // filter unmapped FieldCapabilities unmapped = types.get(UNMAPPED); Set unmappedIndices = unmapped != null ? new HashSet<>(asList(unmapped.indices())) : emptySet(); @@ -566,12 +566,6 @@ private static List buildIndices(DataTypeRegistry typeRegistry, String[ FieldCapabilities typeCap = typeEntry.getValue(); String[] capIndices = typeCap.indices(); - // Skip internal fields (name starting with underscore and its type reported by field_caps starts - // with underscore as well). A meta field named "_version", for example, has the type named "_version". - if (typeEntry.getKey().startsWith("_") && typeCap.getType().startsWith("_")) { - continue; - } - // compute the actual indices - if any are specified, take into account the unmapped indices List concreteIndices = null; if (capIndices != null) { diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java index 535fb8e19b825..3131acff170d8 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java @@ -15,6 +15,7 @@ import java.util.stream.Stream; import org.elasticsearch.action.fieldcaps.FieldCapabilities; +import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.ql.index.EsIndex; import org.elasticsearch.xpack.ql.index.IndexResolution; @@ -246,9 +247,9 @@ public void testMergeIncompatibleCapabilitiesOfObjectFields() throws Exception { addFieldCaps(fieldCaps, fieldName + ".keyword", "keyword", true, true); Map multi = new HashMap<>(); - multi.put("long", new FieldCapabilities(fieldName, "long", true, true, new String[] { "one-index" }, null, null, + multi.put("long", new FieldCapabilities(fieldName, "long", false, true, true, new String[] { "one-index" }, null, null, Collections.emptyMap())); - multi.put("text", new FieldCapabilities(fieldName, "text", true, false, new String[] { "another-index" }, null, null, + multi.put("text", new FieldCapabilities(fieldName, "text", false, true, false, new String[] { "another-index" }, null, null, Collections.emptyMap())); fieldCaps.put(fieldName, multi); @@ -319,7 +320,7 @@ public void testMultipleCompatibleIndicesWithDifferentFields() { public void testIndexWithNoMapping() { Map> versionFC = singletonMap("_version", - singletonMap("_index", new FieldCapabilities("_version", "_version", false, false, + singletonMap("_index", new FieldCapabilities("_version", "_version", true, false, false, null, null, null, Collections.emptyMap()))); assertTrue(mergedMappings("*", new String[] { "empty" }, versionFC).isValid()); } @@ -395,7 +396,7 @@ private static class UpdateableFieldCapabilities extends FieldCapabilities { List nonAggregatableIndices = new ArrayList<>(); UpdateableFieldCapabilities(String name, String type, boolean isSearchable, boolean isAggregatable) { - super(name, type, isSearchable, isAggregatable, null, null, null, Collections.emptyMap()); + super(name, type, false, isSearchable, isAggregatable, null, null, null, Collections.emptyMap()); } @Override @@ -429,17 +430,19 @@ private static void assertEqualsMaps(Map left, Map right) { private void addFieldCaps(Map> fieldCaps, String name, String type, boolean isSearchable, boolean isAggregatable) { Map cap = new HashMap<>(); - cap.put(type, new FieldCapabilities(name, type, isSearchable, isAggregatable, null, null, null, Collections.emptyMap())); + cap.put(type, new FieldCapabilities(name, type, false, isSearchable, isAggregatable, null, null, null, Collections.emptyMap())); fieldCaps.put(name, cap); } private static IndexResolution mergedMappings(String indexPattern, String[] indexNames, Map> fieldCaps) { - return IndexResolver.mergedMappings(SqlDataTypeRegistry.INSTANCE, indexPattern, indexNames, fieldCaps); + return IndexResolver.mergedMappings(SqlDataTypeRegistry.INSTANCE, indexPattern, + new FieldCapabilitiesResponse(indexNames, fieldCaps)); } private static List separateMappings(String javaRegex, String[] indexNames, Map> fieldCaps) { - return IndexResolver.separateMappings(SqlDataTypeRegistry.INSTANCE, javaRegex, indexNames, fieldCaps, null); + return IndexResolver.separateMappings(SqlDataTypeRegistry.INSTANCE, javaRegex, + new FieldCapabilitiesResponse(indexNames, fieldCaps), null); } } diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/common/DocumentConversionUtilsTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/common/DocumentConversionUtilsTests.java index 68192279c69a9..eb583f2f3ce81 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/common/DocumentConversionUtilsTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/common/DocumentConversionUtilsTests.java @@ -117,6 +117,6 @@ public void testExtractFieldMappings() { private static FieldCapabilities createFieldCapabilities(String name, String type) { return new FieldCapabilities( - name, type, true, true, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, Collections.emptyMap()); + name, type, false, true, true, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, Collections.emptyMap()); } } diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationSchemaAndResultTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationSchemaAndResultTests.java index faf74844e56c6..4e9b2d5f69c3c 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationSchemaAndResultTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationSchemaAndResultTests.java @@ -90,7 +90,7 @@ protected void field, Collections.singletonMap( type, - new FieldCapabilities(field, type, true, true, null, null, null, emptyMap()) + new FieldCapabilities(field, type, false, true, true, null, null, null, emptyMap()) ) ); } diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/SchemaUtilTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/SchemaUtilTests.java index b17734122220c..d406393b8b15a 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/SchemaUtilTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/SchemaUtilTests.java @@ -214,6 +214,7 @@ private static FieldCapabilities createFieldCapabilities(String name, String typ return new FieldCapabilities( name, type, + false, true, true, Strings.EMPTY_ARRAY, From 49b4237594dd730855b949fef85a06cbe238b876 Mon Sep 17 00:00:00 2001 From: jimczi Date: Fri, 5 Mar 2021 11:08:20 +0100 Subject: [PATCH 02/12] remove unneeded mock and fix wrong expectation in ExtractedFieldsDetectorTests --- .../extractor/ExtractedFieldsDetector.java | 20 ++++++++----- .../ExtractedFieldsDetectorTests.java | 29 +++++++++---------- 2 files changed, 27 insertions(+), 22 deletions(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetector.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetector.java index 53ef1ff5d69be..a4a9a6e63e8b5 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetector.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetector.java @@ -38,6 +38,7 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.HashSet; @@ -99,13 +100,13 @@ public Tuple> detect() { } private Set getIncludedFields(Set fieldSelection, Set requiredFieldsForProcessors) { + validateFieldsRequireForProcessors(requiredFieldsForProcessors); Set fields = new TreeSet<>(); // filter metadata field fieldCapabilitiesResponse.get().keySet().stream() - .filter(Predicate.not(fieldCapabilitiesResponse::isMetadataField)) + .filter(f -> fieldCapabilitiesResponse.isMetadataField(f) == false + && IGNORE_FIELDS.contains(f) == false) .forEach(fields::add); - validateFieldsRequireForProcessors(requiredFieldsForProcessors); - fields.removeAll(IGNORE_FIELDS); removeFieldsUnderResultsField(fields); removeObjects(fields); applySourceFiltering(fields); @@ -141,9 +142,14 @@ private void validateFieldsRequireForProcessors(Set processorFields) { if (fieldsForProcessor.size() < processorFields.size()) { throw ExceptionsHelper.badRequestException("fields for feature_processors must not be objects"); } - fieldsForProcessor.removeAll(IGNORE_FIELDS); - if (fieldsForProcessor.size() < processorFields.size()) { - throw ExceptionsHelper.badRequestException("the following fields cannot be used in feature_processors {}", IGNORE_FIELDS); + Collection errorFields = new ArrayList<>(); + for (String fieldName : fieldsForProcessor) { + if (fieldCapabilitiesResponse.isMetadataField(fieldName) || IGNORE_FIELDS.contains(fieldName)) { + errorFields.add(fieldName); + } + } + if (errorFields.isEmpty() == false) { + throw ExceptionsHelper.badRequestException("the following fields cannot be used in feature_processors {}", errorFields); } List fieldsMissingInMapping = processorFields.stream() .filter(f -> fieldCapabilitiesResponse.get().containsKey(f) == false) @@ -303,7 +309,7 @@ private void applyIncludesExcludes(Set fields, Set includes, Set while (fieldsIterator.hasNext()) { String field = fieldsIterator.next(); if (includes.contains(field)) { - if (IGNORE_FIELDS.contains(field)) { + if (fieldCapabilitiesResponse.isMetadataField(field) || IGNORE_FIELDS.contains(field)) { throw ExceptionsHelper.badRequestException("field [{}] cannot be analyzed", field); } if (excludes.contains(field)) { diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java index b045a1a6b2406..2b89ee84b8857 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java @@ -45,8 +45,6 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; public class ExtractedFieldsDetectorTests extends ESTestCase { @@ -318,7 +316,7 @@ public void testDetect_GivenClassificationAndDependentVariableHasInvalidCardinal public void testDetect_GivenIgnoredField() { FieldCapabilitiesResponse fieldCapabilities = new MockFieldCapsResponseBuilder() - .addAggregatableField("_id", "float").build(); + .addField("_id", true, true, "float").build(); ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector( buildOutlierDetectionConfig(), 100, fieldCapabilities, Collections.emptyMap()); @@ -330,7 +328,7 @@ public void testDetect_GivenIgnoredField() { public void testDetect_GivenIncludedIgnoredField() { FieldCapabilitiesResponse fieldCapabilities = new MockFieldCapsResponseBuilder() - .addAggregatableField("_id", "float") + .addField("_id", true, false, "float") .build(); analyzedFields = new FetchSourceContext(true, new String[]{"_id"}, new String[0]); @@ -801,17 +799,17 @@ public void testDetect_GivenParentAndMultiFieldBothAggregatable() { buildRegressionConfig("field_2.double"), 100, fieldCapabilities, Collections.emptyMap()); Tuple> fieldExtraction = extractedFieldsDetector.detect(); - assertThat(fieldExtraction.v1().getAllFields(), hasSize(2)); + assertThat(fieldExtraction.v1().getAllFields(), hasSize(3)); List extractedFieldNames = fieldExtraction.v1().getAllFields().stream().map(ExtractedField::getName) .collect(Collectors.toList()); - assertThat(extractedFieldNames, contains("field_1", "field_2.double")); + assertThat(extractedFieldNames, contains("field_1", "field_2.double", "field_2.keyword")); assertFieldSelectionContains(fieldExtraction.v2(), FieldSelection.included("field_1", Collections.singleton("keyword"), false, FieldSelection.FeatureType.CATEGORICAL), FieldSelection.excluded("field_1.keyword", Collections.singleton("keyword"), "[field_1] is preferred because it is aggregatable"), FieldSelection.included("field_2.double", Collections.singleton("double"), true, FieldSelection.FeatureType.NUMERICAL), - FieldSelection.excluded("field_2.keyword", Collections.singleton("float"), "[field_2.double] is required instead") + FieldSelection.included("field_2.keyword", Collections.singleton("float"), false, FieldSelection.FeatureType.NUMERICAL) ); } @@ -940,6 +938,7 @@ public void testDetect_GivenAnalyzedFieldIncludesObjectField() { private static FieldCapabilitiesResponse simpleFieldResponse() { return new MockFieldCapsResponseBuilder() + .addField("_id", true, false, "_id") .addAggregatableField("field_11", "float") .addNonAggregatableField("field_21", "float") .addAggregatableField("field_21.child", "float") @@ -1226,22 +1225,22 @@ private MockFieldCapsResponseBuilder addNonAggregatableField(String field, Strin } private MockFieldCapsResponseBuilder addField(String field, boolean isAggregatable, String... types) { + return addField(field, false, isAggregatable, types); + } + + private MockFieldCapsResponseBuilder addField(String field, boolean isMetadataField, + boolean isAggregatable, String... types) { Map caps = new HashMap<>(); for (String type : types) { - caps.put(type, new FieldCapabilities(field, type, false, true, isAggregatable, null, null, null, Collections.emptyMap())); + caps.put(type, new FieldCapabilities(field, type, + isMetadataField, true, isAggregatable, null, null, null, Collections.emptyMap())); } fieldCaps.put(field, caps); return this; } private FieldCapabilitiesResponse build() { - FieldCapabilitiesResponse response = mock(FieldCapabilitiesResponse.class); - when(response.get()).thenReturn(fieldCaps); - - for (String field : fieldCaps.keySet()) { - when(response.getField(field)).thenReturn(fieldCaps.get(field)); - } - return response; + return new FieldCapabilitiesResponse(new String[] { "test" }, fieldCaps); } } } From be91028161ab41c0fc6c10e1595eb806ae6041ba Mon Sep 17 00:00:00 2001 From: jimczi Date: Fri, 5 Mar 2021 11:34:10 +0100 Subject: [PATCH 03/12] unused import --- .../xpack/ml/dataframe/extractor/ExtractedFieldsDetector.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetector.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetector.java index a4a9a6e63e8b5..6f19524f744dd 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetector.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetector.java @@ -49,7 +49,6 @@ import java.util.Objects; import java.util.Set; import java.util.TreeSet; -import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; From 937bbaf62fb5487882bb5e26aaee33b2f416f74d Mon Sep 17 00:00:00 2001 From: jimczi Date: Fri, 5 Mar 2021 13:10:28 +0100 Subject: [PATCH 04/12] fix sql IndexResolverTests --- .../analysis/index/IndexResolverTests.java | 29 ++++++++++++++----- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java index 3131acff170d8..af11af45c72eb 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java @@ -122,10 +122,10 @@ public void testMultiLevelNestedMappings() throws Exception { public void testMetaFieldsAreIgnored() throws Exception { Map> fieldCaps = new HashMap<>(); - addFieldCaps(fieldCaps, "_version", "_version", false, false); - addFieldCaps(fieldCaps, "_meta_field", "integer", true, true); - addFieldCaps(fieldCaps, "_size", "integer", true, true); - addFieldCaps(fieldCaps, "_doc_count", "long", false, false); + addFieldCaps(fieldCaps, "_version", "_version", true, false, false); + addFieldCaps(fieldCaps, "_not_meta_field", "integer", false, true, true); + addFieldCaps(fieldCaps, "_size", "integer", true, true, true); + addFieldCaps(fieldCaps, "_doc_count", "long", true, false, false); addFieldCaps(fieldCaps, "text", "keyword", true, true); String wildcard = "*"; @@ -137,7 +137,7 @@ public void testMetaFieldsAreIgnored() throws Exception { assertNull(esIndex.mapping().get("_version")); assertNull(esIndex.mapping().get("_size")); assertNull(esIndex.mapping().get("_doc_count")); - assertEquals(INTEGER, esIndex.mapping().get("_meta_field").getDataType()); + assertEquals(INTEGER, esIndex.mapping().get("_not_meta_field").getDataType()); assertEquals(KEYWORD, esIndex.mapping().get("text").getDataType()); } @@ -427,10 +427,23 @@ private static void assertEqualsMaps(Map left, Map right) { } } - private void addFieldCaps(Map> fieldCaps, String name, String type, boolean isSearchable, - boolean isAggregatable) { + private void addFieldCaps(Map> fieldCaps, + String name, + String type, + boolean isSearchable, + boolean isAggregatable) { + addFieldCaps(fieldCaps, name, type, false, isSearchable, isAggregatable); + } + + private void addFieldCaps(Map> fieldCaps, + String name, + String type, + boolean isMetadataField, + boolean isSearchable, + boolean isAggregatable) { Map cap = new HashMap<>(); - cap.put(type, new FieldCapabilities(name, type, false, isSearchable, isAggregatable, null, null, null, Collections.emptyMap())); + cap.put(type, new FieldCapabilities(name, type, isMetadataField, + isSearchable, isAggregatable, null, null, null, Collections.emptyMap())); fieldCaps.put(name, cap); } From 512102261b9567fe695183a21f44f1fea78840cd Mon Sep 17 00:00:00 2001 From: jimczi Date: Mon, 8 Mar 2021 18:24:11 +0100 Subject: [PATCH 05/12] Apply review comments --- .../elasticsearch/action/fieldcaps/FieldCapabilities.java | 8 ++++---- .../fieldcaps/TransportFieldCapabilitiesIndexAction.java | 4 +--- .../action/fieldcaps/FieldCapabilitiesTests.java | 8 ++++---- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java index 7c5e75b45d1d0..2a9b5c874ef2d 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java @@ -39,7 +39,7 @@ public class FieldCapabilities implements Writeable, ToXContentObject { private static final ParseField TYPE_FIELD = new ParseField("type"); - private static final ParseField METADATA_FIELD = new ParseField("metadata_field"); + private static final ParseField IS_METADATA_FIELD = new ParseField("metadata_field"); private static final ParseField SEARCHABLE_FIELD = new ParseField("searchable"); private static final ParseField AGGREGATABLE_FIELD = new ParseField("aggregatable"); private static final ParseField INDICES_FIELD = new ParseField("indices"); @@ -131,7 +131,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(); builder.field(TYPE_FIELD.getPreferredName(), type); if (isMetadataField) { - builder.field(METADATA_FIELD.getPreferredName(), isMetadataField); + builder.field(IS_METADATA_FIELD.getPreferredName(), isMetadataField); } builder.field(SEARCHABLE_FIELD.getPreferredName(), isSearchable); builder.field(AGGREGATABLE_FIELD.getPreferredName(), isAggregatable); @@ -181,7 +181,7 @@ public static FieldCapabilities fromXContent(String name, XContentParser parser) PARSER.declareString(ConstructingObjectParser.constructorArg(), TYPE_FIELD); PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), SEARCHABLE_FIELD); PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), AGGREGATABLE_FIELD); - PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), METADATA_FIELD); + PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), IS_METADATA_FIELD); PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), INDICES_FIELD); PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), NON_SEARCHABLE_INDICES_FIELD); PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), NON_AGGREGATABLE_INDICES_FIELD); @@ -199,7 +199,7 @@ public String getName() { /** * Whether this field is a metadata field. */ - public boolean isMetaField() { + public boolean isMetadataField() { return isMetadataField; } diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java index 9b434c1a8328e..746901ba95625 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java @@ -116,9 +116,7 @@ private FieldCapabilitiesIndexResponse shardOperation(final FieldCapabilitiesInd Set fieldNames = new HashSet<>(); for (String pattern : request.fields()) { - for (String field : searchExecutionContext.simpleMatchToIndexNames(pattern)) { - fieldNames.add(field); - } + fieldNames.addAll(searchExecutionContext.simpleMatchToIndexNames(pattern)); } Predicate fieldPredicate = indicesService.getFieldFilter().apply(shardId.getIndexName()); diff --git a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesTests.java b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesTests.java index dcae8389f0288..6b93bac3d3796 100644 --- a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesTests.java +++ b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesTests.java @@ -155,14 +155,14 @@ static FieldCapabilities randomFieldCaps(String fieldName) { protected FieldCapabilities mutateInstance(FieldCapabilities instance) { String name = instance.getName(); String type = instance.getType(); - boolean isMetaField = instance.isMetaField(); + boolean isMetadataField = instance.isMetadataField(); boolean isSearchable = instance.isSearchable(); boolean isAggregatable = instance.isAggregatable(); String[] indices = instance.indices(); String[] nonSearchableIndices = instance.nonSearchableIndices(); String[] nonAggregatableIndices = instance.nonAggregatableIndices(); Map> meta = instance.meta(); - switch (between(0, 7)) { + switch (between(0, 8)) { case 0: name += randomAlphaOfLengthBetween(1, 10); break; @@ -227,12 +227,12 @@ protected FieldCapabilities mutateInstance(FieldCapabilities instance) { meta = newMeta; break; case 8: - isMetaField = isMetaField == false; + isMetadataField = isMetadataField == false; break; default: throw new AssertionError(); } - return new FieldCapabilities(name, type, isMetaField, isSearchable, isAggregatable, + return new FieldCapabilities(name, type, isMetadataField, isSearchable, isAggregatable, indices, nonSearchableIndices, nonAggregatableIndices, meta); } } From ea0dced56ba5c9317c0915e100df04252c8dfbd6 Mon Sep 17 00:00:00 2001 From: jimczi Date: Tue, 9 Mar 2021 10:02:20 +0100 Subject: [PATCH 06/12] propagate renaming --- .../action/fieldcaps/FieldCapabilitiesResponse.java | 2 +- .../elasticsearch/action/fieldcaps/IndexFieldCapabilities.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java index b6e61fd8d4c7b..173036afd12d6 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java @@ -113,7 +113,7 @@ public boolean isMetadataField(String field) { if (caps == null) { return false; } - return caps.values().stream().anyMatch(FieldCapabilities::isMetaField); + return caps.values().stream().anyMatch(FieldCapabilities::isMetadataField); } private static Map readField(StreamInput in) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/IndexFieldCapabilities.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/IndexFieldCapabilities.java index ec2eabc86b522..957ca7dced6fc 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/IndexFieldCapabilities.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/IndexFieldCapabilities.java @@ -64,7 +64,7 @@ public class IndexFieldCapabilities implements Writeable { FieldCapabilities fieldCaps = new FieldCapabilities(in); this.name = fieldCaps.getName(); this.type = fieldCaps.getType(); - this.isMetadatafield = fieldCaps.isMetaField(); + this.isMetadatafield = fieldCaps.isMetadataField(); this.isSearchable = fieldCaps.isSearchable(); this.isAggregatable = fieldCaps.isAggregatable(); this.meta = fieldCaps.meta().entrySet().stream().collect(Collectors.toMap( From 78fb83ed12c190e52aa24eca0cd8b814271f5bee Mon Sep 17 00:00:00 2001 From: jimczi Date: Wed, 17 Mar 2021 09:34:26 +0100 Subject: [PATCH 07/12] address review --- .../action/fieldcaps/FieldCapabilities.java | 4 +--- .../fieldcaps/TransportFieldCapabilitiesAction.java | 3 +-- .../org/elasticsearch/indices/IndicesService.java | 12 ++++++++---- .../dataframe/extractor/ExtractedFieldsDetector.java | 2 +- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java index 2a9b5c874ef2d..2e80a8b3ad3c9 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java @@ -130,9 +130,7 @@ public void writeTo(StreamOutput out) throws IOException { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field(TYPE_FIELD.getPreferredName(), type); - if (isMetadataField) { - builder.field(IS_METADATA_FIELD.getPreferredName(), isMetadataField); - } + builder.field(IS_METADATA_FIELD.getPreferredName(), isMetadataField); builder.field(SEARCHABLE_FIELD.getPreferredName(), isSearchable); builder.field(AGGREGATABLE_FIELD.getPreferredName(), isAggregatable); if (indices != null) { diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java index 50eccca2a495f..7afb1fa7e7e42 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java @@ -60,8 +60,7 @@ public TransportFieldCapabilitiesAction(TransportService transportService, this.clusterService = clusterService; this.remoteClusterService = transportService.getRemoteClusterService(); this.indexNameExpressionResolver = indexNameExpressionResolver; - final Set metadataFields = indicesService.getMapperRegistry() - .getMetadataMapperParsers(Version.CURRENT).keySet(); + final Set metadataFields = indicesService.getMetadataFields(Version.CURRENT); this.metadataFieldPred = metadataFields::contains; } diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java index b7d56564e711c..f1ac651f5d99f 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -18,6 +18,7 @@ import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ResourceAlreadyExistsException; +import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.stats.CommonStats; import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags.Flag; @@ -829,10 +830,6 @@ public IndicesQueryCache getIndicesQueryCache() { return indicesQueryCache; } - public MapperRegistry getMapperRegistry() { - return mapperRegistry; - } - static class OldShardsStats implements IndexEventListener { final SearchStats searchStats = new SearchStats(); @@ -1579,6 +1576,13 @@ public Function> getFieldFilter() { return mapperRegistry.getFieldFilter(); } + /** + * Returns the registered metadata field names. + */ + public Set getMetadataFields(Version version) { + return mapperRegistry.getMetadataMapperParsers(version).keySet(); + } + /** * Returns true if fielddata is enabled for the {@link IdFieldMapper} field, false otherwise. */ diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetector.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetector.java index 6f19524f744dd..ef1dcc9b86599 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetector.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetector.java @@ -57,7 +57,7 @@ public class ExtractedFieldsDetector { private static final Logger LOGGER = LogManager.getLogger(ExtractedFieldsDetector.class); /** - * Fields to ignore. These are mostly internal meta fields. + * Internal fields to ignore. */ private static final List IGNORE_FIELDS = Collections.singletonList(DestinationIndex.INCREMENTAL_ID); From d908d179209f538f68b86c8937affe8887a811ca Mon Sep 17 00:00:00 2001 From: jimczi Date: Wed, 17 Mar 2021 09:44:40 +0100 Subject: [PATCH 08/12] add docs --- docs/reference/search/field-caps.asciidoc | 10 ++++++++++ .../action/fieldcaps/FieldCapabilities.java | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/reference/search/field-caps.asciidoc b/docs/reference/search/field-caps.asciidoc index d2ce46fedc746..5468484ca08b8 100644 --- a/docs/reference/search/field-caps.asciidoc +++ b/docs/reference/search/field-caps.asciidoc @@ -106,6 +106,8 @@ described using a type family. For example, `keyword`, `constant_keyword` and `w field types are all described as the `keyword` type family. +`is_metadata_field`:: + Whether this field is registered as a <>. `searchable`:: Whether this field is indexed for search on all indices. @@ -162,12 +164,14 @@ The API returns the following response: "fields": { "rating": { <1> "long": { + "is_metadata_field": false, "searchable": true, "aggregatable": false, "indices": [ "index1", "index2" ], "non_aggregatable_indices": [ "index1" ] <2> }, "keyword": { + "is_metadata_field": false, "searchable": false, "aggregatable": true, "indices": [ "index3", "index4" ], @@ -176,6 +180,7 @@ The API returns the following response: }, "title": { <4> "text": { + "is_metadata_field": false, "searchable": true, "aggregatable": false @@ -211,18 +216,21 @@ in some indices but not all: "fields": { "rating": { "long": { + "is_metadata_field": false, "searchable": true, "aggregatable": false, "indices": [ "index1", "index2" ], "non_aggregatable_indices": [ "index1" ] }, "keyword": { + "is_metadata_field": false, "searchable": false, "aggregatable": true, "indices": [ "index3", "index4" ], "non_searchable_indices": [ "index4" ] }, "unmapped": { <1> + "is_metadata_field": false, "indices": [ "index5" ], "searchable": false, "aggregatable": false @@ -230,11 +238,13 @@ in some indices but not all: }, "title": { "text": { + "is_metadata_field": false, "indices": [ "index1", "index2", "index3", "index4" ], "searchable": true, "aggregatable": false }, "unmapped": { <2> + "is_metadata_field": false, "indices": [ "index5" ], "searchable": false, "aggregatable": false diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java index 2e80a8b3ad3c9..0b2f3719d25ad 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java @@ -39,7 +39,7 @@ public class FieldCapabilities implements Writeable, ToXContentObject { private static final ParseField TYPE_FIELD = new ParseField("type"); - private static final ParseField IS_METADATA_FIELD = new ParseField("metadata_field"); + private static final ParseField IS_METADATA_FIELD = new ParseField("is_metadata_field"); private static final ParseField SEARCHABLE_FIELD = new ParseField("searchable"); private static final ParseField AGGREGATABLE_FIELD = new ParseField("aggregatable"); private static final ParseField INDICES_FIELD = new ParseField("indices"); From c1ec5513e20490cd6bd836c6ab686d8c2a932142 Mon Sep 17 00:00:00 2001 From: jimczi Date: Wed, 17 Mar 2021 12:10:36 +0100 Subject: [PATCH 09/12] fix ut --- .../action/fieldcaps/MergedFieldCapabilitiesResponseTests.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/src/test/java/org/elasticsearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java b/server/src/test/java/org/elasticsearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java index 781f67cd20e03..bfd33876b3be9 100644 --- a/server/src/test/java/org/elasticsearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java @@ -110,6 +110,7 @@ public void testToXContent() throws IOException { " \"rating\": { " + " \"keyword\": {" + " \"type\": \"keyword\"," + + " \"is_metadata_field\": false," + " \"searchable\": false," + " \"aggregatable\": true," + " \"indices\": [\"index3\", \"index4\"]," + @@ -117,6 +118,7 @@ public void testToXContent() throws IOException { " }," + " \"long\": {" + " \"type\": \"long\"," + + " \"is_metadata_field\": false," + " \"searchable\": true," + " \"aggregatable\": false," + " \"indices\": [\"index1\", \"index2\"]," + @@ -126,6 +128,7 @@ public void testToXContent() throws IOException { " \"title\": { " + " \"text\": {" + " \"type\": \"text\"," + + " \"is_metadata_field\": false," + " \"searchable\": true," + " \"aggregatable\": false" + " }" + From 87fee7e64bb1c0a443f86d56e3d4440ed0ba3037 Mon Sep 17 00:00:00 2001 From: jimczi Date: Thu, 25 Mar 2021 12:19:25 +0100 Subject: [PATCH 10/12] more review --- .../elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java | 2 +- .../org/elasticsearch/action/fieldcaps/FieldCapabilities.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java index dc8a3ff9d545d..1eb3639c872fd 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java @@ -232,7 +232,7 @@ public void testWithIndexFilter() throws InterruptedException { assertTrue(newField.containsKey("keyword")); } - public void testMetaFields() { + public void testMetadataFields() { for (int i = 0; i < 2; i++) { String[] fields = i == 0 ? new String[] { "*" } : new String[] { "_id", "_test" }; FieldCapabilitiesResponse response = client() diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java index 0b2f3719d25ad..51691c553bc43 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java @@ -304,12 +304,12 @@ static class Builder { /** * Collect the field capabilities for an index. */ - void add(String index, boolean isMetaField, boolean search, boolean agg, Map meta) { + void add(String index, boolean isMetadataField, boolean search, boolean agg, Map meta) { IndexCaps indexCaps = new IndexCaps(index, search, agg); indiceList.add(indexCaps); this.isSearchable &= search; this.isAggregatable &= agg; - this.isMetadataField |= isMetaField; + this.isMetadataField |= isMetadataField; for (Map.Entry entry : meta.entrySet()) { this.meta.computeIfAbsent(entry.getKey(), key -> new HashSet<>()) .add(entry.getValue()); From 24f068a5b8c47e863de4fc60fb1f07acfc6024e5 Mon Sep 17 00:00:00 2001 From: jimczi Date: Fri, 26 Mar 2021 14:26:04 +0100 Subject: [PATCH 11/12] rename is_metadata_field => metadata_field --- docs/reference/search/field-caps.asciidoc | 18 +++++++++--------- .../action/fieldcaps/FieldCapabilities.java | 2 +- .../MergedFieldCapabilitiesResponseTests.java | 6 +++--- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/reference/search/field-caps.asciidoc b/docs/reference/search/field-caps.asciidoc index 5468484ca08b8..3238421166252 100644 --- a/docs/reference/search/field-caps.asciidoc +++ b/docs/reference/search/field-caps.asciidoc @@ -106,7 +106,7 @@ described using a type family. For example, `keyword`, `constant_keyword` and `w field types are all described as the `keyword` type family. -`is_metadata_field`:: +`metadata_field`:: Whether this field is registered as a <>. `searchable`:: @@ -164,14 +164,14 @@ The API returns the following response: "fields": { "rating": { <1> "long": { - "is_metadata_field": false, + "metadata_field": false, "searchable": true, "aggregatable": false, "indices": [ "index1", "index2" ], "non_aggregatable_indices": [ "index1" ] <2> }, "keyword": { - "is_metadata_field": false, + "metadata_field": false, "searchable": false, "aggregatable": true, "indices": [ "index3", "index4" ], @@ -180,7 +180,7 @@ The API returns the following response: }, "title": { <4> "text": { - "is_metadata_field": false, + "metadata_field": false, "searchable": true, "aggregatable": false @@ -216,21 +216,21 @@ in some indices but not all: "fields": { "rating": { "long": { - "is_metadata_field": false, + "metadata_field": false, "searchable": true, "aggregatable": false, "indices": [ "index1", "index2" ], "non_aggregatable_indices": [ "index1" ] }, "keyword": { - "is_metadata_field": false, + "metadata_field": false, "searchable": false, "aggregatable": true, "indices": [ "index3", "index4" ], "non_searchable_indices": [ "index4" ] }, "unmapped": { <1> - "is_metadata_field": false, + "metadata_field": false, "indices": [ "index5" ], "searchable": false, "aggregatable": false @@ -238,13 +238,13 @@ in some indices but not all: }, "title": { "text": { - "is_metadata_field": false, + "metadata_field": false, "indices": [ "index1", "index2", "index3", "index4" ], "searchable": true, "aggregatable": false }, "unmapped": { <2> - "is_metadata_field": false, + "metadata_field": false, "indices": [ "index5" ], "searchable": false, "aggregatable": false diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java index 51691c553bc43..f528ed068a7d3 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java @@ -39,7 +39,7 @@ public class FieldCapabilities implements Writeable, ToXContentObject { private static final ParseField TYPE_FIELD = new ParseField("type"); - private static final ParseField IS_METADATA_FIELD = new ParseField("is_metadata_field"); + private static final ParseField IS_METADATA_FIELD = new ParseField("metadata_field"); private static final ParseField SEARCHABLE_FIELD = new ParseField("searchable"); private static final ParseField AGGREGATABLE_FIELD = new ParseField("aggregatable"); private static final ParseField INDICES_FIELD = new ParseField("indices"); diff --git a/server/src/test/java/org/elasticsearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java b/server/src/test/java/org/elasticsearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java index bfd33876b3be9..e614afcb93455 100644 --- a/server/src/test/java/org/elasticsearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java @@ -110,7 +110,7 @@ public void testToXContent() throws IOException { " \"rating\": { " + " \"keyword\": {" + " \"type\": \"keyword\"," + - " \"is_metadata_field\": false," + + " \"metadata_field\": false," + " \"searchable\": false," + " \"aggregatable\": true," + " \"indices\": [\"index3\", \"index4\"]," + @@ -118,7 +118,7 @@ public void testToXContent() throws IOException { " }," + " \"long\": {" + " \"type\": \"long\"," + - " \"is_metadata_field\": false," + + " \"metadata_field\": false," + " \"searchable\": true," + " \"aggregatable\": false," + " \"indices\": [\"index1\", \"index2\"]," + @@ -128,7 +128,7 @@ public void testToXContent() throws IOException { " \"title\": { " + " \"text\": {" + " \"type\": \"text\"," + - " \"is_metadata_field\": false," + + " \"metadata_field\": false," + " \"searchable\": true," + " \"aggregatable\": false" + " }" + From 0b3c922b782ce57b14254601d65b98cde5c32f7c Mon Sep 17 00:00:00 2001 From: jimczi Date: Fri, 26 Mar 2021 14:34:21 +0100 Subject: [PATCH 12/12] clarify getAllMetadataParsers --- .../fieldcaps/TransportFieldCapabilitiesAction.java | 2 +- .../org/elasticsearch/index/mapper/MapperRegistry.java | 7 +++++++ .../java/org/elasticsearch/indices/IndicesService.java | 9 ++++++++- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java index 7afb1fa7e7e42..bca8676668de7 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java @@ -60,7 +60,7 @@ public TransportFieldCapabilitiesAction(TransportService transportService, this.clusterService = clusterService; this.remoteClusterService = transportService.getRemoteClusterService(); this.indexNameExpressionResolver = indexNameExpressionResolver; - final Set metadataFields = indicesService.getMetadataFields(Version.CURRENT); + final Set metadataFields = indicesService.getAllMetadataFields(); this.metadataFieldPred = metadataFields::contains; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperRegistry.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperRegistry.java index b1f0d9d8a9107..5f53a856d4261 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperRegistry.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperRegistry.java @@ -64,6 +64,13 @@ public Map getMetadataMapperParsers(Vers return metadataMapperParsers7x; } + /** + * Return a map of all meta mappers that have been registered in all compatible versions. + */ + public Map getAllMetadataMapperParsers() { + return metadataMapperParsers; + } + /** * Returns a function that given an index name, returns a predicate that fields must match in order to be returned by get mappings, * get index, get field mappings and field capabilities API. Useful to filter the fields that such API return. diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java index f1ac651f5d99f..fc58bc270a8a9 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -1577,12 +1577,19 @@ public Function> getFieldFilter() { } /** - * Returns the registered metadata field names. + * Returns the registered metadata field names for the provided compatible {@link Version}. */ public Set getMetadataFields(Version version) { return mapperRegistry.getMetadataMapperParsers(version).keySet(); } + /** + * Returns the registered metadata field names for all compatible versions. + */ + public Set getAllMetadataFields() { + return mapperRegistry.getAllMetadataMapperParsers().keySet(); + } + /** * Returns true if fielddata is enabled for the {@link IdFieldMapper} field, false otherwise. */