Expose if a field is a metadata field in the field capabilities response

client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java

@@ -1247,11 +1247,11 @@ public void testFieldCaps() throws IOException {
         assertEquals(2, ratingResponse.size());
 
         FieldCapabilities expectedKeywordCapabilities = new FieldCapabilities(
-            "rating", "keyword", true, true, new String[]{"index2"}, null, null, Collections.emptyMap());
+            "rating", "keyword", false, true, true, new String[]{"index2"}, null, null, Collections.emptyMap());
         assertEquals(expectedKeywordCapabilities, ratingResponse.get("keyword"));
 
         FieldCapabilities expectedLongCapabilities = new FieldCapabilities(
-            "rating", "long", true, true, new String[]{"index1"}, null, null, Collections.emptyMap());
+            "rating", "long", false, true, true, new String[]{"index1"}, null, null, Collections.emptyMap());
         assertEquals(expectedLongCapabilities, ratingResponse.get("long"));
 
         // Check the capabilities for the 'field' field.
@@ -1260,7 +1260,7 @@ public void testFieldCaps() throws IOException {
         assertEquals(1, fieldResponse.size());
 
         FieldCapabilities expectedTextCapabilities = new FieldCapabilities(
-            "field", "text", true, false, null, null, null, Collections.emptyMap());
+            "field", "text", false, true, false, null, null, null, Collections.emptyMap());
         assertEquals(expectedTextCapabilities, fieldResponse.get("text"));
     }
 

server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java

@@ -13,12 +13,16 @@
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.index.mapper.KeywordFieldMapper;
+import org.elasticsearch.index.mapper.MetadataFieldMapper;
+import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.junit.Before;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -86,16 +90,9 @@ public void setUp() throws Exception {
         assertAcked(client().admin().indices().prepareAliases().addAlias("new_index", "current"));
     }
 
-    public static class FieldFilterPlugin extends Plugin implements MapperPlugin {
-        @Override
-        public Function<String, Predicate<String>> getFieldFilter() {
-            return index -> field -> field.equals("playlist") == false;
-        }
-    }
-
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Collections.singleton(FieldFilterPlugin.class);
+        return List.of(TestMapperPlugin.class);
     }
 
     public void testFieldAlias() {
@@ -112,13 +109,13 @@ public void testFieldAlias() {
 
         assertTrue(distance.containsKey("double"));
         assertEquals(
-            new FieldCapabilities("distance", "double", true, true, new String[] {"old_index"}, null, null,
+            new FieldCapabilities("distance", "double", false, true, true, new String[] {"old_index"}, null, null,
                     Collections.emptyMap()),
             distance.get("double"));
 
         assertTrue(distance.containsKey("text"));
         assertEquals(
-            new FieldCapabilities("distance", "text", true, false, new String[] {"new_index"}, null, null,
+            new FieldCapabilities("distance", "text", false, true, false, new String[] {"new_index"}, null, null,
                     Collections.emptyMap()),
             distance.get("text"));
 
@@ -128,7 +125,7 @@ public void testFieldAlias() {
 
         assertTrue(routeLength.containsKey("double"));
         assertEquals(
-            new FieldCapabilities("route_length_miles", "double", true, true, null, null, null, Collections.emptyMap()),
+            new FieldCapabilities("route_length_miles", "double", false, true, true, null, null, null, Collections.emptyMap()),
             routeLength.get("double"));
     }
 
@@ -169,13 +166,13 @@ public void testWithUnmapped() {
 
         assertTrue(oldField.containsKey("long"));
         assertEquals(
-            new FieldCapabilities("old_field", "long", true, true, new String[] {"old_index"}, null, null,
+            new FieldCapabilities("old_field", "long", false, true, true, new String[] {"old_index"}, null, null,
                     Collections.emptyMap()),
             oldField.get("long"));
 
         assertTrue(oldField.containsKey("unmapped"));
         assertEquals(
-            new FieldCapabilities("old_field", "unmapped", false, false, new String[] {"new_index"}, null, null,
+            new FieldCapabilities("old_field", "unmapped", false, false, false, new String[] {"new_index"}, null, null,
                     Collections.emptyMap()),
             oldField.get("unmapped"));
 
@@ -184,7 +181,7 @@ public void testWithUnmapped() {
 
         assertTrue(newField.containsKey("long"));
         assertEquals(
-            new FieldCapabilities("new_field", "long", true, true, null, null, null, Collections.emptyMap()),
+            new FieldCapabilities("new_field", "long", false, true, true, null, null, null, Collections.emptyMap()),
             newField.get("long"));
     }
 
@@ -235,10 +232,67 @@ public void testWithIndexFilter() throws InterruptedException {
         assertTrue(newField.containsKey("keyword"));
     }
 
+    public void testMetaFields() {
+        for (int i = 0; i < 2; i++) {
+            String[] fields = i == 0 ? new String[] { "*" } : new String[] { "_id", "_test" };
+            FieldCapabilitiesResponse response = client()
+                .prepareFieldCaps()
+                .setFields(fields)
+                .get();
+
+            Map<String, FieldCapabilities> idField = response.getField("_id");
+            assertEquals(1, idField.size());
+
+            assertTrue(idField.containsKey("_id"));
+            assertEquals(
+                new FieldCapabilities("_id", "_id", true, true, false, null, null, null, Collections.emptyMap()),
+                idField.get("_id"));
+
+            Map<String, FieldCapabilities> testField = response.getField("_test");
+            assertEquals(1, testField.size());
+
+            assertTrue(testField.containsKey("keyword"));
+            assertEquals(
+                new FieldCapabilities("_test", "keyword", true, true, true, null, null, null, Collections.emptyMap()),
+                testField.get("keyword"));
+        }
+    }
+
     private void assertIndices(FieldCapabilitiesResponse response, String... indices) {
         assertNotNull(response.getIndices());
         Arrays.sort(indices);
         Arrays.sort(response.getIndices());
         assertArrayEquals(indices, response.getIndices());
     }
+
+    public static final class TestMapperPlugin extends Plugin implements MapperPlugin {
+        @Override
+        public Map<String, MetadataFieldMapper.TypeParser> getMetadataMappers() {
+            return Collections.singletonMap(TestMetadataMapper.CONTENT_TYPE, TestMetadataMapper.PARSER);
+        }
+
+        @Override
+        public Function<String, Predicate<String>> getFieldFilter() {
+            return index -> field -> field.equals("playlist") == false;
+        }
+    }
+
+    private static final class TestMetadataMapper extends MetadataFieldMapper {
+        private static final String CONTENT_TYPE = "_test";
+        private static final String FIELD_NAME = "_test";
+
+        protected TestMetadataMapper() {
+            super(new KeywordFieldMapper.KeywordFieldType(FIELD_NAME));
+        }
+
+        @Override
+        protected void parseCreateField(ParseContext context) throws IOException {}
+
+        @Override
+        protected String contentType() {
+            return CONTENT_TYPE;
+        }
+
+        private static final TypeParser PARSER = new FixedTypeParser(c -> new TestMetadataMapper());
+    }
 }

server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java

@@ -39,6 +39,7 @@
 public class FieldCapabilities implements Writeable, ToXContentObject {
 
     private static final ParseField TYPE_FIELD = new ParseField("type");
+    private static final ParseField METADATA_FIELD = new ParseField("metadata_field");
     private static final ParseField SEARCHABLE_FIELD = new ParseField("searchable");
     private static final ParseField AGGREGATABLE_FIELD = new ParseField("aggregatable");
     private static final ParseField INDICES_FIELD = new ParseField("indices");
@@ -48,6 +49,7 @@ public class FieldCapabilities implements Writeable, ToXContentObject {
 
     private final String name;
     private final String type;
+    private final boolean isMetadataField;
     private final boolean isSearchable;
     private final boolean isAggregatable;
 
@@ -61,6 +63,7 @@ public class FieldCapabilities implements Writeable, ToXContentObject {
      * Constructor for a set of indices.
      * @param name The name of the field
      * @param type The type associated with the field.
+     * @param isMetadataField Whether this field is a metadata field.
      * @param isSearchable Whether this field is indexed for search.
      * @param isAggregatable Whether this field can be aggregated on.
      * @param indices The list of indices where this field name is defined as {@code type},
@@ -72,13 +75,16 @@ public class FieldCapabilities implements Writeable, ToXContentObject {
      * @param meta Merged metadata across indices.
      */
     public FieldCapabilities(String name, String type,
-                             boolean isSearchable, boolean isAggregatable,
+                             boolean isMetadataField,
+                             boolean isSearchable,
+                             boolean isAggregatable,
                              String[] indices,
                              String[] nonSearchableIndices,
                              String[] nonAggregatableIndices,
                              Map<String, Set<String>> meta) {
         this.name = name;
         this.type = type;
+        this.isMetadataField = isMetadataField;
         this.isSearchable = isSearchable;
         this.isAggregatable = isAggregatable;
         this.indices = indices;
@@ -90,6 +96,7 @@ public FieldCapabilities(String name, String type,
     FieldCapabilities(StreamInput in) throws IOException {
         this.name = in.readString();
         this.type = in.readString();
+        this.isMetadataField = in.getVersion().onOrAfter(Version.V_8_0_0) ? in.readBoolean() : false;
         this.isSearchable = in.readBoolean();
         this.isAggregatable = in.readBoolean();
         this.indices = in.readOptionalStringArray();
@@ -106,6 +113,9 @@ public FieldCapabilities(String name, String type,
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(name);
         out.writeString(type);
+        if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+            out.writeBoolean(isMetadataField);
+        }
         out.writeBoolean(isSearchable);
         out.writeBoolean(isAggregatable);
         out.writeOptionalStringArray(indices);
@@ -120,6 +130,9 @@ public void writeTo(StreamOutput out) throws IOException {
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
         builder.field(TYPE_FIELD.getPreferredName(), type);
+        if (isMetadataField) {
+            builder.field(METADATA_FIELD.getPreferredName(), isMetadataField);
+        }
         builder.field(SEARCHABLE_FIELD.getPreferredName(), isSearchable);
         builder.field(AGGREGATABLE_FIELD.getPreferredName(), isAggregatable);
         if (indices != null) {
@@ -156,17 +169,19 @@ public static FieldCapabilities fromXContent(String name, XContentParser parser)
         true,
         (a, name) -> new FieldCapabilities(name,
             (String) a[0],
+            a[3] == null ? false : (boolean) a[3],
             (boolean) a[1],
             (boolean) a[2],
-            a[3] != null ? ((List<String>) a[3]).toArray(new String[0]) : null,
             a[4] != null ? ((List<String>) a[4]).toArray(new String[0]) : null,
             a[5] != null ? ((List<String>) a[5]).toArray(new String[0]) : null,
-            a[6] != null ? ((Map<String, Set<String>>) a[6]) : Collections.emptyMap()));
+            a[6] != null ? ((List<String>) a[6]).toArray(new String[0]) : null,
+            a[7] != null ? ((Map<String, Set<String>>) a[7]) : Collections.emptyMap()));
 
     static {
         PARSER.declareString(ConstructingObjectParser.constructorArg(), TYPE_FIELD);
         PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), SEARCHABLE_FIELD);
         PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), AGGREGATABLE_FIELD);
+        PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), METADATA_FIELD);
         PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), INDICES_FIELD);
         PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), NON_SEARCHABLE_INDICES_FIELD);
         PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), NON_AGGREGATABLE_INDICES_FIELD);
@@ -181,6 +196,13 @@ public String getName() {
         return name;
     }
 
+    /**
+     * Whether this field is a metadata field.
+     */
+    public boolean isMetaField() {
+        return isMetadataField;
+    }
+
     /**
      * Whether this field can be aggregated on all indices.
      */
@@ -238,7 +260,8 @@ public boolean equals(Object o) {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
         FieldCapabilities that = (FieldCapabilities) o;
-        return isSearchable == that.isSearchable &&
+        return isMetadataField == that.isMetadataField &&
+            isSearchable == that.isSearchable &&
             isAggregatable == that.isAggregatable &&
             Objects.equals(name, that.name) &&
             Objects.equals(type, that.type) &&
@@ -250,7 +273,7 @@ public boolean equals(Object o) {
 
     @Override
     public int hashCode() {
-        int result = Objects.hash(name, type, isSearchable, isAggregatable, meta);
+        int result = Objects.hash(name, type, isMetadataField, isSearchable, isAggregatable, meta);
         result = 31 * result + Arrays.hashCode(indices);
         result = 31 * result + Arrays.hashCode(nonSearchableIndices);
         result = 31 * result + Arrays.hashCode(nonAggregatableIndices);
@@ -263,8 +286,9 @@ public String toString() {
     }
 
     static class Builder {
-        private String name;
-        private String type;
+        private final String name;
+        private final String type;
+        private boolean isMetadataField;
         private boolean isSearchable;
         private boolean isAggregatable;
         private List<IndexCaps> indiceList;
@@ -282,11 +306,12 @@ static class Builder {
         /**
          * Collect the field capabilities for an index.
          */
-        void add(String index, boolean search, boolean agg, Map<String, String> meta) {
+        void add(String index, boolean isMetaField, boolean search, boolean agg, Map<String, String> meta) {
             IndexCaps indexCaps = new IndexCaps(index, search, agg);
             indiceList.add(indexCaps);
             this.isSearchable &= search;
             this.isAggregatable &= agg;
+            this.isMetadataField |= isMetaField;
             for (Map.Entry<String, String> entry : meta.entrySet()) {
                 this.meta.computeIfAbsent(entry.getKey(), key -> new HashSet<>())
                         .add(entry.getValue());
@@ -310,7 +335,7 @@ FieldCapabilities build(boolean withIndices) {
 
             final String[] nonSearchableIndices;
             if (isSearchable == false &&
-                indiceList.stream().anyMatch((caps) -> caps.isSearchable)) {
+                    indiceList.stream().anyMatch((caps) -> caps.isSearchable)) {
                 // Iff this field is searchable in some indices AND non-searchable in others
                 // we record the list of non-searchable indices
                 nonSearchableIndices = indiceList.stream()
@@ -337,7 +362,7 @@ FieldCapabilities build(boolean withIndices) {
             Map<String, Set<String>> immutableMeta = meta.entrySet().stream()
                     .collect(Collectors.toUnmodifiableMap(
                             Map.Entry::getKey, entryValueFunction.andThen(Set::copyOf)));
-            return new FieldCapabilities(name, type, isSearchable, isAggregatable,
+            return new FieldCapabilities(name, type, isMetadataField, isSearchable, isAggregatable,
                 indices, nonSearchableIndices, nonAggregatableIndices, immutableMeta);
         }
     }

server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexResponse.java

@@ -25,18 +25,21 @@ public class FieldCapabilitiesIndexResponse extends ActionResponse implements Wr
     private final String indexName;
     private final Map<String, IndexFieldCapabilities> responseMap;
     private final boolean canMatch;
+    private final transient Version originVersion;
 
     FieldCapabilitiesIndexResponse(String indexName, Map<String, IndexFieldCapabilities> responseMap, boolean canMatch) {
         this.indexName = indexName;
         this.responseMap = responseMap;
         this.canMatch = canMatch;
+        this.originVersion = Version.CURRENT;
     }
 
     FieldCapabilitiesIndexResponse(StreamInput in) throws IOException {
         super(in);
         this.indexName = in.readString();
         this.responseMap = in.readMap(StreamInput::readString, IndexFieldCapabilities::new);
         this.canMatch = in.getVersion().onOrAfter(Version.V_7_9_0) ? in.readBoolean() : true;
+        this.originVersion = in.getVersion();
     }
 
     /**
@@ -65,6 +68,10 @@ public IndexFieldCapabilities getField(String field) {
         return responseMap.get(field);
     }
 
+    Version getOriginVersion() {
+        return originVersion;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(indexName);

server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java

@@ -105,6 +105,17 @@ public Map<String, FieldCapabilities> getField(String field) {
         return responseMap.get(field);
     }
 
+    /**
+     * Returns <code>true</code> if the provided field is a metadata field.
+     */
+    public boolean isMetadataField(String field) {
+        Map<String, FieldCapabilities> caps = getField(field);
+        if (caps == null) {
+            return false;
+        }
+        return caps.values().stream().anyMatch(FieldCapabilities::isMetaField);
+    }
+
     private static Map<String, FieldCapabilities> readField(StreamInput in) throws IOException {
         return in.readMap(StreamInput::readString, FieldCapabilities::new);
     }