Replace IndexMetaData.Custom with Map-based custom metadata (#32749)

This PR removes the deprecated `Custom` class in `IndexMetaData`, in favor of a `Map<String, DiffableStringMap>` that is used to store custom index metadata. As part of this, there is now no way to set this metadata in a template or create index request (since it's only set by plugins, or dedicated REST endpoints). The `Map<String, DiffableStringMap>` is intended to be a namespaced `Map<String, String>` (`DiffableStringMap` implements `Map<String, String>`, so the signature is more like `Map<String, Map<String, String>>`). This is so we can do things like: ``` java Map<String, String> ccrMeta = indexMetaData.getCustom("ccr"); ``` And then have complete control over the metadata. This also means any plugin/feature that uses this has to manage its own BWC, as the map is just serialized as a map. It also means that if metadata is put in the map that isn't used (for instance, if a plugin were removed), it causes no failures the way an unregistered `Setting` would. The reason I use a custom `DiffableStringMap` here rather than a plain `Map<String, String>` is so the map can be diffed with previous cluster state updates for serialization. Supersedes #32683
elastic · Aug 30, 2018 · 001b78f · 001b78f
1 parent af2eaf2
commit 001b78f
Show file tree

Hide file tree

Showing 17 changed files with 406 additions and 335 deletions.
diff --git a/...a/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java b/...a/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java
@@ -55,8 +55,6 @@ public class CreateIndexClusterStateUpdateRequest extends ClusterStateUpdateRequ
 
     private final Set<Alias> aliases = new HashSet<>();
 
-    private final Map<String, IndexMetaData.Custom> customs = new HashMap<>();
-
     private final Set<ClusterBlock> blocks = new HashSet<>();
 
     private ActiveShardCount waitForActiveShards = ActiveShardCount.DEFAULT;
@@ -83,11 +81,6 @@ public CreateIndexClusterStateUpdateRequest aliases(Set<Alias> aliases) {
         return this;
     }
 
-    public CreateIndexClusterStateUpdateRequest customs(Map<String, IndexMetaData.Custom> customs) {
-        this.customs.putAll(customs);
-        return this;
-    }
-
     public CreateIndexClusterStateUpdateRequest blocks(Set<ClusterBlock> blocks) {
         this.blocks.addAll(blocks);
         return this;
@@ -146,10 +139,6 @@ public Set<Alias> aliases() {
         return aliases;
     }
 
-    public Map<String, IndexMetaData.Custom> customs() {
-        return customs;
-    }
-
     public Set<ClusterBlock> blocks() {
         return blocks;
     }

diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java
@@ -29,7 +29,6 @@
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
@@ -58,9 +57,9 @@
 import java.util.Set;
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
+import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS;
 import static org.elasticsearch.common.settings.Settings.readSettingsFromStream;
 import static org.elasticsearch.common.settings.Settings.writeSettingsToStream;
-import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS;
 
 /**
  * A request to create an index. Best created with {@link org.elasticsearch.client.Requests#createIndexRequest(String)}.
@@ -87,8 +86,6 @@ public class CreateIndexRequest extends AcknowledgedRequest<CreateIndexRequest>
 
     private final Set<Alias> aliases = new HashSet<>();
 
-    private final Map<String, IndexMetaData.Custom> customs = new HashMap<>();
-
     private ActiveShardCount waitForActiveShards = ActiveShardCount.DEFAULT;
 
     public CreateIndexRequest() {
@@ -388,18 +385,7 @@ public CreateIndexRequest source(Map<String, ?> source, DeprecationHandler depre
             } else if (ALIASES.match(name, deprecationHandler)) {
                 aliases((Map<String, Object>) entry.getValue());
             } else {
-                // maybe custom?
-                IndexMetaData.Custom proto = IndexMetaData.lookupPrototype(name);
-                if (proto != null) {
-                    try {
-                        customs.put(name, proto.fromMap((Map<String, Object>) entry.getValue()));
-                    } catch (IOException e) {
-                        throw new ElasticsearchParseException("failed to parse custom metadata for [{}]", name);
-                    }
-                } else {
-                    // found a key which is neither custom defined nor one of the supported ones
-                    throw new ElasticsearchParseException("unknown key [{}] for create index", name);
-                }
+                throw new ElasticsearchParseException("unknown key [{}] for create index", name);
             }
         }
         return this;
@@ -413,18 +399,6 @@ public Set<Alias> aliases() {
         return this.aliases;
     }
 
-    /**
-     * Adds custom metadata to the index to be created.
-     */
-    public CreateIndexRequest custom(IndexMetaData.Custom custom) {
-        customs.put(custom.type(), custom);
-        return this;
-    }
-
-    public Map<String, IndexMetaData.Custom> customs() {
-        return this.customs;
-    }
-
     public ActiveShardCount waitForActiveShards() {
         return waitForActiveShards;
     }
@@ -474,11 +448,13 @@ public void readFrom(StreamInput in) throws IOException {
             }
             mappings.put(type, source);
         }
-        int customSize = in.readVInt();
-        for (int i = 0; i < customSize; i++) {
-            String type = in.readString();
-            IndexMetaData.Custom customIndexMetaData = IndexMetaData.lookupPrototypeSafe(type).readFrom(in);
-            customs.put(type, customIndexMetaData);
+        if (in.getVersion().before(Version.V_7_0_0_alpha1)) {
+            // This used to be the size of custom metadata classes
+            int customSize = in.readVInt();
+            assert customSize == 0 : "unexpected custom metadata when none is supported";
+            if (customSize > 0) {
+                throw new IllegalStateException("unexpected custom metadata when none is supported");
+            }
         }
         int aliasesSize = in.readVInt();
         for (int i = 0; i < aliasesSize; i++) {
@@ -501,10 +477,9 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeString(entry.getKey());
             out.writeString(entry.getValue());
         }
-        out.writeVInt(customs.size());
-        for (Map.Entry<String, IndexMetaData.Custom> entry : customs.entrySet()) {
-            out.writeString(entry.getKey());
-            entry.getValue().writeTo(out);
+        if (out.getVersion().before(Version.V_7_0_0_alpha1)) {
+            // Size of custom index metadata, which is removed
+            out.writeVInt(0);
         }
         out.writeVInt(aliases.size());
         for (Alias alias : aliases) {
@@ -542,10 +517,6 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t
             alias.toXContent(builder, params);
         }
         builder.endObject();
-
-        for (Map.Entry<String, IndexMetaData.Custom> entry : customs.entrySet()) {
-            builder.field(entry.getKey(), entry.getValue(), params);
-        }
         return builder;
     }
 }
diff --git a/...rc/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java b/...rc/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.action.support.master.AcknowledgedRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
@@ -224,14 +223,6 @@ public CreateIndexRequestBuilder setSource(Map<String, ?> source) {
         return this;
     }
 
-    /**
-     * Adds custom metadata to the index to be created.
-     */
-    public CreateIndexRequestBuilder addCustom(IndexMetaData.Custom custom) {
-        request.custom(custom);
-        return this;
-    }
-
     /**
      * Sets the settings and mappings as a single source.
      */

diff --git a/...c/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java b/...c/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java
@@ -75,7 +75,7 @@ protected void masterOperation(final CreateIndexRequest request, final ClusterSt
         final CreateIndexClusterStateUpdateRequest updateRequest = new CreateIndexClusterStateUpdateRequest(request, cause, indexName, request.index())
                 .ackTimeout(request.timeout()).masterNodeTimeout(request.masterNodeTimeout())
                 .settings(request.settings()).mappings(request.mappings())
-                .aliases(request.aliases()).customs(request.customs())
+                .aliases(request.aliases())
                 .waitForActiveShards(request.waitForActiveShards());
 
         createIndexService.createIndex(updateRequest, ActionListener.wrap(response ->

diff --git a/...er/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java b/...er/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java
@@ -185,7 +185,6 @@ static CreateIndexClusterStateUpdateRequest prepareCreateIndexRequest(final Resi
                 .masterNodeTimeout(targetIndex.masterNodeTimeout())
                 .settings(targetIndex.settings())
                 .aliases(targetIndex.aliases())
-                .customs(targetIndex.customs())
                 .waitForActiveShards(targetIndex.waitForActiveShards())
                 .recoverFrom(metaData.getIndex())
                 .resizeType(resizeRequest.getResizeType())

diff --git a/...ain/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java b/...ain/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java
@@ -27,7 +27,6 @@
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.MasterNodeRequest;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -61,9 +60,9 @@
 import java.util.stream.Collectors;
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
+import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS;
 import static org.elasticsearch.common.settings.Settings.readSettingsFromStream;
 import static org.elasticsearch.common.settings.Settings.writeSettingsToStream;
-import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS;
 
 /**
  * A request to create an index template.
@@ -88,8 +87,6 @@ public class PutIndexTemplateRequest extends MasterNodeRequest<PutIndexTemplateR
 
     private final Set<Alias> aliases = new HashSet<>();
 
-    private Map<String, IndexMetaData.Custom> customs = new HashMap<>();
-
     private Integer version;
 
     public PutIndexTemplateRequest() {
@@ -353,15 +350,7 @@ public PutIndexTemplateRequest source(Map<String, Object> templateSource) {
             } else if (name.equals("aliases")) {
                 aliases((Map<String, Object>) entry.getValue());
             } else {
-                // maybe custom?
-                IndexMetaData.Custom proto = IndexMetaData.lookupPrototype(name);
-                if (proto != null) {
-                    try {
-                        customs.put(name, proto.fromMap((Map<String, Object>) entry.getValue()));
-                    } catch (IOException e) {
-                        throw new ElasticsearchParseException("failed to parse custom metadata for [{}]", name);
-                    }
-                }
+                throw new ElasticsearchParseException("unknown key [{}] in the template ", name);
             }
         }
         return this;
@@ -395,15 +384,6 @@ public PutIndexTemplateRequest source(BytesReference source, XContentType xConte
         return source(XContentHelper.convertToMap(source, true, xContentType).v2());
     }
 
-    public PutIndexTemplateRequest custom(IndexMetaData.Custom custom) {
-        customs.put(custom.type(), custom);
-        return this;
-    }
-
-    public Map<String, IndexMetaData.Custom> customs() {
-        return this.customs;
-    }
-
     public Set<Alias> aliases() {
         return this.aliases;
     }
@@ -494,11 +474,13 @@ public void readFrom(StreamInput in) throws IOException {
             String mappingSource = in.readString();
             mappings.put(type, mappingSource);
         }
-        int customSize = in.readVInt();
-        for (int i = 0; i < customSize; i++) {
-            String type = in.readString();
-            IndexMetaData.Custom customIndexMetaData = IndexMetaData.lookupPrototypeSafe(type).readFrom(in);
-            customs.put(type, customIndexMetaData);
+        if (in.getVersion().before(Version.V_7_0_0_alpha1)) {
+            // Used to be used for custom index metadata
+            int customSize = in.readVInt();
+            assert customSize == 0 : "expected not to have any custom metadata";
+            if (customSize > 0) {
+                throw new IllegalStateException("unexpected custom metadata when none is supported");
+            }
         }
         int aliasesSize = in.readVInt();
         for (int i = 0; i < aliasesSize; i++) {
@@ -525,10 +507,8 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeString(entry.getKey());
             out.writeString(entry.getValue());
         }
-        out.writeVInt(customs.size());
-        for (Map.Entry<String, IndexMetaData.Custom> entry : customs.entrySet()) {
-            out.writeString(entry.getKey());
-            entry.getValue().writeTo(out);
+        if (out.getVersion().before(Version.V_7_0_0_alpha1)) {
+            out.writeVInt(0);
         }
         out.writeVInt(aliases.size());
         for (Alias alias : aliases) {
@@ -565,10 +545,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         }
         builder.endObject();
 
-        for (Map.Entry<String, IndexMetaData.Custom> entry : customs.entrySet()) {
-            builder.field(entry.getKey(), entry.getValue(), params);
-        }
-
         return builder;
     }
 }
diff --git a/.../org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java b/.../org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java
@@ -84,7 +84,6 @@ protected void masterOperation(final PutIndexTemplateRequest request, final Clus
                 .settings(templateSettingsBuilder.build())
                 .mappings(request.mappings())
                 .aliases(request.aliases())
-                .customs(request.customs())
                 .create(request.create())
                 .masterTimeout(request.masterNodeTimeout())
                 .version(request.version()),