CORE: Allow Setting Index Autocreate to _template

qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java

@@ -987,6 +987,9 @@ private void checkSnapshot(String snapshotName, int count, Version tookOnVersion
         Map<String, Object> aliases = new HashMap<>();
         aliases.put("alias1", emptyMap());
         aliases.put("alias2", singletonMap("filter", singletonMap("term", singletonMap("version", tookOnVersion.toString()))));
+        if (runningAgainstOldCluster == false) {
+            expectedTemplate.put("auto_create_index", false);
+        }
         expectedTemplate.put("aliases", aliases);
         expectedTemplate = singletonMap("test_template", expectedTemplate);
         if (false == expectedTemplate.equals(getTemplateResponse)) {

server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java

@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.action.admin.indices.template.put;
 
+import java.util.ArrayList;
 import org.elasticsearch.ElasticsearchGenerationException;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.Version;
@@ -75,6 +76,8 @@ public class PutIndexTemplateRequest extends MasterNodeRequest<PutIndexTemplateR
 
     private String cause = "";
 
+    private boolean autoCreateIndex;
+
     private List<String> indexPatterns;
 
     private int order;
@@ -135,6 +138,15 @@ public List<String> patterns() {
         return this.indexPatterns;
     }
 
+    public PutIndexTemplateRequest autoCreateIndex(boolean autoCreateIndex) {
+        this.autoCreateIndex = autoCreateIndex;
+        return this;
+    }
+
+    public boolean autoCreateIndex() {
+        return autoCreateIndex;
+    }
+
     public PutIndexTemplateRequest order(int order) {
         this.order = order;
         return this;
@@ -325,6 +337,8 @@ public PutIndexTemplateRequest source(Map<String, Object> templateSource) {
                 } else {
                     throw new IllegalArgumentException("Malformed [template] value, should be a string or a list of strings");
                 }
+            } else if (name.equals("auto_create_index")) {
+                autoCreateIndex(XContentMapValues.nodeBooleanValue(entry.getValue(), false));
             } else if (name.equals("order")) {
                 order(XContentMapValues.nodeIntegerValue(entry.getValue(), order()));
             } else if ("version".equals(name)) {
@@ -461,7 +475,11 @@ public void readFrom(StreamInput in) throws IOException {
         name = in.readString();
 
         if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) {
-            indexPatterns = in.readList(StreamInput::readString);
+            List<String> patterns = in.readList(StreamInput::readString);
+            if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
+                autoCreateIndex = Boolean.parseBoolean(patterns.remove(patterns.size() - 1));
+            }
+            indexPatterns = patterns;
         } else {
             indexPatterns = Collections.singletonList(in.readString());
         }
@@ -495,7 +513,13 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeString(cause);
         out.writeString(name);
         if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) {
-            out.writeStringList(indexPatterns);
+            if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
+                List<String> patternsAndAutoCreate = new ArrayList<>(indexPatterns);
+                patternsAndAutoCreate.add(Boolean.toString(autoCreateIndex));
+                out.writeStringList(patternsAndAutoCreate);
+            } else {
+                out.writeStringList(indexPatterns);
+            }
         } else {
             out.writeString(indexPatterns.size() > 0 ? indexPatterns.get(0) : "");
         }
@@ -520,6 +544,7 @@ public void writeTo(StreamOutput out) throws IOException {
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.field("index_patterns", indexPatterns);
+        builder.field("auto_create_index", autoCreateIndex);
         builder.field("order", order);
         if (version != null) {
             builder.field("version", version);

server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java

@@ -60,6 +60,11 @@ public PutIndexTemplateRequestBuilder setPatterns(List<String> indexPatterns) {
         return this;
     }
 
+    public PutIndexTemplateRequestBuilder setAutoCreateIndex(boolean autoCreateIndex) {
+        request.autoCreateIndex(autoCreateIndex);
+        return this;
+    }
+
     /**
      * Sets the order of this template if more than one template matches.
      */

server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java

@@ -80,6 +80,7 @@ protected void masterOperation(final PutIndexTemplateRequest request, final Clus
         indexScopedSettings.validate(templateSettingsBuilder.build(), true); // templates must be consistent with regards to dependencies
         indexTemplateService.putTemplate(new MetaDataIndexTemplateService.PutRequest(cause, request.name())
                 .patterns(request.patterns())
+                .autoCreateIndex(request.autoCreateIndex())
                 .order(request.order())
                 .settings(templateSettingsBuilder.build())
                 .mappings(request.mappings())

server/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java

@@ -21,6 +21,7 @@
 
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.MetaDataIndexTemplateService;
 import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.Tuple;
@@ -41,6 +42,8 @@
  */
 public final class AutoCreateIndex {
 
+    public static final String AUTO_CREATE_TEMPLATE_BASED = "_template";
+
     public static final Setting<AutoCreate> AUTO_CREATE_INDEX_SETTING =
         new Setting<>("action.auto_create_index", "true", AutoCreate::new, Property.NodeScope, Setting.Property.Dynamic);
 
@@ -62,6 +65,10 @@ public boolean needToCheck() {
         return this.autoCreate.autoCreateIndex;
     }
 
+    public boolean templateBased() {
+        return autoCreate.templateBased;
+    }
+
     /**
      * Should the index be auto created?
      * @throws IndexNotFoundException if the index doesn't exist and shouldn't be auto created
@@ -70,6 +77,24 @@ public boolean shouldAutoCreate(String index, ClusterState state) {
         if (resolver.hasIndexOrAlias(index, state)) {
             return false;
         }
+        if (templateBased()) {
+            return MetaDataIndexTemplateService.findTemplates(state.metaData(), index).stream().findFirst()
+                .map(template -> {
+                    if (template.autoCreateIndex()) {
+                        return true;
+                    }
+                    throw new IndexNotFoundException(
+                        "no such index and [" + AUTO_CREATE_INDEX_SETTING.getKey()
+                            + "] is set to [_template] but the matching template does not allow for auto creating the index.",
+                        index
+                    );
+                }).orElseThrow(
+                    () -> new IndexNotFoundException(
+                        "no such index and [" + AUTO_CREATE_INDEX_SETTING.getKey()
+                            + "] is set to [_template] but there is no matching template.", index
+                    )
+                );
+        }
         // One volatile read, so that all checks are done against the same instance:
         final AutoCreate autoCreate = this.autoCreate;
         if (autoCreate.autoCreateIndex == false) {
@@ -108,45 +133,58 @@ void setAutoCreate(AutoCreate autoCreate) {
 
     static class AutoCreate {
         private final boolean autoCreateIndex;
+        private final boolean templateBased;
         private final List<Tuple<String, Boolean>> expressions;
         private final String string;
 
         private AutoCreate(String value) {
             boolean autoCreateIndex;
             List<Tuple<String, Boolean>> expressions = new ArrayList<>();
-            try {
-                autoCreateIndex = Booleans.parseBoolean(value);
-            } catch (IllegalArgumentException ex) {
+            if (AUTO_CREATE_TEMPLATE_BASED.equals(value)) {
+                autoCreateIndex = true;
+                templateBased = true;
+            } else {
+                templateBased = false;
                 try {
-                    String[] patterns = Strings.commaDelimitedListToStringArray(value);
-                    for (String pattern : patterns) {
-                        if (pattern == null || pattern.trim().length() == 0) {
-                            throw new IllegalArgumentException("Can't parse [" + value + "] for setting [action.auto_create_index] must "
-                                    + "be either [true, false, or a comma separated list of index patterns]");
-                        }
-                        pattern = pattern.trim();
-                        Tuple<String, Boolean> expression;
-                        if (pattern.startsWith("-")) {
-                            if (pattern.length() == 1) {
-                                throw new IllegalArgumentException("Can't parse [" + value + "] for setting [action.auto_create_index] "
-                                        + "must contain an index name after [-]");
+                    autoCreateIndex = Booleans.parseBoolean(value);
+                } catch (IllegalArgumentException ex) {
+                    try {
+                        String[] patterns = Strings.commaDelimitedListToStringArray(value);
+                        for (String pattern : patterns) {
+                            if (pattern == null || pattern.trim().length() == 0) {
+                                throw new IllegalArgumentException(
+                                    "Can't parse [" + value + "] for setting [action.auto_create_index] must "
+                                    + "be either [true, false, or a comma separated list of index patterns]"
+                                );
                             }
-                            expression = new Tuple<>(pattern.substring(1), false);
-                        } else if(pattern.startsWith("+")) {
-                            if (pattern.length() == 1) {
-                                throw new IllegalArgumentException("Can't parse [" + value + "] for setting [action.auto_create_index] "
-                                        + "must contain an index name after [+]");
+                            pattern = pattern.trim();
+                            Tuple<String, Boolean> expression;
+                            if (pattern.startsWith("-")) {
+                                if (pattern.length() == 1) {
+                                    throw new IllegalArgumentException(
+                                        "Can't parse [" + value + "] for setting [action.auto_create_index] "
+                                        + "must contain an index name after [-]"
+                                    );
+                                }
+                                expression = new Tuple<>(pattern.substring(1), false);
+                            } else if (pattern.startsWith("+")) {
+                                if (pattern.length() == 1) {
+                                    throw new IllegalArgumentException(
+                                        "Can't parse [" + value + "] for setting [action.auto_create_index] "
+                                        + "must contain an index name after [+]"
+                                    );
+                                }
+                                expression = new Tuple<>(pattern.substring(1), true);
+                            } else {
+                                expression = new Tuple<>(pattern, true);
                             }
-                            expression = new Tuple<>(pattern.substring(1), true);
-                        } else {
-                            expression = new Tuple<>(pattern, true);
+                            expressions.add(expression);
                         }
-                        expressions.add(expression);
+                        autoCreateIndex = true;
+                    } catch (IllegalArgumentException ex1) {
+                        ex1.addSuppressed(ex);
+                        throw ex1;
                     }
-                    autoCreateIndex = true;
-                } catch (IllegalArgumentException ex1) {
-                    ex1.addSuppressed(ex);
-                    throw ex1;
                 }
             }
             this.expressions = expressions;

server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java

@@ -80,6 +80,8 @@ public class IndexTemplateMetaData extends AbstractDiffable<IndexTemplateMetaDat
 
     private final List<String> patterns;
 
+    private final boolean autoCreateIndex;
+
     private final Settings settings;
 
     // the mapping source should always include the type as top level
@@ -90,14 +92,16 @@ public class IndexTemplateMetaData extends AbstractDiffable<IndexTemplateMetaDat
     public IndexTemplateMetaData(String name, int order, Integer version,
                                  List<String> patterns, Settings settings,
                                  ImmutableOpenMap<String, CompressedXContent> mappings,
-                                 ImmutableOpenMap<String, AliasMetaData> aliases) {
+                                 ImmutableOpenMap<String, AliasMetaData> aliases,
+                                 boolean autoCreateIndex) {
         if (patterns == null || patterns.isEmpty()) {
             throw new IllegalArgumentException("Index patterns must not be null or empty; got " + patterns);
         }
         this.name = name;
         this.order = order;
         this.version = version;
         this.patterns= patterns;
+        this.autoCreateIndex = autoCreateIndex;
         this.settings = settings;
         this.mappings = mappings;
         this.aliases = aliases;
@@ -137,6 +141,14 @@ public List<String> getPatterns() {
         return this.patterns;
     }
 
+    public boolean getAutoCreateIndex() {
+        return this.autoCreateIndex;
+    }
+
+    public boolean autoCreateIndex() {
+        return this.autoCreateIndex;
+    }
+
     public Settings settings() {
         return this.settings;
     }
@@ -196,7 +208,11 @@ public static IndexTemplateMetaData readFrom(StreamInput in) throws IOException
         Builder builder = new Builder(in.readString());
         builder.order(in.readInt());
         if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) {
-            builder.patterns(in.readList(StreamInput::readString));
+            List<String> indexPatterns = in.readList(StreamInput::readString);
+            if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
+                builder.autoCreateIndex(Boolean.parseBoolean(indexPatterns.remove(indexPatterns.size() - 1)));
+            }
+            builder.patterns(indexPatterns);
         } else {
             builder.patterns(Collections.singletonList(in.readString()));
         }
@@ -231,7 +247,14 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeString(name);
         out.writeInt(order);
         if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) {
-            out.writeStringList(patterns);
+            final List<String> indexPatterns;
+            if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
+                indexPatterns = new ArrayList<>(patterns);
+                indexPatterns.add(Boolean.toString(autoCreateIndex));
+            } else {
+                indexPatterns = patterns;
+            }
+            out.writeStringList(indexPatterns);
         } else {
             out.writeString(patterns.get(0));
         }
@@ -254,7 +277,7 @@ public void writeTo(StreamOutput out) throws IOException {
     public static class Builder {
 
         private static final Set<String> VALID_FIELDS = Sets.newHashSet(
-            "template", "order", "mappings", "settings", "index_patterns", "aliases", "version");
+            "template", "order", "mappings", "settings", "index_patterns", "auto_create_index" , "aliases", "version");
 
         private String name;
 
@@ -264,6 +287,8 @@ public static class Builder {
 
         private List<String> indexPatterns;
 
+        private boolean autoCreateIndex;
+
         private Settings settings = Settings.Builder.EMPTY_SETTINGS;
 
         private final ImmutableOpenMap.Builder<String, CompressedXContent> mappings;
@@ -302,6 +327,10 @@ public Builder patterns(List<String> indexPatterns) {
             return this;
         }
 
+        public Builder autoCreateIndex(boolean autoCreateIndex) {
+            this.autoCreateIndex = autoCreateIndex;
+            return this;
+        }
 
         public Builder settings(Settings.Builder settings) {
             this.settings = settings.build();
@@ -339,7 +368,10 @@ public Builder putAlias(AliasMetaData.Builder aliasMetaData) {
         }
 
         public IndexTemplateMetaData build() {
-            return new IndexTemplateMetaData(name, order, version, indexPatterns, settings, mappings.build(), aliases.build());
+            return new IndexTemplateMetaData(
+                name, order, version, indexPatterns, settings, mappings.build(), aliases.build(), autoCreateIndex
+            );
+
         }
 
         public static void toXContent(IndexTemplateMetaData indexTemplateMetaData, XContentBuilder builder, ToXContent.Params params)
@@ -359,6 +391,7 @@ public static void toInnerXContent(IndexTemplateMetaData indexTemplateMetaData,
                 builder.field("version", indexTemplateMetaData.version());
             }
             builder.field("index_patterns", indexTemplateMetaData.patterns());
+            builder.field("auto_create_index", indexTemplateMetaData.autoCreateIndex());
 
             builder.startObject("settings");
             indexTemplateMetaData.settings().toXContent(builder, params);
@@ -456,6 +489,8 @@ public static IndexTemplateMetaData fromXContent(XContentParser parser, String t
                         builder.order(parser.intValue());
                     } else if ("version".equals(currentFieldName)) {
                         builder.version(parser.intValue());
+                    } else if ("auto_create_index".equals(currentFieldName)) {
+                        builder.autoCreateIndex(parser.booleanValue());
                     }
                 }
             }