Merge remote-tracking branch 'origin/master' into can-match-stack-ove…

…rflow

* origin/master: (59 commits)
  Fix Lucene version of 5.6.1.
  Remove azure deprecated settings (elastic#26099)
  Handle the 5.6.0 release
  Allow plugins to validate cluster-state on join (elastic#26595)
  Remove index mapper dynamic settings (elastic#25734)
  update AWS SDK for ECS Task IAM support in discovery-ec2 (elastic#26479)
  Azure repository: Accelerate the listing of files (used in delete snapshot) (elastic#25710)
  Build: Remove norelease from forbidden patterns (elastic#26592)
  Fix reference to painless inside expression engine (elastic#26528)
  Build: Move javadoc linking to root build.gradle (elastic#26529)
  Test: Remove leftover static bwc test case (elastic#26584)
  Docs: Remove remaining references to file and native scripts (elastic#26580)
  Snapshot fallback should consider build.snapshot
  elastic#26496: Set the correct bwc version after backport to 6.x
  Fix the MapperFieldType.rangeQuery API. (elastic#26552)
  Deduplicate `_field_names`. (elastic#26550)
  [Docs] Update method setSource(byte[] source) (elastic#26561)
  [Docs] Fix typo in javadocs (elastic#26556)
  Allow multiple digits in Vagrant 2.x minor versions
  Support Vagrant 2.x
  ...

build.gradle

@@ -204,25 +204,15 @@ task branchConsistency {
 }
 
 subprojects {
-  project.afterEvaluate {
-    // ignore missing javadocs
-    tasks.withType(Javadoc) { Javadoc javadoc ->
-      // the -quiet here is because of a bug in gradle, in that adding a string option
-      // by itself is not added to the options. By adding quiet, both this option and
-      // the "value" -quiet is added, separated by a space. This is ok since the javadoc
-      // command already adds -quiet, so we are just duplicating it
-      // see https://discuss.gradle.org/t/add-custom-javadoc-option-that-does-not-take-an-argument/5959
-      javadoc.options.encoding='UTF8'
-      javadoc.options.addStringOption('Xdoclint:all,-missing', '-quiet')
-      /*
-      TODO: building javadocs with java 9 b118 is currently broken with weird errors, so
-      for now this is commented out...try again with the next ea build...
-      javadoc.executable = new File(project.javaHome, 'bin/javadoc')
-      if (project.javaVersion == JavaVersion.VERSION_1_9) {
-        // TODO: remove this hack! gradle should be passing this...
-        javadoc.options.addStringOption('source', '8')
-      }*/
-    }
+  // ignore missing javadocs
+  tasks.withType(Javadoc) { Javadoc javadoc ->
+    // the -quiet here is because of a bug in gradle, in that adding a string option
+    // by itself is not added to the options. By adding quiet, both this option and
+    // the "value" -quiet is added, separated by a space. This is ok since the javadoc
+    // command already adds -quiet, so we are just duplicating it
+    // see https://discuss.gradle.org/t/add-custom-javadoc-option-that-does-not-take-an-argument/5959
+    javadoc.options.encoding='UTF8'
+    javadoc.options.addStringOption('Xdoclint:all,-missing', '-quiet')
   }
 
   /* Sets up the dependencies that we build as part of this project but
@@ -280,6 +270,27 @@ subprojects {
         }
       }
     }
+
+    // Handle javadoc dependencies across projects. Order matters: the linksOffline for
+    // org.elasticsearch:elasticsearch must be the last one or all the links for the
+    // other packages (e.g org.elasticsearch.client) will point to core rather than
+    // their own artifacts.
+    if (project.plugins.hasPlugin(BuildPlugin)) {
+      String artifactsHost = VersionProperties.elasticsearch.endsWith("-SNAPSHOT") ? "https://snapshots.elastic.co" : "https://artifacts.elastic.co"
+      Closure sortClosure = { a, b -> b.group <=> a.group }
+      Closure depJavadocClosure = { dep ->
+        if (dep.group != null && dep.group.startsWith('org.elasticsearch')) {
+          String substitution = project.ext.projectSubstitutions.get("${dep.group}:${dep.name}:${dep.version}")
+          if (substitution != null) {
+            project.javadoc.dependsOn substitution + ':javadoc'
+            String artifactPath = dep.group.replaceAll('\\.', '/') + '/' + dep.name.replaceAll('\\.', '/') + '/' + dep.version
+            project.javadoc.options.linksOffline artifactsHost + "/javadoc/" + artifactPath, "${project.project(substitution).buildDir}/docs/javadoc/"
+          }
+        }
+      }
+      project.configurations.compile.dependencies.findAll().toSorted(sortClosure).each(depJavadocClosure)
+      project.configurations.provided.dependencies.findAll().toSorted(sortClosure).each(depJavadocClosure)
+    }
   }
 }
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -455,28 +455,8 @@ class BuildPlugin implements Plugin<Project> {
     }
 
     static void configureJavadoc(Project project) {
-        String artifactsHost = VersionProperties.elasticsearch.endsWith("-SNAPSHOT") ? "https://snapshots.elastic.co" : "https://artifacts.elastic.co"
-        project.afterEvaluate {
-            project.tasks.withType(Javadoc) {
-                executable = new File(project.javaHome, 'bin/javadoc')
-            }
-            /*
-             * Order matters, the linksOffline for org.elasticsearch:elasticsearch must be the last one
-             * or all the links for the other packages (e.g org.elasticsearch.client) will point to core rather than their own artifacts
-             */
-            Closure sortClosure = { a, b -> b.group <=> a.group }
-            Closure depJavadocClosure = { dep ->
-                if (dep.group != null && dep.group.startsWith('org.elasticsearch')) {
-                    String substitution = project.ext.projectSubstitutions.get("${dep.group}:${dep.name}:${dep.version}")
-                    if (substitution != null) {
-                        project.javadoc.dependsOn substitution + ':javadoc'
-                        String artifactPath = dep.group.replaceAll('\\.', '/') + '/' + dep.name.replaceAll('\\.', '/') + '/' + dep.version
-                        project.javadoc.options.linksOffline artifactsHost + "/javadoc/" + artifactPath, "${project.project(substitution).buildDir}/docs/javadoc/"
-                    }
-                }
-            }
-            project.configurations.compile.dependencies.findAll().toSorted(sortClosure).each(depJavadocClosure)
-            project.configurations.provided.dependencies.findAll().toSorted(sortClosure).each(depJavadocClosure)
+        project.tasks.withType(Javadoc) {
+            executable = new File(project.javaHome, 'bin/javadoc')
         }
         configureJavadocJar(project)
     }

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/ForbiddenPatternsTask.groovy

@@ -63,11 +63,6 @@ public class ForbiddenPatternsTask extends DefaultTask {
         patterns.put('nocommit should be all lowercase or all uppercase',
             /((?i)nocommit)(?<!(nocommit|NOCOMMIT))/)
         patterns.put('tab', /\t/)
-        if (System.getProperty('build.snapshot', 'true').equals('false')) {
-            patterns.put('norelease', /norelease|NORELEASE/)
-        }
-        patterns.put('norelease should be all lowercase or all uppercase',
-            /((?i)norelease)(?<!(norelease|NORELEASE))/)
 
 
         inputs.property("excludes", filesFilter.excludes)

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantSupportPlugin.groovy

@@ -45,7 +45,7 @@ class VagrantSupportPlugin implements Plugin<Project> {
             }
             String version = pipe.toString().trim()
             if (runResult.exitValue == 0) {
-                if (version ==~ /Vagrant 1\.(8\.[6-9]|9\.[0-9])+/) {
+                if (version ==~ /Vagrant 1\.(8\.[6-9]|9\.[0-9])+/ || version ==~ /Vagrant 2\.[0-9]+\.[0-9]+/) {
                     return [ 'supported' : true ]
                 } else {
                     return [ 'supported' : false,

client/rest-high-level/build.gradle

@@ -59,4 +59,5 @@ forbiddenApisMain {
   // core does not depend on the httpclient for compile so we add the signatures here. We don't add them for test as they are already
   // specified
   signaturesURLs += [PrecommitTasks.getResource('/forbidden/http-signatures.txt')]
+  signaturesURLs += [file('src/main/resources/forbidden/rest-high-level-signatures.txt').toURI().toURL()]
 }

client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java

@@ -42,6 +42,7 @@
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.unit.TimeValue;
@@ -57,6 +58,7 @@
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.nio.charset.Charset;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Locale;
@@ -139,8 +141,8 @@ static Request bulk(BulkRequest bulkRequest) throws IOException {
             bulkContentType = XContentType.JSON;
         }
 
-        byte separator = bulkContentType.xContent().streamSeparator();
-        ContentType requestContentType = ContentType.create(bulkContentType.mediaType());
+        final byte separator = bulkContentType.xContent().streamSeparator();
+        final ContentType requestContentType = createContentType(bulkContentType);
 
         ByteArrayOutputStream content = new ByteArrayOutputStream();
         for (DocWriteRequest<?> request : bulkRequest.requests()) {
@@ -268,7 +270,7 @@ static Request index(IndexRequest indexRequest) {
         parameters.withWaitForActiveShards(indexRequest.waitForActiveShards());
 
         BytesRef source = indexRequest.source().toBytesRef();
-        ContentType contentType = ContentType.create(indexRequest.getContentType().mediaType());
+        ContentType contentType = createContentType(indexRequest.getContentType());
         HttpEntity entity = new ByteArrayEntity(source.bytes, source.offset, source.length, contentType);
 
         return new Request(method, endpoint, parameters.getParams(), entity);
@@ -352,7 +354,7 @@ static Request clearScroll(ClearScrollRequest clearScrollRequest) throws IOExcep
 
     private static HttpEntity createEntity(ToXContent toXContent, XContentType xContentType) throws IOException {
         BytesRef source = XContentHelper.toXContent(toXContent, xContentType, false).toBytesRef();
-        return new ByteArrayEntity(source.bytes, source.offset, source.length, ContentType.create(xContentType.mediaType()));
+        return new ByteArrayEntity(source.bytes, source.offset, source.length, createContentType(xContentType));
     }
 
     static String endpoint(String[] indices, String[] types, String endpoint) {
@@ -372,6 +374,17 @@ static String endpoint(String... parts) {
         return joiner.toString();
     }
 
+    /**
+     * Returns a {@link ContentType} from a given {@link XContentType}.
+     *
+     * @param xContentType the {@link XContentType}
+     * @return the {@link ContentType}
+     */
+    @SuppressForbidden(reason = "Only allowed place to convert a XContentType to a ContentType")
+    static ContentType createContentType(final XContentType xContentType) {
+        return ContentType.create(xContentType.mediaTypeWithoutParameters(), (Charset) null);
+    }
+
     /**
      * Utility class to build request's parameters map and centralize all parameter names.
      */

client/rest-high-level/src/main/resources/forbidden/rest-high-level-signatures.txt

@@ -0,0 +1,21 @@
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance  with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on
+# an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+# either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+@defaultMessage Use Request#createContentType(XContentType) to be sure to pass the right MIME type
+org.apache.http.entity.ContentType#create(java.lang.String)
+org.apache.http.entity.ContentType#create(java.lang.String,java.lang.String)
+org.apache.http.entity.ContentType#create(java.lang.String,java.nio.charset.Charset)
+org.apache.http.entity.ContentType#create(java.lang.String,org.apache.http.NameValuePair[])

client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java

@@ -278,7 +278,7 @@ public void testIndex() throws IOException {
 
         HttpEntity entity = request.entity;
         assertTrue(entity instanceof ByteArrayEntity);
-        assertEquals(indexRequest.getContentType().mediaType(), entity.getContentType().getValue());
+        assertEquals(indexRequest.getContentType().mediaTypeWithoutParameters(), entity.getContentType().getValue());
         try (XContentParser parser = createParser(xContentType.xContent(), entity.getContent())) {
             assertEquals(nbFields, parser.map().size());
         }
@@ -488,7 +488,7 @@ public void testBulk() throws IOException {
         assertEquals("/_bulk", request.endpoint);
         assertEquals(expectedParams, request.params);
         assertEquals("POST", request.method);
-        assertEquals(xContentType.mediaType(), request.entity.getContentType().getValue());
+        assertEquals(xContentType.mediaTypeWithoutParameters(), request.entity.getContentType().getValue());
         byte[] content = new byte[(int) request.entity.getContentLength()];
         try (InputStream inputStream = request.entity.getContent()) {
             Streams.readFully(inputStream, content);
@@ -541,7 +541,7 @@ public void testBulkWithDifferentContentTypes() throws IOException {
             bulkRequest.add(new DeleteRequest("index", "type", "2"));
 
             Request request = Request.bulk(bulkRequest);
-            assertEquals(XContentType.JSON.mediaType(), request.entity.getContentType().getValue());
+            assertEquals(XContentType.JSON.mediaTypeWithoutParameters(), request.entity.getContentType().getValue());
         }
         {
             XContentType xContentType = randomFrom(XContentType.JSON, XContentType.SMILE);
@@ -551,7 +551,7 @@ public void testBulkWithDifferentContentTypes() throws IOException {
             bulkRequest.add(new DeleteRequest("index", "type", "2"));
 
             Request request = Request.bulk(bulkRequest);
-            assertEquals(xContentType.mediaType(), request.entity.getContentType().getValue());
+            assertEquals(xContentType.mediaTypeWithoutParameters(), request.entity.getContentType().getValue());
         }
         {
             XContentType xContentType = randomFrom(XContentType.JSON, XContentType.SMILE);
@@ -563,7 +563,7 @@ public void testBulkWithDifferentContentTypes() throws IOException {
             }
 
             Request request = Request.bulk(new BulkRequest().add(updateRequest));
-            assertEquals(xContentType.mediaType(), request.entity.getContentType().getValue());
+            assertEquals(xContentType.mediaTypeWithoutParameters(), request.entity.getContentType().getValue());
         }
         {
             BulkRequest bulkRequest = new BulkRequest();
@@ -732,7 +732,7 @@ public void testSearchScroll() throws IOException {
         assertEquals("/_search/scroll", request.endpoint);
         assertEquals(0, request.params.size());
         assertToXContentBody(searchScrollRequest, request.entity);
-        assertEquals(Request.REQUEST_BODY_CONTENT_TYPE.mediaType(), request.entity.getContentType().getValue());
+        assertEquals(Request.REQUEST_BODY_CONTENT_TYPE.mediaTypeWithoutParameters(), request.entity.getContentType().getValue());
     }
 
     public void testClearScroll() throws IOException {
@@ -746,12 +746,12 @@ public void testClearScroll() throws IOException {
         assertEquals("/_search/scroll", request.endpoint);
         assertEquals(0, request.params.size());
         assertToXContentBody(clearScrollRequest, request.entity);
-        assertEquals(Request.REQUEST_BODY_CONTENT_TYPE.mediaType(), request.entity.getContentType().getValue());
+        assertEquals(Request.REQUEST_BODY_CONTENT_TYPE.mediaTypeWithoutParameters(), request.entity.getContentType().getValue());
     }
 
     private static void assertToXContentBody(ToXContent expectedBody, HttpEntity actualEntity) throws IOException {
         BytesReference expectedBytes = XContentHelper.toXContent(expectedBody, Request.REQUEST_BODY_CONTENT_TYPE, false);
-        assertEquals(XContentType.JSON.mediaType(), actualEntity.getContentType().getValue());
+        assertEquals(XContentType.JSON.mediaTypeWithoutParameters(), actualEntity.getContentType().getValue());
         assertEquals(expectedBytes, new BytesArray(EntityUtils.toByteArray(actualEntity)));
     }
 
@@ -793,6 +793,11 @@ public void testEndpoint() {
         assertEquals("/a/_create", Request.endpoint("a", null, null, "_create"));
     }
 
+    public void testCreateContentType() {
+        final XContentType xContentType = randomFrom(XContentType.values());
+        assertEquals(xContentType.mediaTypeWithoutParameters(), Request.createContentType(xContentType).getMimeType());
+    }
+
     public void testEnforceSameContentType() {
         XContentType xContentType = randomFrom(XContentType.JSON, XContentType.SMILE);
         IndexRequest indexRequest = new IndexRequest().source(singletonMap("field", "value"), xContentType);

core/src/main/java/org/elasticsearch/Build.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch;
 
+import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.io.FileSystemUtils;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -59,7 +60,18 @@ public class Build {
             // not running from the official elasticsearch jar file (unit tests, IDE, uber client jar, shadiness)
             shortHash = "Unknown";
             date = "Unknown";
-            isSnapshot = true;
+            final String buildSnapshot = System.getProperty("build.snapshot");
+            if (buildSnapshot != null) {
+                try {
+                    Class.forName("com.carrotsearch.randomizedtesting.RandomizedContext");
+                } catch (final ClassNotFoundException e) {
+                    // we are not in tests but build.snapshot is set, bail hard
+                    throw new IllegalStateException("build.snapshot set to [" + buildSnapshot + "] but not running tests");
+                }
+                isSnapshot = Booleans.parseBoolean(buildSnapshot);
+            } else {
+                isSnapshot = true;
+            }
         }
         if (shortHash == null) {
             throw new IllegalStateException("Error finding the build shortHash. " +

core/src/main/java/org/elasticsearch/Version.java

@@ -92,6 +92,9 @@ public class Version implements Comparable<Version> {
     public static final Version V_5_5_3 = new Version(V_5_5_3_ID, org.apache.lucene.util.Version.LUCENE_6_6_0);
     public static final int V_5_6_0_ID = 5060099;
     public static final Version V_5_6_0 = new Version(V_5_6_0_ID, org.apache.lucene.util.Version.LUCENE_6_6_0);
+    public static final int V_5_6_1_ID = 5060199;
+    // use proper Lucene constant once we are on a Lucene snapshot that knows about 6.6.1
+    public static final Version V_5_6_1 = new Version(V_5_6_1_ID, org.apache.lucene.util.Version.fromBits(6, 6, 1));
     public static final int V_6_0_0_alpha1_ID = 6000001;
     public static final Version V_6_0_0_alpha1 =
             new Version(V_6_0_0_alpha1_ID, org.apache.lucene.util.Version.LUCENE_7_0_0);
@@ -142,6 +145,8 @@ public static Version fromId(int id) {
                 return V_6_0_0_alpha2;
             case V_6_0_0_alpha1_ID:
                 return V_6_0_0_alpha1;
+            case V_5_6_1_ID:
+                return V_5_6_1;
             case V_5_6_0_ID:
                 return V_5_6_0;
             case V_5_5_3_ID:

core/src/main/java/org/elasticsearch/action/DocWriteResponse.java

@@ -33,7 +33,7 @@
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.seqno.SequenceNumbersService;
+import org.elasticsearch.index.seqno.SequenceNumbers;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.rest.RestStatus;
 
@@ -176,7 +176,7 @@ public long getVersion() {
     }
 
     /**
-     * Returns the sequence number assigned for this change. Returns {@link SequenceNumbersService#UNASSIGNED_SEQ_NO} if the operation
+     * Returns the sequence number assigned for this change. Returns {@link SequenceNumbers#UNASSIGNED_SEQ_NO} if the operation
      * wasn't performed (i.e., an update operation that resulted in a NOOP).
      */
     public long getSeqNo() {
@@ -263,7 +263,7 @@ public void readFrom(StreamInput in) throws IOException {
             seqNo = in.readZLong();
             primaryTerm = in.readVLong();
         } else {
-            seqNo = SequenceNumbersService.UNASSIGNED_SEQ_NO;
+            seqNo = SequenceNumbers.UNASSIGNED_SEQ_NO;
             primaryTerm = 0;
         }
         forcedRefresh = in.readBoolean();
@@ -375,7 +375,7 @@ public abstract static class Builder {
         protected Result result = null;
         protected boolean forcedRefresh;
         protected ShardInfo shardInfo = null;
-        protected Long seqNo = SequenceNumbersService.UNASSIGNED_SEQ_NO;
+        protected Long seqNo = SequenceNumbers.UNASSIGNED_SEQ_NO;
         protected Long primaryTerm = 0L;
 
         public ShardId getShardId() {