Update httpclient for JDK 11 TLS engine (#37994)

The apache commons http client implementations recently released
versions that solve TLS compatibility issues with the new TLS engine
that supports TLSv1.3 with JDK 11. This change updates our code to
use these versions since JDK 11 is a supported JDK and we should
allow the use of TLSv1.3.

buildSrc/version.properties

@@ -21,16 +21,13 @@ joda              = 2.10.1
 # test dependencies
 randomizedrunner  = 2.7.1
 junit             = 4.12
-httpclient        = 4.5.2
-# When updating httpcore, please also update server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
-httpcore          = 4.4.5
-# When updating httpasyncclient, please also update server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
-httpasyncclient   = 4.1.2
+httpclient        = 4.5.7
+httpcore          = 4.4.11
+httpasyncclient   = 4.1.4
 commonslogging    = 1.1.3
-commonscodec      = 1.10
+commonscodec      = 1.11
 hamcrest          = 1.3
 securemock        = 1.2
-# When updating mocksocket, please also update server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
 mocksocket        = 1.2
 
 # benchmark dependencies

client/rest/licenses/commons-codec-1.11.jar.sha1

@@ -0,0 +1 @@
+3acb4705652e16236558f0f4f2192cc33c3bd189

client/rest/licenses/httpasyncclient-4.1.4.jar.sha1

@@ -0,0 +1 @@
+f3a3240681faae3fa46b573a4c7e50cec9db0d86

client/rest/licenses/httpclient-4.5.7.jar.sha1

@@ -0,0 +1 @@
+dda059f4908e1b548b7ba68d81a3b05897f27cb0

client/rest/licenses/httpcore-4.4.11.jar.sha1

@@ -0,0 +1 @@
+de748cf874e4e193b42eceea9fe5574fabb9d4df

client/rest/licenses/httpcore-nio-4.4.11.jar.sha1

@@ -0,0 +1 @@
+7d0a97d01d39cff9aa3e6db81f21fddb2435f4e6

client/sniffer/licenses/commons-codec-1.11.jar.sha1

@@ -0,0 +1 @@
+3acb4705652e16236558f0f4f2192cc33c3bd189

client/sniffer/licenses/httpclient-4.5.7.jar.sha1

@@ -0,0 +1 @@
+dda059f4908e1b548b7ba68d81a3b05897f27cb0

client/sniffer/licenses/httpcore-4.4.11.jar.sha1

@@ -0,0 +1 @@
+de748cf874e4e193b42eceea9fe5574fabb9d4df

plugins/analysis-phonetic/licenses/commons-codec-1.11.jar.sha1

@@ -0,0 +1 @@
+3acb4705652e16236558f0f4f2192cc33c3bd189

plugins/discovery-azure-classic/licenses/commons-codec-1.11.jar.sha1

@@ -0,0 +1 @@
+3acb4705652e16236558f0f4f2192cc33c3bd189

plugins/discovery-azure-classic/licenses/httpclient-4.5.7.jar.sha1

@@ -0,0 +1 @@
+dda059f4908e1b548b7ba68d81a3b05897f27cb0

plugins/discovery-azure-classic/licenses/httpcore-4.4.11.jar.sha1

@@ -0,0 +1 @@
+de748cf874e4e193b42eceea9fe5574fabb9d4df

plugins/discovery-ec2/licenses/commons-codec-1.11.jar.sha1

@@ -0,0 +1 @@
+3acb4705652e16236558f0f4f2192cc33c3bd189

plugins/discovery-ec2/licenses/httpclient-4.5.7.jar.sha1

@@ -0,0 +1 @@
+dda059f4908e1b548b7ba68d81a3b05897f27cb0

plugins/discovery-ec2/licenses/httpcore-4.4.11.jar.sha1

@@ -0,0 +1 @@
+de748cf874e4e193b42eceea9fe5574fabb9d4df

plugins/discovery-gce/licenses/commons-codec-1.11.jar.sha1

@@ -0,0 +1 @@
+3acb4705652e16236558f0f4f2192cc33c3bd189

plugins/discovery-gce/licenses/httpclient-4.5.7.jar.sha1

@@ -0,0 +1 @@
+dda059f4908e1b548b7ba68d81a3b05897f27cb0

plugins/discovery-gce/licenses/httpcore-4.4.11.jar.sha1

@@ -0,0 +1 @@
+de748cf874e4e193b42eceea9fe5574fabb9d4df

plugins/ingest-attachment/licenses/commons-codec-1.11.jar.sha1

@@ -0,0 +1 @@
+3acb4705652e16236558f0f4f2192cc33c3bd189

plugins/repository-gcs/licenses/commons-codec-1.11.jar.sha1

@@ -0,0 +1 @@
+3acb4705652e16236558f0f4f2192cc33c3bd189

plugins/repository-gcs/licenses/httpclient-4.5.7.jar.sha1

@@ -0,0 +1 @@
+dda059f4908e1b548b7ba68d81a3b05897f27cb0

plugins/repository-gcs/licenses/httpcore-4.4.11.jar.sha1

@@ -0,0 +1 @@
+de748cf874e4e193b42eceea9fe5574fabb9d4df

plugins/repository-hdfs/build.gradle

@@ -52,7 +52,7 @@ dependencies {
   compile 'com.google.protobuf:protobuf-java:2.5.0'
   compile 'commons-logging:commons-logging:1.1.3'
   compile 'commons-cli:commons-cli:1.2'
-  compile 'commons-codec:commons-codec:1.10'
+  compile "commons-codec:commons-codec:${versions.commonscodec}"
   compile 'commons-collections:commons-collections:3.2.2'
   compile 'commons-configuration:commons-configuration:1.6'
   compile 'commons-io:commons-io:2.4'

plugins/repository-hdfs/licenses/commons-codec-1.11.jar.sha1

@@ -0,0 +1 @@
+3acb4705652e16236558f0f4f2192cc33c3bd189

plugins/repository-s3/licenses/commons-codec-1.11.jar.sha1

@@ -0,0 +1 @@
+3acb4705652e16236558f0f4f2192cc33c3bd189

plugins/repository-s3/licenses/httpclient-4.5.7.jar.sha1

@@ -0,0 +1 @@
+dda059f4908e1b548b7ba68d81a3b05897f27cb0

plugins/repository-s3/licenses/httpcore-4.4.11.jar.sha1

@@ -0,0 +1 @@
+de748cf874e4e193b42eceea9fe5574fabb9d4df

x-pack/plugin/core/licenses/commons-codec-1.11.jar.sha1

@@ -0,0 +1 @@
+3acb4705652e16236558f0f4f2192cc33c3bd189

x-pack/plugin/core/licenses/httpasyncclient-4.1.4.jar.sha1

@@ -0,0 +1 @@
+f3a3240681faae3fa46b573a4c7e50cec9db0d86

x-pack/plugin/core/licenses/httpclient-4.5.7.jar.sha1

@@ -0,0 +1 @@
+dda059f4908e1b548b7ba68d81a3b05897f27cb0

x-pack/plugin/core/licenses/httpcore-4.4.11.jar.sha1

@@ -0,0 +1 @@
+de748cf874e4e193b42eceea9fe5574fabb9d4df

x-pack/plugin/core/licenses/httpcore-nio-4.4.11.jar.sha1

@@ -0,0 +1 @@
+7d0a97d01d39cff9aa3e6db81f21fddb2435f4e6

x-pack/plugin/security/licenses/httpclient-cache-4.5.7.jar.sha1

@@ -0,0 +1 @@
+c13a0ce27c17831e5e5be6c751842006dcecb270

x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpClient.java

@@ -47,6 +47,7 @@
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
@@ -162,7 +163,9 @@ private void setWhitelistAutomaton(List<String> whiteListedHosts) {
     }
 
     public HttpResponse execute(HttpRequest request) throws IOException {
-        URI uri = createURI(request);
+        Tuple<HttpHost, URI> tuple = createURI(request);
+        final URI uri = tuple.v2();
+        final HttpHost httpHost = tuple.v1();
 
         HttpRequestBase internalRequest;
         if (request.method == HttpMethod.HEAD) {
@@ -212,7 +215,7 @@ public HttpResponse execute(HttpRequest request) throws IOException {
             // preemptive auth, no need to wait for a 401 first
             AuthCache authCache = new BasicAuthCache();
             BasicScheme basicAuth = new BasicScheme();
-            authCache.put(new HttpHost(request.host, request.port, request.scheme.scheme()), basicAuth);
+            authCache.put(httpHost, basicAuth);
             localContext.setAuthCache(authCache);
         }
 
@@ -233,7 +236,7 @@ public HttpResponse execute(HttpRequest request) throws IOException {
 
         internalRequest.setConfig(config.build());
 
-        try (CloseableHttpResponse response = SocketAccess.doPrivileged(() -> client.execute(internalRequest, localContext))) {
+        try (CloseableHttpResponse response = SocketAccess.doPrivileged(() -> client.execute(httpHost, internalRequest, localContext))) {
             // headers
             Header[] headers = response.getAllHeaders();
             Map<String, String[]> responseHeaders = new HashMap<>(headers.length);
@@ -310,7 +313,7 @@ private HttpProxy getProxyFromSettings(Settings settings) {
         return HttpProxy.NO_PROXY;
     }
 
-    private URI createURI(HttpRequest request) {
+    private Tuple<HttpHost, URI> createURI(HttpRequest request) {
         // this could be really simple, as the apache http client has a UriBuilder class, however this class is always doing
         // url path escaping, and we have done this already, so this would result in double escaping
         try {
@@ -320,7 +323,23 @@ private URI createURI(HttpRequest request) {
             URI uri = URIUtils.createURI(request.scheme.scheme(), request.host, request.port, request.path,
                     Strings.isNullOrEmpty(format) ? null : format, null);
 
-            return uri;
+            if (uri.isAbsolute() == false) {
+                throw new IllegalStateException("URI [" + uri.toASCIIString() + "] must be absolute");
+            }
+            final HttpHost httpHost = URIUtils.extractHost(uri);
+            // what a mess that we need to do this to workaround https://issues.apache.org/jira/browse/HTTPCLIENT-1968
+            // in some cases the HttpClient will re-write the URI which drops the escaping for
+            // slashes within a path. This rewriting is done to obtain a relative URI when
+            // a proxy is not being used. To avoid this we can handle making it relative ourselves
+            if (request.path != null && request.path.contains("%2F")) {
+                final boolean isUsingProxy = (request.proxy != null && request.proxy.equals(HttpProxy.NO_PROXY) == false) ||
+                    HttpProxy.NO_PROXY.equals(settingsProxy) == false;
+                if (isUsingProxy == false) {
+                    // we need a relative uri
+                    uri = URIUtils.createURI(null, null, -1, request.path, Strings.isNullOrEmpty(format) ? null : format, null);
+                }
+            }
+            return new Tuple<>(httpHost, uri);
         } catch (URISyntaxException e) {
             throw new IllegalArgumentException(e);
         }