SOLR-17012: Update Apache Hadoop to 3.3.6 and Apache Curator to 5.5.0 (…

…#1743) Co-authored-by: Kevin Risden <[email protected]>
apache · Oct 4, 2023 · 621bc2a · 621bc2a
1 parent 540f61c
commit 621bc2a
Show file tree

Hide file tree

Showing 30 changed files with 127 additions and 40 deletions.
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
@@ -84,7 +84,7 @@ Bug Fixes
 
 Dependency Upgrades
 ---------------------
-(No changes)
+* SOLR-17012: Update Apache Hadoop to 3.3.6 and Apache Curator to 5.5.0 (Kevin Risden)
 
 Other Changes
 ---------------------

diff --git a/solr/licenses/curator-client-4.3.0.jar.sha1 b/solr/licenses/curator-client-4.3.0.jar.sha1
diff --git a/solr/licenses/curator-client-5.5.0.jar.sha1 b/solr/licenses/curator-client-5.5.0.jar.sha1
@@ -0,0 +1 @@
+db2d83bdc0bac7b4f25fc113d8ce3eedc0a4e89c
diff --git a/solr/licenses/curator-framework-4.3.0.jar.sha1 b/solr/licenses/curator-framework-4.3.0.jar.sha1
diff --git a/solr/licenses/curator-framework-5.5.0.jar.sha1 b/solr/licenses/curator-framework-5.5.0.jar.sha1
@@ -0,0 +1 @@
+b706a216e49352103bd2527e83b1ec2410924494
diff --git a/solr/licenses/curator-recipes-4.3.0.jar.sha1 b/solr/licenses/curator-recipes-4.3.0.jar.sha1
diff --git a/solr/licenses/curator-recipes-5.5.0.jar.sha1 b/solr/licenses/curator-recipes-5.5.0.jar.sha1
@@ -0,0 +1 @@
+4aa0cfb129c36cd91528fc1b8775705280e60285
diff --git a/solr/licenses/hadoop-annotations-3.3.5.jar.sha1 b/solr/licenses/hadoop-annotations-3.3.5.jar.sha1
diff --git a/solr/licenses/hadoop-annotations-3.3.6.jar.sha1 b/solr/licenses/hadoop-annotations-3.3.6.jar.sha1
@@ -0,0 +1 @@
+451bc97f7519017cfa96c8f11d79e1e8027968b2
diff --git a/solr/licenses/hadoop-auth-3.3.5.jar.sha1 b/solr/licenses/hadoop-auth-3.3.5.jar.sha1
diff --git a/solr/licenses/hadoop-auth-3.3.6.jar.sha1 b/solr/licenses/hadoop-auth-3.3.6.jar.sha1
@@ -0,0 +1 @@
+066aaf67a580910de62f92f21f76e3df170483cf
diff --git a/solr/licenses/hadoop-client-api-3.3.5.jar.sha1 b/solr/licenses/hadoop-client-api-3.3.5.jar.sha1
diff --git a/solr/licenses/hadoop-client-api-3.3.6.jar.sha1 b/solr/licenses/hadoop-client-api-3.3.6.jar.sha1
@@ -0,0 +1 @@
+12ac6f103a0ff29fce17a078c7c64d25320b6165
diff --git a/solr/licenses/hadoop-client-minicluster-3.3.5.jar.sha1 b/solr/licenses/hadoop-client-minicluster-3.3.5.jar.sha1
diff --git a/solr/licenses/hadoop-client-minicluster-3.3.6.jar.sha1 b/solr/licenses/hadoop-client-minicluster-3.3.6.jar.sha1
@@ -0,0 +1 @@
+1d7be37c806e6703ea672d0e5e47fd43ea721acc
diff --git a/solr/licenses/hadoop-client-runtime-3.3.5.jar.sha1 b/solr/licenses/hadoop-client-runtime-3.3.5.jar.sha1
diff --git a/solr/licenses/hadoop-client-runtime-3.3.6.jar.sha1 b/solr/licenses/hadoop-client-runtime-3.3.6.jar.sha1
@@ -0,0 +1 @@
+81065531e63fccbe85fb04a3274709593fb00d3c
diff --git a/solr/licenses/hadoop-common-3.3.5.jar.sha1 b/solr/licenses/hadoop-common-3.3.5.jar.sha1
diff --git a/solr/licenses/hadoop-common-3.3.6.jar.sha1 b/solr/licenses/hadoop-common-3.3.6.jar.sha1
@@ -0,0 +1 @@
+09ca864bec94779e74b99e84ea02dba85a641233
diff --git a/solr/licenses/hadoop-hdfs-3.3.5-tests.jar.sha1 b/solr/licenses/hadoop-hdfs-3.3.5-tests.jar.sha1
diff --git a/solr/licenses/hadoop-hdfs-3.3.5.jar.sha1 b/solr/licenses/hadoop-hdfs-3.3.5.jar.sha1
diff --git a/solr/licenses/hadoop-hdfs-3.3.6-tests.jar.sha1 b/solr/licenses/hadoop-hdfs-3.3.6-tests.jar.sha1
@@ -0,0 +1 @@
+5058b645375c6a68f509e167ad6a6ada9642df09
diff --git a/solr/licenses/hadoop-hdfs-3.3.6.jar.sha1 b/solr/licenses/hadoop-hdfs-3.3.6.jar.sha1
@@ -0,0 +1 @@
+ba40aca60f39599d5b1f1d32b35295bfde1f3c8b
diff --git a/solr/licenses/hadoop-minikdc-3.3.5.jar.sha1 b/solr/licenses/hadoop-minikdc-3.3.5.jar.sha1
diff --git a/solr/licenses/hadoop-minikdc-3.3.6.jar.sha1 b/solr/licenses/hadoop-minikdc-3.3.6.jar.sha1
@@ -0,0 +1 @@
+7f454a44beea61f42f37b414c0d73decbe61de32
diff --git a/...s/hadoop-auth/src/java/org/apache/solr/security/hadoop/DelegationTokenKerberosFilter.java b/...s/hadoop-auth/src/java/org/apache/solr/security/hadoop/DelegationTokenKerberosFilter.java
@@ -16,10 +16,15 @@
  */
 package org.apache.solr.security.hadoop;
 
+import static org.apache.hadoop.security.token.delegation.ZKDelegationTokenSecretManager.ZK_DTSM_ZNODE_WORKING_PATH;
+import static org.apache.hadoop.security.token.delegation.ZKDelegationTokenSecretManager.ZK_DTSM_ZNODE_WORKING_PATH_DEAFULT;
+
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Enumeration;
 import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.ExecutorService;
 import javax.servlet.FilterChain;
 import javax.servlet.FilterConfig;
 import javax.servlet.ServletException;
@@ -41,6 +46,8 @@
 import org.apache.solr.common.cloud.SolrZkClient;
 import org.apache.solr.common.cloud.ZkACLProvider;
 import org.apache.solr.common.cloud.ZkCredentialsProvider;
+import org.apache.solr.common.util.ExecutorUtil;
+import org.apache.solr.common.util.SolrNamedThreadFactory;
 import org.apache.zookeeper.CreateMode;
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.data.ACL;
@@ -51,6 +58,7 @@
  * reuse the authentication of an end-user or another application.
  */
 public class DelegationTokenKerberosFilter extends DelegationTokenAuthenticationFilter {
+  private ExecutorService curatorSafeServiceExecutor;
   private CuratorFramework curatorFramework;
 
   @Override
@@ -62,7 +70,8 @@ public void init(FilterConfig conf) throws ServletException {
       try {
         conf.getServletContext()
             .setAttribute(
-                "signer.secret.provider.zookeeper.curator.client", getCuratorClient(zkClient));
+                "signer.secret.provider.zookeeper.curator.client",
+                getCuratorClientInternal(conf, zkClient));
       } catch (InterruptedException | KeeperException e) {
         throw new ServletException(e);
       }
@@ -123,8 +132,14 @@ public void doFilter(ServletRequest servletRequest, ServletResponse servletRespo
   @Override
   public void destroy() {
     super.destroy();
-    if (curatorFramework != null) curatorFramework.close();
-    curatorFramework = null;
+    if (curatorFramework != null) {
+      curatorFramework.close();
+      curatorFramework = null;
+    }
+    if (curatorSafeServiceExecutor != null) {
+      ExecutorUtil.shutdownNowAndAwaitTermination(curatorSafeServiceExecutor);
+      curatorSafeServiceExecutor = null;
+    }
   }
 
   @Override
@@ -141,6 +156,31 @@ protected void initializeAuthHandler(String authHandlerClassName, FilterConfig f
     newAuthHandler.setAuthHandler(authHandler);
   }
 
+  private CuratorFramework getCuratorClientInternal(FilterConfig conf, SolrZkClient zkClient)
+      throws KeeperException, InterruptedException {
+    // There is a race condition where the znodeWorking path used by ZKDelegationTokenSecretManager
+    // can be created by multiple nodes, but Hadoop doesn't handle this well. This explicitly
+    // creates it up front and handles if the znode already exists. This relates to HADOOP-18452
+    // but didn't solve the underlying issue of the race condition.
+
+    // If namespace parents are implicitly created, they won't have ACLs.
+    // So, let's explicitly create them.
+    CuratorFramework curatorFramework = getCuratorClient(zkClient);
+    CuratorFramework nullNsFw = curatorFramework.usingNamespace(null);
+    try {
+      String znodeWorkingPath =
+          '/'
+              + Objects.requireNonNullElse(
+                  conf.getInitParameter(ZK_DTSM_ZNODE_WORKING_PATH),
+                  ZK_DTSM_ZNODE_WORKING_PATH_DEAFULT)
+              + "/ZKDTSMRoot";
+      nullNsFw.create().creatingParentContainersIfNeeded().forPath(znodeWorkingPath);
+    } catch (Exception ignore) {
+    }
+
+    return curatorFramework;
+  }
+
   protected CuratorFramework getCuratorClient(SolrZkClient zkClient)
       throws InterruptedException, KeeperException {
     // should we try to build a RetryPolicy off of the ZkController?
@@ -163,10 +203,12 @@ protected CuratorFramework getCuratorClient(SolrZkClient zkClient)
     try {
       zkClient.makePath(
           SecurityAwareZkACLProvider.SECURITY_ZNODE_PATH, CreateMode.PERSISTENT, true);
-    } catch (KeeperException.NodeExistsException ex) {
-      // ignore?
+    } catch (KeeperException.NodeExistsException ignore) {
     }
 
+    curatorSafeServiceExecutor =
+        ExecutorUtil.newMDCAwareSingleThreadExecutor(
+            new SolrNamedThreadFactory("delegationtokenkerberosfilter-curator-safeService"));
     curatorFramework =
         CuratorFrameworkFactory.builder()
             .namespace(zkNamespace)
@@ -176,8 +218,10 @@ protected CuratorFramework getCuratorClient(SolrZkClient zkClient)
             .authorization(curatorToSolrZk.getAuthInfos())
             .sessionTimeoutMs(zkClient.getZkClientTimeout())
             .connectionTimeoutMs(connectionTimeoutMs)
+            .runSafeService(curatorSafeServiceExecutor)
             .build();
     curatorFramework.start();
+
     return curatorFramework;
   }
 

diff --git a/solr/modules/hadoop-auth/src/java/org/apache/solr/security/hadoop/HadoopAuthFilter.java b/solr/modules/hadoop-auth/src/java/org/apache/solr/security/hadoop/HadoopAuthFilter.java
@@ -16,9 +16,14 @@
  */
 package org.apache.solr.security.hadoop;
 
+import static org.apache.hadoop.security.token.delegation.ZKDelegationTokenSecretManager.ZK_DTSM_ZNODE_WORKING_PATH;
+import static org.apache.hadoop.security.token.delegation.ZKDelegationTokenSecretManager.ZK_DTSM_ZNODE_WORKING_PATH_DEAFULT;
+
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.ExecutorService;
 import javax.servlet.FilterChain;
 import javax.servlet.FilterConfig;
 import javax.servlet.ServletException;
@@ -39,6 +44,8 @@
 import org.apache.solr.common.cloud.SolrZkClient;
 import org.apache.solr.common.cloud.ZkACLProvider;
 import org.apache.solr.common.cloud.ZkCredentialsProvider;
+import org.apache.solr.common.util.ExecutorUtil;
+import org.apache.solr.common.util.SolrNamedThreadFactory;
 import org.apache.zookeeper.CreateMode;
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.data.ACL;
@@ -53,6 +60,7 @@ public class HadoopAuthFilter extends DelegationTokenAuthenticationFilter {
    */
   static final String DELEGATION_TOKEN_ZK_CLIENT = "solr.kerberos.delegation.token.zk.client";
 
+  private ExecutorService curatorSafeServiceExecutor;
   private CuratorFramework curatorFramework;
 
   @Override
@@ -63,7 +71,8 @@ public void init(FilterConfig conf) throws ServletException {
       try {
         conf.getServletContext()
             .setAttribute(
-                "signer.secret.provider.zookeeper.curator.client", getCuratorClient(zkClient));
+                "signer.secret.provider.zookeeper.curator.client",
+                getCuratorClientInternal(conf, zkClient));
       } catch (KeeperException | InterruptedException e) {
         throw new ServletException(e);
       }
@@ -104,8 +113,12 @@ public void destroy() {
     super.destroy();
     if (curatorFramework != null) {
       curatorFramework.close();
+      curatorFramework = null;
+    }
+    if (curatorSafeServiceExecutor != null) {
+      ExecutorUtil.shutdownNowAndAwaitTermination(curatorSafeServiceExecutor);
+      curatorSafeServiceExecutor = null;
     }
-    curatorFramework = null;
   }
 
   @Override
@@ -122,6 +135,31 @@ protected void initializeAuthHandler(String authHandlerClassName, FilterConfig f
     newAuthHandler.setAuthHandler(authHandler);
   }
 
+  private CuratorFramework getCuratorClientInternal(FilterConfig conf, SolrZkClient zkClient)
+      throws KeeperException, InterruptedException {
+    // There is a race condition where the znodeWorking path used by ZKDelegationTokenSecretManager
+    // can be created by multiple nodes, but Hadoop doesn't handle this well. This explicitly
+    // creates it up front and handles if the znode already exists. This relates to HADOOP-18452
+    // but didn't solve the underlying issue of the race condition.
+
+    // If namespace parents are implicitly created, they won't have ACLs.
+    // So, let's explicitly create them.
+    CuratorFramework curatorFramework = getCuratorClient(zkClient);
+    CuratorFramework nullNsFw = curatorFramework.usingNamespace(null);
+    try {
+      String znodeWorkingPath =
+          '/'
+              + Objects.requireNonNullElse(
+                  conf.getInitParameter(ZK_DTSM_ZNODE_WORKING_PATH),
+                  ZK_DTSM_ZNODE_WORKING_PATH_DEAFULT)
+              + "/ZKDTSMRoot";
+      nullNsFw.create().creatingParentContainersIfNeeded().forPath(znodeWorkingPath);
+    } catch (Exception ignore) {
+    }
+
+    return curatorFramework;
+  }
+
   protected CuratorFramework getCuratorClient(SolrZkClient zkClient)
       throws KeeperException, InterruptedException {
     // should we try to build a RetryPolicy off of the ZkController?
@@ -144,13 +182,12 @@ protected CuratorFramework getCuratorClient(SolrZkClient zkClient)
     try {
       zkClient.makePath(
           SecurityAwareZkACLProvider.SECURITY_ZNODE_PATH, CreateMode.PERSISTENT, true);
-
-    } catch (KeeperException ex) {
-      if (ex.code() != KeeperException.Code.NODEEXISTS) {
-        throw ex;
-      }
+    } catch (KeeperException.NodeExistsException ignore) {
     }
 
+    curatorSafeServiceExecutor =
+        ExecutorUtil.newMDCAwareSingleThreadExecutor(
+            new SolrNamedThreadFactory("hadoopauthfilter-curator-safeService"));
     curatorFramework =
         CuratorFrameworkFactory.builder()
             .namespace(zkNamespace)
@@ -160,8 +197,10 @@ protected CuratorFramework getCuratorClient(SolrZkClient zkClient)
             .authorization(curatorToSolrZk.getAuthInfos())
             .sessionTimeoutMs(zkClient.getZkClientTimeout())
             .connectionTimeoutMs(connectionTimeoutMs)
+            .runSafeService(curatorSafeServiceExecutor)
             .build();
     curatorFramework.start();
+
     return curatorFramework;
   }
 

diff --git a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc
@@ -64,6 +64,10 @@ It is always strongly recommended that you fully reindex your documents after a
 In Solr 8, it was possible to add docValues to a schema without re-indexing via `UninvertDocValuesMergePolicy`, an advanced/expert utility.
 Due to changes in Lucene 9, that isn't possible any more.
 
+== Solr 9.5
+=== Curator upgraded to 5.5.0 and requires Zookeeper 3.5.x or higher
+xref:upgrade-notes:major-changes-in-solr-9.html#solr-8-2[Solr 8.2 recommended using Zookeeper 3.5.5] and now with Curator 5.5.0 requires https://curator.apache.org/docs/breaking-changes/[Zookeeper 3.5.x or higher]. This primarily affects users of `hadoop-auth`, but usage of Curator could affect other parts of Solr.
+
 == Solr 9.4
 === The Built-In Config Sets
 * The build in ConfigSets (`_default` and `sample_techproducts_configs`), now use a default `autoSoftCommit` time of 3 seconds,

diff --git a/versions.lock b/versions.lock
@@ -44,7 +44,7 @@ com.google.cloud:google-cloud-storage:2.27.0 (2 constraints: d71c8a27)
 com.google.code.gson:gson:2.10.1 (7 constraints: 005f69b0)
 com.google.errorprone:error_prone_annotations:2.22.0 (11 constraints: 6e891f5a)
 com.google.guava:failureaccess:1.0.1 (2 constraints: f9199e37)
-com.google.guava:guava:32.1.2-jre (26 constraints: a37b7b56)
+com.google.guava:guava:32.1.2-jre (26 constraints: 407b586c)
 com.google.guava:guava-parent:32.1.2-jre (1 constraints: b80ba5eb)
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava (2 constraints: 4b35b0a0)
 com.google.http-client:google-http-client:1.43.3 (11 constraints: 3fbf96b4)
@@ -177,14 +177,14 @@ org.apache.commons:commons-exec:1.3 (2 constraints: a41056b8)
 org.apache.commons:commons-lang3:3.13.0 (5 constraints: 9c3eb936)
 org.apache.commons:commons-math3:3.6.1 (5 constraints: 57322799)
 org.apache.commons:commons-text:1.10.0 (1 constraints: d911adf8)
-org.apache.curator:curator-client:4.3.0 (2 constraints: e214cba2)
-org.apache.curator:curator-framework:4.3.0 (2 constraints: ff13b474)
-org.apache.curator:curator-recipes:4.3.0 (1 constraints: 09050836)
-org.apache.hadoop:hadoop-annotations:3.3.5 (1 constraints: 0d050836)
-org.apache.hadoop:hadoop-auth:3.3.5 (1 constraints: 0d050836)
-org.apache.hadoop:hadoop-client-api:3.3.5 (3 constraints: 22287160)
-org.apache.hadoop:hadoop-client-runtime:3.3.5 (2 constraints: 6d17a643)
-org.apache.hadoop:hadoop-common:3.3.5 (1 constraints: 0d050836)
+org.apache.curator:curator-client:5.5.0 (2 constraints: e81468a3)
+org.apache.curator:curator-framework:5.5.0 (2 constraints: 05144b75)
+org.apache.curator:curator-recipes:5.5.0 (1 constraints: 0c051336)
+org.apache.hadoop:hadoop-annotations:3.3.6 (1 constraints: 0e050936)
+org.apache.hadoop:hadoop-auth:3.3.6 (1 constraints: 0e050936)
+org.apache.hadoop:hadoop-client-api:3.3.6 (3 constraints: 25280861)
+org.apache.hadoop:hadoop-client-runtime:3.3.6 (2 constraints: 6f17dc43)
+org.apache.hadoop:hadoop-common:3.3.6 (1 constraints: 0e050936)
 org.apache.hadoop.thirdparty:hadoop-shaded-guava:1.1.1 (1 constraints: 0505f435)
 org.apache.httpcomponents:httpclient:4.5.14 (9 constraints: 62806342)
 org.apache.httpcomponents:httpcore:4.4.16 (8 constraints: 256d4617)
@@ -256,7 +256,7 @@ org.apache.tika:tika-core:1.28.5 (2 constraints: d8118f11)
 org.apache.tika:tika-parsers:1.28.5 (1 constraints: 42054a3b)
 org.apache.tomcat:annotations-api:6.0.53 (1 constraints: 40054e3b)
 org.apache.xmlbeans:xmlbeans:5.0.3 (2 constraints: 72173075)
-org.apache.zookeeper:zookeeper:3.9.0 (2 constraints: a0134e5f)
+org.apache.zookeeper:zookeeper:3.9.0 (2 constraints: 9c134e5f)
 org.apache.zookeeper:zookeeper-jute:3.9.0 (2 constraints: 99125f23)
 org.apiguardian:apiguardian-api:1.1.2 (2 constraints: 601bd5a8)
 org.bitbucket.b_c:jose4j:0.9.3 (1 constraints: 0e050936)
@@ -410,9 +410,9 @@ net.bytebuddy:byte-buddy:1.14.6 (1 constraints: 460b44de)
 net.minidev:accessors-smart:2.4.9 (1 constraints: 500a92b8)
 net.minidev:json-smart:2.4.10 (1 constraints: 400e9a7c)
 no.nav.security:mock-oauth2-server:0.5.10 (1 constraints: 3805333b)
-org.apache.hadoop:hadoop-client-minicluster:3.3.5 (1 constraints: 0d050836)
-org.apache.hadoop:hadoop-hdfs:3.3.5 (1 constraints: 0d050836)
-org.apache.hadoop:hadoop-minikdc:3.3.5 (1 constraints: 0d050836)
+org.apache.hadoop:hadoop-client-minicluster:3.3.6 (1 constraints: 0e050936)
+org.apache.hadoop:hadoop-hdfs:3.3.6 (1 constraints: 0e050936)
+org.apache.hadoop:hadoop-minikdc:3.3.6 (1 constraints: 0e050936)
 org.apache.kerby:kerb-admin:1.0.1 (1 constraints: 840d892f)
 org.apache.kerby:kerb-client:1.0.1 (1 constraints: 840d892f)
 org.apache.kerby:kerb-common:1.0.1 (2 constraints: a51841ca)

diff --git a/versions.props b/versions.props
@@ -39,9 +39,9 @@ org.apache.commons:commons-configuration2=2.9.0
 org.apache.commons:commons-exec=1.3
 org.apache.commons:commons-lang3=3.13.0
 org.apache.commons:commons-math3=3.6.1
-org.apache.curator:*=4.3.0
+org.apache.curator:*=5.5.0
 org.apache.hadoop.thirdparty:*=1.1.1
-org.apache.hadoop:*=3.3.5
+org.apache.hadoop:*=3.3.6
 org.apache.httpcomponents:httpclient=4.5.14
 org.apache.httpcomponents:httpcore=4.4.16
 org.apache.httpcomponents:httpmime=4.5.14