Add handling for "etcdserver: revision of auth store is old" error ex…

…perienced by customers when Role changes are made to the ETCD Cluster. We are seeing a use case when ETCD Role grants are updated we are getting an error "etcdserver: revision of auth store is old" on all JETCD clients right after the grant is applied. The only fix now is for the client to restart and reconnect. ETCD Cluster config: ETCD_AUTH_TOKEN: jwt,pub-key=/auth-token/authToken.pub,priv-key=/auth-token/authToken.key,sign-method=RS256,ttl=8h Signed-off-by: darvay <[email protected]>
etcd-io · Nov 9, 2022 · 744a8d9 · 744a8d9
1 parent cadde88
commit 744a8d9
Show file tree

Hide file tree

Showing 4 changed files with 26 additions and 0 deletions.
diff --git a/jetcd-core/src/main/java/io/etcd/jetcd/impl/ClientConnectionManager.java b/jetcd-core/src/main/java/io/etcd/jetcd/impl/ClientConnectionManager.java
@@ -273,6 +273,9 @@ public <S, T> CompletableFuture<T> execute(
                         if (Errors.isInvalidTokenError(error)) {
                             authCredential().refresh();
                         }
+                        if (Errors.isAuthStoreExpired(error)) {
+                            authCredential().refresh();
+                        }
                         if (!execution.retryOn(error)) {
                             // permanent failure
                             wrappedFuture.completeExceptionally(error);

diff --git a/jetcd-core/src/main/java/io/etcd/jetcd/impl/Impl.java b/jetcd-core/src/main/java/io/etcd/jetcd/impl/Impl.java
@@ -16,6 +16,7 @@
 import net.jodah.failsafe.Failsafe;
 import net.jodah.failsafe.RetryPolicy;
 
+import static io.etcd.jetcd.support.Errors.isAuthStoreExpired;
 import static io.etcd.jetcd.support.Errors.isInvalidTokenError;
 
 abstract class Impl {
@@ -113,6 +114,9 @@ protected <S, T> CompletableFuture<T> execute(
                 if (isInvalidTokenError(status)) {
                     connectionManager.authCredential().refresh();
                 }
+                if (isAuthStoreExpired(status)) {
+                    connectionManager.authCredential().refresh();
+                }
                 return doRetry.test(status);
             })
             .onRetriesExceeded(e -> logger.warn("maximum number of auto retries reached"))

diff --git a/jetcd-core/src/main/java/io/etcd/jetcd/support/Errors.java b/jetcd-core/src/main/java/io/etcd/jetcd/support/Errors.java
@@ -21,6 +21,7 @@
 public final class Errors {
     public static final String NO_LEADER_ERROR_MESSAGE = "etcdserver: no leader";
     public static final String INVALID_AUTH_TOKEN_ERROR_MESSAGE = "etcdserver: invalid auth token";
+    public static final String ERROR_AUTH_STORE_OLD = "etcdserver: revision of auth store is old";
 
     private Errors() {
     }
@@ -39,6 +40,16 @@ public static boolean isInvalidTokenError(Status status) {
             && INVALID_AUTH_TOKEN_ERROR_MESSAGE.equals(status.getDescription());
     }
 
+    public static boolean isAuthStoreExpired(Throwable e) {
+        Status status = Status.fromThrowable(e);
+        return isAuthStoreExpired(status);
+    }
+
+    public static boolean isAuthStoreExpired(Status status) {
+        return (status.getCode() == Status.Code.UNAUTHENTICATED || status.getCode() == Status.Code.INVALID_ARGUMENT)
+            && ERROR_AUTH_STORE_OLD.equals(status.getDescription());
+    }
+
     public static boolean isHaltError(final Status status) {
         return status.getCode() != Status.Code.UNAVAILABLE && status.getCode() != Status.Code.INTERNAL;
     }

diff --git a/jetcd-core/src/test/java/io/etcd/jetcd/impl/UtilTest.java b/jetcd-core/src/test/java/io/etcd/jetcd/impl/UtilTest.java
@@ -30,6 +30,14 @@
 @Timeout(value = 30, unit = TimeUnit.SECONDS)
 class UtilTest {
 
+    @Test
+    public void testAuthStoreExpired() {
+        Status authExpiredStatus = Status.INVALID_ARGUMENT
+            .withDescription(Errors.ERROR_AUTH_STORE_OLD);
+        Status status = Status.fromThrowable(new StatusException(authExpiredStatus));
+        assertThat(Errors.isAuthStoreExpired(status)).isTrue();
+    }
+
     @Test
     public void testAuthErrorIsNotRetryable() {
         Status authErrorStatus = Status.UNAUTHENTICATED