Retry when store is down in high layer (#2710)

pingcap · May 30, 2023 · 4ed8391 · 4ed8391
1 parent e438029
commit 4ed8391
Show file tree

Hide file tree

Showing 2 changed files with 32 additions and 5 deletions.
diff --git a/.github/workflows/alter-primary-key-false-test.yml b/.github/workflows/alter-primary-key-false-test.yml
@@ -1,12 +1,12 @@
 name: alter-primary-key-false-test
 
 on:
-  push:
-    branches:
-      - master
   pull_request:
     branches:
-      - master
+      - release-3.1
+      - release-3.0
+      - release-2.5
+      - release-2.4
 
 jobs:
   test:

diff --git a/tikv-client/src/main/java/com/pingcap/tikv/operation/iterator/DAGIterator.java b/tikv-client/src/main/java/com/pingcap/tikv/operation/iterator/DAGIterator.java
@@ -43,6 +43,7 @@
 import org.tikv.common.region.TiRegion;
 import org.tikv.common.region.TiStore;
 import org.tikv.common.region.TiStoreType;
+import org.tikv.common.util.BackOffFunction;
 import org.tikv.common.util.BackOffer;
 import org.tikv.common.util.ConcreteBackOffer;
 import org.tikv.common.util.RangeSplitter;
@@ -206,8 +207,9 @@ private SelectResponse process(RegionTask regionTask) {
     Queue<SelectResponse> responseQueue = new ArrayDeque<>();
     remainTasks.add(regionTask);
     BackOffer backOffer = ConcreteBackOffer.newCopNextMaxBackOff();
-
     HashSet<Long> resolvedLocks = new HashSet<>();
+    BackOffer storeUnreachableBackOffer = ConcreteBackOffer.newCustomBackOff(60 * 1000);
+
     // In case of one region task spilt into several others, we ues a queue to properly handle all
     // the remaining tasks.
     while (!remainTasks.isEmpty()) {
@@ -220,6 +222,31 @@ private SelectResponse process(RegionTask regionTask) {
       TiStore store = task.getStore();
 
       try {
+        if (store == null || !store.isReachable()) {
+          storeUnreachableBackOffer.doBackOffWithMaxSleep(
+              BackOffFunction.BackOffFuncType.BoServerBusy,
+              2000,
+              new TiClientInternalException("retry timeout: store is null or unreachable"));
+          if (store == null) {
+            logger.warn("TiKV store is null, invalid cache and retry");
+          } else {
+            logger.warn(
+                "TiKV store " + store.getAddress() + " is unreachable, invalid cache and retry");
+          }
+          clientSession.getTiKVSession().getRegionManager().invalidateRegion(region);
+          try {
+            remainTasks.addAll(
+                RangeSplitter.newSplitter(clientSession.getTiKVSession().getRegionManager())
+                    .splitRangeByRegion(ranges, storeType));
+          } catch (Exception e) {
+            // If the pd is switching leader, the region invalid exception will be thrown. In this
+            // case, we can retry with the original task.
+            logger.warn("split range by region error, retry with the original task", e);
+            remainTasks.add(task);
+          }
+          continue;
+        }
+
         RegionStoreClient client =
             clientSession
                 .getTiKVSession()