diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/filterrewrite/AggregatorBridge.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/filterrewrite/AggregatorBridge.java
index 095e0cd8cf75b..6b34582b259ea 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/filterrewrite/AggregatorBridge.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/filterrewrite/AggregatorBridge.java
@@ -67,7 +67,7 @@ void setRangesConsumer(Consumer<Ranges> setRanges) {
      *
      * @param leaf the leaf reader context for the segment
      */
-    abstract Ranges prepareFromSegment(LeafReaderContext leaf) throws IOException;
+    abstract Ranges tryBuildRangesFromSegment(LeafReaderContext leaf) throws IOException;
 
     /**
      * Attempts to build aggregation results for a segment
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/filterrewrite/DateHistogramAggregatorBridge.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/filterrewrite/DateHistogramAggregatorBridge.java
index 22206d4dcb4bc..8bff3fdc5fefb 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/filterrewrite/DateHistogramAggregatorBridge.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/filterrewrite/DateHistogramAggregatorBridge.java
@@ -54,7 +54,7 @@ protected void buildRanges(SearchContext context) throws IOException {
     }
 
     @Override
-    final Ranges prepareFromSegment(LeafReaderContext leaf) throws IOException {
+    final Ranges tryBuildRangesFromSegment(LeafReaderContext leaf) throws IOException {
         long[] bounds = Helper.getSegmentBounds(leaf, fieldType.name());
         return buildRanges(bounds, maxRewriteFilters);
     }
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/filterrewrite/FilterRewriteOptimizationContext.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/filterrewrite/FilterRewriteOptimizationContext.java
index a1670035b7909..87faafe4526de 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/filterrewrite/FilterRewriteOptimizationContext.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/filterrewrite/FilterRewriteOptimizationContext.java
@@ -18,8 +18,7 @@
 import org.opensearch.search.internal.SearchContext;
 
 import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.BiConsumer;
 
 import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
@@ -41,14 +40,13 @@ public final class FilterRewriteOptimizationContext {
     private final AggregatorBridge aggregatorBridge;
     private String shardId;
 
-    private Ranges ranges;
-    private final Map<Integer, Ranges> rangesFromSegment = new HashMap<>(); // map of segment ordinal to its ranges
+    private Ranges ranges; // built at shard level
 
     // debug info related fields
-    private int leafNodeVisited;
-    private int innerNodeVisited;
-    private int segments;
-    private int optimizedSegments;
+    private final AtomicInteger leafNodeVisited = new AtomicInteger();
+    private final AtomicInteger innerNodeVisited = new AtomicInteger();
+    private final AtomicInteger segments = new AtomicInteger();
+    private final AtomicInteger optimizedSegments = new AtomicInteger();
 
     public FilterRewriteOptimizationContext(
         AggregatorBridge aggregatorBridge,
@@ -90,19 +88,6 @@ void setRanges(Ranges ranges) {
         this.ranges = ranges;
     }
 
-    void setRangesFromSegment(int leafOrd, Ranges ranges) {
-        this.rangesFromSegment.put(leafOrd, ranges);
-    }
-
-    void clearRangesFromSegment(int leafOrd) {
-        this.rangesFromSegment.remove(leafOrd);
-    }
-
-    Ranges getRanges(int leafOrd) {
-        if (!preparedAtShardLevel) return rangesFromSegment.get(leafOrd);
-        return ranges;
-    }
-
     /**
      * Try to populate the bucket doc counts for aggregation
      * <p>
@@ -113,7 +98,7 @@ Ranges getRanges(int leafOrd) {
      */
     public boolean tryOptimize(final LeafReaderContext leafCtx, final BiConsumer<Long, Long> incrementDocCount, boolean segmentMatchAll)
         throws IOException {
-        segments++;
+        segments.incrementAndGet();
         if (!canOptimize) {
             return false;
         }
@@ -135,62 +120,70 @@ public boolean tryOptimize(final LeafReaderContext leafCtx, final BiConsumer<Lon
             return false;
         }
 
-        Ranges ranges = tryBuildRangesFromSegment(leafCtx, segmentMatchAll);
+        Ranges ranges = getRanges(leafCtx, segmentMatchAll);
         if (ranges == null) return false;
 
-        consumeDebugInfo(aggregatorBridge.tryOptimize(values, incrementDocCount, getRanges(leafCtx.ord)));
+        consumeDebugInfo(aggregatorBridge.tryOptimize(values, incrementDocCount, ranges));
 
-        optimizedSegments++;
+        optimizedSegments.incrementAndGet();
         logger.debug("Fast filter optimization applied to shard {} segment {}", shardId, leafCtx.ord);
         logger.debug("Crossed leaf nodes: {}, inner nodes: {}", leafNodeVisited, innerNodeVisited);
 
-        clearRangesFromSegment(leafCtx.ord);
         return true;
     }
 
+    Ranges getRanges(LeafReaderContext leafCtx, boolean segmentMatchAll) {
+        if (!preparedAtShardLevel) {
+            try {
+                return getRangesFromSegment(leafCtx, segmentMatchAll);
+            } catch (IOException e) {
+                logger.warn("Failed to build ranges from segment.", e);
+                return null;
+            }
+        }
+        return ranges;
+    }
+
     /**
      * Even when ranges cannot be built at shard level, we can still build ranges
      * at segment level when it's functionally match-all at segment level
      */
-    private Ranges tryBuildRangesFromSegment(LeafReaderContext leafCtx, boolean segmentMatchAll) throws IOException {
-        if (!preparedAtShardLevel && !segmentMatchAll) {
+    private Ranges getRangesFromSegment(LeafReaderContext leafCtx, boolean segmentMatchAll) throws IOException {
+        if (!segmentMatchAll) {
             return null;
         }
 
-        if (!preparedAtShardLevel) { // not built at shard level but segment match all
-            logger.debug("Shard {} segment {} functionally match all documents. Build the fast filter", shardId, leafCtx.ord);
-            setRangesFromSegment(leafCtx.ord, aggregatorBridge.prepareFromSegment(leafCtx));
-        }
-        return getRanges(leafCtx.ord);
+        logger.debug("Shard {} segment {} functionally match all documents. Build the fast filter", shardId, leafCtx.ord);
+        return aggregatorBridge.tryBuildRangesFromSegment(leafCtx);
     }
 
     /**
      * Contains debug info of BKD traversal to show in profile
      */
     static class DebugInfo {
-        private int leafNodeVisited = 0; // leaf node visited
-        private int innerNodeVisited = 0; // inner node visited
+        private final AtomicInteger leafNodeVisited = new AtomicInteger(); // leaf node visited
+        private final AtomicInteger innerNodeVisited = new AtomicInteger(); // inner node visited
 
         void visitLeaf() {
-            leafNodeVisited++;
+            leafNodeVisited.incrementAndGet();
         }
 
         void visitInner() {
-            innerNodeVisited++;
+            innerNodeVisited.incrementAndGet();
         }
     }
 
     void consumeDebugInfo(DebugInfo debug) {
-        leafNodeVisited += debug.leafNodeVisited;
-        innerNodeVisited += debug.innerNodeVisited;
+        leafNodeVisited.addAndGet(debug.leafNodeVisited.get());
+        innerNodeVisited.addAndGet(debug.innerNodeVisited.get());
     }
 
     public void populateDebugInfo(BiConsumer<String, Object> add) {
-        if (optimizedSegments > 0) {
-            add.accept("optimized_segments", optimizedSegments);
-            add.accept("unoptimized_segments", segments - optimizedSegments);
-            add.accept("leaf_visited", leafNodeVisited);
-            add.accept("inner_visited", innerNodeVisited);
+        if (optimizedSegments.get() > 0) {
+            add.accept("optimized_segments", optimizedSegments.get());
+            add.accept("unoptimized_segments", segments.get() - optimizedSegments.get());
+            add.accept("leaf_visited", leafNodeVisited.get());
+            add.accept("inner_visited", innerNodeVisited.get());
         }
     }
 }
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/filterrewrite/RangeAggregatorBridge.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/filterrewrite/RangeAggregatorBridge.java
index 8496f7969d882..b590a444c8b04 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/filterrewrite/RangeAggregatorBridge.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/filterrewrite/RangeAggregatorBridge.java
@@ -69,7 +69,7 @@ protected void buildRanges(RangeAggregator.Range[] ranges) {
     }
 
     @Override
-    final Ranges prepareFromSegment(LeafReaderContext leaf) {
+    final Ranges tryBuildRangesFromSegment(LeafReaderContext leaf) {
         throw new UnsupportedOperationException("Range aggregation should not build ranges at segment level");
     }