diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/text/LuceneTextIndexCreator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/text/LuceneTextIndexCreator.java
index 2cdbf13f6af4..c24778ab3728 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/text/LuceneTextIndexCreator.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/text/LuceneTextIndexCreator.java
@@ -36,6 +36,7 @@
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.NoMergeScheduler;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FSDirectory;
@@ -131,6 +132,15 @@ public LuceneTextIndexCreator(String column, File segmentIndexDir, boolean commi
       indexWriterConfig.setCommitOnClose(commit);
       indexWriterConfig.setUseCompoundFile(config.isLuceneUseCompoundFile());
 
+      // For the realtime segment, prevent background merging. The realtime segment will call .commit()
+      // on the IndexWriter when segment conversion occurs. By default, Lucene will sometimes choose to
+      // merge segments in the background, which is problematic because the lucene index directory's
+      // contents is copied to create the immutable segment. If a background merge occurs during this
+      // copy, a FileNotFoundException will be triggered and segment build will fail.
+      if (!_commitOnClose) {
+        indexWriterConfig.setMergeScheduler(NoMergeScheduler.INSTANCE);
+      }
+
       if (_reuseMutableIndex) {
         LOGGER.info("Reusing the realtime lucene index for segment {} and column {}", segmentIndexDir, column);
         indexWriterConfig.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);