diff --git a/packages/vite/src/node/optimizer/optimizer.ts b/packages/vite/src/node/optimizer/optimizer.ts
index a7131db778cf1c..f82dbe0f3bcf85 100644
--- a/packages/vite/src/node/optimizer/optimizer.ts
+++ b/packages/vite/src/node/optimizer/optimizer.ts
@@ -711,7 +711,7 @@ async function createDepsOptimizer(
   }
 }
 
-const runOptimizerIfIdleAfterMs = 50
+const callCrawlEndIfIdleAfterMs = 50
 
 interface CrawlEndFinder {
   ensureFirstRun: () => void
@@ -721,18 +721,17 @@ interface CrawlEndFinder {
 }
 
 function setupOnCrawlEnd(onCrawlEnd: () => void): CrawlEndFinder {
-  let registeredIds: { id: string; done: () => Promise<any> }[] = []
+  const registeredIds = new Set<string>()
   const seenIds = new Set<string>()
   const workersSources = new Set<string>()
-  const waitingOn = new Map<string, () => void>()
-  let firstRunEnsured = false
-  let crawlEndCalled = false
+  let timeoutHandle: NodeJS.Timeout | undefined
 
   let cancelled = false
   function cancel() {
     cancelled = true
   }
 
+  let crawlEndCalled = false
   function callOnCrawlEnd() {
     if (!cancelled && !crawlEndCalled) {
       crawlEndCalled = true
@@ -743,84 +742,56 @@ function setupOnCrawlEnd(onCrawlEnd: () => void): CrawlEndFinder {
   // If all the inputs are dependencies, we aren't going to get any
   // delayDepsOptimizerUntil(id) calls. We need to guard against this
   // by forcing a rerun if no deps have been registered
+  let firstRunEnsured = false
   function ensureFirstRun() {
     if (!firstRunEnsured && seenIds.size === 0) {
       setTimeout(() => {
         if (seenIds.size === 0) {
           callOnCrawlEnd()
         }
-      }, runOptimizerIfIdleAfterMs)
+      }, 200)
     }
     firstRunEnsured = true
   }
 
   function registerWorkersSource(id: string): void {
     workersSources.add(id)
+
     // Avoid waiting for this id, as it may be blocked by the rollup
     // bundling process of the worker that also depends on the optimizer
-    registeredIds = registeredIds.filter((registered) => registered.id !== id)
+    registeredIds.delete(id)
 
-    const resolve = waitingOn.get(id)
-    // Forced resolve to avoid waiting for the bundling of the worker to finish
-    resolve?.()
+    checkIfCrawlEndAfterTimeout()
   }
 
   function delayDepsOptimizerUntil(id: string, done: () => Promise<any>): void {
     if (!seenIds.has(id)) {
       seenIds.add(id)
-      registeredIds.push({ id, done })
-      callOnCrawlEndWhenIdle()
+      if (!workersSources.has(id)) {
+        registeredIds.add(id)
+        done()
+          .catch(() => {})
+          .finally(() => markIdAsDone(id))
+      }
     }
   }
+  function markIdAsDone(id: string): void {
+    registeredIds.delete(id)
+    checkIfCrawlEndAfterTimeout()
+  }
 
-  async function callOnCrawlEndWhenIdle() {
-    if (cancelled || waitingOn.size > 0) return
-
-    const processingRegisteredIds = registeredIds
-    registeredIds = []
-
-    const donePromises = processingRegisteredIds.map(async (registeredId) => {
-      // During build, we need to cancel workers
-      let resolve: () => void
-      const waitUntilDone = new Promise<void>((_resolve) => {
-        resolve = _resolve
-        registeredId
-          .done()
-          .catch(() => {
-            // Ignore errors
-          })
-          .finally(() => resolve())
-      })
-      waitingOn.set(registeredId.id, () => resolve())
-
-      await waitUntilDone
-      waitingOn.delete(registeredId.id)
-    })
-
-    const afterLoad = () => {
-      if (cancelled) return
-      if (
-        registeredIds.length > 0 &&
-        registeredIds.every((registeredId) =>
-          workersSources.has(registeredId.id),
-        )
-      ) {
-        return
-      }
-
-      if (registeredIds.length > 0) {
-        callOnCrawlEndWhenIdle()
-      } else {
-        callOnCrawlEnd()
-      }
-    }
+  function checkIfCrawlEndAfterTimeout() {
+    if (cancelled || registeredIds.size > 0) return
 
-    await Promise.allSettled(donePromises)
-    if (registeredIds.length > 0) {
-      afterLoad()
-    } else {
-      setTimeout(afterLoad, runOptimizerIfIdleAfterMs)
-    }
+    if (timeoutHandle) clearTimeout(timeoutHandle)
+    timeoutHandle = setTimeout(
+      callOnCrawlEndWhenIdle,
+      callCrawlEndIfIdleAfterMs,
+    )
+  }
+  async function callOnCrawlEndWhenIdle() {
+    if (cancelled || registeredIds.size > 0) return
+    callOnCrawlEnd()
   }
 
   return {
diff --git a/packages/vite/src/node/plugins/optimizedDeps.ts b/packages/vite/src/node/plugins/optimizedDeps.ts
index a91c961fddba6c..f832e5da395c6c 100644
--- a/packages/vite/src/node/plugins/optimizedDeps.ts
+++ b/packages/vite/src/node/plugins/optimizedDeps.ts
@@ -110,6 +110,7 @@ export function optimizedDepsBuildPlugin(config: ResolvedConfig): Plugin {
           depsOptimizer.delayDepsOptimizerUntil(resolved.id, async () => {
             await this.load(resolved)
           })
+          return resolved
         }
       }
     },
diff --git a/playground/optimize-deps/__tests__/optimize-deps.spec.ts b/playground/optimize-deps/__tests__/optimize-deps.spec.ts
index f023d1c22ded8d..73e7593a31e288 100644
--- a/playground/optimize-deps/__tests__/optimize-deps.spec.ts
+++ b/playground/optimize-deps/__tests__/optimize-deps.spec.ts
@@ -6,6 +6,7 @@ import {
   isBuild,
   isServe,
   page,
+  serverLogs,
   viteTestUrl,
 } from '~utils'
 
@@ -214,3 +215,10 @@ test('pre bundle css require', async () => {
 
   expect(await getColor('.css-require')).toBe('red')
 })
+
+test.runIf(isBuild)('no missing deps during build', async () => {
+  serverLogs.forEach((log) => {
+    // no warning from esbuild css minifier
+    expect(log).not.toMatch('Missing dependency found after crawling ended')
+  })
+})