diff --git a/src/node/server/serverPluginServeStatic.ts b/src/node/server/serverPluginServeStatic.ts
index 7c9a875cd327c2..f91650c3a8c8f0 100644
--- a/src/node/server/serverPluginServeStatic.ts
+++ b/src/node/server/serverPluginServeStatic.ts
@@ -3,6 +3,8 @@ import { ServerPlugin } from '.'
 const send = require('koa-send')
 const debug = require('debug')('vite:history')
 
+export const seenUrls = new Set()
+
 export const serveStaticPlugin: ServerPlugin = ({
   root,
   app,
@@ -56,7 +58,14 @@ export const serveStaticPlugin: ServerPlugin = ({
   })
 
   if (!config.serviceWorker) {
-    app.use(require('koa-conditional-get')())
+    app.use(async (ctx, next) => {
+      await next()
+      // the first request to the server should never 304
+      if (seenUrls.has(ctx.url) && ctx.fresh) {
+        ctx.status = 304
+      }
+      seenUrls.add(ctx.url)
+    })
   }
   app.use(require('koa-etag')())
 
diff --git a/src/node/server/serverPluginVue.ts b/src/node/server/serverPluginVue.ts
index 0d74fc09c147ed..69556ab765eacf 100644
--- a/src/node/server/serverPluginVue.ts
+++ b/src/node/server/serverPluginVue.ts
@@ -30,6 +30,7 @@ import { Context } from 'koa'
 import { transform } from '../esbuildService'
 import { InternalResolver } from '../resolver'
 import qs from 'querystring'
+import { seenUrls } from './serverPluginServeStatic'
 
 const debug = require('debug')('vite:sfc')
 const getEtag = require('etag')
@@ -58,7 +59,11 @@ export const vuePlugin: ServerPlugin = ({
     ctx.etag = getEtag(ctx.body)
     // only add 304 tag check if not using service worker to cache user code
     if (!config.serviceWorker) {
-      ctx.status = ctx.etag === ctx.get('If-None-Match') ? 304 : 200
+      ctx.status =
+        seenUrls.has(ctx.url) && ctx.etag === ctx.get('If-None-Match')
+          ? 304
+          : 200
+      seenUrls.add(ctx.url)
     }
   }
 
diff --git a/src/node/utils/fsUtils.ts b/src/node/utils/fsUtils.ts
index 95e8f754fda5bd..432fa1c701236a 100644
--- a/src/node/utils/fsUtils.ts
+++ b/src/node/utils/fsUtils.ts
@@ -3,6 +3,7 @@ import fs from 'fs-extra'
 import LRUCache from 'lru-cache'
 import { Context } from 'koa'
 import { Readable } from 'stream'
+import { seenUrls } from '../server/serverPluginServeStatic'
 
 const getETag = require('etag')
 
@@ -36,10 +37,12 @@ export async function cachedRead(
       ctx.lastModified = new Date(cached.lastModified)
       if (
         ctx.__serviceWorker !== true &&
-        ctx.get('If-None-Match') === ctx.etag
+        ctx.get('If-None-Match') === ctx.etag &&
+        seenUrls.has(ctx.url)
       ) {
         ctx.status = 304
       }
+      seenUrls.add(ctx.url)
       ctx.body = cached.content
     }
     return cached.content