feat(streaming-ssr): Various bug fixes and migrate to FetchAPI/Web streams

packages/project-config/src/__tests__/paths.test.ts

@@ -108,6 +108,7 @@ describe('paths', () => {
           layouts: path.join(FIXTURE_BASEDIR, 'web', 'src', 'layouts/'),
           src: path.join(FIXTURE_BASEDIR, 'web', 'src'),
           generators: path.join(FIXTURE_BASEDIR, 'web', 'generators'),
+          document: null, // this fixture doesnt have a document
           app: path.join(FIXTURE_BASEDIR, 'web', 'src', 'App.tsx'),
           index: null,
           html: path.join(FIXTURE_BASEDIR, 'web', 'src', 'index.html'),
@@ -158,6 +159,13 @@ describe('paths', () => {
             'routeHooks'
           ),
           distServer: path.join(FIXTURE_BASEDIR, 'web', 'dist', 'server'),
+          distDocumentServer: path.join(
+            FIXTURE_BASEDIR,
+            'web',
+            'dist',
+            'server',
+            'Document.js'
+          ),
           distServerEntries: path.join(
             FIXTURE_BASEDIR,
             'web',
@@ -380,6 +388,7 @@ describe('paths', () => {
           src: path.join(FIXTURE_BASEDIR, 'web', 'src'),
           generators: path.join(FIXTURE_BASEDIR, 'web', 'generators'),
           app: path.join(FIXTURE_BASEDIR, 'web', 'src', 'App.js'),
+          document: null, // this fixture doesnt have a document
           index: null,
           html: path.join(FIXTURE_BASEDIR, 'web', 'src', 'index.html'),
           config: path.join(FIXTURE_BASEDIR, 'web', 'config'),
@@ -422,6 +431,13 @@ describe('paths', () => {
             'server',
             'entry.server.js'
           ),
+          distDocumentServer: path.join(
+            FIXTURE_BASEDIR,
+            'web',
+            'dist',
+            'server',
+            'Document.js'
+          ),
           distRouteHooks: path.join(
             FIXTURE_BASEDIR,
             'web',
@@ -696,6 +712,7 @@ describe('paths', () => {
           components: path.join(FIXTURE_BASEDIR, 'web', 'src', 'components'),
           layouts: path.join(FIXTURE_BASEDIR, 'web', 'src', 'layouts/'),
           src: path.join(FIXTURE_BASEDIR, 'web', 'src'),
+          document: null, // this fixture doesnt have a document
           generators: path.join(FIXTURE_BASEDIR, 'web', 'generators'),
           app: null,
           index: path.join(FIXTURE_BASEDIR, 'web', 'src', 'index.js'),
@@ -743,6 +760,13 @@ describe('paths', () => {
             'server',
             'entry.server.js'
           ),
+          distDocumentServer: path.join(
+            FIXTURE_BASEDIR,
+            'web',
+            'dist',
+            'server',
+            'Document.js'
+          ), // this is constructed regardless of presence of src/Document
           distRouteHooks: path.join(
             FIXTURE_BASEDIR,
             'web',
@@ -971,6 +995,7 @@ describe('paths', () => {
           pages: path.join(FIXTURE_BASEDIR, 'web', 'src', 'pages/'),
           components: path.join(FIXTURE_BASEDIR, 'web', 'src', 'components'),
           layouts: path.join(FIXTURE_BASEDIR, 'web', 'src', 'layouts/'),
+          document: null, // this fixture doesnt have a document
           src: path.join(FIXTURE_BASEDIR, 'web', 'src'),
           generators: path.join(FIXTURE_BASEDIR, 'web', 'generators'),
           app: path.join(FIXTURE_BASEDIR, 'web', 'src', 'App.tsx'),
@@ -1015,6 +1040,13 @@ describe('paths', () => {
             'server',
             'entry.server.js'
           ),
+          distDocumentServer: path.join(
+            FIXTURE_BASEDIR,
+            'web',
+            'dist',
+            'server',
+            'Document.js'
+          ),
           distRouteHooks: path.join(
             FIXTURE_BASEDIR,
             'web',

packages/project-config/src/paths.ts

@@ -29,6 +29,7 @@ export interface WebPaths {
   base: string
   src: string
   app: string
+  document: string
   generators: string
   index: string | null
   html: string
@@ -49,6 +50,7 @@ export interface WebPaths {
   dist: string
   distServer: string
   distEntryServer: string
+  distDocumentServer: string
   distRouteHooks: string
   distServerEntries: string
   routeManifest: string
@@ -103,6 +105,7 @@ const PATH_WEB_DIR_PAGES = 'web/src/pages/'
 const PATH_WEB_DIR_COMPONENTS = 'web/src/components'
 const PATH_WEB_DIR_SRC = 'web/src'
 const PATH_WEB_DIR_SRC_APP = 'web/src/App'
+const PATH_WEB_DIR_SRC_DOCUMENT = 'web/src/Document'
 const PATH_WEB_DIR_SRC_INDEX = 'web/src/index' // .jsx|.tsx
 const PATH_WEB_INDEX_HTML = 'web/src/index.html'
 const PATH_WEB_DIR_GENERATORS = 'web/generators'
@@ -122,6 +125,8 @@ const PATH_WEB_DIR_CONFIG_STORYBOOK_MANAGER = 'web/config/storybook.manager.js'
 const PATH_WEB_DIR_DIST = 'web/dist'
 const PATH_WEB_DIR_DIST_SERVER = 'web/dist/server'
 const PATH_WEB_DIR_DIST_SERVER_ENTRY_SERVER = 'web/dist/server/entry.server.js'
+const PATH_WEB_DIR_DIST_DOCUMENT = 'web/dist/server/Document.js'
+
 const PATH_WEB_DIR_DIST_SERVER_ROUTEHOOKS = 'web/dist/server/routeHooks'
 const PATH_WEB_DIR_DIST_SERVER_ENTRIES = 'web/dist/server/entries.js'
 const PATH_WEB_DIR_ROUTE_MANIFEST = 'web/dist/server/route-manifest.json'
@@ -211,6 +216,9 @@ export const getPaths = (BASE_DIR: string = getBaseDir()): Paths => {
       src: path.join(BASE_DIR, PATH_WEB_DIR_SRC),
       generators: path.join(BASE_DIR, PATH_WEB_DIR_GENERATORS),
       app: resolveFile(path.join(BASE_DIR, PATH_WEB_DIR_SRC_APP)) as string,
+      document: resolveFile(
+        path.join(BASE_DIR, PATH_WEB_DIR_SRC_DOCUMENT)
+      ) as string,
       index: resolveFile(path.join(BASE_DIR, PATH_WEB_DIR_SRC_INDEX)), // old webpack entry point
       html: path.join(BASE_DIR, PATH_WEB_INDEX_HTML),
       config: path.join(BASE_DIR, PATH_WEB_DIR_CONFIG),
@@ -235,6 +243,7 @@ export const getPaths = (BASE_DIR: string = getBaseDir()): Paths => {
         BASE_DIR,
         PATH_WEB_DIR_DIST_SERVER_ENTRY_SERVER
       ),
+      distDocumentServer: path.join(BASE_DIR, PATH_WEB_DIR_DIST_DOCUMENT),
       distRouteHooks: path.join(BASE_DIR, PATH_WEB_DIR_DIST_SERVER_ROUTEHOOKS),
       distServerEntries: path.join(BASE_DIR, PATH_WEB_DIR_DIST_SERVER_ENTRIES),
       routeManifest: path.join(BASE_DIR, PATH_WEB_DIR_ROUTE_MANIFEST),

packages/vite/package.json

@@ -69,6 +69,7 @@
     "@redwoodjs/web": "6.0.7",
     "@swc/core": "1.3.60",
     "@vitejs/plugin-react": "4.0.4",
+    "@whatwg-node/server": "0.9.14",
     "acorn-loose": "8.3.0",
     "buffer": "6.0.3",
     "busboy": "^1.6.0",

packages/vite/src/devFeServer.ts

@@ -1,5 +1,4 @@
-// TODO (STREAMING) Merge with runFeServer so we only have one file
-
+import { createServerAdapter } from '@whatwg-node/server'
 import express from 'express'
 import { createServer as createViteServer } from 'vite'
 
@@ -77,11 +76,12 @@ async function createServer() {
       continue
     }
 
+    // @TODO we no longer need to use the regex
     const expressPathDef = route.hasParams
       ? route.matchRegexString
       : route.pathDefinition
 
-    app.get(expressPathDef, routeHandler)
+    app.get(expressPathDef, createServerAdapter(routeHandler))
   }
 
   const port = getConfig().web.port

packages/vite/src/runFeServer.ts

@@ -6,6 +6,7 @@
 import fs from 'fs/promises'
 import path from 'path'
 
+import { createServerAdapter } from '@whatwg-node/server'
 // @ts-expect-error We will remove dotenv-defaults from this package anyway
 import { config as loadDotEnv } from 'dotenv-defaults'
 import express from 'express'
@@ -112,11 +113,14 @@ export async function runFeServer() {
       continue
     }
 
+    // @TODO: we don't need regexes here
+    // Param matching, etc. all handled within the route handler now
     const expressPathDef = route.hasParams
       ? route.matchRegexString
       : route.pathDefinition
 
-    app.get(expressPathDef, routeHandler)
+    // Wrap with whatg/server adapter. Express handler -> Fetch API handler
+    app.get(expressPathDef, createServerAdapter(routeHandler))
   }
 
   const server = app.listen(

packages/vite/src/streaming/createReactStreamingHandler.ts

@@ -1,16 +1,16 @@
 import path from 'path'
 
-import type { Request, Response } from 'express'
 import isbot from 'isbot'
 import type { ViteDevServer } from 'vite'
 
 import type { RWRouteManifestItem } from '@redwoodjs/internal'
 import { getAppRouteHook, getPaths } from '@redwoodjs/project-config'
+import { matchPath } from '@redwoodjs/router'
 import type { TagDescriptor } from '@redwoodjs/web'
 
 // import { stripQueryStringAndHashFromPath } from '../utils'
 
-import { reactRenderToStream } from './streamHelpers'
+import { reactRenderToStreamResponse } from './streamHelpers'
 import { loadAndRunRouteHooks } from './triggerRouteHooks'
 
 interface CreateReactStreamingHandlerOptions {
@@ -32,14 +32,22 @@ export const createReactStreamingHandler = async (
   const isProd = !viteDevServer
 
   let entryServerImport: any
+  let fallbackDocumentImport: any
 
   if (isProd) {
     entryServerImport = await import(rwPaths.web.distEntryServer)
+    fallbackDocumentImport = await import(rwPaths.web.distDocumentServer)
   }
 
-  return async (req: Request, res: Response) => {
+  // @NOTE: we are returning a FetchAPI handler
+  return async (req: Request) => {
     if (redirect) {
-      res.redirect(redirect.to)
+      return new Response(null, {
+        status: 302,
+        headers: {
+          Location: redirect.to,
+        },
+      })
     }
 
     // Do this inside the handler for **dev-only**.
@@ -48,12 +56,21 @@ export const createReactStreamingHandler = async (
       entryServerImport = await viteDevServer.ssrLoadModule(
         rwPaths.web.entryServer as string // already validated in dev server
       )
+      fallbackDocumentImport = await viteDevServer.ssrLoadModule(
+        rwPaths.web.document
+      )
     }
 
     const ServerEntry =
       entryServerImport.ServerEntry || entryServerImport.default
 
-    const currentPathName = req.path
+    const FallbackDocument =
+      fallbackDocumentImport.Document || fallbackDocumentImport.default
+
+    const { pathname: currentPathName } = new URL(req.url)
+
+    // @TODO validate this is correct
+    const parsedParams = matchPath(route.pathDefinition, currentPathName)
 
     let metaTags: TagDescriptor[] = []
 
@@ -70,7 +87,7 @@ export const createReactStreamingHandler = async (
       paths: [getAppRouteHook(isProd), routeHookPath],
       reqMeta: {
         req,
-        parsedParams: req.params,
+        parsedParams,
       },
       viteDevServer,
     })
@@ -82,21 +99,25 @@ export const createReactStreamingHandler = async (
       bundle && '/' + bundle,
     ].filter(Boolean) as string[]
 
-    const isSeoCrawler = checkUaForSeoCrawler(req.headers['user-agent'] || '')
+    const isSeoCrawler = checkUaForSeoCrawler(
+      req.headers.get('user-agent') || ''
+    )
 
-    reactRenderToStream(
+    const reactResponse = await reactRenderToStreamResponse(
       {
         ServerEntry,
+        FallbackDocument,
         currentPathName,
         metaTags,
         cssLinks,
         isProd,
         jsBundles,
-        res,
       },
       {
         waitForAllReady: isSeoCrawler,
       }
     )
+
+    return reactResponse
   }
 }