feat(prerenderer): support for query links exploration (#1474)

Co-authored-by: Pooya Parsa <[email protected]>
nitrojs · Aug 7, 2023 · cd56e14 · cd56e14
1 parent 6669fb2
commit cd56e14
Show file tree

Hide file tree

Showing 5 changed files with 65 additions and 49 deletions.
diff --git a/src/prerender.ts b/src/prerender.ts
@@ -89,12 +89,34 @@ export async function prerender(nitro: Nitro) {
   const failedRoutes = new Set<PrerenderRoute>();
   const skippedRoutes = new Set();
   const displayedLengthWarns = new Set();
+
   const canPrerender = (route = "/") => {
     // Skip if route is already generated or skipped
     if (generatedRoutes.has(route) || skippedRoutes.has(route)) {
       return false;
     }
 
+    // Check for explicitly ignored routes
+    for (const ignore of nitro.options.prerender.ignore) {
+      if (route.startsWith(ignore)) {
+        return false;
+      }
+    }
+
+    // Check for route rules explicitly disabling prerender
+    if (_getRouteRules(route).prerender === false) {
+      return false;
+    }
+
+    return true;
+  };
+
+  const canWriteToDisk = (route: PrerenderRoute) => {
+    // Cannot write routes with query
+    if (route.route.includes("?")) {
+      return false;
+    }
+
     // Ensure length is not too long for filesystem
     // https://en.wikipedia.org/wiki/Comparison_of_file_systems#Limits
     const FS_MAX_SEGMENT = 255;
@@ -104,15 +126,15 @@ export async function prerender(nitro: Nitro) {
       FS_MAX_PATH - (nitro.options.output.publicDir.length + 10);
 
     if (
-      (route.length >= FS_MAX_PATH_PUBLIC_HTML ||
-        route.split("/").some((s) => s.length > FS_MAX_SEGMENT)) &&
+      (route.route.length >= FS_MAX_PATH_PUBLIC_HTML ||
+        route.route.split("/").some((s) => s.length > FS_MAX_SEGMENT)) &&
       !displayedLengthWarns.has(route)
     ) {
       displayedLengthWarns.add(route);
-      const _route = route.slice(0, 60) + "...";
-      if (route.length >= FS_MAX_PATH_PUBLIC_HTML) {
+      const _route = route.route.slice(0, 60) + "...";
+      if (route.route.length >= FS_MAX_PATH_PUBLIC_HTML) {
         nitro.logger.warn(
-          `Prerendering long route "${_route}" (${route.length}) can cause filesystem issues since it exceeds ${FS_MAX_PATH_PUBLIC_HTML}-character limit when writing to \`${nitro.options.output.publicDir}\`.`
+          `Prerendering long route "${_route}" (${route.route.length}) can cause filesystem issues since it exceeds ${FS_MAX_PATH_PUBLIC_HTML}-character limit when writing to \`${nitro.options.output.publicDir}\`.`
         );
       } else {
         nitro.logger.warn(
@@ -122,18 +144,6 @@ export async function prerender(nitro: Nitro) {
       }
     }
 
-    // Check for explicitly ignored routes
-    for (const ignore of nitro.options.prerender.ignore) {
-      if (route.startsWith(ignore)) {
-        return false;
-      }
-    }
-
-    // Check for route rules explicitly disabling prerender
-    if (_getRouteRules(route).prerender === false) {
-      return false;
-    }
-
     return true;
   };
 
@@ -148,7 +158,7 @@ export async function prerender(nitro: Nitro) {
     generatedRoutes.add(route);
 
     // Create result object
-    const _route: PrerenderRoute & { skip?: boolean } = { route };
+    const _route: PrerenderRoute = { route };
 
     // Fetch the route
     const encodedRoute = encodeURI(route);
@@ -194,33 +204,38 @@ export async function prerender(nitro: Nitro) {
       failedRoutes.add(_route);
     }
 
-    // Write to the file
+    // Measure actual time taken for generating route
+    _route.generateTimeMS = Date.now() - start;
+
+    // Guess route type and populate fileName
     const isImplicitHTML =
       !route.endsWith(".html") &&
       (res.headers.get("content-type") || "").includes("html");
     const routeWithIndex = route.endsWith("/") ? route + "index" : route;
-    _route.fileName = isImplicitHTML
-      ? joinURL(route, "index.html")
-      : routeWithIndex;
-    _route.fileName = withoutBase(_route.fileName, nitro.options.baseURL);
+    _route.fileName = withoutBase(
+      isImplicitHTML ? joinURL(route, "index.html") : routeWithIndex,
+      nitro.options.baseURL
+    );
 
+    // Allow hooking before generate
     await nitro.hooks.callHook("prerender:generate", _route, nitro);
 
-    // Measure actual time taken for generating route
-    _route.generateTimeMS = Date.now() - start;
-
-    // Check if route skipped or has errors
+    // Check if route is skipped or has errors
     if (_route.skip || _route.error) {
       await nitro.hooks.callHook("prerender:route", _route);
       nitro.logger.log(formatPrerenderRoute(_route));
+      dataBuff = undefined; // Free memory
       return _route;
     }
 
-    const filePath = join(nitro.options.output.publicDir, _route.fileName);
-
-    await writeFile(filePath, dataBuff);
-
-    nitro._prerenderedRoutes.push(_route);
+    // Write to the disk
+    if (canWriteToDisk(_route)) {
+      const filePath = join(nitro.options.output.publicDir, _route.fileName);
+      await writeFile(filePath, dataBuff);
+      nitro._prerenderedRoutes.push(_route);
+    } else {
+      _route.skip = true;
+    }
 
     // Crawl route links
     if (!_route.error && isImplicitHTML) {
@@ -240,9 +255,7 @@ export async function prerender(nitro: Nitro) {
     await nitro.hooks.callHook("prerender:route", _route);
     nitro.logger.log(formatPrerenderRoute(_route));
 
-    // Free memory
-    dataBuff = undefined;
-
+    dataBuff = undefined; // Free memory
     return _route;
   };
 
@@ -349,16 +362,15 @@ function extractLinks(
   );
 
   for (const link of _links.filter(Boolean)) {
-    const parsed = parseURL(link);
-    if (parsed.protocol) {
+    const _link = parseURL(link);
+    if (_link.protocol) {
       continue;
     }
-    let { pathname } = parsed;
-    if (!pathname.startsWith("/")) {
+    if (!_link.pathname.startsWith("/")) {
       const fromURL = new URL(from, "http://localhost");
-      pathname = new URL(pathname, fromURL).pathname;
+      _link.pathname = new URL(_link.pathname, fromURL).pathname;
     }
-    links.push(pathname);
+    links.push(_link.pathname + _link.search);
   }
   for (const link of links) {
     const _parents = linkParents.get(link);
@@ -394,5 +406,9 @@ function formatPrerenderRoute(route: PrerenderRoute) {
     }
   }
 
+  if (route.skip) {
+    str += chalk.gray(" (skipped)");
+  }
+
   return chalk.gray(str);
 }
diff --git a/src/types/nitro.ts b/src/types/nitro.ts
@@ -69,12 +69,11 @@ export interface PrerenderRoute {
   fileName?: string;
   error?: Error & { statusCode: number; statusMessage: string };
   generateTimeMS?: number;
+  skip?: boolean;
 }
 
 /** @deprecated Internal type will be removed in future versions */
-export interface PrerenderGenerateRoute extends PrerenderRoute {
-  skip?: boolean;
-}
+export type PrerenderGenerateRoute = PrerenderRoute;
 
 type HookResult = void | Promise<void>;
 export interface NitroHooks {
@@ -88,10 +87,7 @@ export interface NitroHooks {
   "prerender:routes": (routes: Set<string>) => HookResult;
   "prerender:config": (config: NitroConfig) => HookResult;
   "prerender:init": (prerenderer: Nitro) => HookResult;
-  "prerender:generate": (
-    route: PrerenderRoute & { skip?: boolean },
-    nitro: Nitro
-  ) => HookResult;
+  "prerender:generate": (route: PrerenderRoute, nitro: Nitro) => HookResult;
   "prerender:route": (route: PrerenderRoute) => HookResult;
   "prerender:done": (result: {
     prerenderedRoutes: PrerenderRoute[];

diff --git a/test/fixture/routes/prerender.ts b/test/fixture/routes/prerender.ts
@@ -10,6 +10,7 @@ export default defineEventHandler((event) => {
     "../api/hey",
     "/api/param/foo.json",
     "/api/param/foo.css",
+    event.path.includes("?") ? "/api/param/hidden" : "/prerender?withQuery",
   ];
 
   appendHeader(
@@ -34,7 +35,6 @@ ${links.map((link) => `    <li><a href="${link}">${link}</a></li>`).join("\n")}
 
   /* Bad Link Examples */
   <a x-href="/500?x-href">x-href attr</a>
-
   &lt;a href=&quot;/500&lt;/a&gt;
 </body>
 </html>`;

diff --git a/test/presets/cloudflare-pages.test.ts b/test/presets/cloudflare-pages.test.ts
@@ -51,6 +51,7 @@ describe("nitro:preset:cloudflare-pages", async () => {
           "/prerender/index.html.gz",
           "/api/hey/index.html",
           "/api/param/foo.json/index.html",
+          "/api/param/hidden/index.html",
           "/api/param/prerender1/index.html",
           "/api/param/prerender3/index.html",
           "/api/param/prerender4/index.html",

diff --git a/test/presets/vercel.test.ts b/test/presets/vercel.test.ts
@@ -31,6 +31,9 @@ describe("nitro:preset:vercel", async () => {
               "api/param/foo.json/index.html": {
                 "path": "api/param/foo.json",
               },
+              "api/param/hidden/index.html": {
+                "path": "api/param/hidden",
+              },
               "api/param/prerender1/index.html": {
                 "path": "api/param/prerender1",
               },