[metadata] add option of configuring ua of async metadata

add config schema

update test

use a different config

serve differently based on the config

rename

packages/next/src/build/index.ts

@@ -80,6 +80,7 @@ import {
   UNDERSCORE_NOT_FOUND_ROUTE_ENTRY,
   UNDERSCORE_NOT_FOUND_ROUTE,
   DYNAMIC_CSS_MANIFEST,
+  RESPONSE_CONFIG_MANIFEST,
 } from '../shared/lib/constants'
 import {
   getSortedRoutes,
@@ -214,6 +215,7 @@ import {
   getParsedNodeOptionsWithoutInspect,
 } from '../server/lib/utils'
 import { InvariantError } from '../shared/lib/invariant-error'
+import { HTML_LIMITED_BOT_UA_RE } from '../shared/lib/router/utils/is-bot'
 
 type Fallback = null | boolean | string
 
@@ -1304,6 +1306,22 @@ export default async function build(
         NextBuildContext.clientRouterFilters = clientRouterFilters
       }
 
+      // Write html limited bots config to response-config-manifest
+      const responseConfigManifestPath = path.join(
+        distDir,
+        RESPONSE_CONFIG_MANIFEST
+      )
+      const responseConfigManifest: {
+        version: number
+        htmlLimitedBots: string
+      } = {
+        version: 0,
+        htmlLimitedBots: (
+          config.experimental.htmlLimitedBots || HTML_LIMITED_BOT_UA_RE
+        ).source,
+      }
+      await writeManifest(responseConfigManifestPath, responseConfigManifest)
+
       // Ensure commonjs handling is used for files in the distDir (generally .next)
       // Files outside of the distDir can be "type": "module"
       await writeFileUtf8(
@@ -2482,6 +2500,7 @@ export default async function build(
               path.relative(distDir, pagesManifestPath),
               BUILD_MANIFEST,
               PRERENDER_MANIFEST,
+              RESPONSE_CONFIG_MANIFEST,
               path.join(SERVER_DIRECTORY, MIDDLEWARE_MANIFEST),
               path.join(SERVER_DIRECTORY, MIDDLEWARE_BUILD_MANIFEST + '.js'),
               ...(!process.env.TURBOPACK

packages/next/src/export/index.ts

@@ -360,6 +360,7 @@ async function exportAppImpl(
       inlineCss: nextConfig.experimental.inlineCss ?? false,
       authInterrupts: !!nextConfig.experimental.authInterrupts,
       streamingMetadata: !!nextConfig.experimental.streamingMetadata,
+      htmlLimitedBots: nextConfig.experimental.htmlLimitedBots,
     },
     reactMaxHeadersLength: nextConfig.reactMaxHeadersLength,
   }

packages/next/src/server/app-render/types.ts

@@ -216,6 +216,7 @@ export interface RenderOptsPartial {
     inlineCss: boolean
     authInterrupts: boolean
     streamingMetadata: boolean
+    htmlLimitedBots: RegExp | undefined
   }
   postponed?: string
 

packages/next/src/server/base-server.ts

@@ -83,7 +83,10 @@ import {
 } from './lib/revalidate'
 import { execOnce } from '../shared/lib/utils'
 import { isBlockedPage } from './utils'
-import { isBot, isHtmlLimitedBotUA } from '../shared/lib/router/utils/is-bot'
+import {
+  HTML_LIMITED_BOT_UA_RE,
+  isBot,
+} from '../shared/lib/router/utils/is-bot'
 import RenderResult from './render-result'
 import { removeTrailingSlash } from '../shared/lib/router/utils/remove-trailing-slash'
 import { denormalizePagePath } from '../shared/lib/page-path/denormalize-page-path'
@@ -329,6 +332,24 @@ export type NextEnabledDirectories = {
   readonly app: boolean
 }
 
+function shouldServeStreamingMetadata(
+  userAgent: string,
+  {
+    streamingMetadata,
+    htmlLimitedBots,
+  }: {
+    streamingMetadata: boolean
+    htmlLimitedBots: RegExp | undefined
+  }
+): boolean {
+  if (!streamingMetadata) {
+    return false
+  }
+
+  const blockingMetadataUA = htmlLimitedBots || HTML_LIMITED_BOT_UA_RE
+  return !blockingMetadataUA.test(userAgent)
+}
+
 export default abstract class Server<
   ServerOptions extends Options = Options,
   ServerRequest extends BaseNextRequest = BaseNextRequest,
@@ -595,6 +616,7 @@ export default abstract class Server<
         inlineCss: this.nextConfig.experimental.inlineCss ?? false,
         authInterrupts: !!this.nextConfig.experimental.authInterrupts,
         streamingMetadata: !!this.nextConfig.experimental.streamingMetadata,
+        htmlLimitedBots: this.nextConfig.experimental.htmlLimitedBots,
       },
       onInstrumentationRequestError:
         this.instrumentationOnRequestError.bind(this),
@@ -1675,11 +1697,13 @@ export default abstract class Server<
       renderOpts: {
         ...this.renderOpts,
         supportsDynamicResponse: !isBotRequest,
-        serveStreamingMetadata:
-          this.renderOpts.experimental.streamingMetadata &&
-          !isHtmlLimitedBotUA(ua),
+        serveStreamingMetadata: shouldServeStreamingMetadata(
+          ua,
+          this.renderOpts.experimental
+        ),
       },
     }
+
     const payload = await fn(ctx)
     if (payload === null) {
       return
@@ -2181,8 +2205,10 @@ export default abstract class Server<
       // cache if there are no dynamic data requirements
       opts.supportsDynamicResponse =
         !isSSG && !isBotRequest && !query.amp && isSupportedDocument
-      opts.serveStreamingMetadata =
-        opts.experimental.streamingMetadata && !isHtmlLimitedBotUA(ua)
+      opts.serveStreamingMetadata = shouldServeStreamingMetadata(
+        ua,
+        this.renderOpts.experimental
+      )
     }
 
     // In development, we always want to generate dynamic HTML.

packages/next/src/server/config-schema.ts

@@ -441,6 +441,7 @@ export const configSchema: zod.ZodType<NextConfig> = z.lazy(() =>
         authInterrupts: z.boolean().optional(),
         newDevOverlay: z.boolean().optional(),
         streamingMetadata: z.boolean().optional(),
+        htmlLimitedBots: z.instanceof(RegExp).optional(),
       })
       .optional(),
     exportPathMap: z

packages/next/src/server/config-shared.ts

@@ -14,6 +14,7 @@ import type { ExpireTime } from './lib/revalidate'
 import type { SupportedTestRunners } from '../cli/next-test'
 import type { ExperimentalPPRConfig } from './lib/experimental/ppr'
 import { INFINITE_CACHE } from '../lib/constants'
+import { HTML_LIMITED_BOT_UA_RE } from '../shared/lib/router/utils/is-bot'
 
 export type NextConfigComplete = Required<NextConfig> & {
   images: Required<ImageConfigComplete>
@@ -572,6 +573,12 @@ export interface ExperimentalConfig {
    * When enabled will cause async metadata calls to stream rather than block the render.
    */
   streamingMetadata?: boolean
+
+  /**
+   * User Agent of bots that can handle streaming metadata.
+   * Besides the default behavior, Next.js act differently on serving metadata to bots based on their capability.
+   */
+  htmlLimitedBots?: RegExp
 }
 
 export type ExportPathMap = {
@@ -1197,6 +1204,7 @@ export const defaultConfig: NextConfig = {
     inlineCss: false,
     newDevOverlay: false,
     streamingMetadata: false,
+    htmlLimitedBots: HTML_LIMITED_BOT_UA_RE,
   },
   bundlePagesRouterDependencies: false,
 }

packages/next/src/shared/lib/constants.ts

@@ -42,6 +42,7 @@ export const EXPORT_DETAIL = 'export-detail.json'
 export const PRERENDER_MANIFEST = 'prerender-manifest.json'
 export const ROUTES_MANIFEST = 'routes-manifest.json'
 export const IMAGES_MANIFEST = 'images-manifest.json'
+export const RESPONSE_CONFIG_MANIFEST = 'response-config-manifest.json'
 export const SERVER_FILES_MANIFEST = 'required-server-files.json'
 export const DEV_CLIENT_PAGES_MANIFEST = '_devPagesManifest.json'
 export const MIDDLEWARE_MANIFEST = 'middleware-manifest.json'

packages/next/src/shared/lib/router/utils/is-bot.ts

@@ -4,14 +4,14 @@ const HEADLESS_BROWSER_BOT_UA_RE =
 
 // This regex contains the bots that we need to do a blocking render for and can't safely stream the response
 // due to how they parse the DOM. For example, they might explicitly check for metadata in the `head` tag, so we can't stream metadata tags after the `head` was sent.
-const HTML_LIMITED_BOT_UA_RE =
+export const HTML_LIMITED_BOT_UA_RE =
   /Mediapartners-Google|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview/i
 
 function isHeadlessBrowserBotUA(userAgent: string) {
   return HEADLESS_BROWSER_BOT_UA_RE.test(userAgent)
 }
 
-export function isHtmlLimitedBotUA(userAgent: string) {
+function isHtmlLimitedBotUA(userAgent: string) {
   return HTML_LIMITED_BOT_UA_RE.test(userAgent)
 }
 

test/e2e/app-dir/metadata-streaming/metadata-streaming-customized-rule.test.ts

@@ -0,0 +1,43 @@
+import { nextTestSetup } from 'e2e-utils'
+
+describe('app-dir - metadata-streaming-customized-rule', () => {
+  const { next } = nextTestSetup({
+    files: __dirname,
+    overrideFiles: {
+      'next.config.js': `
+        module.exports = {
+          experimental: {
+            streamingMetadata: true,
+            htmlLimitedBots: /Minibot/i,
+          }
+        }
+      `,
+    },
+  })
+
+  it('should send the blocking response for html limited bots', async () => {
+    const $ = await next.render$(
+      '/',
+      undefined, // no query
+      {
+        headers: {
+          'user-agent': 'Minibot',
+        },
+      }
+    )
+    expect(await $('title').text()).toBe('index page')
+  })
+
+  it('should send streaming response for headless browser bots', async () => {
+    const $ = await next.render$(
+      '/',
+      undefined, // no query
+      {
+        headers: {
+          'user-agent': 'Weebot',
+        },
+      }
+    )
+    expect(await $('title').length).toBe(0)
+  })
+})

test/e2e/app-dir/metadata-streaming/metadata-streaming.test.ts

@@ -1,7 +1,7 @@
 import { nextTestSetup } from 'e2e-utils'
 import { retry, createMultiDomMatcher } from 'next-test-utils'
 
-describe('metadata-streaming', () => {
+describe('app-dir - metadata-streaming', () => {
   const { next } = nextTestSetup({
     files: __dirname,
   })

test/lib/next-modes/base.ts

@@ -23,8 +23,12 @@ export type PackageJson = {
   dependencies?: { [key: string]: string }
   [key: string]: unknown
 }
+
+type ResolvedFileConfig = FileRef | { [filename: string]: string | FileRef }
+type FilesConfig = ResolvedFileConfig | string
 export interface NextInstanceOpts {
-  files: FileRef | string | { [filename: string]: string | FileRef }
+  files: FilesConfig
+  overrideFiles?: FilesConfig
   dependencies?: { [name: string]: string }
   resolutions?: { [name: string]: string }
   packageJson?: PackageJson
@@ -55,7 +59,8 @@ type OmitFirstArgument<F> = F extends (
 const nextjsReactPeerVersion = "^19.0.0";
 
 export class NextInstance {
-  protected files: FileRef | { [filename: string]: string | FileRef }
+  protected files: ResolvedFileConfig
+  protected overrideFiles: ResolvedFileConfig
   protected nextConfig?: NextConfig
   protected installCommand?: InstallCommand
   protected buildCommand?: string
@@ -96,10 +101,10 @@ export class NextInstance {
     }
   }
 
-  protected async writeInitialFiles() {
+  private async writeFiles(filesConfig: FilesConfig, testDir: string) {
     // Handle case where files is a directory string
     const files =
-      typeof this.files === 'string' ? new FileRef(this.files) : this.files
+      typeof filesConfig === 'string' ? new FileRef(filesConfig) : filesConfig
     if (files instanceof FileRef) {
       // if a FileRef is passed directly to `files` we copy the
       // entire folder to the test directory
@@ -111,7 +116,7 @@ export class NextInstance {
         )
       }
 
-      await fs.cp(files.fsPath, this.testDir, {
+      await fs.cp(files.fsPath, testDir, {
         recursive: true,
         filter(source) {
           // we don't copy a package.json as it's manually written
@@ -125,7 +130,7 @@ export class NextInstance {
     } else {
       for (const filename of Object.keys(files)) {
         const item = files[filename]
-        const outputFilename = path.join(this.testDir, filename)
+        const outputFilename = path.join(testDir, filename)
 
         if (typeof item === 'string') {
           await fs.mkdir(path.dirname(outputFilename), { recursive: true })
@@ -137,6 +142,16 @@ export class NextInstance {
     }
   }
 
+  protected async writeInitialFiles() {
+    return this.writeFiles(this.files, this.testDir)
+  }
+
+  protected async writeOverrideFiles() {
+    if (this.overrideFiles) {
+      return this.writeFiles(this.overrideFiles, this.testDir)
+    }
+  }
+
   protected async createTestDir({
     skipInstall = false,
     parentSpan,
@@ -242,6 +257,12 @@ export class NextInstance {
             await this.writeInitialFiles()
           })
 
+        await rootSpan
+          .traceChild('writeOverrideFiles')
+          .traceAsyncFn(async () => {
+            await this.writeOverrideFiles()
+          })
+
         const testDirFiles = await fs.readdir(this.testDir)
 
         let nextConfigFile = testDirFiles.find((file) =>

test/production/app-dir/metadata-streaming-config/app/layout.tsx

@@ -0,0 +1,8 @@
+import { ReactNode } from 'react'
+export default function Root({ children }: { children: ReactNode }) {
+  return (
+    <html>
+      <body>{children}</body>
+    </html>
+  )
+}

test/production/app-dir/metadata-streaming-config/app/page.tsx

@@ -0,0 +1,3 @@
+export default function Page() {
+  return <p>hello world</p>
+}

test/production/app-dir/metadata-streaming-config/metadata-streaming-config-customized.test.ts

@@ -0,0 +1,36 @@
+import { nextTestSetup } from 'e2e-utils'
+
+describe('app-dir - metadata-streaming-config-customized', () => {
+  const { next } = nextTestSetup({
+    files: __dirname,
+    overrideFiles: {
+      'next.config.js': `
+        module.exports = {
+          experimental: {
+            htmlLimitedBots: /MyBot/i,
+          }
+        }
+      `,
+    },
+  })
+
+  it('should have the default streaming metadata config output in routes-manifest.json', async () => {
+    const requiredServerFiles = JSON.parse(
+      await next.readFile('.next/required-server-files.json')
+    )
+    expect(requiredServerFiles.files).toContain(
+      '.next/response-config-manifest.json'
+    )
+
+    const responseConfigManifest = JSON.parse(
+      await next.readFile('.next/response-config-manifest.json')
+    )
+
+    expect(responseConfigManifest).toMatchInlineSnapshot(`
+     {
+       "htmlLimitedBots": "MyBot",
+       "version": 0,
+     }
+    `)
+  })
+})