[metadata] add option of configuring ua of async metadata

packages/next/src/build/index.ts

@@ -80,6 +80,7 @@ import {
   UNDERSCORE_NOT_FOUND_ROUTE_ENTRY,
   UNDERSCORE_NOT_FOUND_ROUTE,
   DYNAMIC_CSS_MANIFEST,
+  RESPONSE_CONFIG_MANIFEST,
 } from '../shared/lib/constants'
 import {
   getSortedRoutes,
@@ -214,6 +215,7 @@ import {
   getParsedNodeOptionsWithoutInspect,
 } from '../server/lib/utils'
 import { InvariantError } from '../shared/lib/invariant-error'
+import { HTML_LIMITED_BOT_UA_RE_STRING } from '../shared/lib/router/utils/is-bot'
 
 type Fallback = null | boolean | string
 
@@ -1307,6 +1309,21 @@ export default async function build(
         NextBuildContext.clientRouterFilters = clientRouterFilters
       }
 
+      // Write html limited bots config to response-config-manifest
+      const responseConfigManifestPath = path.join(
+        distDir,
+        RESPONSE_CONFIG_MANIFEST
+      )
+      const responseConfigManifest: {
+        version: number
+        htmlLimitedBots: string
+      } = {
+        version: 0,
+        htmlLimitedBots:
+          config.experimental.htmlLimitedBots || HTML_LIMITED_BOT_UA_RE_STRING,
+      }
+      await writeManifest(responseConfigManifestPath, responseConfigManifest)
+
       // Ensure commonjs handling is used for files in the distDir (generally .next)
       // Files outside of the distDir can be "type": "module"
       await writeFileUtf8(
@@ -2485,6 +2502,7 @@ export default async function build(
               path.relative(distDir, pagesManifestPath),
               BUILD_MANIFEST,
               PRERENDER_MANIFEST,
+              RESPONSE_CONFIG_MANIFEST,
               path.join(SERVER_DIRECTORY, MIDDLEWARE_MANIFEST),
               path.join(SERVER_DIRECTORY, MIDDLEWARE_BUILD_MANIFEST + '.js'),
               ...(!process.env.TURBOPACK

packages/next/src/build/utils.ts

@@ -1,4 +1,4 @@
-import type { NextConfig, NextConfigComplete } from '../server/config-shared'
+import type { NextConfigComplete } from '../server/config-shared'
 import type { ExperimentalPPRConfig } from '../server/lib/experimental/ppr'
 import type { AppBuildManifest } from './webpack/plugins/app-build-manifest-plugin'
 import type { AssetBinding } from './webpack/loaders/get-module-build-info'
@@ -1466,7 +1466,7 @@ export async function copyTracedFiles(
   pageKeys: readonly string[],
   appPageKeys: readonly string[] | undefined,
   tracingRoot: string,
-  serverConfig: NextConfig,
+  serverConfig: NextConfigComplete,
   middlewareManifest: MiddlewareManifest,
   hasInstrumentationHook: boolean,
   staticPages: Set<string>

packages/next/src/export/index.ts

@@ -360,6 +360,7 @@ async function exportAppImpl(
       inlineCss: nextConfig.experimental.inlineCss ?? false,
       authInterrupts: !!nextConfig.experimental.authInterrupts,
       streamingMetadata: !!nextConfig.experimental.streamingMetadata,
+      htmlLimitedBots: nextConfig.experimental.htmlLimitedBots,
     },
     reactMaxHeadersLength: nextConfig.reactMaxHeadersLength,
   }

packages/next/src/server/app-render/types.ts

@@ -216,6 +216,7 @@ export interface RenderOptsPartial {
     inlineCss: boolean
     authInterrupts: boolean
     streamingMetadata: boolean
+    htmlLimitedBots: string | undefined
   }
   postponed?: string
 

packages/next/src/server/base-server.ts

@@ -83,7 +83,7 @@ import {
 } from './lib/revalidate'
 import { execOnce } from '../shared/lib/utils'
 import { isBlockedPage } from './utils'
-import { isBot, isHtmlLimitedBotUA } from '../shared/lib/router/utils/is-bot'
+import { isBot } from '../shared/lib/router/utils/is-bot'
 import RenderResult from './render-result'
 import { removeTrailingSlash } from '../shared/lib/router/utils/remove-trailing-slash'
 import { denormalizePagePath } from '../shared/lib/page-path/denormalize-page-path'
@@ -175,6 +175,7 @@ import type { RouteModule } from './route-modules/route-module'
 import { FallbackMode, parseFallbackField } from '../lib/fallback'
 import { toResponseCacheEntry } from './response-cache/utils'
 import { scheduleOnNextTick } from '../lib/scheduler'
+import { shouldServeStreamingMetadata } from './lib/streaming-metadata'
 
 export type FindComponentsResult = {
   components: LoadComponentsReturnType
@@ -595,6 +596,7 @@ export default abstract class Server<
         inlineCss: this.nextConfig.experimental.inlineCss ?? false,
         authInterrupts: !!this.nextConfig.experimental.authInterrupts,
         streamingMetadata: !!this.nextConfig.experimental.streamingMetadata,
+        htmlLimitedBots: this.nextConfig.experimental.htmlLimitedBots,
       },
       onInstrumentationRequestError:
         this.instrumentationOnRequestError.bind(this),
@@ -1675,11 +1677,13 @@ export default abstract class Server<
       renderOpts: {
         ...this.renderOpts,
         supportsDynamicResponse: !isBotRequest,
-        serveStreamingMetadata:
-          this.renderOpts.experimental.streamingMetadata &&
-          !isHtmlLimitedBotUA(ua),
+        serveStreamingMetadata: shouldServeStreamingMetadata(
+          ua,
+          this.renderOpts.experimental
+        ),
       },
     }
+
     const payload = await fn(ctx)
     if (payload === null) {
       return
@@ -2181,8 +2185,6 @@ export default abstract class Server<
       // cache if there are no dynamic data requirements
       opts.supportsDynamicResponse =
         !isSSG && !isBotRequest && !query.amp && isSupportedDocument
-      opts.serveStreamingMetadata =
-        opts.experimental.streamingMetadata && !isHtmlLimitedBotUA(ua)
     }
 
     // In development, we always want to generate dynamic HTML.

packages/next/src/server/config-schema.ts

@@ -441,6 +441,7 @@ export const configSchema: zod.ZodType<NextConfig> = z.lazy(() =>
         authInterrupts: z.boolean().optional(),
         newDevOverlay: z.boolean().optional(),
         streamingMetadata: z.boolean().optional(),
+        htmlLimitedBots: z.instanceof(RegExp).optional(),
       })
       .optional(),
     exportPathMap: z

packages/next/src/server/config-shared.ts

@@ -21,6 +21,11 @@ export type NextConfigComplete = Required<NextConfig> & {
   configOrigin?: string
   configFile?: string
   configFileName: string
+  // override NextConfigComplete.experimental.htmlLimitedBots to string
+  // because it's not defined in NextConfigComplete.experimental
+  experimental: Omit<ExperimentalConfig, 'htmlLimitedBots'> & {
+    htmlLimitedBots: string | undefined
+  }
 }
 
 export type I18NDomains = readonly DomainLocale[]
@@ -572,6 +577,12 @@ export interface ExperimentalConfig {
    * When enabled will cause async metadata calls to stream rather than block the render.
    */
   streamingMetadata?: boolean
+
+  /**
+   * User Agent of bots that can handle streaming metadata.
+   * Besides the default behavior, Next.js act differently on serving metadata to bots based on their capability.
+   */
+  htmlLimitedBots?: RegExp
 }
 
 export type ExportPathMap = {
@@ -1197,6 +1208,7 @@ export const defaultConfig: NextConfig = {
     inlineCss: false,
     newDevOverlay: false,
     streamingMetadata: false,
+    htmlLimitedBots: undefined,
   },
   bundlePagesRouterDependencies: false,
 }

packages/next/src/server/config.ts

@@ -27,6 +27,7 @@ import { hasNextSupport } from '../server/ci-info'
 import { transpileConfig } from '../build/next-config-ts/transpile-config'
 import { dset } from '../shared/lib/dset'
 import { normalizeZodErrors } from '../shared/lib/zod'
+import { HTML_LIMITED_BOT_UA_RE_STRING } from '../shared/lib/router/utils/is-bot'
 
 export { normalizeConfig } from './config-shared'
 export type { DomainLocale, NextConfig } from './config-shared'
@@ -1004,6 +1005,11 @@ function assignDefaults(
     ]),
   ]
 
+  if (!result.experimental.htmlLimitedBots) {
+    // @ts-expect-error: override the htmlLimitedBots with default string, type covert: RegExp -> string
+    result.experimental.htmlLimitedBots = HTML_LIMITED_BOT_UA_RE_STRING
+  }
+
   return result
 }
 
@@ -1222,6 +1228,12 @@ export default async function loadConfig(
       }
     }
 
+    // serialize the regex config into string
+    if (userConfig.experimental?.htmlLimitedBots instanceof RegExp) {
+      userConfig.experimental.htmlLimitedBots =
+        userConfig.experimental.htmlLimitedBots.source
+    }
+
     onLoadUserConfig?.(userConfig)
     const completeConfig = assignDefaults(
       dir,

packages/next/src/server/lib/streaming-metadata.ts

@@ -0,0 +1,22 @@
+import { HTML_LIMITED_BOT_UA_RE_STRING } from '../../shared/lib/router/utils/is-bot'
+
+export function shouldServeStreamingMetadata(
+  userAgent: string,
+  {
+    streamingMetadata,
+    htmlLimitedBots,
+  }: {
+    streamingMetadata: boolean
+    htmlLimitedBots: string | undefined
+  }
+): boolean {
+  if (!streamingMetadata) {
+    return false
+  }
+
+  const blockingMetadataUARegex = new RegExp(
+    htmlLimitedBots || HTML_LIMITED_BOT_UA_RE_STRING,
+    'i'
+  )
+  return !blockingMetadataUARegex.test(userAgent)
+}

packages/next/src/shared/lib/constants.ts

@@ -42,6 +42,7 @@ export const EXPORT_DETAIL = 'export-detail.json'
 export const PRERENDER_MANIFEST = 'prerender-manifest.json'
 export const ROUTES_MANIFEST = 'routes-manifest.json'
 export const IMAGES_MANIFEST = 'images-manifest.json'
+export const RESPONSE_CONFIG_MANIFEST = 'response-config-manifest.json'
 export const SERVER_FILES_MANIFEST = 'required-server-files.json'
 export const DEV_CLIENT_PAGES_MANIFEST = '_devPagesManifest.json'
 export const MIDDLEWARE_MANIFEST = 'middleware-manifest.json'

packages/next/src/shared/lib/router/utils/is-bot.ts

@@ -4,14 +4,19 @@ const HEADLESS_BROWSER_BOT_UA_RE =
 
 // This regex contains the bots that we need to do a blocking render for and can't safely stream the response
 // due to how they parse the DOM. For example, they might explicitly check for metadata in the `head` tag, so we can't stream metadata tags after the `head` was sent.
-const HTML_LIMITED_BOT_UA_RE =
-  /Mediapartners-Google|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview/i
+export const HTML_LIMITED_BOT_UA_RE_STRING =
+  'Mediapartners-Google|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview'
+
+export const HTML_LIMITED_BOT_UA_RE = new RegExp(
+  HTML_LIMITED_BOT_UA_RE_STRING,
+  'i'
+)
 
 function isHeadlessBrowserBotUA(userAgent: string) {
   return HEADLESS_BROWSER_BOT_UA_RE.test(userAgent)
 }
 
-export function isHtmlLimitedBotUA(userAgent: string) {
+function isHtmlLimitedBotUA(userAgent: string) {
   return HTML_LIMITED_BOT_UA_RE.test(userAgent)
 }
 

test/e2e/app-dir/metadata-streaming/metadata-streaming-customized-rule.test.ts

@@ -0,0 +1,43 @@
+import { nextTestSetup } from 'e2e-utils'
+
+describe('app-dir - metadata-streaming-customized-rule', () => {
+  const { next } = nextTestSetup({
+    files: __dirname,
+    overrideFiles: {
+      'next.config.js': `
+        module.exports = {
+          experimental: {
+            streamingMetadata: true,
+            htmlLimitedBots: /Minibot/i,
+          }
+        }
+      `,
+    },
+  })
+
+  it('should send the blocking response for html limited bots', async () => {
+    const $ = await next.render$(
+      '/',
+      undefined, // no query
+      {
+        headers: {
+          'user-agent': 'Minibot',
+        },
+      }
+    )
+    expect(await $('title').text()).toBe('index page')
+  })
+
+  it('should send streaming response for headless browser bots', async () => {
+    const $ = await next.render$(
+      '/',
+      undefined, // no query
+      {
+        headers: {
+          'user-agent': 'Weebot',
+        },
+      }
+    )
+    expect(await $('title').length).toBe(0)
+  })
+})

test/e2e/app-dir/metadata-streaming/metadata-streaming.test.ts

@@ -1,7 +1,7 @@
 import { nextTestSetup } from 'e2e-utils'
 import { retry, createMultiDomMatcher } from 'next-test-utils'
 
-describe('metadata-streaming', () => {
+describe('app-dir - metadata-streaming', () => {
   const { next } = nextTestSetup({
     files: __dirname,
   })

test/lib/next-modes/base.ts

@@ -23,8 +23,12 @@ export type PackageJson = {
   dependencies?: { [key: string]: string }
   [key: string]: unknown
 }
+
+type ResolvedFileConfig = FileRef | { [filename: string]: string | FileRef }
+type FilesConfig = ResolvedFileConfig | string
 export interface NextInstanceOpts {
-  files: FileRef | string | { [filename: string]: string | FileRef }
+  files: FilesConfig
+  overrideFiles?: FilesConfig
   dependencies?: { [name: string]: string }
   resolutions?: { [name: string]: string }
   packageJson?: PackageJson
@@ -55,7 +59,8 @@ type OmitFirstArgument<F> = F extends (
 const nextjsReactPeerVersion = "^19.0.0";
 
 export class NextInstance {
-  protected files: FileRef | { [filename: string]: string | FileRef }
+  protected files: ResolvedFileConfig
+  protected overrideFiles: ResolvedFileConfig
   protected nextConfig?: NextConfig
   protected installCommand?: InstallCommand
   protected buildCommand?: string
@@ -96,10 +101,10 @@ export class NextInstance {
     }
   }
 
-  protected async writeInitialFiles() {
+  private async writeFiles(filesConfig: FilesConfig, testDir: string) {
     // Handle case where files is a directory string
     const files =
-      typeof this.files === 'string' ? new FileRef(this.files) : this.files
+      typeof filesConfig === 'string' ? new FileRef(filesConfig) : filesConfig
     if (files instanceof FileRef) {
       // if a FileRef is passed directly to `files` we copy the
       // entire folder to the test directory
@@ -111,7 +116,7 @@ export class NextInstance {
         )
       }
 
-      await fs.cp(files.fsPath, this.testDir, {
+      await fs.cp(files.fsPath, testDir, {
         recursive: true,
         filter(source) {
           // we don't copy a package.json as it's manually written
@@ -125,7 +130,7 @@ export class NextInstance {
     } else {
       for (const filename of Object.keys(files)) {
         const item = files[filename]
-        const outputFilename = path.join(this.testDir, filename)
+        const outputFilename = path.join(testDir, filename)
 
         if (typeof item === 'string') {
           await fs.mkdir(path.dirname(outputFilename), { recursive: true })
@@ -137,6 +142,16 @@ export class NextInstance {
     }
   }
 
+  protected async writeInitialFiles() {
+    return this.writeFiles(this.files, this.testDir)
+  }
+
+  protected async writeOverrideFiles() {
+    if (this.overrideFiles) {
+      return this.writeFiles(this.overrideFiles, this.testDir)
+    }
+  }
+
   protected async createTestDir({
     skipInstall = false,
     parentSpan,
@@ -242,6 +257,12 @@ export class NextInstance {
             await this.writeInitialFiles()
           })
 
+        await rootSpan
+          .traceChild('writeOverrideFiles')
+          .traceAsyncFn(async () => {
+            await this.writeOverrideFiles()
+          })
+
         const testDirFiles = await fs.readdir(this.testDir)
 
         let nextConfigFile = testDirFiles.find((file) =>