From 734c0b6716c356ca2ee5dd3da6d796925a4190de Mon Sep 17 00:00:00 2001
From: Claas Augner <495429+caugner@users.noreply.github.com>
Date: Mon, 27 May 2024 23:04:09 +0200
Subject: [PATCH] fix(search): use "noindex, follow" robots tag + remove
 robots.txt exclude (#11140)

* fix(spas): set "noindex, follow" for search

Previously, it was "index, follow", but we do not want to index
MDN full-text search results.

* fix(robots): remove /search from robots.txt
---
 build/spas.ts            | 5 +++--
 libs/types/hydration.ts  | 3 ++-
 ssr/render.ts            | 7 +++++--
 tool/build-robots-txt.ts | 9 +--------
 4 files changed, 11 insertions(+), 13 deletions(-)

diff --git a/build/spas.ts b/build/spas.ts
index 8a8198f6b3cb..b66f23975330 100644
--- a/build/spas.ts
+++ b/build/spas.ts
@@ -140,7 +140,7 @@ export async function buildSPAs(options: {
 
       const SPAs = [
         { prefix: "play", pageTitle: "Playground | MDN" },
-        { prefix: "search", pageTitle: "Search" },
+        { prefix: "search", pageTitle: "Search", onlyFollow: true },
         { prefix: "plus", pageTitle: MDN_PLUS_TITLE },
         {
           prefix: "plus/ai-help",
@@ -177,12 +177,13 @@ export async function buildSPAs(options: {
         },
       ];
       const locale = VALID_LOCALES.get(pathLocale) || pathLocale;
-      for (const { prefix, pageTitle, noIndexing } of SPAs) {
+      for (const { prefix, pageTitle, noIndexing, onlyFollow } of SPAs) {
         const url = `/${locale}/${prefix}`;
         const context = {
           pageTitle,
           locale,
           noIndexing,
+          onlyFollow,
         };
 
         const html = renderCanonicalHTML(url, context);
diff --git a/libs/types/hydration.ts b/libs/types/hydration.ts
index 454675a941d4..295fcbd86116 100644
--- a/libs/types/hydration.ts
+++ b/libs/types/hydration.ts
@@ -8,7 +8,8 @@ interface HydrationData<T = any, S = any> {
   pageTitle?: any;
   possibleLocales?: any;
   locale?: any;
-  noIndexing?: any;
+  noIndexing?: boolean;
+  onlyFollow?: boolean;
   image?: string | null;
 }
 
diff --git a/ssr/render.ts b/ssr/render.ts
index 69f32bfa318f..ed8fcc59dbed 100644
--- a/ssr/render.ts
+++ b/ssr/render.ts
@@ -106,7 +106,8 @@ export default function render(
     pageTitle = null,
     possibleLocales = null,
     locale = null,
-    noIndexing = null,
+    noIndexing = false,
+    onlyFollow = false,
     image = null,
     blogMeta = null,
   }: HydrationData = {}
@@ -211,7 +212,9 @@ export default function render(
   const robotsContent =
     !ALWAYS_ALLOW_ROBOTS || (doc && doc.noIndexing) || noIndexing
       ? "noindex, nofollow"
-      : "";
+      : onlyFollow
+        ? "noindex, follow"
+        : "";
   const robotsMeta = robotsContent
     ? `<meta name="robots" content="${robotsContent}">`
     : "";
diff --git a/tool/build-robots-txt.ts b/tool/build-robots-txt.ts
index ab6d4f0d46b3..f39827779635 100644
--- a/tool/build-robots-txt.ts
+++ b/tool/build-robots-txt.ts
@@ -5,7 +5,6 @@
  */
 import fs from "node:fs";
 
-import { VALID_LOCALES } from "../libs/constants/index.js";
 import { ALWAYS_ALLOW_ROBOTS } from "../libs/env/index.js";
 
 const ALLOW_TEXT = `
@@ -25,12 +24,6 @@ Disallow: /
 `;
 
 export async function runBuildRobotsTxt(outfile: string) {
-  let content = ALWAYS_ALLOW_ROBOTS ? ALLOW_TEXT : DISALLOW_TEXT;
-  if (ALWAYS_ALLOW_ROBOTS) {
-    // Append extra lines specifically when we do allow robots.
-    for (const locale of VALID_LOCALES.values()) {
-      content += `Disallow: /${locale}/search\n`;
-    }
-  }
+  const content = ALWAYS_ALLOW_ROBOTS ? ALLOW_TEXT : DISALLOW_TEXT;
   fs.writeFileSync(outfile, `${content.trim()}\n`, "utf-8");
 }