From 72f11d031d44c5cb7fa93e86623cac201c594eb8 Mon Sep 17 00:00:00 2001
From: Claas Augner <caugner@mozilla.com>
Date: Wed, 15 May 2024 18:54:20 -0700
Subject: [PATCH 1/2] fix(ssr): remove unnecessary robots tags

1. Robots tag for 404 page is unnecessary, as it is ignored anyways.
2. Robots tag with value "index, follow" is unnecessary, as it is the default.
---
 ssr/render.ts | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/ssr/render.ts b/ssr/render.ts
index 12f9d27c8624..69f32bfa318f 100644
--- a/ssr/render.ts
+++ b/ssr/render.ts
@@ -209,13 +209,12 @@ export default function render(
   }</script>`;
 
   const robotsContent =
-    !ALWAYS_ALLOW_ROBOTS ||
-    (doc && doc.noIndexing) ||
-    pageNotFound ||
-    noIndexing
+    !ALWAYS_ALLOW_ROBOTS || (doc && doc.noIndexing) || noIndexing
       ? "noindex, nofollow"
-      : "index, follow";
-  const robotsMeta = `<meta name="robots" content="${robotsContent}">`;
+      : "";
+  const robotsMeta = robotsContent
+    ? `<meta name="robots" content="${robotsContent}">`
+    : "";
   const rssLink = `<link rel="alternate" type="application/rss+xml" title="MDN Blog RSS Feed" href="${BASE_URL}/${DEFAULT_LOCALE}/blog/rss.xml" hreflang="en" />`;
   const ssr_data = [...translations, ...WEBFONT_TAGS, rssLink, robotsMeta];
   let html = buildHtml;

From bbacbacebb8e96019b9efda2a2803ae9fdb9a338 Mon Sep 17 00:00:00 2001
From: Claas Augner <caugner@mozilla.com>
Date: Wed, 15 May 2024 20:52:12 -0700
Subject: [PATCH 2/2] test(spas): update expectations

---
 testing/tests/index.test.ts | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/testing/tests/index.test.ts b/testing/tests/index.test.ts
index 8ed91f99b4e4..9eded792f560 100644
--- a/testing/tests/index.test.ts
+++ b/testing/tests/index.test.ts
@@ -119,7 +119,7 @@ test("content built foo page", () => {
     `https://developer.mozilla.org${doc.mdn_url}`
   );
 
-  expect($('meta[name="robots"]').attr("content")).toBe("index, follow");
+  expect($('meta[name="robots"]')).toHaveLength(0);
 
   // The HTML should contain the Google Analytics snippet.
   // The ID should match what's set in `.env.testing`.
@@ -1179,7 +1179,7 @@ test("404 page", () => {
   const $ = cheerio.load(html);
   expect($("title").text()).toContain("Page not found");
   expect($("h1").text()).toContain("Page not found");
-  expect($('meta[name="robots"]').attr("content")).toBe("noindex, nofollow");
+  expect($('meta[name="robots"]')).toHaveLength(0);
   expect($('meta[property="og:locale"]').attr("content")).toBe("en_US");
 });
 
@@ -1190,7 +1190,7 @@ test("plus page", () => {
   const html = fs.readFileSync(htmlFile, "utf-8");
   const $ = cheerio.load(html);
   expect($("title").text()).toContain("Plus");
-  expect($('meta[name="robots"]').attr("content")).toBe("index, follow");
+  expect($('meta[name="robots"]')).toHaveLength(0);
 });
 
 test("plus collections page", () => {