fix /ffxiv/craftsim crawl (#512)

* fix /ffxiv/craftsim crawl * remove pages with url endings from sitemap
ff14-advanced-market-search · Oct 23, 2024 · 25c7783 · 25c7783
1 parent 3e362aa
commit 25c7783
Show file tree

Hide file tree

Showing 2 changed files with 31 additions and 31 deletions.
diff --git a/app/routes/[robots.txt].tsx b/app/routes/[robots.txt].tsx
@@ -43,7 +43,7 @@ export const loader: LoaderFunction = () => {
   Disallow: /build/
   Disallow: /queries/full-scan
   Disallow: /ffxiv/marketshare
-  Disallow: /ffxiv/craftsim/queries
+  Disallow: /ffxiv/craftsim
   Disallow: /wow/best-deals
   Disallow: /wow/export-search
   Disallow: /wow/marketshare

diff --git a/app/routes/[sitemap.xml].tsx b/app/routes/[sitemap.xml].tsx
@@ -116,16 +116,16 @@ export const loader: LoaderFunction = async () => {
   <lastmod>2024-05-07T00:27:48+00:00</lastmod>
   <priority>0.80</priority>
 </url>
-<url>
-  <loc>https://saddlebagexchange.com/wow/export-search</loc>
-  <lastmod>2024-05-07T00:27:48+00:00</lastmod>
-  <priority>0.80</priority>
-</url>
-<url>
-  <loc>https://saddlebagexchange.com/wow/marketshare</loc>
-  <lastmod>2024-05-07T00:27:48+00:00</lastmod>
-  <priority>0.80</priority>
-</url>
+// <url>
+//   <loc>https://saddlebagexchange.com/wow/export-search</loc>
+//   <lastmod>2024-05-07T00:27:48+00:00</lastmod>
+//   <priority>0.80</priority>
+// </url>
+// <url>
+//   <loc>https://saddlebagexchange.com/wow/marketshare</loc>
+//   <lastmod>2024-05-07T00:27:48+00:00</lastmod>
+//   <priority>0.80</priority>
+// </url>
 <url>
   <loc>https://saddlebagexchange.com/wow/marketshare/recommended</loc>
   <lastmod>2024-06-21T00:27:48+00:00</lastmod>
@@ -181,26 +181,26 @@ export const loader: LoaderFunction = async () => {
   <lastmod>2024-05-07T00:27:48+00:00</lastmod>
   <priority>0.80</priority>
 </url>
-<url>
-  <loc>https://saddlebagexchange.com/queries/full-scan</loc>
-  <lastmod>2024-05-07T00:27:48+00:00</lastmod>
-  <priority>0.80</priority>
-</url>
-<url>
-  <loc>https://saddlebagexchange.com/ffxiv/marketshare</loc>
-  <lastmod>2024-05-07T00:27:48+00:00</lastmod>
-  <priority>0.80</priority>
-</url>
-<url>
-  <loc>https://saddlebagexchange.com/ffxiv/craftsim</loc>
-  <lastmod>2024-05-07T00:27:48+00:00</lastmod>
-  <priority>0.80</priority>
-</url>
-<url>
-  <loc>https://saddlebagexchange.com/wow/best-deals</loc>
-  <lastmod>2024-05-07T00:27:48+00:00</lastmod>
-  <priority>0.80</priority>
-</url>
+// <url>
+//   <loc>https://saddlebagexchange.com/queries/full-scan</loc>
+//   <lastmod>2024-05-07T00:27:48+00:00</lastmod>
+//   <priority>0.80</priority>
+// </url>
+// <url>
+//   <loc>https://saddlebagexchange.com/ffxiv/marketshare</loc>
+//   <lastmod>2024-05-07T00:27:48+00:00</lastmod>
+//   <priority>0.80</priority>
+// </url>
+// <url>
+//   <loc>https://saddlebagexchange.com/ffxiv/craftsim</loc>
+//   <lastmod>2024-05-07T00:27:48+00:00</lastmod>
+//   <priority>0.80</priority>
+// </url>
+// <url>
+//   <loc>https://saddlebagexchange.com/wow/best-deals</loc>
+//   <lastmod>2024-05-07T00:27:48+00:00</lastmod>
+//   <priority>0.80</priority>
+// </url>
 <url>
   <loc>https://saddlebagexchange.com/wow/pet-marketshare</loc>
   <lastmod>2024-05-11T00:27:48+00:00</lastmod>