Update robots.txt.sample (#2307)

robots.txt.sample

@@ -1,12 +1,12 @@
 ############################################
-## For more information about the robots.txt standard, see:
+## For more information about the robots.txt standard visit
 ## http://www.robotstxt.org/orig.html
 ##
-## For syntax checking, see:
+## For syntax checking visit
 ## http://tool.motoricerca.info/robots-checker.phtml
 
 ############################################
-## Crawl the Sitemap. Set the correct url before uncomment
+## Crawl the Sitemap. Set the correct URL before uncomment
 
 # Sitemap: https://example.com/sitemap.xml
 
@@ -15,26 +15,37 @@
 
 User-agent: *
 
+## How many seconds a crawler should wait before loading and crawling page content
+## Set a custom crawl rate if you are experiencing traffic issues with your server
+## https://www.contentkingapp.com/academy/robotstxt/faq/crawl-delay-10/
+
+Crawl-delay: 10
+
 ############################################
 ## Allow to crawl paging (paging inside a listing with more params are disallowed below)
+
 Allow: /*?p=
 
 ############################################
-## Do not crawl non-SEF paths and generated content (if you use a shop id in URL, you must prefix with * or copy for each shop)
+## Do not crawl non-SEF paths and generated content (if you use a store id in URL you must prefix with * or copy for each store)
 
 Disallow: */index.php/
 Disallow: */catalog/product_compare/
 Disallow: */catalog/category/view/
 Disallow: */catalog/product/view/
 Disallow: */catalog/product/gallery/
 Disallow: */catalogsearch/
+#Allow: */catalogsearch/seo_sitemap
+#Allow: */catalogsearch/term/popular
+Disallow: */checkout/
 Disallow: */control/
 Disallow: */contacts/
 Disallow: */customer/
 Disallow: */customize/
 Disallow: */newsletter/
 Disallow: */poll/
 Disallow: */review/
+Disallow: */sales/
 Disallow: */sendfriend/
 Disallow: */tag/
 Disallow: */wishlist/
@@ -59,3 +70,32 @@ Disallow: /*?q=*
 Disallow: /*?p=*&
 Disallow: /*.php$
 Disallow: /*?SID=
+
+############################################
+## Do not allow media indexing for the following bots
+## Disallow all or add custom paths. For example */media/ or */skin/
+
+# User-agent: baiduspider-image
+# Disallow: /
+# Disallow: */media/
+# Disallow: */skin/
+
+# User-agent: baiduspider-video
+# Disallow: /
+# Disallow: */media/
+# Disallow: */skin/
+
+# User-agent: msnbot-media
+# Disallow: /
+# Disallow: */media/
+# Disallow: */skin/
+
+# User-agent: Googlebot-Image
+# Disallow: /
+# Disallow: */media/
+# Disallow: */skin/
+
+# User-agent: Googlebot-Video
+# Disallow: /
+# Disallow: */media/
+# Disallow: */skin/