Generate a sitemap and robots.txt for search engine indexing

Gemfile

@@ -57,3 +57,4 @@ gem 'dotenv-rails', '~> 2.8'
 # pinned to < 9.3 until https://github.com/ilyakatz/data-migrate/issues/302 resolved
 gem 'data_migrate', '~> 9.2', '< 9.3'
 gem 'rubyzip', '~> 2.3'
+gem 'sitemap_generator'

Gemfile.lock

@@ -251,6 +251,8 @@ GEM
     sidekiq-status (2.1.3)
       chronic_duration
       sidekiq (>= 5.0)
+    sitemap_generator (6.3.0)
+      builder (~> 3.0)
     spring (2.1.1)
     spring-watcher-listen (2.0.1)
       listen (>= 2.7, < 4.0)
@@ -321,6 +323,7 @@ DEPENDENCIES
   rubyzip (~> 2.3)
   sidekiq (~> 6.5.1)
   sidekiq-status (~> 2.1.1)
+  sitemap_generator
   spring
   spring-watcher-listen (~> 2.0.0)
   storyblok-richtext-renderer!

config/routes.rb

@@ -1,3 +1,5 @@
+require 'robots_generator'
+
 Rails.application.routes.draw do
   resources :user_project_permissions, except: :index
   resources :document_folders, except: :index
@@ -40,6 +42,10 @@
   post '/rails/active_storage/direct_uploads' => 'direct_uploads#create'
   post '/projects/:id/create_export' => 'projects#create_export'
   get '/projects/:id/exports' => 'projects#exports'
+  match '/robots.txt', to: RobotsGenerator, via: :all
+  if ENV['AWS_ACCESS_KEY_ID'].present?
+    get 'sitemap.xml.gz', to: redirect("https://#{ENV['AWS_BUCKET']}.s3.#{ENV['AWS_REGION']}.amazonaws.com/sitemaps/sitemap.xml.gz")
+  end
 
   get '*path', to: "application#fallback_index_html", constraints: ->(request) do
     !request.xhr? && request.format.html?

config/sitemap.rb

@@ -0,0 +1,46 @@
+# Sitemap config
+SitemapGenerator::Sitemap.default_host = "#{ENV['PROTOCOL'] || 'http'}://#{ENV['HOSTNAME']}"
+SitemapGenerator::Sitemap.sitemaps_host = "#{ENV['PROTOCOL'] || 'http'}://#{ENV['HOSTNAME']}"
+
+# Set the sitemap storage details
+if ENV['AWS_ACCESS_KEY_ID'].present?
+  SitemapGenerator::Sitemap.sitemaps_path = 'sitemaps/'
+  SitemapGenerator::Sitemap.adapter = SitemapGenerator::AwsSdkAdapter.new(ENV['AWS_BUCKET'],
+    acl: 'public-read', # Optional. This is the default.
+    cache_control: 'private, max-age=0, no-cache', # Optional. This is the default.
+    access_key_id: ENV['AWS_ACCESS_KEY_ID'],
+    secret_access_key: ENV['AWS_SECRET_ACCESS_KEY'],
+    region: ENV['AWS_REGION'],
+  )
+end
+
+SitemapGenerator::Sitemap.create do
+  # Put links creation logic here.
+  #
+  # The root path '/' and sitemap index file are added automatically for you.
+  # Links are added to the Sitemap in the order they are specified.
+  #
+  # Usage: add(path, options={})
+  #        (default options are used if you don't specify)
+  #
+  # Defaults: :priority => 0.5, :changefreq => 'weekly',
+  #           :lastmod => Time.now, :host => default_host
+  #
+  # Examples:
+  #
+  # Add '/articles'
+  #
+  #   add articles_path, :priority => 0.7, :changefreq => 'daily'
+  #
+  # Add all articles:
+  #
+  #   Article.find_each do |article|
+  #     add article_path(article), :lastmod => article.updated_at
+  #   end
+  Project.find_each do |project|
+    add project_path(project), :lastmod => project.updated_at
+  end
+  Document.find_each do |document|
+    add project_path(document.project, { "document" => document.id }), :lastmod => document.updated_at
+  end
+end

lib/robots_generator.rb

@@ -0,0 +1,10 @@
+class RobotsGenerator
+  # http://avandamiri.com/2011/10/11/serving-different-robots-using-rack.html
+  def self.call(env)
+    body = "Sitemap: #{ENV['PROTOCOL'] || 'http'}://#{ENV['HOSTNAME']}/sitemap.xml.gz"
+    headers = {
+      'Content-Type'  => 'text/plain',
+    }
+    [200, headers, [body]]
+  end
+end