From 74d77c1e239af1dcbc0c2ce5504c096d38751b0b Mon Sep 17 00:00:00 2001
From: evazion <noizave@gmail.com>
Date: Tue, 28 Jan 2020 23:45:36 -0600
Subject: [PATCH] Update robots.txt.

* Generate /robots.txt dynamically.
* Include link to sitemap.
* Update list of allowed urls.
* Disallow crawling of non-canonical subdomains.
---
 app/controllers/robots_controller.rb |  6 ++++
 app/views/robots/index.text.erb      | 23 ++++++++++++
 config/routes.rb                     |  1 +
 public/robots.txt                    | 52 ----------------------------
 4 files changed, 30 insertions(+), 52 deletions(-)
 create mode 100644 app/controllers/robots_controller.rb
 create mode 100644 app/views/robots/index.text.erb
 delete mode 100644 public/robots.txt

diff --git a/app/controllers/robots_controller.rb b/app/controllers/robots_controller.rb
new file mode 100644
index 000000000..fda10c506
--- /dev/null
+++ b/app/controllers/robots_controller.rb
@@ -0,0 +1,6 @@
+class RobotsController < ApplicationController
+  respond_to :text
+
+  def index
+  end
+end
diff --git a/app/views/robots/index.text.erb b/app/views/robots/index.text.erb
new file mode 100644
index 000000000..c7f1c0983
--- /dev/null
+++ b/app/views/robots/index.text.erb
@@ -0,0 +1,23 @@
+Sitemap: <%= root_url %>sitemap.xml
+
+User-agent: *
+Disallow: /
+
+<% if Rails.env.production? && Danbooru.config.hostname == request.host %>
+Allow: /artists
+Allow: /artist_commentaries
+Allow: /comments
+Allow: /explore
+Allow: /forum_posts
+Allow: /forum_topics
+Allow: /notes
+Allow: /pools
+Allow: /posts
+Allow: /sessions
+Allow: /static
+Allow: /tags
+Allow: /uploads
+Allow: /user_upgrades
+Allow: /users
+Allow: /wiki_pages
+<% end %>
diff --git a/config/routes.rb b/config/routes.rb
index ea87fbdff..00530dca6 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -232,6 +232,7 @@ Rails.application.routes.draw do
   get "reports/uploads" => "reports#uploads"
   get "reports/upload_tags" => "reports#upload_tags"
   resources :recommended_posts, only: [:index]
+  resources :robots, only: [:index]
   resources :saved_searches, :except => [:show] do
     collection do
       get :labels
diff --git a/public/robots.txt b/public/robots.txt
deleted file mode 100644
index 8e267d4a2..000000000
--- a/public/robots.txt
+++ /dev/null
@@ -1,52 +0,0 @@
-User-agent: *
-Disallow: /admin
-Disallow: /advertisements
-Disallow: /artists
-Disallow: /artist_commentaries
-Disallow: /artist_commentary_versions
-Disallow: /artist_versions
-Disallow: /bans
-Disallow: /comment_votes
-Disallow: /comments
-Disallow: /counts
-Disallow: /delayed_jobs
-Disallow: /dmails
-Disallow: /favorite
-Disallow: /iqdb_queries
-Disallow: /ip_bans
-Disallow: /maintenance
-Disallow: /m/
-Disallow: /mod_actions
-Disallow: /moderator
-Disallow: /news_updates
-Disallow: /note_versions
-Disallow: /notes
-Disallow: /pool_elements
-Disallow: /pool_orders
-Disallow: /pool_versions
-Disallow: /post_appeals
-Disallow: /post_flags
-Disallow: /post_sets
-Disallow: /post_versions
-Disallow: /post_votes
-Disallow: /posts.atom
-Disallow: /posts/random
-Disallow: /posts/copy_notes
-Disallow: /posts/revert
-Disallow: /reports
-Disallow: /session
-Disallow: /sources
-Disallow: /static
-Disallow: /tag_alias_corrections
-Disallow: /tag_alias_requests
-Disallow: /tag_aliases
-Disallow: /tag_implication_requests
-Disallow: /tag_implications
-Disallow: /tags
-Disallow: /uploads
-Disallow: /user_feedback
-Disallow: /users
-Disallow: /wiki_page_versions
-Disallow: /wiki_pages/new
-Disallow: /wiki_pages/revert
-Disallow: /wiki_pages/show_or_new