Update robots.txt.

* Generate /robots.txt dynamically.
* Include link to sitemap.
* Update list of allowed urls.
* Disallow crawling of non-canonical subdomains.

app/controllers/robots_controller.rb

@@ -0,0 +1,6 @@
+class RobotsController < ApplicationController
+  respond_to :text
+
+  def index
+  end
+end

app/views/robots/index.text.erb

@@ -0,0 +1,23 @@
+Sitemap: <%= root_url %>sitemap.xml
+
+User-agent: *
+Disallow: /
+
+<% if Rails.env.production? && Danbooru.config.hostname == request.host %>
+Allow: /artists
+Allow: /artist_commentaries
+Allow: /comments
+Allow: /explore
+Allow: /forum_posts
+Allow: /forum_topics
+Allow: /notes
+Allow: /pools
+Allow: /posts
+Allow: /sessions
+Allow: /static
+Allow: /tags
+Allow: /uploads
+Allow: /user_upgrades
+Allow: /users
+Allow: /wiki_pages
+<% end %>

config/routes.rb

@@ -232,6 +232,7 @@ Rails.application.routes.draw do
   get "reports/uploads" => "reports#uploads"
   get "reports/upload_tags" => "reports#upload_tags"
   resources :recommended_posts, only: [:index]
+  resources :robots, only: [:index]
   resources :saved_searches, :except => [:show] do
     collection do
       get :labels

public/robots.txt

@@ -1,52 +0,0 @@
-User-agent: *
-Disallow: /admin
-Disallow: /advertisements
-Disallow: /artists
-Disallow: /artist_commentaries
-Disallow: /artist_commentary_versions
-Disallow: /artist_versions
-Disallow: /bans
-Disallow: /comment_votes
-Disallow: /comments
-Disallow: /counts
-Disallow: /delayed_jobs
-Disallow: /dmails
-Disallow: /favorite
-Disallow: /iqdb_queries
-Disallow: /ip_bans
-Disallow: /maintenance
-Disallow: /m/
-Disallow: /mod_actions
-Disallow: /moderator
-Disallow: /news_updates
-Disallow: /note_versions
-Disallow: /notes
-Disallow: /pool_elements
-Disallow: /pool_orders
-Disallow: /pool_versions
-Disallow: /post_appeals
-Disallow: /post_flags
-Disallow: /post_sets
-Disallow: /post_versions
-Disallow: /post_votes
-Disallow: /posts.atom
-Disallow: /posts/random
-Disallow: /posts/copy_notes
-Disallow: /posts/revert
-Disallow: /reports
-Disallow: /session
-Disallow: /sources
-Disallow: /static
-Disallow: /tag_alias_corrections
-Disallow: /tag_alias_requests
-Disallow: /tag_aliases
-Disallow: /tag_implication_requests
-Disallow: /tag_implications
-Disallow: /tags
-Disallow: /uploads
-Disallow: /user_feedback
-Disallow: /users
-Disallow: /wiki_page_versions
-Disallow: /wiki_pages/new
-Disallow: /wiki_pages/revert
-Disallow: /wiki_pages/show_or_new