Update robots.txt.

* Generate /robots.txt dynamically.
* Include link to sitemap.
* Update list of allowed urls.
* Disallow crawling of non-canonical subdomains.

public/robots.txt

@@ -1,52 +0,0 @@
-User-agent: *
-Disallow: /admin
-Disallow: /advertisements
-Disallow: /artists
-Disallow: /artist_commentaries
-Disallow: /artist_commentary_versions
-Disallow: /artist_versions
-Disallow: /bans
-Disallow: /comment_votes
-Disallow: /comments
-Disallow: /counts
-Disallow: /delayed_jobs
-Disallow: /dmails
-Disallow: /favorite
-Disallow: /iqdb_queries
-Disallow: /ip_bans
-Disallow: /maintenance
-Disallow: /m/
-Disallow: /mod_actions
-Disallow: /moderator
-Disallow: /news_updates
-Disallow: /note_versions
-Disallow: /notes
-Disallow: /pool_elements
-Disallow: /pool_orders
-Disallow: /pool_versions
-Disallow: /post_appeals
-Disallow: /post_flags
-Disallow: /post_sets
-Disallow: /post_versions
-Disallow: /post_votes
-Disallow: /posts.atom
-Disallow: /posts/random
-Disallow: /posts/copy_notes
-Disallow: /posts/revert
-Disallow: /reports
-Disallow: /session
-Disallow: /sources
-Disallow: /static
-Disallow: /tag_alias_corrections
-Disallow: /tag_alias_requests
-Disallow: /tag_aliases
-Disallow: /tag_implication_requests
-Disallow: /tag_implications
-Disallow: /tags
-Disallow: /uploads
-Disallow: /user_feedback
-Disallow: /users
-Disallow: /wiki_page_versions
-Disallow: /wiki_pages/new
-Disallow: /wiki_pages/revert
-Disallow: /wiki_pages/show_or_new