updated robots.txt

public/robots.txt

@@ -1,5 +1,55 @@
-# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file
-#
-# To ban all spiders from the entire site uncomment the next two lines:
-# User-Agent: *
-# Disallow: /
+User-agent: *
+Disallow: /admin
+Disallow: /advertisements
+Disallow: /artist_commentaries
+Disallow: /artist_commentary_versions
+Disallow: /artist_versions
+Disallow: /bans
+Disallow: /comment_votes
+Disallow: /comments
+Disallow: /counts
+Disallow: /delayed_jobs
+Disallow: /dmails
+Disallow: /favorite
+Disallow: /ip_bans
+Disallow: /janitor_trials
+Disallow: /legacy
+Disallow: /maintenance
+Disallow: /mobile/
+Disallow: /m/
+Disallow: /mod_actions
+Disallow: /moderator
+Disallow: /news_updates
+Disallow: /note_versions
+Disallow: /notes
+Disallow: /pool_elements
+Disallow: /pool_orders
+Disallow: /pool_versions
+Disallow: /post_appeals
+Disallow: /post_flags
+Disallow: /post_sets
+Disallow: /post_versions
+Disallow: /post_votes
+Disallow: /posts/show_seq
+Disallow: /posts/random
+Disallow: /posts/copy_notes
+Disallow: /posts/revert
+Disallow: /reports
+Disallow: /session
+Disallow: /sources
+Disallow: /static
+Disallow: /tag_alias_corrections
+Disallow: /tag_alias_requests
+Disallow: /tag_aliases
+Disallow: /tag_corrections
+Disallow: /tag_implication_requests
+Disallow: /tag_implications
+Disallow: /tag_subscriptions
+Disallow: /tags
+Disallow: /uploads
+Disallow: /user_feedback
+Disallow: /users
+Disallow: /wiki_page_versions
+Disallow: /wiki_pages/new
+Disallow: /wiki_pages/revert
+Disallow: /wiki_pages/show_or_new