hide pages from crawler for degenerate queries

app/logical/post_sets/post.rb

@@ -140,6 +140,19 @@ module PostSets
       post_count == Danbooru.config.blank_tag_search_fast_count
     end
 
+    def hide_from_crawler?
+      return true if !is_single_tag?
+      return true if is_pattern_search?
+      return true if params[:page].to_i > 1
+      return true if is_metatag_search?
+      false
+    end
+
+    def is_metatag_search?
+      # filter out some common metatags
+      tag_string =~ /(?:rating|user|fav|status|order|source|score|width|height):/
+    end
+
     def is_single_tag?
       tag_array.size == 1
     end

app/views/posts/index.html.erb

@@ -60,4 +60,7 @@
     <meta name="description" content="<%= @post_set.wiki_page.presenter.blurb %>"></meta>
   <% end %>
   <%= missed_post_search_count_js %>
+  <% if @post_set.hide_from_crawler? %>
+    <meta name="robots" content="none">
+  <% end %>
 <% end %>