From 1207a39b6673b664c1467b54d58c06121caa3571 Mon Sep 17 00:00:00 2001
From: r888888888 <r888888888@gmail.com>
Date: Thu, 4 Feb 2016 13:53:46 -0800
Subject: [PATCH] hide pages from crawler for degenerate queries

---
 app/logical/post_sets/post.rb  | 13 +++++++++++++
 app/views/posts/index.html.erb |  3 +++
 2 files changed, 16 insertions(+)

diff --git a/app/logical/post_sets/post.rb b/app/logical/post_sets/post.rb
index 1985ccd2c..1ac980714 100644
--- a/app/logical/post_sets/post.rb
+++ b/app/logical/post_sets/post.rb
@@ -140,6 +140,19 @@ module PostSets
       post_count == Danbooru.config.blank_tag_search_fast_count
     end
 
+    def hide_from_crawler?
+      return true if !is_single_tag?
+      return true if is_pattern_search?
+      return true if params[:page].to_i > 1
+      return true if is_metatag_search?
+      false
+    end
+
+    def is_metatag_search?
+      # filter out some common metatags
+      tag_string =~ /(?:rating|user|fav|status|order|source|score|width|height):/
+    end
+
     def is_single_tag?
       tag_array.size == 1
     end
diff --git a/app/views/posts/index.html.erb b/app/views/posts/index.html.erb
index 5422bdee1..fe99c6c09 100644
--- a/app/views/posts/index.html.erb
+++ b/app/views/posts/index.html.erb
@@ -60,4 +60,7 @@
     <meta name="description" content="<%= @post_set.wiki_page.presenter.blurb %>"></meta>
   <% end %>
   <%= missed_post_search_count_js %>
+  <% if @post_set.hide_from_crawler? %>
+    <meta name="robots" content="none">
+  <% end %>
 <% end %>