eliminate listbooru references

Gemfile

@@ -51,6 +51,7 @@ gem 'ptools'
 gem 'jquery-rails'
 gem 'webpacker', '>= 4.0.x'
 gem 'retriable'
+gem 'redis'
 
 # needed for looser jpeg header compat
 gem 'ruby-imagespec', :require => "image_spec", :git => "https://github.com/r888888888/ruby-imagespec.git", :branch => "exif-fixes"

Gemfile.lock

@@ -314,6 +314,7 @@ GEM
       ffi (>= 0.5.0, < 2)
     recaptcha (4.8.0)
       json
+    redis (4.0.3)
     ref (2.0.0)
     representable (2.3.0)
       uber (~> 0.0.7)
@@ -488,6 +489,7 @@ DEPENDENCIES
   rails (~> 5.2)
   rakismet
   recaptcha
+  redis
   responders
   retriable
   ruby-imagespec!

app/models/saved_search.rb

@@ -1,92 +1,103 @@
 class SavedSearch < ApplicationRecord
-  module ListbooruMethods
+  REDIS_EXPIRY = 3600
-    extend ActiveSupport::Concern
+  QUERY_LIMIT = 500
 
-    module ClassMethods
+  concerning :Redis do
+    class_methods do
       def enabled?
-        Danbooru.config.aws_sqs_saved_search_url.present?
+        Danbooru.config.redis_url.present?
       end
 
-      def posts_search_available?
+      def post_ids_for(user_id, label: nil)
-        enabled? && CurrentUser.is_gold?
+        redis = Redis.new(url: Danbooru.config.redis_url)
-      end
-
-      def sqs_service
-        SqsService.new(Danbooru.config.aws_sqs_saved_search_url)
-      end
-
-      def post_ids(user_id, label = nil)
-        return [] unless enabled?
         label = normalize_label(label) if label
-
+        queries = queries_for(user_id, label: label)
-        Cache.get("ss-#{user_id}-#{Cache.hash(label)}", 60) do
+        post_ids = Set.new
-          queries = SavedSearch.queries_for(user_id, label)
+        queries.each do |query|
-          return [] if queries.empty?
+          query_hash = Cache.hash(query)
-
+          redis_key = "search:#{query_hash}"
-          json = {
+          if redis.exists(redis_key)
-            "key" => Danbooru.config.listbooru_auth_key,
+            sub_ids = redis.smembers(redis_key)
-            "queries" => queries
+            post_ids.merge(sub_ids)
-          }.to_json
+            redis.expire(redis_key, REDIS_EXPIRY)
-
-          uri = "#{Danbooru.config.listbooru_server}/v2/search"
-
-          resp = HTTParty.post(uri, Danbooru.config.httparty_options.merge(body: json))
-          if resp.success?
-            resp.body.to_s.scan(/\d+/).map(&:to_i)
           else
-            raise "HTTP error code: #{resp.code} #{resp.message}"
+            SavedSearch.delay(queue: "default").populate(query)
           end
         end
+        post_ids.to_a.sort
+      end
+    end
+  end
+
+  concerning :Labels do
+    class_methods do
+      def normalize_label(label)
+        label.
+          to_s.
+          strip.
+          downcase.
+          gsub(/[[:space:]]/, "_")
+      end
+
+      def search_labels(user_id, params)
+        labels = labels_for(user_id)
+
+        if params[:label].present?
+          query = Regexp.escape(params[:label]).gsub("\\*", ".*")
+          query = ".*#{query}.*" unless query.include?("*")
+          query = /\A#{query}\z/
+          labels = labels.grep(query)
+        end
+
+        labels
+      end
+
+      def self.labels_for(user_id)
+        Cache.get(cache_key(user_id)) do
+          SavedSearch.
+            where(user_id: user_id).
+            order("label").
+            pluck(Arel.sql("distinct unnest(labels) as label"))
+        end
       end
     end
   end
 
-  include ListbooruMethods
+  concerning :Search do
+    class_methods do
+      def populate(query)
+        CurrentUser.as_system do
+          query_hash = Cache.hash(query)
+          redis_key = "search:#{query_hash}"
+          redis = Redis.new(url: Danbooru.config.redis_url)
+          return if redis.exists(redis_key)
+          post_ids = Post.tag_match(query, true).limit(QUERY_LIMIT).pluck(:id)
+          redis.sadd(redis_key, post_ids)
+          redis.expire(redis_key, REDIS_EXPIRY)
+        end
+      rescue
+        # swallow
+      end
+    end
+  end
 
   attr_accessor :disable_labels
   belongs_to :user
-  validates :query, :presence => true
+  validates :query, presence: true
   validate :validate_count
   before_create :update_user_on_create
   after_destroy :update_user_on_destroy
-  after_save {|rec| Cache.delete(SavedSearch.cache_key(rec.user_id))}
-  after_destroy {|rec| Cache.delete(SavedSearch.cache_key(rec.user_id))}
   before_validation :normalize
   scope :labeled, ->(label) { where("labels @> string_to_array(?, '~~~~')", label)}
 
-  def self.normalize_label(label)
+  def self.queries_for(user_id, label: nil, options: {})
-    label.to_s.strip.downcase.gsub(/[[:space:]]/, "_")
+    SavedSearch.
-  end
+      where(user_id: user_id).
-
+      tap {|arel| label ? arel.labeled(label) : arel}.
-  def self.search_labels(user_id, params)
+      pluck(:query).
-    labels = labels_for(user_id)
+      map {|x| Tag.normalize_query(x, sort: true)}.
-
+      sort.
-    if params[:label].present?
+      uniq
-      query = Regexp.escape(params[:label]).gsub("\\*", ".*")
-      query = ".*#{query}.*" unless query.include?("*")
-      query = /\A#{query}\z/
-      labels = labels.grep(query)
-    end
-
-    labels
-  end
-
-  def self.labels_for(user_id)
-    Cache.get(cache_key(user_id)) do
-      SavedSearch.where(user_id: user_id).order("label").pluck(Arel.sql("distinct unnest(labels) as label"))
-    end
-  end
-
-  def self.cache_key(user_id)
-    "ss-labels-#{user_id}"
-  end
-
-  def self.queries_for(user_id, label = nil, options = {})
-    if label
-      SavedSearch.where(user_id: user_id).labeled(label).map(&:normalized_query).sort.uniq
-    else
-      SavedSearch.where(user_id: user_id).map(&:normalized_query).sort.uniq
-    end
   end
 
   def normalized_query
@@ -95,7 +106,9 @@ class SavedSearch < ApplicationRecord
 
   def normalize
     self.query = Tag.normalize_query(query, sort: false)
-    self.labels = labels.map {|x| SavedSearch.normalize_label(x)}.reject {|x| x.blank?}
+    self.labels = labels.
+      map {|x| SavedSearch.normalize_label(x)}.
+      reject {|x| x.blank?}
   end
 
   def validate_count
@@ -106,13 +119,13 @@ class SavedSearch < ApplicationRecord
 
   def update_user_on_create
     if !user.has_saved_searches?
-      user.update_attribute(:has_saved_searches, true)
+      user.update(has_saved_searches: true)
     end
   end
 
   def update_user_on_destroy
     if user.saved_searches.count == 0
-      user.update_attribute(:has_saved_searches, false)
+      user.update(has_saved_searches: false)
     end
   end
 

config/danbooru_default_config.rb

@@ -796,6 +796,9 @@ module Danbooru
 
     def recommender_key
     end
+
+    def redis_url
+    end
   end
 
   class EnvironmentConfiguration