Merge pull request #2975 from evazion/feat-rails-cache

Use Rails.cache as caching backend.

app/logical/cache.rb

@@ -1,83 +1,46 @@
 class Cache
-  def self.incr(key)
+  def self.get_multi(keys, prefix)
-    MEMCACHE.incr(key)
+    sanitized_key_to_key_hash = keys.map do |key|
-    ActiveRecord::Base.logger.debug('MemCache Incr %s' % [key])
+      ["#{prefix}:#{Cache.sanitize(key)}", key]
+    end.to_h
+
+    sanitized_keys = sanitized_key_to_key_hash.keys
+    sanitized_key_to_value_hash = Rails.cache.fetch_multi(*sanitized_keys) do |sanitized_key|
+      key = sanitized_key_to_key_hash[sanitized_key]
+      yield key
     end
 
-  def self.decr(key)
+    keys_to_values_hash = sanitized_key_to_value_hash.transform_keys(&sanitized_key_to_key_hash)
-    MEMCACHE.decr(key)
+    keys_to_values_hash
-    ActiveRecord::Base.logger.debug('MemCache Decr %s' % [key])
   end
 
-  def self.get_multi(keys, prefix, expiry_in_seconds = nil)
+  def self.get(key, expiry_in_seconds = nil, &block)
-    key_to_sanitized_key_hash = keys.inject({}) do |hash, x|
+    Rails.cache.fetch(key, expires_in: expiry_in_seconds, &block)
-      hash[x] = "#{prefix}:#{Cache.sanitize(x)}"
-      hash
-    end
-    start_time = Time.now
-    sanitized_key_to_value_hash = MEMCACHE.get_multi(key_to_sanitized_key_hash.values)
-    elapsed = Time.now - start_time
-    {}.tap do |result_hash|
-      key_to_sanitized_key_hash.each do |key, sanitized_key|
-        if sanitized_key_to_value_hash.has_key?(sanitized_key)
-          result_hash[key] = sanitized_key_to_value_hash[sanitized_key]
-        else
-          result_hash[key] = yield(key)
-          Cache.put(sanitized_key, result_hash[key], expiry_in_seconds)
-        end
-      end
-
-      ActiveRecord::Base.logger.debug('MemCache Multi-Get (%0.6f)  %s' % [elapsed, keys.join(",")])
-    end
-  end
-
-  def self.get(key, expiry_in_seconds = nil)
-    start_time = Time.now
-    value = MEMCACHE.get key
-    elapsed = Time.now - start_time
-    if expiry_in_seconds
-      expiry = expiry_in_seconds
-    else
-      expiry = 0
-    end
-    ActiveRecord::Base.logger.debug('MemCache Get (%0.6f)  %s -> %s' % [elapsed, key, value])
-    if value.nil? and block_given? then
-      value = yield
-      MEMCACHE.set key, value, expiry
-    end
-    value
   rescue => err
-    ActiveRecord::Base.logger.debug "MemCache Error: #{err.message}"
+    Rails.logger.debug { "MemCache Error: #{err.message}" }
     nil
   end
 
   def self.put(key, value, expiry_in_seconds = nil)
-    start_time = Time.now
+    Rails.cache.write(key, value, expires_in: expiry_in_seconds)
-    if expiry_in_seconds
-      expiry = expiry_in_seconds
-    else
-      expiry = 0
-    end      
-    MEMCACHE.set key, value, expiry
-    elapsed = Time.now - start_time
-    ActiveRecord::Base.logger.debug('MemCache Set (%0.6f)  %s -> %s' % [elapsed, key, value])
     value
   rescue => err
-    ActiveRecord::Base.logger.debug "MemCache Error: #{err.message}"
+    Rails.logger.debug { "MemCache Error: #{err.message}" }
     nil
   end
 
-  def self.delete(key, delay = nil)
+  def self.delete(key)
-    start_time = Time.now
+    Rails.cache.delete(key)
-    MEMCACHE.delete key
-    elapsed = Time.now - start_time
-    ActiveRecord::Base.logger.debug('MemCache Delete (%0.6f)  %s' % [elapsed, key])
     nil
   rescue => err
-    ActiveRecord::Base.logger.debug "MemCache Error: #{err.message}"
+    Rails.logger.debug { "MemCache Error: #{err.message}" }
     nil
   end
 
+  def self.clear
+    Rails.cache.clear
+  end
+
   def self.sanitize(key)
     key.gsub(/\W/) {|x| "%#{x.ord}"}.slice(0, 230)
   end

config/initializers/z_dalli.rb

@@ -1,9 +1,14 @@
-unless defined?(MEMCACHE)
+Rails.application.configure do
-  MEMCACHE = Dalli::Client.new(Danbooru.config.memcached_servers, :namespace => Danbooru.config.app_name.gsub(/[^A-Za-z0-9]/, "_"))
-end
-
   begin
-  MEMCACHE.get("x")
+    if Rails.env.test?
+      config.cache_store = :memory_store, { size: 32.megabytes }
+      Rails.cache = ActiveSupport::Cache.lookup_store(Rails.application.config.cache_store)
+    else
+      config.cache_store = :dalli_store, Danbooru.config.memcached_servers, { namespace: Danbooru.config.safe_app_name }
+      Rails.cache = ActiveSupport::Cache.lookup_store(Rails.application.config.cache_store)
+
+      Rails.cache.dalli.alive!
+    end
   rescue Dalli::RingError => e
     puts "-" * 40
     puts "WARNING! MEMCACHE SERVER NOT FOUND! You will experience performance degradation."
@@ -14,11 +19,8 @@ rescue Dalli::RingError => e
     end
     puts "-- END STACKTRACE --"
     puts "-" * 40
-  MEMCACHE = MemcacheMock.new
-end
 
-if Rails.env.production?
+    config.cache_store = :memory_store, { size: 32.megabytes }
-  Rails.application.configure do
+    Rails.cache = ActiveSupport::Cache.lookup_store(Rails.application.config.cache_store)
-    config.cache_store = :dalli_store, Danbooru.config.memcached_servers
   end
 end

test/functional/moderator/invitations_controller_test.rb

@@ -7,7 +7,6 @@ module Moderator
         @mod = FactoryGirl.create(:moderator_user)
         CurrentUser.user = @mod
         CurrentUser.ip_addr = "127.0.0.1"
-        MEMCACHE.flush_all
 
         @user_1 = FactoryGirl.create(:user)
         @user_2 = FactoryGirl.create(:user, :inviter_id => @mod.id)

test/functional/moderator/ip_addrs_controller_test.rb

@@ -8,7 +8,6 @@ module Moderator
         CurrentUser.user = @user
         CurrentUser.ip_addr = "127.0.0.1"
         FactoryGirl.create(:comment)
-        MEMCACHE.flush_all
       end
 
       should "find by ip addr" do

test/functional/posts_controller_test.rb

@@ -8,7 +8,6 @@ class PostsControllerTest < ActionController::TestCase
       CurrentUser.user = @user
       CurrentUser.ip_addr = "127.0.0.1"
       @post = FactoryGirl.create(:post, :uploader_id => @user.id, :tag_string => "aaaa")
-      MEMCACHE.flush_all
     end
 
     teardown do

test/functional/tag_alias_corrections_controller_test.rb

@@ -6,7 +6,6 @@ class TagAliasCorrectionsControllerTest < ActionController::TestCase
       @admin = FactoryGirl.create(:admin_user)
       CurrentUser.user = @admin
       CurrentUser.ip_addr = "127.0.0.1"
-      MEMCACHE.flush_all
       @tag_alias = FactoryGirl.create(:tag_alias, :antecedent_name => "aaa", :consequent_name => "bbb")
     end
 

test/functional/tag_alias_requests_controller_test.rb

@@ -6,7 +6,6 @@ class TagAliasRequestsControllerTest < ActionController::TestCase
       @user = FactoryGirl.create(:user)
       CurrentUser.user = @user
       CurrentUser.ip_addr = "127.0.0.1"
-      MEMCACHE.flush_all
     end
 
     teardown do

test/functional/tag_aliases_controller_test.rb

@@ -6,7 +6,6 @@ class TagAliasesControllerTest < ActionController::TestCase
       @user = FactoryGirl.create(:admin_user)
       CurrentUser.user = @user
       CurrentUser.ip_addr = "127.0.0.1"
-      MEMCACHE.flush_all
     end
 
     teardown do

test/functional/tag_implication_requests_controller_test.rb

@@ -8,7 +8,6 @@ class TagImplicationRequestsControllerTest < ActionController::TestCase
       end
       CurrentUser.user = @user
       CurrentUser.ip_addr = "127.0.0.1"
-      MEMCACHE.flush_all
     end
 
     teardown do

test/functional/tag_implications_controller_test.rb

@@ -6,7 +6,6 @@ class TagImplicationsControllerTest < ActionController::TestCase
       @user = FactoryGirl.create(:admin_user)
       CurrentUser.user = @user
       CurrentUser.ip_addr = "127.0.0.1"
-      MEMCACHE.flush_all
     end
 
     teardown do

test/test_helper.rb

@@ -21,12 +21,12 @@ Shoulda::Matchers.configure do |config|
   end
 end
 
-if defined?(MEMCACHE)
-  Object.send(:remove_const, :MEMCACHE)
-end
-
 class ActiveSupport::TestCase
   include PostArchiveTestHelper
+
+  teardown do
+    Cache.clear
+  end
 end
 
 class ActionController::TestCase
@@ -41,9 +41,12 @@ class ActionController::TestCase
     __send__(http_method, action, params, session.merge(:user_id => @users[role].id))
     assert_redirected_to(new_sessions_path)
   end
+
+  teardown do
+    Cache.clear
+  end
 end
 
-MEMCACHE = MemcacheMock.new
 Delayed::Worker.delay_jobs = false
 
 require "helpers/reportbooru_helper"

test/unit/artist_test.rb

@@ -18,7 +18,6 @@ class ArtistTest < ActiveSupport::TestCase
       user = Timecop.travel(1.month.ago) {FactoryGirl.create(:user)}
       CurrentUser.user = user
       CurrentUser.ip_addr = "127.0.0.1"
-      MEMCACHE.flush_all
     end
 
     teardown do

test/unit/artist_url_test.rb

@@ -3,7 +3,6 @@ require 'test_helper'
 class ArtistUrlTest < ActiveSupport::TestCase
   context "An artist url" do
     setup do
-      MEMCACHE.flush_all
       CurrentUser.user = FactoryGirl.create(:user)
       CurrentUser.ip_addr = "127.0.0.1"
     end

test/unit/comment_test.rb

@@ -6,7 +6,6 @@ class CommentTest < ActiveSupport::TestCase
       user = FactoryGirl.create(:user)
       CurrentUser.user = user
       CurrentUser.ip_addr = "127.0.0.1"
-      MEMCACHE.flush_all
     end
 
     teardown do

test/unit/dmail_test.rb

@@ -3,7 +3,6 @@ require 'test_helper'
 class DmailTest < ActiveSupport::TestCase
   context "A dmail" do
     setup do
-      MEMCACHE.flush_all
       @user = FactoryGirl.create(:user)
       CurrentUser.user = @user
       CurrentUser.ip_addr = "1.2.3.4"

test/unit/favorite_test.rb

@@ -5,7 +5,6 @@ class FavoriteTest < ActiveSupport::TestCase
     user = FactoryGirl.create(:user)
     CurrentUser.user = user
     CurrentUser.ip_addr = "127.0.0.1"
-    MEMCACHE.flush_all
     Favorite # need to force loading the favorite model
   end
 

test/unit/ip_ban_test.rb

@@ -5,7 +5,6 @@ class IpBanTest < ActiveSupport::TestCase
     @user = FactoryGirl.create(:user)
     CurrentUser.user = @user
     CurrentUser.ip_addr = "127.0.0.1"
-    MEMCACHE.flush_all
     Danbooru.config.stubs(:member_comment_time_threshold).returns(1.week.from_now)
   end
 

test/unit/janitor_trial_test.rb

@@ -7,7 +7,6 @@ class JanitorTrialTest < ActiveSupport::TestCase
       @user = FactoryGirl.create(:user)
       CurrentUser.user = @admin
       CurrentUser.ip_addr = "127.0.0.1"
-      MEMCACHE.flush_all
     end
 
     teardown do

test/unit/moderator/ip_addr_search_test.rb

@@ -9,7 +9,6 @@ module Moderator
         CurrentUser.ip_addr = "127.0.0.1"
         Danbooru.config.stubs(:member_comment_time_threshold).returns(1.week.from_now)
         FactoryGirl.create(:comment)
-        MEMCACHE.flush_all
       end
 
       teardown do

test/unit/note_test.rb

@@ -6,7 +6,6 @@ class NoteTest < ActiveSupport::TestCase
       @user = FactoryGirl.create(:user)
       CurrentUser.user = @user
       CurrentUser.ip_addr = "127.0.0.1"
-      MEMCACHE.flush_all
     end
 
     teardown do

test/unit/pool_test.rb

@@ -13,7 +13,6 @@ class PoolTest < ActiveSupport::TestCase
     end
 
     CurrentUser.ip_addr = "127.0.0.1"
-    MEMCACHE.flush_all
 
     mock_pool_archive_service!
     PoolArchive.sqs_service.stubs(:merge?).returns(false)

test/unit/post_appeal_test.rb

@@ -6,7 +6,6 @@ class PostAppealTest < ActiveSupport::TestCase
       @alice = FactoryGirl.create(:user)
       CurrentUser.user = @alice
       CurrentUser.ip_addr = "127.0.0.1"
-      MEMCACHE.flush_all
       Danbooru.config.stubs(:max_appeals_per_day).returns(5)
     end
 

test/unit/post_archive_test.rb

@@ -8,7 +8,6 @@ class PostArchiveTest < ActiveSupport::TestCase
       end
       CurrentUser.user = @user
       CurrentUser.ip_addr = "127.0.0.1"
-      MEMCACHE.flush_all
     end
 
     teardown do

test/unit/post_disapproval_test.rb

@@ -6,7 +6,6 @@ class PostDisapprovalTest < ActiveSupport::TestCase
       @alice = FactoryGirl.create(:moderator_user)
       CurrentUser.user = @alice
       CurrentUser.ip_addr = "127.0.0.1"
-      MEMCACHE.flush_all
     end
 
     teardown do

test/unit/post_flag_test.rb

@@ -8,7 +8,6 @@ class PostFlagTest < ActiveSupport::TestCase
       end
       CurrentUser.user = @alice
       CurrentUser.ip_addr = "127.0.0.1"
-      MEMCACHE.flush_all
       @post = FactoryGirl.create(:post, :tag_string => "aaa")
     end
 

test/unit/post_pruner_test.rb

@@ -5,7 +5,6 @@ class PostPrunerTest < ActiveSupport::TestCase
     @user = FactoryGirl.create(:admin_user)
     CurrentUser.user = @user
     CurrentUser.ip_addr = "127.0.0.1"
-    MEMCACHE.flush_all
 
     Timecop.travel(2.weeks.ago) do
       @flagger = FactoryGirl.create(:gold_user)

test/unit/post_sets/favorite_test.rb

@@ -7,7 +7,6 @@ module PostSets
         @user = FactoryGirl.create(:user)
         CurrentUser.user = @user
         CurrentUser.ip_addr = "127.0.0.1"
-        MEMCACHE.flush_all
 
         @post_1 = FactoryGirl.create(:post)
         @post_2 = FactoryGirl.create(:post)

test/unit/post_sets/intro_test.rb

@@ -8,7 +8,6 @@ module PostSets
         @user = FactoryGirl.create(:user)
         CurrentUser.user = @user
         CurrentUser.ip_addr = "127.0.0.1"
-        MEMCACHE.flush_all
 
         @post_1 = FactoryGirl.create(:post, :tag_string => "a")
         @post_2 = FactoryGirl.create(:post, :tag_string => "b")

test/unit/post_sets/pool_test.rb

@@ -10,7 +10,6 @@ module PostSets
         @user = FactoryGirl.create(:user)
         CurrentUser.user = @user
         CurrentUser.ip_addr = "127.0.0.1"
-        MEMCACHE.flush_all
 
         mock_pool_archive_service!
         start_pool_archive_transaction

test/unit/post_sets/post_test.rb

@@ -8,7 +8,6 @@ module PostSets
         @user = FactoryGirl.create(:user)
         CurrentUser.user = @user
         CurrentUser.ip_addr = "127.0.0.1"
-        MEMCACHE.flush_all
 
         @post_1 = FactoryGirl.create(:post, :tag_string => "a")
         @post_2 = FactoryGirl.create(:post, :tag_string => "b")

test/unit/post_test.rb

@@ -18,7 +18,6 @@ class PostTest < ActiveSupport::TestCase
     end
     CurrentUser.user = @user
     CurrentUser.ip_addr = "127.0.0.1"
-    MEMCACHE.flush_all
     mock_saved_search_service!
   end
 

test/unit/post_version_test.rb

@@ -8,7 +8,6 @@ class PostVersionTest < ActiveSupport::TestCase
       end
       CurrentUser.user = @user
       CurrentUser.ip_addr = "127.0.0.1"
-      MEMCACHE.flush_all
     end
 
     teardown do

test/unit/post_vote_test.rb

@@ -8,7 +8,6 @@ class PostVoteTest < ActiveSupport::TestCase
     @user = FactoryGirl.create(:user)
     CurrentUser.user = @user
     CurrentUser.ip_addr = "127.0.0.1"
-    MEMCACHE.flush_all
 
     @post = FactoryGirl.create(:post)
   end

test/unit/related_tag_calculator_test.rb

@@ -5,7 +5,6 @@ class RelatedTagCalculatorTest < ActiveSupport::TestCase
     user = FactoryGirl.create(:user)
     CurrentUser.user = user
     CurrentUser.ip_addr = "127.0.0.1"
-    MEMCACHE.flush_all
   end
 
   teardown do

test/unit/related_tag_query_test.rb

@@ -5,7 +5,6 @@ class RelatedTagQueryTest < ActiveSupport::TestCase
     user = FactoryGirl.create(:user)
     CurrentUser.user = user
     CurrentUser.ip_addr = "127.0.0.1"
-    MEMCACHE.flush_all
   end
 
   context "a related tag query without a category constraint" do

test/unit/saved_search_test.rb

@@ -41,10 +41,6 @@ class SavedSearchTest < ActiveSupport::TestCase
   end
 
   context "Fetching the post ids for a search" do
-    setup do
-      MEMCACHE.expects(:get).returns(nil)
-    end
-
     teardown do
       FakeWeb.clean_registry
     end

test/unit/tag_alias_correction_test.rb

@@ -6,13 +6,11 @@ class TagAliasCorrectionTest < ActiveSupport::TestCase
       @mod = FactoryGirl.create(:moderator_user)
       CurrentUser.user = @mod
       CurrentUser.ip_addr = "127.0.0.1"
-      MEMCACHE.flush_all
       @post = FactoryGirl.create(:post, :tag_string => "aaa")
       @tag_alias = FactoryGirl.create(:tag_alias, :antecedent_name => "aaa", :consequent_name => "bbb")
     end
 
     teardown do
-      MEMCACHE.flush_all
       CurrentUser.user = nil
       CurrentUser.ip_addr = nil
     end

test/unit/tag_alias_request_test.rb

@@ -6,11 +6,9 @@ class TagAliasRequestTest < ActiveSupport::TestCase
       @user = FactoryGirl.create(:user)
       CurrentUser.user = @user
       CurrentUser.ip_addr = "127.0.0.1"
-      MEMCACHE.flush_all
     end
 
     teardown do
-      MEMCACHE.flush_all
       CurrentUser.user = nil
       CurrentUser.ip_addr = nil
     end

test/unit/tag_alias_test.rb

@@ -11,7 +11,6 @@ class TagAliasTest < ActiveSupport::TestCase
         CurrentUser.user = user
       end
       CurrentUser.ip_addr = "127.0.0.1"
-      MEMCACHE.flush_all
       mock_saved_search_service!
     end
 

test/unit/tag_implication_request_test.rb

@@ -6,11 +6,9 @@ class TagImplicationRequestTest < ActiveSupport::TestCase
       @user = FactoryGirl.create(:user)
       CurrentUser.user = @user
       CurrentUser.ip_addr = "127.0.0.1"
-      MEMCACHE.flush_all
     end
 
     teardown do
-      MEMCACHE.flush_all
       CurrentUser.user = nil
       CurrentUser.ip_addr = nil
     end

test/unit/tag_implication_test.rb

@@ -7,7 +7,6 @@ class TagImplicationTest < ActiveSupport::TestCase
       CurrentUser.user = user
       CurrentUser.ip_addr = "127.0.0.1"
       @user = FactoryGirl.create(:user)
-      MEMCACHE.flush_all
     end
 
     teardown do

test/unit/tag_subscription_test.rb

@@ -5,7 +5,6 @@ class TagSubscriptionTest < ActiveSupport::TestCase
     user = FactoryGirl.create(:user)
     CurrentUser.user = user
     CurrentUser.ip_addr = "127.0.0.1"
-    MEMCACHE.flush_all
   end
 
   teardown do

test/unit/tag_test.rb

@@ -5,7 +5,6 @@ class TagTest < ActiveSupport::TestCase
     user = FactoryGirl.create(:builder_user)
     CurrentUser.user = user
     CurrentUser.ip_addr = "127.0.0.1"
-    MEMCACHE.flush_all
   end
 
   teardown do
@@ -32,10 +31,6 @@ class TagTest < ActiveSupport::TestCase
   end
 
   context "A tag category fetcher" do
-    setup do
-      MEMCACHE.flush_all
-    end
-
     should "fetch for a single tag" do
       FactoryGirl.create(:artist_tag, :name => "test")
       assert_equal(Tag.categories.artist, Tag.category_for("test"))
@@ -57,10 +52,6 @@ class TagTest < ActiveSupport::TestCase
   end
 
   context "A tag category mapping" do
-    setup do
-      MEMCACHE.flush_all
-    end
-
     should "exist" do
       assert_nothing_raised {Tag.categories}
     end
@@ -97,10 +88,6 @@ class TagTest < ActiveSupport::TestCase
   end
 
   context "A tag" do
-    setup do
-      MEMCACHE.flush_all
-    end
-
     should "be lockable by a moderator" do
       @tag = FactoryGirl.create(:tag)
       @tag.update_attributes({:is_locked => true}, :as => :moderator)
@@ -123,11 +110,11 @@ class TagTest < ActiveSupport::TestCase
     should "reset its category after updating" do
       tag = FactoryGirl.create(:artist_tag)
       tag.update_category_cache_for_all
-      assert_equal(Tag.categories.artist, MEMCACHE.get("tc:#{tag.name}"))
+      assert_equal(Tag.categories.artist, Cache.get("tc:#{tag.name}"))
 
       tag.update_attribute(:category, Tag.categories.copyright)
       tag.update_category_cache_for_all
-      assert_equal(Tag.categories.copyright, MEMCACHE.get("tc:#{tag.name}"))
+      assert_equal(Tag.categories.copyright, Cache.get("tc:#{tag.name}"))
     end
   end
 

test/unit/upload_test.rb

@@ -17,7 +17,6 @@ class UploadTest < ActiveSupport::TestCase
       user = FactoryGirl.create(:contributor_user)
       CurrentUser.user = user
       CurrentUser.ip_addr = "127.0.0.1"
-      MEMCACHE.flush_all
     end
 
     teardown do

test/unit/user_feedback_test.rb

@@ -4,7 +4,6 @@ class UserFeedbackTest < ActiveSupport::TestCase
   context "A user's feedback" do
     setup do
       CurrentUser.ip_addr = "127.0.0.1"
-      MEMCACHE.flush_all
     end
 
     teardown do

test/unit/user_test.rb

@@ -6,7 +6,6 @@ class UserTest < ActiveSupport::TestCase
       @user = FactoryGirl.create(:user)
       CurrentUser.user = @user
       CurrentUser.ip_addr = "127.0.0.1"
-      MEMCACHE.flush_all
     end
 
     teardown do

test/unit/wiki_page_test.rb

@@ -2,7 +2,6 @@ require 'test_helper'
 
 class WikiPageTest < ActiveSupport::TestCase
   setup do
-    MEMCACHE.flush_all
     CurrentUser.ip_addr = "127.0.0.1"
   end