jaculabilis/danbooru
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #2975 from evazion/feat-rails-cache

Browse Source 
Use Rails.cache as caching backend.
This commit is contained in:
Albert Yi
2017-04-17 13:00:52 -07:00
committed by GitHub
parent 6d693799a0 c0e5c400a7
commit 24dd7b05c1
47 changed files with 57 additions and 151 deletions
Download Patch File Download Diff File Expand all files Collapse all files

87
app/logical/cache.rb
View File

@@ -1,83 +1,46 @@
class Cache
def self.incr(key)
MEMCACHE.incr(key)
ActiveRecord::Base.logger.debug('MemCache Incr %s' % [key])
def self.get_multi(keys, prefix)
sanitized_key_to_key_hash = keys.map do |key|
["#{prefix}:#{Cache.sanitize(key)}", key]
end.to_h
sanitized_keys = sanitized_key_to_key_hash.keys
sanitized_key_to_value_hash = Rails.cache.fetch_multi(*sanitized_keys) do |sanitized_key|
key = sanitized_key_to_key_hash[sanitized_key]
yield key
end
keys_to_values_hash = sanitized_key_to_value_hash.transform_keys(&sanitized_key_to_key_hash)
keys_to_values_hash
end
def self.decr(key)
MEMCACHE.decr(key)
ActiveRecord::Base.logger.debug('MemCache Decr %s' % [key])
end
def self.get_multi(keys, prefix, expiry_in_seconds = nil)
key_to_sanitized_key_hash = keys.inject({}) do |hash, x|
hash[x] = "#{prefix}:#{Cache.sanitize(x)}"
hash
end
start_time = Time.now
sanitized_key_to_value_hash = MEMCACHE.get_multi(key_to_sanitized_key_hash.values)
elapsed = Time.now - start_time
{}.tap do |result_hash|
key_to_sanitized_key_hash.each do |key, sanitized_key|
if sanitized_key_to_value_hash.has_key?(sanitized_key)
result_hash[key] = sanitized_key_to_value_hash[sanitized_key]
else
result_hash[key] = yield(key)
Cache.put(sanitized_key, result_hash[key], expiry_in_seconds)
end
end
ActiveRecord::Base.logger.debug('MemCache Multi-Get (%0.6f) %s' % [elapsed, keys.join(",")])
end
end
def self.get(key, expiry_in_seconds = nil)
start_time = Time.now
value = MEMCACHE.get key
elapsed = Time.now - start_time
if expiry_in_seconds
expiry = expiry_in_seconds
else
expiry = 0
end
ActiveRecord::Base.logger.debug('MemCache Get (%0.6f) %s -> %s' % [elapsed, key, value])
if value.nil? and block_given? then
value = yield
MEMCACHE.set key, value, expiry
end
value
def self.get(key, expiry_in_seconds = nil, &block)
Rails.cache.fetch(key, expires_in: expiry_in_seconds, &block)
rescue => err
ActiveRecord::Base.logger.debug "MemCache Error: #{err.message}"
Rails.logger.debug { "MemCache Error: #{err.message}" }
nil
end
def self.put(key, value, expiry_in_seconds = nil)
start_time = Time.now
if expiry_in_seconds
expiry = expiry_in_seconds
else
expiry = 0
end
MEMCACHE.set key, value, expiry
elapsed = Time.now - start_time
ActiveRecord::Base.logger.debug('MemCache Set (%0.6f) %s -> %s' % [elapsed, key, value])
Rails.cache.write(key, value, expires_in: expiry_in_seconds)
value
rescue => err
ActiveRecord::Base.logger.debug "MemCache Error: #{err.message}"
Rails.logger.debug { "MemCache Error: #{err.message}" }
nil
end
def self.delete(key, delay = nil)
start_time = Time.now
MEMCACHE.delete key
elapsed = Time.now - start_time
ActiveRecord::Base.logger.debug('MemCache Delete (%0.6f) %s' % [elapsed, key])
def self.delete(key)
Rails.cache.delete(key)
nil
rescue => err
ActiveRecord::Base.logger.debug "MemCache Error: #{err.message}"
Rails.logger.debug { "MemCache Error: #{err.message}" }
nil
end
def self.clear
Rails.cache.clear
end
def self.sanitize(key)
key.gsub(/\W/) {|x| "%#{x.ord}"}.slice(0, 230)
end

42
config/initializers/z_dalli.rb
View File

@@ -1,24 +1,26 @@
unless defined?(MEMCACHE)
MEMCACHE = Dalli::Client.new(Danbooru.config.memcached_servers, :namespace => Danbooru.config.app_name.gsub(/[^A-Za-z0-9]/, "_"))
end
Rails.application.configure do
begin
if Rails.env.test?
config.cache_store = :memory_store, { size: 32.megabytes }
Rails.cache = ActiveSupport::Cache.lookup_store(Rails.application.config.cache_store)
else
config.cache_store = :dalli_store, Danbooru.config.memcached_servers, { namespace: Danbooru.config.safe_app_name }
Rails.cache = ActiveSupport::Cache.lookup_store(Rails.application.config.cache_store)
begin
MEMCACHE.get("x")
rescue Dalli::RingError => e
puts "-" * 40
puts "WARNING! MEMCACHE SERVER NOT FOUND! You will experience performance degradation."
puts e.to_s
puts "-- BEGIN STACKTRACE --"
e.backtrace.each do |line|
puts line
end
puts "-- END STACKTRACE --"
puts "-" * 40
MEMCACHE = MemcacheMock.new
end
Rails.cache.dalli.alive!
end
rescue Dalli::RingError => e
puts "-" * 40
puts "WARNING! MEMCACHE SERVER NOT FOUND! You will experience performance degradation."
puts e.to_s
puts "-- BEGIN STACKTRACE --"
e.backtrace.each do |line|
puts line
end
puts "-- END STACKTRACE --"
puts "-" * 40
if Rails.env.production?
Rails.application.configure do
config.cache_store = :dalli_store, Danbooru.config.memcached_servers
config.cache_store = :memory_store, { size: 32.megabytes }
Rails.cache = ActiveSupport::Cache.lookup_store(Rails.application.config.cache_store)
end
end

1
test/functional/moderator/invitations_controller_test.rb
View File

@@ -7,7 +7,6 @@ module Moderator
@mod = FactoryGirl.create(:moderator_user)
CurrentUser.user = @mod
CurrentUser.ip_addr = "127.0.0.1"
MEMCACHE.flush_all
@user_1 = FactoryGirl.create(:user)
@user_2 = FactoryGirl.create(:user, :inviter_id => @mod.id)

1
test/functional/moderator/ip_addrs_controller_test.rb
View File

@@ -8,7 +8,6 @@ module Moderator
CurrentUser.user = @user
CurrentUser.ip_addr = "127.0.0.1"
FactoryGirl.create(:comment)
MEMCACHE.flush_all
end
should "find by ip addr" do

1
test/functional/posts_controller_test.rb
View File

@@ -8,7 +8,6 @@ class PostsControllerTest < ActionController::TestCase
CurrentUser.user = @user
CurrentUser.ip_addr = "127.0.0.1"
@post = FactoryGirl.create(:post, :uploader_id => @user.id, :tag_string => "aaaa")
MEMCACHE.flush_all
end
teardown do

1
test/functional/tag_alias_corrections_controller_test.rb
View File

@@ -6,7 +6,6 @@ class TagAliasCorrectionsControllerTest < ActionController::TestCase
@admin = FactoryGirl.create(:admin_user)
CurrentUser.user = @admin
CurrentUser.ip_addr = "127.0.0.1"
MEMCACHE.flush_all
@tag_alias = FactoryGirl.create(:tag_alias, :antecedent_name => "aaa", :consequent_name => "bbb")
end

1
test/functional/tag_alias_requests_controller_test.rb
View File

@@ -6,7 +6,6 @@ class TagAliasRequestsControllerTest < ActionController::TestCase
@user = FactoryGirl.create(:user)
CurrentUser.user = @user
CurrentUser.ip_addr = "127.0.0.1"
MEMCACHE.flush_all
end
teardown do

1
test/functional/tag_aliases_controller_test.rb
View File

@@ -6,7 +6,6 @@ class TagAliasesControllerTest < ActionController::TestCase
@user = FactoryGirl.create(:admin_user)
CurrentUser.user = @user
CurrentUser.ip_addr = "127.0.0.1"
MEMCACHE.flush_all
end
teardown do

1
test/functional/tag_implication_requests_controller_test.rb
View File

@@ -8,7 +8,6 @@ class TagImplicationRequestsControllerTest < ActionController::TestCase
end
CurrentUser.user = @user
CurrentUser.ip_addr = "127.0.0.1"
MEMCACHE.flush_all
end
teardown do

1
test/functional/tag_implications_controller_test.rb
View File

@@ -6,7 +6,6 @@ class TagImplicationsControllerTest < ActionController::TestCase
@user = FactoryGirl.create(:admin_user)
CurrentUser.user = @user
CurrentUser.ip_addr = "127.0.0.1"
MEMCACHE.flush_all
end
teardown do

13
test/test_helper.rb
View File

@@ -21,12 +21,12 @@ Shoulda::Matchers.configure do |config|
end
end
if defined?(MEMCACHE)
Object.send(:remove_const, :MEMCACHE)
end
class ActiveSupport::TestCase
include PostArchiveTestHelper
teardown do
Cache.clear
end
end
class ActionController::TestCase
@@ -41,9 +41,12 @@ class ActionController::TestCase
__send__(http_method, action, params, session.merge(:user_id => @users[role].id))
assert_redirected_to(new_sessions_path)
end
teardown do
Cache.clear
end
end
MEMCACHE = MemcacheMock.new
Delayed::Worker.delay_jobs = false
require "helpers/reportbooru_helper"

1
test/unit/artist_test.rb
View File

@@ -18,7 +18,6 @@ class ArtistTest < ActiveSupport::TestCase
user = Timecop.travel(1.month.ago) {FactoryGirl.create(:user)}
CurrentUser.user = user
CurrentUser.ip_addr = "127.0.0.1"
MEMCACHE.flush_all
end
teardown do

1
test/unit/artist_url_test.rb
View File

@@ -3,7 +3,6 @@ require 'test_helper'
class ArtistUrlTest < ActiveSupport::TestCase
context "An artist url" do
setup do
MEMCACHE.flush_all
CurrentUser.user = FactoryGirl.create(:user)
CurrentUser.ip_addr = "127.0.0.1"
end

1
test/unit/comment_test.rb
View File

@@ -6,7 +6,6 @@ class CommentTest < ActiveSupport::TestCase
user = FactoryGirl.create(:user)
CurrentUser.user = user
CurrentUser.ip_addr = "127.0.0.1"
MEMCACHE.flush_all
end
teardown do

1
test/unit/dmail_test.rb
View File

@@ -3,7 +3,6 @@ require 'test_helper'
class DmailTest < ActiveSupport::TestCase
context "A dmail" do
setup do
MEMCACHE.flush_all
@user = FactoryGirl.create(:user)
CurrentUser.user = @user
CurrentUser.ip_addr = "1.2.3.4"

1
test/unit/favorite_test.rb
View File

@@ -5,7 +5,6 @@ class FavoriteTest < ActiveSupport::TestCase
user = FactoryGirl.create(:user)
CurrentUser.user = user
CurrentUser.ip_addr = "127.0.0.1"
MEMCACHE.flush_all
Favorite # need to force loading the favorite model
end

1
test/unit/ip_ban_test.rb
View File

@@ -5,7 +5,6 @@ class IpBanTest < ActiveSupport::TestCase
@user = FactoryGirl.create(:user)
CurrentUser.user = @user
CurrentUser.ip_addr = "127.0.0.1"
MEMCACHE.flush_all
Danbooru.config.stubs(:member_comment_time_threshold).returns(1.week.from_now)
end

1
test/unit/janitor_trial_test.rb
View File

@@ -7,7 +7,6 @@ class JanitorTrialTest < ActiveSupport::TestCase
@user = FactoryGirl.create(:user)
CurrentUser.user = @admin
CurrentUser.ip_addr = "127.0.0.1"
MEMCACHE.flush_all
end
teardown do

1
test/unit/moderator/ip_addr_search_test.rb
View File

@@ -9,7 +9,6 @@ module Moderator
CurrentUser.ip_addr = "127.0.0.1"
Danbooru.config.stubs(:member_comment_time_threshold).returns(1.week.from_now)
FactoryGirl.create(:comment)
MEMCACHE.flush_all
end
teardown do

1
test/unit/note_test.rb
View File

@@ -6,7 +6,6 @@ class NoteTest < ActiveSupport::TestCase
@user = FactoryGirl.create(:user)
CurrentUser.user = @user
CurrentUser.ip_addr = "127.0.0.1"
MEMCACHE.flush_all
end
teardown do

1
test/unit/pool_test.rb
View File

@@ -13,7 +13,6 @@ class PoolTest < ActiveSupport::TestCase
end
CurrentUser.ip_addr = "127.0.0.1"
MEMCACHE.flush_all
mock_pool_archive_service!
PoolArchive.sqs_service.stubs(:merge?).returns(false)

1
test/unit/post_appeal_test.rb
View File

@@ -6,7 +6,6 @@ class PostAppealTest < ActiveSupport::TestCase
@alice = FactoryGirl.create(:user)
CurrentUser.user = @alice
CurrentUser.ip_addr = "127.0.0.1"
MEMCACHE.flush_all
Danbooru.config.stubs(:max_appeals_per_day).returns(5)
end

1
test/unit/post_archive_test.rb
View File

@@ -8,7 +8,6 @@ class PostArchiveTest < ActiveSupport::TestCase
end
CurrentUser.user = @user
CurrentUser.ip_addr = "127.0.0.1"
MEMCACHE.flush_all
end
teardown do

1
test/unit/post_disapproval_test.rb
View File

@@ -6,7 +6,6 @@ class PostDisapprovalTest < ActiveSupport::TestCase
@alice = FactoryGirl.create(:moderator_user)
CurrentUser.user = @alice
CurrentUser.ip_addr = "127.0.0.1"
MEMCACHE.flush_all
end
teardown do

1
test/unit/post_flag_test.rb
View File

@@ -8,7 +8,6 @@ class PostFlagTest < ActiveSupport::TestCase
end
CurrentUser.user = @alice
CurrentUser.ip_addr = "127.0.0.1"
MEMCACHE.flush_all
@post = FactoryGirl.create(:post, :tag_string => "aaa")
end

1
test/unit/post_pruner_test.rb
View File

@@ -5,7 +5,6 @@ class PostPrunerTest < ActiveSupport::TestCase
@user = FactoryGirl.create(:admin_user)
CurrentUser.user = @user
CurrentUser.ip_addr = "127.0.0.1"
MEMCACHE.flush_all
Timecop.travel(2.weeks.ago) do
@flagger = FactoryGirl.create(:gold_user)

1
test/unit/post_sets/favorite_test.rb
View File

@@ -7,7 +7,6 @@ module PostSets
@user = FactoryGirl.create(:user)
CurrentUser.user = @user
CurrentUser.ip_addr = "127.0.0.1"
MEMCACHE.flush_all
@post_1 = FactoryGirl.create(:post)
@post_2 = FactoryGirl.create(:post)

1
test/unit/post_sets/intro_test.rb
View File

@@ -8,7 +8,6 @@ module PostSets
@user = FactoryGirl.create(:user)
CurrentUser.user = @user
CurrentUser.ip_addr = "127.0.0.1"
MEMCACHE.flush_all
@post_1 = FactoryGirl.create(:post, :tag_string => "a")
@post_2 = FactoryGirl.create(:post, :tag_string => "b")

1
test/unit/post_sets/pool_test.rb
View File

@@ -10,7 +10,6 @@ module PostSets
@user = FactoryGirl.create(:user)
CurrentUser.user = @user
CurrentUser.ip_addr = "127.0.0.1"
MEMCACHE.flush_all
mock_pool_archive_service!
start_pool_archive_transaction

1
test/unit/post_sets/post_test.rb
View File

@@ -8,7 +8,6 @@ module PostSets
@user = FactoryGirl.create(:user)
CurrentUser.user = @user
CurrentUser.ip_addr = "127.0.0.1"
MEMCACHE.flush_all
@post_1 = FactoryGirl.create(:post, :tag_string => "a")
@post_2 = FactoryGirl.create(:post, :tag_string => "b")

1
test/unit/post_test.rb
View File

@@ -18,7 +18,6 @@ class PostTest < ActiveSupport::TestCase
end
CurrentUser.user = @user
CurrentUser.ip_addr = "127.0.0.1"
MEMCACHE.flush_all
mock_saved_search_service!
end

1
test/unit/post_version_test.rb
View File

@@ -8,7 +8,6 @@ class PostVersionTest < ActiveSupport::TestCase
end
CurrentUser.user = @user
CurrentUser.ip_addr = "127.0.0.1"
MEMCACHE.flush_all
end
teardown do

1
test/unit/post_vote_test.rb
View File

@@ -8,7 +8,6 @@ class PostVoteTest < ActiveSupport::TestCase
@user = FactoryGirl.create(:user)
CurrentUser.user = @user
CurrentUser.ip_addr = "127.0.0.1"
MEMCACHE.flush_all
@post = FactoryGirl.create(:post)
end

1
test/unit/related_tag_calculator_test.rb
View File

@@ -5,7 +5,6 @@ class RelatedTagCalculatorTest < ActiveSupport::TestCase
user = FactoryGirl.create(:user)
CurrentUser.user = user
CurrentUser.ip_addr = "127.0.0.1"
MEMCACHE.flush_all
end
teardown do

1
test/unit/related_tag_query_test.rb
View File

@@ -5,7 +5,6 @@ class RelatedTagQueryTest < ActiveSupport::TestCase
user = FactoryGirl.create(:user)
CurrentUser.user = user
CurrentUser.ip_addr = "127.0.0.1"
MEMCACHE.flush_all
end
context "a related tag query without a category constraint" do

4
test/unit/saved_search_test.rb
View File

@@ -41,10 +41,6 @@ class SavedSearchTest < ActiveSupport::TestCase
end
context "Fetching the post ids for a search" do
setup do
MEMCACHE.expects(:get).returns(nil)
end
teardown do
FakeWeb.clean_registry
end

2
test/unit/tag_alias_correction_test.rb
View File

@@ -6,13 +6,11 @@ class TagAliasCorrectionTest < ActiveSupport::TestCase
@mod = FactoryGirl.create(:moderator_user)
CurrentUser.user = @mod
CurrentUser.ip_addr = "127.0.0.1"
MEMCACHE.flush_all
@post = FactoryGirl.create(:post, :tag_string => "aaa")
@tag_alias = FactoryGirl.create(:tag_alias, :antecedent_name => "aaa", :consequent_name => "bbb")
end
teardown do
MEMCACHE.flush_all
CurrentUser.user = nil
CurrentUser.ip_addr = nil
end

2
test/unit/tag_alias_request_test.rb
View File

@@ -6,11 +6,9 @@ class TagAliasRequestTest < ActiveSupport::TestCase
@user = FactoryGirl.create(:user)
CurrentUser.user = @user
CurrentUser.ip_addr = "127.0.0.1"
MEMCACHE.flush_all
end
teardown do
MEMCACHE.flush_all
CurrentUser.user = nil
CurrentUser.ip_addr = nil
end

1
test/unit/tag_alias_test.rb
View File

@@ -11,7 +11,6 @@ class TagAliasTest < ActiveSupport::TestCase
CurrentUser.user = user
end
CurrentUser.ip_addr = "127.0.0.1"
MEMCACHE.flush_all
mock_saved_search_service!
end

2
test/unit/tag_implication_request_test.rb
View File

@@ -6,11 +6,9 @@ class TagImplicationRequestTest < ActiveSupport::TestCase
@user = FactoryGirl.create(:user)
CurrentUser.user = @user
CurrentUser.ip_addr = "127.0.0.1"
MEMCACHE.flush_all
end
teardown do
MEMCACHE.flush_all
CurrentUser.user = nil
CurrentUser.ip_addr = nil
end

1
test/unit/tag_implication_test.rb
View File

@@ -7,7 +7,6 @@ class TagImplicationTest < ActiveSupport::TestCase
CurrentUser.user = user
CurrentUser.ip_addr = "127.0.0.1"
@user = FactoryGirl.create(:user)
MEMCACHE.flush_all
end
teardown do

1
test/unit/tag_subscription_test.rb
View File

@@ -5,7 +5,6 @@ class TagSubscriptionTest < ActiveSupport::TestCase
user = FactoryGirl.create(:user)
CurrentUser.user = user
CurrentUser.ip_addr = "127.0.0.1"
MEMCACHE.flush_all
end
teardown do

17
test/unit/tag_test.rb
View File

@@ -5,7 +5,6 @@ class TagTest < ActiveSupport::TestCase
user = FactoryGirl.create(:builder_user)
CurrentUser.user = user
CurrentUser.ip_addr = "127.0.0.1"
MEMCACHE.flush_all
end
teardown do
@@ -32,10 +31,6 @@ class TagTest < ActiveSupport::TestCase
end
context "A tag category fetcher" do
setup do
MEMCACHE.flush_all
end
should "fetch for a single tag" do
FactoryGirl.create(:artist_tag, :name => "test")
assert_equal(Tag.categories.artist, Tag.category_for("test"))
@@ -57,10 +52,6 @@ class TagTest < ActiveSupport::TestCase
end
context "A tag category mapping" do
setup do
MEMCACHE.flush_all
end
should "exist" do
assert_nothing_raised {Tag.categories}
end
@@ -97,10 +88,6 @@ class TagTest < ActiveSupport::TestCase
end
context "A tag" do
setup do
MEMCACHE.flush_all
end
should "be lockable by a moderator" do
@tag = FactoryGirl.create(:tag)
@tag.update_attributes({:is_locked => true}, :as => :moderator)
@@ -123,11 +110,11 @@ class TagTest < ActiveSupport::TestCase
should "reset its category after updating" do
tag = FactoryGirl.create(:artist_tag)
tag.update_category_cache_for_all
assert_equal(Tag.categories.artist, MEMCACHE.get("tc:#{tag.name}"))
assert_equal(Tag.categories.artist, Cache.get("tc:#{tag.name}"))
tag.update_attribute(:category, Tag.categories.copyright)
tag.update_category_cache_for_all
assert_equal(Tag.categories.copyright, MEMCACHE.get("tc:#{tag.name}"))
assert_equal(Tag.categories.copyright, Cache.get("tc:#{tag.name}"))
end
end

1
test/unit/upload_test.rb
View File

@@ -17,7 +17,6 @@ class UploadTest < ActiveSupport::TestCase
user = FactoryGirl.create(:contributor_user)
CurrentUser.user = user
CurrentUser.ip_addr = "127.0.0.1"
MEMCACHE.flush_all
end
teardown do

1
test/unit/user_feedback_test.rb
View File

@@ -4,7 +4,6 @@ class UserFeedbackTest < ActiveSupport::TestCase
context "A user's feedback" do
setup do
CurrentUser.ip_addr = "127.0.0.1"
MEMCACHE.flush_all
end
teardown do

1
test/unit/user_test.rb
View File

@@ -6,7 +6,6 @@ class UserTest < ActiveSupport::TestCase
@user = FactoryGirl.create(:user)
CurrentUser.user = @user
CurrentUser.ip_addr = "127.0.0.1"
MEMCACHE.flush_all
end
teardown do

1
test/unit/wiki_page_test.rb
View File

@@ -2,7 +2,6 @@ require 'test_helper'
class WikiPageTest < ActiveSupport::TestCase
setup do
MEMCACHE.flush_all
CurrentUser.ip_addr = "127.0.0.1"
end