From aeb5edaff65217d4743c85e7104f47b87a467528 Mon Sep 17 00:00:00 2001 From: evazion Date: Tue, 5 Nov 2019 12:53:52 -0600 Subject: [PATCH] Fix deadlocks while updating tags counts. Sometimes uploads fail with this error: ActiveRecord::QueryCanceled - PG::QueryCanceled: ERROR: canceling statement due to statement timeout CONTEXT: while updating tuple (2808,110) in relation "tags" This is caused by two uploads trying to update tag counts for the same tags at the same time. The UPDATE statement doesn't guarantee the tags will be updated in any particular order, which can cause deadlocks when two UPDATEs try to update the same tags in a conflicting order. For example, if one process tries to increment '1girl' and 'solo' (in that order), while at the same time another process tries to increment 'solo' and '1girl' (in that order), the result will be a deadlock. The solution is to explicitly lock the tags in a consistent order before the update statement. --- app/models/tag.rb | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/app/models/tag.rb b/app/models/tag.rb index c1c74e58f..31bb57ed6 100644 --- a/app/models/tag.rb +++ b/app/models/tag.rb @@ -91,12 +91,19 @@ class Tag < ApplicationRecord extend ActiveSupport::Concern module ClassMethods + # Lock the tags first in alphabetical order to avoid deadlocks under concurrent updates. + # + # https://stackoverflow.com/questions/44660368/postgres-update-with-order-by-how-to-do-it + # https://www.postgresql.org/message-id/flat/freemail.20070030161126.43285%40fm10.freemail.hu + # https://www.postgresql.org/message-id/flat/CAKOSWNkb3Zy_YFQzwyRw3MRrU10LrMj04%2BHdByfQu6M1S5B7mg%40mail.gmail.com#9dc514507357472bdf22d3109d9c7957 def increment_post_counts(tag_names) - Tag.where(:name => tag_names).update_all("post_count = post_count + 1") + Tag.where(name: tag_names).order(:name).lock("FOR UPDATE").pluck(1) + Tag.where(name: tag_names).update_all("post_count = post_count + 1") end def decrement_post_counts(tag_names) - Tag.where(:name => tag_names).update_all("post_count = post_count - 1") + Tag.where(name: tag_names).order(:name).lock("FOR UPDATE").pluck(1) + Tag.where(name: tag_names).update_all("post_count = post_count - 1") end def regenerate_post_counts!