iqdb: update API client to use new version of IQDB.

Replace the old IQDB API client with a new client for the new forked
version of IQDB at https://github.com/danbooru/iqdb.

Changes:

* The /iqdb_queries endpoint now returns `hash` and `signature` fields.
  The `signature` is the full decoded Haar signature, while the `hash`
  is a encoded version of the signature.
* The /iqdb_queries endpoint no longer returns `width` and `height`
  fields in the response (these were always 128x128).
* We no longer need the IQDBs frontend server, now we talk to the IQDB
  instance directly.
* We no longer send add/remove image commands to IQDB through AWS SQS,
  now we send them to IQDB directly. They are sent in a delayed job so
  that if IQDB is down, uploading images is still possible, the add
  image commands will just get queued up.
* Fix a bug where regenerating an image's thumbnails didn't regenerate
  IQDB, because IQDB silently ignored add image commands when the image
  already existed in the database.
This commit is contained in:
evazion
2021-06-16 05:14:15 -05:00
parent 5b208ddb78
commit 0f36bbf8d3
22 changed files with 136 additions and 164 deletions

View File

@@ -36,8 +36,6 @@ class Post < ApplicationRecord
after_save :update_parent_on_save
after_save :apply_post_metatags
after_commit :delete_files, :on => :destroy
after_commit :remove_iqdb_async, :on => :destroy
after_commit :update_iqdb_async, :on => :create
belongs_to :approver, class_name: "User", optional: true
belongs_to :uploader, :class_name => "User", :counter_cache => "post_upload_count"
@@ -977,6 +975,8 @@ class Post < ApplicationRecord
update_parent_on_destroy
end
end
remove_iqdb # this is non-transactional
end
def ban!
@@ -1319,16 +1319,15 @@ class Post < ApplicationRecord
def regenerate!(category, user)
if category == "iqdb"
update_iqdb_async
update_iqdb
ModAction.log("<@#{user.name}> regenerated IQDB for post ##{id}", :post_regenerate_iqdb, user)
else
media_file = MediaFile.open(file, frame_data: pixiv_ugoira_frame_data)
UploadService::Utils.process_resizes(self, nil, id, media_file: media_file)
# XXX This may be racy; the thumbnail may not be purged from Cloudflare by the time IQDB tries to download it.
purge_cached_urls!
update_iqdb_async
update_iqdb
ModAction.log("<@#{user.name}> regenerated image samples for post ##{id}", :post_regenerate, user)
end
@@ -1340,33 +1339,15 @@ class Post < ApplicationRecord
end
end
module IqdbMethods
extend ActiveSupport::Concern
module ClassMethods
def iqdb_sqs_service
SqsService.new(Danbooru.config.aws_sqs_iqdb_url)
end
def iqdb_enabled?
Danbooru.config.aws_sqs_iqdb_url.present?
end
def remove_iqdb(post_id)
if iqdb_enabled?
iqdb_sqs_service.send_message("remove\n#{post_id}")
end
end
concerning :IqdbMethods do
def update_iqdb
# performs IqdbClient.new.add_post(post)
IqdbAddPostJob.perform_later(self)
end
def update_iqdb_async
if Post.iqdb_enabled? && has_preview?
Post.iqdb_sqs_service.send_message("update\n#{id}\n#{preview_file_url}")
end
end
def remove_iqdb_async
Post.remove_iqdb(id)
def remove_iqdb
# performs IqdbClient.new.remove(id)
IqdbRemovePostJob.perform_later(id)
end
end
@@ -1460,7 +1441,6 @@ class Post < ApplicationRecord
include ApiMethods
extend SearchMethods
include PixivMethods
include IqdbMethods
include ValidationMethods
has_bit_flags ["has_embedded_notes", "has_cropped"]