jaculabilis/danbooru
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

iqdb: update API client to use new version of IQDB.

Browse Source 
Replace the old IQDB API client with a new client for the new forked
version of IQDB at https://github.com/danbooru/iqdb.

Changes:

* The /iqdb_queries endpoint now returns `hash` and `signature` fields.
  The `signature` is the full decoded Haar signature, while the `hash`
  is a encoded version of the signature.
* The /iqdb_queries endpoint no longer returns `width` and `height`
  fields in the response (these were always 128x128).
* We no longer need the IQDBs frontend server, now we talk to the IQDB
  instance directly.
* We no longer send add/remove image commands to IQDB through AWS SQS,
  now we send them to IQDB directly. They are sent in a delayed job so
  that if IQDB is down, uploading images is still possible, the add
  image commands will just get queued up.
* Fix a bug where regenerating an image's thumbnails didn't regenerate
  IQDB, because IQDB silently ignored add image commands when the image
  already existed in the database.
This commit is contained in:
evazion
2021-06-16 05:14:15 -05:00
parent 5b208ddb78
commit 0f36bbf8d3
22 changed files with 136 additions and 164 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.env
View File

@@ -78,8 +78,6 @@
# export DANBOORU_AWS_ACCESS_KEY_ID=
# export DANBOORU_AWS_SECRET_ACCESS_KEY=
# export DANBOORU_AWS_SQS_REGION=
# export DANBOORU_IQDBS_AUTH_KEY=
# export DANBOORU_IQDBS_SERVER=
# export DANBOORU_CLOUDFLARE_KEY=
# export DANBOORU_CLOUDFLARE_EMAIL=
# export DANBOORU_CLOUDFLARE_ZONE=

3
Procfile
View File

@@ -1,5 +1,2 @@
unicorn: bin/rails server -p 3000
jobs: bin/rake jobs:work
recommender: bundle exec ruby script/mock_services/recommender.rb
iqdbs: bundle exec ruby script/mock_services/iqdbs.rb
reportbooru: bundle exec ruby script/mock_services/reportbooru.rb

2
README.md
View File

@@ -88,7 +88,7 @@ The following features requires a Google API account:
### IQDB Service
IQDB integration is delegated to the [IQDBS service](https://github.com/r888888888/iqdbs).
IQDB integration is delegated to the [IQDB service](https://github.com/danbooru/iqdb).
### Archive Service

3
app/controllers/iqdb_queries_controller.rb
View File

@@ -5,7 +5,8 @@ class IqdbQueriesController < ApplicationController
# XXX allow bare search params for backwards compatibility.
search_params.merge!(params.slice(:url, :image_url, :file_url, :post_id, :limit, :similarity, :high_similarity).permit!)
@high_similarity_matches, @low_similarity_matches, @matches = IqdbProxy.new.search(search_params)
iqdb_params = search_params.to_h.symbolize_keys
@high_similarity_matches, @low_similarity_matches, @matches = IqdbClient.new.search(**iqdb_params)
respond_with(@matches, template: "iqdb_queries/show")
end

2
app/controllers/mock_services_controller.rb
View File

@@ -31,7 +31,7 @@ class MockServicesController < ApplicationController
render json: @data
end
def iqdbs_similar
def iqdb_query
@data = posts.map { |post| { post_id: post.id, score: rand(0..100)} }
render json: @data
end

5
app/jobs/iqdb_add_post_job.rb Normal file
View File

@@ -0,0 +1,5 @@
class IqdbAddPostJob < ApplicationJob
def perform(post)
IqdbClient.new.add_post(post)
end
end

5
app/jobs/iqdb_remove_post_job.rb Normal file
View File

@@ -0,0 +1,5 @@
class IqdbRemovePostJob < ApplicationJob
def perform(post_id)
IqdbClient.new.remove(post_id)
end
end

86
app/logical/iqdb_client.rb Normal file
View File

@@ -0,0 +1,86 @@
class IqdbClient
class Error < StandardError; end
attr_reader :iqdb_url, :http
def initialize(iqdb_url: Danbooru.config.iqdb_url.to_s, http: Danbooru::Http.new)
@iqdb_url = iqdb_url.chomp("/")
@http = http
end
concerning :QueryMethods do
def search(post_id: nil, file: nil, url: nil, image_url: nil, file_url: nil, similarity: 0.0, high_similarity: 65.0, limit: 20)
limit = limit.to_i.clamp(1, 1000)
similarity = similarity.to_f.clamp(0.0, 100.0)
high_similarity = high_similarity.to_f.clamp(0.0, 100.0)
if file.blank?
if url.present?
file = download(url, :preview_url)
elsif image_url.present?
file = download(image_url, :url)
elsif file_url.present?
file = download(file_url, :image_url)
elsif post_id.present?
file = Post.find(post_id).file(:preview)
else
return [[], [], []]
end
end
results = query(file, limit: limit)
results = results.select { |result| result["score"] >= similarity }.take(limit)
matches = decorate_posts(results)
high_similarity_matches, low_similarity_matches = matches.partition { |match| match["score"] >= high_similarity }
[high_similarity_matches, low_similarity_matches, matches]
ensure
file.try(:close)
end
def download(url, type)
strategy = Sources::Strategies.find(url)
download_url = strategy.send(type)
file = strategy.download_file!(download_url)
file
end
def decorate_posts(json)
post_ids = json.map { |match| match["post_id"] }
posts = Post.where(id: post_ids).group_by(&:id).transform_values(&:first)
json.map do |match|
post = posts.fetch(match["post_id"], nil)
match.with_indifferent_access.merge(post: post) if post
end.compact
end
end
def add_post(post)
return unless post.has_preview?
preview_file = post.file(:preview)
add(post.id, preview_file)
end
concerning :HttpMethods do
def query(file, limit: 20)
file = HTTP::FormData::File.new(file)
request(:post, "query", form: { file: file }, params: { limit: limit })
end
def add(post_id, file)
file = HTTP::FormData::File.new(file)
request(:post, "images/#{post_id}", form: { file: file })
end
def remove(post_id)
request(:delete, "images/#{post_id}")
end
def request(method, url, **options)
return [] if iqdb_url.blank? # do nothing if iqdb isn't configured
response = http.timeout(30).send(method, "#{iqdb_url}/#{url}", **options)
raise Error, "IQDB error: #{response.parse}" if response.status != 200
response.parse
end
end
end

77
app/logical/iqdb_proxy.rb
View File

@@ -1,77 +0,0 @@
class IqdbProxy
class Error < StandardError; end
attr_reader :http, :iqdbs_server
def initialize(http: Danbooru::Http.new, iqdbs_server: Danbooru.config.iqdbs_server)
@iqdbs_server = iqdbs_server
@http = http
end
def enabled?
iqdbs_server.present?
end
def download(url, type)
strategy = Sources::Strategies.find(url)
download_url = strategy.send(type)
file = strategy.download_file!(download_url)
file
end
def search(params)
limit = params[:limit]&.to_i&.clamp(1, 1000) || 20
similarity = params[:similarity]&.to_f&.clamp(0.0, 100.0) || 0.0
high_similarity = params[:high_similarity]&.to_f&.clamp(0.0, 100.0) || 65.0
if params[:file].present?
file = params[:file]
results = query(file: file, limit: limit)
elsif params[:url].present?
file = download(params[:url], :preview_url)
results = query(file: file, limit: limit)
elsif params[:image_url].present?
file = download(params[:image_url], :url)
results = query(file: file, limit: limit)
elsif params[:file_url].present?
file = download(params[:file_url], :image_url)
results = query(file: file, limit: limit)
elsif params[:post_id].present?
url = Post.find(params[:post_id]).preview_file_url
results = query(url: url, limit: limit)
else
results = []
end
results = results.select { |result| result["score"] >= similarity }.take(limit)
matches = decorate_posts(results)
high_similarity_matches, low_similarity_matches = matches.partition { |match| match["score"] >= high_similarity }
[high_similarity_matches, low_similarity_matches, matches]
ensure
file.try(:close)
end
def query(file: nil, url: nil, limit: 20)
raise NotImplementedError, "the IQDBs service isn't configured" unless enabled?
file = HTTP::FormData::File.new(file) if file
form = { file: file, url: url, limit: limit }.compact
response = http.timeout(30).post("#{iqdbs_server}/similar", form: form)
raise Error, "IQDB error: #{response.status}" if response.status != 200
raise Error, "IQDB error: #{response.parse["error"]}" if response.parse.is_a?(Hash)
raise Error, "IQDB error: #{response.parse.first}" if response.parse.try(:first).is_a?(String)
response.parse
end
def decorate_posts(json)
post_ids = json.map { |match| match["post_id"] }
posts = Post.where(id: post_ids).group_by(&:id).transform_values(&:first)
json.map do |match|
post = posts.fetch(match["post_id"], nil)
match.with_indifferent_access.merge(post: post) if post
end.compact
end
end

2
app/logical/upload_service.rb
View File

@@ -85,6 +85,8 @@ class UploadService
)
end
@post.update_iqdb
upload.update(status: "completed", post_id: @post.id)
@post

2
app/logical/upload_service/replacer.rb
View File

@@ -128,7 +128,7 @@ class UploadService
replacement.save!
post.save!
post.update_iqdb_async
post.update_iqdb
end
def purge_cached_urls(post)

42
app/models/post.rb
View File

@@ -36,8 +36,6 @@ class Post < ApplicationRecord
after_save :update_parent_on_save
after_save :apply_post_metatags
after_commit :delete_files, :on => :destroy
after_commit :remove_iqdb_async, :on => :destroy
after_commit :update_iqdb_async, :on => :create
belongs_to :approver, class_name: "User", optional: true
belongs_to :uploader, :class_name => "User", :counter_cache => "post_upload_count"
@@ -977,6 +975,8 @@ class Post < ApplicationRecord
update_parent_on_destroy
end
end
remove_iqdb # this is non-transactional
end
def ban!
@@ -1319,16 +1319,15 @@ class Post < ApplicationRecord
def regenerate!(category, user)
if category == "iqdb"
update_iqdb_async
update_iqdb
ModAction.log("<@#{user.name}> regenerated IQDB for post ##{id}", :post_regenerate_iqdb, user)
else
media_file = MediaFile.open(file, frame_data: pixiv_ugoira_frame_data)
UploadService::Utils.process_resizes(self, nil, id, media_file: media_file)
# XXX This may be racy; the thumbnail may not be purged from Cloudflare by the time IQDB tries to download it.
purge_cached_urls!
update_iqdb_async
update_iqdb
ModAction.log("<@#{user.name}> regenerated image samples for post ##{id}", :post_regenerate, user)
end
@@ -1340,33 +1339,15 @@ class Post < ApplicationRecord
end
end
module IqdbMethods
extend ActiveSupport::Concern
module ClassMethods
def iqdb_sqs_service
SqsService.new(Danbooru.config.aws_sqs_iqdb_url)
end
def iqdb_enabled?
Danbooru.config.aws_sqs_iqdb_url.present?
end
def remove_iqdb(post_id)
if iqdb_enabled?
iqdb_sqs_service.send_message("remove\n#{post_id}")
end
end
concerning :IqdbMethods do
def update_iqdb
# performs IqdbClient.new.add_post(post)
IqdbAddPostJob.perform_later(self)
end
def update_iqdb_async
if Post.iqdb_enabled? && has_preview?
Post.iqdb_sqs_service.send_message("update\n#{id}\n#{preview_file_url}")
end
end
def remove_iqdb_async
Post.remove_iqdb(id)
def remove_iqdb
# performs IqdbClient.new.remove(id)
IqdbRemovePostJob.perform_later(id)
end
end
@@ -1460,7 +1441,6 @@ class Post < ApplicationRecord
include ApiMethods
extend SearchMethods
include PixivMethods
include IqdbMethods
include ValidationMethods
has_bit_flags ["has_embedded_notes", "has_cropped"]

2
app/views/uploads/_related_posts.html.erb
View File

@@ -16,7 +16,7 @@
</div>
<% end %>
<% if Danbooru.config.iqdbs_server %>
<% if Danbooru.config.iqdb_url %>
<% if params[:url] %>
<div class="input" id="iqdb-similar">
<p><em>Loading similar...</em></p>

12
config/danbooru_default_config.rb
View File

@@ -481,11 +481,12 @@ module Danbooru
def reportbooru_key
end
# The URL for the IQDBs server (https://github.com/evazion/iqdbs).
# Optional. Used for dupe detection and reverse image searches.
# Set to http://localhost/mock/iqdbs to enable a fake iqdb server for
# The URL for the IQDB server (https://github.com/danbooru/iqdb). Optional.
# Used for dupe detection and reverse image searches. Set this to
# http://localhost:3000/mock/iqdb to enable a fake iqdb server for
# development purposes.
def iqdbs_server
def iqdb_url
# "http://localhost:3000/mock/iqdb"
end
def aws_credentials
@@ -501,9 +502,6 @@ module Danbooru
def aws_sqs_region
end
def aws_sqs_iqdb_url
end
def aws_sqs_archives_url
end

4
config/routes.rb
View File

@@ -329,8 +329,8 @@ Rails.application.routes.draw do
get "/mock/reportbooru/missed_searches" => "mock_services#reportbooru_missed_searches", as: "mock_reportbooru_missed_searches"
get "/mock/reportbooru/post_searches/rank" => "mock_services#reportbooru_post_searches", as: "mock_reportbooru_post_searches"
get "/mock/reportbooru/post_views/rank" => "mock_services#reportbooru_post_views", as: "mock_reportbooru_post_views"
get "/mock/iqdbs/similar" => "mock_services#iqdbs_similar", as: "mock_iqdbs_similar"
post "/mock/iqdbs/similar" => "mock_services#iqdbs_similar"
get "/mock/iqdb/query" => "mock_services#iqdb_query", as: "mock_iqdb_query"
post "/mock/iqdb/query" => "mock_services#iqdb_query"
match "*other", to: "static#not_found", via: :all
end

3
lib/tasks/iqdb.rake
View File

@@ -2,8 +2,7 @@ namespace :iqdb do
task reindex_posts: :environment do
STDIN.each_line do |post_id|
puts post_id
post.remove_iqdb_async
post.update_iqdb_async
post.update_iqdb
end
end
end

9
test/functional/iqdb_queries_controller_test.rb
View File

@@ -11,7 +11,7 @@ class IqdbQueriesControllerTest < ActionDispatch::IntegrationTest
context "with a url parameter" do
should "render a response" do
@url = "https://google.com"
@matches = [{ "post_id" => @post.id, "width" => 128, "height" => 128, "score" => 95.0 }]
@matches = [{ post_id: @post.id, score: 95.0 }]
mock_iqdb_matches(@matches)
get_auth iqdb_queries_path, @user, as: :javascript, params: { url: @url }
@@ -22,10 +22,13 @@ class IqdbQueriesControllerTest < ActionDispatch::IntegrationTest
end
context "with a post_id parameter" do
should "redirect to iqdbs" do
@matches = [{ "post_id" => @post.id, "width" => 128, "height" => 128, "score" => 95.0 }]
should "render a response" do
@matches = [{ post_id: @post.id, score: 95.0 }]
mock_iqdb_matches(@matches)
# Make the call to `@post.file(:preview)` work.
Post.any_instance.stubs(:file).returns(File.open("test/files/test.jpg"))
get_auth iqdb_queries_path, @user, params: { post_id: @post.id }
assert_response :success

2
test/functional/mock_services_controller_test.rb
View File

@@ -15,7 +15,7 @@ class MockServicesControllerTest < ActionDispatch::IntegrationTest
mock_reportbooru_missed_searches_path,
mock_reportbooru_post_searches_path,
mock_reportbooru_post_views_path,
mock_iqdbs_similar_path,
mock_iqdb_query_path,
]
paths.each do |path|

4
test/functional/post_regenerations_controller_test.rb
View File

@@ -6,6 +6,7 @@ class PostRegenerationsControllerTest < ActionDispatch::IntegrationTest
@mod = create(:moderator_user, name: "yukari", created_at: 1.month.ago)
@upload = assert_successful_upload("test/files/test.jpg", user: @mod)
@post = @upload.post
perform_enqueued_jobs # add post to iqdb
end
context "create action" do
@@ -23,9 +24,6 @@ class PostRegenerationsControllerTest < ActionDispatch::IntegrationTest
context "for an IQDB regeneration" do
should "regenerate IQDB" do
mock_iqdb_service!
Post.iqdb_sqs_service.expects(:send_message).with("update\n#{@post.id}\n#{@post.preview_file_url}")
post_auth post_regenerations_path, @mod, params: { post_id: @post.id, category: "iqdb" }
perform_enqueued_jobs
end

1
test/functional/uploads_controller_test.rb
View File

@@ -4,7 +4,6 @@ class UploadsControllerTest < ActionDispatch::IntegrationTest
context "The uploads controller" do
setup do
@user = create(:contributor_user, name: "marisa")
mock_iqdb_service!
end
context "image proxy action" do

24
test/test_helpers/iqdb_test_helper.rb
View File

@@ -1,28 +1,6 @@
module IqdbTestHelper
def mock_iqdb_service!
mock_sqs_service = Class.new do
def initialize
@commands = []
end
def commands
@commands
end
def send_message(msg)
@commands << msg.split(/\n/).first
end
end
service = mock_sqs_service.new
Post.stubs(:iqdb_sqs_service).returns(service)
Post.stubs(:iqdb_enabled?).returns(true)
Danbooru.config.stubs(:iqdbs_server).returns("http://localhost:3004")
end
def mock_iqdb_matches(matches)
Danbooru.config.stubs(:iqdbs_server).returns("http://localhost:3004")
Danbooru.config.stubs(:iqdb_url).returns("http://localhost:5588")
response = HTTP::Response.new(status: 200, body: matches.to_json, headers: { "Content-Type": "application/json" }, version: "1.1")
HTTP::Client.any_instance.stubs(:post).returns(response)
end

6
test/unit/post_test.rb
View File

@@ -35,6 +35,7 @@ class PostTest < ActiveSupport::TestCase
@post = @upload.post
Favorite.add(post: @post, user: @user)
create(:favorite_group, post_ids: [@post.id])
perform_enqueued_jobs # perform IqdbAddPostJob
end
should "delete the files" do
@@ -85,10 +86,9 @@ class PostTest < ActiveSupport::TestCase
end
should "remove the post from iqdb" do
mock_iqdb_service!
Post.iqdb_sqs_service.expects(:send_message).with("remove\n#{@post.id}")
@post.expunge!
perform_enqueued_jobs
assert_performed_jobs(1, only: IqdbRemovePostJob)
end
context "that is status locked" do