jaculabilis/danbooru
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

remove old iqdb code

Browse Source
This commit is contained in:
Albert Yi
2016-12-02 13:21:56 -08:00
parent 8e5331d234
commit 7aab50f726
23 changed files with 13 additions and 349 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
README.md
View File

@@ -39,8 +39,7 @@ debug your Nginx configuration file.
### IQDB Integration ### IQDB Integration
In order to enable IQDB integration, you must compile and run the IQDB server somewhere (preferably on the local server). There are two Danbooru configuration settings that you must then set: iqdb_hostname_and_port and iqdb_file. IQDB integration is now delegated to the [IQDBS service](https://github.com/r888888888/iqdbs).
You must then populate the initial database. There is a fix script called 028_iqdb_import.rb to do this for you. You will need to install your own copy and enable the appropriate
configuration settings.
From then on, all new uploads will asynchronously create two tasks: one to update the IQDB database through the server interface (which updates the in-memory representation), and another to the command interface (which updates the disk representation). Expunging posts will also update the database.

2
app/controllers/iqdb_queries_controller.rb
View File

@@ -3,7 +3,7 @@ class IqdbQueriesController < ApplicationController
before_filter :member_only before_filter :member_only
def create def create
if !Danbooru.config.iqdb_hostname_and_port if !Danbooru.config.iqdbs_server
render :nothing => true render :nothing => true
return return
end end

31
app/logical/iqdb/command.rb
View File

@@ -1,31 +0,0 @@
module Iqdb
class Command
attr_reader :database
def initialize(database)
@database = database
end
def process(&block)
IO.popen("iqdb command #{database}", "w+", &block)
end
def add(post)
hex = post.id.to_s(16)
process do |io|
io.puts "add 0 #{hex}:#{post.preview_file_path}"
io.puts "quit"
io.read
end
end
def remove(post_id)
hex = post_id.to_s(16)
process do |io|
io.puts "remove 0 #{hex}"
io.puts "quit"
io.read
end
end
end
end

13
app/logical/iqdb/download.rb
View File

@@ -42,18 +42,7 @@ module Iqdb
end end
end end
else else
begin raise NotImplementedError
tempfile = Tempfile.new("iqdb-#{$PROCESS_ID}")
@download = Downloads::File.new(source, tempfile.path, :get_thumbnail => true)
@download.download!
if Danbooru.config.iqdb_hostname_and_port
@matches = Iqdb::Server.default.query(3, @download.file_path).matches
end
ensure
tempfile.close
tempfile.unlink
end
end end
end end
end end

16
app/logical/iqdb/importer.rb
View File

@@ -1,16 +0,0 @@
module Iqdb
class Importer
def import!
Post.find_each do |post|
IO.popen("iqdb command #{Danbooru.config.iqdb_file}", "w+") do |io|
if File.exists?(post.preview_file_path)
hex = post.id.to_s(16)
io.puts "add 0 #{hex}:#{post.preview_file_path}"
end
io.puts "quit"
io.read
end
end
end
end
end

6
app/logical/iqdb/responses/base.rb
View File

@@ -1,6 +0,0 @@
module Iqdb
module Responses
class Base
end
end
end

27
app/logical/iqdb/responses/collection.rb
View File

@@ -1,27 +0,0 @@
module Iqdb
module Responses
class Collection
attr_reader :responses
include Enumerable
delegate :<=>, :each, :to => :matches
def initialize(response_string)
@responses = response_string.split(/\n/).map do |string|
::Iqdb::Responses.const_get("Response_#{string[0..2]}").new(string[4..-1])
end
end
def matches
@matches ||= responses.select {|x| x.is_a?(Iqdb::Responses::Response_200) && x.score >= 80}
end
def errored?
errors.any?
end
def errors
@errors ||= responses.select {|x| x.is_a?(Iqdb::Responses::Error)}.map {|x| x.to_s}
end
end
end
end

6
app/logical/iqdb/responses/error.rb
View File

@@ -1,6 +0,0 @@
module Iqdb
module Responses
class Error < Base
end
end
end

8
app/logical/iqdb/responses/response_000.rb
View File

@@ -1,8 +0,0 @@
module Iqdb
module Responses
class Response_000 < Base
def initialize(response_string)
end
end
end
end

11
app/logical/iqdb/responses/response_100.rb
View File

@@ -1,11 +0,0 @@
module Iqdb
module Responses
class Response_100 < Base
attr_reader :message
def initialize(response_string)
@message = response_string
end
end
end
end

11
app/logical/iqdb/responses/response_101.rb
View File

@@ -1,11 +0,0 @@
module Iqdb
module Responses
class Response_101 < Base
attr_reader :key, :value
def initialize(response_string)
@key, @value = response_string.split(/\=/)
end
end
end
end

11
app/logical/iqdb/responses/response_102.rb
View File

@@ -1,11 +0,0 @@
module Iqdb
module Responses
class Response_102 < Base
attr_reader :dbid, :filename
def initialize(response_string)
@dbid, @filename = response_string.split(/ /)
end
end
end
end

18
app/logical/iqdb/responses/response_200.rb
View File

@@ -1,18 +0,0 @@
module Iqdb
module Responses
class Response_200 < Base
attr_reader :imgid, :score, :width, :height
def initialize(response_string)
@imgid, @score, @width, @height = response_string.split(/ /)
@score = score.to_f
@width = width.to_i
@height = height.to_i
end
def post_id
imgid.to_i(16)
end
end
end
end

19
app/logical/iqdb/responses/response_201.rb
View File

@@ -1,19 +0,0 @@
module Iqdb
module Responses
class Response_201 < Base
attr_reader :dbid, :imgid, :score, :width, :height
def initialize(response_string)
@dbid, @imgid, @score, @width, @height = response_string.split(/ /)
@dbid = dbid.to_i
@score = score.to_f
@width = width.to_i
@height = height.to_i
end
def post_id
imgid.to_i(16)
end
end
end
end

19
app/logical/iqdb/responses/response_202.rb
View File

@@ -1,19 +0,0 @@
module Iqdb
module Responses
class Response_202 < Base
attr_reader :original_id, :stddev, :dupes
def initialize(response_string)
response_string =~ /^(\d+)=([0-9.]+)/
@original_id = $1
@stddev = $2
@dupes = response_string.scan(/(\d+):([0-9.]+)/).map {|x| [x[0].to_i(16), x[1].to_f]}
end
def original_post_id
original_id.to_i(16)
end
end
end
end

15
app/logical/iqdb/responses/response_300.rb
View File

@@ -1,15 +0,0 @@
module Iqdb
module Responses
class Response_300 < Error
attr_reader :message
def initialize(response_string)
@message = response_string
end
def to_s
"Error: #{message}"
end
end
end
end

17
app/logical/iqdb/responses/response_301.rb
View File

@@ -1,17 +0,0 @@
module Iqdb
module Responses
class Response_301 < Error
attr_reader :exception, :description
def initialize(response_string)
response_string =~ /^(\S+) (.+)/
@exception = $1
@description = $2
end
def to_s
"Exception: #{exception}: #{description}"
end
end
end
end

17
app/logical/iqdb/responses/response_302.rb
View File

@@ -1,17 +0,0 @@
module Iqdb
module Responses
class Response_302 < Error
attr_reader :exception, :description
def initialize(response_string)
response_string =~ /^(\S+) (.+)/
@exception = $1
@description = $2
end
def to_s
"Fatal Exception: #{exception}: #{description}"
end
end
end
end

60
app/logical/iqdb/server.rb
View File

@@ -1,60 +0,0 @@
module Iqdb
class Server
FLAG_SKETCH = 0x01
FLAG_GRAYSCALE = 0x02
FLAG_WIDTH_AS_SET = 0x08
FLAG_DISCARD_COMMON_COEFFS = 0x10
attr_reader :hostname, :port, :socket
def self.default
new(*Danbooru.config.iqdb_hostname_and_port)
end
def initialize(hostname, port)
@hostname = hostname
@port = port
end
def open
@socket = TCPSocket.open(hostname, port)
end
def close
socket.close
end
def request
open
yield
ensure
close
end
def add(post)
request do
hex = post.id.to_s(16)
socket.puts "add 0 #{hex}:#{post.preview_file_path}"
socket.puts "done"
socket.read
end
end
def remove(post_id)
request do
hex = post_id.to_s(16)
socket.puts "remove 0 #{hex}"
socket.puts "done"
socket.read
end
end
def query(results, filename, flags = 0)
request do
socket.puts "query 0 #{flags} #{results} #{filename}"
socket.puts "done"
responses = Responses::Collection.new(socket.read)
end
end
end
end

36
app/models/post.rb
View File

@@ -1697,42 +1697,21 @@ class Post < ActiveRecord::Base
if Danbooru.config.aws_sqs_iqdb_url if Danbooru.config.aws_sqs_iqdb_url
client = SqsService.new(Danbooru.config.aws_sqs_iqdb_url) client = SqsService.new(Danbooru.config.aws_sqs_iqdb_url)
client.send_message("remove\n#{post_id}") client.send_message("remove\n#{post_id}")
else
Iqdb::Server.new(*Danbooru.config.iqdb_hostname_and_port).remove(post_id)
Iqdb::Command.new(Danbooru.config.iqdb_file).remove(post_id)
end end
end end
end end
def update_iqdb_async def update_iqdb_async
if File.exists?(preview_file_path) if File.exists?(preview_file_path) && Danbooru.config.aws_sqs_iqdb_url
if Danbooru.config.aws_sqs_iqdb_url client = SqsService.new(Danbooru.config.aws_sqs_iqdb_url)
client = SqsService.new(Danbooru.config.aws_sqs_iqdb_url) client.send_message("update\n#{id}\n#{complete_preview_file_url}")
client.send_message("update\n#{id}\n#{complete_preview_file_url}")
elsif Danbooru.config.iqdb_hostname_and_port
Danbooru.config.all_server_hosts.each do |host|
if has_tag?("ugoira")
run_at = 10.seconds.from_now
else
run_at = Time.now
end
delay(:queue => host, :run_at => run_at).update_iqdb
end
end
end end
end end
def remove_iqdb_async def remove_iqdb_async
if File.exists?(preview_file_path) if File.exists?(preview_file_path) && Danbooru.config.aws_sqs_iqdb_url
if Danbooru.config.aws_sqs_iqdb_url client = SqsService.new(Danbooru.config.aws_sqs_iqdb_url)
client = SqsService.new(Danbooru.config.aws_sqs_iqdb_url) client.send_message("remove\n#{id}")
client.send_message("remove\n#{id}")
elsif Danbooru.config.iqdb_hostname_and_port
Danbooru.config.all_server_hosts.each do |host|
Post.delay(:queue => host).remove_iqdb(id)
end
end
end end
end end
@@ -1740,9 +1719,6 @@ class Post < ActiveRecord::Base
if Danbooru.config.aws_sqs_iqdb_url if Danbooru.config.aws_sqs_iqdb_url
client = SqsService.new(Danbooru.config.aws_sqs_iqdb_url) client = SqsService.new(Danbooru.config.aws_sqs_iqdb_url)
client.send_message("update\n#{id}\n#{complete_preview_file_url}") client.send_message("update\n#{id}\n#{complete_preview_file_url}")
else
Iqdb::Server.new(*Danbooru.config.iqdb_hostname_and_port).add(self)
Iqdb::Command.new(Danbooru.config.iqdb_file).add(self)
end end
end end
end end

2
app/views/posts/partials/show/_edit.html.erb
View File

@@ -72,7 +72,7 @@
<%= button_tag "Artist", :id => "find-artist-button", :type => "button" %> <%= button_tag "Artist", :id => "find-artist-button", :type => "button" %>
</div> </div>
<% if Danbooru.config.iqdb_hostname_and_port %> <% if Danbooru.config.iqdbs_server %>
<div class="input" id="iqdb-similar" style="display: none;"></div> <div class="input" id="iqdb-similar" style="display: none;"></div>
<% end %> <% end %>

2
app/views/uploads/new.html.erb
View File

@@ -88,7 +88,7 @@
</div> </div>
<% end %> <% end %>
<% if Danbooru.config.iqdb_hostname_and_port %> <% if Danbooru.config.iqdbs_server %>
<% if params[:url] %> <% if params[:url] %>
<div class="input" id="iqdb-similar"> <div class="input" id="iqdb-similar">
<p><em>Loading similar...</em></p> <p><em>Loading similar...</em></p>

8
config/danbooru_default_config.rb
View File

@@ -454,14 +454,6 @@ module Danbooru
def iqdbs_server def iqdbs_server
end end
def iqdb_hostname_and_port # TODO: remove
nil
end
def iqdb_file # TODO: remove
nil
end
# google api options # google api options
def google_api_project def google_api_project
end end