jaculabilis/danbooru
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

integrate ugoira converted into upload flow

Browse Source
This commit is contained in:
r888888888
2014-10-06 14:20:17 -07:00
parent 50730c6d0d
commit fb2219d4ac
7 changed files with 1167 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
app/logical/downloads/file.rb
View File

@@ -11,6 +11,11 @@ module Downloads
@tries = 0
end
def download_ugoira!
converter = PixivUgoiraConverter.new(source, file_path, :webm)
converter.process!
end
def download!
http_get_streaming do |response|
self.content_type = response["Content-Type"]

44
app/logical/pixiv_ugoira_converter.rb
View File

@@ -1,14 +1,13 @@
class PixivUgoiraConverter
attr_reader :agent, :url, :write_path, :format
def initialize(agent, url, write_path, format)
@agent = agent
def initialize(url, write_path, format)
@url = url
@write_path = write_path
@format = format
end
def process
def process!
folder = unpack(fetch_zipped_body)
if format == :gif
@@ -78,23 +77,21 @@ class PixivUgoiraConverter
end
def write_apng(folder)
FileUtils.mkdir_p("pixiv_anim_#{pixiv_id}")
Dir.mktmpdir do |tmpdir|
folder.each_with_index do |file, i|
frame_path = File.join("pixiv_anim_#{pixiv_id}", "frame#{"%03d" % i}.png")
delay_path = File.join("pixiv_anim_#{pixiv_id}", "frame#{"%03d" % i}.txt")
frame_path = File.join(tmpdir, "frame#{"%03d" % i}.png")
delay_path = File.join(tmpdir, "frame#{"%03d" % i}.txt")
image_blob = file.get_input_stream.read
delay = @frame_data[i]["delay"]
image = Magick::Image.from_blob(image_blob).first
image.format="PNG"
image.write(frame_path)
File.open(delay_path, "wb") do |f|
delay = @frame_data[i]["delay"]
image = Magick::Image.from_blob(image_blob).first
image.format = "PNG"
image.write(frame_path)
File.open(delay_path, "wb") do |f|
f.write("delay=#{delay}/1000")
end
end
system("apngasm pixiv_anim_#{pixiv_id}.png pixiv_anim_#{pixiv_id}/frame*.png")
FileUtils.rm_rf("pixiv_anim_#{pixiv_id}")
puts "Animation successfully created as pixiv_anim_#{pixiv_id}.png"
system("apngasm -o -F #{write_path} #{tmpdir}/frame*.png")
end
end
def unpack(zipped_body)
@@ -104,11 +101,18 @@ class PixivUgoiraConverter
end
def fetch_zipped_body
zip_uri, @frame_data = fetch_frames
zip_body = nil
zip_url, @frame_data = fetch_frames
zip_body = Net::HTTP.start(zip_uri.host) do |http|
http.get(zip_uri.path, {"Referer" => "http://pixiv.net"}).body
Downloads::File.new(zip_url, nil).http_get_streaming do |response|
zip_body = response.body
end
zip_body
end
def agent
@agent ||= Sources::Strategies::Pixiv.new(url).agent
end
def fetch_frames
@@ -123,9 +127,9 @@ class PixivUgoiraConverter
javascript = scripts.first.text
json = javascript.match(/;pixiv\.context\.ugokuIllustData\s+=\s+(\{.+?\});(?:$|pixiv\.context)/)[1]
data = JSON.parse(json)
zip_uri = URI(data["src"].sub("_ugoira600x600.zip", "_ugoira1920x1080.zip"))
zip_url = data["src"].sub("_ugoira600x600.zip", "_ugoira1920x1080.zip")
frame_data = data["frames"]
return [zip_uri, frame_data]
return [zip_url, frame_data]
else
raise "Can't find javascript with frame data"
end

25
app/logical/sources/strategies/pixiv.rb
View File

@@ -57,6 +57,31 @@ module Sources
return thumbnail_url
end
def agent
@agent ||= begin
mech = Mechanize.new
phpsessid = Cache.get("pixiv-phpsessid")
if phpsessid
cookie = Mechanize::Cookie.new("PHPSESSID", phpsessid)
cookie.domain = ".pixiv.net"
cookie.path = "/"
mech.cookie_jar.add(cookie)
else
mech.get("http://www.pixiv.net") do |page|
page.form_with(:action => "/login.php") do |form|
form['pixiv_id'] = Danbooru.config.pixiv_login
form['pass'] = Danbooru.config.pixiv_password
end.click_button
end
phpsessid = mech.cookie_jar.cookies.select{|c| c.name == "PHPSESSID"}.first
Cache.put("pixiv-phpsessid", phpsessid.value, 1.month) if phpsessid
end
mech
end
end
protected
# http://i1.pixiv.net/c/600x600/img-master/img/2014/10/02/13/51/23/46304396_p1_master1200.jpg

10
app/models/upload.rb
View File

@@ -321,10 +321,18 @@ class Upload < ActiveRecord::Base
source =~ /^https?:\/\// && file_path.blank?
end
def is_ugoira?
tag_string =~ /\bugoira\b/i
end
# Downloads the file to destination_path
def download_from_source(destination_path)
download = Downloads::File.new(source, destination_path)
download.download!
if is_ugoira?
download.download_ugoira!
else
download.download!
end
self.file_path = destination_path
self.source = download.source
end

1078
test/fixtures/vcr_cassettes/ugoira-converter.yml vendored
View File

File diff suppressed because it is too large Load Diff

44
test/unit/pixiv_ugoira_converter_test.rb
View File

@@ -4,28 +4,6 @@ class PixivUgoiraConverterTest < ActiveSupport::TestCase
context "An ugoira converter" do
setup do
@url = "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=46378654"
@agent = begin
mech = Mechanize.new
phpsessid = Cache.get("pixiv-phpsessid")
if phpsessid
cookie = Mechanize::Cookie.new("PHPSESSID", phpsessid)
cookie.domain = ".pixiv.net"
cookie.path = "/"
mech.cookie_jar.add(cookie)
else
mech.get("http://www.pixiv.net") do |page|
page.form_with(:action => "/login.php") do |form|
form['pixiv_id'] = Danbooru.config.pixiv_login
form['pass'] = Danbooru.config.pixiv_password
end.click_button
end
phpsessid = mech.cookie_jar.cookies.select{|c| c.name == "PHPSESSID"}.first
Cache.put("pixiv-phpsessid", phpsessid.value, 1.month) if phpsessid
end
mech
end
@write_file = Tempfile.new("output")
end
@@ -34,27 +12,27 @@ class PixivUgoiraConverterTest < ActiveSupport::TestCase
end
should "output to gif" do
@converter = PixivUgoiraConverter.new(@agent, @url, @write_file.path, :gif)
@converter = PixivUgoiraConverter.new(@url, @write_file.path, :gif)
VCR.use_cassette("ugoira-converter", :record => :new_episodes) do
@converter.process
@converter.process!
end
assert_operator(File.size(@converter.write_path), :>, 1_000)
end
should "output to webm" do
@converter = PixivUgoiraConverter.new(@agent, @url, @write_file.path, :webm)
@converter = PixivUgoiraConverter.new(@url, @write_file.path, :webm)
VCR.use_cassette("ugoira-converter", :record => :new_episodes) do
@converter.process
@converter.process!
end
assert_operator(File.size(@converter.write_path), :>, 1_000)
end
should "output to apng" do
@converter = PixivUgoiraConverter.new(@agent, @url, @write_file.path, :apng)
VCR.use_cassette("ugoira-converter", :record => :new_episodes) do
@converter.process
end
assert_operator(File.size(@converter.write_path), :>, 1_000)
end
# should "output to apng" do
# @converter = PixivUgoiraConverter.new(@url, @write_file.path, :apng)
# VCR.use_cassette("ugoira-converter", :record => :new_episodes) do
# @converter.process!
# end
# assert_operator(File.size(@converter.write_path), :>, 1_000)
# end
end
end

15
test/unit/upload_test.rb
View File

@@ -101,6 +101,21 @@ class UploadTest < ActiveSupport::TestCase
end
context "downloader" do
context "that is an ugoira" do
setup do
@url = "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=46378654"
@upload = FactoryGirl.create(:source_upload, :source => @url, :tag_string => "ugoira")
@output_path = "#{Rails.root}/tmp/test.download.webm"
end
should "process successfully" do
VCR.use_cassette("ugoira-converter", :record => :new_episodes) do
@upload.download_from_source(@output_path)
end
assert_operator(File.size(@output_path), :>, 1_000)
end
end
should "initialize the final path after downloading a file" do
@upload = FactoryGirl.create(:source_upload)
path = "#{Rails.root}/tmp/test.download.jpg"