jaculabilis/danbooru
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

integrate ugoiras into zip+webm+preview

Browse Source
This commit is contained in:
r888888888
2014-10-09 17:05:47 -07:00
parent 0a61aac231
commit 3bb06c2be4
28 changed files with 1800 additions and 1125 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
app/assets/stylesheets/specific/comments.css.scss
View File

@@ -98,7 +98,7 @@ div#c-comments {
}
div.post-preview {
&[data-tags~=animated], &[data-file-ext=swf], &[data-file-ext=webm] {
&[data-tags~=animated], &[data-file-ext=swf], &[data-file-ext=webm], &[data-file-ext=zip] {
div.preview {
position: relative;

2
app/assets/stylesheets/specific/posts.css.scss
View File

@@ -22,7 +22,7 @@ article.post-preview {
margin: auto;
}
&[data-tags~=animated]:before, &[data-file-ext=swf]:before, &[data-file-ext=webm]:before {
&[data-tags~=animated]:before, &[data-file-ext=swf]:before, &[data-file-ext=webm]:before, &[data-file-ext=zip]:before {
content: "►";
position: absolute;
width: 20px;

57
app/logical/downloads/file.rb
View File

@@ -2,45 +2,49 @@ module Downloads
class File
class Error < Exception ; end
attr_reader :tries
attr_reader :data
attr_accessor :source, :content_type, :file_path
def initialize(source, file_path)
def initialize(source, file_path, options = {})
# source can potentially get rewritten in the course
# of downloading a file, so check it again
@source = source
# where to save the download
@file_path = file_path
@tries = 0
# we sometimes need to capture data from the source page
@data = {:is_ugoira => options[:is_ugoira]}
end
def download!
http_get_streaming do |response|
@source, @data = http_get_streaming(@source, @data) do |response|
self.content_type = response["Content-Type"]
::File.open(file_path, "wb") do |out|
::File.open(@file_path, "wb") do |out|
response.read_body(out)
end
end
after_download
@source = after_download(@source)
end
def before_download(url, headers)
def before_download(url, headers, datums)
RewriteStrategies::Base.strategies.each do |strategy|
url, headers = strategy.new.rewrite(url, headers)
url, headers, datums = strategy.new.rewrite(url, headers, datums)
end
return [url, headers]
return [url, headers, datums]
end
def after_download
fix_image_board_sources
def after_download(src)
fix_image_board_sources(src)
end
def url
URI.parse(source)
end
def http_get_streaming(options = {})
def http_get_streaming(src, datums = {}, options = {})
max_size = options[:max_size] || Danbooru.config.max_file_size
max_size = nil if max_size == 0 # unlimited
limit = 4
tries = 0
url = URI.parse(src)
while true
unless url.is_a?(URI::HTTP) || url.is_a?(URI::HTTPS)
@@ -50,7 +54,8 @@ module Downloads
headers = {
"User-Agent" => "#{Danbooru.config.safe_app_name}/#{Danbooru.config.version}"
}
@source, headers = before_download(source, headers)
src, headers, datums = before_download(src, headers, datums)
url = URI.parse(src)
begin
Net::HTTP.start(url.host, url.port, :use_ssl => url.is_a?(URI::HTTPS)) do |http|
@@ -63,13 +68,13 @@ module Downloads
raise Error.new("File is too large (#{len} bytes)") if len && len.to_i > max_size
end
yield(res)
return
return [src, datums]
when Net::HTTPRedirection then
if limit == 0 then
raise Error.new("Too many redirects")
end
@source = res["location"]
src = res["location"]
limit -= 1
else
@@ -78,19 +83,23 @@ module Downloads
end # http.request_get
end # http.start
rescue Errno::ECONNRESET, Errno::ETIMEDOUT, Errno::EIO, Errno::EHOSTUNREACH, Errno::ECONNREFUSED, IOError => x
@tries += 1
if @tries < 3
tries += 1
if tries < 3
retry
else
raise
end
end
end # while
[src, datums]
end # def
def fix_image_board_sources
if source =~ /i\.4cdn\.org|\/src\/\d{12,}|urnc\.yi\.org|yui\.cynthia\.bne\.jp/
@source = "Image board"
def fix_image_board_sources(src)
if src =~ /i\.4cdn\.org|\/src\/\d{12,}|urnc\.yi\.org|yui\.cynthia\.bne\.jp/
"Image board"
else
src
end
end
end

4
app/logical/downloads/rewrite_strategies/base.rb
View File

@@ -5,8 +5,8 @@ module Downloads
[Pixiv, NicoSeiga, Twitpic, DeviantArt, Tumblr, Moebooru]
end
def rewrite(url, headers)
return [url, headers]
def rewrite(url, headers, data = {})
return [url, headers, data]
end
protected

4
app/logical/downloads/rewrite_strategies/deviant_art.rb
View File

@@ -1,13 +1,13 @@
module Downloads
module RewriteStrategies
class DeviantArt < Base
def rewrite(url, headers)
def rewrite(url, headers, data = {})
if url =~ /https?:\/\/(?:.+?\.)?deviantart\.(?:com|net)/
url, headers = rewrite_html_pages(url, headers)
url, headers = rewrite_thumbnails(url, headers)
end
return [url, headers]
return [url, headers, data]
end
protected

4
app/logical/downloads/rewrite_strategies/moebooru.rb
View File

@@ -3,12 +3,12 @@ module Downloads
class Moebooru < Base
DOMAINS = '(?:[^.]+\.)?yande\.re|konachan\.com'
def rewrite(url, headers)
def rewrite(url, headers, data = {})
if url =~ %r{https?://(?:#{DOMAINS})}
url, headers = rewrite_jpeg_versions(url, headers)
end
return [url, headers]
return [url, headers, data]
end
protected

4
app/logical/downloads/rewrite_strategies/nico_seiga.rb
View File

@@ -1,14 +1,14 @@
module Downloads
module RewriteStrategies
class NicoSeiga < Base
def rewrite(url, headers)
def rewrite(url, headers, data = {})
if url =~ %r{https?://lohas\.nicoseiga\.jp} || url =~ %r{https?://seiga\.nicovideo\.jp}
url, headers = rewrite_headers(url, headers)
url, headers = rewrite_html_pages(url, headers)
url, headers = rewrite_thumbnails(url, headers)
end
return [url, headers]
return [url, headers, data]
end
protected

13
app/logical/downloads/rewrite_strategies/pixiv.rb
View File

@@ -1,16 +1,16 @@
module Downloads
module RewriteStrategies
class Pixiv < Base
def rewrite(url, headers)
def rewrite(url, headers, data = {})
if url =~ /https?:\/\/(?:\w+\.)?pixiv\.net/
url, headers = rewrite_headers(url, headers)
url, headers = rewrite_cdn(url, headers)
url, headers = rewrite_html_pages(url, headers)
url, headers, data = rewrite_html_pages(url, headers, data)
url, headers = rewrite_thumbnails(url, headers)
url, headers = rewrite_old_small_manga_pages(url, headers)
end
return [url, headers]
return [url, headers, data]
end
protected
@@ -31,9 +31,12 @@ module Downloads
if url =~ /illust_id=\d+/i || url =~ %r!pixiv\.net/img-inf/img/!i
source = ::Sources::Strategies::Pixiv.new(url)
source.get
return [source.image_url, headers]
data[:ugoira_frame_data] = source.ugoira_frame_data
data[:ugoira_width] = source.ugoira_width
data[:ugoira_height] = source.ugoira_height
return [source.file_url, headers, data]
else
return [url, headers]
return [url, headers, data]
end
end

4
app/logical/downloads/rewrite_strategies/tumblr.rb
View File

@@ -1,13 +1,13 @@
module Downloads
module RewriteStrategies
class Tumblr < Base
def rewrite(url, headers)
def rewrite(url, headers, data = {})
if url =~ %r{^https?://.*tumblr\.com}
url, headers = rewrite_cdn(url, headers)
url, headers = rewrite_thumbnails(url, headers)
end
return [url, headers]
return [url, headers, data]
end
protected

4
app/logical/downloads/rewrite_strategies/twitpic.rb
View File

@@ -1,13 +1,13 @@
module Downloads
module RewriteStrategies
class Twitpic < Base
def rewrite(url, headers)
def rewrite(url, headers, data = {})
if url =~ %r{https?://twitpic\.com} || url =~ %r{^https?://d3j5vwomefv46c\.cloudfront\.net}
url, headers = rewrite_html_pages(url, headers)
url, headers = rewrite_thumbnails(url, headers)
end
return [url, headers]
return [url, headers, data]
end
protected

109
app/logical/pixiv_ugoira_converter.rb
View File

@@ -1,43 +1,11 @@
class PixivUgoiraConverter
attr_reader :agent, :url, :write_path, :format
def initialize(url, write_path, format)
@url = url
@write_path = write_path
@format = format
def convert(source_path, output_path, preview_path, frame_data)
folder = unpack(File.open(source_path))
write_webm(folder, output_path, frame_data)
write_preview(folder, preview_path)
end
def process!
folder = unpack(fetch_zipped_body)
if format == :gif
write_gif(folder)
elsif format == :webm
write_webm(folder)
elsif format == :apng
write_apng(folder)
end
end
def write_gif(folder)
anim = Magick::ImageList.new
delay_sum = 0
folder.each_with_index do |file, i|
image_blob = file.get_input_stream.read
image = Magick::Image.from_blob(image_blob).first
image.ticks_per_second = 1000
delay = @frame_data[i]["delay"]
rounded_delay = (delay_sum + delay).round(-1) - delay_sum.round(-1)
image.delay = rounded_delay
delay_sum += delay
anim << image
end
anim = anim.optimize_layers(Magick::OptimizeTransLayer)
anim.write("gif:" + write_path)
end
def write_webm(folder)
def write_webm(folder, write_path, frame_data)
Dir.mktmpdir do |tmpdir|
FileUtils.mkdir_p("#{tmpdir}/images")
folder.each_with_index do |file, i|
@@ -62,7 +30,7 @@ class PixivUgoiraConverter
timecodes_path = File.join(tmpdir, "timecodes.tc")
File.open(timecodes_path, "w+") do |f|
f.write("# timecode format v2\n")
@frame_data.each do |img|
frame_data.each do |img|
f.write("#{delay_sum}\n")
delay_sum += img["delay"]
end
@@ -71,68 +39,21 @@ class PixivUgoiraConverter
end
ext = folder.first.name.match(/\.(\w{,4})$/)[1]
system("ffmpeg -i #{tmpdir}/images/%06d.#{ext} -codec:v libvpx -crf 4 -b:v 5000k -an #{tmpdir}/tmp.webm")
system("mkvmerge -o #{write_path} --webm --timecodes 0:#{tmpdir}/timecodes.tc #{tmpdir}/tmp.webm")
system("ffmpeg -loglevel quiet -i #{tmpdir}/images/%06d.#{ext} -codec:v libvpx -crf 4 -b:v 5000k -an #{tmpdir}/tmp.webm")
system("mkvmerge -q -o #{write_path} --webm --timecodes 0:#{tmpdir}/timecodes.tc #{tmpdir}/tmp.webm")
end
end
def write_apng(folder)
Dir.mktmpdir do |tmpdir|
folder.each_with_index do |file, i|
frame_path = File.join(tmpdir, "frame#{"%03d" % i}.png")
delay_path = File.join(tmpdir, "frame#{"%03d" % i}.txt")
image_blob = file.get_input_stream.read
delay = @frame_data[i]["delay"]
image = Magick::Image.from_blob(image_blob).first
image.format = "PNG"
image.write(frame_path)
File.open(delay_path, "wb") do |f|
f.write("delay=#{delay}/1000")
end
end
system("apngasm -o -F #{write_path} #{tmpdir}/frame*.png")
end
def write_preview(folder, path)
file = folder.first
image_blob = file.get_input_stream.read
image = Magick::Image.from_blob(image_blob).first
image.write(path)
end
def unpack(zipped_body)
def unpack(zip_file)
folder = Zip::CentralDirectory.new
folder.read_from_stream(StringIO.new(zipped_body))
folder.read_from_stream(zip_file)
folder
end
def fetch_zipped_body
zip_body = nil
zip_url, @frame_data = fetch_frames
Downloads::File.new(zip_url, nil).http_get_streaming do |response|
zip_body = response.body
end
zip_body
end
def agent
@agent ||= Sources::Strategies::Pixiv.new(url).agent
end
def fetch_frames
agent.get(url) do |page|
# Get the zip url and frame delay by parsing javascript contained in a <script> tag on the page.
# Not a neat solution, but I haven't found any other location that has the frame delays listed.
scripts = page.search("body script").find_all do |node|
node.text =~ /_ugoira600x600\.zip/
end
if scripts.any?
javascript = scripts.first.text
json = javascript.match(/;pixiv\.context\.ugokuIllustData\s+=\s+(\{.+?\});(?:$|pixiv\.context)/)[1]
data = JSON.parse(json)
zip_url = data["src"].sub("_ugoira600x600.zip", "_ugoira1920x1080.zip")
frame_data = data["frames"]
return [zip_url, frame_data]
else
raise "Can't find javascript with frame data"
end
end
end
end

21
app/logical/pixiv_ugoira_service.rb Normal file
View File

@@ -0,0 +1,21 @@
class PixivUgoiraService
attr_reader :width, :height, :frame_data
def process(post)
save_frame_data(post)
end
def save_frame_data(post)
PixivUgoiraFrameData.create(:data => @frame_data, :post_id => post.id)
end
def generate_resizes(source_path, output_path, preview_path)
PixivUgoiraConverter.new.convert(source_path, output_path, preview_path, @frame_data)
end
def load(data)
@frame_data = data[:ugoira_frame_data]
@width = data[:ugoira_width]
@height = data[:ugoira_height]
end
end

24
app/logical/pixiv_web_agent.rb Normal file
View File

@@ -0,0 +1,24 @@
class PixivWebAgent
def self.build
mech = Mechanize.new
phpsessid = Cache.get("pixiv-phpsessid")
if phpsessid
cookie = Mechanize::Cookie.new("PHPSESSID", phpsessid)
cookie.domain = ".pixiv.net"
cookie.path = "/"
mech.cookie_jar.add(cookie)
else
mech.get("http://www.pixiv.net") do |page|
page.form_with(:action => "/login.php") do |form|
form['pixiv_id'] = Danbooru.config.pixiv_login
form['pass'] = Danbooru.config.pixiv_password
end.click_button
end
phpsessid = mech.cookie_jar.cookies.select{|c| c.name == "PHPSESSID"}.first
Cache.put("pixiv-phpsessid", phpsessid.value, 1.month) if phpsessid
end
mech
end
end

2
app/logical/sources/site.rb
View File

@@ -5,7 +5,7 @@ module Sources
class Site
attr_reader :url, :strategy
delegate :get, :referer_url, :site_name, :artist_name, :profile_url, :image_url, :tags, :artist_record, :unique_id, :page_count, :to => :strategy
delegate :get, :referer_url, :site_name, :artist_name, :profile_url, :image_url, :tags, :artist_record, :unique_id, :page_count, :file_url, :ugoira_frame_data, :ugoira_width, :ugoira_height, :to => :strategy
def self.strategies
[Strategies::Pixiv, Strategies::NicoSeiga, Strategies::DeviantArt, Strategies::Nijie]

54
app/logical/sources/strategies/pixiv.rb
View File

@@ -5,6 +5,8 @@ require 'csv'
module Sources
module Strategies
class Pixiv < Base
attr_reader :zip_url, :ugoira_frame_data, :ugoira_width, :ugoira_height
def self.url_match?(url)
url =~ /^https?:\/\/(?:\w+\.)?pixiv\.net/
end
@@ -43,6 +45,8 @@ module Sources
agent.get(URI.parse(normalized_url)) do |page|
@artist_name, @profile_url = get_profile_from_page(page)
@pixiv_moniker = get_moniker_from_page(page)
@image_url = get_image_url_from_page(page)
@zip_url, @ugoira_frame_data, @ugoira_width, @ugoira_height = get_zip_url_from_page(page)
@tags = get_tags_from_page(page)
@page_count = get_page_count_from_page(page)
@@ -58,28 +62,11 @@ module Sources
end
def agent
@agent ||= begin
mech = Mechanize.new
@agent ||= PixivWebAgent.build
end
phpsessid = Cache.get("pixiv-phpsessid")
if phpsessid
cookie = Mechanize::Cookie.new("PHPSESSID", phpsessid)
cookie.domain = ".pixiv.net"
cookie.path = "/"
mech.cookie_jar.add(cookie)
else
mech.get("http://www.pixiv.net") do |page|
page.form_with(:action => "/login.php") do |form|
form['pixiv_id'] = Danbooru.config.pixiv_login
form['pass'] = Danbooru.config.pixiv_password
end.click_button
end
phpsessid = mech.cookie_jar.cookies.select{|c| c.name == "PHPSESSID"}.first
Cache.put("pixiv-phpsessid", phpsessid.value, 1.month) if phpsessid
end
mech
end
def file_url
image_url || zip_url
end
protected
@@ -191,6 +178,31 @@ module Sources
end
end
def get_zip_url_from_page(page)
scripts = page.search("body script").find_all do |node|
node.text =~ /_ugoira600x600\.zip/
end
if scripts.any?
javascript = scripts.first.text
json = javascript.match(/;pixiv\.context\.ugokuIllustData\s+=\s+(\{.+?\});(?:$|pixiv\.context)/)[1]
data = JSON.parse(json)
zip_url = data["src"].sub("_ugoira600x600.zip", "_ugoira1920x1080.zip")
frame_data = data["frames"]
if javascript =~ /illustSize\s*=\s*\[\s*(\d+)\s*,\s*(\d+)\s*\]/
image_width = $1.to_i
image_height = $2.to_i
else
image_width = 600
image_height = 600
end
return [zip_url, frame_data, image_width, image_height]
end
end
def get_tags_from_page(page)
# puts page.root.to_xhtml

4
app/models/pixiv_ugoira_frame_data.rb Normal file
View File

@@ -0,0 +1,4 @@
class PixivUgoiraFrameData < ActiveRecord::Base
attr_accessible :post_id, :data
serialize :data
end

13
app/models/post.rb
View File

@@ -26,6 +26,7 @@ class Post < ActiveRecord::Base
belongs_to :parent, :class_name => "Post"
has_one :upload, :dependent => :destroy
has_one :artist_commentary, :dependent => :destroy
has_one :pixiv_ugoira_frame_data, :class_name => "PixivUgoiraFrameData"
has_many :flags, :class_name => "PostFlag", :dependent => :destroy
has_many :appeals, :class_name => "PostAppeal", :dependent => :destroy
has_many :versions, lambda {order("post_versions.updated_at ASC, post_versions.id ASC")}, :class_name => "PostVersion", :dependent => :destroy
@@ -70,12 +71,20 @@ class Post < ActiveRecord::Base
def large_file_path
if has_large?
"#{Rails.root}/public/data/sample/#{file_path_prefix}#{Danbooru.config.large_image_prefix}#{md5}.jpg"
"#{Rails.root}/public/data/sample/#{file_path_prefix}#{Danbooru.config.large_image_prefix}#{md5}.#{large_file_ext}"
else
file_path
end
end
def large_file_ext
if is_ugoira?
"webm"
else
"jpg"
end
end
def preview_file_path
"#{Rails.root}/public/data/preview/#{file_path_prefix}#{md5}.jpg"
end
@@ -129,7 +138,7 @@ class Post < ActiveRecord::Base
end
def is_video?
file_ext =~ /webm/i
file_ext =~ /webm|zip/i
end
def has_preview?

61
app/models/upload.rb
View File

@@ -112,6 +112,7 @@ class Upload < ActiveRecord::Base
post.distribute_files
if post.save
CurrentUser.increment!(:post_upload_count)
ugoira_service.process(post)
update_attributes(:status => "completed", :post_id => post.id)
else
update_attribute(:status, "error: " + post.errors.full_messages.join(", "))
@@ -140,6 +141,10 @@ class Upload < ActiveRecord::Base
delete_temp_file
end
def ugoira_service
@ugoira_service ||= PixivUgoiraService.new
end
def convert_to_post
Post.new.tap do |p|
p.tag_string = tag_string
@@ -190,16 +195,32 @@ class Upload < ActiveRecord::Base
def is_video?
%w(webm).include?(file_ext)
end
def is_ugoira?
%w(zip).include?(file_ext)
end
end
module ResizerMethods
def generate_resizes(source_path)
generate_resize_for(Danbooru.config.small_image_width, Danbooru.config.small_image_width, source_path, 85)
if is_image? && image_width > Danbooru.config.large_image_width
generate_resize_for(Danbooru.config.large_image_width, nil, source_path)
end
end
def generate_video_preview_for(width, height, output_path)
dimension_ratio = image_width.to_f / image_height
if dimension_ratio > 1
height = (width / dimension_ratio).to_i
else
width = (height * dimension_ratio).to_i
end
video.screenshot(output_path, {:seek_time => 0, :resolution => "#{width}x#{height}"})
FileUtils.chmod(0664, output_path)
end
def generate_resize_for(width, height, source_path, quality = 90)
unless File.exists?(source_path)
raise Error.new("file not found")
@@ -208,15 +229,10 @@ class Upload < ActiveRecord::Base
output_path = resized_file_path_for(width)
if is_image?
Danbooru.resize(source_path, output_path, width, height, quality)
elsif is_ugoira?
ugoira_service.generate_resizes(source_path, resized_file_path_for(Danbooru.config.large_image_width), resized_file_path_for(Danbooru.config.small_image_width))
elsif is_video?
dimension_ratio = image_width.to_f / image_height
if dimension_ratio > 1
height = (width / dimension_ratio).to_i
else
width = (height * dimension_ratio).to_i
end
video.screenshot(output_path, {:seek_time => 0, :resolution => "#{width}x#{height}"})
FileUtils.chmod(0664, output_path)
generate_video_preview_for(width, height, output_path)
end
end
end
@@ -227,6 +243,9 @@ class Upload < ActiveRecord::Base
if is_video?
self.image_width = video.width
self.image_height = video.height
elsif is_ugoira?
self.image_width = ugoira_service.width
self.image_height = ugoira_service.height
else
File.open(file_path, "rb") do |file|
image_size = ImageSpec.new(file)
@@ -238,13 +257,13 @@ class Upload < ActiveRecord::Base
# Does this file have image dimensions?
def has_dimensions?
%w(jpg gif png swf webm).include?(file_ext)
%w(jpg gif png swf webm zip).include?(file_ext)
end
end
module ContentTypeMethods
def is_valid_content_type?
file_ext =~ /jpg|gif|png|swf|webm/
file_ext =~ /jpg|gif|png|swf|webm|zip/
end
def content_type_to_file_ext(content_type)
@@ -264,6 +283,9 @@ class Upload < ActiveRecord::Base
when "video/webm"
"webm"
when "application/zip"
"zip"
else
"bin"
end
@@ -286,6 +308,9 @@ class Upload < ActiveRecord::Base
when /^\x1a\x45\xdf\xa3/
"video/webm"
when /^PK\x03\x04/
"application/zip"
else
"application/octet-stream"
end
@@ -321,23 +346,17 @@ class Upload < ActiveRecord::Base
source =~ /^https?:\/\// && file_path.blank?
end
def is_ugoira?
def has_ugoira_tag?
tag_string =~ /\bugoira\b/i
end
# Downloads the file to destination_path
def download_from_source(destination_path)
self.file_path = destination_path
if is_ugoira?
converter = PixivUgoiraConverter.new(source, destination_path, :webm)
converter.process!
self.source = source
else
download = Downloads::File.new(source, destination_path)
download.download!
self.source = download.source
end
download = Downloads::File.new(source, destination_path, :is_ugoira => has_ugoira_tag?)
download.download!
self.source = download.source
ugoira_service.load(download.data)
end
end