jaculabilis/danbooru
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

integrate ugoiras into zip+webm+preview

Browse Source
This commit is contained in:
r888888888
2014-10-09 17:05:47 -07:00
parent 0a61aac231
commit 3bb06c2be4
28 changed files with 1800 additions and 1125 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
app/logical/downloads/file.rb
View File

@@ -2,45 +2,49 @@ module Downloads
class File
class Error < Exception ; end
attr_reader :tries
attr_reader :data
attr_accessor :source, :content_type, :file_path
def initialize(source, file_path)
def initialize(source, file_path, options = {})
# source can potentially get rewritten in the course
# of downloading a file, so check it again
@source = source
# where to save the download
@file_path = file_path
@tries = 0
# we sometimes need to capture data from the source page
@data = {:is_ugoira => options[:is_ugoira]}
end
def download!
http_get_streaming do |response|
@source, @data = http_get_streaming(@source, @data) do |response|
self.content_type = response["Content-Type"]
::File.open(file_path, "wb") do |out|
::File.open(@file_path, "wb") do |out|
response.read_body(out)
end
end
after_download
@source = after_download(@source)
end
def before_download(url, headers)
def before_download(url, headers, datums)
RewriteStrategies::Base.strategies.each do |strategy|
url, headers = strategy.new.rewrite(url, headers)
url, headers, datums = strategy.new.rewrite(url, headers, datums)
end
return [url, headers]
return [url, headers, datums]
end
def after_download
fix_image_board_sources
def after_download(src)
fix_image_board_sources(src)
end
def url
URI.parse(source)
end
def http_get_streaming(options = {})
def http_get_streaming(src, datums = {}, options = {})
max_size = options[:max_size] || Danbooru.config.max_file_size
max_size = nil if max_size == 0 # unlimited
limit = 4
tries = 0
url = URI.parse(src)
while true
unless url.is_a?(URI::HTTP) || url.is_a?(URI::HTTPS)
@@ -50,7 +54,8 @@ module Downloads
headers = {
"User-Agent" => "#{Danbooru.config.safe_app_name}/#{Danbooru.config.version}"
}
@source, headers = before_download(source, headers)
src, headers, datums = before_download(src, headers, datums)
url = URI.parse(src)
begin
Net::HTTP.start(url.host, url.port, :use_ssl => url.is_a?(URI::HTTPS)) do |http|
@@ -63,13 +68,13 @@ module Downloads
raise Error.new("File is too large (#{len} bytes)") if len && len.to_i > max_size
end
yield(res)
return
return [src, datums]
when Net::HTTPRedirection then
if limit == 0 then
raise Error.new("Too many redirects")
end
@source = res["location"]
src = res["location"]
limit -= 1
else
@@ -78,19 +83,23 @@ module Downloads
end # http.request_get
end # http.start
rescue Errno::ECONNRESET, Errno::ETIMEDOUT, Errno::EIO, Errno::EHOSTUNREACH, Errno::ECONNREFUSED, IOError => x
@tries += 1
if @tries < 3
tries += 1
if tries < 3
retry
else
raise
end
end
end # while
[src, datums]
end # def
def fix_image_board_sources
if source =~ /i\.4cdn\.org|\/src\/\d{12,}|urnc\.yi\.org|yui\.cynthia\.bne\.jp/
@source = "Image board"
def fix_image_board_sources(src)
if src =~ /i\.4cdn\.org|\/src\/\d{12,}|urnc\.yi\.org|yui\.cynthia\.bne\.jp/
"Image board"
else
src
end
end
end

4
app/logical/downloads/rewrite_strategies/base.rb
View File

@@ -5,8 +5,8 @@ module Downloads
[Pixiv, NicoSeiga, Twitpic, DeviantArt, Tumblr, Moebooru]
end
def rewrite(url, headers)
return [url, headers]
def rewrite(url, headers, data = {})
return [url, headers, data]
end
protected

4
app/logical/downloads/rewrite_strategies/deviant_art.rb
View File

@@ -1,13 +1,13 @@
module Downloads
module RewriteStrategies
class DeviantArt < Base
def rewrite(url, headers)
def rewrite(url, headers, data = {})
if url =~ /https?:\/\/(?:.+?\.)?deviantart\.(?:com|net)/
url, headers = rewrite_html_pages(url, headers)
url, headers = rewrite_thumbnails(url, headers)
end
return [url, headers]
return [url, headers, data]
end
protected

4
app/logical/downloads/rewrite_strategies/moebooru.rb
View File

@@ -3,12 +3,12 @@ module Downloads
class Moebooru < Base
DOMAINS = '(?:[^.]+\.)?yande\.re|konachan\.com'
def rewrite(url, headers)
def rewrite(url, headers, data = {})
if url =~ %r{https?://(?:#{DOMAINS})}
url, headers = rewrite_jpeg_versions(url, headers)
end
return [url, headers]
return [url, headers, data]
end
protected

4
app/logical/downloads/rewrite_strategies/nico_seiga.rb
View File

@@ -1,14 +1,14 @@
module Downloads
module RewriteStrategies
class NicoSeiga < Base
def rewrite(url, headers)
def rewrite(url, headers, data = {})
if url =~ %r{https?://lohas\.nicoseiga\.jp} || url =~ %r{https?://seiga\.nicovideo\.jp}
url, headers = rewrite_headers(url, headers)
url, headers = rewrite_html_pages(url, headers)
url, headers = rewrite_thumbnails(url, headers)
end
return [url, headers]
return [url, headers, data]
end
protected

13
app/logical/downloads/rewrite_strategies/pixiv.rb
View File

@@ -1,16 +1,16 @@
module Downloads
module RewriteStrategies
class Pixiv < Base
def rewrite(url, headers)
def rewrite(url, headers, data = {})
if url =~ /https?:\/\/(?:\w+\.)?pixiv\.net/
url, headers = rewrite_headers(url, headers)
url, headers = rewrite_cdn(url, headers)
url, headers = rewrite_html_pages(url, headers)
url, headers, data = rewrite_html_pages(url, headers, data)
url, headers = rewrite_thumbnails(url, headers)
url, headers = rewrite_old_small_manga_pages(url, headers)
end
return [url, headers]
return [url, headers, data]
end
protected
@@ -31,9 +31,12 @@ module Downloads
if url =~ /illust_id=\d+/i || url =~ %r!pixiv\.net/img-inf/img/!i
source = ::Sources::Strategies::Pixiv.new(url)
source.get
return [source.image_url, headers]
data[:ugoira_frame_data] = source.ugoira_frame_data
data[:ugoira_width] = source.ugoira_width
data[:ugoira_height] = source.ugoira_height
return [source.file_url, headers, data]
else
return [url, headers]
return [url, headers, data]
end
end

4
app/logical/downloads/rewrite_strategies/tumblr.rb
View File

@@ -1,13 +1,13 @@
module Downloads
module RewriteStrategies
class Tumblr < Base
def rewrite(url, headers)
def rewrite(url, headers, data = {})
if url =~ %r{^https?://.*tumblr\.com}
url, headers = rewrite_cdn(url, headers)
url, headers = rewrite_thumbnails(url, headers)
end
return [url, headers]
return [url, headers, data]
end
protected

4
app/logical/downloads/rewrite_strategies/twitpic.rb
View File

@@ -1,13 +1,13 @@
module Downloads
module RewriteStrategies
class Twitpic < Base
def rewrite(url, headers)
def rewrite(url, headers, data = {})
if url =~ %r{https?://twitpic\.com} || url =~ %r{^https?://d3j5vwomefv46c\.cloudfront\.net}
url, headers = rewrite_html_pages(url, headers)
url, headers = rewrite_thumbnails(url, headers)
end
return [url, headers]
return [url, headers, data]
end
protected