diff --git a/app/logical/downloads/file.rb b/app/logical/downloads/file.rb index 9699b0942..9997fc802 100644 --- a/app/logical/downloads/file.rb +++ b/app/logical/downloads/file.rb @@ -26,20 +26,23 @@ module Downloads end def size - @source, headers, @data = before_download(@source, @data) + url, headers, _ = before_download(@source, @data) options = { timeout: 3, headers: headers }.deep_merge(Danbooru.config.httparty_options) - res = HTTParty.head(@source, options) + res = HTTParty.head(url, options) res.content_length end def download! + url, headers, @data = before_download(@source, @data) + ::File.open(@file_path, "wb") do |out| - @source, @data = http_get_streaming(@source, @data) do |response| + http_get_streaming(uncached_url(url), headers) do |response| out.write(response) end end - @downloaded_source = @source - @source = after_download(@source) + + @downloaded_source = url + @source = after_download(url) end def before_download(url, datums) @@ -52,6 +55,13 @@ module Downloads return [url, headers, datums] end + # Prevent transparent proxies (namely Cloudflare) from potentially mangling the image. See issue #3528. + def uncached_url(url) + url = Addressable::URI.parse(url) + url.query_values = (url.query_values || {}).merge(danbooru_no_cache: SecureRandom.uuid) + url + end + def after_download(src) src = fix_twitter_sources(src) if options[:referer_url].present? @@ -67,7 +77,7 @@ module Downloads end end - def http_get_streaming(src, datums = {}, options = {}, &block) + def http_get_streaming(src, headers = {}, options = {}, &block) max_size = options[:max_size] || Danbooru.config.max_file_size max_size = nil if max_size == 0 # unlimited limit = 4 @@ -79,9 +89,6 @@ module Downloads raise Error.new("URL must be HTTP or HTTPS") end - src, headers, datums = before_download(src, datums) - url = URI.parse(src) - validate_local_hosts(url) begin @@ -96,7 +103,7 @@ module Downloads @content_type = res["Content-Type"] - return [src, datums] + return else raise Error.new("HTTP error code: #{res.code} #{res.message}") end @@ -109,8 +116,6 @@ module Downloads end end end # while - - [src, datums] end # def def fix_twitter_sources(src) diff --git a/test/unit/downloads/art_station_test.rb b/test/unit/downloads/art_station_test.rb index a7db904d5..077007b8d 100644 --- a/test/unit/downloads/art_station_test.rb +++ b/test/unit/downloads/art_station_test.rb @@ -28,6 +28,13 @@ module Downloads end end + context "a download for an ArtStation image hosted on CloudFlare" do + should "return the original file, not the polished file" do + @source = "https://cdnb.artstation.com/p/assets/images/images/003/716/071/large/aoi-ogata-hate-city.jpg?1476754974" + assert_downloaded(517_706, @source) # polished size: 502_052 + end + end + context "a download for a https://$artist.artstation.com/projects/$id page" do setup do @source = "https://dantewontdie.artstation.com/projects/YZK5q"