Fix #3910: Corrupted images during upload.
Use a fresh tempfile for each download attempt instead of reusing the same file (and having to rewind/truncate it after each failed attempt).
This commit is contained in:
@@ -54,17 +54,15 @@ module Downloads
|
|||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
def download!(tries: 3)
|
def download!(tries: 3, **options)
|
||||||
strategy = Sources::Strategies.find(source, referer)
|
strategy = Sources::Strategies.find(source, referer)
|
||||||
output_file = Tempfile.new(binmode: true)
|
|
||||||
url = self.class.uncached_url(strategy.file_url, strategy.headers)
|
url = self.class.uncached_url(strategy.file_url, strategy.headers)
|
||||||
@data = strategy.data
|
@data = strategy.data
|
||||||
|
|
||||||
Retriable.retriable(on: RETRIABLE_ERRORS, tries: tries, base_interval: 0) do
|
Retriable.retriable(on: RETRIABLE_ERRORS, tries: tries, base_interval: 0) do
|
||||||
http_get_streaming(url, output_file, strategy.headers)
|
file = http_get_streaming(url, headers: strategy.headers, **options)
|
||||||
|
return [file, strategy]
|
||||||
end
|
end
|
||||||
|
|
||||||
[output_file, strategy]
|
|
||||||
end
|
end
|
||||||
|
|
||||||
def validate_local_hosts(url)
|
def validate_local_hosts(url)
|
||||||
@@ -74,7 +72,7 @@ module Downloads
|
|||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
def http_get_streaming(src, file, headers = {}, max_size: Danbooru.config.max_file_size)
|
def http_get_streaming(src, file: Tempfile.new(binmode: true), headers: {}, max_size: Danbooru.config.max_file_size)
|
||||||
url = URI.parse(src)
|
url = URI.parse(src)
|
||||||
|
|
||||||
unless url.is_a?(URI::HTTP) || url.is_a?(URI::HTTPS)
|
unless url.is_a?(URI::HTTP) || url.is_a?(URI::HTTPS)
|
||||||
|
|||||||
@@ -6,7 +6,6 @@ module Downloads
|
|||||||
setup do
|
setup do
|
||||||
@source = "http://www.google.com/intl/en_ALL/images/logo.gif"
|
@source = "http://www.google.com/intl/en_ALL/images/logo.gif"
|
||||||
@download = Downloads::File.new(@source)
|
@download = Downloads::File.new(@source)
|
||||||
@tempfile = Tempfile.new("danbooru-test")
|
|
||||||
end
|
end
|
||||||
|
|
||||||
context "that fails" do
|
context "that fails" do
|
||||||
@@ -14,11 +13,22 @@ module Downloads
|
|||||||
HTTParty.expects(:get).times(3).raises(Errno::ETIMEDOUT)
|
HTTParty.expects(:get).times(3).raises(Errno::ETIMEDOUT)
|
||||||
assert_raises(Errno::ETIMEDOUT) { @download.download! }
|
assert_raises(Errno::ETIMEDOUT) { @download.download! }
|
||||||
end
|
end
|
||||||
|
|
||||||
|
should "return an uncorrupted file on the second try" do
|
||||||
|
bomb = stub("bomb")
|
||||||
|
bomb.expects(:size).raises(IOError)
|
||||||
|
resp = stub("resp", success?: true)
|
||||||
|
|
||||||
|
HTTParty.expects(:get).twice.multiple_yields("a", bomb, "b", "c").then.multiple_yields("a", "b", "c").returns(resp)
|
||||||
|
tempfile, _ = @download.download!
|
||||||
|
|
||||||
|
assert_equal("abc", tempfile.read)
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
should "throw an exception when the file is larger than the maximum" do
|
should "throw an exception when the file is larger than the maximum" do
|
||||||
assert_raise(Downloads::File::Error) do
|
assert_raise(Downloads::File::Error) do
|
||||||
@download.http_get_streaming(@source, @tempfile, {}, max_size: 1)
|
@download.download!(max_size: 1)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user