Merge pull request #3563 from evazion/fix-3528

Fix #3528: Add method to prevent image-hosting CloudFlare sites from altering the image

app/logical/downloads/file.rb

@@ -26,20 +26,23 @@ module Downloads
     end
 
     def size
-      @source, headers, @data = before_download(@source, @data)
+      url, headers, _ = before_download(@source, @data)
       options = { timeout: 3, headers: headers }.deep_merge(Danbooru.config.httparty_options)
-      res = HTTParty.head(@source, options)
+      res = HTTParty.head(url, options)
       res.content_length
     end
 
     def download!
+      url, headers, @data = before_download(@source, @data)
+
       ::File.open(@file_path, "wb") do |out|
-        @source, @data = http_get_streaming(@source, @data) do |response|
+        http_get_streaming(uncached_url(url, headers), headers) do |response|
           out.write(response)
         end
       end
-      @downloaded_source = @source
+
-      @source = after_download(@source)
+      @downloaded_source = url
+      @source = after_download(url)
     end
 
     def before_download(url, datums)
@@ -67,7 +70,7 @@ module Downloads
       end
     end
 
-    def http_get_streaming(src, datums = {}, options = {}, &block)
+    def http_get_streaming(src, headers = {}, options = {}, &block)
       max_size = options[:max_size] || Danbooru.config.max_file_size
       max_size = nil if max_size == 0 # unlimited
       limit = 4
@@ -79,9 +82,6 @@ module Downloads
           raise Error.new("URL must be HTTP or HTTPS")
         end
 
-        src, headers, datums = before_download(src, datums)
-        url = URI.parse(src)
-
         validate_local_hosts(url)
 
         begin
@@ -96,7 +96,7 @@ module Downloads
 
             @content_type = res["Content-Type"]
 
-            return [src, datums]
+            return
           else
             raise Error.new("HTTP error code: #{res.code} #{res.message}")
           end
@@ -109,8 +109,6 @@ module Downloads
           end
         end
       end # while
-
-      [src, datums]
     end # def
 
     def fix_twitter_sources(src)
@@ -135,5 +133,27 @@ module Downloads
         src
       end
     end
+
+    private
+
+    # Prevent Cloudflare from potentially mangling the image. See issue #3528.
+    def uncached_url(url, headers = {})
+      url = Addressable::URI.parse(url)
+
+      if is_cloudflare?(url, headers)
+        url.query_values = (url.query_values || {}).merge(danbooru_no_cache: SecureRandom.uuid)
+      end
+
+      url
+    end
+
+    def is_cloudflare?(url, headers = {})
+      Cache.get("is_cloudflare:#{url.origin}", 4.hours) do
+        res = HTTParty.head(url, { headers: headers }.deep_merge(Danbooru.config.httparty_options))
+        raise Error.new("HTTP error code: #{res.code} #{res.message}") unless res.success?
+
+        res.key?("CF-Ray")
+      end
+    end
   end
 end

test/test_helpers/download_helper.rb

@@ -15,7 +15,7 @@ module DownloadTestHelper
     download = Downloads::File.new(test_source, tempfile.path)
 
     rewritten_source, _, _ = download.before_download(test_source, {})
-    assert_equal(expected_source, rewritten_source, "Tested source URL: #{test_source}")
+    assert_match(expected_source, rewritten_source, "Tested source URL: #{test_source}")
   end
 
   def assert_not_rewritten(source)

test/unit/downloads/art_station_test.rb

@@ -28,6 +28,13 @@ module Downloads
       end
     end
 
+    context "a download for an ArtStation image hosted on CloudFlare" do
+      should "return the original file, not the polished file" do
+        @source = "https://cdnb.artstation.com/p/assets/images/images/003/716/071/large/aoi-ogata-hate-city.jpg?1476754974"
+        assert_downloaded(517_706, @source) # polished size: 502_052
+      end
+    end
+
     context "a download for a https://$artist.artstation.com/projects/$id page" do
       setup do
         @source = "https://dantewontdie.artstation.com/projects/YZK5q"

test/unit/downloads/moebooru_test.rb

@@ -0,0 +1,14 @@
+require "test_helper"
+
+module Downloads
+  class MoebooruTest < ActiveSupport::TestCase
+    context "downloading a 'https://yande.re/jpeg/:hash/:file.jpg' jpeg sample url" do
+      should "download the original file" do
+        @source = "https://yande.re/jpeg/2c6876ac2317fce617e3c5f1a642123b/yande.re%20292092%20hatsune_miku%20tid%20vocaloid.jpg"
+        @rewrite = "https://yande.re/image/2c6876ac2317fce617e3c5f1a642123b/yande.re%20292092%20hatsune_miku%20tid%20vocaloid.png"
+        assert_rewritten(@rewrite, @source)
+        assert_downloaded(1_050_117, @source)
+      end
+    end
+  end
+end

test/unit/downloads/nico_seiga_test.rb

@@ -0,0 +1,32 @@
+require "test_helper"
+
+module Downloads
+  class NicoSeigaTest < ActiveSupport::TestCase
+    context "downloading a 'http://seiga.nicovideo.jp/seiga/:id' url" do
+      should "download the original file" do
+        @source = "http://seiga.nicovideo.jp/seiga/im4937663"
+        @rewrite = %r!http://lohas.nicoseiga.jp/priv/\h{40}/\d+/4937663!
+        assert_rewritten(@rewrite, @source)
+        assert_downloaded(2032, @source)
+      end
+    end
+
+    context "downloading a 'http://lohas.nicoseiga.jp/o/:hash/:id' url" do
+      should "download the original file" do
+        @source = "http://lohas.nicoseiga.jp/o/910aecf08e542285862954017f8a33a8c32a8aec/1433298801/4937663"
+        @rewrite = %r!http://lohas.nicoseiga.jp/priv/\h{40}/\d+/4937663!
+        assert_rewritten(@rewrite, @source)
+        assert_downloaded(2032, @source)
+      end
+    end
+
+    context "downloading a 'https://lohas.nicoseiga.jp/thumb/:id' url" do
+      should "download the original file" do
+        @source = "https://lohas.nicoseiga.jp/thumb/4937663i"
+        @rewrite = %r!http://lohas.nicoseiga.jp/priv/\h{40}/\d+/4937663!
+        assert_rewritten(@rewrite, @source)
+        assert_downloaded(2032, @source)
+      end
+    end
+  end
+end

test/unit/downloads/nijie_test.rb

@@ -0,0 +1,22 @@
+require "test_helper"
+
+module Downloads
+  class NijieTest < ActiveSupport::TestCase
+    context "downloading a 'http://nijie.info/view.php?id=:id' url" do
+      should "download the original file" do
+        @source = "http://nijie.info/view.php?id=213043"
+        @rewrite = "https://pic03.nijie.info/nijie_picture/728995_20170505014820_0.jpg"
+        assert_rewritten(@rewrite, @source)
+        assert_downloaded(132_555, @source)
+      end
+    end
+
+    context "downloading a 'https://pic*.nijie.info/nijie_picture/:id.jpg' url" do
+      should "download the original file" do
+        @source = "https://pic03.nijie.info/nijie_picture/728995_20170505014820_0.jpg"
+        assert_not_rewritten(@source)
+        assert_downloaded(132_555, @source)
+      end
+    end
+  end
+end

test/unit/downloads/pawoo_test.rb

@@ -0,0 +1,32 @@
+require "test_helper"
+
+module Downloads
+  class PawooTest < ActiveSupport::TestCase
+    context "downloading a 'https://pawoo.net/web/statuses/:id' url" do
+      should "download the original file" do
+        @source = "https://pawoo.net/web/statuses/1202176"
+        @rewrite = "https://img.pawoo.net/media_attachments/files/000/128/953/original/4c0a06087b03343f.png"
+        assert_rewritten(@rewrite, @source)
+        assert_downloaded(7680, @source)
+      end
+    end
+
+    context "downloading a 'https://pawoo.net/:user/:id' url" do
+      should "download the original file" do
+        @source = "https://pawoo.net/@9ed00e924818/1202176"
+        @rewrite = "https://img.pawoo.net/media_attachments/files/000/128/953/original/4c0a06087b03343f.png"
+        assert_rewritten(@rewrite, @source)
+        assert_downloaded(7680, @source)
+      end
+    end
+
+    context "downloading a 'https://img.pawoo.net/media_attachments/:id/small/:file' url" do
+      should "download the original file" do
+        @source = "https://img.pawoo.net/media_attachments/files/000/128/953/small/4c0a06087b03343f.png"
+        @rewrite = "https://img.pawoo.net/media_attachments/files/000/128/953/original/4c0a06087b03343f.png"
+        assert_rewritten(@rewrite, @source)
+        assert_downloaded(7680, @source)
+      end
+    end
+  end
+end

test/unit/downloads/twitter_test.rb

@@ -0,0 +1,41 @@
+require "test_helper"
+
+module Downloads
+  class TwitterTest < ActiveSupport::TestCase
+    context "downloading a 'https://twitter.com/:user/status/:id' url containing a video" do
+      should "download the largest video" do
+        @source = "https://twitter.com/CincinnatiZoo/status/859073537713328129"
+        @rewrite = "https://video.twimg.com/ext_tw_video/859073467769126913/pu/vid/1280x720/cPGgVROXHy3yrK6u.mp4"
+        assert_rewritten(@rewrite, @source)
+        assert_downloaded(8_602_983, @source)
+      end
+    end
+
+    context "downloading a 'https://twitter.com/:user/status/:id/photo/:n' card url" do
+      should "download the orig file" do
+        @source = "https://twitter.com/masayasuf/status/870734961778630656/photo/1"
+        @rewrite = "https://pbs.twimg.com/media/DBV40M2UIAAHYlt.jpg:orig"
+        assert_rewritten(@rewrite, @source)
+        assert_downloaded(788_206, @source)
+      end
+    end
+
+    context "downloading a 'https://mobile.twitter.com/:user/status/:id/photo/:n' mobile url" do
+      should "download the orig file" do
+        @source = "https://mobile.twitter.com/Strangestone/status/556440271961858051"
+        @rewrite = "https://pbs.twimg.com/media/B7jfc1JCcAEyeJh.png:orig"
+        assert_rewritten(@rewrite, @source)
+        assert_downloaded(280_150, @source)
+      end
+    end
+
+    context "downloading a 'https://pbs.twimg.com/media/*:large' url" do
+      should "download the orig file" do
+        @source = "https://pbs.twimg.com/media/B4HSEP5CUAA4xyu.png:large"
+        @rewrite = "https://pbs.twimg.com/media/B4HSEP5CUAA4xyu.png:orig"
+        assert_rewritten(@rewrite, @source)
+        assert_downloaded(9800, @source)
+      end
+    end
+  end
+end