Merge pull request #3226 from evazion/fix-3224

Fix #3224: 403 Forbidden when logging in to DeviantArt.

app/controllers/uploads_controller.rb

@@ -6,19 +6,13 @@ class UploadsController < ApplicationController
     @upload = Upload.new
     @upload_notice_wiki = WikiPage.titled(Danbooru.config.upload_notice_wiki_page).first
     if params[:url]
-      @normalized_url = params[:url]
+      download = Downloads::File.new(params[:url], ".")
-      headers = default_headers()
+      @normalized_url, _, _ = download.before_download(params[:url], {})
-      data = {}
-
-      Downloads::RewriteStrategies::Base.strategies.each do |strategy|
-        @normalized_url, headers, data = strategy.new(@normalized_url).rewrite(@normalized_url, headers, data)
-      end
-
       @post = find_post_by_url(@normalized_url)
 
       begin
         @source = Sources::Site.new(params[:url], :referer_url => params[:ref])
-        @remote_size = Downloads::File.new(@normalized_url, ".").size
+        @remote_size = download.size
       rescue Exception
       end
     end
@@ -79,12 +73,6 @@ protected
     end
   end
 
-  def default_headers
-    {
-      "User-Agent" => "#{Danbooru.config.safe_app_name}/#{Danbooru.config.version}"
-    }
-  end
-
   def save_recent_tags
     if @upload
       tags = Tag.scan_tags(@upload.tag_string)

app/logical/downloads/file.rb

@@ -26,10 +26,7 @@ module Downloads
     end
 
     def size
-      headers = {
+      @source, _, @data = before_download(@source, @data)
-        "User-Agent" => "#{Danbooru.config.safe_app_name}/#{Danbooru.config.version}"
-      }
-      @source, headers, @data = before_download(@source, headers, @data)
       url = URI.parse(@source)
       res = HTTParty.head(url, Danbooru.config.httparty_options.reverse_merge(timeout: 3))
       res.content_length
@@ -45,7 +42,9 @@ module Downloads
       @source = after_download(@source)
     end
 
-    def before_download(url, headers, datums)
+    def before_download(url, datums)
+      headers = Danbooru.config.http_headers
+
       RewriteStrategies::Base.strategies.each do |strategy|
         url, headers, datums = strategy.new(url).rewrite(url, headers, datums)
       end
@@ -80,16 +79,14 @@ module Downloads
           raise Error.new("URL must be HTTP or HTTPS")
         end
 
-        headers = {
+        src, headers, datums = before_download(src, datums)
-          "User-Agent" => "#{Danbooru.config.safe_app_name}/#{Danbooru.config.version}"
-        }
-        src, headers, datums = before_download(src, headers, datums)
         url = URI.parse(src)
 
         validate_local_hosts(url)
 
         begin
-          res = HTTParty.get(url, Danbooru.config.httparty_options.reverse_merge(stream_body: true, timeout: 10, headers: headers), &block)
+          options = { stream_body: true, timeout: 10, headers: headers }
+          res = HTTParty.get(url, options.deep_merge(Danbooru.config.httparty_options), &block)
 
           if res.success?
             if max_size

app/logical/downloads/rewrite_strategies/art_station.rb

@@ -5,7 +5,7 @@ module Downloads
         # example: https://cdnb3.artstation.com/p/assets/images/images/003/716/071/large/aoi-ogata-hate-city.jpg?1476754974
         if url =~ %r!^https?://cdn\w*\.artstation\.com/p/assets/images/images/\d+/\d+/\d+/(?:medium|small|large)/!
           original_url, headers = rewrite_large_url(url, headers)
-          if test_original(original_url)
+          if http_exists?(original_url, headers)
             url = original_url
           end
         else
@@ -16,11 +16,6 @@ module Downloads
       end
 
     protected
-      def test_original(url)
-        res = http_head_request(url, {})
-        res.success?
-      end
-
       def rewrite_html_url(url, headers)
         return [url, headers] unless Sources::Strategies::ArtStation.url_match?(url)
 

app/logical/downloads/rewrite_strategies/base.rb

@@ -20,12 +20,8 @@ module Downloads
       end
 
     protected
-      def http_head_request(url, headers)
-        HTTParty.head(url, Danbooru.config.httparty_options.merge(headers: headers))
-      end
-
       def http_exists?(url, headers)
-        res = http_head_request(url, headers)
+        res = HTTParty.head(url, Danbooru.config.httparty_options.deep_merge(headers: headers))
         res.success?
       end
     end

app/logical/image_proxy.rb

@@ -16,11 +16,7 @@ class ImageProxy
       raise "Proxy not allowed for this site"
     end
 
-    headers = {
+    response = HTTParty.get(url, Danbooru.config.httparty_options.deep_merge(headers: {"Referer" => fake_referer_for(url)}))
-      "Referer" => fake_referer_for(url),
-      "User-Agent" => "#{Danbooru.config.safe_app_name}/#{Danbooru.config.version}"
-    }
-    response = HTTParty.get(url, Danbooru.config.httparty_options.merge(headers: headers))
     if response.success?
       return response
     else

app/logical/pixiv_api_client.rb

@@ -123,19 +123,18 @@ class PixivApiClient
   end
 
   def works(illust_id)
-    headers = {
+    headers = Danbooru.config.http_headers.merge(
       "Referer" => "http://www.pixiv.net",
-      "User-Agent" => "#{Danbooru.config.safe_app_name}/#{Danbooru.config.version}",
       "Content-Type" => "application/x-www-form-urlencoded",
       "Authorization" => "Bearer #{access_token}"
-    }
+    )
     params = {
       "image_sizes" => "large",
       "include_stats" => "true"
     }
 
     url = "https://public-api.secure.pixiv.net/v#{API_VERSION}/works/#{illust_id.to_i}.json"
-    resp = HTTParty.get(url, Danbooru.config.httparty_options.merge(query: params, headers: headers))
+    resp = HTTParty.get(url, Danbooru.config.httparty_options.deep_merge(query: params, headers: headers))
 
     if resp.success?
       json = parse_api_json(resp.body)
@@ -171,7 +170,7 @@ private
       }
       url = "https://oauth.secure.pixiv.net/auth/token"
 
-      resp = HTTParty.post(url, Danbooru.config.httparty_options.merge(body: params, headers: headers))
+      resp = HTTParty.post(url, Danbooru.config.httparty_options.deep_merge(body: params, headers: headers))
       if resp.success?
         json = JSON.parse(resp.body)
         access_token = json["response"]["access_token"]

app/logical/sources/strategies/deviant_art.rb

@@ -142,6 +142,8 @@ module Sources
 
       def session_cookies(mech)
         Cache.get(DEVIANTART_SESSION_CACHE_KEY, 2.hours) do
+          mech.request_headers = Danbooru.config.http_headers
+
           page = mech.get("https://www.deviantart.com/users/login")
           validate_key = page.search('input[name="validate_key"]').attribute("value").value
           validate_token = page.search('input[name="validate_token"]').attribute("value").value

config/danbooru_default_config.rb

@@ -417,10 +417,20 @@ module Danbooru
       false
     end
 
+    # The default headers to be sent with outgoing http requests. Some external
+    # services will fail if you don't set a valid User-Agent.
+    def http_headers
+      {
+        "User-Agent" => "#{Danbooru.config.safe_app_name}/#{Danbooru.config.version}",
+      }
+    end
+
     def httparty_options
       # proxy example:
       # {http_proxyaddr: "", http_proxyport: "", http_proxyuser: nil, http_proxypass: nil}
-      {}
+      {
+        headers: Danbooru.config.http_headers,
+      }
     end
 
     # you should override this

test/helpers/download_helper.rb

@@ -14,7 +14,7 @@ module DownloadTestHelper
     tempfile = Tempfile.new("danbooru-test")
     download = Downloads::File.new(test_source, tempfile.path)
 
-    rewritten_source, headers, _ = download.before_download(test_source, {}, {})
+    rewritten_source, _, _ = download.before_download(test_source, {})
     assert_equal(expected_source, rewritten_source, "Tested source URL: #{test_source}")
   end