Set the default User-Agent sent for outgoing http requests.

* Add a `Danbooru.config.http_headers` config option.

* Refactor various places to use this option instead of setting the user
  agent manually.

app/logical/downloads/file.rb

@@ -26,10 +26,7 @@ module Downloads
     end
 
     def size
-      headers = {
-        "User-Agent" => "#{Danbooru.config.safe_app_name}/#{Danbooru.config.version}"
-      }
-      @source, headers, @data = before_download(@source, headers, @data)
+      @source, _, @data = before_download(@source, @data)
       url = URI.parse(@source)
       res = HTTParty.head(url, Danbooru.config.httparty_options.reverse_merge(timeout: 3))
       res.content_length
@@ -45,7 +42,9 @@ module Downloads
       @source = after_download(@source)
     end
 
-    def before_download(url, headers, datums)
+    def before_download(url, datums)
+      headers = Danbooru.config.http_headers
+
       RewriteStrategies::Base.strategies.each do |strategy|
         url, headers, datums = strategy.new(url).rewrite(url, headers, datums)
       end
@@ -80,16 +79,14 @@ module Downloads
           raise Error.new("URL must be HTTP or HTTPS")
         end
 
-        headers = {
-          "User-Agent" => "#{Danbooru.config.safe_app_name}/#{Danbooru.config.version}"
-        }
-        src, headers, datums = before_download(src, headers, datums)
+        src, headers, datums = before_download(src, datums)
         url = URI.parse(src)
 
         validate_local_hosts(url)
 
         begin
-          res = HTTParty.get(url, Danbooru.config.httparty_options.reverse_merge(stream_body: true, timeout: 10, headers: headers), &block)
+          options = { stream_body: true, timeout: 10, headers: headers }
+          res = HTTParty.get(url, options.deep_merge(Danbooru.config.httparty_options), &block)
 
           if res.success?
             if max_size

app/logical/downloads/rewrite_strategies/art_station.rb

@@ -5,7 +5,7 @@ module Downloads
         # example: https://cdnb3.artstation.com/p/assets/images/images/003/716/071/large/aoi-ogata-hate-city.jpg?1476754974
         if url =~ %r!^https?://cdn\w*\.artstation\.com/p/assets/images/images/\d+/\d+/\d+/(?:medium|small|large)/!
           original_url, headers = rewrite_large_url(url, headers)
-          if test_original(original_url)
+          if http_exists?(original_url, headers)
             url = original_url
           end
         else
@@ -16,11 +16,6 @@ module Downloads
       end
 
     protected
-      def test_original(url)
-        res = http_head_request(url, {})
-        res.success?
-      end
-
       def rewrite_html_url(url, headers)
         return [url, headers] unless Sources::Strategies::ArtStation.url_match?(url)
 

app/logical/downloads/rewrite_strategies/base.rb

@@ -20,12 +20,8 @@ module Downloads
       end
 
     protected
-      def http_head_request(url, headers)
-        HTTParty.head(url, Danbooru.config.httparty_options.merge(headers: headers))
-      end
-
       def http_exists?(url, headers)
-        res = http_head_request(url, headers)
+        res = HTTParty.head(url, Danbooru.config.httparty_options.deep_merge(headers: headers))
         res.success?
       end
     end