From 8125d8e796dc95f5cc8724228665fe955f7176e3 Mon Sep 17 00:00:00 2001 From: evazion Date: Wed, 19 Jul 2017 19:37:14 -0500 Subject: [PATCH] Set the default User-Agent sent for outgoing http requests. * Add a `Danbooru.config.http_headers` config option. * Refactor various places to use this option instead of setting the user agent manually. --- app/controllers/uploads_controller.rb | 18 +++--------------- app/logical/downloads/file.rb | 17 +++++++---------- .../rewrite_strategies/art_station.rb | 7 +------ .../downloads/rewrite_strategies/base.rb | 6 +----- app/logical/image_proxy.rb | 6 +----- app/logical/pixiv_api_client.rb | 9 ++++----- config/danbooru_default_config.rb | 12 +++++++++++- test/helpers/download_helper.rb | 2 +- 8 files changed, 29 insertions(+), 48 deletions(-) diff --git a/app/controllers/uploads_controller.rb b/app/controllers/uploads_controller.rb index d2228ac30..fa2655d9f 100644 --- a/app/controllers/uploads_controller.rb +++ b/app/controllers/uploads_controller.rb @@ -6,19 +6,13 @@ class UploadsController < ApplicationController @upload = Upload.new @upload_notice_wiki = WikiPage.titled(Danbooru.config.upload_notice_wiki_page).first if params[:url] - @normalized_url = params[:url] - headers = default_headers() - data = {} - - Downloads::RewriteStrategies::Base.strategies.each do |strategy| - @normalized_url, headers, data = strategy.new(@normalized_url).rewrite(@normalized_url, headers, data) - end - + download = Downloads::File.new(params[:url], ".") + @normalized_url, _, _ = download.before_download(params[:url], {}) @post = find_post_by_url(@normalized_url) begin @source = Sources::Site.new(params[:url], :referer_url => params[:ref]) - @remote_size = Downloads::File.new(@normalized_url, ".").size + @remote_size = download.size rescue Exception end end @@ -79,12 +73,6 @@ protected end end - def default_headers - { - "User-Agent" => "#{Danbooru.config.safe_app_name}/#{Danbooru.config.version}" - } - end - def save_recent_tags if @upload tags = Tag.scan_tags(@upload.tag_string) diff --git a/app/logical/downloads/file.rb b/app/logical/downloads/file.rb index 21b72d265..d8b596ec7 100644 --- a/app/logical/downloads/file.rb +++ b/app/logical/downloads/file.rb @@ -26,10 +26,7 @@ module Downloads end def size - headers = { - "User-Agent" => "#{Danbooru.config.safe_app_name}/#{Danbooru.config.version}" - } - @source, headers, @data = before_download(@source, headers, @data) + @source, _, @data = before_download(@source, @data) url = URI.parse(@source) res = HTTParty.head(url, Danbooru.config.httparty_options.reverse_merge(timeout: 3)) res.content_length @@ -45,7 +42,9 @@ module Downloads @source = after_download(@source) end - def before_download(url, headers, datums) + def before_download(url, datums) + headers = Danbooru.config.http_headers + RewriteStrategies::Base.strategies.each do |strategy| url, headers, datums = strategy.new(url).rewrite(url, headers, datums) end @@ -80,16 +79,14 @@ module Downloads raise Error.new("URL must be HTTP or HTTPS") end - headers = { - "User-Agent" => "#{Danbooru.config.safe_app_name}/#{Danbooru.config.version}" - } - src, headers, datums = before_download(src, headers, datums) + src, headers, datums = before_download(src, datums) url = URI.parse(src) validate_local_hosts(url) begin - res = HTTParty.get(url, Danbooru.config.httparty_options.reverse_merge(stream_body: true, timeout: 10, headers: headers), &block) + options = { stream_body: true, timeout: 10, headers: headers } + res = HTTParty.get(url, options.deep_merge(Danbooru.config.httparty_options), &block) if res.success? if max_size diff --git a/app/logical/downloads/rewrite_strategies/art_station.rb b/app/logical/downloads/rewrite_strategies/art_station.rb index 06fedf13e..454044f9e 100644 --- a/app/logical/downloads/rewrite_strategies/art_station.rb +++ b/app/logical/downloads/rewrite_strategies/art_station.rb @@ -5,7 +5,7 @@ module Downloads # example: https://cdnb3.artstation.com/p/assets/images/images/003/716/071/large/aoi-ogata-hate-city.jpg?1476754974 if url =~ %r!^https?://cdn\w*\.artstation\.com/p/assets/images/images/\d+/\d+/\d+/(?:medium|small|large)/! original_url, headers = rewrite_large_url(url, headers) - if test_original(original_url) + if http_exists?(original_url, headers) url = original_url end else @@ -16,11 +16,6 @@ module Downloads end protected - def test_original(url) - res = http_head_request(url, {}) - res.success? - end - def rewrite_html_url(url, headers) return [url, headers] unless Sources::Strategies::ArtStation.url_match?(url) diff --git a/app/logical/downloads/rewrite_strategies/base.rb b/app/logical/downloads/rewrite_strategies/base.rb index 578b4fec9..d1219397d 100644 --- a/app/logical/downloads/rewrite_strategies/base.rb +++ b/app/logical/downloads/rewrite_strategies/base.rb @@ -20,12 +20,8 @@ module Downloads end protected - def http_head_request(url, headers) - HTTParty.head(url, Danbooru.config.httparty_options.merge(headers: headers)) - end - def http_exists?(url, headers) - res = http_head_request(url, headers) + res = HTTParty.head(url, Danbooru.config.httparty_options.deep_merge(headers: headers)) res.success? end end diff --git a/app/logical/image_proxy.rb b/app/logical/image_proxy.rb index 5c2094511..6a93969aa 100644 --- a/app/logical/image_proxy.rb +++ b/app/logical/image_proxy.rb @@ -16,11 +16,7 @@ class ImageProxy raise "Proxy not allowed for this site" end - headers = { - "Referer" => fake_referer_for(url), - "User-Agent" => "#{Danbooru.config.safe_app_name}/#{Danbooru.config.version}" - } - response = HTTParty.get(url, Danbooru.config.httparty_options.merge(headers: headers)) + response = HTTParty.get(url, Danbooru.config.httparty_options.deep_merge(headers: {"Referer" => fake_referer_for(url)})) if response.success? return response else diff --git a/app/logical/pixiv_api_client.rb b/app/logical/pixiv_api_client.rb index b45849b40..cc123228f 100644 --- a/app/logical/pixiv_api_client.rb +++ b/app/logical/pixiv_api_client.rb @@ -123,19 +123,18 @@ class PixivApiClient end def works(illust_id) - headers = { + headers = Danbooru.config.http_headers.merge( "Referer" => "http://www.pixiv.net", - "User-Agent" => "#{Danbooru.config.safe_app_name}/#{Danbooru.config.version}", "Content-Type" => "application/x-www-form-urlencoded", "Authorization" => "Bearer #{access_token}" - } + ) params = { "image_sizes" => "large", "include_stats" => "true" } url = "https://public-api.secure.pixiv.net/v#{API_VERSION}/works/#{illust_id.to_i}.json" - resp = HTTParty.get(url, Danbooru.config.httparty_options.merge(query: params, headers: headers)) + resp = HTTParty.get(url, Danbooru.config.httparty_options.deep_merge(query: params, headers: headers)) if resp.success? json = parse_api_json(resp.body) @@ -171,7 +170,7 @@ private } url = "https://oauth.secure.pixiv.net/auth/token" - resp = HTTParty.post(url, Danbooru.config.httparty_options.merge(body: params, headers: headers)) + resp = HTTParty.post(url, Danbooru.config.httparty_options.deep_merge(body: params, headers: headers)) if resp.success? json = JSON.parse(resp.body) access_token = json["response"]["access_token"] diff --git a/config/danbooru_default_config.rb b/config/danbooru_default_config.rb index ca6862920..735c621b7 100644 --- a/config/danbooru_default_config.rb +++ b/config/danbooru_default_config.rb @@ -417,10 +417,20 @@ module Danbooru false end + # The default headers to be sent with outgoing http requests. Some external + # services will fail if you don't set a valid User-Agent. + def http_headers + { + "User-Agent" => "#{Danbooru.config.safe_app_name}/#{Danbooru.config.version}", + } + end + def httparty_options # proxy example: # {http_proxyaddr: "", http_proxyport: "", http_proxyuser: nil, http_proxypass: nil} - {} + { + headers: Danbooru.config.http_headers, + } end # you should override this diff --git a/test/helpers/download_helper.rb b/test/helpers/download_helper.rb index 5a8efd928..95a843644 100644 --- a/test/helpers/download_helper.rb +++ b/test/helpers/download_helper.rb @@ -14,7 +14,7 @@ module DownloadTestHelper tempfile = Tempfile.new("danbooru-test") download = Downloads::File.new(test_source, tempfile.path) - rewritten_source, headers, _ = download.before_download(test_source, {}, {}) + rewritten_source, _, _ = download.before_download(test_source, {}) assert_equal(expected_source, rewritten_source, "Tested source URL: #{test_source}") end