diff --git a/app/logical/danbooru/http.rb b/app/logical/danbooru/http.rb index ed25aa804..d5c223b57 100644 --- a/app/logical/danbooru/http.rb +++ b/app/logical/danbooru/http.rb @@ -4,6 +4,7 @@ require "danbooru/http/cache" require "danbooru/http/redirector" require "danbooru/http/retriable" require "danbooru/http/session" +require "danbooru/http/spoof_referrer" module Danbooru class Http @@ -25,6 +26,7 @@ module Danbooru .timeout(DEFAULT_TIMEOUT) .headers("Accept-Encoding" => "gzip") .headers("User-Agent": "#{Danbooru.config.canonical_app_name}/#{Rails.application.config.x.git_hash}") + .use(:spoof_referrer) .use(:auto_inflate) .use(redirector: { max_redirects: MAX_REDIRECTS }) .use(:session) @@ -97,8 +99,7 @@ module Danbooru concerning :DownloadMethods do def download_media(url, no_polish: true, **options) - url = Addressable::URI.heuristic_parse(url) - response = headers(Referer: url.origin).get(url) + response = get(url) # prevent Cloudflare Polish from modifying images. if no_polish && response.headers["CF-Polished"].present? diff --git a/app/logical/danbooru/http/spoof_referrer.rb b/app/logical/danbooru/http/spoof_referrer.rb new file mode 100644 index 000000000..7df6faef9 --- /dev/null +++ b/app/logical/danbooru/http/spoof_referrer.rb @@ -0,0 +1,13 @@ +module Danbooru + class Http + class SpoofReferrer < HTTP::Feature + HTTP::Options.register_feature :spoof_referrer, self + + def perform(request, &block) + request.headers["Referer"] = request.uri.origin unless request.headers["Referer"].present? + response = yield request + response + end + end + end +end diff --git a/test/unit/danbooru_http_test.rb b/test/unit/danbooru_http_test.rb index 79ba9e0d5..7f3e0a2ad 100644 --- a/test/unit/danbooru_http_test.rb +++ b/test/unit/danbooru_http_test.rb @@ -127,6 +127,15 @@ class DanbooruHttpTest < ActiveSupport::TestCase end end + context "spoof referrer feature" do + should "spoof the referer" do + response = Danbooru::Http.get("https://httpbin.org/anything") + + assert_equal(200, response.status) + assert_equal("https://httpbin.org", response.parse.dig("headers", "Referer")) + end + end + context "#download method" do should "download files" do response, file = Danbooru::Http.download_media("https://httpbin.org/bytes/1000")