danbooru::http: factor out referrer spoofing.

Factor out referrer spoofing so that it can be used outside of downloading
files. We also need to spoof the referrer when determining the remote
filesize of images on the uploads page.

app/logical/danbooru/http.rb

@@ -4,6 +4,7 @@ require "danbooru/http/cache"
 require "danbooru/http/redirector"
 require "danbooru/http/retriable"
 require "danbooru/http/session"
+require "danbooru/http/spoof_referrer"
 
 module Danbooru
   class Http
@@ -25,6 +26,7 @@ module Danbooru
         .timeout(DEFAULT_TIMEOUT)
         .headers("Accept-Encoding" => "gzip")
         .headers("User-Agent": "#{Danbooru.config.canonical_app_name}/#{Rails.application.config.x.git_hash}")
+        .use(:spoof_referrer)
         .use(:auto_inflate)
         .use(redirector: { max_redirects: MAX_REDIRECTS })
         .use(:session)
@@ -97,8 +99,7 @@ module Danbooru
 
     concerning :DownloadMethods do
       def download_media(url, no_polish: true, **options)
-        url = Addressable::URI.heuristic_parse(url)
-        response = headers(Referer: url.origin).get(url)
+        response = get(url)
 
         # prevent Cloudflare Polish from modifying images.
         if no_polish && response.headers["CF-Polished"].present?

app/logical/danbooru/http/spoof_referrer.rb

@@ -0,0 +1,13 @@
+module Danbooru
+  class Http
+    class SpoofReferrer < HTTP::Feature
+      HTTP::Options.register_feature :spoof_referrer, self
+
+      def perform(request, &block)
+        request.headers["Referer"] = request.uri.origin unless request.headers["Referer"].present?
+        response = yield request
+        response
+      end
+    end
+  end
+end

test/unit/danbooru_http_test.rb

@@ -127,6 +127,15 @@ class DanbooruHttpTest < ActiveSupport::TestCase
       end
     end
 
+    context "spoof referrer feature" do
+      should "spoof the referer" do
+        response = Danbooru::Http.get("https://httpbin.org/anything")
+
+        assert_equal(200, response.status)
+        assert_equal("https://httpbin.org", response.parse.dig("headers", "Referer"))
+      end
+    end
+
     context "#download method" do
       should "download files" do
         response, file = Danbooru::Http.download_media("https://httpbin.org/bytes/1000")