From d3bb5c67ee42cebc80ecfdef4101de15ccbe9708 Mon Sep 17 00:00:00 2001
From: evazion <noizave@gmail.com>
Date: Wed, 24 Jun 2020 20:52:29 -0500
Subject: [PATCH] danbooru::http: factor out referrer spoofing.

Factor out referrer spoofing so that it can be used outside of downloading
files. We also need to spoof the referrer when determining the remote
filesize of images on the uploads page.
---
 app/logical/danbooru/http.rb                |  5 +++--
 app/logical/danbooru/http/spoof_referrer.rb | 13 +++++++++++++
 test/unit/danbooru_http_test.rb             |  9 +++++++++
 3 files changed, 25 insertions(+), 2 deletions(-)
 create mode 100644 app/logical/danbooru/http/spoof_referrer.rb

diff --git a/app/logical/danbooru/http.rb b/app/logical/danbooru/http.rb
index ed25aa804..d5c223b57 100644
--- a/app/logical/danbooru/http.rb
+++ b/app/logical/danbooru/http.rb
@@ -4,6 +4,7 @@ require "danbooru/http/cache"
 require "danbooru/http/redirector"
 require "danbooru/http/retriable"
 require "danbooru/http/session"
+require "danbooru/http/spoof_referrer"
 
 module Danbooru
   class Http
@@ -25,6 +26,7 @@ module Danbooru
         .timeout(DEFAULT_TIMEOUT)
         .headers("Accept-Encoding" => "gzip")
         .headers("User-Agent": "#{Danbooru.config.canonical_app_name}/#{Rails.application.config.x.git_hash}")
+        .use(:spoof_referrer)
         .use(:auto_inflate)
         .use(redirector: { max_redirects: MAX_REDIRECTS })
         .use(:session)
@@ -97,8 +99,7 @@ module Danbooru
 
     concerning :DownloadMethods do
       def download_media(url, no_polish: true, **options)
-        url = Addressable::URI.heuristic_parse(url)
-        response = headers(Referer: url.origin).get(url)
+        response = get(url)
 
         # prevent Cloudflare Polish from modifying images.
         if no_polish && response.headers["CF-Polished"].present?
diff --git a/app/logical/danbooru/http/spoof_referrer.rb b/app/logical/danbooru/http/spoof_referrer.rb
new file mode 100644
index 000000000..7df6faef9
--- /dev/null
+++ b/app/logical/danbooru/http/spoof_referrer.rb
@@ -0,0 +1,13 @@
+module Danbooru
+  class Http
+    class SpoofReferrer < HTTP::Feature
+      HTTP::Options.register_feature :spoof_referrer, self
+
+      def perform(request, &block)
+        request.headers["Referer"] = request.uri.origin unless request.headers["Referer"].present?
+        response = yield request
+        response
+      end
+    end
+  end
+end
diff --git a/test/unit/danbooru_http_test.rb b/test/unit/danbooru_http_test.rb
index 79ba9e0d5..7f3e0a2ad 100644
--- a/test/unit/danbooru_http_test.rb
+++ b/test/unit/danbooru_http_test.rb
@@ -127,6 +127,15 @@ class DanbooruHttpTest < ActiveSupport::TestCase
       end
     end
 
+    context "spoof referrer feature" do
+      should "spoof the referer" do
+        response = Danbooru::Http.get("https://httpbin.org/anything")
+
+        assert_equal(200, response.status)
+        assert_equal("https://httpbin.org", response.parse.dig("headers", "Referer"))
+      end
+    end
+
     context "#download method" do
       should "download files" do
         response, file = Danbooru::Http.download_media("https://httpbin.org/bytes/1000")