From 412b7f2727c6ef670de55a0dced24274e04c4da8 Mon Sep 17 00:00:00 2001
From: evazion <noizave@gmail.com>
Date: Tue, 18 Oct 2022 22:38:19 -0500
Subject: [PATCH] http: split requests into internal and external requests.

Split requests made by Danbooru::Http into either internal or external
requests. Internal requests are API calls to internal services run by
Danbooru. External requests are requests to external websites, for
example fetching sources or downloading files. External requests may use
a HTTP proxy if one is configured. Internal requests don't.

Fixes a few source extractors not using the HTTP proxy for certain API calls.
---
 app/logical/authorize_net_client.rb         |  2 +-
 app/logical/autotagger_client.rb            |  2 +-
 app/logical/danbooru/http.rb                | 26 +++++++++++++++------
 app/logical/deep_danbooru_client.rb         |  2 +-
 app/logical/deviant_art_api_client.rb       |  8 +++----
 app/logical/discord_api_client.rb           |  2 +-
 app/logical/discord_webhook_service.rb      |  2 +-
 app/logical/iqdb_client.rb                  |  2 +-
 app/logical/nico_seiga_api_client.rb        |  2 +-
 app/logical/pixiv_ajax_client.rb            |  2 +-
 app/logical/reportbooru_service.rb          |  2 +-
 app/logical/source/extractor.rb             |  2 +-
 app/logical/source/extractor/deviant_art.rb |  5 +---
 app/logical/source/extractor/fantia.rb      |  2 +-
 app/logical/source/extractor/furaffinity.rb |  2 +-
 app/logical/source/extractor/twitter.rb     |  2 +-
 app/logical/twitter_api_client.rb           |  2 +-
 17 files changed, 38 insertions(+), 29 deletions(-)

diff --git a/app/logical/authorize_net_client.rb b/app/logical/authorize_net_client.rb
index 3f30850f1..47faa05c6 100644
--- a/app/logical/authorize_net_client.rb
+++ b/app/logical/authorize_net_client.rb
@@ -9,7 +9,7 @@ class AuthorizeNetClient
 
   attr_reader :login_id, :transaction_key, :test_mode, :http
 
-  def initialize(login_id: Danbooru.config.authorize_net_login_id, transaction_key: Danbooru.config.authorize_net_transaction_key, test_mode: Danbooru.config.authorize_net_test_mode, http: Danbooru::Http.new)
+  def initialize(login_id: Danbooru.config.authorize_net_login_id, transaction_key: Danbooru.config.authorize_net_transaction_key, test_mode: Danbooru.config.authorize_net_test_mode, http: Danbooru::Http.external)
     @login_id = login_id
     @transaction_key = transaction_key
     @test_mode = test_mode
diff --git a/app/logical/autotagger_client.rb b/app/logical/autotagger_client.rb
index d628a74db..adf64a3a2 100644
--- a/app/logical/autotagger_client.rb
+++ b/app/logical/autotagger_client.rb
@@ -7,7 +7,7 @@
 class AutotaggerClient
   attr_reader :autotagger_url, :http
 
-  def initialize(autotagger_url: Danbooru.config.autotagger_url.to_s, http: Danbooru::Http.new)
+  def initialize(autotagger_url: Danbooru.config.autotagger_url.to_s, http: Danbooru::Http.internal)
     @autotagger_url = autotagger_url.chomp("/")
     @http = http
   end
diff --git a/app/logical/danbooru/http.rb b/app/logical/danbooru/http.rb
index 83d88c105..43a0384a7 100644
--- a/app/logical/danbooru/http.rb
+++ b/app/logical/danbooru/http.rb
@@ -40,21 +40,33 @@ module Danbooru
     attr_accessor :max_size, :http
 
     class << self
-      delegate :get, :head, :put, :post, :delete, :cache, :follow, :max_size, :timeout, :auth, :basic_auth, :headers, :cookies, :use, :public_only, :with_legacy_ssl, :download_media, to: :new
+      delegate :get, :head, :put, :post, :delete, :cache, :follow, :max_size, :timeout, :auth, :basic_auth, :headers, :cookies, :use, :proxy, :public_only, :with_legacy_ssl, :download_media, to: :new
     end
 
-    def initialize
-      @http ||=
-        ::Danbooru::Http::ApplicationClient.new
+    # The default HTTP client.
+    def self.default
+      Danbooru::Http::ApplicationClient.new
         .timeout(DEFAULT_TIMEOUT)
-        .headers("Accept-Encoding" => "gzip")
-        .headers("User-Agent": "#{Danbooru.config.canonical_app_name}/#{Rails.application.config.x.git_hash}")
-        #.headers("User-Agent": Danbooru.config.canonical_app_name)
+        .headers("Accept-Encoding": "gzip")
         .use(:auto_inflate)
         .use(redirector: { max_redirects: MAX_REDIRECTS })
         .use(:session)
     end
 
+    # The default HTTP client for requests to external websites. This includes API calls to external services, fetching source data, and downloading images.
+    def self.external
+      new.proxy.public_only.headers("User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:102.0) Gecko/20100101 Firefox/102.0")
+    end
+
+    # The default HTTP client for API calls to internal services controlled by Danbooru.
+    def self.internal
+      new.headers("User-Agent": "#{Danbooru.config.canonical_app_name}/#{Rails.application.config.x.git_hash}")
+    end
+
+    def initialize
+      @http ||= Danbooru::Http.default
+    end
+
     def get(url, **options)
       request(:get, url, **options)
     end
diff --git a/app/logical/deep_danbooru_client.rb b/app/logical/deep_danbooru_client.rb
index 26da348fe..f6ab17605 100644
--- a/app/logical/deep_danbooru_client.rb
+++ b/app/logical/deep_danbooru_client.rb
@@ -3,7 +3,7 @@
 class DeepDanbooruClient
   attr_reader :http
 
-  def initialize(http: Danbooru::Http.new)
+  def initialize(http: Danbooru::Http.internal)
     @http = http
   end
 
diff --git a/app/logical/deviant_art_api_client.rb b/app/logical/deviant_art_api_client.rb
index e8372ee3e..b67d63ddf 100644
--- a/app/logical/deviant_art_api_client.rb
+++ b/app/logical/deviant_art_api_client.rb
@@ -21,10 +21,10 @@ class DeviantArtApiClient
   class Error < StandardError; end
   BASE_URL = "https://www.deviantart.com/api/v1/oauth2/"
 
-  attr_reader :client_id, :client_secret
+  attr_reader :client_id, :client_secret, :http
 
-  def initialize(client_id, client_secret)
-    @client_id, @client_secret = client_id, client_secret
+  def initialize(client_id, client_secret, http)
+    @client_id, @client_secret, @http = client_id, client_secret, http
   end
 
   # https://www.deviantart.com/developers/http/v1/20160316/deviation_single/bcc296bdf3b5e40636825a942a514816
@@ -54,7 +54,7 @@ class DeviantArtApiClient
     params = { access_token: access_token.token, **params }
 
     url = URI.join(BASE_URL, url).to_s
-    response = Danbooru::Http.cache(1.minute).get(url, params: params)
+    response = http.cache(1.minute).get(url, params: params)
     response.parse.with_indifferent_access
   end
 
diff --git a/app/logical/discord_api_client.rb b/app/logical/discord_api_client.rb
index dd42f089b..75b1ca06f 100644
--- a/app/logical/discord_api_client.rb
+++ b/app/logical/discord_api_client.rb
@@ -14,7 +14,7 @@ class DiscordApiClient
 
   attr_reader :application_id, :bot_token, :http
 
-  def initialize(application_id: Danbooru.config.discord_application_client_id, bot_token: Danbooru.config.discord_bot_token, http: Danbooru::Http.new)
+  def initialize(application_id: Danbooru.config.discord_application_client_id, bot_token: Danbooru.config.discord_bot_token, http: Danbooru::Http.external)
     @application_id = application_id
     @bot_token = bot_token
     @http = http
diff --git a/app/logical/discord_webhook_service.rb b/app/logical/discord_webhook_service.rb
index 815a3d9e5..aadfda873 100644
--- a/app/logical/discord_webhook_service.rb
+++ b/app/logical/discord_webhook_service.rb
@@ -6,7 +6,7 @@
 class DiscordWebhookService
   attr_reader :webhook_id, :webhook_secret, :http
 
-  def initialize(webhook_id: Danbooru.config.discord_webhook_id, webhook_secret: Danbooru.config.discord_webhook_secret, http: Danbooru::Http.new)
+  def initialize(webhook_id: Danbooru.config.discord_webhook_id, webhook_secret: Danbooru.config.discord_webhook_secret, http: Danbooru::Http.external)
     @webhook_id = webhook_id
     @webhook_secret = webhook_secret
     @http = http
diff --git a/app/logical/iqdb_client.rb b/app/logical/iqdb_client.rb
index a40a77733..c0e0c7933 100644
--- a/app/logical/iqdb_client.rb
+++ b/app/logical/iqdb_client.rb
@@ -11,7 +11,7 @@ class IqdbClient
   # Create a new IQDB API client.
   # @param iqdb_url [String] the base URL of the IQDB server
   # @param http [Danbooru::Http] the HTTP client to use
-  def initialize(iqdb_url: Danbooru.config.iqdb_url.to_s, http: Danbooru::Http.new)
+  def initialize(iqdb_url: Danbooru.config.iqdb_url.to_s, http: Danbooru::Http.internal)
     @iqdb_url = iqdb_url.chomp("/")
     @http = http
   end
diff --git a/app/logical/nico_seiga_api_client.rb b/app/logical/nico_seiga_api_client.rb
index 15dd53b92..74a68cc56 100644
--- a/app/logical/nico_seiga_api_client.rb
+++ b/app/logical/nico_seiga_api_client.rb
@@ -7,7 +7,7 @@ class NicoSeigaApiClient
 
   attr_reader :http, :user_session
 
-  def initialize(work_id:, type:, user_session: Danbooru.config.nico_seiga_user_session, http: Danbooru::Http.new)
+  def initialize(work_id:, type:, user_session: Danbooru.config.nico_seiga_user_session, http:)
     @work_id = work_id
     @work_type = type
     @user_session = user_session
diff --git a/app/logical/pixiv_ajax_client.rb b/app/logical/pixiv_ajax_client.rb
index b6f438c48..ecf5cf3b1 100644
--- a/app/logical/pixiv_ajax_client.rb
+++ b/app/logical/pixiv_ajax_client.rb
@@ -365,7 +365,7 @@ class PixivAjaxClient
 
   # @param phpsessid [String] the Pixiv login cookie
   # @param http [Danbooru::Http] the HTTP client to use for Pixiv
-  def initialize(phpsessid, http: Danbooru::Http.new)
+  def initialize(phpsessid, http:)
     @phpsessid = phpsessid
     @http = http
   end
diff --git a/app/logical/reportbooru_service.rb b/app/logical/reportbooru_service.rb
index 2c56368bf..6f23b1acc 100644
--- a/app/logical/reportbooru_service.rb
+++ b/app/logical/reportbooru_service.rb
@@ -7,7 +7,7 @@
 class ReportbooruService
   attr_reader :http, :reportbooru_server
 
-  def initialize(http: Danbooru::Http.new, reportbooru_server: Danbooru.config.reportbooru_server)
+  def initialize(http: Danbooru::Http.internal, reportbooru_server: Danbooru.config.reportbooru_server)
     @reportbooru_server = reportbooru_server
     @http = http.timeout(1)
   end
diff --git a/app/logical/source/extractor.rb b/app/logical/source/extractor.rb
index f720e6a01..3113451c4 100644
--- a/app/logical/source/extractor.rb
+++ b/app/logical/source/extractor.rb
@@ -196,7 +196,7 @@ module Source
 
     # A http client for API requests.
     def http
-      Danbooru::Http.new.proxy.public_only
+      Danbooru::Http.external
     end
 
     # A http client for downloading files.
diff --git a/app/logical/source/extractor/deviant_art.rb b/app/logical/source/extractor/deviant_art.rb
index d9c0f9441..054fcf6c3 100644
--- a/app/logical/source/extractor/deviant_art.rb
+++ b/app/logical/source/extractor/deviant_art.rb
@@ -189,10 +189,7 @@ module Source
       memoize :uuid
 
       def api_client
-        api_client = DeviantArtApiClient.new(
-          Danbooru.config.deviantart_client_id,
-          Danbooru.config.deviantart_client_secret
-        )
+        api_client = DeviantArtApiClient.new(Danbooru.config.deviantart_client_id, Danbooru.config.deviantart_client_secret, http)
         api_client.access_token = Cache.get("da-access-token", 11.weeks) do
           api_client.access_token.to_hash
         end
diff --git a/app/logical/source/extractor/fantia.rb b/app/logical/source/extractor/fantia.rb
index 847213bb3..7bb61ddf8 100644
--- a/app/logical/source/extractor/fantia.rb
+++ b/app/logical/source/extractor/fantia.rb
@@ -157,7 +157,7 @@ class Source::Extractor
     end
 
     def http
-      Danbooru::Http.new.cookies(_session_id: Danbooru.config.fantia_session_id)
+      super.cookies(_session_id: Danbooru.config.fantia_session_id)
     end
   end
 end
diff --git a/app/logical/source/extractor/furaffinity.rb b/app/logical/source/extractor/furaffinity.rb
index 40e733d23..e0b5deb92 100644
--- a/app/logical/source/extractor/furaffinity.rb
+++ b/app/logical/source/extractor/furaffinity.rb
@@ -67,7 +67,7 @@ class Source::Extractor
     end
 
     def http
-      Danbooru::Http.new.cookies(a: Danbooru.config.furaffinity_cookie_a, b: Danbooru.config.furaffinity_cookie_b, sfw: 0)
+      super.cookies(a: Danbooru.config.furaffinity_cookie_a, b: Danbooru.config.furaffinity_cookie_b, sfw: 0)
     end
   end
 end
diff --git a/app/logical/source/extractor/twitter.rb b/app/logical/source/extractor/twitter.rb
index edbfb4f28..c9e8e1450 100644
--- a/app/logical/source/extractor/twitter.rb
+++ b/app/logical/source/extractor/twitter.rb
@@ -130,7 +130,7 @@ class Source::Extractor
     end
 
     def api_client
-      TwitterApiClient.new(Danbooru.config.twitter_api_key, Danbooru.config.twitter_api_secret)
+      TwitterApiClient.new(Danbooru.config.twitter_api_key, Danbooru.config.twitter_api_secret, http: http)
     end
 
     def api_response
diff --git a/app/logical/twitter_api_client.rb b/app/logical/twitter_api_client.rb
index 9e52b055f..b77122a6c 100644
--- a/app/logical/twitter_api_client.rb
+++ b/app/logical/twitter_api_client.rb
@@ -10,7 +10,7 @@ class TwitterApiClient
   # Create a Twitter API client
   # @param api_key [String] the Twitter API key
   # @param api_secret [String] the Twitter API secret
-  def initialize(api_key, api_secret, http: Danbooru::Http.new)
+  def initialize(api_key, api_secret, http:)
     @api_key, @api_secret, @http = api_key, api_secret, http
   end