Merge pull request #3886 from r888888888/source-api-caching

cache api clients

app/logical/deviant_art_api_client.rb

@@ -47,18 +47,20 @@ class DeviantArtApiClient
   end
 
   def request(url, **params)
-    options = httparty_options.deep_merge(
+    options = {
       base_uri: BASE_URL,
-      query: { access_token: access_token.token, **params },
+      params: { access_token: access_token.token, **params },
       headers: { "Accept-Encoding" => "gzip" },
       format: :plain,
-    )
+    }
 
-    resp = HTTParty.get(url, **options)
+    body, code = HttpartyCache.get(url, **options)
-    json = JSON.parse(Zlib.gunzip(resp.body), symbolize_names: true)
 
-    raise Error, "HTTP error #{resp.code}: #{json}" unless resp.success?
+    if code == 200
-    json
+      return JSON.parse(Zlib.gunzip(body), symbolize_names: true)
+    end
+
+    raise "DeviantArtApiClient call failed (code=#{code}, url=#{url}, body=#{body})"
   end
 
   def oauth

app/logical/httparty_cache.rb

@@ -0,0 +1,12 @@
+module HttpartyCache
+  extend self
+
+  def get(url, headers: {}, params: {}, base_uri: nil, format: :html, expiry: 60)
+    key = Cache.hash({url: url, headers: headers, params: params, base_uri: base_uri, format: format}.to_s)
+    Cache.get("cachedget:#{key}", expiry) do
+      resp = HTTParty.get(url, Danbooru.config.httparty_options.deep_merge(query: params, headers: headers, base_uri: base_uri, format: format))
+      body = resp.body.force_encoding("utf-8")
+      [body, resp.code]
+    end
+  end
+end

app/logical/nico_seiga_api_client.rb

@@ -1,43 +1,52 @@
 class NicoSeigaApiClient
+  extend Memoist
   BASE_URL = "http://seiga.nicovideo.jp/api"
-  attr_reader :user_id, :moniker, :image_id, :title, :desc
+  attr_reader :illust_id
 
-  def initialize(illust_id)
+  def initialize(illust_id:, user_id: nil)
-    get_illust(illust_id)
+    @illust_id = illust_id
-    get_artist(user_id)
+    @user_id = user_id
   end
 
-  def get_illust(id)
+  def image_id
-    uri = "#{BASE_URL}/illust/info?id=#{id}"
+    illust_xml["response"]["image"]["id"].to_i
-    resp = HTTParty.get(uri, Danbooru.config.httparty_options)
+  end
-    if resp.success?
+
-      parse_illust_xml_response(resp.body)
+  def user_id
+    @user_id || illust_xml["response"]["image"]["user_id"].to_i
+  end
+
+  def title
+    illust_xml["response"]["image"]["title"]
+  end
+
+  def desc
+    illust_xml["response"]["image"]["description"] || illust_xml["response"]["image"]["summary"]
+  end
+
+  def moniker
+    artist_xml["response"]["user"]["nickname"]
+  end
+
+  def illust_xml
+    uri = "#{BASE_URL}/illust/info?id=#{illust_id}"
+    body, code = HttpartyCache.get(uri)
+    if code == 200
+      Hash.from_xml(body)
     else
-      raise HTTParty::ResponseError.new(resp)
+      raise "nico seiga api call failed (code=#{code}, body=#{body})"
     end
   end
+  memoize :illust_xml
 
-  def get_artist(id)
+  def artist_xml
-    uri = "#{BASE_URL}/user/info?id=#{id}"
+    uri = "#{BASE_URL}/user/info?id=#{user_id}"
-    resp = HTTParty.get(uri, Danbooru.config.httparty_options)
+    body, code = HttpartyCache.get(uri)
-    if resp.success?
+    if code == 200
-      parse_artist_xml_response(resp.body)
+      Hash.from_xml(body)
     else
-      raise HTTParty::ResponseError.new(resp)
+      raise "nico seiga api call failed (code=#{code}, body=#{body})"
     end
   end
-
+  memoize :artist_xml
-  def parse_artist_xml_response(text)
-    doc = Hash.from_xml(text)
-    @moniker = doc["response"]["user"]["nickname"]
-  end
-
-  def parse_illust_xml_response(text)
-    doc = Hash.from_xml(text)
-    image = doc["response"]["image"]
-    @image_id = image["id"].to_i
-    @user_id = image["user_id"].to_i
-    @title = image["title"]
-    @desc = image["description"] || image["summary"]
-  end
 end

app/logical/pixiv_api_client.rb

@@ -164,19 +164,19 @@ class PixivApiClient
     }
 
     url = "https://public-api.secure.pixiv.net/v#{API_VERSION}/works/#{illust_id.to_i}.json"
-    resp = HTTParty.get(url, Danbooru.config.httparty_options.deep_merge(query: params, headers: headers))
+    body, code = HttpartyCache.get(url, headers: headers, params: params)
-    body = resp.body.force_encoding("utf-8")
     json = JSON.parse(body)
 
-    if resp.success?
+    if code == 200
       WorkResponse.new(json["response"][0])
     elsif json["status"] == "failure" && json.dig("errors", "system", "message") =~ /対象のイラストは見つかりませんでした。/
       raise BadIDError.new("Pixiv ##{illust_id} not found: work was deleted, made private, or ID is invalid.")
     else
-      raise Error.new("Pixiv API call failed (status=#{resp.code} body=#{body})")
+      raise Error.new("Pixiv API call failed (status=#{code} body=#{body})")
     end
+
   rescue JSON::ParserError
-    raise Error.new("Pixiv API call failed (status=#{resp.code} body=#{body})")
+    raise Error.new("Pixiv API call failed (status=#{code} body=#{body})")
   end
 
   def fanbox(fanbox_id)

app/logical/sources/strategies/nico_seiga.rb

@@ -123,7 +123,7 @@ module Sources
     public
 
       def api_client
-        NicoSeigaApiClient.new(illust_id)
+        NicoSeigaApiClient.new(illust_id: illust_id)
       end
       memoize :api_client
 

app/logical/sources/strategies/twitter.rb

@@ -53,13 +53,17 @@ module Sources::Strategies
     end
 
     def profile_url
-      if url =~ %r{\Ahttps?://(?:mobile\.)?twitter\.com/(\w+)}i && $1 != "i"
+      if url =~ %r{\Ahttps?://(?:mobile\.)?twitter\.com/(\w+)}i
-        "https://twitter.com/#{$1}"
+        if $1 != "i"
-      elsif artist_name.present?
+          return "https://twitter.com/#{$1}"
-        "https://twitter.com/" + artist_name
+        end
-      else
-        ""
       end
+
+      if artist_name.present?
+        return "https://twitter.com/" + artist_name
+      end
+
+      ""
     end
 
     def artists
@@ -129,7 +133,7 @@ module Sources::Strategies
     memoize :service
 
     def api_response
-      service.client.status(status_id, tweet_mode: "extended")
+      service.status(status_id, tweet_mode: "extended")
     rescue ::Twitter::Error::NotFound
       {}
     end

app/logical/tumblr_api_client.rb

@@ -1,9 +1,14 @@
 class TumblrApiClient < Struct.new(:api_key)
-  include HTTParty
-  base_uri "https://api.tumblr.com/v2/blog/"
-
   def posts(blog_name, post_id)
-    response = self.class.get("/#{blog_name}/posts", Danbooru.config.httparty_options.merge(query: { id: post_id, api_key: api_key }))
+    body, code = HttpartyCache.get("/#{blog_name}/posts", 
-    response.parsed_response.with_indifferent_access[:response]
+      params: { id: post_id, api_key: api_key },
+      base_uri: "https://api.tumblr.com/v2/blog/"
+    )
+
+    if code == 200
+      return JSON.parse(body)["response"].with_indifferent_access
+    end
+
+    raise "TumblrApiClient call failed (code=#{code}, body=#{body}, blog_name=#{blog_name}, post_id=#{post_id})"
   end
 end

app/logical/twitter_service.rb

@@ -18,6 +18,12 @@ class TwitterService
   end
   memoize :client
 
+  def status(id, options = {})
+    Cache.get("twitterapi:#{id}", 60) do
+      client.status(id, options)      
+    end
+  end
+
   def extract_urls_for_status(tweet)
     tweet.media.map do |obj|
       if obj.is_a?(Twitter::Media::Photo)

test/models/upload_service_test.rb

@@ -1121,7 +1121,7 @@ class UploadServiceTest < ActiveSupport::TestCase
 
     context "for nijie" do
       should "record the canonical source" do
-        page_url = "https://nijie.info/view.php?id=213043"
+        page_url = "https://nijie.info/view.php?id=728995"
         image_url = "https://pic03.nijie.info/nijie_picture/728995_20170505014820_0.jpg"
         upload = FactoryBot.create(:jpg_upload, file_size: 1000, md5: "12345", file_ext: "jpg", image_width: 100, image_height: 100, source: image_url, referer_url: page_url)
 

test/unit/downloads/tumblr_test.rb

@@ -80,7 +80,7 @@ module Downloads
         @source = "http://media.tumblr.com/tumblr_m24kbxqKAX1rszquso1_250.jpg"
         @rewrite = "https://media.tumblr.com/tumblr_m24kbxqKAX1rszquso1_1280.jpg"
         assert_rewritten(@rewrite, @source, @ref)
-        assert_downloaded(101869, @source, @ref)
+        assert_downloaded(105963, @source, @ref)
         # assert_downloaded(296_399, @source)
       end
     end

test/unit/downloads/twitter_test.rb

@@ -17,10 +17,10 @@ module Downloads
     context "downloading a 'https://twitter.com/:user/status/:id/photo/:n' card url" do
       should "download the orig file" do
         skip "Twitter key is not set" unless Danbooru.config.twitter_api_key
-        @source = "https://twitter.com/paxiti/status/1035511366629568512/photo/1"
+        @source = "https://twitter.com/uroobnad/status/1039308544644763648/photo/1"
-        @rewrite = "https://pbs.twimg.com/media/Dl7f3G4VsAEoZXz.jpg:orig"
+        @rewrite = "https://danbooru.donmai.us/data/sample/sample-1cfa3153f9d5a546d055d5977905ebb4.jpg"
         assert_rewritten(@rewrite, @source)
-        assert_downloaded(131_525, @source)
+        assert_downloaded(179493, @source)
       end
     end
 

test/unit/sources/pixiv_test.rb

@@ -165,11 +165,9 @@ module Sources
       end
 
       context "fetching source data for a deleted work" do
-        should "raise a bad id error" do
+        should "return the same url" do
-          assert_raise(::PixivApiClient::BadIDError) do
+          get_source("https://i.pximg.net/img-original/img/2017/11/22/01/06/44/65991677_p0.png")
-            get_source("https://i.pximg.net/img-original/img/2017/11/22/01/06/44/65991677_p0.png")
+          assert_equal(["https://i.pximg.net/img-original/img/2017/11/22/01/06/44/65991677_p0.png"], @site.image_urls)
-            @site.image_urls
-          end
         end
       end