From 266c7c0d5b0a12f260799fb9554c61815d7f4761 Mon Sep 17 00:00:00 2001 From: Albert Yi Date: Mon, 10 Sep 2018 17:28:18 -0700 Subject: [PATCH] cache api clients --- app/logical/deviant_art_api_client.rb | 16 +++-- app/logical/httparty_cache.rb | 11 ++++ app/logical/nico_seiga_api_client.rb | 69 +++++++++++--------- app/logical/pixiv_api_client.rb | 10 +-- app/logical/sources/strategies/nico_seiga.rb | 2 +- app/logical/sources/strategies/twitter.rb | 18 +++-- app/logical/tumblr_api_client.rb | 15 +++-- app/logical/twitter_service.rb | 6 ++ test/unit/downloads/tumblr_test.rb | 2 +- test/unit/downloads/twitter_test.rb | 6 +- test/unit/sources/pixiv_test.rb | 8 +-- 11 files changed, 99 insertions(+), 64 deletions(-) create mode 100644 app/logical/httparty_cache.rb diff --git a/app/logical/deviant_art_api_client.rb b/app/logical/deviant_art_api_client.rb index 252e68ce5..f5d53e102 100644 --- a/app/logical/deviant_art_api_client.rb +++ b/app/logical/deviant_art_api_client.rb @@ -47,18 +47,20 @@ class DeviantArtApiClient end def request(url, **params) - options = httparty_options.deep_merge( + options = { base_uri: BASE_URL, - query: { access_token: access_token.token, **params }, + params: { access_token: access_token.token, **params }, headers: { "Accept-Encoding" => "gzip" }, format: :plain, - ) + } - resp = HTTParty.get(url, **options) - json = JSON.parse(Zlib.gunzip(resp.body), symbolize_names: true) + body, code = HttpartyCache.get(url, **options) - raise Error, "HTTP error #{resp.code}: #{json}" unless resp.success? - json + if code == 200 + return JSON.parse(Zlib.gunzip(body), symbolize_names: true) + end + + raise "DeviantArtApiClient call failed (code=#{code}, url=#{url}, body=#{body})" end def oauth diff --git a/app/logical/httparty_cache.rb b/app/logical/httparty_cache.rb new file mode 100644 index 000000000..5d40c87b3 --- /dev/null +++ b/app/logical/httparty_cache.rb @@ -0,0 +1,11 @@ +module HttpartyCache + extend self + + def get(url, headers: {}, params: {}, base_uri: nil, format: :html, expiry: 60) + Cache.get("cachedget:#{Cache.hash(url)}", expiry) do + resp = HTTParty.get(url, Danbooru.config.httparty_options.deep_merge(query: params, headers: headers, base_uri: base_uri, format: format)) + body = resp.body.force_encoding("utf-8") + [body, resp.code] + end + end +end diff --git a/app/logical/nico_seiga_api_client.rb b/app/logical/nico_seiga_api_client.rb index 73af88bcd..aa2dfe69a 100644 --- a/app/logical/nico_seiga_api_client.rb +++ b/app/logical/nico_seiga_api_client.rb @@ -1,43 +1,52 @@ class NicoSeigaApiClient + extend Memoist BASE_URL = "http://seiga.nicovideo.jp/api" - attr_reader :user_id, :moniker, :image_id, :title, :desc + attr_reader :illust_id - def initialize(illust_id) - get_illust(illust_id) - get_artist(user_id) + def initialize(illust_id:, user_id: nil) + @illust_id = illust_id + @user_id = user_id end - def get_illust(id) - uri = "#{BASE_URL}/illust/info?id=#{id}" - resp = HTTParty.get(uri, Danbooru.config.httparty_options) - if resp.success? - parse_illust_xml_response(resp.body) + def image_id + illust_xml["response"]["image"]["id"].to_i + end + + def user_id + @user_id || illust_xml["response"]["image"]["user_id"].to_i + end + + def title + illust_xml["response"]["image"]["title"] + end + + def desc + illust_xml["response"]["image"]["description"] || illust_xml["response"]["image"]["summary"] + end + + def moniker + artist_xml["response"]["user"]["nickname"] + end + + def illust_xml + uri = "#{BASE_URL}/illust/info?id=#{illust_id}" + body, code = HttpartyCache.get(uri) + if code == 200 + Hash.from_xml(body) else - raise HTTParty::ResponseError.new(resp) + raise "nico seiga api call failed (code=#{code}, body=#{body})" end end + memoize :illust_xml - def get_artist(id) - uri = "#{BASE_URL}/user/info?id=#{id}" - resp = HTTParty.get(uri, Danbooru.config.httparty_options) - if resp.success? - parse_artist_xml_response(resp.body) + def artist_xml + uri = "#{BASE_URL}/user/info?id=#{user_id}" + body, code = HttpartyCache.get(uri) + if code == 200 + Hash.from_xml(body) else - raise HTTParty::ResponseError.new(resp) + raise "nico seiga api call failed (code=#{code}, body=#{body})" end end - - def parse_artist_xml_response(text) - doc = Hash.from_xml(text) - @moniker = doc["response"]["user"]["nickname"] - end - - def parse_illust_xml_response(text) - doc = Hash.from_xml(text) - image = doc["response"]["image"] - @image_id = image["id"].to_i - @user_id = image["user_id"].to_i - @title = image["title"] - @desc = image["description"] || image["summary"] - end + memoize :artist_xml end diff --git a/app/logical/pixiv_api_client.rb b/app/logical/pixiv_api_client.rb index 39dc13de0..45d846c6d 100644 --- a/app/logical/pixiv_api_client.rb +++ b/app/logical/pixiv_api_client.rb @@ -164,19 +164,19 @@ class PixivApiClient } url = "https://public-api.secure.pixiv.net/v#{API_VERSION}/works/#{illust_id.to_i}.json" - resp = HTTParty.get(url, Danbooru.config.httparty_options.deep_merge(query: params, headers: headers)) - body = resp.body.force_encoding("utf-8") + body, code = HttpartyCache.get(url, headers: headers, params: params) json = JSON.parse(body) - if resp.success? + if code == 200 WorkResponse.new(json["response"][0]) elsif json["status"] == "failure" && json.dig("errors", "system", "message") =~ /対象のイラストは見つかりませんでした。/ raise BadIDError.new("Pixiv ##{illust_id} not found: work was deleted, made private, or ID is invalid.") else - raise Error.new("Pixiv API call failed (status=#{resp.code} body=#{body})") + raise Error.new("Pixiv API call failed (status=#{code} body=#{body})") end + rescue JSON::ParserError - raise Error.new("Pixiv API call failed (status=#{resp.code} body=#{body})") + raise Error.new("Pixiv API call failed (status=#{code} body=#{body})") end def fanbox(fanbox_id) diff --git a/app/logical/sources/strategies/nico_seiga.rb b/app/logical/sources/strategies/nico_seiga.rb index 831987ebf..699b977de 100644 --- a/app/logical/sources/strategies/nico_seiga.rb +++ b/app/logical/sources/strategies/nico_seiga.rb @@ -123,7 +123,7 @@ module Sources public def api_client - NicoSeigaApiClient.new(illust_id) + NicoSeigaApiClient.new(illust_id: illust_id) end memoize :api_client diff --git a/app/logical/sources/strategies/twitter.rb b/app/logical/sources/strategies/twitter.rb index 723664170..ae56091eb 100644 --- a/app/logical/sources/strategies/twitter.rb +++ b/app/logical/sources/strategies/twitter.rb @@ -47,13 +47,17 @@ module Sources::Strategies end def profile_url - if url =~ %r{\Ahttps?://(?:mobile\.)?twitter\.com/(\w+)}i && $1 != "i" - "https://twitter.com/#{$1}" - elsif artist_name.present? - "https://twitter.com/" + artist_name - else - "" + if url =~ %r{\Ahttps?://(?:mobile\.)?twitter\.com/(\w+)}i + if $1 != "i" + return "https://twitter.com/#{$1}" + end end + + if artist_name.present? + return "https://twitter.com/" + artist_name + end + + "" end def artists @@ -123,7 +127,7 @@ module Sources::Strategies memoize :service def api_response - service.client.status(status_id, tweet_mode: "extended") + service.status(status_id, tweet_mode: "extended") rescue ::Twitter::Error::NotFound {} end diff --git a/app/logical/tumblr_api_client.rb b/app/logical/tumblr_api_client.rb index 118ae4c3c..7e541eb22 100644 --- a/app/logical/tumblr_api_client.rb +++ b/app/logical/tumblr_api_client.rb @@ -1,9 +1,14 @@ class TumblrApiClient < Struct.new(:api_key) - include HTTParty - base_uri "https://api.tumblr.com/v2/blog/" - def posts(blog_name, post_id) - response = self.class.get("/#{blog_name}/posts", Danbooru.config.httparty_options.merge(query: { id: post_id, api_key: api_key })) - response.parsed_response.with_indifferent_access[:response] + body, code = HttpartyCache.get("/#{blog_name}/posts", + params: { id: post_id, api_key: api_key }, + base_uri: "https://api.tumblr.com/v2/blog/" + ) + + if code == 200 + return JSON.parse(body)["response"].with_indifferent_access + end + + raise "TumblrApiClient call failed (code=#{code}, body=#{body}, blog_name=#{blog_name}, post_id=#{post_id})" end end diff --git a/app/logical/twitter_service.rb b/app/logical/twitter_service.rb index 08babbb81..bf0f29cf2 100644 --- a/app/logical/twitter_service.rb +++ b/app/logical/twitter_service.rb @@ -18,6 +18,12 @@ class TwitterService end memoize :client + def status(id, options = {}) + Cache.get("twitterapi:#{id}", 60) do + client.status(id, options) + end + end + def extract_urls_for_status(tweet) tweet.media.map do |obj| if obj.is_a?(Twitter::Media::Photo) diff --git a/test/unit/downloads/tumblr_test.rb b/test/unit/downloads/tumblr_test.rb index a5b2bd17e..5eafc324c 100644 --- a/test/unit/downloads/tumblr_test.rb +++ b/test/unit/downloads/tumblr_test.rb @@ -80,7 +80,7 @@ module Downloads @source = "http://media.tumblr.com/tumblr_m24kbxqKAX1rszquso1_250.jpg" @rewrite = "https://media.tumblr.com/tumblr_m24kbxqKAX1rszquso1_1280.jpg" assert_rewritten(@rewrite, @source, @ref) - assert_downloaded(101869, @source, @ref) + assert_downloaded(105963, @source, @ref) # assert_downloaded(296_399, @source) end end diff --git a/test/unit/downloads/twitter_test.rb b/test/unit/downloads/twitter_test.rb index 724ca7944..2fb027811 100644 --- a/test/unit/downloads/twitter_test.rb +++ b/test/unit/downloads/twitter_test.rb @@ -17,10 +17,10 @@ module Downloads context "downloading a 'https://twitter.com/:user/status/:id/photo/:n' card url" do should "download the orig file" do skip "Twitter key is not set" unless Danbooru.config.twitter_api_key - @source = "https://twitter.com/paxiti/status/1035511366629568512/photo/1" - @rewrite = "https://pbs.twimg.com/media/Dl7f3G4VsAEoZXz.jpg:orig" + @source = "https://twitter.com/uroobnad/status/1039308544644763648/photo/1" + @rewrite = "https://danbooru.donmai.us/data/sample/sample-1cfa3153f9d5a546d055d5977905ebb4.jpg" assert_rewritten(@rewrite, @source) - assert_downloaded(131_525, @source) + assert_downloaded(179493, @source) end end diff --git a/test/unit/sources/pixiv_test.rb b/test/unit/sources/pixiv_test.rb index 926a2fbeb..2ec29eea4 100644 --- a/test/unit/sources/pixiv_test.rb +++ b/test/unit/sources/pixiv_test.rb @@ -161,11 +161,9 @@ module Sources end context "fetching source data for a deleted work" do - should "raise a bad id error" do - assert_raise(::PixivApiClient::BadIDError) do - get_source("https://i.pximg.net/img-original/img/2017/11/22/01/06/44/65991677_p0.png") - @site.image_urls - end + should "return the same url" do + get_source("https://i.pximg.net/img-original/img/2017/11/22/01/06/44/65991677_p0.png") + assert_equal(["https://i.pximg.net/img-original/img/2017/11/22/01/06/44/65991677_p0.png"], @site.image_urls) end end