jaculabilis/danbooru
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

twitter: replace twitter gem with our own API client.

Browse Source 
The twitter gem had several problems:

* It's been unmaintained for over a year.
* It pulled in a lot of dependencies, many of which were outdated. In
  particular, it locked the `http` gem to version 3.3, preventing us
  from upgrading to 4.2.
* It raised exceptions on normal error conditions, like for deleted
  tweets or suspended users, which we really don't want.
* We had to wrap it to provide caching.

Changes:

* Fixes #4226 (Exception when creating new artists entries for suspended
  Twitter accounts)
* Drop support for scraping images from summary cards. Summary cards
  are the previews you get when you link to a website in a tweet. These
  preview images aren't always the best image.
This commit is contained in:
evazion
2019-12-13 17:27:03 -06:00
parent 0b556ece1c
commit da84e3a2f2
6 changed files with 73 additions and 145 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
app/logical/sources/strategies/twitter.rb
View File

@@ -16,7 +16,7 @@ module Sources::Strategies
RESERVED_USERNAMES = %w[home i intent search]
def self.enabled?
TwitterService.new.enabled?
Danbooru.config.twitter_api_key.present? && Danbooru.config.twitter_api_secret.present?
end
# https://twitter.com/i/web/status/943446161586733056
@@ -49,12 +49,20 @@ module Sources::Strategies
if url =~ IMAGE_URL
["https://pbs.twimg.com/media/#{$~[:file_name]}.#{$~[:file_ext]}:orig"]
elsif api_response.present?
service.image_urls(api_response)
api_response.dig(:extended_entities, :media).to_a.map do |media|
if media[:type] == "photo"
media[:media_url_https] + ":orig"
elsif media[:type].in?(["video", "animated_gif"])
variants = media.dig(:video_info, :variants)
videos = variants.select { |variant| variant[:content_type] == "video/mp4" }
video = videos.max_by { |video| video[:bitrate].to_i }
video[:url]
end
end
else
[url]
end
end
memoize :image_urls
def preview_urls
image_urls.map do |x|
@@ -73,9 +81,8 @@ module Sources::Strategies
end
def intent_url
return nil if api_response.blank?
user_id = api_response.attrs[:user][:id_str]
user_id = api_response.dig(:user, :id_str)
return nil if user_id.blank?
"https://twitter.com/intent/user?user_id=#{user_id}"
end
@@ -87,7 +94,7 @@ module Sources::Strategies
if artist_name_from_url.present?
artist_name_from_url
elsif api_response.present?
api_response.attrs[:user][:screen_name]
api_response.dig(:user, :screen_name)
else
""
end
@@ -98,8 +105,7 @@ module Sources::Strategies
end
def artist_commentary_desc
return "" if api_response.blank?
api_response.attrs[:full_text]
api_response[:full_text].to_s
end
def normalizable_for_artist_finder?
@@ -111,22 +117,19 @@ module Sources::Strategies
end
def tags
return [] if api_response.blank?
api_response.attrs[:entities][:hashtags].map do |text:, indices:|
[text, "https://twitter.com/hashtag/#{text}"]
api_response.dig(:entities, :hashtags).to_a.map do |hashtag|
[hashtag[:text], "https://twitter.com/hashtag/#{hashtag[:text]}"]
end
end
memoize :tags
def dtext_artist_commentary_desc
return "" if artist_commentary_desc.blank?
url_replacements = api_response.urls.map do |obj|
[obj.url.to_s, obj.expanded_url.to_s]
url_replacements = api_response.dig(:entities, :urls).to_a.map do |obj|
[obj[:url], obj[:expanded_url]]
end
url_replacements += api_response.media.map do |obj|
[obj.url.to_s, ""]
url_replacements += api_response.dig(:extended_entities, :media).to_a.map do |obj|
[obj[:url], ""]
end
url_replacements = url_replacements.to_h
@@ -137,30 +140,26 @@ module Sources::Strategies
desc = desc.gsub(%r!@([a-zA-Z0-9_]+)!, '"@\\1":[https://twitter.com/\\1]')
desc.strip
end
memoize :dtext_artist_commentary_desc
public
def service
TwitterService.new
def api_client
TwitterApiClient.new(Danbooru.config.twitter_api_key, Danbooru.config.twitter_api_secret)
end
memoize :service
def api_response
return {} if !service.enabled?
service.status(status_id, tweet_mode: "extended")
rescue ::Twitter::Error::NotFound
{}
return {} if !self.class.enabled?
api_client.status(status_id)
end
memoize :api_response
def status_id
[url, referer_url].map {|x| self.class.status_id_from_url(x)}.compact.first
end
memoize :status_id
def artist_name_from_url
[url, referer_url].map {|x| self.class.artist_name_from_url(x)}.compact.first
end
memoize :api_response
end
end

26
app/logical/twitter_api_client.rb Normal file
View File

@@ -0,0 +1,26 @@
class TwitterApiClient
extend Memoist
attr_reader :api_key, :api_secret
def initialize(api_key, api_secret)
@api_key, @api_secret = api_key, api_secret
end
def bearer_token(token_expiry = 24.hours)
http = Danbooru::Http.basic_auth(user: api_key, pass: api_secret)
response = http.cache(token_expiry).post("https://api.twitter.com/oauth2/token", form: { grant_type: :client_credentials })
response.parse["access_token"]
end
def client
Danbooru::Http.auth("Bearer #{bearer_token}")
end
def status(id, cache: 1.minute)
response = client.cache(cache).get("https://api.twitter.com/1.1/statuses/show.json?id=#{id}&tweet_mode=extended")
response.parse.with_indifferent_access
end
memoize :bearer_token, :client
end

74
app/logical/twitter_service.rb
View File

@@ -1,74 +0,0 @@
class TwitterService
class Error < Exception ; end
extend Memoist
def enabled?
Danbooru.config.twitter_api_key.present? && Danbooru.config.twitter_api_secret.present?
end
def client
raise Error, "Twitter API keys not set" if !enabled?
rest_client = ::Twitter::REST::Client.new do |config|
config.consumer_key = Danbooru.config.twitter_api_key
config.consumer_secret = Danbooru.config.twitter_api_secret
if bearer_token = Cache.get("twitter-api-token")
config.bearer_token = bearer_token
end
end
Cache.put("twitter-api-token", rest_client.bearer_token)
rest_client
end
memoize :client
def status(id, options = {})
Cache.get("twitterapi:#{id}", 60) do
client.status(id, options)
end
end
def extract_urls_for_status(tweet)
tweet.media.map do |obj|
if obj.is_a?(Twitter::Media::Photo)
obj.media_url_https.to_s + ":orig"
elsif obj.is_a?(Twitter::Media::Video)
video = obj.video_info.variants.select do |x|
x.content_type == "video/mp4"
end.max_by {|y| y.bitrate}
if video
video.url.to_s
end
end
end.compact.uniq
end
def extract_og_image_from_page(url)
resp = HTTParty.get(url, Danbooru.config.httparty_options)
if resp.success?
doc = Nokogiri::HTML(resp.body)
images = doc.css("meta[property='og:image']")
return images.first.attr("content").sub(":large", ":orig")
end
end
def extract_urls_for_card(attrs)
urls = attrs.urls.map {|x| x.expanded_url}
url = urls.reject {|x| x.host == "twitter.com"}.first
if url.nil?
url = urls.first
end
[extract_og_image_from_page(url)].compact
end
def image_urls(tweet)
if tweet.media.any?
extract_urls_for_status(tweet)
elsif tweet.urls.any?
extract_urls_for_card(tweet)
else
[]
end
end
end