Merge pull request #3190 from evazion/feat-tumblr-support

Add Tumblr integration (#3184)

app/logical/downloads/file.rb

@@ -140,8 +140,8 @@ module Downloads
     def set_source_to_referer(src, referer)
       if Sources::Strategies::Nijie.url_match?(src) ||
          Sources::Strategies::Twitter.url_match?(src) ||
-         Sources::Strategies::Tumblr.url_match?(src) ||
          Sources::Strategies::Pawoo.url_match?(src) ||
+         Sources::Strategies::Tumblr.url_match?(src) || Sources::Strategies::Tumblr.url_match?(referer)
          Sources::Strategies::ArtStation.url_match?(src) || Sources::Strategies::ArtStation.url_match?(referer)
         strategy = Sources::Site.new(src, :referer_url => referer)
         strategy.referer_url

app/logical/downloads/rewrite_strategies/tumblr.rb

@@ -10,6 +10,7 @@ module Downloads
       def rewrite(url, headers, data = {})
         url = rewrite_cdn(url)
         url = rewrite_samples(url, headers)
+        url = rewrite_html_pages(url)
 
         return [url, headers, data]
       end
@@ -56,6 +57,14 @@ module Downloads
         url.sub!(%r!\Ahttps?://gs1\.wac\.edgecastcdn\.net/8019B6/data\.tumblr\.com!i, "http://data.tumblr.com")
         url
       end
+
+      def rewrite_html_pages(url)
+        if Sources::Strategies::Tumblr.url_match?(url)
+          url = Sources::Strategies::Tumblr.new(url).image_url
+        end
+
+        url
+      end
     end
   end
 end

app/logical/sources/strategies/tumblr.rb

@@ -1,28 +1,130 @@
 module Sources::Strategies
   class Tumblr < Base
+    extend Memoist
+
     def self.url_match?(url)
-      url =~ %r{^https?://.+\.tumblr\.com/(?:\w+/)?(?:tumblr_)?(\w+_)(\d+)\..+$} || url =~ %r{^https?://[^.]+\.tumblr\.com/(?:post|image)/\d+}
+      blog_name, post_id = parse_info_from_url(url)
+      blog_name.present? && post_id.present?
     end
 
     def referer_url
-      if @referer_url =~ %r{^https?://[^.]+\.tumblr\.com/post/\d+} && @url =~ %r{^https?://.+\.tumblr\.com/(?:\w+/)?(?:tumblr_)?(\w+_)(\d+)\..+$}
-        @referer_url
-      elsif @referer_url =~ %r{^https?://[^.]+\.tumblr\.com/image/\d+} && @url =~ %r{^https?://.+\.tumblr\.com/(?:\w+/)?(?:tumblr_)?(\w+_)(\d+)\..+$}
-        @referer_url.sub("/image/", "/post/")
-      else
-        @url
-      end
+      blog_name, post_id = self.class.parse_info_from_url(normalized_url)
+      "https://#{blog_name}.tumblr.com/post/#{post_id}"
     end
 
     def tags
-      []
+      post[:tags].map do |tag|
+        # normalize tags: space, underscore, and hyphen are equivalent in tumblr tags.
+        [tag.tr(" _-", "_"), "https://tumblr.com/tagged/#{CGI::escape(tag.tr(" _-", "-"))}"]
+      end.uniq
     end
 
     def site_name
       "Tumblr"
     end
 
+    def profile_url
+      "https://#{artist_name}.tumblr.com/"
+    end
+
+    def artist_name
+      post[:blog_name]
+    end
+
+    def artist_commentary_title
+      case post[:type]
+      when "text", "link"
+        post[:title]
+      else
+        nil
+      end
+    end
+
+    def artist_commentary_desc
+      case post[:type]
+      when "text"
+        post[:body]
+      when "link"
+        post[:description]
+      when "photo", "video"
+        post[:caption]
+      else
+        nil
+      end
+    end
+
+    def image_url
+      image_urls.first
+    end
+
+    def image_urls
+      urls = case post[:type]
+      when "photo"
+        post[:photos].map do |photo|
+          self.class.normalize_image_url(photo[:original_size][:url])
+        end
+      when "video"
+        [post[:video_url]]
+      else
+        []
+      end
+
+      urls += self.class.parse_inline_images(artist_commentary_desc)
+      urls
+    end
+
     def get
     end
+
+    module HelperMethods
+      extend ActiveSupport::Concern
+
+      module ClassMethods
+        def parse_info_from_url(url)
+          url =~ %r!\Ahttps?://(?<blog_name>[^.]+)\.tumblr\.com/(?:post|image)/(?<post_id>\d+)!i
+          [$1, $2]
+        end
+
+        def parse_inline_images(text)
+          html = Nokogiri::HTML.fragment(text)
+          image_urls = html.css("img").map { |node| node["src"] }
+          image_urls = image_urls.map(&method(:normalize_image_url))
+          image_urls
+        end
+
+        def normalize_image_url(url)
+          url, _, _ = Downloads::RewriteStrategies::Tumblr.new.rewrite(url, {})
+          url
+        end
+      end
+
+      def normalized_url
+        if self.class.url_match?(@referer_url)
+          @referer_url
+        elsif self.class.url_match?(@url)
+          @url
+        end
+      end
+    end
+
+    module ApiMethods
+      def client
+        ::TumblrApiClient.new(Danbooru.config.tumblr_consumer_key)
+      end
+
+      def api_response
+        blog_name, post_id = self.class.parse_info_from_url(normalized_url)
+        client.posts(blog_name, post_id)
+      end
+
+      def post
+        api_response[:posts].first
+      end
+    end
+
+    include ApiMethods
+    include HelperMethods
+
+    memoize :client, :api_response
   end
 end

app/logical/tumblr_api_client.rb

@@ -0,0 +1,9 @@
+class TumblrApiClient < Struct.new(:api_key)
+  include HTTParty
+  base_uri "https://api.tumblr.com/v2/blog/"
+
+  def posts(blog_name, post_id)
+    response = self.class.get("/#{blog_name}/posts", query: { id: post_id, api_key: api_key })
+    response.parsed_response.with_indifferent_access[:response]
+  end
+end