Merge pull request #3175 from evazion/fix-pawoo

Multiple Pawoo fixes (closes #2981)

app/logical/downloads/rewrite_strategies/base.rb

@@ -5,6 +5,8 @@
 module Downloads
   module RewriteStrategies
     class Base
+      attr_reader :url
+
       def initialize(url = nil)
         @url = url
       end

app/logical/downloads/rewrite_strategies/pawoo.rb

@@ -1,17 +1,13 @@
 module Downloads
   module RewriteStrategies
     class Pawoo < Base
-      attr_accessor :url, :source
-
-      def initialize(url)
-        @url  = url
-      end
-
       def rewrite(url, headers, data = {})
-        if PawooApiClient::Status.is_match?(url)
+        if Sources::Strategies::Pawoo.url_match?(url)
-          client = PawooApiClient.new
+          source = Sources::Strategies::Pawoo.new(url)
-          response = client.get_status(url)
+          source.get
-          url = response.image_url
+          url = source.image_url
+        elsif url =~ %r!\Ahttps?://img\.pawoo\.net/media_attachments/files/(\d+/\d+/\d+)/small/([a-z0-9]+\.\w+)\z!i
+          url = "https://img.pawoo.net/media_attachments/files/#{$1}/original/#{$2}"
         end
 
         return [url, headers, data]

app/logical/pawoo_api_client.rb

@@ -1,81 +1,92 @@
 class PawooApiClient
-	extend Memoist
+  extend Memoist
 
-	class MissingConfigurationError < Exception ; end
+  class MissingConfigurationError < Exception ; end
 
-	class Account
+  class Account
-		attr_reader :json
+    attr_reader :json
 
-		def self.is_match?(url)
+    def self.is_match?(url)
-			url =~ %r!https?://pawoo.net/web/accounts/(\d+)!
+      url =~ %r!https?://pawoo.net/web/accounts/(\d+)!
-			$1
+      $1
-		end
+    end
 
-		def initialize(json)
+    def initialize(json)
-			@json = get
+      @json = get
-		end
+    end
 
-		def profile_url
+    def profile_url
-			json["url"]
+      json["url"]
-		end
+    end
-	end
+  end
 
-	class Status
+  class Status
-		attr_reader :json
+    attr_reader :json
 
-		def self.is_match?(url)
+    def self.is_match?(url)
-			url =~ %r!https?://pawoo.net/web/statuses/(\d+)! || url =~ %r!https?://pawoo.net/@.+?/(\d+)!
+      url =~ %r!https?://pawoo.net/web/statuses/(\d+)! || url =~ %r!https?://pawoo.net/@.+?/(\d+)!
-			$1
+      $1
-		end
+    end
 
-		def initialize(json)
+    def initialize(json)
-			@json = json
+      @json = json
-		end
+    end
 
-		def account_profile_url
+    def account_profile_url
-			json["account"]["url"]
+      json["account"]["url"]
-		end
+    end
 
-		def account_name
+    def account_name
-			json["account"]["username"]
+      json["account"]["username"]
-		end
+    end
 
-		def image_url
+    def image_url
-			image_urls.first
+      image_urls.first
-		end
+    end
 
-		def image_urls
+    def image_urls
-			json["media_attachments"].map {|x| x["url"]}
+      json["media_attachments"].map {|x| x["url"]}
-		end
+    end
-	end
 
-	def get_status(url)
+    def tags
-		if id = Status.is_match?(url)
+      json["tags"].map { |tag| [tag["name"], tag["url"]] }
-			Status.new(JSON.parse(access_token.get("/api/v1/statuses/#{id}").body))
+    end
-		else
-			nil
-		end
-	end
 
-	private
+    def commentary
+      commentary = ""
+      commentary << "<p>#{json["spoiler_text"]}</p>" if json["spoiler_text"].present?
+      commentary << json["content"]
+      commentary
+    end
+  end
 
-	def fetch_access_token
+  def get_status(url)
-		raise MissingConfigurationError.new("missing pawoo client id") if Danbooru.config.pawoo_client_id.nil?
+    if id = Status.is_match?(url)
-		raise MissingConfigurationError.new("missing pawoo client secret") if Danbooru.config.pawoo_client_secret.nil?
+      Status.new(JSON.parse(access_token.get("/api/v1/statuses/#{id}").body))
+    else
+      nil
+    end
+  end
 
-		Rails.cache.fetch("pawoo-token") do
+  private
-			result = client.client_credentials.get_token
-			result.token
-		end
-	end
 
-	def access_token
+  def fetch_access_token
-		OAuth2::AccessToken.new(client, fetch_access_token)
+    raise MissingConfigurationError.new("missing pawoo client id") if Danbooru.config.pawoo_client_id.nil?
-	end
+    raise MissingConfigurationError.new("missing pawoo client secret") if Danbooru.config.pawoo_client_secret.nil?
 
-	def client
+    Cache.get("pawoo-token") do
-		OAuth2::Client.new(Danbooru.config.pawoo_client_id, Danbooru.config.pawoo_client_secret, :site => "https://pawoo.net")
+      result = client.client_credentials.get_token
-	end
+      result.token
+    end
+  end
+
+  def access_token
+    OAuth2::AccessToken.new(client, fetch_access_token)
+  end
+
+  def client
+    OAuth2::Client.new(Danbooru.config.pawoo_client_id, Danbooru.config.pawoo_client_secret, :site => "https://pawoo.net")
+  end
 
   memoize :client
 end

app/logical/sources/strategies/pawoo.rb

@@ -1,3 +1,16 @@
+# html page urls:
+#   https://pawoo.net/@evazion/19451018
+#   https://pawoo.net/web/statuses/19451018
+#
+# image urls:
+#   https://img.pawoo.net/media_attachments/files/001/297/997/small/c4272a09570757c2.png
+#   https://img.pawoo.net/media_attachments/files/001/297/997/original/c4272a09570757c2.png
+#   https://pawoo.net/media/lU2uV7C1MMQSb1czwvg
+#
+# artist urls:
+#   https://pawoo.net/@evazion
+#   https://pawoo.net/web/accounts/47806
+
 module Sources::Strategies
   class Pawoo < Base
     attr_reader :image_urls
@@ -7,23 +20,67 @@ module Sources::Strategies
     end
 
     def referer_url
-      @url
+      normalized_url
     end
 
     def site_name
       "Pawoo"
     end
 
+    def api_response
+      @response ||= PawooApiClient.new.get_status(normalized_url)
+    end
+
     def get
-      response = PawooApiClient.new.get_status(url)
+      response = api_response
       @artist_name = response.account_name
       @profile_url = response.account_profile_url
       @image_url = response.image_urls.first
       @image_urls = response.image_urls
+      @tags = response.tags
+      @artist_commentary_title = nil
+      @artist_commentary_desc = response.commentary
+    end
+
+    def normalized_url
+      if self.class.url_match?(@url)
+        @url
+      elsif self.class.url_match?(@referer_url)
+        @referer_url
+      end
     end
 
     def normalizable_for_artist_finder?
       true
     end
+
+    def dtext_artist_commentary_desc
+      to_dtext(artist_commentary_desc)
+    end
+
+    def to_dtext(text)
+      html = Nokogiri::HTML.fragment(text)
+
+      dtext = html.children.map do |element|
+        case element.name
+        when "text"
+          element.content
+        when "p"
+          to_dtext(element.inner_html) + "\n\n"
+        when "a"
+          # don't include links to the toot itself.
+          media_urls = api_response.json["media_attachments"].map { |attr| attr["text_url"] }
+          next if element.attribute("href").value.in?(media_urls)
+
+          title = to_dtext(element.inner_html)
+          url = element.attributes["href"].value
+          %("#{title}":[#{url}])
+        else
+          to_dtext(element.inner_html)
+        end
+      end.join.strip
+
+      dtext
+    end
   end
 end

app/models/artist.rb

@@ -48,6 +48,7 @@ class Artist < ApplicationRecord
           break if url =~ /(?:data|media)\.tumblr\.com\/[a-z0-9]+\/$/i
           break if url =~ /deviantart\.net\//i
           break if url =~ %r!\Ahttps?://(?:mobile\.)?twitter\.com/\Z!i
+          break if url =~ %r!pawoo\.net/(?:web/)?$!i
         end
 
         artists.inject({}) {|h, x| h[x.name] = x; h}.values.slice(0, 20)

config/danbooru_default_config.rb

@@ -373,6 +373,15 @@ module Danbooru
       nil
     end
 
+    # http://tinysubversions.com/notes/mastodon-bot/
+    def pawoo_client_id
+      nil
+    end
+
+    def pawoo_client_secret
+      nil
+    end
+
     def enable_dimension_autotagging
       true
     end

test/unit/artist_test.rb

@@ -291,6 +291,25 @@ class ArtistTest < ActiveSupport::TestCase
       end
     end
 
+    context "when finding pawoo artists" do
+      setup do
+        FactoryGirl.create(:artist, :name => "evazion", :url_string => "https://pawoo.net/@evazion")
+        FactoryGirl.create(:artist, :name => "yasumo01", :url_string => "https://pawoo.net/web/accounts/28816")
+      end
+
+      should "find the artist" do
+        assert_artist_found("evazion", "https://pawoo.net/@evazion/19451018")
+        assert_artist_found("evazion", "https://pawoo.net/web/statuses/19451018")
+        assert_artist_found("yasumo01", "https://pawoo.net/@yasumo01/222337")
+        assert_artist_found("yasumo01", "https://pawoo.net/web/statuses/222337")
+      end
+
+      should "return nothing for unknown pawoo sources" do
+        assert_artist_not_found("https://pawoo.net/@9ed00e924818/1202176")
+        assert_artist_not_found("https://pawoo.net/web/statuses/1202176")
+      end
+    end
+
     should "normalize its other names" do
       artist = FactoryGirl.create(:artist, :name => "a1", :other_names_comma => "aaa, bbb, ccc ddd")
       assert_equal("aaa, bbb, ccc_ddd", artist.other_names_comma)

test/unit/sources/pawoo_test.rb

@@ -2,7 +2,7 @@ require 'test_helper'
 
 module Sources
   class PawooTest < ActiveSupport::TestCase
-    context "The source site for pawoo" do
+    context "The source site for a https://pawoo.net/web/status/$id url"  do
       setup do
         @site = Sources::Site.new("https://pawoo.net/web/statuses/1202176")
         @site.get
@@ -19,6 +19,80 @@ module Sources
       should "get the image url" do
         assert_equal("https://img.pawoo.net/media_attachments/files/000/128/953/original/4c0a06087b03343f.png", @site.image_url)
       end
+
+      should "get the commentary" do
+        desc = '<p>a mind forever voyaging through strange seas of thought alone <a href="https://pawoo.net/media/9hJzXvwxVl1CezW0ecM" rel="nofollow noopener" target="_blank"><span class="invisible">https://</span><span class="ellipsis">pawoo.net/media/9hJzXvwxVl1Cez</span><span class="invisible">W0ecM</span></a></p>'
+        assert_equal(desc, @site.artist_commentary_desc)
+      end
+
+      should "get the dtext-ified commentary" do
+        desc = 'a mind forever voyaging through strange seas of thought alone'
+        assert_equal(desc, @site.dtext_artist_commentary_desc)
+      end
+    end
+
+    context "The source site for a https://pawoo.net/$user/$id url"  do
+      setup do
+        @site = Sources::Site.new("https://pawoo.net/@evazion/19451018")
+        @site.get
+      end
+
+      should "get the profile" do
+        assert_equal("https://pawoo.net/@evazion", @site.profile_url)
+      end
+
+      should "get the artist name" do
+        assert_equal("evazion", @site.artist_name)
+      end
+
+      should "get the image urls" do
+        urls = %w[
+          https://img.pawoo.net/media_attachments/files/001/297/997/original/c4272a09570757c2.png
+          https://img.pawoo.net/media_attachments/files/001/298/028/original/55a6fd252778454b.mp4
+          https://img.pawoo.net/media_attachments/files/001/298/081/original/2588ee9ba808f38f.webm
+          https://img.pawoo.net/media_attachments/files/001/298/084/original/media.mp4
+        ]
+
+        assert_equal(urls, @site.image_urls)
+      end
+
+      should "get the tags" do
+        assert_equal(%w[baz bar foo], @site.tags.map(&:first))
+      end
+
+      should "get the commentary" do
+        desc = "<p>test post please ignore</p><p>blah blah blah</p><p>this is a test 🍕</p><p><a href=\"https://pawoo.net/tags/foo\" class=\"mention hashtag\" rel=\"tag\">#<span>foo</span></a> <a href=\"https://pawoo.net/tags/bar\" class=\"mention hashtag\" rel=\"tag\">#<span>bar</span></a> <a href=\"https://pawoo.net/tags/baz\" class=\"mention hashtag\" rel=\"tag\">#<span>baz</span></a></p>"
+
+        assert_nil(@site.artist_commentary_title)
+        assert_equal(desc, @site.artist_commentary_desc)
+      end
+
+      should "get the dtext-ified commentary" do
+        desc = <<-EOS.strip_heredoc.chomp
+          test post please ignore
+
+          blah blah blah
+
+          this is a test 🍕
+
+          "#foo":[https://pawoo.net/tags/foo] "#bar":[https://pawoo.net/tags/bar] "#baz":[https://pawoo.net/tags/baz]
+        EOS
+
+        assert_equal(desc, @site.dtext_artist_commentary_desc)
+      end
+    end
+
+    context "The source site for a https://img.pawoo.net/ url"  do
+      setup do
+        @url = "https://img.pawoo.net/media_attachments/files/001/298/028/original/55a6fd252778454b.mp4"
+        @ref = "https://pawoo.net/@evazion/19451018"
+        @site = Sources::Site.new(@url, referer_url: @ref)
+        @site.get
+      end
+
+      should "fetch the source data" do
+        assert_equal("evazion", @site.artist_name)
+      end
     end
   end
 end