diff --git a/app/logical/downloads/rewrite_strategies/base.rb b/app/logical/downloads/rewrite_strategies/base.rb index 732fdb09d..f64805ab1 100644 --- a/app/logical/downloads/rewrite_strategies/base.rb +++ b/app/logical/downloads/rewrite_strategies/base.rb @@ -5,6 +5,8 @@ module Downloads module RewriteStrategies class Base + attr_reader :url + def initialize(url = nil) @url = url end diff --git a/app/logical/downloads/rewrite_strategies/pawoo.rb b/app/logical/downloads/rewrite_strategies/pawoo.rb index 0f1f7941a..3a0e3690e 100644 --- a/app/logical/downloads/rewrite_strategies/pawoo.rb +++ b/app/logical/downloads/rewrite_strategies/pawoo.rb @@ -1,17 +1,13 @@ module Downloads module RewriteStrategies class Pawoo < Base - attr_accessor :url, :source - - def initialize(url) - @url = url - end - def rewrite(url, headers, data = {}) - if PawooApiClient::Status.is_match?(url) - client = PawooApiClient.new - response = client.get_status(url) - url = response.image_url + if Sources::Strategies::Pawoo.url_match?(url) + source = Sources::Strategies::Pawoo.new(url) + source.get + url = source.image_url + elsif url =~ %r!\Ahttps?://img\.pawoo\.net/media_attachments/files/(\d+/\d+/\d+)/small/([a-z0-9]+\.\w+)\z!i + url = "https://img.pawoo.net/media_attachments/files/#{$1}/original/#{$2}" end return [url, headers, data] diff --git a/app/logical/pawoo_api_client.rb b/app/logical/pawoo_api_client.rb index 4e5913b1b..ad7526fce 100644 --- a/app/logical/pawoo_api_client.rb +++ b/app/logical/pawoo_api_client.rb @@ -1,81 +1,92 @@ class PawooApiClient - extend Memoist + extend Memoist - class MissingConfigurationError < Exception ; end + class MissingConfigurationError < Exception ; end - class Account - attr_reader :json + class Account + attr_reader :json - def self.is_match?(url) - url =~ %r!https?://pawoo.net/web/accounts/(\d+)! - $1 - end + def self.is_match?(url) + url =~ %r!https?://pawoo.net/web/accounts/(\d+)! + $1 + end - def initialize(json) - @json = get - end + def initialize(json) + @json = get + end - def profile_url - json["url"] - end - end + def profile_url + json["url"] + end + end - class Status - attr_reader :json + class Status + attr_reader :json - def self.is_match?(url) - url =~ %r!https?://pawoo.net/web/statuses/(\d+)! || url =~ %r!https?://pawoo.net/@.+?/(\d+)! - $1 - end + def self.is_match?(url) + url =~ %r!https?://pawoo.net/web/statuses/(\d+)! || url =~ %r!https?://pawoo.net/@.+?/(\d+)! + $1 + end - def initialize(json) - @json = json - end + def initialize(json) + @json = json + end - def account_profile_url - json["account"]["url"] - end + def account_profile_url + json["account"]["url"] + end - def account_name - json["account"]["username"] - end + def account_name + json["account"]["username"] + end - def image_url - image_urls.first - end + def image_url + image_urls.first + end - def image_urls - json["media_attachments"].map {|x| x["url"]} - end - end + def image_urls + json["media_attachments"].map {|x| x["url"]} + end - def get_status(url) - if id = Status.is_match?(url) - Status.new(JSON.parse(access_token.get("/api/v1/statuses/#{id}").body)) - else - nil - end - end + def tags + json["tags"].map { |tag| [tag["name"], tag["url"]] } + end - private + def commentary + commentary = "" + commentary << "

#{json["spoiler_text"]}

" if json["spoiler_text"].present? + commentary << json["content"] + commentary + end + end - def fetch_access_token - raise MissingConfigurationError.new("missing pawoo client id") if Danbooru.config.pawoo_client_id.nil? - raise MissingConfigurationError.new("missing pawoo client secret") if Danbooru.config.pawoo_client_secret.nil? + def get_status(url) + if id = Status.is_match?(url) + Status.new(JSON.parse(access_token.get("/api/v1/statuses/#{id}").body)) + else + nil + end + end - Rails.cache.fetch("pawoo-token") do - result = client.client_credentials.get_token - result.token - end - end + private - def access_token - OAuth2::AccessToken.new(client, fetch_access_token) - end + def fetch_access_token + raise MissingConfigurationError.new("missing pawoo client id") if Danbooru.config.pawoo_client_id.nil? + raise MissingConfigurationError.new("missing pawoo client secret") if Danbooru.config.pawoo_client_secret.nil? - def client - OAuth2::Client.new(Danbooru.config.pawoo_client_id, Danbooru.config.pawoo_client_secret, :site => "https://pawoo.net") - end + Cache.get("pawoo-token") do + result = client.client_credentials.get_token + result.token + end + end + + def access_token + OAuth2::AccessToken.new(client, fetch_access_token) + end + + def client + OAuth2::Client.new(Danbooru.config.pawoo_client_id, Danbooru.config.pawoo_client_secret, :site => "https://pawoo.net") + end memoize :client end diff --git a/app/logical/sources/strategies/pawoo.rb b/app/logical/sources/strategies/pawoo.rb index 1cc6dda0c..1605332e0 100644 --- a/app/logical/sources/strategies/pawoo.rb +++ b/app/logical/sources/strategies/pawoo.rb @@ -1,3 +1,16 @@ +# html page urls: +# https://pawoo.net/@evazion/19451018 +# https://pawoo.net/web/statuses/19451018 +# +# image urls: +# https://img.pawoo.net/media_attachments/files/001/297/997/small/c4272a09570757c2.png +# https://img.pawoo.net/media_attachments/files/001/297/997/original/c4272a09570757c2.png +# https://pawoo.net/media/lU2uV7C1MMQSb1czwvg +# +# artist urls: +# https://pawoo.net/@evazion +# https://pawoo.net/web/accounts/47806 + module Sources::Strategies class Pawoo < Base attr_reader :image_urls @@ -7,23 +20,67 @@ module Sources::Strategies end def referer_url - @url + normalized_url end def site_name "Pawoo" end + def api_response + @response ||= PawooApiClient.new.get_status(normalized_url) + end + def get - response = PawooApiClient.new.get_status(url) + response = api_response @artist_name = response.account_name @profile_url = response.account_profile_url @image_url = response.image_urls.first @image_urls = response.image_urls + @tags = response.tags + @artist_commentary_title = nil + @artist_commentary_desc = response.commentary + end + + def normalized_url + if self.class.url_match?(@url) + @url + elsif self.class.url_match?(@referer_url) + @referer_url + end end def normalizable_for_artist_finder? true end + + def dtext_artist_commentary_desc + to_dtext(artist_commentary_desc) + end + + def to_dtext(text) + html = Nokogiri::HTML.fragment(text) + + dtext = html.children.map do |element| + case element.name + when "text" + element.content + when "p" + to_dtext(element.inner_html) + "\n\n" + when "a" + # don't include links to the toot itself. + media_urls = api_response.json["media_attachments"].map { |attr| attr["text_url"] } + next if element.attribute("href").value.in?(media_urls) + + title = to_dtext(element.inner_html) + url = element.attributes["href"].value + %("#{title}":[#{url}]) + else + to_dtext(element.inner_html) + end + end.join.strip + + dtext + end end end diff --git a/app/models/artist.rb b/app/models/artist.rb index 3e9b436a5..152ca48d7 100644 --- a/app/models/artist.rb +++ b/app/models/artist.rb @@ -48,6 +48,7 @@ class Artist < ApplicationRecord break if url =~ /(?:data|media)\.tumblr\.com\/[a-z0-9]+\/$/i break if url =~ /deviantart\.net\//i break if url =~ %r!\Ahttps?://(?:mobile\.)?twitter\.com/\Z!i + break if url =~ %r!pawoo\.net/(?:web/)?$!i end artists.inject({}) {|h, x| h[x.name] = x; h}.values.slice(0, 20) diff --git a/config/danbooru_default_config.rb b/config/danbooru_default_config.rb index bac24967e..78a4c26bc 100644 --- a/config/danbooru_default_config.rb +++ b/config/danbooru_default_config.rb @@ -373,6 +373,15 @@ module Danbooru nil end + # http://tinysubversions.com/notes/mastodon-bot/ + def pawoo_client_id + nil + end + + def pawoo_client_secret + nil + end + def enable_dimension_autotagging true end diff --git a/test/unit/artist_test.rb b/test/unit/artist_test.rb index 5e4f703c8..7cd5f06ba 100644 --- a/test/unit/artist_test.rb +++ b/test/unit/artist_test.rb @@ -291,6 +291,25 @@ class ArtistTest < ActiveSupport::TestCase end end + context "when finding pawoo artists" do + setup do + FactoryGirl.create(:artist, :name => "evazion", :url_string => "https://pawoo.net/@evazion") + FactoryGirl.create(:artist, :name => "yasumo01", :url_string => "https://pawoo.net/web/accounts/28816") + end + + should "find the artist" do + assert_artist_found("evazion", "https://pawoo.net/@evazion/19451018") + assert_artist_found("evazion", "https://pawoo.net/web/statuses/19451018") + assert_artist_found("yasumo01", "https://pawoo.net/@yasumo01/222337") + assert_artist_found("yasumo01", "https://pawoo.net/web/statuses/222337") + end + + should "return nothing for unknown pawoo sources" do + assert_artist_not_found("https://pawoo.net/@9ed00e924818/1202176") + assert_artist_not_found("https://pawoo.net/web/statuses/1202176") + end + end + should "normalize its other names" do artist = FactoryGirl.create(:artist, :name => "a1", :other_names_comma => "aaa, bbb, ccc ddd") assert_equal("aaa, bbb, ccc_ddd", artist.other_names_comma) diff --git a/test/unit/sources/pawoo_test.rb b/test/unit/sources/pawoo_test.rb index a97652ca6..bfa34a3cd 100644 --- a/test/unit/sources/pawoo_test.rb +++ b/test/unit/sources/pawoo_test.rb @@ -2,7 +2,7 @@ require 'test_helper' module Sources class PawooTest < ActiveSupport::TestCase - context "The source site for pawoo" do + context "The source site for a https://pawoo.net/web/status/$id url" do setup do @site = Sources::Site.new("https://pawoo.net/web/statuses/1202176") @site.get @@ -19,6 +19,80 @@ module Sources should "get the image url" do assert_equal("https://img.pawoo.net/media_attachments/files/000/128/953/original/4c0a06087b03343f.png", @site.image_url) end + + should "get the commentary" do + desc = '

a mind forever voyaging through strange seas of thought alone pawoo.net/media/9hJzXvwxVl1Cez

' + assert_equal(desc, @site.artist_commentary_desc) + end + + should "get the dtext-ified commentary" do + desc = 'a mind forever voyaging through strange seas of thought alone' + assert_equal(desc, @site.dtext_artist_commentary_desc) + end + end + + context "The source site for a https://pawoo.net/$user/$id url" do + setup do + @site = Sources::Site.new("https://pawoo.net/@evazion/19451018") + @site.get + end + + should "get the profile" do + assert_equal("https://pawoo.net/@evazion", @site.profile_url) + end + + should "get the artist name" do + assert_equal("evazion", @site.artist_name) + end + + should "get the image urls" do + urls = %w[ + https://img.pawoo.net/media_attachments/files/001/297/997/original/c4272a09570757c2.png + https://img.pawoo.net/media_attachments/files/001/298/028/original/55a6fd252778454b.mp4 + https://img.pawoo.net/media_attachments/files/001/298/081/original/2588ee9ba808f38f.webm + https://img.pawoo.net/media_attachments/files/001/298/084/original/media.mp4 + ] + + assert_equal(urls, @site.image_urls) + end + + should "get the tags" do + assert_equal(%w[baz bar foo], @site.tags.map(&:first)) + end + + should "get the commentary" do + desc = "

test post please ignore

blah blah blah

this is a test 🍕

#foo #bar #baz

" + + assert_nil(@site.artist_commentary_title) + assert_equal(desc, @site.artist_commentary_desc) + end + + should "get the dtext-ified commentary" do + desc = <<-EOS.strip_heredoc.chomp + test post please ignore + + blah blah blah + + this is a test 🍕 + + "#foo":[https://pawoo.net/tags/foo] "#bar":[https://pawoo.net/tags/bar] "#baz":[https://pawoo.net/tags/baz] + EOS + + assert_equal(desc, @site.dtext_artist_commentary_desc) + end + end + + context "The source site for a https://img.pawoo.net/ url" do + setup do + @url = "https://img.pawoo.net/media_attachments/files/001/298/028/original/55a6fd252778454b.mp4" + @ref = "https://pawoo.net/@evazion/19451018" + @site = Sources::Site.new(@url, referer_url: @ref) + @site.get + end + + should "fetch the source data" do + assert_equal("evazion", @site.artist_name) + end end end end