jaculabilis/danbooru
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #3175 from evazion/fix-pawoo

Browse Source 
Multiple Pawoo fixes (closes #2981)
This commit is contained in:
Albert Yi
2017-06-19 15:22:51 -07:00
committed by GitHub
parent f2e451c851 3560bbbd47
commit 05f7226fdd
8 changed files with 241 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
app/logical/downloads/rewrite_strategies/base.rb
View File

@@ -5,6 +5,8 @@
module Downloads
module RewriteStrategies
class Base
attr_reader :url
def initialize(url = nil)
@url = url
end

16
app/logical/downloads/rewrite_strategies/pawoo.rb
View File

@@ -1,17 +1,13 @@
module Downloads
module RewriteStrategies
class Pawoo < Base
attr_accessor :url, :source
def initialize(url)
@url = url
end
def rewrite(url, headers, data = {})
if PawooApiClient::Status.is_match?(url)
client = PawooApiClient.new
response = client.get_status(url)
url = response.image_url
if Sources::Strategies::Pawoo.url_match?(url)
source = Sources::Strategies::Pawoo.new(url)
source.get
url = source.image_url
elsif url =~ %r!\Ahttps?://img\.pawoo\.net/media_attachments/files/(\d+/\d+/\d+)/small/([a-z0-9]+\.\w+)\z!i
url = "https://img.pawoo.net/media_attachments/files/#{$1}/original/#{$2}"
end
return [url, headers, data]

129
app/logical/pawoo_api_client.rb
View File

@@ -1,81 +1,92 @@
class PawooApiClient
extend Memoist
extend Memoist
class MissingConfigurationError < Exception ; end
class MissingConfigurationError < Exception ; end
class Account
attr_reader :json
class Account
attr_reader :json
def self.is_match?(url)
url =~ %r!https?://pawoo.net/web/accounts/(\d+)!
$1
end
def self.is_match?(url)
url =~ %r!https?://pawoo.net/web/accounts/(\d+)!
$1
end
def initialize(json)
@json = get
end
def initialize(json)
@json = get
end
def profile_url
json["url"]
end
end
def profile_url
json["url"]
end
end
class Status
attr_reader :json
class Status
attr_reader :json
def self.is_match?(url)
url =~ %r!https?://pawoo.net/web/statuses/(\d+)! || url =~ %r!https?://pawoo.net/@.+?/(\d+)!
$1
end
def self.is_match?(url)
url =~ %r!https?://pawoo.net/web/statuses/(\d+)! || url =~ %r!https?://pawoo.net/@.+?/(\d+)!
$1
end
def initialize(json)
@json = json
end
def initialize(json)
@json = json
end
def account_profile_url
json["account"]["url"]
end
def account_profile_url
json["account"]["url"]
end
def account_name
json["account"]["username"]
end
def account_name
json["account"]["username"]
end
def image_url
image_urls.first
end
def image_url
image_urls.first
end
def image_urls
json["media_attachments"].map {|x| x["url"]}
end
end
def image_urls
json["media_attachments"].map {|x| x["url"]}
end
def get_status(url)
if id = Status.is_match?(url)
Status.new(JSON.parse(access_token.get("/api/v1/statuses/#{id}").body))
else
nil
end
end
def tags
json["tags"].map { |tag| [tag["name"], tag["url"]] }
end
private
def commentary
commentary = ""
commentary << "<p>#{json["spoiler_text"]}</p>" if json["spoiler_text"].present?
commentary << json["content"]
commentary
end
end
def fetch_access_token
raise MissingConfigurationError.new("missing pawoo client id") if Danbooru.config.pawoo_client_id.nil?
raise MissingConfigurationError.new("missing pawoo client secret") if Danbooru.config.pawoo_client_secret.nil?
def get_status(url)
if id = Status.is_match?(url)
Status.new(JSON.parse(access_token.get("/api/v1/statuses/#{id}").body))
else
nil
end
end
Rails.cache.fetch("pawoo-token") do
result = client.client_credentials.get_token
result.token
end
end
private
def access_token
OAuth2::AccessToken.new(client, fetch_access_token)
end
def fetch_access_token
raise MissingConfigurationError.new("missing pawoo client id") if Danbooru.config.pawoo_client_id.nil?
raise MissingConfigurationError.new("missing pawoo client secret") if Danbooru.config.pawoo_client_secret.nil?
def client
OAuth2::Client.new(Danbooru.config.pawoo_client_id, Danbooru.config.pawoo_client_secret, :site => "https://pawoo.net")
end
Cache.get("pawoo-token") do
result = client.client_credentials.get_token
result.token
end
end
def access_token
OAuth2::AccessToken.new(client, fetch_access_token)
end
def client
OAuth2::Client.new(Danbooru.config.pawoo_client_id, Danbooru.config.pawoo_client_secret, :site => "https://pawoo.net")
end
memoize :client
end

61
app/logical/sources/strategies/pawoo.rb
View File

@@ -1,3 +1,16 @@
# html page urls:
# https://pawoo.net/@evazion/19451018
# https://pawoo.net/web/statuses/19451018
#
# image urls:
# https://img.pawoo.net/media_attachments/files/001/297/997/small/c4272a09570757c2.png
# https://img.pawoo.net/media_attachments/files/001/297/997/original/c4272a09570757c2.png
# https://pawoo.net/media/lU2uV7C1MMQSb1czwvg
#
# artist urls:
# https://pawoo.net/@evazion
# https://pawoo.net/web/accounts/47806
module Sources::Strategies
class Pawoo < Base
attr_reader :image_urls
@@ -7,23 +20,67 @@ module Sources::Strategies
end
def referer_url
@url
normalized_url
end
def site_name
"Pawoo"
end
def api_response
@response ||= PawooApiClient.new.get_status(normalized_url)
end
def get
response = PawooApiClient.new.get_status(url)
response = api_response
@artist_name = response.account_name
@profile_url = response.account_profile_url
@image_url = response.image_urls.first
@image_urls = response.image_urls
@tags = response.tags
@artist_commentary_title = nil
@artist_commentary_desc = response.commentary
end
def normalized_url
if self.class.url_match?(@url)
@url
elsif self.class.url_match?(@referer_url)
@referer_url
end
end
def normalizable_for_artist_finder?
true
end
def dtext_artist_commentary_desc
to_dtext(artist_commentary_desc)
end
def to_dtext(text)
html = Nokogiri::HTML.fragment(text)
dtext = html.children.map do |element|
case element.name
when "text"
element.content
when "p"
to_dtext(element.inner_html) + "\n\n"
when "a"
# don't include links to the toot itself.
media_urls = api_response.json["media_attachments"].map { |attr| attr["text_url"] }
next if element.attribute("href").value.in?(media_urls)
title = to_dtext(element.inner_html)
url = element.attributes["href"].value
%("#{title}":[#{url}])
else
to_dtext(element.inner_html)
end
end.join.strip
dtext
end
end
end

1
app/models/artist.rb
View File

@@ -48,6 +48,7 @@ class Artist < ApplicationRecord
break if url =~ /(?:data|media)\.tumblr\.com\/[a-z0-9]+\/$/i
break if url =~ /deviantart\.net\//i
break if url =~ %r!\Ahttps?://(?:mobile\.)?twitter\.com/\Z!i
break if url =~ %r!pawoo\.net/(?:web/)?$!i
end
artists.inject({}) {|h, x| h[x.name] = x; h}.values.slice(0, 20)

9
config/danbooru_default_config.rb
View File

@@ -373,6 +373,15 @@ module Danbooru
nil
end
# http://tinysubversions.com/notes/mastodon-bot/
def pawoo_client_id
nil
end
def pawoo_client_secret
nil
end
def enable_dimension_autotagging
true
end

19
test/unit/artist_test.rb
View File

@@ -291,6 +291,25 @@ class ArtistTest < ActiveSupport::TestCase
end
end
context "when finding pawoo artists" do
setup do
FactoryGirl.create(:artist, :name => "evazion", :url_string => "https://pawoo.net/@evazion")
FactoryGirl.create(:artist, :name => "yasumo01", :url_string => "https://pawoo.net/web/accounts/28816")
end
should "find the artist" do
assert_artist_found("evazion", "https://pawoo.net/@evazion/19451018")
assert_artist_found("evazion", "https://pawoo.net/web/statuses/19451018")
assert_artist_found("yasumo01", "https://pawoo.net/@yasumo01/222337")
assert_artist_found("yasumo01", "https://pawoo.net/web/statuses/222337")
end
should "return nothing for unknown pawoo sources" do
assert_artist_not_found("https://pawoo.net/@9ed00e924818/1202176")
assert_artist_not_found("https://pawoo.net/web/statuses/1202176")
end
end
should "normalize its other names" do
artist = FactoryGirl.create(:artist, :name => "a1", :other_names_comma => "aaa, bbb, ccc ddd")
assert_equal("aaa, bbb, ccc_ddd", artist.other_names_comma)

76
test/unit/sources/pawoo_test.rb
View File

@@ -2,7 +2,7 @@ require 'test_helper'
module Sources
class PawooTest < ActiveSupport::TestCase
context "The source site for pawoo" do
context "The source site for a https://pawoo.net/web/status/$id url" do
setup do
@site = Sources::Site.new("https://pawoo.net/web/statuses/1202176")
@site.get
@@ -19,6 +19,80 @@ module Sources
should "get the image url" do
assert_equal("https://img.pawoo.net/media_attachments/files/000/128/953/original/4c0a06087b03343f.png", @site.image_url)
end
should "get the commentary" do
desc = '<p>a mind forever voyaging through strange seas of thought alone <a href="https://pawoo.net/media/9hJzXvwxVl1CezW0ecM" rel="nofollow noopener" target="_blank"><span class="invisible">https://</span><span class="ellipsis">pawoo.net/media/9hJzXvwxVl1Cez</span><span class="invisible">W0ecM</span></a></p>'
assert_equal(desc, @site.artist_commentary_desc)
end
should "get the dtext-ified commentary" do
desc = 'a mind forever voyaging through strange seas of thought alone'
assert_equal(desc, @site.dtext_artist_commentary_desc)
end
end
context "The source site for a https://pawoo.net/$user/$id url" do
setup do
@site = Sources::Site.new("https://pawoo.net/@evazion/19451018")
@site.get
end
should "get the profile" do
assert_equal("https://pawoo.net/@evazion", @site.profile_url)
end
should "get the artist name" do
assert_equal("evazion", @site.artist_name)
end
should "get the image urls" do
urls = %w[
https://img.pawoo.net/media_attachments/files/001/297/997/original/c4272a09570757c2.png
https://img.pawoo.net/media_attachments/files/001/298/028/original/55a6fd252778454b.mp4
https://img.pawoo.net/media_attachments/files/001/298/081/original/2588ee9ba808f38f.webm
https://img.pawoo.net/media_attachments/files/001/298/084/original/media.mp4
]
assert_equal(urls, @site.image_urls)
end
should "get the tags" do
assert_equal(%w[baz bar foo], @site.tags.map(&:first))
end
should "get the commentary" do
desc = "<p>test post please ignore</p><p>blah blah blah</p><p>this is a test 🍕</p><p><a href=\"https://pawoo.net/tags/foo\" class=\"mention hashtag\" rel=\"tag\">#<span>foo</span></a> <a href=\"https://pawoo.net/tags/bar\" class=\"mention hashtag\" rel=\"tag\">#<span>bar</span></a> <a href=\"https://pawoo.net/tags/baz\" class=\"mention hashtag\" rel=\"tag\">#<span>baz</span></a></p>"
assert_nil(@site.artist_commentary_title)
assert_equal(desc, @site.artist_commentary_desc)
end
should "get the dtext-ified commentary" do
desc = <<-EOS.strip_heredoc.chomp
test post please ignore
blah blah blah
this is a test 🍕
"#foo":[https://pawoo.net/tags/foo] "#bar":[https://pawoo.net/tags/bar] "#baz":[https://pawoo.net/tags/baz]
EOS
assert_equal(desc, @site.dtext_artist_commentary_desc)
end
end
context "The source site for a https://img.pawoo.net/ url" do
setup do
@url = "https://img.pawoo.net/media_attachments/files/001/298/028/original/55a6fd252778454b.mp4"
@ref = "https://pawoo.net/@evazion/19451018"
@site = Sources::Site.new(@url, referer_url: @ref)
@site.get
end
should "fetch the source data" do
assert_equal("evazion", @site.artist_name)
end
end
end
end