From b053a2d7835c1ad4b888d5e17a4ff4b6df508461 Mon Sep 17 00:00:00 2001 From: r888888888 Date: Mon, 17 Apr 2017 14:15:05 -0700 Subject: [PATCH] add source + rewrite strategy for pawoo --- Gemfile | 1 + Gemfile.lock | 8 ++ .../downloads/rewrite_strategies/base.rb | 2 +- .../downloads/rewrite_strategies/pawoo.rb | 21 +++++ app/logical/pawoo_api_client.rb | 81 +++++++++++++++++++ app/logical/sources/site.rb | 2 +- app/logical/sources/strategies/pawoo.rb | 25 ++++++ ...414233617_add_updated_at_index_to_pools.rb | 5 ++ db/structure.sql | 9 +++ test/unit/sources/pawoo_test.rb | 24 ++++++ 10 files changed, 176 insertions(+), 2 deletions(-) create mode 100644 app/logical/downloads/rewrite_strategies/pawoo.rb create mode 100644 app/logical/pawoo_api_client.rb create mode 100644 app/logical/sources/strategies/pawoo.rb create mode 100644 db/migrate/20170414233617_add_updated_at_index_to_pools.rb create mode 100644 test/unit/sources/pawoo_test.rb diff --git a/Gemfile b/Gemfile index 299d66564..042cef674 100644 --- a/Gemfile +++ b/Gemfile @@ -42,6 +42,7 @@ gem 'bigquery', :git => "https://github.com/abronte/BigQuery.git", :ref => "b92b gem 'memcache_mock' gem 'memoist' gem 'daemons' +gem 'oauth2' # needed for looser jpeg header compat gem 'ruby-imagespec', :require => "image_spec", :git => "https://github.com/r888888888/ruby-imagespec.git", :branch => "exif-fixes" diff --git a/Gemfile.lock b/Gemfile.lock index 592076025..083e1efff 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -198,6 +198,7 @@ GEM mocha (1.2.1) metaclass (~> 0.0.1) multi_json (1.12.1) + multi_xml (0.6.0) multipart-post (2.0.0) naught (1.0.0) net-http-digest_auth (1.4.1) @@ -216,6 +217,12 @@ GEM nokogumbo (1.2.0) nokogiri ntlm-http (0.1.1) + oauth2 (1.3.0) + faraday (>= 0.8, < 0.11) + jwt (~> 1.0) + multi_json (~> 1.3) + multi_xml (~> 0.5) + rack (>= 1.2, < 3) os (0.9.6) pg (0.19.0) pg (0.19.0-x64-mingw32) @@ -401,6 +408,7 @@ DEPENDENCIES mocha net-sftp newrelic_rpm + oauth2 pg protected_attributes pry-byebug diff --git a/app/logical/downloads/rewrite_strategies/base.rb b/app/logical/downloads/rewrite_strategies/base.rb index 9fd98da3c..732fdb09d 100644 --- a/app/logical/downloads/rewrite_strategies/base.rb +++ b/app/logical/downloads/rewrite_strategies/base.rb @@ -10,7 +10,7 @@ module Downloads end def self.strategies - [Downloads::RewriteStrategies::Pixiv, Downloads::RewriteStrategies::NicoSeiga, Downloads::RewriteStrategies::ArtStation, Downloads::RewriteStrategies::Twitpic, Downloads::RewriteStrategies::DeviantArt, Downloads::RewriteStrategies::Tumblr, Downloads::RewriteStrategies::Moebooru, Downloads::RewriteStrategies::Twitter, Downloads::RewriteStrategies::Nijie] + [Downloads::RewriteStrategies::Pixiv, Downloads::RewriteStrategies::NicoSeiga, Downloads::RewriteStrategies::ArtStation, Downloads::RewriteStrategies::Twitpic, Downloads::RewriteStrategies::DeviantArt, Downloads::RewriteStrategies::Tumblr, Downloads::RewriteStrategies::Moebooru, Downloads::RewriteStrategies::Twitter, Downloads::RewriteStrategies::Nijie, Downloads::RewriteStrategies::Pawoo] end def rewrite(url, headers, data = {}) diff --git a/app/logical/downloads/rewrite_strategies/pawoo.rb b/app/logical/downloads/rewrite_strategies/pawoo.rb new file mode 100644 index 000000000..a5854f740 --- /dev/null +++ b/app/logical/downloads/rewrite_strategies/pawoo.rb @@ -0,0 +1,21 @@ +module Downloads + module RewriteStrategies + class Pawoo < Base + attr_accessor :url, :source + + def initialize(url) + @url = url + end + + def rewrite(url, headers, data = {}) + if PawooApiClient::Status.is_match?(url) + client = PawooApiClient.new + response = client.get_status(url) + url = client.image_url + end + + return [url, headers, data] + end + end + end +end diff --git a/app/logical/pawoo_api_client.rb b/app/logical/pawoo_api_client.rb new file mode 100644 index 000000000..4b4e7b016 --- /dev/null +++ b/app/logical/pawoo_api_client.rb @@ -0,0 +1,81 @@ +class PawooApiClient + extend Memoist + + class MissingConfigurationError < Exception ; end + + class Account + attr_reader :json + + def self.is_match?(url) + url =~ %r!https://pawoo.net/web/accounts/(\d+)! + $1 + end + + def initialize(json) + @json = get + end + + def profile_url + json["url"] + end + end + + class Status + attr_reader :json + + def self.is_match?(url) + url =~ %r!https?://pawoo.net/web/statuses/(\d+)! || url =~ %r!https?://pawoo.net/@.+?/(\d+)! + $1 + end + + def initialize(json) + @json = json + end + + def account_profile_url + json["account"]["url"] + end + + def account_name + json["account"]["username"] + end + + def image_url + image_urls.first + end + + def image_urls + json["media_attachments"].map {|x| x["url"]} + end + end + + def get_status(url) + if id = Status.is_match?(url) + Status.new(JSON.parse(access_token.get("/api/v1/statuses/#{id}").body)) + else + nil + end + end + + private + + def fetch_access_token + raise MissingConfigurationError.new("missing pawoo client id") if Danbooru.config.pawoo_client_id.nil? + raise MissingConfigurationError.new("missing pawoo client secret") if Danbooru.config.pawoo_client_secret.nil? + + Rails.cache.fetch("pawoo-token") do + result = client.client_credentials.get_token + result.token + end + end + + def access_token + OAuth2::AccessToken.new(client, fetch_access_token) + end + + def client + OAuth2::Client.new(Danbooru.config.pawoo_client_id, Danbooru.config.pawoo_client_secret, :site => "https://pawoo.net") + end + + memoize :client +end diff --git a/app/logical/sources/site.rb b/app/logical/sources/site.rb index babe63bea..e03ce08e9 100644 --- a/app/logical/sources/site.rb +++ b/app/logical/sources/site.rb @@ -10,7 +10,7 @@ module Sources :artist_commentary_desc, :rewrite_thumbnails, :illust_id_from_url, :to => :strategy def self.strategies - [Strategies::PixivWhitecube, Strategies::Pixiv, Strategies::NicoSeiga, Strategies::DeviantArt, Strategies::ArtStation, Strategies::Nijie, Strategies::Twitter, Strategies::Tumblr] + [Strategies::PixivWhitecube, Strategies::Pixiv, Strategies::NicoSeiga, Strategies::DeviantArt, Strategies::ArtStation, Strategies::Nijie, Strategies::Twitter, Strategies::Tumblr, Strategies::Pawoo] end def initialize(url, options = {}) diff --git a/app/logical/sources/strategies/pawoo.rb b/app/logical/sources/strategies/pawoo.rb new file mode 100644 index 000000000..93b4de407 --- /dev/null +++ b/app/logical/sources/strategies/pawoo.rb @@ -0,0 +1,25 @@ +module Sources::Strategies + class Pawoo < Base + attr_reader :image_urls + + def self.url_match?(url) + PawooApiClient::Status.is_match?(url) + end + + def site_name + "Pawoo" + end + + def get + response = PawooApiClient.new.get_status(url) + @artist_name = response.account_name + @profile_url = response.account_profile_url + @image_url = response.image_urls.first + @image_urls = response.image_urls + end + + def normalizable_for_artist_finder? + true + end + end +end diff --git a/db/migrate/20170414233617_add_updated_at_index_to_pools.rb b/db/migrate/20170414233617_add_updated_at_index_to_pools.rb new file mode 100644 index 000000000..8c8542854 --- /dev/null +++ b/db/migrate/20170414233617_add_updated_at_index_to_pools.rb @@ -0,0 +1,5 @@ +class AddUpdatedAtIndexToPools < ActiveRecord::Migration + def change + add_index :pools, :updated_at + end +end diff --git a/db/structure.sql b/db/structure.sql index da5f8c5c0..776ee531f 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -6682,6 +6682,13 @@ CREATE INDEX index_pools_on_name ON pools USING btree (name); CREATE INDEX index_pools_on_name_trgm ON pools USING gin (lower((name)::text) gin_trgm_ops); +-- +-- Name: index_pools_on_updated_at; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_pools_on_updated_at ON pools USING btree (updated_at); + + -- -- Name: index_post_appeals_on_created_at; Type: INDEX; Schema: public; Owner: - -- @@ -7465,5 +7472,7 @@ INSERT INTO schema_migrations (version) VALUES ('20170414005856'); INSERT INTO schema_migrations (version) VALUES ('20170414233426'); +INSERT INTO schema_migrations (version) VALUES ('20170414233617'); + INSERT INTO schema_migrations (version) VALUES ('20170416224142'); diff --git a/test/unit/sources/pawoo_test.rb b/test/unit/sources/pawoo_test.rb new file mode 100644 index 000000000..4a22bc64c --- /dev/null +++ b/test/unit/sources/pawoo_test.rb @@ -0,0 +1,24 @@ +require 'test_helper' + +module Sources + class PawooTest < ActiveSupport::TestCase + context "The source site for pawoo" do + setup do + @site = Sources::Site.new("https://pawoo.net/web/statuses/1202176") + @site.get + end + + should "get the profile" do + assert_equal("https://pawoo.net/@9ed00e924818", @site.profile_url) + end + + should "get the artist name" do + assert_equal("9ed00e924818", @site.artist_name) + end + + should "get the image url" do + assert_equal("https://img.pawoo.net/media_attachments/files/000/128/953/original/4c0a06087b03343f.png?1492461815", @site.image_url) + end + end + end +end