diff --git a/app/controllers/sources_controller.rb b/app/controllers/sources_controller.rb index 4dea558c4..32abcff66 100644 --- a/app/controllers/sources_controller.rb +++ b/app/controllers/sources_controller.rb @@ -4,7 +4,7 @@ class SourcesController < ApplicationController respond_to :js, :json, :xml def show - @source = Sources::Strategies.find(params[:url], params[:ref]) + @source = Source::Extractor.find(params[:url], params[:ref]) respond_with(@source.to_h) do |format| format.xml { render xml: @source.to_h.to_xml(root: "source") } diff --git a/app/logical/iqdb_client.rb b/app/logical/iqdb_client.rb index 51f5997ba..a40a77733 100644 --- a/app/logical/iqdb_client.rb +++ b/app/logical/iqdb_client.rb @@ -30,15 +30,15 @@ class IqdbClient if file.present? file = file.tempfile elsif url.present? - strategy = Sources::Strategies.find(url) - raise Error, "Can't do reverse image search: #{url} has multiple images. Enter the URL of a single image." if strategy.image_urls.size > 1 + extractor = Source::Extractor.find(url) + raise Error, "Can't do reverse image search: #{url} has multiple images. Enter the URL of a single image." if extractor.image_urls.size > 1 - download_url = strategy.image_urls.first - file = Sources::Strategies.find(download_url).download_file!(download_url) + download_url = extractor.image_urls.first + file = Source::Extractor.find(download_url).download_file!(download_url) elsif image_url.present? - file = Sources::Strategies.find(image_url).download_file!(image_url) + file = Source::Extractor.find(image_url).download_file!(image_url) elsif file_url.present? - file = Sources::Strategies.find(file_url).download_file!(file_url) + file = Source::Extractor.find(file_url).download_file!(file_url) elsif post_id.present? file = Post.find(post_id).file(:preview) elsif media_asset_id.present? diff --git a/app/logical/post_replacement_processor.rb b/app/logical/post_replacement_processor.rb index 47da83460..524d08ec1 100644 --- a/app/logical/post_replacement_processor.rb +++ b/app/logical/post_replacement_processor.rb @@ -66,14 +66,14 @@ class PostReplacementProcessor return MediaFile.open(file) if file.present? raise "No file or source URL provided" if source_url.blank? - strategy = Sources::Strategies.find(source_url, referer_url) - raise NotImplementedError, "No login credentials configured for #{strategy.site_name}." unless strategy.class.enabled? + extractor = Source::Extractor.find(source_url, referer_url) + raise NotImplementedError, "No login credentials configured for #{extractor.site_name}." unless extractor.class.enabled? - image_urls = strategy.image_urls + image_urls = extractor.image_urls raise "#{source_url} contains multiple images" if image_urls.size > 1 image_url = image_urls.first - file = strategy.download_file!(image_url) + file = extractor.download_file!(image_url) [file, image_url] end diff --git a/app/logical/source/extractor.rb b/app/logical/source/extractor.rb new file mode 100644 index 000000000..0ae192984 --- /dev/null +++ b/app/logical/source/extractor.rb @@ -0,0 +1,317 @@ +# frozen_string_literal: true + +# A source extractor is used to extract information from a given source URL. It +# extracts all the images and videos from the URL, as well as metadata such as +# the tags, commentary, artist name, profile URL, and additional names and URLs +# for new artist entries. +# +# To add a new site, create a subclass of Source::Extractor and implement the following methods: +# +# * match? - True if the extractor should be used for this URL. +# * image_urls - The list of images or videos at this URL. Used during uploads. +# * page_url - The page containing the images. Used for post sources. +# * profile_url - The URL of the artist's profile page. Used for artist finding. +# * profile_urls - Extra profile URLs to add to the artist entry. +# * tag_name - The artist's login name. Used as the default name for new artist tags. +# * artist_name - The artist's display name. Used as an other name in new artist entries. +# * other_names - Extra names used in new artist entries. +# * tags - The artist's tags for the work. Used by translated tags. +# * artist_commentary_title - The artist's title of the work. Used for artist commentaries. +# * artist_commentary_desc - The artist's description of the work. Used for artist commentaries. +# +module Source + class Extractor + extend Memoist + + # The http timeout to download a file. + DOWNLOAD_TIMEOUT = 60 + + attr_reader :url, :referer_url, :parsed_url, :parsed_referer + delegate :site_name, to: :parsed_url + + SUBCLASSES = [ + Source::Extractor::Pixiv, + Source::Extractor::Twitter, + Source::Extractor::Tumblr, + Source::Extractor::NicoSeiga, + Source::Extractor::DeviantArt, + Source::Extractor::Moebooru, + Source::Extractor::Nijie, + Source::Extractor::ArtStation, + Source::Extractor::HentaiFoundry, + Source::Extractor::Fanbox, + Source::Extractor::Mastodon, + Source::Extractor::PixivSketch, + Source::Extractor::Weibo, + Source::Extractor::Newgrounds, + Source::Extractor::Skeb, + Source::Extractor::Lofter, + Source::Extractor::Foundation, + Source::Extractor::Plurk, + Source::Extractor::Tinami, + Source::Extractor::Fantia, + ] + + # Should return true if the extractor is configured correctly. Return false + # if the extractor requires api keys that have not been configured. + def self.enabled? + true + end + + # Return the extractor for the given `url`. The `url` may be either a + # direct image URL, or the URL of a page containing one or more images. + # + # The `referer_url` is optionally provided when uploading direct image URLs + # with the bookmarklet. This will be the page containing the image. This + # lets us extract information from sites like Twitter, where the image URL by + # itself doesn't have enough information to find the page containing the image. + # + # @param url [String] The URL to extract information from. + # @param referer_url [String, nil] The page URL if `url` is an image URL. + # @return [Source::Extractor] + def self.find(url, referer_url = nil, default: Extractor::Null) + extractor = SUBCLASSES.lazy.map { |extractor| extractor.new(url, referer_url) }.find(&:match?) + extractor || default&.new(url, referer_url) + end + + # Initialize an extractor. Normally one should call `Source::Extractor.find` + # instead of instantiating an extractor directly. + # + # @param url [String] The URL to extract information form. + # @param referer_url [String, nil] The page URL if `url` is an image URL. + def initialize(url, referer_url = nil) + @url = url.to_s + @referer_url = referer_url&.to_s + + @parsed_url = Source::URL.parse(url) + @parsed_referer = Source::URL.parse(referer_url) if referer_url.present? + @parsed_referer = nil if parsed_url&.site_name != parsed_referer&.site_name + end + + # Should return true if this extractor should be used for this URL. + # Normally, this should check if the URL is from the right site. + # + # @return [Boolean] + def match? + false + end + + # The list of image (or video) URLs extracted from the target URL. + # + # If the target URL is a page, this should be every image on the page. If + # the target URL is a single image, this should be the image itself. + # + # @return [Array] + def image_urls + [] + end + + # The URL of the page containing the image, or nil if it can't be found. + # + # The source of the post will be set to the page URL if it's not possible + # to convert the image URL to a page URL for this site. + # + # For example, for sites like Twitter and Tumblr, it's not possible to + # convert image URLs to page URLs, so the page URL will be used as the + # source for these sites. For sites like Pixiv and DeviantArt, it is + # possible to convert image URLs to page URLs, so the image URL will be + # used as the source for these sites. This is determined by whether + # `Source::URL#page_url` returns a URL or nil. + # + # @return [String, nil] + def page_url + nil + end + + # A name to suggest as the artist's tag name when creating a new artist. + # This should usually be the artist's login name. It should be plain ASCII, + # hopefully unique, and it should follow the rules for tag names (see + # TagNameValidator). + # + # @return [String, nil] + def tag_name + artist_name + end + + # The artists's primary name. If an artist has both a display name and a + # login name, this should be the display name. This will be used as an + # other name for new artist entries. + # + # @return [String, nil] + def artist_name + nil + end + + # A list of all names associated with the artist. These names will be suggested + # as other names when creating a new artist. + # + # @return [Array] + def other_names + [artist_name, tag_name].compact.uniq + end + + # A link to the artist's profile page on the site. This will be used for + # artist finding purposes, so it needs to match the URL in the artist entry. + # + # @return [String, nil] + def profile_url + nil + end + + # A list of all profile urls associated with the artist. These urls will + # be suggested when creating a new artist. + # + # @return [Array] + def profile_urls + [profile_url].compact + end + + # The artist's title of the work. Used for the artist commentary. + # + # @return [String, nil] + def artist_commentary_title + nil + end + + # The artist's description of the work. Used for the artist commentary. + # + # @return [String, nil] + def artist_commentary_desc + nil + end + + # Download the file at the given url. Raises Danbooru::Http::DownloadError if the download fails, or + # Danbooru::Http::FileTooLargeError if the file is too large. + # + # @return [MediaFile] the downloaded file + def download_file!(download_url) + response, file = http_downloader.download_media(download_url) + file + end + + # A http client for API requests. + def http + Danbooru::Http.new.proxy.public_only + end + + # A http client for downloading files. + def http_downloader + http.timeout(DOWNLOAD_TIMEOUT).max_size(Danbooru.config.max_file_size).use(:spoof_referrer).use(:unpolish_cloudflare) + end + + def artists + ArtistFinder.find_artists(profile_url) + end + + # A new artist entry with suggested defaults for when the artist doesn't + # exist. Used in Artist.new_with_defaults to prefill the new artist form. + def new_artist + Artist.new( + name: tag_name, + other_names: other_names, + url_string: profile_urls.join("\n") + ) + end + + def tags + (@tags || []).uniq + end + + def normalized_tags + tags.map { |tag, _url| normalize_tag(tag) }.sort.uniq + end + + def normalize_tag(tag) + WikiPage.normalize_other_name(tag).downcase + end + + def translated_tags + translated_tags = normalized_tags.flat_map(&method(:translate_tag)).uniq.sort + translated_tags.reject(&:artist?) + end + + # Given a tag from the source site, should return an array of corresponding Danbooru tags. + def translate_tag(untranslated_tag) + return [] if untranslated_tag.blank? + + translated_tag_names = WikiPage.active.other_names_include(untranslated_tag).uniq.pluck(:title) + translated_tag_names = TagAlias.to_aliased(translated_tag_names) + translated_tags = Tag.where(name: translated_tag_names) + + if translated_tags.empty? + normalized_name = TagAlias.to_aliased([Tag.normalize_name(untranslated_tag)]) + translated_tags = Tag.nonempty.where(name: normalized_name) + end + + translated_tags + end + + def dtext_artist_commentary_title + self.class.to_dtext(artist_commentary_title) + end + + def dtext_artist_commentary_desc + self.class.to_dtext(artist_commentary_desc) + end + + # A search query that should return any posts that were previously + # uploaded from the same source. These may be duplicates, or they may be + # other posts from the same gallery. + def related_posts_search_query + "source:#{url}" + end + + def related_posts(limit = 5) + Post.system_tag_match(related_posts_search_query).paginate(1, limit: limit) + end + + # A hash containing the results of any API calls made by the extractor. For debugging purposes only. + def api_response + nil + end + + def to_h + { + :artist => { + :name => artist_name, + :tag_name => tag_name, + :other_names => other_names, + :profile_url => profile_url, + :profile_urls => profile_urls + }, + :artists => artists.as_json(include: :sorted_urls), + :image_urls => image_urls, + :page_url => page_url, + :tags => tags, + :normalized_tags => normalized_tags, + :translated_tags => translated_tags, + :artist_commentary => { + :title => artist_commentary_title, + :description => artist_commentary_desc, + :dtext_title => dtext_artist_commentary_title, + :dtext_description => dtext_artist_commentary_desc + }, + :api_response => api_response.to_h + } + end + + def to_json(*_args) + to_h.to_json + end + + def http_exists?(url) + http_downloader.head(url).status.success? + end + + # Convert commentary to dtext by stripping html tags. Sites can override + # this to customize how their markup is translated to dtext. + def self.to_dtext(text) + text = text.to_s + text = Rails::Html::FullSanitizer.new.sanitize(text, encode_special_chars: false) + text = CGI.unescapeHTML(text) + text + end + + memoize :http, :http_downloader, :related_posts + end +end diff --git a/app/logical/sources/strategies/art_station.rb b/app/logical/source/extractor/art_station.rb similarity index 97% rename from app/logical/sources/strategies/art_station.rb rename to app/logical/source/extractor/art_station.rb index 570966d79..d77611252 100644 --- a/app/logical/sources/strategies/art_station.rb +++ b/app/logical/source/extractor/art_station.rb @@ -1,8 +1,8 @@ # frozen_string_literal: true # @see Source::URL::ArtStation -module Sources::Strategies - class ArtStation < Base +class Source::Extractor + class ArtStation < Source::Extractor def match? Source::URL::ArtStation === parsed_url end diff --git a/app/logical/sources/strategies/deviant_art.rb b/app/logical/source/extractor/deviant_art.rb similarity index 98% rename from app/logical/sources/strategies/deviant_art.rb rename to app/logical/source/extractor/deviant_art.rb index ab0190695..b7987ea70 100644 --- a/app/logical/sources/strategies/deviant_art.rb +++ b/app/logical/source/extractor/deviant_art.rb @@ -1,8 +1,8 @@ # frozen_string_literal: true -module Sources - module Strategies - class DeviantArt < Base +module Source + class Extractor + class DeviantArt < Source::Extractor def self.enabled? Danbooru.config.deviantart_client_id.present? && Danbooru.config.deviantart_client_secret.present? end diff --git a/app/logical/sources/strategies/fanbox.rb b/app/logical/source/extractor/fanbox.rb similarity index 98% rename from app/logical/sources/strategies/fanbox.rb rename to app/logical/source/extractor/fanbox.rb index c257370b0..362647798 100644 --- a/app/logical/sources/strategies/fanbox.rb +++ b/app/logical/source/extractor/fanbox.rb @@ -1,9 +1,9 @@ # frozen_string_literal: true # @see Source::URL::Fanbox -module Sources - module Strategies - class Fanbox < Base +module Source + class Extractor + class Fanbox < Source::Extractor def match? Source::URL::Fanbox === parsed_url end diff --git a/app/logical/sources/strategies/fantia.rb b/app/logical/source/extractor/fantia.rb similarity index 98% rename from app/logical/sources/strategies/fantia.rb rename to app/logical/source/extractor/fantia.rb index fabe37640..a848c02a4 100644 --- a/app/logical/sources/strategies/fantia.rb +++ b/app/logical/source/extractor/fantia.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true -module Sources::Strategies - class Fantia < Base +class Source::Extractor + class Fantia < Source::Extractor def self.enabled? Danbooru.config.fantia_session_id.present? end diff --git a/app/logical/sources/strategies/foundation.rb b/app/logical/source/extractor/foundation.rb similarity index 96% rename from app/logical/sources/strategies/foundation.rb rename to app/logical/source/extractor/foundation.rb index 599d26258..ea1f8847f 100644 --- a/app/logical/sources/strategies/foundation.rb +++ b/app/logical/source/extractor/foundation.rb @@ -1,9 +1,9 @@ # frozen_string_literal: true # @see Source::URL::Foundation -module Sources - module Strategies - class Foundation < Base +module Source + class Extractor + class Foundation < Source::Extractor def match? Source::URL::Foundation === parsed_url end diff --git a/app/logical/sources/strategies/hentai_foundry.rb b/app/logical/source/extractor/hentai_foundry.rb similarity index 95% rename from app/logical/sources/strategies/hentai_foundry.rb rename to app/logical/source/extractor/hentai_foundry.rb index 07fd68d6b..d3499ae14 100644 --- a/app/logical/sources/strategies/hentai_foundry.rb +++ b/app/logical/source/extractor/hentai_foundry.rb @@ -1,9 +1,9 @@ # frozen_string_literal: true # @see Source::URL::HentaiFoundry -module Sources - module Strategies - class HentaiFoundry < Base +module Source + class Extractor + class HentaiFoundry < Source::Extractor def match? Source::URL::HentaiFoundry === parsed_url end diff --git a/app/logical/sources/strategies/lofter.rb b/app/logical/source/extractor/lofter.rb similarity index 95% rename from app/logical/sources/strategies/lofter.rb rename to app/logical/source/extractor/lofter.rb index 3a31d55de..aa4feabaf 100644 --- a/app/logical/sources/strategies/lofter.rb +++ b/app/logical/source/extractor/lofter.rb @@ -1,9 +1,9 @@ # frozen_string_literal: true # @see Source::URL::Lofter -module Sources - module Strategies - class Lofter < Base +module Source + class Extractor + class Lofter < Source::Extractor def match? Source::URL::Lofter === parsed_url end diff --git a/app/logical/sources/strategies/mastodon.rb b/app/logical/source/extractor/mastodon.rb similarity index 97% rename from app/logical/sources/strategies/mastodon.rb rename to app/logical/source/extractor/mastodon.rb index 92a00ee8b..ef8fd0695 100644 --- a/app/logical/sources/strategies/mastodon.rb +++ b/app/logical/source/extractor/mastodon.rb @@ -1,8 +1,8 @@ # frozen_string_literal: true # @see Source::URL::Mastodon -module Sources::Strategies - class Mastodon < Base +class Source::Extractor + class Mastodon < Source::Extractor def match? Source::URL::Mastodon === parsed_url end diff --git a/app/logical/sources/strategies/moebooru.rb b/app/logical/source/extractor/moebooru.rb similarity index 87% rename from app/logical/sources/strategies/moebooru.rb rename to app/logical/source/extractor/moebooru.rb index 277eeb0d5..eac927752 100644 --- a/app/logical/sources/strategies/moebooru.rb +++ b/app/logical/source/extractor/moebooru.rb @@ -1,10 +1,10 @@ # frozen_string_literal: true # @see Source::URL::Moebooru -module Sources - module Strategies - class Moebooru < Base - delegate :artist_name, :profile_url, :tag_name, :artist_commentary_title, :artist_commentary_desc, :dtext_artist_commentary_title, :dtext_artist_commentary_desc, to: :sub_strategy, allow_nil: true +module Source + class Extractor + class Moebooru < Source::Extractor + delegate :artist_name, :profile_url, :tag_name, :artist_commentary_title, :artist_commentary_desc, :dtext_artist_commentary_title, :dtext_artist_commentary_desc, to: :sub_extractor, allow_nil: true delegate :site_name, :domain, to: :parsed_url def match? @@ -27,7 +27,7 @@ module Sources end end - # XXX the base strategy excludes artist tags from the translated tags; we don't want that for moebooru. + # XXX the base extractor excludes artist tags from the translated tags; we don't want that for moebooru. def translated_tags tags.map(&:first).flat_map(&method(:translate_tag)).uniq.sort end @@ -50,8 +50,8 @@ module Sources memoize :api_response concerning :HelperMethods do - def sub_strategy - @sub_strategy ||= Sources::Strategies.find(api_response[:source], default: nil) + def sub_extractor + @sub_extractor ||= Source::Extractor.find(api_response[:source], default: nil) end def file_ext diff --git a/app/logical/sources/strategies/newgrounds.rb b/app/logical/source/extractor/newgrounds.rb similarity index 96% rename from app/logical/sources/strategies/newgrounds.rb rename to app/logical/source/extractor/newgrounds.rb index 6b022424b..d16410045 100644 --- a/app/logical/sources/strategies/newgrounds.rb +++ b/app/logical/source/extractor/newgrounds.rb @@ -1,9 +1,9 @@ # frozen_string_literal: true # @see Source::URL::Newgrounds -module Sources - module Strategies - class Newgrounds < Base +module Source + class Extractor + class Newgrounds < Source::Extractor def match? Source::URL::Newgrounds === parsed_url end diff --git a/app/logical/sources/strategies/nico_seiga.rb b/app/logical/source/extractor/nico_seiga.rb similarity index 97% rename from app/logical/sources/strategies/nico_seiga.rb rename to app/logical/source/extractor/nico_seiga.rb index 86c8dbb92..845344802 100644 --- a/app/logical/sources/strategies/nico_seiga.rb +++ b/app/logical/source/extractor/nico_seiga.rb @@ -1,9 +1,9 @@ # frozen_string_literal: true # @see Source::URL::NicoSeiga -module Sources - module Strategies - class NicoSeiga < Base +module Source + class Extractor + class NicoSeiga < Source::Extractor def self.enabled? Danbooru.config.nico_seiga_user_session.present? end diff --git a/app/logical/sources/strategies/nijie.rb b/app/logical/source/extractor/nijie.rb similarity index 98% rename from app/logical/sources/strategies/nijie.rb rename to app/logical/source/extractor/nijie.rb index 67484f0cd..3e41f66c4 100644 --- a/app/logical/sources/strategies/nijie.rb +++ b/app/logical/source/extractor/nijie.rb @@ -1,9 +1,9 @@ # frozen_string_literal: true # @see Source::URL::Nijie -module Sources - module Strategies - class Nijie < Base +module Source + class Extractor + class Nijie < Source::Extractor def self.enabled? Danbooru.config.nijie_login.present? && Danbooru.config.nijie_password.present? end diff --git a/app/logical/sources/strategies/null.rb b/app/logical/source/extractor/null.rb similarity index 75% rename from app/logical/sources/strategies/null.rb rename to app/logical/source/extractor/null.rb index c7df33599..afaaed690 100644 --- a/app/logical/sources/strategies/null.rb +++ b/app/logical/source/extractor/null.rb @@ -1,8 +1,8 @@ # frozen_string_literal: true -module Sources - module Strategies - class Null < Base +module Source + class Extractor + class Null < Source::Extractor def image_urls [url] end diff --git a/app/logical/sources/strategies/pixiv.rb b/app/logical/source/extractor/pixiv.rb similarity index 98% rename from app/logical/sources/strategies/pixiv.rb rename to app/logical/source/extractor/pixiv.rb index d180b1f35..482632c0b 100644 --- a/app/logical/sources/strategies/pixiv.rb +++ b/app/logical/source/extractor/pixiv.rb @@ -1,9 +1,9 @@ # frozen_string_literal: true # @see Source::URL::Pixiv -module Sources - module Strategies - class Pixiv < Base +module Source + class Extractor + class Pixiv < Source::Extractor def self.enabled? Danbooru.config.pixiv_phpsessid.present? end diff --git a/app/logical/sources/strategies/pixiv_sketch.rb b/app/logical/source/extractor/pixiv_sketch.rb similarity index 96% rename from app/logical/sources/strategies/pixiv_sketch.rb rename to app/logical/source/extractor/pixiv_sketch.rb index 5fe19eb51..4b4e941af 100644 --- a/app/logical/sources/strategies/pixiv_sketch.rb +++ b/app/logical/source/extractor/pixiv_sketch.rb @@ -1,9 +1,9 @@ # frozen_string_literal: true # @see Source::URL::PixivSketch -module Sources - module Strategies - class PixivSketch < Base +module Source + class Extractor + class PixivSketch < Source::Extractor def match? Source::URL::PixivSketch === parsed_url end diff --git a/app/logical/sources/strategies/plurk.rb b/app/logical/source/extractor/plurk.rb similarity index 98% rename from app/logical/sources/strategies/plurk.rb rename to app/logical/source/extractor/plurk.rb index 484e62db4..983b060a4 100644 --- a/app/logical/sources/strategies/plurk.rb +++ b/app/logical/source/extractor/plurk.rb @@ -1,9 +1,9 @@ # frozen_string_literal: true # @see Source::URL::Plurk -module Sources - module Strategies - class Plurk < Base +module Source + class Extractor + class Plurk < Source::Extractor def match? Source::URL::Plurk === parsed_url end diff --git a/app/logical/sources/strategies/skeb.rb b/app/logical/source/extractor/skeb.rb similarity index 98% rename from app/logical/sources/strategies/skeb.rb rename to app/logical/source/extractor/skeb.rb index 55bfd3b22..8a82bd1b7 100644 --- a/app/logical/sources/strategies/skeb.rb +++ b/app/logical/source/extractor/skeb.rb @@ -1,9 +1,9 @@ # frozen_string_literal: true # @see Source::URL::Skeb -module Sources - module Strategies - class Skeb < Base +module Source + class Extractor + class Skeb < Extractor def match? Source::URL::Skeb === parsed_url end diff --git a/app/logical/sources/strategies/tinami.rb b/app/logical/source/extractor/tinami.rb similarity index 97% rename from app/logical/sources/strategies/tinami.rb rename to app/logical/source/extractor/tinami.rb index 03598c002..6632e270c 100644 --- a/app/logical/sources/strategies/tinami.rb +++ b/app/logical/source/extractor/tinami.rb @@ -1,9 +1,9 @@ # frozen_string_literal: true # @see Source::URL::Tinami -module Sources - module Strategies - class Tinami < Base +module Source + class Extractor + class Tinami < Source::Extractor def match? Source::URL::Tinami === parsed_url diff --git a/app/logical/sources/strategies/tumblr.rb b/app/logical/source/extractor/tumblr.rb similarity index 98% rename from app/logical/sources/strategies/tumblr.rb rename to app/logical/source/extractor/tumblr.rb index bd1e879ee..7bc3f2780 100644 --- a/app/logical/sources/strategies/tumblr.rb +++ b/app/logical/source/extractor/tumblr.rb @@ -1,8 +1,8 @@ # frozen_string_literal: true # @see Source::URL::Tumblr -module Sources::Strategies - class Tumblr < Base +class Source::Extractor + class Tumblr < Source::Extractor def self.enabled? Danbooru.config.tumblr_consumer_key.present? end diff --git a/app/logical/sources/strategies/twitter.rb b/app/logical/source/extractor/twitter.rb similarity index 98% rename from app/logical/sources/strategies/twitter.rb rename to app/logical/source/extractor/twitter.rb index 053a3d196..62034bff9 100644 --- a/app/logical/sources/strategies/twitter.rb +++ b/app/logical/source/extractor/twitter.rb @@ -1,8 +1,8 @@ # frozen_string_literal: true # @see Source::URL::Twitter -module Sources::Strategies - class Twitter < Base +class Source::Extractor + class Twitter < Source::Extractor # List of hashtag suffixes attached to tag other names # Ex: 西住みほ生誕祭2019 should be checked as 西住みほ # The regexes will not match if there is nothing preceding diff --git a/app/logical/sources/strategies/weibo.rb b/app/logical/source/extractor/weibo.rb similarity index 97% rename from app/logical/sources/strategies/weibo.rb rename to app/logical/source/extractor/weibo.rb index 1abb24d11..0569a6e7f 100644 --- a/app/logical/sources/strategies/weibo.rb +++ b/app/logical/source/extractor/weibo.rb @@ -1,10 +1,9 @@ # frozen_string_literal: true # @see Source::URL::Weibo -module Sources - module Strategies - class Weibo < Base - +module Source + class Extractor + class Weibo < Source::Extractor def match? Source::URL::Weibo === parsed_url end diff --git a/app/logical/source/url.rb b/app/logical/source/url.rb index 930d051a4..039381e03 100644 --- a/app/logical/source/url.rb +++ b/app/logical/source/url.rb @@ -3,8 +3,8 @@ # A Source::URL is a URL from a source site, such as Twitter, Pixiv, etc. Each site has a # subclass responsible for parsing and extracting information from URLs for that site. # -# Sources::Strategies are the main user of Source::URLs. Each Source::URL subclass usually -# has a corresponding strategy for extracting data from that site. +# Source::Extractors are the main user of Source::URLs. Each Source::URL subclass usually +# has a corresponding extractor for extracting data from that site. # # To add a new site, create a subclass of Source::URL and implement `#match?` to define # which URLs belong to the site, and `#parse` to parse and extract information from the URL. diff --git a/app/logical/sources/error.rb b/app/logical/sources/error.rb deleted file mode 100644 index 79ca185ee..000000000 --- a/app/logical/sources/error.rb +++ /dev/null @@ -1,6 +0,0 @@ -# frozen_string_literal: true - -module Sources - class Error < StandardError - end -end diff --git a/app/logical/sources/strategies.rb b/app/logical/sources/strategies.rb deleted file mode 100644 index c59169286..000000000 --- a/app/logical/sources/strategies.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -module Sources - module Strategies - def self.all - [ - Strategies::Pixiv, - Strategies::Twitter, - Strategies::Tumblr, - Strategies::NicoSeiga, - Strategies::DeviantArt, - Strategies::Moebooru, - Strategies::Nijie, - Strategies::ArtStation, - Strategies::HentaiFoundry, - Strategies::Fanbox, - Strategies::Mastodon, - Strategies::PixivSketch, - Strategies::Weibo, - Strategies::Newgrounds, - Strategies::Skeb, - Strategies::Lofter, - Strategies::Foundation, - Strategies::Plurk, - Strategies::Tinami, - Strategies::Fantia, - ] - end - - def self.find(url, referer = nil, default: Strategies::Null) - strategy = all.lazy.map { |s| s.new(url, referer) }.detect(&:match?) - strategy || default&.new(url, referer) - end - end -end diff --git a/app/logical/sources/strategies/base.rb b/app/logical/sources/strategies/base.rb deleted file mode 100644 index 3f87e5a2c..000000000 --- a/app/logical/sources/strategies/base.rb +++ /dev/null @@ -1,259 +0,0 @@ -# frozen_string_literal: true - -# This is a collection of strategies for extracting information about a -# resource. At a minimum it tries to extract the artist name and a canonical -# URL to download the image from. But it can also be used to normalize a URL -# for use with the artist finder. -# -# Design Principles -# -# In general you should minimize state. You can safely assume that url -# and referer_url will not change over the lifetime of an instance, -# so you can safely memoize methods and their results. A common pattern is -# conditionally making an external API call and parsing its response. You should -# make this call on demand and memoize the response. - -module Sources - module Strategies - class Base - extend Memoist - - # The http timeout to download a file. - DOWNLOAD_TIMEOUT = 60 - - attr_reader :url, :referer_url, :parsed_url, :parsed_referer - delegate :site_name, to: :parsed_url - - # Should return true if all prerequisites for using the strategy are met. - # Return false if the strategy requires api keys that have not been configured. - def self.enabled? - true - end - - # Extract information from a target URL. The target URL may be either a - # direct image URL, or the URL of a HTML page containing one or more - # images. - # - # The referer URL is optionally provided when uploading direct image URLs - # with the bookmarklet. This lets us find the page containing the image - # for sites like Twitter, where the image URL by itself doesn't have - # enough information to find the page containing the image. - # - # @param url [String] The target URL - # @param referer_url [String] If the the target URL is an image URL, this - # should be the HTML page containing the image. - def initialize(url, referer_url = nil) - @url = url.to_s - @referer_url = referer_url&.to_s - - @parsed_url = Source::URL.parse(url) - @parsed_referer = Source::URL.parse(referer_url) if referer_url.present? - @parsed_referer = nil if parsed_url&.site_name != parsed_referer&.site_name - end - - # Should return true if this strategy should be used. By default, checks - # if the main url belongs to any of the domains associated with this site. - def match? - false - end - - # Whatever url is, this method should return the direct links - # to the canonical binary files. It should not be an HTML page. It should - # be a list of JPEG, PNG, GIF, WEBM, MP4, ZIP, etc. It is what the - # downloader will fetch and save to disk. - def image_urls - [] - end - - # The URL of the page containing the image, or nil if it can't be found. - # - # The source of the post will be set to the page URL if it's not possible - # to convert the image URL to a page URL for this site. - # - # For example, for sites like Twitter and Tumblr, it's not possible to - # convert image URLs to page URLs, so the page URL will be used as the - # source for these sites. For sites like Pixiv and DeviantArt, it is - # possible to convert image URLs to page URLs, so the image URL will be - # used as the source for these sites. This is determined by whether - # `Source::URL#page_url` returns a URL or nil. - # - # @return [String, nil] - def page_url - nil - end - - # A name to suggest as the artist's tag name when creating a new artist. - # This should usually be the artist's account name. - def tag_name - artist_name - end - - # The artists's primary name. If an artist has both a display name and an - # account name, this should be the display name. - def artist_name - nil - end - - # A list of all names associated with the artist. These names will be suggested - # as other names when creating a new artist. - def other_names - [artist_name, tag_name].compact.uniq - end - - # A link to the artist's profile page on the site. This will be used for - # artist finding purposes, so it needs to match the URL in the artist entry. - def profile_url - nil - end - - # A list of all profile urls associated with the artist. These urls will - # be suggested when creating a new artist. - def profile_urls - [profile_url].compact - end - - def artist_commentary_title - nil - end - - def artist_commentary_desc - nil - end - - # Download the file at the given url. Raises Danbooru::Http::DownloadError if the download fails, or - # Danbooru::Http::FileTooLargeError if the file is too large. - # - # @return [MediaFile] the downloaded file - def download_file!(download_url) - response, file = http_downloader.download_media(download_url) - file - end - - # A http client for API requests. - def http - Danbooru::Http.new.proxy.public_only - end - memoize :http - - # A http client for downloading files. - def http_downloader - http.timeout(DOWNLOAD_TIMEOUT).max_size(Danbooru.config.max_file_size).use(:spoof_referrer).use(:unpolish_cloudflare) - end - memoize :http_downloader - - def artists - ArtistFinder.find_artists(profile_url) - end - - # A new artist entry with suggested defaults for when the artist doesn't - # exist. Used in Artist.new_with_defaults to prefill the new artist form. - def new_artist - Artist.new( - name: tag_name, - other_names: other_names, - url_string: profile_urls.join("\n") - ) - end - - def tags - (@tags || []).uniq - end - - def normalized_tags - tags.map { |tag, _url| normalize_tag(tag) }.sort.uniq - end - - def normalize_tag(tag) - WikiPage.normalize_other_name(tag).downcase - end - - def translated_tags - translated_tags = normalized_tags.flat_map(&method(:translate_tag)).uniq.sort - translated_tags.reject(&:artist?) - end - - # Given a tag from the source site, should return an array of corresponding Danbooru tags. - def translate_tag(untranslated_tag) - return [] if untranslated_tag.blank? - - translated_tag_names = WikiPage.active.other_names_include(untranslated_tag).uniq.pluck(:title) - translated_tag_names = TagAlias.to_aliased(translated_tag_names) - translated_tags = Tag.where(name: translated_tag_names) - - if translated_tags.empty? - normalized_name = TagAlias.to_aliased([Tag.normalize_name(untranslated_tag)]) - translated_tags = Tag.nonempty.where(name: normalized_name) - end - - translated_tags - end - - def dtext_artist_commentary_title - self.class.to_dtext(artist_commentary_title) - end - - def dtext_artist_commentary_desc - self.class.to_dtext(artist_commentary_desc) - end - - # A search query that should return any posts that were previously - # uploaded from the same source. These may be duplicates, or they may be - # other posts from the same gallery. - def related_posts_search_query - "source:#{url}" - end - - def related_posts(limit = 5) - Post.system_tag_match(related_posts_search_query).paginate(1, limit: limit) - end - memoize :related_posts - - # A hash containing the results of any API calls made by the strategy. For debugging purposes only. - def api_response - nil - end - - def to_h - { - :artist => { - :name => artist_name, - :tag_name => tag_name, - :other_names => other_names, - :profile_url => profile_url, - :profile_urls => profile_urls - }, - :artists => artists.as_json(include: :sorted_urls), - :image_urls => image_urls, - :page_url => page_url, - :tags => tags, - :normalized_tags => normalized_tags, - :translated_tags => translated_tags, - :artist_commentary => { - :title => artist_commentary_title, - :description => artist_commentary_desc, - :dtext_title => dtext_artist_commentary_title, - :dtext_description => dtext_artist_commentary_desc - }, - :api_response => api_response.to_h - } - end - - def to_json(*_args) - to_h.to_json - end - - def http_exists?(url) - http_downloader.head(url).status.success? - end - - # Convert commentary to dtext by stripping html tags. Sites can override - # this to customize how their markup is translated to dtext. - def self.to_dtext(text) - text = text.to_s - text = Rails::Html::FullSanitizer.new.sanitize(text, encode_special_chars: false) - text = CGI.unescapeHTML(text) - text - end - end - end -end diff --git a/app/models/artist.rb b/app/models/artist.rb index f32ffd138..33f83192b 100644 --- a/app/models/artist.rb +++ b/app/models/artist.rb @@ -156,7 +156,7 @@ class Artist < ApplicationRecord end if source.present? - artist = Sources::Strategies.find(source).new_artist + artist = Source::Extractor.find(source).new_artist artist.attributes = params else artist = Artist.new(params) @@ -252,7 +252,7 @@ class Artist < ApplicationRecord elsif query.include?("*") where(id: ArtistURL.where_like(:url, query).select(:artist_id)) elsif query =~ %r{\Ahttps?://}i - url = Sources::Strategies.find(query).profile_url || query + url = Source::Extractor.find(query).profile_url || query ArtistFinder.find_artists(url) else where(id: ArtistURL.where_like(:url, "*#{query}*").select(:artist_id)) diff --git a/app/models/artist_url.rb b/app/models/artist_url.rb index a1d25dbcc..8bd185683 100644 --- a/app/models/artist_url.rb +++ b/app/models/artist_url.rb @@ -53,7 +53,7 @@ class ArtistURL < ApplicationRecord elsif url.include?("*") where_ilike(attr, url) else - profile_url = Sources::Strategies.find(url).profile_url || url + profile_url = Source::Extractor.find(url).profile_url || url where(attr => normalize_normalized_url(profile_url)) end end diff --git a/app/models/post.rb b/app/models/post.rb index 41cdf049f..8cf9bb0e5 100644 --- a/app/models/post.rb +++ b/app/models/post.rb @@ -87,7 +87,7 @@ class Post < ApplicationRecord ) if add_artist_tag - tag_string = "#{tag_string} #{upload_media_asset.source_strategy&.artists.to_a.map(&:tag).map(&:name).join(" ")}".strip + tag_string = "#{tag_string} #{upload_media_asset.source_extractor&.artists.to_a.map(&:tag).map(&:name).join(" ")}".strip tag_string += " " if tag_string.present? end @@ -1157,7 +1157,7 @@ class Post < ApplicationRecord self.pixiv_id = nil return unless web_source? - site = Sources::Strategies::Pixiv.new(source) + site = Source::Extractor::Pixiv.new(source) if site.match? self.pixiv_id = site.illust_id end @@ -1265,7 +1265,7 @@ class Post < ApplicationRecord return if !web_source? return if has_tag?("artist_request") || has_tag?("official_art") return if tags.any?(&:artist?) - return if Sources::Strategies.find(source).is_a?(Sources::Strategies::Null) + return if Source::Extractor.find(source).is_a?(Source::Extractor::Null) new_artist_path = Routes.new_artist_path(artist: { source: source }) warnings.add(:base, "Artist tag is required. \"Create new artist tag\":[#{new_artist_path}]. Ask on the forum if you need naming help") diff --git a/app/models/upload.rb b/app/models/upload.rb index 1a5ac9a8e..06580a8fb 100644 --- a/app/models/upload.rb +++ b/app/models/upload.rb @@ -117,8 +117,8 @@ class Upload < ApplicationRecord UploadMediaAsset.new(file: file.tempfile, source_url: "file://#{file.original_filename}") end elsif source.present? - page_url = source_strategy.page_url - image_urls = source_strategy.image_urls + page_url = source_extractor.page_url + image_urls = source_extractor.image_urls if image_urls.empty? raise Error, "#{source} doesn't contain any images" @@ -136,14 +136,14 @@ class Upload < ApplicationRecord update!(status: "error", error: e.message) end - def source_strategy + def source_extractor return nil if source.blank? - Sources::Strategies.find(source, referer_url) + Source::Extractor.find(source, referer_url) end def self.available_includes [:uploader, :upload_media_assets, :media_assets, :posts] end - memoize :source_strategy + memoize :source_extractor end diff --git a/app/models/upload_media_asset.rb b/app/models/upload_media_asset.rb index 841963ead..fb2cc5f2f 100644 --- a/app/models/upload_media_asset.rb +++ b/app/models/upload_media_asset.rb @@ -79,9 +79,9 @@ class UploadMediaAsset < ApplicationRecord end end - def source_strategy + def source_extractor return nil if source_url.blank? - Sources::Strategies.find(source_url, page_url) + Source::Extractor.find(source_url, page_url) end def async_process_upload! @@ -98,7 +98,7 @@ class UploadMediaAsset < ApplicationRecord if file.present? media_file = MediaFile.open(file) else - media_file = source_strategy.download_file!(source_url) + media_file = source_extractor.download_file!(source_url) end MediaAsset.upload!(media_file) do |media_asset| @@ -120,5 +120,5 @@ class UploadMediaAsset < ApplicationRecord end end - memoize :source_strategy + memoize :source_extractor end diff --git a/app/views/upload_media_assets/index.html.erb b/app/views/upload_media_assets/index.html.erb index 690718e50..332284c0c 100644 --- a/app/views/upload_media_assets/index.html.erb +++ b/app/views/upload_media_assets/index.html.erb @@ -2,8 +2,8 @@

Upload

- <% if policy(@upload).show? && @upload.source_strategy.present? %> - <%= render_source_data(@upload.source_strategy) %> + <% if policy(@upload).show? && @upload.source_extractor.present? %> + <%= render_source_data(@upload.source_extractor) %> <% end %>
diff --git a/app/views/uploads/_single_asset_upload.html.erb b/app/views/uploads/_single_asset_upload.html.erb index f68acf8bf..17c63b25a 100644 --- a/app/views/uploads/_single_asset_upload.html.erb +++ b/app/views/uploads/_single_asset_upload.html.erb @@ -34,10 +34,10 @@

- <%= render "uploads/related_posts", source: upload_media_asset.source_strategy %> + <%= render "uploads/related_posts", source: upload_media_asset.source_extractor %> - <% if upload_media_asset.source_strategy.present? %> - <%= render_source_data(upload_media_asset.source_strategy) %> + <% if upload_media_asset.source_extractor.present? %> + <%= render_source_data(upload_media_asset.source_extractor) %> <% end %> <% post = Post.new_from_upload(upload_media_asset, add_artist_tag: true, source: upload_media_asset.canonical_url, **permitted_attributes(Post).to_h.symbolize_keys) %> diff --git a/test/functional/post_replacements_controller_test.rb b/test/functional/post_replacements_controller_test.rb index abfddbca8..05aa2a7ae 100644 --- a/test/functional/post_replacements_controller_test.rb +++ b/test/functional/post_replacements_controller_test.rb @@ -108,7 +108,7 @@ class PostReplacementsControllerTest < ActionDispatch::IntegrationTest context "replacing a post with a Pixiv ugoira" do should "save the frame data" do - skip "Pixiv credentials not configured" unless Sources::Strategies::Pixiv.enabled? + skip "Pixiv credentials not configured" unless Source::Extractor::Pixiv.enabled? @post = create(:post) post_auth post_replacements_path, create(:moderator_user), params: { @@ -134,7 +134,7 @@ class PostReplacementsControllerTest < ActionDispatch::IntegrationTest context "replacing a post with notes" do should "rescale the notes" do - skip "Pixiv credentials not configured" unless Sources::Strategies::Pixiv.enabled? + skip "Pixiv credentials not configured" unless Source::Extractor::Pixiv.enabled? as(create(:user)) do @post = create(:post, image_width: 160, image_height: 164) diff --git a/test/test_helpers/download_test_helper.rb b/test/test_helpers/download_test_helper.rb index 6297bbabb..3d3e9771d 100644 --- a/test/test_helpers/download_test_helper.rb +++ b/test/test_helpers/download_test_helper.rb @@ -1,12 +1,12 @@ module DownloadTestHelper def assert_downloaded(expected_filesize, source, referer = nil) - strategy = Sources::Strategies.find(source, referer) + strategy = Source::Extractor.find(source, referer) file = strategy.download_file!(strategy.image_urls.sole) assert_equal(expected_filesize, file.size, "Tested source URL: #{source}") end def assert_rewritten(expected_source, test_source, test_referer = nil) - strategy = Sources::Strategies.find(test_source, test_referer) + strategy = Source::Extractor.find(test_source, test_referer) rewritten_source = strategy.image_urls.sole assert_match(expected_source, rewritten_source, "Tested source URL: #{test_source}") end diff --git a/test/test_helpers/upload_test_helper.rb b/test/test_helpers/upload_test_helper.rb index 775a38513..9271a4d30 100644 --- a/test/test_helpers/upload_test_helper.rb +++ b/test/test_helpers/upload_test_helper.rb @@ -3,7 +3,7 @@ module UploadTestHelper def create_upload!(source_or_file_path, user:, **params) if source_or_file_path =~ %r{\Ahttps?://}i - skip "Login credentials not configured for #{source_or_file_path}" unless Sources::Strategies.find(source_or_file_path).class.enabled? + skip "Login credentials not configured for #{source_or_file_path}" unless Source::Extractor.find(source_or_file_path).class.enabled? source = { source: source_or_file_path } else file = Rack::Test::UploadedFile.new(Rails.root.join(source_or_file_path)) diff --git a/test/unit/artist_test.rb b/test/unit/artist_test.rb index 3a4a616a1..83bac5588 100644 --- a/test/unit/artist_test.rb +++ b/test/unit/artist_test.rb @@ -355,7 +355,7 @@ class ArtistTest < ActiveSupport::TestCase end should "find the artist" do - skip "Nijie credentials not configured" unless Sources::Strategies::Nijie.enabled? + skip "Nijie credentials not configured" unless Source::Extractor::Nijie.enabled? assert_artist_found("evazion", "http://nijie.info/view.php?id=218944") assert_artist_found("728995", "http://nijie.info/view.php?id=213043") end diff --git a/test/unit/downloads/pixiv_test.rb b/test/unit/downloads/pixiv_test.rb index a23d09906..d04d4b76f 100644 --- a/test/unit/downloads/pixiv_test.rb +++ b/test/unit/downloads/pixiv_test.rb @@ -54,7 +54,7 @@ module Downloads end should "download the full size image instead of the thumbnail" do - skip "Pixiv credentials not configured" unless Sources::Strategies::Pixiv.enabled? + skip "Pixiv credentials not configured" unless Source::Extractor::Pixiv.enabled? assert_rewritten(@p0_full_size_image, @p0_large_thumbnail) assert_rewritten(@p1_full_size_image, @p1_large_thumbnail) @@ -119,7 +119,7 @@ module Downloads context "An ugoira site for pixiv" do should "capture the data" do - @strategy = Sources::Strategies.find("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364") + @strategy = Source::Extractor.find("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364") media_file = @strategy.download_file!(@strategy.image_urls.sole) assert_equal(2, media_file.frame_data.size) diff --git a/test/unit/sources/art_station_test.rb b/test/unit/sources/art_station_test.rb index 07e35da91..2003870ad 100644 --- a/test/unit/sources/art_station_test.rb +++ b/test/unit/sources/art_station_test.rb @@ -4,7 +4,7 @@ module Sources class ArtStationTest < ActiveSupport::TestCase context "The source site for an art station artwork page" do setup do - @site = Sources::Strategies.find("https://www.artstation.com/artwork/04XA4") + @site = Source::Extractor.find("https://www.artstation.com/artwork/04XA4") end should "get the image url" do @@ -35,7 +35,7 @@ module Sources context "The source site for an art station projects page" do setup do - @site = Sources::Strategies.find("https://dantewontdie.artstation.com/projects/YZK5q") + @site = Source::Extractor.find("https://dantewontdie.artstation.com/projects/YZK5q") end should "get the image url" do @@ -68,7 +68,7 @@ module Sources context "The source site for a www.artstation.com/artwork/$slug page" do setup do - @site = Sources::Strategies.find("https://www.artstation.com/artwork/cody-from-sf") + @site = Source::Extractor.find("https://www.artstation.com/artwork/cody-from-sf") end should "get the image url" do @@ -90,7 +90,7 @@ module Sources context "with a referer" do should "work" do - site = Sources::Strategies.find(@url, @ref) + site = Source::Extractor.find(@url, @ref) assert_equal(["https://cdn.artstation.com/p/assets/images/images/006/029/978/4k/amama-l-z.jpg"], site.image_urls) assert_equal("https://amama.artstation.com/projects/4BWW2", site.page_url) @@ -102,7 +102,7 @@ module Sources context "without a referer" do should "work" do - site = Sources::Strategies.find(@url) + site = Source::Extractor.find(@url) assert_equal(["https://cdn.artstation.com/p/assets/images/images/006/029/978/4k/amama-l-z.jpg"], site.image_urls) assert_nil(site.page_url) @@ -117,7 +117,7 @@ module Sources context "A 4k asset url" do context "without a referer" do should "work" do - site = Sources::Strategies.find("https://cdna.artstation.com/p/assets/images/images/007/253/680/4k/ina-wong-demon-girl-done-ttd-comp.jpg?1504793833") + site = Source::Extractor.find("https://cdna.artstation.com/p/assets/images/images/007/253/680/4k/ina-wong-demon-girl-done-ttd-comp.jpg?1504793833") assert_equal(["https://cdn.artstation.com/p/assets/images/images/007/253/680/4k/ina-wong-demon-girl-done-ttd-comp.jpg?1504793833"], site.image_urls) assert_nothing_raised { site.to_h } @@ -128,7 +128,7 @@ module Sources context "A cover url" do should "work" do url = "https://cdna.artstation.com/p/assets/covers/images/007/262/828/large/monica-kyrie-1.jpg?1504865060" - site = Sources::Strategies.find(url) + site = Source::Extractor.find(url) assert_equal(["https://cdn.artstation.com/p/assets/covers/images/007/262/828/original/monica-kyrie-1.jpg?1504865060"], site.image_urls) end @@ -136,7 +136,7 @@ module Sources context "The source site for an ArtStation gallery" do setup do - @site = Sources::Strategies.find("https://www.artstation.com/artwork/BDxrA") + @site = Source::Extractor.find("https://www.artstation.com/artwork/BDxrA") end should "get only image urls, not video urls" do @@ -147,7 +147,7 @@ module Sources context "A work that includes video clips" do should_eventually "include the video clips in the image urls" do - @source = Sources::Strategies.find("https://www.artstation.com/artwork/0nP1e8") + @source = Source::Extractor.find("https://www.artstation.com/artwork/0nP1e8") assert_equal(%w[ https://cdn.artstation.com/p/assets/images/images/040/979/418/original/yusuf-umar-workout-10mb.gif?1630425406 @@ -163,7 +163,7 @@ module Sources end should "work for the video itself" do - @source = Sources::Strategies.find("https://cdn-animation.artstation.com/p/video_sources/000/466/622/workout.mp4") + @source = Source::Extractor.find("https://cdn-animation.artstation.com/p/video_sources/000/466/622/workout.mp4") assert_equal(["https://cdn-animation.artstation.com/p/video_sources/000/466/622/workout.mp4"], @source.image_urls) end @@ -172,7 +172,7 @@ module Sources context "A work that has been deleted" do should "work" do url = "https://fiship.artstation.com/projects/x8n8XT" - site = Sources::Strategies.find(url) + site = Source::Extractor.find(url) assert_equal("fiship", site.artist_name) assert_equal("https://www.artstation.com/fiship", site.profile_url) @@ -183,12 +183,12 @@ module Sources end should "work for artists with underscores in their name" do - site = Sources::Strategies.find("https://hosi_na.artstation.com/projects/3oEk3B") + site = Source::Extractor.find("https://hosi_na.artstation.com/projects/3oEk3B") assert_equal("hosi_na", site.artist_name) end should "work for artists with dashes in their name" do - site = Sources::Strategies.find("https://sa-dui.artstation.com/projects/DVERn") + site = Source::Extractor.find("https://sa-dui.artstation.com/projects/DVERn") assert_equal("sa-dui", site.artist_name) end diff --git a/test/unit/sources/deviant_art_test.rb b/test/unit/sources/deviant_art_test.rb index f82ed4e6a..84ee33de5 100644 --- a/test/unit/sources/deviant_art_test.rb +++ b/test/unit/sources/deviant_art_test.rb @@ -9,7 +9,7 @@ module Sources context "A page url" do setup do - @site = Sources::Strategies.find("https://www.deviantart.com/aeror404/art/Holiday-Elincia-424551484") + @site = Source::Extractor.find("https://www.deviantart.com/aeror404/art/Holiday-Elincia-424551484") end should "work" do @@ -26,7 +26,7 @@ module Sources context "The source for a deleted DeviantArt image URL" do should "work" do - @site = Sources::Strategies.find("https://pre00.deviantart.net/423b/th/pre/i/2017/281/e/0/mindflayer_girl01_by_nickbeja-dbpxdt8.png") + @site = Source::Extractor.find("https://pre00.deviantart.net/423b/th/pre/i/2017/281/e/0/mindflayer_girl01_by_nickbeja-dbpxdt8.png") @artist = create(:artist, name: "nickbeja", url_string: "https://nickbeja.deviantart.com") assert_equal(["https://pre00.deviantart.net/423b/th/pre/i/2017/281/e/0/mindflayer_girl01_by_nickbeja-dbpxdt8.png"], @site.image_urls) @@ -40,7 +40,7 @@ module Sources context "The source for a download-disabled DeviantArt artwork page" do should "get the image url" do - @site = Sources::Strategies.find("https://noizave.deviantart.com/art/test-no-download-697415967") + @site = Source::Extractor.find("https://noizave.deviantart.com/art/test-no-download-697415967") # https://img00.deviantart.net/56ee/i/2017/219/2/3/test__no_download_by_noizave-dbj81lr.jpg (md5: 25a03b5a6744b6b914a13b3cd50e3c2c, size: 37638) # orig file: https://danbooru.donmai.us/posts/463438 (md5: eb97244675e47dbd77ffcd2d7e15aeab, size: 59401) @@ -56,7 +56,7 @@ module Sources context "The source for a download-enabled DeviantArt artwork page" do should "get the download image url" do - @site = Sources::Strategies.find("https://www.deviantart.com/len1/art/All-that-Glitters-II-774592781") + @site = Source::Extractor.find("https://www.deviantart.com/len1/art/All-that-Glitters-II-774592781") # http://origin-orig.deviantart.net/a713/f/2018/333/3/6/all_that_glitters_ii_by_len1-dct67m5.jpg (md5: d16bb8620600334caa029ebb9bc426a6, size: 1402017) assert_match(%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/a6289ca5-2205-4118-af55-c6934fba0930/dct67m5-51e8db38-9167-4f5c-931d-561ea4d3810d\.jpg}, @site.image_urls.sole) @@ -71,7 +71,7 @@ module Sources context "The source for a DeviantArt image url" do should "fetch the source data" do - @site = Sources::Strategies.find("https://pre00.deviantart.net/b5e6/th/pre/f/2016/265/3/5/legend_of_galactic_heroes_by_hideyoshi-daihpha.jpg") + @site = Source::Extractor.find("https://pre00.deviantart.net/b5e6/th/pre/f/2016/265/3/5/legend_of_galactic_heroes_by_hideyoshi-daihpha.jpg") # http://origin-orig.deviantart.net/9e1f/f/2016/265/3/5/legend_of_galactic_heroes_by_hideyoshi-daihpha.jpg (md5: 4cfec3d50ebbb924077cc5c90e705d4e, size: 906621) assert_match(%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/b1f96af6-56a3-47a8-b7f4-406f243af3a3/daihpha-9f1fcd2e-7557-4db5-951b-9aedca9a3ae7\.jpg}, @site.image_urls.sole) @@ -86,7 +86,7 @@ module Sources context "The source for a origin-orig.deviantart.net image url without a referer" do should "work" do - @site = Sources::Strategies.find("http://origin-orig.deviantart.net/7b5b/f/2017/160/c/5/test_post_please_ignore_by_noizave-dbc3a48.png") + @site = Source::Extractor.find("http://origin-orig.deviantart.net/7b5b/f/2017/160/c/5/test_post_please_ignore_by_noizave-dbc3a48.png") # md5: 9dec050536dbdb09ab63cb9c5a48f8b7 assert_match(%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/83d3eb4d-13e5-4aea-a08f-8d4331d033c4/dbc3a48-10b9e2e8-b176-4820-ab9e-23449c11e7c9\.png}, @site.image_urls.sole) @@ -103,7 +103,7 @@ module Sources context "The source for a img00.deviantart.net sample image url" do should "return the full size image url" do - @site = Sources::Strategies.find("https://img00.deviantart.net/a233/i/2017/160/5/1/test_post_please_ignore_by_noizave-dbc3a48.png") + @site = Source::Extractor.find("https://img00.deviantart.net/a233/i/2017/160/5/1/test_post_please_ignore_by_noizave-dbc3a48.png") assert_match(%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/83d3eb4d-13e5-4aea-a08f-8d4331d033c4/dbc3a48-10b9e2e8-b176-4820-ab9e-23449c11e7c9\.png}, @site.image_urls.sole) assert_downloaded(3619, @site.image_urls.sole) @@ -114,7 +114,7 @@ module Sources context "The source for a th00.deviantart.net/*/PRE/* thumbnail url" do should "return the full size image url" do - @site = Sources::Strategies.find("http://th00.deviantart.net/fs71/PRE/f/2014/065/3/b/goruto_by_xyelkiltrox-d797tit.png") + @site = Source::Extractor.find("http://th00.deviantart.net/fs71/PRE/f/2014/065/3/b/goruto_by_xyelkiltrox-d797tit.png") # http://origin-orig.deviantart.net/0f1e/f/2014/065/3/b/goruto_by_xyelkiltrox-d797tit.png (md5: d779f5a7da29ec90d777a8db38d07994, size: 3391584) assert_match(%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/d8995973-0b32-4a7d-8cd8-d847d083689a/d797tit-1eac22e0-38b6-4eae-adcb-1b72843fd62a\.png}, @site.image_urls.sole) @@ -126,7 +126,7 @@ module Sources context "A source for a *.deviantart.net/*/:title_by_:artist.jpg url artist name containing underscores" do should "find the correct artist" do - @site = Sources::Strategies.find("https://orig00.deviantart.net/4274/f/2010/230/8/a/pkmn_king_and_queen_by_mikoto_chan.jpg") + @site = Source::Extractor.find("https://orig00.deviantart.net/4274/f/2010/230/8/a/pkmn_king_and_queen_by_mikoto_chan.jpg") @artist = create(:artist, name: "mikoto-chan", url_string: "https://www.deviantart.com/mikoto-chan") assert_equal("mikoto-chan", @site.artist_name) @@ -144,7 +144,7 @@ module Sources context "without a referer" do should "work" do - @site = Sources::Strategies.find(@url) + @site = Source::Extractor.find(@url) assert_equal([@site.url], @site.image_urls) assert_equal("47ness", @site.artist_name) @@ -157,7 +157,7 @@ module Sources context "with a referer" do should "work" do - @site = Sources::Strategies.find(@url, @ref) + @site = Source::Extractor.find(@url, @ref) # http://origin-orig.deviantart.net/a418/f/2007/120/c/9/cool_like_me_by_47ness.jpg (md5: da78e7c192d42470acda7d87ade64849, size: 265496) assert_match(%r{\Ahttps://wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/ece2238f-5c8f-48e4-afda-304cab294acd/dwcohb-8189be91-691d-4212-b3a0-0b77e86a57d1\.jpg}, @site.image_urls.sole) @@ -181,7 +181,7 @@ module Sources context "without a referer" do should "work" do - @site = Sources::Strategies.find(@url) + @site = Source::Extractor.find(@url) assert_equal([@url], @site.image_urls) assert_nil(@site.artist_name) @@ -194,7 +194,7 @@ module Sources context "with a referer" do should "work" do - @site = Sources::Strategies.find(@url, @ref) + @site = Source::Extractor.find(@url, @ref) assert_match(%r!\Ahttps://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/8b472d70-a0d6-41b5-9a66-c35687090acc/d23jbr4-8a06af02-70cb-46da-8a96-42a6ba73cdb4.jpg!, @site.image_urls.sole) assert_equal("edsfox", @site.artist_name) @@ -215,7 +215,7 @@ module Sources context "with a referer" do should "work" do - @site = Sources::Strategies.find(@url, @ref) + @site = Source::Extractor.find(@url, @ref) assert_match(%r!\Ahttps://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/8b472d70-a0d6-41b5-9a66-c35687090acc/d23jbr4-8a06af02-70cb-46da-8a96-42a6ba73cdb4.jpg!, @site.image_urls.sole) assert_equal("edsfox", @site.artist_name) @@ -236,7 +236,7 @@ module Sources context "with a referer" do should "work" do - @site = Sources::Strategies.find(@url, @ref) + @site = Source::Extractor.find(@url, @ref) assert_equal(@ref, @site.page_url) assert_equal([@artist], @site.artists) @@ -247,7 +247,7 @@ module Sources context "The source for a non-downloadable animated gif with id<=790677560" do should "return working image url" do - @site = Sources::Strategies.find("https://www.deviantart.com/heartgear/art/Silent-Night-579982816") + @site = Source::Extractor.find("https://www.deviantart.com/heartgear/art/Silent-Night-579982816") # md5: 62caac1863aa264a56d548b4b7607097 assert_match(%r!\Ahttps://images-wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/f/ea95be00-c5aa-4063-bd55-f5a9183912f7/d9lb1ls-7d625444-0003-4123-bf00-274737ca7fdd.gif\?token=!, @site.image_urls.sole) @@ -258,7 +258,7 @@ module Sources context "The source for a non-downloadable flash file" do should "return working image url" do skip - @site = Sources::Strategies.find("https://www.deviantart.com/heartgear/art/SL-40v3-522007633") + @site = Source::Extractor.find("https://www.deviantart.com/heartgear/art/SL-40v3-522007633") # md5: 6adf1a3d532f898f44cf9948cbc7db7d assert_match(%r!\Ahttps://api-da\.wixmp\.com/_api/download/file\?downloadToken=!, @site.image_urls.sole) @@ -268,7 +268,7 @@ module Sources context "The source for a non-downloadable video file" do should "return working image url" do - @site = Sources::Strategies.find("https://www.deviantart.com/gs-mantis/art/Chen-Goes-Fishing-505847233") + @site = Source::Extractor.find("https://www.deviantart.com/gs-mantis/art/Chen-Goes-Fishing-505847233") # md5: 344ac2b9fd5a87982af4b648aa2b2b0d assert_equal(["https://wixmp-ed30a86b8c4ca887773594c2.wixmp.com/v/mp4/fe046bc7-4d68-4699-96c1-19aa464edff6/d8d6281-91959e92-214f-4b2d-a138-ace09f4b6d09.1080p.8e57939eba634743a9fa41185e398d00.mp4"], @site.image_urls) @@ -278,7 +278,7 @@ module Sources context "The source for an DeviantArt artwork page" do setup do - @site = Sources::Strategies.find("http://noizave.deviantart.com/art/test-post-please-ignore-685436408") + @site = Source::Extractor.find("http://noizave.deviantart.com/art/test-post-please-ignore-685436408") end should "get the image url" do @@ -338,7 +338,7 @@ module Sources context "The source for a login-only DeviantArt artwork page" do setup do - @site = Sources::Strategies.find("http://noizave.deviantart.com/art/hidden-work-685458369") + @site = Source::Extractor.find("http://noizave.deviantart.com/art/hidden-work-685458369") end should "get the image url" do @@ -350,7 +350,7 @@ module Sources context "A source with malformed links in the artist commentary" do should "fix the links" do - @site = Sources::Strategies.find("https://teemutaiga.deviantart.com/art/Kisu-620666655") + @site = Source::Extractor.find("https://teemutaiga.deviantart.com/art/Kisu-620666655") assert_match(%r!"Print available at Inprnt":\[http://www.inprnt.com/gallery/teemutaiga/kisu\]!, @site.dtext_artist_commentary_desc) end @@ -358,7 +358,7 @@ module Sources context "An artist entry with a profile url that is missing the 'www'" do should "still find the artist" do - @site = Sources::Strategies.find("http://noizave.deviantart.com/art/test-post-please-ignore-685436408") + @site = Source::Extractor.find("http://noizave.deviantart.com/art/test-post-please-ignore-685436408") @artist = create(:artist, name: "noizave", url_string: "https://deviantart.com/noizave") assert_equal([@artist], @site.artists) diff --git a/test/unit/sources/fanbox_test.rb b/test/unit/sources/fanbox_test.rb index 6f1f16fec..057223704 100644 --- a/test/unit/sources/fanbox_test.rb +++ b/test/unit/sources/fanbox_test.rb @@ -4,9 +4,9 @@ module Sources class FanboxTest < ActiveSupport::TestCase context "A free Pixiv Fanbox post" do setup do - @post1 = Sources::Strategies.find("https://yanmi0308.fanbox.cc/posts/1141325") - @post2 = Sources::Strategies.find("https://chanxco.fanbox.cc/posts/209386") - @post3 = Sources::Strategies.find("https://downloads.fanbox.cc/images/post/209386/w/1200/8dRNHXkFqAwSt31W2Bg8fSdL.jpeg") + @post1 = Source::Extractor.find("https://yanmi0308.fanbox.cc/posts/1141325") + @post2 = Source::Extractor.find("https://chanxco.fanbox.cc/posts/209386") + @post3 = Source::Extractor.find("https://downloads.fanbox.cc/images/post/209386/w/1200/8dRNHXkFqAwSt31W2Bg8fSdL.jpeg") assert_nothing_raised { @post1.to_h } assert_nothing_raised { @post2.to_h } @@ -95,7 +95,7 @@ module Sources context "an age-restricted fanbox post" do should "work" do - @source = Sources::Strategies.find("https://mfr.fanbox.cc/posts/1306390") + @source = Source::Extractor.find("https://mfr.fanbox.cc/posts/1306390") assert_nothing_raised { @source.to_h } assert_equal("mfr", @source.artist_name) @@ -105,7 +105,7 @@ module Sources context "A link in the old format" do should "still work" do - post = Sources::Strategies.find("https://www.pixiv.net/fanbox/creator/1566167/post/39714") + post = Source::Extractor.find("https://www.pixiv.net/fanbox/creator/1566167/post/39714") assert_nothing_raised { post.to_h } assert_equal("https://omu001.fanbox.cc", post.profile_url) assert_equal("https://omu001.fanbox.cc/posts/39714", post.page_url) @@ -116,7 +116,7 @@ module Sources context "A cover image" do should "still work" do - post = Sources::Strategies.find("https://pixiv.pximg.net/c/1620x580_90_a2_g5/fanbox/public/images/creator/1566167/cover/QqxYtuWdy4XWQx1ZLIqr4wvA.jpeg") + post = Source::Extractor.find("https://pixiv.pximg.net/c/1620x580_90_a2_g5/fanbox/public/images/creator/1566167/cover/QqxYtuWdy4XWQx1ZLIqr4wvA.jpeg") assert_nothing_raised { post.to_h } assert_downloaded(750_484, post.image_urls.sole) assert_equal("https://omu001.fanbox.cc", post.profile_url) @@ -128,7 +128,7 @@ module Sources context "A dead profile picture from the old domain" do should "still find the artist" do - post = Sources::Strategies.find("https://pixiv.pximg.net/c/400x400_90_a2_g5/fanbox/public/images/creator/1566167/profile/Ix6bnJmTaOAFZhXHLbWyIY1e.jpeg") + post = Source::Extractor.find("https://pixiv.pximg.net/c/400x400_90_a2_g5/fanbox/public/images/creator/1566167/profile/Ix6bnJmTaOAFZhXHLbWyIY1e.jpeg") assert_equal("https://omu001.fanbox.cc", post.profile_url) artist = FactoryBot.create(:artist, name: "omu", url_string: "https://omu001.fanbox.cc") assert_equal([artist], post.artists) diff --git a/test/unit/sources/fantia_test.rb b/test/unit/sources/fantia_test.rb index fd036f3e8..041e77110 100644 --- a/test/unit/sources/fantia_test.rb +++ b/test/unit/sources/fantia_test.rb @@ -10,7 +10,7 @@ module Sources context "A c.fantia.jp/uploads/post/file/ url" do should "work" do url = "https://c.fantia.jp/uploads/post/file/1070093/16faf0b1-58d8-4aac-9e86-b243063eaaf1.jpeg" - source = Sources::Strategies.find(url) + source = Source::Extractor.find(url) assert_equal([url], source.image_urls) assert_equal("豆ラッコ", source.other_names.first) @@ -28,7 +28,7 @@ module Sources should "work" do url = "https://cc.fantia.jp/uploads/post_content_photo/file/7087182/main_7f04ff3c-1f08-450f-bd98-796c290fc2d1.jpg?Key-Pair-Id=APKAIOCKYZS7WKBB6G7A&Policy=eyJTdGF0ZW1lbnQiOlt7IlJlc291cmNlIjoiaHR0cHM6Ly9jYy5mYW50aWEuanAvdXBsb2Fkcy9wb3N0X2NvbnRlbnRfcGhvdG8vZmlsZS83MDg3MTgyL21haW5fN2YwNGZmM2MtMWYwOC00NTBmLWJkOTgtNzk2YzI5MGZjMmQxLmpwZyIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTY0NjkyODAzN319fV19&Signature=wl2Nr9i1O5R5dDc7FB-8CKtRvyZPS6ZEFXn7Q74rBh9R2PZkpKuQUDDsJubgkYaHrqHEapcOdZczzZaM5kbRLXGPOnVFUE7vHKnXZTO~Z1-Z8Cqt823NKCR-AXBjYPhQoGP0pITLYkjhofy0FXg6RYJ0oNJPdKkdjcnwzr-nZfyaFgkrrQ5~LRDhW5HOgSNfvhJleMRLRgLtXbbgNnVwHmpFWNkFSwwmDcUTXTh4hrhQrOJ~xJmiQesSP1wPAE5ZZSBGsbUstOa5Y1nVu540wItR4VWLm-jjuMk9OIr-Nvxg0ocoP9WU13WrRbeMeL5X0xhxBYSxgVIKXko2BqMf5w__" ref = "https://fantia.jp/posts/1132267" - source = Sources::Strategies.find(url, ref) + source = Source::Extractor.find(url, ref) assert_equal("稲光伸二", source.other_names.first) assert_equal("https://fantia.jp/fanclubs/1096", source.profile_url) @@ -43,7 +43,7 @@ module Sources context "A c.fantia.jp/uploads/product/image/ url" do should "work" do url = "https://c.fantia.jp/uploads/product/image/249638/fd5aef8f-c217-49d0-83e8-289efb33dfc4.jpg" - source = Sources::Strategies.find(url) + source = Source::Extractor.find(url) tags = ["イラスト集", "CG集", "PNG", "オリジナル", "宮前詩帆", "春川朱璃愛", "夏川黒羽", "ASMR", "音声", "原神", "シニョーラ"] assert_equal([url], source.image_urls) @@ -62,7 +62,7 @@ module Sources should "work" do url = "https://c.fantia.jp/uploads/product_image/file/219407/main_bd7419c2-2450-4c53-a28a-90101fa466ab.jpg" ref = "https://fantia.jp/products/249638" - source = Sources::Strategies.find(url, ref) + source = Source::Extractor.find(url, ref) assert_equal(["https://c.fantia.jp/uploads/product_image/file/219407/bd7419c2-2450-4c53-a28a-90101fa466ab.jpg"], source.image_urls) assert_equal("https://fantia.jp/fanclubs/7", source.profile_url) @@ -76,7 +76,7 @@ module Sources context "A fantia.jp/posts/$id/download url" do should "work" do url = "https://fantia.jp/posts/1143951/download/1830956" - source = Sources::Strategies.find(url) + source = Source::Extractor.find(url) assert_match(%r{1830956/cbcdfcbe_20220224_120_040_100.png}, source.image_urls.sole) assert_equal("松永紅葉", source.other_names.first) @@ -93,7 +93,7 @@ module Sources context "A fantia.jp/posts/$id url" do should "work" do url = "https://fantia.jp/posts/1143951" - source = Sources::Strategies.find(url) + source = Source::Extractor.find(url) assert_equal("https://c.fantia.jp/uploads/post/file/1143951/47491020-a6c6-47db-b09e-815b0530c0bc.png", source.image_urls.first) assert_match(%r{1830956/cbcdfcbe_20220224_120_040_100.png}, source.image_urls.second) @@ -112,7 +112,7 @@ module Sources context "A fantia.jp/products/$id url" do should "work" do url = "https://fantia.jp/products/249638" - source = Sources::Strategies.find(url) + source = Source::Extractor.find(url) image_urls = %w[ https://c.fantia.jp/uploads/product/image/249638/fd5aef8f-c217-49d0-83e8-289efb33dfc4.jpg https://c.fantia.jp/uploads/product_image/file/219406/c73bd7f9-a13a-48f7-9ac7-35309faa88c3.jpg @@ -137,7 +137,7 @@ module Sources context "A product url with no images" do should "not get placeholder images" do - source = Sources::Strategies.find("https://fantia.jp/products/10000") + source = Source::Extractor.find("https://fantia.jp/products/10000") assert_equal([], source.image_urls) assert_nothing_raised { source.to_h } end @@ -148,8 +148,8 @@ module Sources url1 = "https://fantia.jp/posts/12345678901234567890" url2 = "https://fantia.jp/products/12345678901234567890" - source1 = Sources::Strategies.find(url1) - source2 = Sources::Strategies.find(url2) + source1 = Source::Extractor.find(url1) + source2 = Source::Extractor.find(url2) assert_equal([], source1.image_urls) assert_equal([], source2.image_urls) diff --git a/test/unit/sources/foundation_test.rb b/test/unit/sources/foundation_test.rb index 0c3d1650f..25f84fa60 100644 --- a/test/unit/sources/foundation_test.rb +++ b/test/unit/sources/foundation_test.rb @@ -7,9 +7,9 @@ module Sources @post_url = "https://foundation.app/@dadachyo/~/103724" @post_with_video = "https://foundation.app/@huwari/~/88982" @image_url = "https://f8n-ipfs-production.imgix.net/QmPhpz6E9TFRpvdVTviM8Hy9o9rxrnPW5Ywj471NnSNkpi/nft.jpg" - @image1 = Sources::Strategies.find(@post_url) - @image2 = Sources::Strategies.find(@image_url) - @image3 = Sources::Strategies.find(@post_with_video) + @image1 = Source::Extractor.find(@post_url) + @image2 = Source::Extractor.find(@image_url) + @image3 = Source::Extractor.find(@post_with_video) end should "get the artist name" do @@ -57,7 +57,7 @@ module Sources should "work" do page_url = "https://foundation.app/@asuka111art/dinner-with-cats-82426" image_url = "https://f8n-ipfs-production.imgix.net/Qma7Lz2LfFb4swoqzr1V43oRGh9xikgigM11g3EukdU61R/nft.png" - source = Sources::Strategies.find(page_url) + source = Source::Extractor.find(page_url) assert_equal("asuka111art", source.artist_name) assert_equal(["https://foundation.app/@asuka111art", "https://foundation.app/0x9A94f94626352566e0A9105F1e3DA0439E3e3783"], source.profile_urls) @@ -69,7 +69,7 @@ module Sources context "for a f8n-production-collection-assets.imgix.net URL" do should "work" do image_url = "https://f8n-production-collection-assets.imgix.net/0x3B3ee1931Dc30C1957379FAc9aba94D1C48a5405/128711/QmcBfbeCMSxqYB3L1owPAxFencFx3jLzCPFx6xUBxgSCkH/nft.png?q=80&auto=format%2Ccompress&cs=srgb&h=640" - source = Sources::Strategies.find(image_url) + source = Source::Extractor.find(image_url) assert_equal("mochiiimo", source.artist_name) assert_equal(["https://foundation.app/@mochiiimo", "https://foundation.app/0x7E2ef75C0C09b2fc6BCd1C68B6D409720CcD58d2"], source.profile_urls) @@ -82,25 +82,25 @@ module Sources should "get the image urls" do assert_equal( ["https://f8n-ipfs-production.imgix.net/QmX4MotNAAj9Rcyew43KdgGDxU1QtXemMHoUTNacMLLSjQ/nft.png"], - Sources::Strategies.find("https://foundation.app/@mochiiimo/~/97376").image_urls, + Source::Extractor.find("https://foundation.app/@mochiiimo/~/97376").image_urls, ) assert_equal( ["https://f8n-ipfs-production.imgix.net/QmX4MotNAAj9Rcyew43KdgGDxU1QtXemMHoUTNacMLLSjQ/nft.png"], - Sources::Strategies.find("https://foundation.app/@mochiiimo/foundation/97376").image_urls, + Source::Extractor.find("https://foundation.app/@mochiiimo/foundation/97376").image_urls, ) assert_equal( ["https://f8n-production-collection-assets.imgix.net/0xFb0a8e1bB97fD7231Cd73c489dA4732Ae87995F0/4/nft.png"], - Sources::Strategies.find("https://foundation.app/@KILLERGF/kgfgen/4").image_urls, + Source::Extractor.find("https://foundation.app/@KILLERGF/kgfgen/4").image_urls, ) end end context "non-alphanumeric usernames" do should "still work" do - case1 = Sources::Strategies.find("https://foundation.app/@brandon.dalmer/~/6792") - case2 = Sources::Strategies.find("https://foundation.app/@~/~/6792") + case1 = Source::Extractor.find("https://foundation.app/@brandon.dalmer/~/6792") + case2 = Source::Extractor.find("https://foundation.app/@~/~/6792") image = "https://f8n-ipfs-production.imgix.net/QmVnpe39qodMjTe8v3fijPfB1tjwhT8hgobtgLPtsangqc/nft.png" assert_nothing_raised { case1.to_h } assert_nothing_raised { case2.to_h } @@ -110,7 +110,7 @@ module Sources end should "parse UTF-8 commentaries correctly" do - source = Sources::Strategies.find("https://foundation.app/@SimaEnaga/~/107338") + source = Source::Extractor.find("https://foundation.app/@SimaEnaga/~/107338") assert_equal(<<~EOS, source.dtext_artist_commentary_desc) 【須佐之男尊/Susanoo-no-Mikoto】 diff --git a/test/unit/sources/hentai_foundry_test.rb b/test/unit/sources/hentai_foundry_test.rb index b97abb02e..449ee0a15 100644 --- a/test/unit/sources/hentai_foundry_test.rb +++ b/test/unit/sources/hentai_foundry_test.rb @@ -4,8 +4,8 @@ module Sources class HentaiFoundryTest < ActiveSupport::TestCase context "The source for a hentai foundry picture" do setup do - @image_1 = Sources::Strategies.find("https://www.hentai-foundry.com/pictures/user/Afrobull/795025/kuroeda") - @image_2 = Sources::Strategies.find("https://pictures.hentai-foundry.com/a/Afrobull/795025/Afrobull-795025-kuroeda.png") + @image_1 = Source::Extractor.find("https://www.hentai-foundry.com/pictures/user/Afrobull/795025/kuroeda") + @image_2 = Source::Extractor.find("https://pictures.hentai-foundry.com/a/Afrobull/795025/Afrobull-795025-kuroeda.png") end should "get the illustration id" do @@ -52,7 +52,7 @@ module Sources context "An artist profile url" do setup do - @site = Sources::Strategies.find("https://www.hentai-foundry.com/user/Afrobull/profile") + @site = Source::Extractor.find("https://www.hentai-foundry.com/user/Afrobull/profile") end should "get the profile url" do @@ -66,7 +66,7 @@ module Sources context "A deleted picture" do setup do - @image = Sources::Strategies.find("https://www.hentai-foundry.com/pictures/user/faustsketcher/279498") + @image = Source::Extractor.find("https://www.hentai-foundry.com/pictures/user/faustsketcher/279498") @artist = FactoryBot.create(:artist, name: "faustsketcher", url_string: @image.url) end @@ -92,7 +92,7 @@ module Sources context "a post with a deeply nested commentary" do should "work" do - @source = Sources::Strategies.find("https://hentai-foundry.com/pictures/user/LumiNyu/867562/Mona-patreon-winner") + @source = Source::Extractor.find("https://hentai-foundry.com/pictures/user/LumiNyu/867562/Mona-patreon-winner") assert_nothing_raised { @source.to_h } end end diff --git a/test/unit/sources/lofter_test.rb b/test/unit/sources/lofter_test.rb index c81fe1309..92ac0bfea 100644 --- a/test/unit/sources/lofter_test.rb +++ b/test/unit/sources/lofter_test.rb @@ -6,8 +6,8 @@ module Sources setup do @img = "https://imglf4.lf127.net/img/S1d2QlVsWkJhSW1qcnpIS0ZSa3ZJUFczb2RKSVlpMHJkNy9kc3BSQVQvQm5DNzB4eVhxay9nPT0.png?imageView&thumbnail=1680x0&quality=96&stripmeta=0" @ref = "https://gengar563.lofter.com/post/1e82da8c_1c98dae1b" - @source = Sources::Strategies.find(@img, @ref) - @source2 = Sources::Strategies.find(@ref) + @source = Source::Extractor.find(@img, @ref) + @source2 = Source::Extractor.find(@ref) end should "get the artist name" do @@ -47,7 +47,7 @@ module Sources context "A different CSS schema" do should "still find all the data" do - source1 = Sources::Strategies.find("https://yuli031458.lofter.com/post/3163d871_1cbdc5f6d") + source1 = Source::Extractor.find("https://yuli031458.lofter.com/post/3163d871_1cbdc5f6d") assert_equal(["https://imglf5.lf127.net/img/Mm55d3lNK2tJUWpNTjVLN0MvaTRDc1UvQUFLMGszOHRvSjV6S3VSa1lwa3BDWUtVOWpBTHBnPT0.jpg"], source1.image_urls) assert_not_empty(source1.tags) @@ -56,7 +56,7 @@ module Sources context "A bad link" do should "correctly get the full size" do - source = Sources::Strategies.find("https://imglf4.lf127.net/img/S1d2QlVsWkJhSW1qcnpIS0ZSa3ZJUFczb2RKSVlpMHJkNy9kc3BSQVQvQm5DNzB4eVhxay9nPT0.png?imageView&thumbnail=1680x0&quality=96&stripmeta=0") + source = Source::Extractor.find("https://imglf4.lf127.net/img/S1d2QlVsWkJhSW1qcnpIS0ZSa3ZJUFczb2RKSVlpMHJkNy9kc3BSQVQvQm5DNzB4eVhxay9nPT0.png?imageView&thumbnail=1680x0&quality=96&stripmeta=0") assert_equal(["https://imglf4.lf127.net/img/S1d2QlVsWkJhSW1qcnpIS0ZSa3ZJUFczb2RKSVlpMHJkNy9kc3BSQVQvQm5DNzB4eVhxay9nPT0.png"], source.image_urls) assert_nothing_raised { source.to_h } end @@ -64,7 +64,7 @@ module Sources context "A dead link" do should "not raise anything" do - source = Sources::Strategies.find("https://gxszdddd.lofter.com/post/322595b1_1ca5e6f66") + source = Source::Extractor.find("https://gxszdddd.lofter.com/post/322595b1_1ca5e6f66") assert_nothing_raised { source.to_h } end end diff --git a/test/unit/sources/mastodon_test.rb b/test/unit/sources/mastodon_test.rb index 1046615ec..54bff9c66 100644 --- a/test/unit/sources/mastodon_test.rb +++ b/test/unit/sources/mastodon_test.rb @@ -5,7 +5,7 @@ module Sources context "The source site for a https://pawoo.net/web/status/$id url" do setup do skip "Pawoo keys not set" unless Danbooru.config.pawoo_client_id - @site = Sources::Strategies.find("https://pawoo.net/web/statuses/1202176") + @site = Source::Extractor.find("https://pawoo.net/web/statuses/1202176") end should "get the profile" do @@ -34,7 +34,7 @@ module Sources context "The source site for a https://pawoo.net/$user/$id url" do setup do skip "Pawoo keys not set" unless Danbooru.config.pawoo_client_id - @site = Sources::Strategies.find("https://pawoo.net/@evazion/19451018") + @site = Source::Extractor.find("https://pawoo.net/@evazion/19451018") end should "get the profile" do @@ -89,7 +89,7 @@ module Sources skip "Pawoo keys not set" unless Danbooru.config.pawoo_client_id @url = "https://img.pawoo.net/media_attachments/files/001/298/028/original/55a6fd252778454b.mp4" @ref = "https://pawoo.net/@evazion/19451018" - @site = Sources::Strategies.find(@url, @ref) + @site = Source::Extractor.find(@url, @ref) end should "fetch the source data" do @@ -105,11 +105,11 @@ module Sources setup do skip "Baraag keys not set" unless Danbooru.config.baraag_client_id @url = "https://baraag.net/@bardbot/105732813175612920" - @site1 = Sources::Strategies.find(@url) + @site1 = Source::Extractor.find(@url) @img = "https://baraag.net/system/media_attachments/files/105/803/948/862/719/091/original/54e1cb7ca33ec449.png" @ref = "https://baraag.net/@Nakamura/105803949565505009" - @site2 = Sources::Strategies.find(@img, @ref) + @site2 = Source::Extractor.find(@img, @ref) end should "work" do @@ -140,8 +140,8 @@ module Sources setup do skip "Pawoo keys not set" unless Danbooru.config.pawoo_client_id - @site1 = Sources::Strategies.find("https://pawoo.net/@nantokakun/105643037682139899") # 404 - @site2 = Sources::Strategies.find("https://img.pawoo.net/media_attachments/files/001/297/997/original/c4272a09570757c2.png") + @site1 = Source::Extractor.find("https://pawoo.net/@nantokakun/105643037682139899") # 404 + @site2 = Source::Extractor.find("https://img.pawoo.net/media_attachments/files/001/297/997/original/c4272a09570757c2.png") assert_nothing_raised { @site1.to_h } assert_nothing_raised { @site2.to_h } diff --git a/test/unit/sources/moebooru_test.rb b/test/unit/sources/moebooru_test.rb index a8a9d68b8..76c0ac7e1 100644 --- a/test/unit/sources/moebooru_test.rb +++ b/test/unit/sources/moebooru_test.rb @@ -3,7 +3,7 @@ require "test_helper" module Sources class MoebooruTest < ActiveSupport::TestCase def assert_source_data_equals(url, referer = nil, site_name: nil, image_url: nil, page_url: nil, size: nil, tags: [], profile_url: nil, **params) - site = Sources::Strategies.find(url, referer) + site = Source::Extractor.find(url, referer) assert_equal(site_name, site.site_name) assert_equal([image_url], site.image_urls) diff --git a/test/unit/sources/newgrounds_test.rb b/test/unit/sources/newgrounds_test.rb index 280f66284..f7d5121db 100644 --- a/test/unit/sources/newgrounds_test.rb +++ b/test/unit/sources/newgrounds_test.rb @@ -6,8 +6,8 @@ module Sources setup do @url = "https://www.newgrounds.com/art/view/hcnone/sephiroth" @image_url = "https://art.ngfiles.com/images/1539000/1539538_hcnone_sephiroth.png?f1607668234" - @image_1 = Sources::Strategies.find(@url) - @image_2 = Sources::Strategies.find(@image_url) + @image_1 = Source::Extractor.find(@url) + @image_2 = Source::Extractor.find(@image_url) end should "get the artist name" do @@ -64,7 +64,7 @@ module Sources context "A multi-image Newgrounds post" do should "get all the images" do - source = Sources::Strategies.find("https://www.newgrounds.com/art/view/natthelich/weaver") + source = Source::Extractor.find("https://www.newgrounds.com/art/view/natthelich/weaver") image_urls = [ "https://art.ngfiles.com/images/1520000/1520217_natthelich_weaver.jpg?f1606365031", "https://art.ngfiles.com/comments/199000/iu_199826_7115981.jpg", @@ -76,13 +76,13 @@ module Sources context "A deleted or not existing picture" do setup do - @fake_1 = Sources::Strategies.find("https://www.newgrounds.com/art/view/ThisUser/DoesNotExist") + @fake_1 = Source::Extractor.find("https://www.newgrounds.com/art/view/ThisUser/DoesNotExist") @artist_1 = create(:artist, name: "thisuser", url_string: "https://thisuser.newgrounds.com") - @fake_2 = Sources::Strategies.find("https://www.newgrounds.com/art/view/natthelich/nopicture") + @fake_2 = Source::Extractor.find("https://www.newgrounds.com/art/view/natthelich/nopicture") @artist_2 = create(:artist, name: "natthelich", url_string: "https://natthelich.newgrounds.com") - @fake_3 = Sources::Strategies.find("https://www.newgrounds.com/art/view/theolebrave/sensitive-pochaco") + @fake_3 = Source::Extractor.find("https://www.newgrounds.com/art/view/theolebrave/sensitive-pochaco") @artist_3 = create(:artist, name: "taffytoad", url_string: "https://taffytoad.newgrounds.com") end diff --git a/test/unit/sources/nico_seiga_test.rb b/test/unit/sources/nico_seiga_test.rb index b80e88c66..941aa79be 100644 --- a/test/unit/sources/nico_seiga_test.rb +++ b/test/unit/sources/nico_seiga_test.rb @@ -3,14 +3,14 @@ require 'test_helper' module Sources class NicoSeigaTest < ActiveSupport::TestCase setup do - skip "NicoSeiga credentials not configured" unless Sources::Strategies::NicoSeiga.enabled? + skip "NicoSeiga credentials not configured" unless Source::Extractor::NicoSeiga.enabled? end context "The source site for nico seiga" do setup do - @site_1 = Sources::Strategies.find("http://lohas.nicoseiga.jp/o/910aecf08e542285862954017f8a33a8c32a8aec/1433298801/4937663") - @site_2 = Sources::Strategies.find("http://seiga.nicovideo.jp/seiga/im4937663") - @site_3 = Sources::Strategies.find("https://seiga.nicovideo.jp/watch/mg470189?track=ct_episode") + @site_1 = Source::Extractor.find("http://lohas.nicoseiga.jp/o/910aecf08e542285862954017f8a33a8c32a8aec/1433298801/4937663") + @site_2 = Source::Extractor.find("http://seiga.nicovideo.jp/seiga/im4937663") + @site_3 = Source::Extractor.find("https://seiga.nicovideo.jp/watch/mg470189?track=ct_episode") end should "get the profile" do @@ -80,7 +80,7 @@ module Sources end should "work for a https://lohas.nicoseiga.jp/thumb/${id}i url" do - site = Sources::Strategies.find("https://lohas.nicoseiga.jp/thumb/6844226i") + site = Source::Extractor.find("https://lohas.nicoseiga.jp/thumb/6844226i") assert_match(%r!https?://lohas.nicoseiga.jp/priv/[a-f0-9]{40}/[0-9]+/6844226!, site.image_urls.sole) assert_match("https://seiga.nicovideo.jp/seiga/im6844226", site.page_url) @@ -91,7 +91,7 @@ module Sources setup do @url = "https://seiga.nicovideo.jp/image/source/9146749" @ref = "https://seiga.nicovideo.jp/watch/mg389884" - @site = Sources::Strategies.find(@url, @ref) + @site = Source::Extractor.find(@url, @ref) end should "get the correct pic" do @@ -105,7 +105,7 @@ module Sources context "A manga image" do should "work" do - @source = Sources::Strategies.find("https://drm.cdn.nicomanga.jp/image/d4a2faa68ec34f95497db6601a4323fde2ccd451_9537/8017978p?1570012695") + @source = Source::Extractor.find("https://drm.cdn.nicomanga.jp/image/d4a2faa68ec34f95497db6601a4323fde2ccd451_9537/8017978p?1570012695") assert_match(%r{\Ahttps://lohas\.nicoseiga\.jp/priv/\h{40}/\d+/8017978\z}, @source.image_urls.sole) end @@ -113,7 +113,7 @@ module Sources context "A nico.ms illust URL" do should "work" do - @source = Sources::Strategies.find("https://nico.ms/im10922621") + @source = Source::Extractor.find("https://nico.ms/im10922621") assert_match(%r{\Ahttps://lohas\.nicoseiga\.jp/priv/\h{40}/\d+/10922621\z}, @source.image_urls.sole) end @@ -121,7 +121,7 @@ module Sources context "A nico.ms manga URL" do should "work" do - @source = Sources::Strategies.find("https://nico.ms/mg310193") + @source = Source::Extractor.find("https://nico.ms/mg310193") assert_equal(19, @source.image_urls.size) assert_equal("https://seiga.nicovideo.jp/watch/mg310193", @source.page_url) @@ -130,14 +130,14 @@ module Sources context "A nicoseiga video" do should "not raise anything" do - site = Sources::Strategies.find("https://www.nicovideo.jp/watch/sm36465441") + site = Source::Extractor.find("https://www.nicovideo.jp/watch/sm36465441") assert_nothing_raised { site.to_h } end end context "An anonymous picture" do should "still work" do - site = Sources::Strategies.find("https://seiga.nicovideo.jp/seiga/im520647") + site = Source::Extractor.find("https://seiga.nicovideo.jp/seiga/im520647") assert_nothing_raised { site.to_h } end @@ -145,7 +145,7 @@ module Sources context "An age-restricted picture" do should "still work" do - site = Sources::Strategies.find("http://seiga.nicovideo.jp/seiga/im9208126") + site = Source::Extractor.find("http://seiga.nicovideo.jp/seiga/im9208126") assert_match(%r!https?://lohas.nicoseiga.jp/priv/[a-f0-9]{40}/[0-9]+/9208126!, site.image_urls.sole) assert_nothing_raised { site.to_h } @@ -154,14 +154,14 @@ module Sources context "An oekaki picture" do should "still work" do - site = Sources::Strategies.find("https://dic.nicovideo.jp/oekaki/52833.png") + site = Source::Extractor.find("https://dic.nicovideo.jp/oekaki/52833.png") assert_nothing_raised { site.to_h } end end context "A commentary with spoiler" do should "correctly add spoiler tags" do - site = Sources::Strategies.find("https://seiga.nicovideo.jp/seiga/im8992650") + site = Source::Extractor.find("https://seiga.nicovideo.jp/seiga/im8992650") commentary = <<~COMM.chomp SLVN大好き。ホントニアコガレテル。 diff --git a/test/unit/sources/nijie_test.rb b/test/unit/sources/nijie_test.rb index 6287591a3..cb378a7cc 100644 --- a/test/unit/sources/nijie_test.rb +++ b/test/unit/sources/nijie_test.rb @@ -3,7 +3,7 @@ require 'test_helper' module Sources class NijieTest < ActiveSupport::TestCase setup do - skip "Nijie credentials not configured" unless Sources::Strategies::Nijie.enabled? + skip "Nijie credentials not configured" unless Source::Extractor::Nijie.enabled? skip if ENV["CI"].present? end @@ -43,7 +43,7 @@ module Sources CurrentUser.user = FactoryBot.create(:user) CurrentUser.ip_addr = "127.0.0.1" - @site = Sources::Strategies.find("https://nijie.info/view.php?id=213043") + @site = Source::Extractor.find("https://nijie.info/view.php?id=213043") end should "get the image url" do @@ -79,7 +79,7 @@ module Sources FactoryBot.create(:tag, :name => "kaga") FactoryBot.create(:wiki_page, :title => "kaga", :other_names => "加賀(艦これ)") - @site = Sources::Strategies.find("https://nijie.info/view.php?id=208316") + @site = Source::Extractor.find("https://nijie.info/view.php?id=208316") assert_includes(@site.tags.map(&:first), "加賀(艦これ)") assert_includes(@site.translated_tags.map(&:name), "kaga") @@ -96,7 +96,7 @@ module Sources context "For long commentaries that may be truncated" do should "get the full commentary" do - site = Sources::Strategies.find("http://nijie.info/view.php?id=266532") + site = Source::Extractor.find("http://nijie.info/view.php?id=266532") title = "ラミアの里" desc = <<~EOS.chomp サークルaskot様より販売されました「ラミアの里 ~ラミアはぁれむで搾られて~」にて前回に引き続きフラウのイラストを担当させて頂きました。 @@ -113,7 +113,7 @@ module Sources context "The source site for a nijie referer url" do setup do - @site = Sources::Strategies.find("http://pic.nijie.net/03/nijie_picture/728995_20170505014820_0.jpg", "https://nijie.info/view_popup.php?id=213043") + @site = Source::Extractor.find("http://pic.nijie.net/03/nijie_picture/728995_20170505014820_0.jpg", "https://nijie.info/view_popup.php?id=213043") end should "get the image url" do @@ -135,7 +135,7 @@ module Sources context "The source site for a nijie popup" do setup do - @site = Sources::Strategies.find("https://nijie.info/view_popup.php?id=213043") + @site = Source::Extractor.find("https://nijie.info/view_popup.php?id=213043") end should "get the image url" do @@ -157,7 +157,7 @@ module Sources context "The source site for a nijie gallery" do setup do - @site = Sources::Strategies.find("https://nijie.info/view.php?id=218856") + @site = Source::Extractor.find("https://nijie.info/view.php?id=218856") end should "get the image urls" do @@ -187,7 +187,7 @@ module Sources context "The source site for a nijie image url without referer" do should "get the correct urls" do image_url = "https://pic.nijie.net/03/nijie_picture/236014_20170620101426_0.png" - site = Sources::Strategies.find(image_url) + site = Source::Extractor.find(image_url) assert_nil(site.page_url) assert_equal([image_url], site.image_urls) @@ -198,7 +198,7 @@ module Sources context "An image url that contains the illust id and artist id (format 1)" do should "fetch all the data" do - site = Sources::Strategies.find("https://pic.nijie.net/03/nijie_picture/diff/main/218856_4_236014_20170620101333.png") + site = Source::Extractor.find("https://pic.nijie.net/03/nijie_picture/diff/main/218856_4_236014_20170620101333.png") assert_equal("https://nijie.info/view.php?id=218856", site.page_url) assert_equal("https://nijie.info/members.php?id=236014", site.profile_url) @@ -209,7 +209,7 @@ module Sources context "An image url that contains the illust id and artist id (format 2)" do should "fetch all the data" do - site = Sources::Strategies.find("https://pic.nijie.net/04/nijie_picture/diff/main/287736_161475_20181112032855_1.png") + site = Source::Extractor.find("https://pic.nijie.net/04/nijie_picture/diff/main/287736_161475_20181112032855_1.png") assert_equal("https://nijie.info/view.php?id=287736", site.page_url) assert_equal("https://nijie.info/members.php?id=161475", site.profile_url) @@ -220,7 +220,7 @@ module Sources context "An mp4 post" do should "find the mp4 file" do - site = Sources::Strategies.find("http://nijie.info/view.php?id=324604") + site = Source::Extractor.find("http://nijie.info/view.php?id=324604") assert_equal(%w[ https://pic.nijie.net/01/nijie/19/69/1349569/illust/0_0_a20b709587eb7713_30b409.mp4 @@ -231,7 +231,7 @@ module Sources context "An artist profile url" do should "not fail" do - site = Sources::Strategies.find("https://nijie.info/members_illust.php?id=236014") + site = Source::Extractor.find("https://nijie.info/members_illust.php?id=236014") assert_equal("https://nijie.info/members.php?id=236014", site.profile_url) assert_nothing_raised { site.to_h } end @@ -239,7 +239,7 @@ module Sources context "An url that is invalid" do should "not fail" do - site = Sources::Strategies.find("http://nijie.info/index.php") + site = Source::Extractor.find("http://nijie.info/index.php") assert_nothing_raised { site.to_h } end end @@ -247,7 +247,7 @@ module Sources context "A deleted work" do context "for an image url" do should "find the profile url" do - site = Sources::Strategies.find("https://pic.nijie.net/01/nijie_picture/diff/main/196201_20150201033106_0.jpg") + site = Source::Extractor.find("https://pic.nijie.net/01/nijie_picture/diff/main/196201_20150201033106_0.jpg") assert_nothing_raised { site.to_h } assert_equal("https://nijie.info/members.php?id=196201", site.profile_url) @@ -257,7 +257,7 @@ module Sources context "for a page url" do should "not fail" do - site = Sources::Strategies.find("http://www.nijie.info/view_popup.php?id=212355") + site = Source::Extractor.find("http://www.nijie.info/view_popup.php?id=212355") assert_equal("https://nijie.info/view.php?id=212355", site.page_url) assert_nil(site.profile_url) @@ -272,7 +272,7 @@ module Sources context "a post requiring login" do should "not fail" do - site = Sources::Strategies.find("https://nijie.info/view.php?id=203688") + site = Source::Extractor.find("https://nijie.info/view.php?id=203688") urls = %w[ https://pic.nijie.net/07/nijie/17/27/676327/illust/0_0_2e46f254324c90c8_dbfc1a.jpg @@ -284,7 +284,7 @@ module Sources context "when the cached session cookie is invalid" do should "clear the cached cookie after failing to fetch the data" do - site = Sources::Strategies.find("https://nijie.info/view.php?id=203688") + site = Source::Extractor.find("https://nijie.info/view.php?id=203688") Cache.put("nijie-session-cookie", { "NIJIEIJIEID" => "fake", "nijie_tok" => "fake" }) assert_equal({ "NIJIEIJIEID" => "fake", "nijie_tok" => "fake" }, site.cached_session_cookie) @@ -298,7 +298,7 @@ module Sources should "work" do image = "https://pic.nijie.net/01/dojin_main/dojin_sam/20120213044700%E3%82%B3%E3%83%94%E3%83%BC%20%EF%BD%9E%200011%E3%81%AE%E3%82%B3%E3%83%94%E3%83%BC.jpg" page = "https://nijie.info/view.php?id=53023" - site = Sources::Strategies.find(image, page) + site = Source::Extractor.find(image, page) tags = [%w[中出し https://nijie.info/search_dojin.php?word=%E4%B8%AD%E5%87%BA%E3%81%97], %w[フェラ https://nijie.info/search_dojin.php?word=%E3%83%95%E3%82%A7%E3%83%A9], @@ -338,7 +338,7 @@ module Sources should "not break the bookmarklet" do image_url = "https://pic.nijie.net/01/nijie_picture/diff/main/201207181053373205_0.jpg" ref = "https://nijie.info/view_popup.php?id=18858&#diff_1" - source = Sources::Strategies.find(image_url, ref) + source = Source::Extractor.find(image_url, ref) assert_equal([image_url], source.image_urls) end diff --git a/test/unit/sources/null_test.rb b/test/unit/sources/null_test.rb index af521fc77..3a08a15ae 100644 --- a/test/unit/sources/null_test.rb +++ b/test/unit/sources/null_test.rb @@ -4,11 +4,11 @@ module Sources class NullTest < ActiveSupport::TestCase context "A source from an unknown site" do setup do - @site = Sources::Strategies.find("http://oremuhax.x0.com/yoro1603.jpg", "http://oremuhax.x0.com/yo125.htm") + @site = Source::Extractor.find("http://oremuhax.x0.com/yoro1603.jpg", "http://oremuhax.x0.com/yo125.htm") end should "be handled by the null strategy" do - assert(@site.is_a?(Sources::Strategies::Null)) + assert(@site.is_a?(Source::Extractor::Null)) end should "find the metadata" do diff --git a/test/unit/sources/pixiv_sketch_test.rb b/test/unit/sources/pixiv_sketch_test.rb index 058944fbc..a4dc216f8 100644 --- a/test/unit/sources/pixiv_sketch_test.rb +++ b/test/unit/sources/pixiv_sketch_test.rb @@ -4,7 +4,7 @@ module Sources class PixivSketchTest < ActiveSupport::TestCase context "A Pixiv Sketch source" do should "work for a post with a single image" do - source = Sources::Strategies.find("https://sketch.pixiv.net/items/5835314698645024323") + source = Source::Extractor.find("https://sketch.pixiv.net/items/5835314698645024323") assert_equal("Pixiv Sketch", source.site_name) assert_equal(["https://img-sketch.pixiv.net/uploads/medium/file/9986983/8431631593768139653.jpg"], source.image_urls) @@ -20,7 +20,7 @@ module Sources should "work for an image url without a referer" do # page: https://sketch.pixiv.net/items/8052785510155853613 - source = Sources::Strategies.find("https://img-sketch.pixiv.net/uploads/medium/file/9988973/7216948861306830496.jpg") + source = Source::Extractor.find("https://img-sketch.pixiv.net/uploads/medium/file/9988973/7216948861306830496.jpg") assert_equal(["https://img-sketch.pixiv.net/uploads/medium/file/9988973/7216948861306830496.jpg"], source.image_urls) assert_nil(source.page_url) @@ -34,7 +34,7 @@ module Sources end should "work for an image url with a referer" do - source = Sources::Strategies.find("https://img-sketch.pixiv.net/uploads/medium/file/9988973/7216948861306830496.jpg", "https://sketch.pixiv.net/items/8052785510155853613") + source = Source::Extractor.find("https://img-sketch.pixiv.net/uploads/medium/file/9988973/7216948861306830496.jpg", "https://sketch.pixiv.net/items/8052785510155853613") assert_equal("https://sketch.pixiv.net/items/8052785510155853613", source.page_url) assert_equal("https://sketch.pixiv.net/@op-one", source.profile_url) @@ -47,7 +47,7 @@ module Sources end should "work for a NSFW post" do - source = Sources::Strategies.find("https://sketch.pixiv.net/items/193462611994864256") + source = Source::Extractor.find("https://sketch.pixiv.net/items/193462611994864256") assert_equal(["https://img-sketch.pixiv.net/uploads/medium/file/884876/4909517173982299587.jpg"], source.image_urls) assert_equal("https://sketch.pixiv.net/items/193462611994864256", source.page_url) @@ -61,7 +61,7 @@ module Sources end should "work for a post with a multiple images" do - source = Sources::Strategies.find("https://sketch.pixiv.net/items/8052785510155853613") + source = Source::Extractor.find("https://sketch.pixiv.net/items/8052785510155853613") assert_equal(%w[ https://img-sketch.pixiv.net/uploads/medium/file/9988964/1564052114639195387.png diff --git a/test/unit/sources/pixiv_test.rb b/test/unit/sources/pixiv_test.rb index b40c31289..aebf0d8bc 100644 --- a/test/unit/sources/pixiv_test.rb +++ b/test/unit/sources/pixiv_test.rb @@ -3,29 +3,29 @@ require 'test_helper' module Sources class PixivTest < ActiveSupport::TestCase setup do - skip "Pixiv credentials not configured" unless Sources::Strategies::Pixiv.enabled? + skip "Pixiv credentials not configured" unless Source::Extractor::Pixiv.enabled? end def assert_illust_id(illust_id, url) - site = Sources::Strategies.find(url) + site = Source::Extractor.find(url) assert_equal(illust_id, site.illust_id.to_i) assert_nothing_raised { site.to_h } end def assert_nil_illust_id(url) - site = Sources::Strategies.find(url) + site = Source::Extractor.find(url) assert_nil(site.illust_id) end def get_source(source) - @site = Sources::Strategies.find(source) + @site = Source::Extractor.find(source) @site end context "in all cases" do context "A gallery page" do setup do - @site = Sources::Strategies.find("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=49270482") + @site = Source::Extractor.find("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=49270482") @image_urls = @site.image_urls end @@ -36,7 +36,7 @@ module Sources context "An ugoira source site for pixiv" do setup do - @site = Sources::Strategies.find("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364") + @site = Source::Extractor.find("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364") end should "get the file url" do @@ -53,7 +53,7 @@ module Sources context "A https://i.pximg.net/img-zip/ugoira/* source" do should "get the metadata" do - @site = Sources::Strategies.find("https://i.pximg.net/img-zip-ugoira/img/2017/04/04/08/57/38/62247364_ugoira1920x1080.zip") + @site = Source::Extractor.find("https://i.pximg.net/img-zip-ugoira/img/2017/04/04/08/57/38/62247364_ugoira1920x1080.zip") assert_equal("uroobnad2", @site.artist_name) end @@ -61,7 +61,7 @@ module Sources context "A https://tc-pximg01.techorus-cdn.com/img-original/img/* source" do should "get the metadata" do - @site = Sources::Strategies.find("https://tc-pximg01.techorus-cdn.com/img-original/img/2017/09/18/03/18/24/65015428_p4.png") + @site = Source::Extractor.find("https://tc-pximg01.techorus-cdn.com/img-original/img/2017/09/18/03/18/24/65015428_p4.png") assert_equal(["https://i.pximg.net/img-original/img/2017/09/18/03/18/24/65015428_p4.png"], @site.image_urls) assert_equal("赤井さしみ", @site.artist_name) @@ -70,12 +70,12 @@ module Sources context "A https://www.pixiv.net/*/artworks/* source" do should "work" do - @site = Sources::Strategies.find("https://www.pixiv.net/en/artworks/64476642") + @site = Source::Extractor.find("https://www.pixiv.net/en/artworks/64476642") assert_equal(["https://i.pximg.net/img-original/img/2017/08/18/00/09/21/64476642_p0.jpg"], @site.image_urls) assert_equal("https://www.pixiv.net/artworks/64476642", @site.page_url) - @site = Sources::Strategies.find("https://www.pixiv.net/artworks/64476642") + @site = Source::Extractor.find("https://www.pixiv.net/artworks/64476642") assert_equal(["https://i.pximg.net/img-original/img/2017/08/18/00/09/21/64476642_p0.jpg"], @site.image_urls) assert_equal("https://www.pixiv.net/artworks/64476642", @site.page_url) end diff --git a/test/unit/sources/plurk_test.rb b/test/unit/sources/plurk_test.rb index 9749a4eeb..e5a926c55 100644 --- a/test/unit/sources/plurk_test.rb +++ b/test/unit/sources/plurk_test.rb @@ -8,10 +8,10 @@ module Sources @adult_post_url = "https://www.plurk.com/p/omc64y" @image_url = "https://images.plurk.com/5wj6WD0r6y4rLN0DL3sqag.jpg" @profile_url = "https://www.plurk.com/redeyehare" - @post1 = Sources::Strategies.find(@post_url) - @post2 = Sources::Strategies.find(@image_url) - @post3 = Sources::Strategies.find(@profile_url) - @post4 = Sources::Strategies.find(@adult_post_url) + @post1 = Source::Extractor.find(@post_url) + @post2 = Source::Extractor.find(@image_url) + @post3 = Source::Extractor.find(@profile_url) + @post4 = Source::Extractor.find(@adult_post_url) end should "not raise errors" do diff --git a/test/unit/sources/skeb_test.rb b/test/unit/sources/skeb_test.rb index 45b1414ed..28e369c26 100644 --- a/test/unit/sources/skeb_test.rb +++ b/test/unit/sources/skeb_test.rb @@ -4,7 +4,7 @@ module Sources class SkebTest < ActiveSupport::TestCase context "The source for a skeb picture" do setup do - @site = Sources::Strategies.find("https://skeb.jp/@kokuzou593/works/45") + @site = Source::Extractor.find("https://skeb.jp/@kokuzou593/works/45") end should "get the artist name" do @@ -35,7 +35,7 @@ module Sources context "A private or non-existent skeb url" do setup do - @site = Sources::Strategies.find("https://skeb.jp/@kai_chiisame/works/2") + @site = Source::Extractor.find("https://skeb.jp/@kai_chiisame/works/2") end should "not raise anything" do @@ -50,14 +50,14 @@ module Sources context "A post with a smaller unwatermarked version" do should "get the smaller but clean picture" do - site = Sources::Strategies.find("https://skeb.jp/@2gi0gi_/works/13") + site = Source::Extractor.find("https://skeb.jp/@2gi0gi_/works/13") assert_equal(["https://skeb.imgix.net/requests/191942_0?bg=%23fff&fm=jpg&q=45&w=696&s=5783ee951cc55d183713395926389453"], site.image_urls) end end context "An animated post with a smaller static unwatermarked version" do should "still get the watermarked gif" do - site = Sources::Strategies.find("https://skeb.jp/@tontaro_/works/316") + site = Source::Extractor.find("https://skeb.jp/@tontaro_/works/316") assert_equal(%w[ https://skeb.imgix.net/uploads/origins/5097b1e1-18ce-418e-82f0-e7e2cdab1cea?bg=%23fff&auto=format&txtfont=bold&txtshad=70&txtclr=BFFFFFFF&txtalign=middle%2Ccenter&txtsize=150&txt=SAMPLE&fm=mp4&w=800&s=fcff06871e114b3dbf505c04f27b5ed1 https://skeb.imgix.net/uploads/origins/23123cfd-9b03-40f6-a8ae-7d74f9118c6f?bg=%23fff&auto=format&txtfont=bold&txtshad=70&txtclr=BFFFFFFF&txtalign=middle%2Ccenter&txtsize=150&txt=SAMPLE&fm=mp4&w=800&s=984626d69b45c040d295e357a67f281e @@ -68,14 +68,14 @@ module Sources context "A post with both the small and large version clean" do should "just get the bigger image" do - site = Sources::Strategies.find("https://skeb.jp/@goma_feet/works/1") + site = Source::Extractor.find("https://skeb.jp/@goma_feet/works/1") assert_equal(["https://skeb.imgix.net/uploads/origins/78ca23dc-a053-4ebe-894f-d5a06e228af8?bg=%23fff&auto=format&w=800&s=3de55b04236059113659f99fd6900d7d"], site.image_urls) end end context "A post with two images" do should "get both correctly and in the right order" do - site = Sources::Strategies.find("https://skeb.jp/@LambOic029/works/146") + site = Source::Extractor.find("https://skeb.jp/@LambOic029/works/146") image_urls = %w[ https://skeb.imgix.net/uploads/origins/3fc062c5-231d-400f-921f-22d77cde54df?bg=%23fff&auto=format&txtfont=bold&txtshad=70&txtclr=BFFFFFFF&txtalign=middle%2Ccenter&txtsize=150&txt=SAMPLE&w=800&s=80a1373b3f8e9bf0108d201fba34de71 https://skeb.imgix.net/uploads/origins/e888bb27-e1a6-48ec-a317-7615252ff818?bg=%23fff&auto=format&txtfont=bold&txtshad=70&txtclr=BFFFFFFF&txtalign=middle%2Ccenter&txtsize=150&txt=SAMPLE&w=800&s=9df9b46bbfad404d3a65c7c56b0cbf40 @@ -87,14 +87,14 @@ module Sources context "A post with a video" do should "get it correctly" do - site = Sources::Strategies.find("https://skeb.jp/@kaisouafuro/works/112") + site = Source::Extractor.find("https://skeb.jp/@kaisouafuro/works/112") assert_match(%r{\Ahttps://skeb-production.s3.ap-northeast-1.amazonaws.com/uploads/outputs/20f9d68f-50ec-44ae-8630-173fc38a2d6a\?response-content-disposition=attachment%3B%20filename%3D%22458093-1.output.mp4%22%3B%20filename%2A%3DUTF-8%27%27458093-1.output.mp4&response-content-type=video%2Fmp4&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=.*&X-Amz-Date=.*&X-Amz-Expires=604800&X-Amz-SignedHeaders=host&X-Amz-Signature=.*}, site.image_urls.sole) end end context "A post with both original and autotranslated commentary" do should "get the original commentary" do - site = Sources::Strategies.find("https://skeb.jp/@kaisouafuro/works/112") + site = Source::Extractor.find("https://skeb.jp/@kaisouafuro/works/112") assert_match(/I would like to request an animation screen for my Twitch channel. My character is a catgirl/, site.dtext_artist_commentary_desc) end end diff --git a/test/unit/sources/stash_test.rb b/test/unit/sources/stash_test.rb index d3b39f561..e680ad5e9 100644 --- a/test/unit/sources/stash_test.rb +++ b/test/unit/sources/stash_test.rb @@ -9,7 +9,7 @@ module Sources context "A https://sta.sh/:id page url" do should "work" do - @site = Sources::Strategies.find("https://sta.sh/0wxs31o7nn2") + @site = Source::Extractor.find("https://sta.sh/0wxs31o7nn2") assert_equal("noizave", @site.artist_name) assert_equal("https://www.deviantart.com/noizave", @site.profile_url) @@ -25,7 +25,7 @@ module Sources context "A https://orig00.deviantart.net/* image url" do context "with a https://sta.sh/:id referer" do should "work" do - @site = Sources::Strategies.find("https://orig00.deviantart.net/0fd2/f/2018/252/9/c/a_pepe_by_noizave-dcmga0s.png", "https://sta.sh/0wxs31o7nn2") + @site = Source::Extractor.find("https://orig00.deviantart.net/0fd2/f/2018/252/9/c/a_pepe_by_noizave-dcmga0s.png", "https://sta.sh/0wxs31o7nn2") assert_equal("noizave", @site.artist_name) assert_equal("https://www.deviantart.com/noizave", @site.profile_url) @@ -40,7 +40,7 @@ module Sources context "without a referer" do should "use the base deviantart strategy" do - @site = Sources::Strategies.find("https://orig00.deviantart.net/0fd2/f/2018/252/9/c/a_pepe_by_noizave-dcmga0s.png") + @site = Source::Extractor.find("https://orig00.deviantart.net/0fd2/f/2018/252/9/c/a_pepe_by_noizave-dcmga0s.png") # if all we have is the image url, then we can't tell that this is really a sta.sh image. assert_equal("Deviant Art", @site.site_name) diff --git a/test/unit/sources/tinami_test.rb b/test/unit/sources/tinami_test.rb index 97fe6e5cf..1badad8c8 100644 --- a/test/unit/sources/tinami_test.rb +++ b/test/unit/sources/tinami_test.rb @@ -5,7 +5,7 @@ module Sources context "Tinami:" do context "A 'http://www.tinami.com/view/:id' post with one image" do should "work" do - source = Sources::Strategies.find("http://www.tinami.com/view/1087268") + source = Source::Extractor.find("http://www.tinami.com/view/1087268") assert_equal("みぐめ", source.artist_name) assert_equal("https://www.tinami.com/view/1087268", source.page_url) @@ -19,7 +19,7 @@ module Sources context "A 'http://www.tinami.com/view/:id' post with multiple images (type one)" do should "work" do - source = Sources::Strategies.find("http://www.tinami.com/view/1087271") + source = Source::Extractor.find("http://www.tinami.com/view/1087271") assert_equal("Shimaken", source.artist_name) assert_equal("https://www.tinami.com/view/1087271", source.page_url) @@ -37,7 +37,7 @@ module Sources context "A 'http://www.tinami.com/view/:id' post with multiple images (type two)" do should "work" do - source = Sources::Strategies.find("http://www.tinami.com/view/1087270") + source = Source::Extractor.find("http://www.tinami.com/view/1087270") assert_equal("セラ箱", source.artist_name) assert_equal("https://www.tinami.com/view/1087270", source.page_url) @@ -56,7 +56,7 @@ module Sources assert_equal(<<~EOS.chomp, source.artist_commentary_desc) リゼロのレムのプライズをクリアドレス仕様にリペイント。透け透けキラキラな感じに改装してみたものです。 - >https://youtu.be/nkjZkEALg94 + >https://youtu.be/nkjZkEALg94 製作日記的な動画です( ´∀` ) @@ -67,7 +67,7 @@ module Sources context "A Tinami image URL without a referer" do should "work" do - source = Sources::Strategies.find("https://img.tinami.com/illust2/img/647/6234fe5588e97.jpg") + source = Source::Extractor.find("https://img.tinami.com/illust2/img/647/6234fe5588e97.jpg") assert_nil(source.artist_name) assert_nil(source.page_url) @@ -81,7 +81,7 @@ module Sources context "A Tinami image URL with a referer" do should "work" do - source = Sources::Strategies.find("https://img.tinami.com/illust2/img/647/6234fe5588e97.jpg", "http://www.tinami.com/view/1087268") + source = Source::Extractor.find("https://img.tinami.com/illust2/img/647/6234fe5588e97.jpg", "http://www.tinami.com/view/1087268") assert_equal("みぐめ", source.artist_name) assert_equal("https://www.tinami.com/view/1087268", source.page_url) @@ -95,7 +95,7 @@ module Sources context "A deleted Tinami post" do should "work" do - source = Sources::Strategies.find("http://www.tinami.com/view/774077") + source = Source::Extractor.find("http://www.tinami.com/view/774077") assert_nil(source.artist_name) assert_equal("https://www.tinami.com/view/774077", source.page_url) diff --git a/test/unit/sources/tumblr_test.rb b/test/unit/sources/tumblr_test.rb index a9fd4ab4b..f847628b0 100644 --- a/test/unit/sources/tumblr_test.rb +++ b/test/unit/sources/tumblr_test.rb @@ -3,12 +3,12 @@ require "test_helper" module Sources class TumblrTest < ActiveSupport::TestCase def setup - skip "Tumblr key is not configured" unless Sources::Strategies::Tumblr.enabled? + skip "Tumblr key is not configured" unless Source::Extractor::Tumblr.enabled? end context "The source for a 'http://*.tumblr.com/post/*' photo post with a single image" do setup do - @site = Sources::Strategies.find("https://noizave.tumblr.com/post/162206271767") + @site = Source::Extractor.find("https://noizave.tumblr.com/post/162206271767") end should "get the artist name" do @@ -85,7 +85,7 @@ module Sources context "The source for a 'http://*.tumblr.com/image/*' image page" do setup do - @site = Sources::Strategies.find("https://noizave.tumblr.com/image/162206271767") + @site = Source::Extractor.find("https://noizave.tumblr.com/image/162206271767") end should "get the image url" do @@ -111,7 +111,7 @@ module Sources context "with a referer" do should "get all the metadata" do - site = Sources::Strategies.find(@url, @ref) + site = Source::Extractor.find(@url, @ref) assert_equal("noizave", site.artist_name) assert_equal("https://noizave.tumblr.com", site.profile_url) @@ -123,7 +123,7 @@ module Sources context "without a referer" do should "still find all the relevant information" do - site = Sources::Strategies.find(@url) + site = Source::Extractor.find(@url) assert_equal("noizave", site.artist_name) assert_equal("https://noizave.tumblr.com", site.profile_url) @@ -136,7 +136,7 @@ module Sources context "The source for a 'http://*.tumblr.com/post/*' text post with inline images" do setup do - @site = Sources::Strategies.find("https://noizave.tumblr.com/post/162221502947") + @site = Source::Extractor.find("https://noizave.tumblr.com/post/162221502947") end should "get the image urls" do @@ -158,7 +158,7 @@ module Sources context "A video post with inline images" do should "get the video and inline images" do url = "https://noizave.tumblr.com/post/162222617101" - site = Sources::Strategies.find(url) + site = Source::Extractor.find(url) urls = %w[ https://va.media.tumblr.com/tumblr_os31dkexhK1wsfqep.mp4 https://media.tumblr.com/afed9f5b3c33c39dc8c967e262955de2/tumblr_inline_os31dclyCR1v11u29_1280.png @@ -171,7 +171,7 @@ module Sources context "The source for a 'http://*.tumblr.com/post/*' answer post with inline images" do setup do - @site = Sources::Strategies.find("https://noizave.tumblr.com/post/171237880542/test-ask") + @site = Source::Extractor.find("https://noizave.tumblr.com/post/171237880542/test-ask") end should "get the image urls" do @@ -193,7 +193,7 @@ module Sources should "return the correct image url" do image_url = "https://64.media.tumblr.com/3dfdab77d913ad1ea59f22407d6ac6f3/b1764aa0f9c378d0-23/s1280x1920/46f4af7ec94456f8fef380ee6311eb81178ce7e9.jpg" page_url = "https://make-do5.tumblr.com/post/619663949657423872" - strategy = Sources::Strategies.find(image_url, page_url) + strategy = Source::Extractor.find(image_url, page_url) assert_match(%r{/3dfdab77d913ad1ea59f22407d6ac6f3/b1764aa0f9c378d0-23/s\d+x\d+/}i, image_url) assert_equal(page_url, strategy.page_url) @@ -203,7 +203,7 @@ module Sources context "A deleted tumblr post" do should "extract the info from the url" do - site = Sources::Strategies.find("http://shimetsukage.tumblr.com/post/176805588268/20180809-ssb-coolboy") + site = Source::Extractor.find("http://shimetsukage.tumblr.com/post/176805588268/20180809-ssb-coolboy") assert_nothing_raised { site.to_h } assert_equal("shimetsukage", site.artist_name) @@ -220,7 +220,7 @@ module Sources page = "https://natsuki-teru.tumblr.com/post/178728919271" image = "https://66.media.tumblr.com/b9395771b2d0435fe4efee926a5a7d9c/tumblr_pg2wu1L9DM1trd056o2_#{size}.png" full = "https://media.tumblr.com/b9395771b2d0435fe4efee926a5a7d9c/tumblr_pg2wu1L9DM1trd056o2_1280.png" - site = Sources::Strategies.find(image, page) + site = Source::Extractor.find(image, page) assert_equal([full], site.image_urls) end diff --git a/test/unit/sources/twitter_test.rb b/test/unit/sources/twitter_test.rb index 51c927709..ce76faa55 100644 --- a/test/unit/sources/twitter_test.rb +++ b/test/unit/sources/twitter_test.rb @@ -3,17 +3,17 @@ require 'test_helper' module Sources class TwitterTest < ActiveSupport::TestCase setup do - skip "Twitter credentials are not configured" if !Sources::Strategies::Twitter.enabled? + skip "Twitter credentials are not configured" if !Source::Extractor::Twitter.enabled? end context "An extended tweet" do should "extract the correct image url" do - @site = Sources::Strategies.find("https://twitter.com/onsen_musume_jp/status/865534101918330881") + @site = Source::Extractor.find("https://twitter.com/onsen_musume_jp/status/865534101918330881") assert_equal(["https://pbs.twimg.com/media/DAL-ntWV0AEbhes.jpg:orig"], @site.image_urls) end should "extract all the image urls" do - @site = Sources::Strategies.find("https://twitter.com/baalbuddy/status/1455330043828264963") + @site = Source::Extractor.find("https://twitter.com/baalbuddy/status/1455330043828264963") urls = %w[ https://pbs.twimg.com/media/FDJekEfX0AQZ-Mx.png:orig @@ -28,33 +28,33 @@ module Sources context "A video" do should "get the correct urls" do - @site = Sources::Strategies.find("https://twitter.com/CincinnatiZoo/status/859073537713328129") + @site = Source::Extractor.find("https://twitter.com/CincinnatiZoo/status/859073537713328129") assert_equal(["https://video.twimg.com/ext_tw_video/859073467769126913/pu/vid/1280x720/cPGgVROXHy3yrK6u.mp4"], @site.image_urls) assert_equal("https://twitter.com/CincinnatiZoo/status/859073537713328129", @site.page_url) end should "work when given a video thumbnail" do # https://twitter.com/Kekeflipnote/status/1241038667898118144 - @site = Sources::Strategies.find("https://pbs.twimg.com/tweet_video_thumb/ETkN_L3X0AMy1aT.jpg:small") + @site = Source::Extractor.find("https://pbs.twimg.com/tweet_video_thumb/ETkN_L3X0AMy1aT.jpg:small") assert_equal(["https://pbs.twimg.com/tweet_video_thumb/ETkN_L3X0AMy1aT.jpg:orig"], @site.image_urls) end should "work when given an external video thumbnail" do # https://twitter.com/chivedips/status/1243850897056133121 - @site = Sources::Strategies.find("https://pbs.twimg.com/ext_tw_video_thumb/1243725361986375680/pu/img/JDA7g7lcw7wK-PIv.jpg:small") + @site = Source::Extractor.find("https://pbs.twimg.com/ext_tw_video_thumb/1243725361986375680/pu/img/JDA7g7lcw7wK-PIv.jpg:small") assert_equal(["https://pbs.twimg.com/ext_tw_video_thumb/1243725361986375680/pu/img/JDA7g7lcw7wK-PIv.jpg:orig"], @site.image_urls) end should "work when given an amplify video thumbnail" do # https://twitter.com/UNITED_CINEMAS/status/1223138847417978881 - @site = Sources::Strategies.find("https://pbs.twimg.com/amplify_video_thumb/1215590775364259840/img/lolCkEEioFZTb5dl.jpg:small") + @site = Source::Extractor.find("https://pbs.twimg.com/amplify_video_thumb/1215590775364259840/img/lolCkEEioFZTb5dl.jpg:small") assert_equal(["https://pbs.twimg.com/amplify_video_thumb/1215590775364259840/img/lolCkEEioFZTb5dl.jpg:orig"], @site.image_urls) end end context "An animated gif" do setup do - @site = Sources::Strategies.find("https://twitter.com/i/web/status/1252517866059907073") + @site = Source::Extractor.find("https://twitter.com/i/web/status/1252517866059907073") end should "get the image url" do @@ -64,7 +64,7 @@ module Sources context "A twitter summary card from twitter with a :large image" do setup do - @site = Sources::Strategies.find("https://twitter.com/aranobu/status/817736083567820800") + @site = Source::Extractor.find("https://twitter.com/aranobu/status/817736083567820800") end should "get the image url" do @@ -78,7 +78,7 @@ module Sources context "The source site for a restricted twitter" do setup do - @site = Sources::Strategies.find("https://mobile.twitter.com/Strangestone/status/556440271961858051") + @site = Source::Extractor.find("https://mobile.twitter.com/Strangestone/status/556440271961858051") end should "get the urls" do @@ -89,7 +89,7 @@ module Sources context "A tweet without any images" do should "not fail" do - @site = Sources::Strategies.find("https://twitter.com/teruyo/status/1058452066060853248") + @site = Source::Extractor.find("https://twitter.com/teruyo/status/1058452066060853248") assert_equal([], @site.image_urls) assert_nothing_raised { @site.to_h } @@ -98,7 +98,7 @@ module Sources context "The source site for twitter" do setup do - @site = Sources::Strategies.find("https://mobile.twitter.com/nounproject/status/540944400767922176") + @site = Source::Extractor.find("https://mobile.twitter.com/nounproject/status/540944400767922176") end should "get the main profile url" do @@ -139,7 +139,7 @@ module Sources context "The source site for a direct image and a referer" do setup do - @site = Sources::Strategies.find("https://pbs.twimg.com/media/B4HSEP5CUAA4xyu.png:large", "https://twitter.com/nounproject/status/540944400767922176") + @site = Source::Extractor.find("https://pbs.twimg.com/media/B4HSEP5CUAA4xyu.png:large", "https://twitter.com/nounproject/status/540944400767922176") end should "get the source data" do @@ -151,7 +151,7 @@ module Sources context "The source site for a direct image url (pbs.twimg.com/media/*.jpg) without a referer url" do setup do - @site = Sources::Strategies.find("https://pbs.twimg.com/media/B4HSEP5CUAA4xyu.png:large") + @site = Source::Extractor.find("https://pbs.twimg.com/media/B4HSEP5CUAA4xyu.png:large") end should "work" do @@ -168,7 +168,7 @@ module Sources context "The source site for a direct image url (pbs.twimg.com/media/*?format=jpg&name=*) without a referer url" do setup do - @site = Sources::Strategies.find("https://pbs.twimg.com/media/EBGp2YdUYAA19Uj?format=jpg&name=small") + @site = Source::Extractor.find("https://pbs.twimg.com/media/EBGp2YdUYAA19Uj?format=jpg&name=small") end should "work" do @@ -176,14 +176,14 @@ module Sources end should "work for filenames containing dashes" do - @site = Sources::Strategies.find("https://pbs.twimg.com/media/EAjc-OWVAAAxAgQ.jpg", "https://twitter.com/asteroid_ill/status/1155420330128625664") + @site = Source::Extractor.find("https://pbs.twimg.com/media/EAjc-OWVAAAxAgQ.jpg", "https://twitter.com/asteroid_ill/status/1155420330128625664") assert_equal(["https://pbs.twimg.com/media/EAjc-OWVAAAxAgQ.jpg:orig"], @site.image_urls) end end context "The source site for a https://twitter.com/i/web/status/:id url" do setup do - @site = Sources::Strategies.find("https://twitter.com/i/web/status/943446161586733056") + @site = Source::Extractor.find("https://twitter.com/i/web/status/943446161586733056") end should "fetch the source data" do @@ -197,7 +197,7 @@ module Sources context "A deleted tweet" do should "still find the artist name" do - @site = Sources::Strategies.find("https://twitter.com/masayasuf/status/870734961778630656") + @site = Source::Extractor.find("https://twitter.com/masayasuf/status/870734961778630656") @artist = FactoryBot.create(:artist, name: "masayasuf", url_string: @site.url) assert_equal("masayasuf", @site.tag_name) @@ -208,7 +208,7 @@ module Sources context "A tweet" do setup do - @site = Sources::Strategies.find("https://twitter.com/noizave/status/875768175136317440") + @site = Source::Extractor.find("https://twitter.com/noizave/status/875768175136317440") end should "convert urls, hashtags, and mentions to dtext" do @@ -228,7 +228,7 @@ module Sources context "A profile banner image" do should "work" do - @site = Sources::Strategies.find("https://pbs.twimg.com/profile_banners/1225702850002468864/1588597370/1500x500") + @site = Source::Extractor.find("https://pbs.twimg.com/profile_banners/1225702850002468864/1588597370/1500x500") assert_equal([@site.url], @site.image_urls) assert_nothing_raised { @site.to_h } end @@ -236,7 +236,7 @@ module Sources context "A tweet containing non-normalized Unicode text" do should "be normalized to nfkc" do - site = Sources::Strategies.find("https://twitter.com/aprilarcus/status/367557195186970624") + site = Source::Extractor.find("https://twitter.com/aprilarcus/status/367557195186970624") desc1 = "𝖸𝗈 𝐔𝐧𝐢𝐜𝐨𝐝𝐞 𝗅 𝗁𝖾𝗋𝖽 𝕌 𝗅𝗂𝗄𝖾 𝑡𝑦𝑝𝑒𝑓𝑎𝑐𝑒𝑠 𝗌𝗈 𝗐𝖾 𝗉𝗎𝗍 𝗌𝗈𝗆𝖾 𝚌𝚘𝚍𝚎𝚙𝚘𝚒𝚗𝚝𝚜 𝗂𝗇 𝗒𝗈𝗎𝗋 𝔖𝔲𝔭𝔭𝔩𝔢𝔪𝔢𝔫𝔱𝔞𝔯𝔶 𝔚𝔲𝔩𝔱𝔦𝔩𝔦𝔫𝔤𝔳𝔞𝔩 𝔓𝔩𝔞𝔫𝔢 𝗌𝗈 𝗒𝗈𝗎 𝖼𝖺𝗇 𝓮𝓷𝓬𝓸𝓭𝓮 𝕗𝕠𝕟𝕥𝕤 𝗂𝗇 𝗒𝗈𝗎𝗋 𝒇𝒐𝒏𝒕𝒔." desc2 = "Yo Unicode l herd U like typefaces so we put some codepoints in your Supplementary Wultilingval Plane so you can encode fonts in your fonts." @@ -245,7 +245,7 @@ module Sources end should "normalize full-width hashtags" do - site = Sources::Strategies.find("https://twitter.com/corpsmanWelt/status/1037724260075069441") + site = Source::Extractor.find("https://twitter.com/corpsmanWelt/status/1037724260075069441") desc1 = %{新しいおともだち\n#けものフレンズ https://t.co/sEAuu16yAQ} desc2 = %{新しいおともだち\n"#けものフレンズ":[https://twitter.com/hashtag/けものフレンズ]} @@ -256,7 +256,7 @@ module Sources context "A twitter post with a pixiv referer" do should "use the twitter strategy" do - site = Sources::Strategies.find("https://twitter.com/Mityubi/status/849630665603665920", "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=56735489") + site = Source::Extractor.find("https://twitter.com/Mityubi/status/849630665603665920", "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=56735489") assert_equal(site.site_name, "Twitter") assert_equal(["https://pbs.twimg.com/media/C8p-gPhVoAMZupS.png:orig"], site.image_urls) @@ -265,7 +265,7 @@ module Sources context "A tweet from a suspended user" do should "not fail" do - site = Sources::Strategies.find("https://twitter.com/tanso_panz/status/1192429800717029377") + site = Source::Extractor.find("https://twitter.com/tanso_panz/status/1192429800717029377") assert_equal(site.site_name, "Twitter") assert_equal("tanso_panz", site.tag_name) @@ -281,7 +281,7 @@ module Sources create(:wiki_page, title: "nishizumi_miho", other_names: "西住みほ") end - site = Sources::Strategies.find("https://twitter.com/kasaishin100/status/1186658635226607616") + site = Source::Extractor.find("https://twitter.com/kasaishin100/status/1186658635226607616") assert_includes(site.tags.map(&:first), "西住みほ生誕祭2019") assert_includes(site.normalized_tags, "西住みほ") diff --git a/test/unit/sources/weibo_test.rb b/test/unit/sources/weibo_test.rb index e6c860b79..84507ebcd 100644 --- a/test/unit/sources/weibo_test.rb +++ b/test/unit/sources/weibo_test.rb @@ -9,7 +9,7 @@ module Sources context "A post with multiple pictures" do setup do - @site = Sources::Strategies.find("https://www.weibo.com/5501756072/J2UNKfbqV?type=comment#_rnd1590548401855") + @site = Source::Extractor.find("https://www.weibo.com/5501756072/J2UNKfbqV?type=comment#_rnd1590548401855") end should "extract all the image urls" do @@ -55,7 +55,7 @@ module Sources context "A deleted or not existing picture" do should "still find the artist name" do - site = Sources::Strategies.find("https://www.weibo.com/5501756072/AsdAsdAsd") + site = Source::Extractor.find("https://www.weibo.com/5501756072/AsdAsdAsd") artist = FactoryBot.create(:artist, name: "nipi27", url_string: "https://www.weibo.com/u/5501756072") assert_equal([artist], site.artists) @@ -64,7 +64,7 @@ module Sources context "A post with video" do should "get the correct video" do - site = Sources::Strategies.find("https://www.weibo.com/5501756072/IF9fugHzj") + site = Source::Extractor.find("https://www.weibo.com/5501756072/IF9fugHzj") assert_downloaded(7_676_656, site.image_urls.sole) end @@ -72,7 +72,7 @@ module Sources context "A direct image sample upload" do should "get the largest version" do - sample = Sources::Strategies.find("https://wx3.sinaimg.cn/mw690/a00fa34cly1gf62g2n8z3j21yu2jo1ky.jpg") + sample = Source::Extractor.find("https://wx3.sinaimg.cn/mw690/a00fa34cly1gf62g2n8z3j21yu2jo1ky.jpg") assert_equal(["https://wx3.sinaimg.cn/large/a00fa34cly1gf62g2n8z3j21yu2jo1ky.jpg"], sample.image_urls) end @@ -82,7 +82,7 @@ module Sources should "get the page url" do url = "https://wx1.sinaimg.cn/large/7eb64558gy1fnbryriihwj20dw104wtu.jpg" ref = "https://photo.weibo.com/2125874520/wbphotos/large/mid/4194742441135220/pid/7eb64558gy1fnbryb5nzoj20dw10419t" - site = Sources::Strategies.find(url, ref) + site = Source::Extractor.find(url, ref) assert_equal("https://www.weibo.com/2125874520/FDKGo4Lk0", site.page_url) end @@ -91,13 +91,13 @@ module Sources context "A deleted url" do should "not raise errors" do url = "https://weibo.com/5265069929/LiLnMENgs" - assert_nothing_raised { Sources::Strategies.find(url).to_h } + assert_nothing_raised { Source::Extractor.find(url).to_h } end end context "A m.weibo.cn/detail url" do should "work" do - @site = Sources::Strategies.find("https://m.weibo.cn/detail/4506950043618873") + @site = Source::Extractor.find("https://m.weibo.cn/detail/4506950043618873") assert_equal(%w[ https://wx1.sinaimg.cn/large/0060kO5aly1gezsyt5xvhj30ok0sgtc9.jpg