From 307df3b3e4dda7d446b4c856184111952979e11b Mon Sep 17 00:00:00 2001 From: nonamethanks Date: Sat, 16 May 2020 23:03:09 +0200 Subject: [PATCH] Refactor source normalization * Move the source normalization logic out of the post model and into individual sources' strategies. * Rewrite normalization tests to be handled into each source's test, and expand them significantly. Previously we were only testing a very small subset of domains and variants. * Fix up normalization for several sites. * Normalize fav.me urls into normal deviantart urls. --- app/logical/sources/strategies.rb | 4 + app/logical/sources/strategies/art_station.rb | 13 +- app/logical/sources/strategies/base.rb | 6 + app/logical/sources/strategies/deviant_art.rb | 96 +++++---- .../sources/strategies/hentai_foundry.rb | 24 ++- app/logical/sources/strategies/moebooru.rb | 14 +- app/logical/sources/strategies/nico_seiga.rb | 26 ++- app/logical/sources/strategies/nijie.rb | 6 + app/logical/sources/strategies/null.rb | 111 +++++++++++ app/logical/sources/strategies/pawoo.rb | 57 +++++- app/logical/sources/strategies/pixiv.rb | 6 + app/logical/sources/strategies/tumblr.rb | 8 + app/logical/sources/strategies/twitter.rb | 21 +- app/models/post.rb | 182 +----------------- test/unit/post_test.rb | 54 ------ test/unit/sources/art_station_test.rb | 19 ++ test/unit/sources/deviant_art_test.rb | 29 ++- test/unit/sources/hentai_foundry_test.rb | 17 ++ test/unit/sources/moebooru_test.rb | 43 ++--- test/unit/sources/nico_seiga_test.rb | 19 ++ test/unit/sources/nijie_test.rb | 20 ++ test/unit/sources/null_test.rb | 142 +++++++++++++- test/unit/sources/pawoo_test.rb | 18 ++ test/unit/sources/pixiv_test.rb | 16 ++ test/unit/sources/tumblr_test.rb | 19 ++ test/unit/sources/twitter_test.rb | 19 ++ 26 files changed, 674 insertions(+), 315 deletions(-) diff --git a/app/logical/sources/strategies.rb b/app/logical/sources/strategies.rb index 7f2b3be98..170b4849f 100644 --- a/app/logical/sources/strategies.rb +++ b/app/logical/sources/strategies.rb @@ -24,5 +24,9 @@ module Sources def self.canonical(url, referer) find(url, referer).canonical_url end + + def self.normalize_source(url) + find(url).normalize_for_source || url + end end end diff --git a/app/logical/sources/strategies/art_station.rb b/app/logical/sources/strategies/art_station.rb index 1c7d1ba09..30f2b5744 100644 --- a/app/logical/sources/strategies/art_station.rb +++ b/app/logical/sources/strategies/art_station.rb @@ -3,6 +3,7 @@ # * https://www.artstation.com/artwork/04XA4 # * https://www.artstation.com/artwork/cody-from-sf # * https://sa-dui.artstation.com/projects/DVERn +# * https://dudeunderscore.artstation.com/projects/NoNmD?album_id=23041 # # Profile URLs: # @@ -20,7 +21,7 @@ module Sources::Strategies class ArtStation < Base PROJECT1 = %r!\Ahttps?://www\.artstation\.com/artwork/(?[a-z0-9-]+)/?\z!i - PROJECT2 = %r!\Ahttps?://(?[\w-]+)\.artstation\.com/projects/(?[a-z0-9-]+)/?\z!i + PROJECT2 = %r!\Ahttps?://(?[\w-]+)\.artstation\.com/projects/(?[a-z0-9-]+)(?:/|\?[\w=-]+)?\z!i PROJECT = Regexp.union(PROJECT1, PROJECT2) ARTIST1 = %r{\Ahttps?://(?[\w-]+)(?[\w-]+)/?\z}i @@ -84,6 +85,16 @@ module Sources::Strategies profile_url.present? && url == profile_url end + def normalize_for_source + return if project_id.blank? + + if artist_name_from_url.present? + "https://#{artist_name_from_url}.artstation.com/projects/#{project_id}" + else + "https://www.artstation.com/artwork/#{project_id}" + end + end + def image_urls_sub if url.match?(ASSET) return [url] diff --git a/app/logical/sources/strategies/base.rb b/app/logical/sources/strategies/base.rb index 7521697d9..b2d23fa12 100644 --- a/app/logical/sources/strategies/base.rb +++ b/app/logical/sources/strategies/base.rb @@ -177,6 +177,12 @@ module Sources profile_url.presence || url end + # Given a post/image url, this is the normalized url that will be displayed in a post's page in its stead. + # This function should never make any network call, even indirectly. Return nil to never normalize. + def normalize_for_source + nil + end + def artists ArtistFinder.find_artists(normalize_for_artist_finder.to_s) end diff --git a/app/logical/sources/strategies/deviant_art.rb b/app/logical/sources/strategies/deviant_art.rb index eb60440b7..786a37576 100644 --- a/app/logical/sources/strategies/deviant_art.rb +++ b/app/logical/sources/strategies/deviant_art.rb @@ -1,25 +1,35 @@ # Asset URLs: # -# * http://orig12.deviantart.net/9b69/f/2017/023/7/c/illustration___tokyo_encount_oei__by_melisaongmiqin-dawi58s.png -# * http://pre15.deviantart.net/81de/th/pre/f/2015/063/5/f/inha_by_inhaestudios-d8kfzm5.jpg -# * http://th00.deviantart.net/fs71/PRE/f/2014/065/3/b/goruto_by_xyelkiltrox-d797tit.png +## NORMALIZABLE +## * http://orig12.deviantart.net/9b69/f/2017/023/7/c/illustration___tokyo_encount_oei__by_melisaongmiqin-dawi58s.png +## * http://pre15.deviantart.net/81de/th/pre/f/2015/063/5/f/inha_by_inhaestudios-d8kfzm5.jpg +## * http://th00.deviantart.net/fs71/PRE/f/2014/065/3/b/goruto_by_xyelkiltrox-d797tit.png +## +## * http://fc00.deviantart.net/fs71/f/2013/234/d/8/d84e05f26f0695b1153e9dab3a962f16-d6j8jl9.jpg +## * http://th04.deviantart.net/fs71/PRE/f/2013/337/3/5/35081351f62b432f84eaeddeb4693caf-d6wlrqs.jpg +## +## * https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/52c4a3ad-d416-42f0-90f6-570983e36797/dczr28f-bd255304-01bf-4765-8cd3-e53983d3f78a.jpg +## * https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/8b472d70-a0d6-41b5-9a66-c35687090acc/d23jbr4-8a06af02-70cb-46da-8a96-42a6ba73cdb4.jpg/v1/fill/w_786,h_1017,q_70,strp/silverhawks_quicksilver_by_edsfox_d23jbr4-pre.jpg +## * https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/f/76098ac8-04ab-4784-b382-88ca082ba9b1/d9x7lmk-595099de-fe8f-48e5-9841-7254f9b2ab8d.png?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ1cm46YXBwOiIsImlzcyI6InVybjphcHA6Iiwib2JqIjpbW3sicGF0aCI6IlwvZlwvNzYwOThhYzgtMDRhYi00Nzg0LWIzODItODhjYTA4MmJhOWIxXC9kOXg3bG1rLTU5NTA5OWRlLWZlOGYtNDhlNS05ODQxLTcyNTRmOWIyYWI4ZC5wbmcifV1dLCJhdWQiOlsidXJuOnNlcnZpY2U6ZmlsZS5kb3dubG9hZCJdfQ.KFOVXAiF8MTlLb3oM-FlD0nnDvODmjqEhFYN5I2X5Bc +## * https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/f/fe7ab27f-7530-4252-99ef-2baaf81b36fd/dddf6pe-1a4a091c-768c-4395-9465-5d33899be1eb.png/v1/fill/w_800,h_1130,q_80,strp/stay_hydrated_and_in_the_shade_by_raikoart_dddf6pe-fullview.jpg?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ1cm46YXBwOjdlMGQxODg5ODIyNjQzNzNhNWYwZDQxNWVhMGQyNmUwIiwiaXNzIjoidXJuOmFwcDo3ZTBkMTg4OTgyMjY0MzczYTVmMGQ0MTVlYTBkMjZlMCIsIm9iaiI6W1t7ImhlaWdodCI6Ijw9MTEzMCIsInBhdGgiOiJcL2ZcL2ZlN2FiMjdmLTc1MzAtNDI1Mi05OWVmLTJiYWFmODFiMzZmZFwvZGRkZjZwZS0xYTRhMDkxYy03NjhjLTQzOTUtOTQ2NS01ZDMzODk5YmUxZWIucG5nIiwid2lkdGgiOiI8PTgwMCJ9XV0sImF1ZCI6WyJ1cm46c2VydmljZTppbWFnZS5vcGVyYXRpb25zIl19.J0W4k-iV6Mg8Kt_5Lr_L_JbBq4lyr7aCausWWJ_Fsbw # -# * http://th04.deviantart.net/fs70/300W/f/2009/364/4/d/Alphes_Mimic___Rika_by_Juriesute.png -# * http://fc02.deviantart.net/fs48/f/2009/186/2/c/Animation_by_epe_tohri.swf -# * http://fc08.deviantart.net/files/f/2007/120/c/9/Cool_Like_Me_by_47ness.jpg +## * http://www.deviantart.com/download/135944599/Touhou___Suwako_Moriya_Colored_by_Turtle_Chibi.png +## * https://www.deviantart.com/download/549677536/countdown_to_midnight_by_kawacy-d939hwg.jpg?token=92090cd3910d52089b566661e8c2f749755ed5f8&ts=1438535525 # -# * http://fc08.deviantart.net/images3/i/2004/088/8/f/Blackrose_for_MuzicFreq.jpg -# * http://img04.deviantart.net/720b/i/2003/37/9/6/princess_peach.jpg +## NOT NORMALIZABLE +## * http://th04.deviantart.net/fs70/300W/f/2009/364/4/d/Alphes_Mimic___Rika_by_Juriesute.png +## * http://fc02.deviantart.net/fs48/f/2009/186/2/c/Animation_by_epe_tohri.swf +## * http://fc08.deviantart.net/files/f/2007/120/c/9/Cool_Like_Me_by_47ness.jpg +## +## * http://fc08.deviantart.net/images3/i/2004/088/8/f/Blackrose_for_MuzicFreq.jpg +## * http://img04.deviantart.net/720b/i/2003/37/9/6/princess_peach.jpg +## +## * http://prnt00.deviantart.net/9b74/b/2016/101/4/468a9d89f52a835d4f6f1c8caca0dfb2-pnjfbh.jpg +## * http://other00.deviantart.net/8863/o/2009/197/3/7/37ac79eaeef9fb32e6ae998e9a77d8dd.jpg +## * http://fc09.deviantart.net/fs22/o/2009/197/3/7/37ac79eaeef9fb32e6ae998e9a77d8dd.jpg +## * http://pre06.deviantart.net/8497/th/pre/f/2009/173/c/c/cc9686111dcffffffb5fcfaf0cf069fb.jpg # -# * http://prnt00.deviantart.net/9b74/b/2016/101/4/468a9d89f52a835d4f6f1c8caca0dfb2-pnjfbh.jpg -# * http://fc00.deviantart.net/fs71/f/2013/234/d/8/d84e05f26f0695b1153e9dab3a962f16-d6j8jl9.jpg -# * http://th04.deviantart.net/fs71/PRE/f/2013/337/3/5/35081351f62b432f84eaeddeb4693caf-d6wlrqs.jpg -# -# * http://fc09.deviantart.net/fs22/o/2009/197/3/7/37ac79eaeef9fb32e6ae998e9a77d8dd.jpg -# * http://pre06.deviantart.net/8497/th/pre/f/2009/173/c/c/cc9686111dcffffffb5fcfaf0cf069fb.jpg -# -# * https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/8b472d70-a0d6-41b5-9a66-c35687090acc/d23jbr4-8a06af02-70cb-46da-8a96-42a6ba73cdb4.jpg/v1/fill/w_786,h_1017,q_70,strp/silverhawks_quicksilver_by_edsfox_d23jbr4-pre.jpg -# * https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/f/fe7ab27f-7530-4252-99ef-2baaf81b36fd/dddf6pe-1a4a091c-768c-4395-9465-5d33899be1eb.png/v1/fill/w_800,h_1130,q_80,strp/stay_hydrated_and_in_the_shade_by_raikoart_dddf6pe-fullview.jpg?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ1cm46YXBwOjdlMGQxODg5ODIyNjQzNzNhNWYwZDQxNWVhMGQyNmUwIiwiaXNzIjoidXJuOmFwcDo3ZTBkMTg4OTgyMjY0MzczYTVmMGQ0MTVlYTBkMjZlMCIsIm9iaiI6W1t7ImhlaWdodCI6Ijw9MTEzMCIsInBhdGgiOiJcL2ZcL2ZlN2FiMjdmLTc1MzAtNDI1Mi05OWVmLTJiYWFmODFiMzZmZFwvZGRkZjZwZS0xYTRhMDkxYy03NjhjLTQzOTUtOTQ2NS01ZDMzODk5YmUxZWIucG5nIiwid2lkdGgiOiI8PTgwMCJ9XV0sImF1ZCI6WyJ1cm46c2VydmljZTppbWFnZS5vcGVyYXRpb25zIl19.J0W4k-iV6Mg8Kt_5Lr_L_JbBq4lyr7aCausWWJ_Fsbw +######################## # # Page URLs: # @@ -39,26 +49,36 @@ module Sources class DeviantArt < Base ASSET_SUBDOMAINS = %r{(?:fc|th|pre|img|orig|origin-orig)\d*}i RESERVED_SUBDOMAINS = %r{\Ahttps?://(?:#{ASSET_SUBDOMAINS}|www)\.}i + MAIN_DOMAIN = %r{\Ahttps?://(?:www\.)?deviantart.com}i TITLE = %r{(?[a-z0-9_-]+?)}i ARTIST = %r{(?<artist>[a-z0-9_-]+?)}i DEVIATION_ID = %r{(?<deviation_id>[0-9]+)}i - DA_FILENAME = %r{#{TITLE}(?:_by_#{ARTIST}(?:-d(?<base36_deviation_id>[a-z0-9]+))?)?\.}i - WIX_FILENAME = %r{#{TITLE}_by_#{ARTIST}_d(?<base36_deviation_id>[a-z0-9]+)-[a-z0-9]+\.}i + DA_FILENAME_1 = %r{[a-f0-9]{32}-d(?<base36_deviation_id>[a-z0-9]+)\.}i + DA_FILENAME_2 = %r{#{TITLE}(?:_by_#{ARTIST}(?:-d(?<base36_deviation_id>[a-z0-9]+))?)?\.}i + DA_FILENAME = Regexp.union(DA_FILENAME_1, DA_FILENAME_2) + WIX_FILENAME = %r{d(?<base36_deviation_id>[a-z0-9]+)[0-9a-f-]+\.\w+(?:/\w+/\w+/[\w,]+/(?<title>[\w-]+)_by_(?<artist>[\w-]+)_d\w+-\w+\.\w+)?.+}i + + NOT_NORMALIZABLE_ASSET = %r{\Ahttps?://#{ASSET_SUBDOMAINS}\.deviantart\.net/.+/[0-9a-f]{32}(?:-[^d]\w+)?\.}i DA_ASSET = %r{\Ahttps?://#{ASSET_SUBDOMAINS}\.deviantart\.net/.+/#{DA_FILENAME}}i - WIX_ASSET = %r{\Ahttps?://images-wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/.+/#{WIX_FILENAME}}i + WIX_ASSET = %r{\Ahttps?://images-wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/(?:intermediary/)?\w/[0-9a-f-]+/#{WIX_FILENAME}}i ASSET = Regexp.union(DA_ASSET, WIX_ASSET) - PATH_ART = %r{\Ahttps?://www\.deviantart\.com/#{ARTIST}/art/#{TITLE}-#{DEVIATION_ID}\z}i + DA_DOWNLOAD = %r{#{MAIN_DOMAIN}/download/#{DEVIATION_ID}/#{DA_FILENAME_2}?}i + + DEVIATION_ART = %r{#{MAIN_DOMAIN}/deviation/#{DEVIATION_ID}\z}i + PATH_ART = %r{#{MAIN_DOMAIN}/#{ARTIST}/art/#{TITLE}-#{DEVIATION_ID}\z}i SUBDOMAIN_ART = %r{\Ahttps?://#{ARTIST}\.deviantart\.com/art/#{TITLE}-#{DEVIATION_ID}\z}i - PATH_PROFILE = %r{\Ahttps?://(www\.)?deviantart\.com/#{ARTIST}/?\z}i + PATH_PROFILE = %r{#{MAIN_DOMAIN}/#{ARTIST}/?\z}i SUBDOMAIN_PROFILE = %r{\Ahttps?://#{ARTIST}\.deviantart\.com/?\z}i + FAVME = %r{\Ahttps?://(www\.)?fav\.me/d(?<base36_deviation_id>[a-z0-9]+)\z}i + def domains - ["deviantart.net", "deviantart.com"] + ["deviantart.net", "deviantart.com", "fav.me"] end def site_name @@ -70,14 +90,6 @@ module Sources parsed_url.domain.in?(domains) || parsed_url.host == "images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com" end - def canonical_url - if self.class.deviation_id_from_url(image_url).present? || page_url.blank? - image_url - else - page_url - end - end - def image_urls [image_url] end @@ -120,12 +132,16 @@ module Sources if artist.present? && title.present? && id.present? "https://www.deviantart.com/#{artist}/art/#{title}-#{id}" elsif id.present? - "https://deviantart.com/deviation/#{id}" + "https://www.deviantart.com/deviation/#{id}" else nil end end + def normalize_for_source + page_url_from_image_url + end + def profile_url return nil if artist_name.blank? "https://www.deviantart.com/#{artist_name.downcase}" @@ -204,9 +220,11 @@ module Sources end def self.deviation_id_from_url(url) - if url =~ ASSET + if url =~ NOT_NORMALIZABLE_ASSET + nil + elsif url =~ ASSET || url =~ FAVME $~[:base36_deviation_id].try(:to_i, 36) - elsif url =~ PATH_ART || (url !~ RESERVED_SUBDOMAINS && url =~ SUBDOMAIN_ART) + elsif url =~ PATH_ART || (url !~ RESERVED_SUBDOMAINS && url =~ SUBDOMAIN_ART) || url =~ DA_DOWNLOAD || url =~ DEVIATION_ART $~[:deviation_id].to_i else nil @@ -214,7 +232,9 @@ module Sources end def self.artist_name_from_url(url) - if url =~ ASSET || url =~ PATH_ART || url =~ PATH_PROFILE + if url =~ NOT_NORMALIZABLE_ASSET + nil + elsif url =~ ASSET || url =~ PATH_ART || url =~ PATH_PROFILE || url =~ DA_DOWNLOAD $~[:artist].try(:dasherize) elsif url !~ RESERVED_SUBDOMAINS && (url =~ SUBDOMAIN_ART || url =~ SUBDOMAIN_PROFILE) $~[:artist] @@ -224,9 +244,11 @@ module Sources end def self.title_from_url(url) - if url =~ ASSET || url =~ PATH_ART || url =~ PATH_PROFILE + if url =~ NOT_NORMALIZABLE_ASSET + nil + elsif url =~ ASSET || url =~ PATH_ART || url =~ DA_DOWNLOAD $~[:title].to_s.titleize.strip.squeeze(" ").tr(" ", "-").presence - elsif url !~ RESERVED_SUBDOMAINS && (url =~ SUBDOMAIN_ART || url =~ SUBDOMAIN_PROFILE) + elsif url !~ RESERVED_SUBDOMAINS && url =~ SUBDOMAIN_ART $~[:title].to_s.titleize.strip.squeeze(" ").tr(" ", "-").presence else nil diff --git a/app/logical/sources/strategies/hentai_foundry.rb b/app/logical/sources/strategies/hentai_foundry.rb index 6b2d2fda0..24befca7f 100644 --- a/app/logical/sources/strategies/hentai_foundry.rb +++ b/app/logical/sources/strategies/hentai_foundry.rb @@ -5,7 +5,9 @@ # Page URLs # # * https://www.hentai-foundry.com/pictures/user/Afrobull/795025/kuroeda -# * https://www.hentai-foundry.com/pictures/user/Afrobull/795025o +# * https://www.hentai-foundry.com/pictures/user/Afrobull/795025 +# * http://www.hentai-foundry.com/pic-795025 +# * http://www.hentai-foundry.com/pictures/user/Ganassa/457176/LOL-Swimsuit---Caitlyn-reworked-nude-ver. # # Preview URLs # @@ -22,9 +24,10 @@ module Sources module Strategies class HentaiFoundry < Base BASE_URL = %r!\Ahttps?://(?:www\.)?hentai-foundry\.com!i - PAGE_URL = %r!#{BASE_URL}/pictures/user/(?<artist_name>[\w-]+)/(?<illust_id>\d+)(?:/[\w-]*)?(\?[\w=]*)?\z!i + PAGE_URL = %r!#{BASE_URL}/pictures/user/(?<artist_name>[\w-]+)/(?<illust_id>\d+)(?:/[\w.-]*)?(\?[\w=]*)?\z!i + OLD_PAGE = %r!#{BASE_URL}/pic-(?<illust_id>\d+)(?:\.html)?\z!i PROFILE_URL = %r!#{BASE_URL}/(?:pictures/)?user/(?<artist_name>[\w-]+)(?:/[a-z]*)?\z!i - IMAGE_URL = %r!\Ahttps?://pictures\.hentai-foundry\.com/\w/(?<artist_name>[\w-]+)/(?<illust_id>\d+)/[\w-]+\.\w+\z!i + IMAGE_URL = %r!\Ahttps?://pictures\.hentai-foundry\.com/+\w/(?<artist_name>[\w-]+)/(?<illust_id>\d+)(?:(?:/[\w.-]+)?\.\w+)?\z!i def domains ["hentai-foundry.com"] @@ -49,8 +52,13 @@ module Sources end def page_url - return nil if illust_id.blank? || artist_name.blank? - "https://www.hentai-foundry.com/pictures/user/#{artist_name}/#{illust_id}" + return nil if illust_id.blank? + + if artist_name.blank? + "https://www.hentai-foundry.com/pic-#{illust_id}" + else + "https://www.hentai-foundry.com/pictures/user/#{artist_name}/#{illust_id}" + end end def page @@ -104,8 +112,12 @@ module Sources url =~ PROFILE_URL end + def normalize_for_source + page_url + end + def illust_id - url[PAGE_URL, :illust_id] || url[IMAGE_URL, :illust_id] + url[PAGE_URL, :illust_id] || url[IMAGE_URL, :illust_id] || url[OLD_PAGE, :illust_id] end end end diff --git a/app/logical/sources/strategies/moebooru.rb b/app/logical/sources/strategies/moebooru.rb index 9482f899b..af149682e 100644 --- a/app/logical/sources/strategies/moebooru.rb +++ b/app/logical/sources/strategies/moebooru.rb @@ -1,6 +1,7 @@ # Original images: # # * https://yande.re/image/b4b1d11facd1700544554e4805d47bb6/.png +# * https://files.yande.re/image/e4c2ba38de88ff1640aaebff84c84e81/469784.jpg # * https://files.yande.re/image/2a5d1d688f565cb08a69ecf4e35017ab/yande.re%20349790%20breast_hold%20kurashima_tomoyasu%20mahouka_koukou_no_rettousei%20naked%20nipples.jpg # * https://ayase.yande.re/image/2d0d229fd8465a325ee7686fcc7f75d2/yande.re%20192481%20animal_ears%20bunny_ears%20garter_belt%20headphones%20mitha%20stockings%20thighhighs.jpg # * https://yuno.yande.re/image/1764b95ae99e1562854791c232e3444b/yande.re%20281544%20cameltoe%20erect_nipples%20fundoshi%20horns%20loli%20miyama-zero%20sarashi%20sling_bikini%20swimsuits.jpg @@ -33,7 +34,7 @@ module Sources class Moebooru < Base BASE_URL = %r!\Ahttps?://(?:[^.]+\.)?(?<domain>yande\.re|konachan\.com)!i POST_URL = %r!#{BASE_URL}/post/show/(?<id>\d+)!i - URL_SLUG = %r!/(?:yande\.re%20|Konachan\.com%20-%20)(?<id>\d+).*!i + URL_SLUG = %r!/(?:yande\.re%20|Konachan\.com%20-%20)?(?<id>\d+)?.*!i IMAGE_URL = %r!#{BASE_URL}/(?<type>image|jpeg|sample)/(?<md5>\h{32})#{URL_SLUG}?\.(?<ext>jpg|jpeg|png|gif)\z!i delegate :artist_name, :profile_url, :tag_name, :artist_commentary_title, :artist_commentary_desc, :dtext_artist_commentary_title, :dtext_artist_commentary_desc, to: :sub_strategy, allow_nil: true @@ -75,6 +76,17 @@ module Sources image_url end + def normalize_for_source + id = post_id_from_url + md5 = post_md5_from_url + + if id.present? + "https://#{site_name}/post/show/#{id}" + elsif md5.present? + "https://#{site_name}/post?tags=md5:#{md5}" + end + end + def tags api_response[:tags].to_s.split.map do |tag| [tag, "https://#{site_name}/post?tags=#{CGI.escape(tag)}"] diff --git a/app/logical/sources/strategies/nico_seiga.rb b/app/logical/sources/strategies/nico_seiga.rb index d4741b283..ceb93a0cb 100644 --- a/app/logical/sources/strategies/nico_seiga.rb +++ b/app/logical/sources/strategies/nico_seiga.rb @@ -1,3 +1,15 @@ +# Image Direct URL +# * https://lohas.nicoseiga.jp/o/971eb8af9bbcde5c2e51d5ef3a2f62d6d9ff5552/1589933964/3583893 +# * http://lohas.nicoseiga.jp/priv/3521156?e=1382558156&h=f2e089256abd1d453a455ec8f317a6c703e2cedf +# * http://lohas.nicoseiga.jp/priv/b80f86c0d8591b217e7513a9e175e94e00f3c7a1/1384936074/3583893 +# * http://seiga.nicovideo.jp/image/source?id=3312222 +# +# Image Page URL +# * https://seiga.nicovideo.jp/seiga/im3521156 +# +# Manga Page URL +# * http://seiga.nicovideo.jp/watch/mg316708 + module Sources module Strategies class NicoSeiga < Base @@ -77,6 +89,10 @@ module Sources if x =~ %r{/watch/mg\d+} return x end + + if x =~ %r{/image/source\?id=(\d+)} + return "http://seiga.nicovideo.jp/seiga/im#{$1}" + end end return super @@ -111,13 +127,21 @@ module Sources end def normalizable_for_artist_finder? - url =~ PAGE || url =~ MANGA_PAGE || url =~ PROFILE || url =~ DIRECT1 || url =~ DIRECT2 + url =~ PAGE || url =~ MANGA_PAGE || url =~ PROFILE || url =~ DIRECT1 || url =~ DIRECT2 || url =~ PAGE2 end def normalize_for_artist_finder "#{profile_url}/" end + def normalize_for_source + if illust_id.present? + "https://seiga.nicovideo.jp/seiga/im#{illust_id}" + elsif theme_id.present? + "http://seiga.nicovideo.jp/watch/mg#{theme_id}" + end + end + def tag_name "nicoseiga#{api_client.user_id}" end diff --git a/app/logical/sources/strategies/nijie.rb b/app/logical/sources/strategies/nijie.rb index 0898416d4..69f90f1af 100644 --- a/app/logical/sources/strategies/nijie.rb +++ b/app/logical/sources/strategies/nijie.rb @@ -162,6 +162,12 @@ module Sources artist_id_from_url || artist_id_from_page end + def normalize_for_source + return if illust_id.blank? + + "https://nijie.info/view.php?id=#{illust_id}" + end + def page return nil if page_url.blank? diff --git a/app/logical/sources/strategies/null.rb b/app/logical/sources/strategies/null.rb index 38fd16dbd..db0e3775d 100644 --- a/app/logical/sources/strategies/null.rb +++ b/app/logical/sources/strategies/null.rb @@ -24,6 +24,117 @@ module Sources def normalize_for_artist_finder url end + + def normalize_for_source + case url + when %r{\Ahttps?://(?:d3j5vwomefv46c|dn3pm25xmtlyu)\.cloudfront\.net/photos/large/(\d+)\.}i + base_10_id = $1.to_i + base_36_id = base_10_id.to_s(36) + "https://twitpic.com/#{base_36_id}" + + when %r{\Ahttp://www\.karabako\.net/images(?:ub)?/karabako_(\d+)(?:_\d+)?\.}i + "http://www.karabako.net/post/view/#{$1}" + + # XXX http://twipple.jp is defunct + # http://p.twpl.jp/show/orig/myRVs + when %r{\Ahttp://p\.twpl\.jp/show/(?:large|orig)/([a-z0-9]+)}i + "http://p.twipple.jp/#{$1}" + + when %r{\Ahttps?://blog(?:(?:-imgs-)?\d*(?:-origin)?)?\.fc2\.com/(?:(?:[^/]/){3}|(?:[^/]/))([^/]+)/(?:file/)?([^\.]+\.[^\?]+)}i + username = $1 + filename = $2 + "http://#{username}.blog.fc2.com/img/#{filename}/" + + when %r{\Ahttps?://diary(\d)?\.fc2\.com/user/([^/]+)/img/(\d+)_(\d+)/(\d+)\.}i + server_id = $1 + username = $2 + year = $3 + month = $4 + day = $5 + "http://diary#{server_id}.fc2.com/cgi-sys/ed.cgi/#{username}?Y=#{year}&M=#{month}&D=#{day}" + + when %r{\Ahttps?://(?:fbcdn-)?s(?:content|photos)-[^/]+\.(?:fbcdn|akamaihd)\.net/hphotos-.+/\d+_(\d+)_(?:\d+_){1,3}[no]\.}i + "https://www.facebook.com/photo.php?fbid=#{$1}" + + when %r{\Ahttps?://c(?:s|han|[1-4])\.sankakucomplex\.com/data(?:/sample)?/(?:[a-f0-9]{2}/){2}(?:sample-|preview)?([a-f0-9]{32})}i + "https://chan.sankakucomplex.com/en/post/show?md5=#{$1}" + + when %r{\Ahttps?://(?:www|s(?:tatic|[1-4]))\.zerochan\.net/.+(?:\.|\/)(\d+)(?:\.(?:jpe?g?))?\z}i + "https://www.zerochan.net/#{$1}#full" + + when %r{\Ahttps?://static[1-6]?\.minitokyo\.net/(?:downloads|view)/(?:\d{2}/){2}(\d+)}i + "http://gallery.minitokyo.net/download/#{$1}" + + # https://gelbooru.com//images/ee/5c/ee5c9a69db9602c95debdb9b98fb3e3e.jpeg + # http://simg.gelbooru.com//images/2003/edd1d2b3881cf70c3acf540780507531.png + # https://simg3.gelbooru.com//samples/0b/3a/sample_0b3ae5e225072b8e391c827cb470d29c.jpg + when %r{\Ahttps?://(?:\w+\.)?gelbooru\.com//?(?:images|samples)/(?:\d+|\h\h/\h\h)/(?:sample_)?(?<md5>\h{32})\.}i + "https://gelbooru.com/index.php?page=post&s=list&md5=#{$~[:md5]}" + + when %r{\Ahttps?://(?:slot\d*\.)?im(?:g|ages)\d*\.wikia\.(?:nocookie\.net|com)/(?:_{2}cb\d{14}/)?([^/]+)(?:/[a-z]{2})?/images/(?:(?:thumb|archive)?/)?[a-f0-9]/[a-f0-9]{2}/(?:\d{14}(?:!|%21))?([^/]+)}i + subdomain = $1 + filename = $2 + "https://#{subdomain}.wikia.com/wiki/File:#{filename}" + + when %r{\Ahttps?://vignette(?:\d*)\.wikia\.nocookie\.net/([^/]+)/images/[a-f0-9]/[a-f0-9]{2}/([^/]+)}i + subdomain = $1 + filename = $2 + "https://#{subdomain}.wikia.com/wiki/File:#{filename}" + + when %r{\Ahttps?://e-shuushuu.net/images/\d{4}-(?:\d{2}-){2}(\d+)}i + "https://e-shuushuu.net/image/#{$1}" + + when %r{\Ahttps?://jpg\.nijigen-daiaru\.com/(\d+)}i + "http://nijigen-daiaru.com/book.php?idb=#{$1}" + + when %r{\Ahttps?://sozai\.doujinantena\.com/contents_jpg/([a-f0-9]{32})/}i + "http://doujinantena.com/page.php?id=#{$1}" + + when %r{\Ahttps?://rule34-(?:data-\d{3}|images)\.paheal\.net/(?:_images/)?([a-f0-9]{32})}i + "https://rule34.paheal.net/post/list/md5:#{$1}/1" + + when %r{\Ahttps?://shimmie\.katawa-shoujo\.com/image/(\d+)}i + "https://shimmie.katawa-shoujo.com/post/view/#{$1}" + + when %r{\Ahttps://(?:(?:\w+\.)?rule34\.xxx|img\.booru\.org/(?:rule34|r34))(?:/(?:img/rule34|r34))?/{1,2}images/\d+/([a-f0-9]{32})\.}i + "https://rule34.xxx/index.php?page=post&s=list&md5=#{$1}" + + when %r{(\Ahttps?://.+)/diarypro/d(?:ata/upfile/|iary\.cgi\?mode=image&upfile=)(\d+)}i + base_url = $1 + entry_no = $2 + "#{base_url}/diarypro/diary.cgi?no=#{entry_no}" + + # XXX site is defunct + when %r{\Ahttps?://i(?:\d)?\.minus\.com/(?:i|j)([^\.]{12,})}i + "http://minus.com/i/#{$1}" + + # http://art59.photozou.jp/pub/212/1986212/photo/118493247_org.v1534644005.jpg + # http://kura3.photozou.jp/pub/794/1481794/photo/161537258_org.v1364829097.jpg + when %r{\Ahttps?://\w+\.photozou\.jp/pub/\d+/(?<artist_id>\d+)/photo/(?<photo_id>\d+)_.*$}i + "https://photozou.jp/photo/show/#{$~[:artist_id]}/#{$~[:photo_id]}" + + # http://img.toranoana.jp/popup_img/04/0030/09/76/040030097695-2p.jpg + # http://img.toranoana.jp/popup_img18/04/0010/22/87/040010228714-1p.jpg + # http://img.toranoana.jp/popup_blimg/04/0030/08/30/040030083068-1p.jpg + # https://ecdnimg.toranoana.jp/ec/img/04/0030/65/34/040030653417-6p.jpg + when %r{\Ahttps?://(\w+\.)?toranoana\.jp/(?:popup_(?:bl)?img\d*|ec/img)/\d{2}/\d{4}/\d{2}/\d{2}/(?<work_id>\d+)}i + "https://ec.toranoana.jp/tora_r/ec/item/#{$~[:work_id]}/" + + # https://a.hitomi.la/galleries/907838/1.png + # https://0a.hitomi.la/galleries/1169701/23.png + # https://aa.hitomi.la/galleries/990722/003_01_002.jpg + # https://la.hitomi.la/galleries/1054851/001_main_image.jpg + when %r{\Ahttps?://\w+\.hitomi\.la/galleries/(?<gallery_id>\d+)/(?<image_id>\d+)\w*\.[a-z]+\z}i + "https://hitomi.la/reader/#{$~[:gallery_id]}.html##{$~[:image_id].to_i}" + + # https://aa.hitomi.la/galleries/883451/t_rena1g.png + when %r{\Ahttps?://\w+\.hitomi\.la/galleries/(?<gallery_id>\d+)/\w*\.[a-z]+\z}i + "https://hitomi.la/galleries/#{$~[:gallery_id]}.html" + + else + nil + end + end end end end diff --git a/app/logical/sources/strategies/pawoo.rb b/app/logical/sources/strategies/pawoo.rb index 897932a5d..49a04b5ae 100644 --- a/app/logical/sources/strategies/pawoo.rb +++ b/app/logical/sources/strategies/pawoo.rb @@ -1,6 +1,28 @@ +# Image URLS: +# * https://img.pawoo.net/media_attachments/files/001/297/997/small/c4272a09570757c2.png +# * https://img.pawoo.net/media_attachments/files/001/297/997/original/c4272a09570757c2.png +# * https://pawoo.net/media/lU2uV7C1MMQSb1czwvg +# +# Page URLS: +# * https://pawoo.net/@evazion/19451018 +# * https://pawoo.net/web/statuses/19451018 +# +# Account URLS: +# * https://pawoo.net/@evazion +# * https://pawoo.net/web/accounts/47806 +# +# OAUTH URLS: (NOTE: ID IS DIFFERENT FROM ACCOUNT URL ID) +# * https://pawoo.net/oauth_authentications/17230064 + module Sources::Strategies class Pawoo < Base + HOST = %r!\Ahttps?://(www\.)?pawoo\.net!i IMAGE = %r!\Ahttps?://img\.pawoo\.net/media_attachments/files/(\d+/\d+/\d+)! + NAMED_PROFILE = %r!#{HOST}/@(?<artist_name>\w+)!i + ID_PROFILE = %r!#{HOST}/web/accounts/(?<artist_id>\d+)! + + STATUS1 = %r!\A#{HOST}/web/statuses/(?<status_id>\d+)! + STATUS2 = %r!\A#{NAMED_PROFILE}/(?<status_id>\d+)! def domains ["pawoo.net"] @@ -14,9 +36,6 @@ module Sources::Strategies image_urls.first end - # https://img.pawoo.net/media_attachments/files/001/297/997/small/c4272a09570757c2.png - # https://img.pawoo.net/media_attachments/files/001/297/997/original/c4272a09570757c2.png - # https://pawoo.net/media/lU2uV7C1MMQSb1czwvg def image_urls if url =~ %r!#{IMAGE}/small/([a-z0-9]+\.\w+)\z!i return ["https://img.pawoo.net/media_attachments/files/#{$1}/original/#{$2}"] @@ -29,8 +48,6 @@ module Sources::Strategies return api_response.image_urls end - # https://pawoo.net/@evazion/19451018 - # https://pawoo.net/web/statuses/19451018 def page_url [url, referer_url].each do |x| if PawooApiClient::Status.is_match?(x) @@ -41,8 +58,6 @@ module Sources::Strategies return super end - # https://pawoo.net/@evazion - # https://pawoo.net/web/accounts/47806 def profile_url if url =~ PawooApiClient::PROFILE2 return "https://pawoo.net/@#{$1}" @@ -56,6 +71,22 @@ module Sources::Strategies api_response.account_name end + def artist_name_from_url + if url =~ NAMED_PROFILE + url[NAMED_PROFILE, :artist_name] + end + end + + def artist_id_from_url + if url =~ ID_PROFILE + url[ID_PROFILE, :artist_name] + end + end + + def status_id_from_url + urls.map { |url| url[STATUS1, :status_id] || url[STATUS2, :status_id] }.compact.first + end + def artist_commentary_title nil end @@ -76,6 +107,18 @@ module Sources::Strategies profile_url end + def normalize_for_source + artist_name = artist_name_from_url + status_id = status_id_from_url + return unless status_id.present? + + if artist_name.present? + "https://pawoo.net/@#{artist_name}/#{status_id}" + else + "https://pawoo.net/web/statuses/#{status_id}" + end + end + def dtext_artist_commentary_desc DText.from_html(artist_commentary_desc) do |element| if element.name == "a" diff --git a/app/logical/sources/strategies/pixiv.rb b/app/logical/sources/strategies/pixiv.rb index 9e78c31ee..4b9ce625d 100644 --- a/app/logical/sources/strategies/pixiv.rb +++ b/app/logical/sources/strategies/pixiv.rb @@ -213,6 +213,12 @@ module Sources illust_id.present? || novel_id.present? || fanbox_id.present? || fanbox_account_id.present? end + def normalize_for_source + return if illust_id.blank? + + "https://www.pixiv.net/artworks/#{illust_id}" + end + def tag_name moniker end diff --git a/app/logical/sources/strategies/tumblr.rb b/app/logical/sources/strategies/tumblr.rb index 7cae9a3a2..c6d2fc5bc 100644 --- a/app/logical/sources/strategies/tumblr.rb +++ b/app/logical/sources/strategies/tumblr.rb @@ -5,6 +5,8 @@ # https://emlan.tumblr.com/post/189469423572/kuro-attempts-to-buy-a-racy-book-at-comiket-but # https://66.media.tumblr.com/168dabd09d5ad69eb5fedcf94c45c31a/3dbfaec9b9e0c2e3-72/s640x960/bf33a1324f3f36d2dc64f011bfeab4867da62bc8.png # https://66.media.tumblr.com/5a2c3fe25c977e2281392752ab971c90/3dbfaec9b9e0c2e3-92/s500x750/4f92bbaaf95c0b4e7970e62b1d2e1415859dd659.png +# +# https://superboin.tumblr.com/post/141169066579/photoset_iframe/superboin/tumblr_o45miiAOts1u6rxu8/500/false module Sources::Strategies class Tumblr < Base @@ -131,6 +133,12 @@ module Sources::Strategies super(tag) end + def normalize_for_source + return unless blog_name.present? && post_id.present? + + "https://#{blog_name}.tumblr.com/post/#{post_id}" + end + def dtext_artist_commentary_desc DText.from_html(artist_commentary_desc).strip end diff --git a/app/logical/sources/strategies/twitter.rb b/app/logical/sources/strategies/twitter.rb index 2fcfb5e89..69f363363 100644 --- a/app/logical/sources/strategies/twitter.rb +++ b/app/logical/sources/strategies/twitter.rb @@ -43,7 +43,7 @@ module Sources::Strategies # https://twitter.com/i/web/status/943446161586733056 # https://twitter.com/motty08111213/status/943446161586733056 def self.status_id_from_url(url) - if url =~ %r{\Ahttps?://(?:mobile\.)?twitter\.com/(?:i/web|\w+)/status/(\d+)}i + if url =~ %r{\Ahttps?://(?:(?:www|mobile)\.)?twitter\.com/(?:i/web|\w+)/status/(\d+)}i return $1 end @@ -145,6 +145,25 @@ module Sources::Strategies profile_url.try(:downcase).presence || url end + def normalize_for_source + status_id = self.class.status_id_from_url(url) + if status_id.present? + "https://twitter.com/i/web/status/#{status_id}" + elsif url =~ %r{\Ahttps?://(?:o|image-proxy-origin)\.twimg\.com/\d/proxy\.jpg\?t=(\w+)&}i + str = Base64.decode64($1) + source = URI.extract(str, ['http', 'https']) + if source.any? + source = source[0] + if source =~ %r{^https?://twitpic.com/show/large/[a-z0-9]+}i + source.gsub!(%r{show/large/}, "") + index = source.rindex('.') + source = source[0..index - 1] + end + source + end + end + end + def tags api_response.dig(:entities, :hashtags).to_a.map do |hashtag| [hashtag[:text], "https://twitter.com/hashtag/#{hashtag[:text]}"] diff --git a/app/models/post.rb b/app/models/post.rb index 5a01ee579..e37453447 100644 --- a/app/models/post.rb +++ b/app/models/post.rb @@ -334,187 +334,7 @@ class Post < ApplicationRecord end def normalized_source - if pixiv_id.present? - return "https://www.pixiv.net/artworks/#{pixiv_id}" - end - - case source - when %r{\Ahttps?://(?:mobile\.)?twitter.com/[^/]+/status/(\d+)(?:/(?:photo/\d)?|\?s=\d+)?\z}i - "https://twitter.com/i/web/status/#{$1}" - - when %r{\Ahttps?://lohas\.nicoseiga\.jp/priv/(\d+)\?e=\d+&h=[a-f0-9]+}i, - %r{\Ahttps?://lohas\.nicoseiga\.jp/priv/[a-f0-9]+/\d+/(\d+)}i - "https://seiga.nicovideo.jp/seiga/im#{$1}" - - when %r{\Ahttps?://(?:d3j5vwomefv46c|dn3pm25xmtlyu)\.cloudfront\.net/photos/large/(\d+)\.}i - base_10_id = $1.to_i - base_36_id = base_10_id.to_s(36) - "https://twitpic.com/#{base_36_id}" - - # http://prnt00.deviantart.net/9b74/b/2016/101/4/468a9d89f52a835d4f6f1c8caca0dfb2-pnjfbh.jpg - # http://fc00.deviantart.net/fs71/f/2013/234/d/8/d84e05f26f0695b1153e9dab3a962f16-d6j8jl9.jpg - # http://th04.deviantart.net/fs71/PRE/f/2013/337/3/5/35081351f62b432f84eaeddeb4693caf-d6wlrqs.jpg - # http://fc09.deviantart.net/fs22/o/2009/197/3/7/37ac79eaeef9fb32e6ae998e9a77d8dd.jpg - when %r{\Ahttps?://(?:fc|th|pre|orig|img|prnt)\d{2}\.deviantart\.net/.+/[a-f0-9]{32}-d(?<id>[a-z0-9]+)\.}i - id = $~[:id].to_i(36) - "https://deviantart.com/deviation/#{id}" - - when Sources::Strategies::DeviantArt::ASSET - Sources::Strategies::DeviantArt.new(source).page_url_from_image_url || source - - when %r{\Ahttp://www\.karabako\.net/images(?:ub)?/karabako_(\d+)(?:_\d+)?\.}i - "http://www.karabako.net/post/view/#{$1}" - - # XXX http://twipple.jp is defunct - # http://p.twpl.jp/show/orig/myRVs - when %r{\Ahttp://p\.twpl\.jp/show/(?:large|orig)/([a-z0-9]+)}i - "http://p.twipple.jp/#{$1}" - - when %r{\Ahttps?://pictures\.hentai-foundry\.com//?[^/]/([^/]+)/(\d+)}i - "https://www.hentai-foundry.com/pictures/user/#{$1}/#{$2}" - - when %r{\Ahttp://blog(?:(?:-imgs-)?\d*(?:-origin)?)?\.fc2\.com/(?:(?:[^/]/){3}|(?:[^/]/))([^/]+)/(?:file/)?([^\.]+\.[^\?]+)}i - username = $1 - filename = $2 - "http://#{username}.blog.fc2.com/img/#{filename}/" - - when %r{\Ahttp://diary(\d)?\.fc2\.com/user/([^/]+)/img/(\d+)_(\d+)/(\d+)\.}i - server_id = $1 - username = $2 - year = $3 - month = $4 - day = $5 - "http://diary#{server_id}.fc2.com/cgi-sys/ed.cgi/#{username}?Y=#{year}&M=#{month}&D=#{day}" - - when %r{\Ahttps?://(?:fbcdn-)?s(?:content|photos)-[^/]+\.(?:fbcdn|akamaihd)\.net/hphotos-.+/\d+_(\d+)_(?:\d+_){1,3}[no]\.}i - "https://www.facebook.com/photo.php?fbid=#{$1}" - - when %r{\Ahttps?://c(?:s|han|[1-4])\.sankakucomplex\.com/data(?:/sample)?/(?:[a-f0-9]{2}/){2}(?:sample-|preview)?([a-f0-9]{32})}i - "https://chan.sankakucomplex.com/en/post/show?md5=#{$1}" - - when %r{\Ahttp://s(?:tatic|[1-4])\.zerochan\.net/.+(?:\.|\/)(\d+)\.(?:jpe?g?)\z}i - "https://www.zerochan.net/#{$1}#full" - - when %r{\Ahttp://static[1-6]?\.minitokyo\.net/(?:downloads|view)/(?:\d{2}/){2}(\d+)}i - "http://gallery.minitokyo.net/download/#{$1}" - - # https://gelbooru.com//images/ee/5c/ee5c9a69db9602c95debdb9b98fb3e3e.jpeg - # http://simg.gelbooru.com//images/2003/edd1d2b3881cf70c3acf540780507531.png - # https://simg3.gelbooru.com//samples/0b/3a/sample_0b3ae5e225072b8e391c827cb470d29c.jpg - when %r{\Ahttps?://(?:\w+\.)?gelbooru\.com//?(?:images|samples)/(?:\d+|\h\h/\h\h)/(?:sample_)?(?<md5>\h{32})\.}i - "https://gelbooru.com/index.php?page=post&s=list&md5=#{$~[:md5]}" - - when %r{\Ahttps?://(?:slot\d*\.)?im(?:g|ages)\d*\.wikia\.(?:nocookie\.net|com)/(?:_{2}cb\d{14}/)?([^/]+)(?:/[a-z]{2})?/images/(?:(?:thumb|archive)?/)?[a-f0-9]/[a-f0-9]{2}/(?:\d{14}(?:!|%21))?([^/]+)}i - subdomain = $1 - filename = $2 - "http://#{subdomain}.wikia.com/wiki/File:#{filename}" - - when %r{\Ahttps?://vignette(?:\d*)\.wikia\.nocookie\.net/([^/]+)/images/[a-f0-9]/[a-f0-9]{2}/([^/]+)}i - subdomain = $1 - filename = $2 - "http://#{subdomain}.wikia.com/wiki/File:#{filename}" - - when %r{\Ahttp://(?:(?:\d{1,3}\.){3}\d{1,3}):(?:\d{1,5})/h/([a-f0-9]{40})-(?:\d+-){3}(?:png|gif|(?:jpe?g?))/keystamp=\d+-[a-f0-9]{10}/([^/]+)}i - sha1hash = $1 - filename = $2 - "http://g.e-hentai.org/?f_shash=#{sha1hash}&fs_from=#{filename}" - - when %r{\Ahttp://e-shuushuu.net/images/\d{4}-(?:\d{2}-){2}(\d+)}i - "http://e-shuushuu.net/image/#{$1}" - - when %r{\Ahttp://jpg\.nijigen-daiaru\.com/(\d+)}i - "http://nijigen-daiaru.com/book.php?idb=#{$1}" - - when %r{\Ahttps?://sozai\.doujinantena\.com/contents_jpg/([a-f0-9]{32})/}i - "http://doujinantena.com/page.php?id=#{$1}" - - when %r{\Ahttp://rule34-(?:data-\d{3}|images)\.paheal\.net/(?:_images/)?([a-f0-9]{32})}i - "https://rule34.paheal.net/post/list/md5:#{$1}/1" - - when %r{\Ahttp://shimmie\.katawa-shoujo\.com/image/(\d+)}i - "https://shimmie.katawa-shoujo.com/post/view/#{$1}" - - when %r{\Ahttp://(?:(?:(?:img\d?|cdn)\.)?rule34\.xxx|img\.booru\.org/(?:rule34|r34))(?:/(?:img/rule34|r34))?/{1,2}images/\d+/(?:[a-f0-9]{32}|[a-f0-9]{40})\.}i - "https://rule34.xxx/index.php?page=post&s=list&md5=#{md5}" - - when %r{(\Ahttp://.+)/diarypro/d(?:ata/upfile/|iary\.cgi\?mode=image&upfile=)(\d+)}i - base_url = $1 - entry_no = $2 - "#{base_url}/diarypro/diary.cgi?no=#{entry_no}" - - # XXX site is defunct - when %r{\Ahttp://i(?:\d)?\.minus\.com/(?:i|j)([^\.]{12,})}i - "http://minus.com/i/#{$1}" - - when %r{\Ahttps?://pic0[1-4]\.nijie\.info/nijie_picture/(?:diff/main/)?\d+_(\d+)_(?:\d+{10}|\d+_\d+{14})}i - "https://nijie.info/view.php?id=#{$1}" - - # http://ayase.yande.re/image/2d0d229fd8465a325ee7686fcc7f75d2/yande.re%20192481%20animal_ears%20bunny_ears%20garter_belt%20headphones%20mitha%20stockings%20thighhighs.jpg - # https://yuno.yande.re/image/1764b95ae99e1562854791c232e3444b/yande.re%20281544%20cameltoe%20erect_nipples%20fundoshi%20horns%20loli%20miyama-zero%20sarashi%20sling_bikini%20swimsuits.jpg - # https://files.yande.re/image/2a5d1d688f565cb08a69ecf4e35017ab/yande.re%20349790%20breast_hold%20kurashima_tomoyasu%20mahouka_koukou_no_rettousei%20naked%20nipples.jpg - # https://files.yande.re/sample/0d79447ce2c89138146f64ba93633568/yande.re%20290757%20sample%20seifuku%20thighhighs%20tsukudani_norio.jpg - when %r{\Ahttps?://(?:[^.]+\.)?yande\.re/(?:image|jpeg|sample)/\h{32}/yande\.re%20(?<post_id>\d+)}i - "https://yande.re/post/show/#{$~[:post_id]}" - - # https://yande.re/jpeg/0c9ec0ffcaa40470093cb44c3fd40056/yande.re%2064649%20animal_ears%20cameltoe%20fixme%20nekomimi%20nipples%20ryohka%20school_swimsuit%20see_through%20shiraishi_nagomi%20suzuya%20swimsuits%20tail%20thighhighs.jpg - # https://yande.re/jpeg/22577d2344fe694cf47f80563031b3cd.jpg - # https://yande.re/image/b4b1d11facd1700544554e4805d47bb6/.png - # https://yande.re/sample/ceb6a12e87945413a95b90fada406f91/.jpg - when %r{\Ahttps?://(?:[^.]+\.)?yande\.re/(?:image|jpeg|sample)/(?<md5>\h{32})(?:/yande\.re.*|/?\.(?:jpg|png))\z}i - "https://yande.re/post?tags=md5:#{$~[:md5]}" - - when %r{\Ahttps?://(?:[^.]+\.)?konachan\.com/(?:image|jpeg|sample)/\h{32}/Konachan\.com%20-%20(?<post_id>\d+)}i - "https://konachan.com/post/show/#{$~[:post_id]}" - - when %r{\Ahttps?://(?:[^.]+\.)?konachan\.com/(?:image|jpeg|sample)/(?<md5>\h{32})(?:/Konachan\.com%20-%20.*|/?\.(?:jpg|png))\z}i - "https://konachan.com/post?tags=md5:#{$~[:md5]}" - - # https://gfee_li.artstation.com/projects/XPGOD - # https://gfee_li.artstation.com/projects/asuka-7 - when %r{\Ahttps?://\w+\.artstation.com/(?:artwork|projects)/(?<project_id>[a-z0-9-]+)\z/}i - "https://www.artstation.com/artwork/#{$~[:project_id]}" - - when %r{\Ahttps?://(?:o|image-proxy-origin)\.twimg\.com/\d/proxy\.jpg\?t=(\w+)&}i - str = Base64.decode64($1) - url = URI.extract(str, ['http', 'https']) - if url.any? - url = url[0] - if url =~ /^https?:\/\/twitpic.com\/show\/large\/[a-z0-9]+/i - url.gsub!(/show\/large\//, "") - index = url.rindex('.') - url = url[0..index - 1] - end - url - else - source - end - - # http://art59.photozou.jp/pub/212/1986212/photo/118493247_org.v1534644005.jpg - # http://kura3.photozou.jp/pub/794/1481794/photo/161537258_org.v1364829097.jpg - when %r{\Ahttps?://\w+\.photozou\.jp/pub/\d+/(?<artist_id>\d+)/photo/(?<photo_id>\d+)_.*$}i - "https://photozou.jp/photo/show/#{$~[:artist_id]}/#{$~[:photo_id]}" - - # http://img.toranoana.jp/popup_img/04/0030/09/76/040030097695-2p.jpg - # http://img.toranoana.jp/popup_img18/04/0010/22/87/040010228714-1p.jpg - # http://img.toranoana.jp/popup_blimg/04/0030/08/30/040030083068-1p.jpg - # https://ecdnimg.toranoana.jp/ec/img/04/0030/65/34/040030653417-6p.jpg - when %r{\Ahttps?://(\w+\.)?toranoana\.jp/(?:popup_(?:bl)?img\d*|ec/img)/\d{2}/\d{4}/\d{2}/\d{2}/(?<work_id>\d+)}i - "https://ec.toranoana.jp/tora_r/ec/item/#{$~[:work_id]}/" - - # https://a.hitomi.la/galleries/907838/1.png - # https://0a.hitomi.la/galleries/1169701/23.png - # https://aa.hitomi.la/galleries/990722/003_01_002.jpg - # https://la.hitomi.la/galleries/1054851/001_main_image.jpg - when %r{\Ahttps?://\w+\.hitomi\.la/galleries/(?<gallery_id>\d+)/(?<image_id>\d+)\w*\.[a-z]+\z}i - "https://hitomi.la/reader/#{$~[:gallery_id]}.html##{$~[:image_id].to_i}" - - # https://aa.hitomi.la/galleries/883451/t_rena1g.png - when %r{\Ahttps?://\w+\.hitomi\.la/galleries/(?<gallery_id>\d+)/\w*\.[a-z]+\z}i - "https://hitomi.la/galleries/#{$~[:gallery_id]}.html" - - else - source - end + Sources::Strategies.normalize_source(source) end def source_domain diff --git a/test/unit/post_test.rb b/test/unit/post_test.rb index 555509e4e..ba03cae91 100644 --- a/test/unit/post_test.rb +++ b/test/unit/post_test.rb @@ -1531,60 +1531,6 @@ class PostTest < ActiveSupport::TestCase @post.pixiv_id = nil end end - - should "normalize pixiv links" do - @post.update!(source: "http://i2.pixiv.net/img12/img/zenze/39749565.png") - assert_equal("https://www.pixiv.net/artworks/39749565", @post.normalized_source) - - @post.update!(source: "http://i1.pixiv.net/img53/img/themare/39735353_big_p1.jpg") - assert_equal("https://www.pixiv.net/artworks/39735353", @post.normalized_source) - - @post.update!(source: "http://i1.pixiv.net/c/150x150/img-master/img/2010/11/30/08/39/58/14901720_p0_master1200.jpg") - assert_equal("https://www.pixiv.net/artworks/14901720", @post.normalized_source) - - @post.update!(source: "http://i1.pixiv.net/img-original/img/2010/11/30/08/39/58/14901720_p0.png") - assert_equal("https://www.pixiv.net/artworks/14901720", @post.normalized_source) - - @post.update!(source: "http://i2.pixiv.net/img-zip-ugoira/img/2014/08/05/06/01/10/44524589_ugoira1920x1080.zip") - assert_equal("https://www.pixiv.net/artworks/44524589", @post.normalized_source) - end - - should "normalize nicoseiga links" do - @post.source = "http://lohas.nicoseiga.jp/priv/3521156?e=1382558156&h=f2e089256abd1d453a455ec8f317a6c703e2cedf" - assert_equal("https://seiga.nicovideo.jp/seiga/im3521156", @post.normalized_source) - @post.source = "http://lohas.nicoseiga.jp/priv/b80f86c0d8591b217e7513a9e175e94e00f3c7a1/1384936074/3583893" - assert_equal("https://seiga.nicovideo.jp/seiga/im3583893", @post.normalized_source) - end - - should "normalize twitpic links" do - @post.source = "http://d3j5vwomefv46c.cloudfront.net/photos/large/820960031.jpg?1384107199" - assert_equal("https://twitpic.com/dks0tb", @post.normalized_source) - end - - should "normalize deviantart links" do - @post.source = "http://fc06.deviantart.net/fs71/f/2013/295/d/7/you_are_already_dead__by_mar11co-d6rgm0e.jpg" - assert_equal("https://www.deviantart.com/mar11co/art/You-Are-Already-Dead-408921710", @post.normalized_source) - @post.source = "http://fc00.deviantart.net/fs71/f/2013/337/3/5/35081351f62b432f84eaeddeb4693caf-d6wlrqs.jpg" - assert_equal("https://deviantart.com/deviation/417560500", @post.normalized_source) - end - - should "normalize karabako links" do - @post.source = "http://www.karabako.net/images/karabako_38835.jpg" - assert_equal("http://www.karabako.net/post/view/38835", @post.normalized_source) - end - - should "normalize twipple links" do - @post.source = "http://p.twpl.jp/show/orig/mI2c3" - assert_equal("http://p.twipple.jp/mI2c3", @post.normalized_source) - end - - should "normalize hentai foundry links" do - @post.source = "http://pictures.hentai-foundry.com//a/AnimeFlux/219123.jpg" - assert_equal("https://www.hentai-foundry.com/pictures/user/AnimeFlux/219123", @post.normalized_source) - - @post.source = "http://pictures.hentai-foundry.com/a/AnimeFlux/219123/Mobile-Suit-Equestria-rainbow-run.jpg" - assert_equal("https://www.hentai-foundry.com/pictures/user/AnimeFlux/219123", @post.normalized_source) - end end context "when validating tags" do diff --git a/test/unit/sources/art_station_test.rb b/test/unit/sources/art_station_test.rb index f0dec1b7b..51ffc59a8 100644 --- a/test/unit/sources/art_station_test.rb +++ b/test/unit/sources/art_station_test.rb @@ -160,5 +160,24 @@ module Sources site = Sources::Strategies.find("https://sa-dui.artstation.com/projects/DVERn") assert_equal("sa-dui", site.artist_name) end + + context "normalizing for source" do + should "normalize correctly" do + source1 = "https://www.artstation.com/artwork/ghost-in-the-shell-fandom" + source2 = "https://anubis1982918.artstation.com/projects/qPVGP/" + source3 = "https://dudeunderscore.artstation.com/projects/NoNmD?album_id=23041" + + assert_equal(source1, Sources::Strategies.normalize_source(source1)) + assert_equal("https://anubis1982918.artstation.com/projects/qPVGP", Sources::Strategies.normalize_source(source2)) + assert_equal("https://dudeunderscore.artstation.com/projects/NoNmD", Sources::Strategies.normalize_source(source3)) + end + + should "avoid normalizing unnormalizable urls" do + bad_source1 = "http://cdna.artstation.com/p/assets/images/images/005/804/224/large/titapa-khemakavat-sa-dui-srevere.jpg?1493887236" + bad_source2 = "https://www.artstation.com" + assert_equal(bad_source1, Sources::Strategies.normalize_source(bad_source1)) + assert_equal(bad_source2, Sources::Strategies.normalize_source(bad_source2)) + end + end end end diff --git a/test/unit/sources/deviant_art_test.rb b/test/unit/sources/deviant_art_test.rb index 9b321f6d2..9a0cfd01d 100644 --- a/test/unit/sources/deviant_art_test.rb +++ b/test/unit/sources/deviant_art_test.rb @@ -32,7 +32,7 @@ module Sources @artist = create(:artist, name: "nickbeja", url_string: "https://nickbeja.deviantart.com") assert_equal("https://pre00.deviantart.net/423b/th/pre/i/2017/281/e/0/mindflayer_girl01_by_nickbeja-dbpxdt8.png", @site.image_url) - assert_equal(@site.image_url, @site.canonical_url) + assert_equal(@site.page_url, @site.canonical_url) assert_equal("nickbeja", @site.artist_name) assert_equal("https://www.deviantart.com/nickbeja", @site.profile_url) assert_equal("https://www.deviantart.com/nickbeja/art/Mindflayer-Girl01-708675884", @site.page_url_from_image_url) @@ -362,5 +362,32 @@ module Sources assert_equal([@artist], @site.artists) end end + + context "normalizing for source" do + should "normalize correctly" do + source1 = "http://fc06.deviantart.net/fs71/f/2013/295/d/7/you_are_already_dead__by_mar11co-d6rgm0e.jpg" + source2 = "https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/8b472d70-a0d6-41b5-9a66-c35687090acc/d23jbr4-8a06af02-70cb-46da-8a96-42a6ba73cdb4.jpg/v1/fill/w_786,h_1017,q_70,strp/silverhawks_quicksilver_by_edsfox_d23jbr4-pre.jpg" + source3 = "http://orig12.deviantart.net/9b69/f/2017/023/7/c/illustration___tokyo_encount_oei__by_melisaongmiqin-dawi58s.png" + source4 = "http://fc00.deviantart.net/fs71/f/2013/337/3/5/35081351f62b432f84eaeddeb4693caf-d6wlrqs.jpg" + source5 = "https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/f/76098ac8-04ab-4784-b382-88ca082ba9b1/d9x7lmk-595099de-fe8f-48e5-9841-7254f9b2ab8d.png?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ1cm46YXBwOiIsImlzcyI6InVybjphcHA6Iiwib2JqIjpbW3sicGF0aCI6IlwvZlwvNzYwOThhYzgtMDRhYi00Nzg0LWIzODItODhjYTA4MmJhOWIxXC9kOXg3bG1rLTU5NTA5OWRlLWZlOGYtNDhlNS05ODQxLTcyNTRmOWIyYWI4ZC5wbmcifV1dLCJhdWQiOlsidXJuOnNlcnZpY2U6ZmlsZS5kb3dubG9hZCJdfQ.KFOVXAiF8MTlLb3oM-FlD0nnDvODmjqEhFYN5I2X5Bc" + source6 = "https://fav.me/dbc3a48" + + assert_equal("https://www.deviantart.com/mar11co/art/You-Are-Already-Dead-408921710", Sources::Strategies.normalize_source(source1)) + assert_equal("https://www.deviantart.com/edsfox/art/Silverhawks-Quicksilver-126872896", Sources::Strategies.normalize_source(source2)) + assert_equal("https://www.deviantart.com/melisaongmiqin/art/Illustration-Tokyo-Encount-Oei-659256076", Sources::Strategies.normalize_source(source3)) + assert_equal("https://www.deviantart.com/deviation/417560500", Sources::Strategies.normalize_source(source4)) + assert_equal("https://www.deviantart.com/deviation/599977532", Sources::Strategies.normalize_source(source5)) + assert_equal("https://www.deviantart.com/deviation/685436408", Sources::Strategies.normalize_source(source6)) + end + + should "avoid normalizing unnormalizable urls" do + bad_source1 = "http://fc08.deviantart.net/images3/i/2004/088/8/f/Blackrose_for_MuzicFreq.jpg" + bad_source2 = "http://prnt00.deviantart.net/9b74/b/2016/101/4/468a9d89f52a835d4f6f1c8caca0dfb2-pnjfbh.jpg" + bad_source3 = "https://deviantart.net" + assert_equal(bad_source1, Sources::Strategies.normalize_source(bad_source1)) + assert_equal(bad_source2, Sources::Strategies.normalize_source(bad_source2)) + assert_equal(bad_source3, Sources::Strategies.normalize_source(bad_source3)) + end + end end end diff --git a/test/unit/sources/hentai_foundry_test.rb b/test/unit/sources/hentai_foundry_test.rb index 22fb39ec4..af9002c2f 100644 --- a/test/unit/sources/hentai_foundry_test.rb +++ b/test/unit/sources/hentai_foundry_test.rb @@ -85,5 +85,22 @@ module Sources assert_equal([@artist], @image.artists) end end + + context "normalizing for source" do + should "normalize correctly" do + source1 = "http://pictures.hentai-foundry.com//a/AnimeFlux/219123.jpg" + source2 = "http://pictures.hentai-foundry.com/a/AnimeFlux/219123/Mobile-Suit-Equestria-rainbow-run.jpg" + source3 = "http://www.hentai-foundry.com/pictures/user/Ganassa/457176/LOL-Swimsuit---Caitlyn-reworked-nude-ver." + + assert_equal("https://www.hentai-foundry.com/pictures/user/AnimeFlux/219123", Sources::Strategies.normalize_source(source1)) + assert_equal("https://www.hentai-foundry.com/pictures/user/AnimeFlux/219123", Sources::Strategies.normalize_source(source2)) + assert_equal("https://www.hentai-foundry.com/pictures/user/Ganassa/457176", Sources::Strategies.normalize_source(source3)) + end + + should "avoid normalizing unnormalizable urls" do + bad_source = "https://pictures.hentai-foundry.com/a/AnimeFlux" + assert_equal(bad_source, Sources::Strategies.normalize_source(bad_source)) + end + end end end diff --git a/test/unit/sources/moebooru_test.rb b/test/unit/sources/moebooru_test.rb index 2c200230d..008051ad3 100644 --- a/test/unit/sources/moebooru_test.rb +++ b/test/unit/sources/moebooru_test.rb @@ -114,34 +114,29 @@ module Sources end end - context "Post#normalized_source" do - should "convert yande.re image urls to post urls" do - @post = FactoryBot.build(:post) + context "normalizing for source" do + should "normalize yande.re sources correctly" do + source1 = "https://files.yande.re/image/b66909b940e8d77accab7c9b25aa4dc3/yande.re%20377828.png" + source2 = "https://files.yande.re/image/2a5d1d688f565cb08a69ecf4e35017ab/yande.re%20349790%20breast_hold%20kurashima_tomoyasu%20mahouka_koukou_no_rettousei%20naked%20nipples.jpg" + source3 = "https://files.yande.re/image/e4c2ba38de88ff1640aaebff84c84e81/469784.jpg" + source4 = "https://yande.re/image/b4b1d11facd1700544554e4805d47bb6/.png" + source5 = "https://yande.re/jpeg/22577d2344fe694cf47f80563031b3cd.jpg" - @post.source = "https://files.yande.re/image/b66909b940e8d77accab7c9b25aa4dc3/yande.re%20377828.png" - assert_equal("https://yande.re/post/show/377828", @post.normalized_source) - - @post.source = "https://files.yande.re/image/2a5d1d688f565cb08a69ecf4e35017ab/yande.re%20349790%20breast_hold%20kurashima_tomoyasu%20mahouka_koukou_no_rettousei%20naked%20nipples.jpg" - assert_equal("https://yande.re/post/show/349790", @post.normalized_source) - - @post.source = "https://yande.re/image/b4b1d11facd1700544554e4805d47bb6/.png" - assert_equal("https://yande.re/post?tags=md5:b4b1d11facd1700544554e4805d47bb6", @post.normalized_source) - - @post.source = "https://yande.re/jpeg/22577d2344fe694cf47f80563031b3cd.jpg" - assert_equal("https://yande.re/post?tags=md5:22577d2344fe694cf47f80563031b3cd", @post.normalized_source) + assert_equal("https://yande.re/post/show/377828", Sources::Strategies.normalize_source(source1)) + assert_equal("https://yande.re/post/show/349790", Sources::Strategies.normalize_source(source2)) + assert_equal("https://yande.re/post/show/469784", Sources::Strategies.normalize_source(source3)) + assert_equal("https://yande.re/post?tags=md5:b4b1d11facd1700544554e4805d47bb6", Sources::Strategies.normalize_source(source4)) + assert_equal("https://yande.re/post?tags=md5:22577d2344fe694cf47f80563031b3cd", Sources::Strategies.normalize_source(source5)) end - should "convert konachan.com image urls to post urls" do - @post = FactoryBot.build(:post) + should "normalize konachan.com sources correctly" do + source1 = "https://konachan.com/image/5d633771614e4bf5c17df19a0f0f333f/Konachan.com%20-%20270807%20black_hair%20bokuden%20clouds%20grass%20landscape%20long_hair%20original%20phone%20rope%20scenic%20seifuku%20skirt%20sky%20summer%20torii%20tree.jpg" + source2 = "https://konachan.com/sample/e2e2994bae738ff52fff7f4f50b069d5/Konachan.com%20-%20270803%20sample.jpg" + source3 = "https://konachan.com/image/99a3c4f10c327d54486259a74173fc0b.jpg" - @post.source = "https://konachan.com/image/5d633771614e4bf5c17df19a0f0f333f/Konachan.com%20-%20270807%20black_hair%20bokuden%20clouds%20grass%20landscape%20long_hair%20original%20phone%20rope%20scenic%20seifuku%20skirt%20sky%20summer%20torii%20tree.jpg" - assert_equal("https://konachan.com/post/show/270807", @post.normalized_source) - - @post.source = "https://konachan.com/sample/e2e2994bae738ff52fff7f4f50b069d5/Konachan.com%20-%20270803%20sample.jpg" - assert_equal("https://konachan.com/post/show/270803", @post.normalized_source) - - @post.source = "https://konachan.com/image/99a3c4f10c327d54486259a74173fc0b.jpg" - assert_equal("https://konachan.com/post?tags=md5:99a3c4f10c327d54486259a74173fc0b", @post.normalized_source) + assert_equal("https://konachan.com/post/show/270807", Sources::Strategies.normalize_source(source1)) + assert_equal("https://konachan.com/post/show/270803", Sources::Strategies.normalize_source(source2)) + assert_equal("https://konachan.com/post?tags=md5:99a3c4f10c327d54486259a74173fc0b", Sources::Strategies.normalize_source(source3)) end end end diff --git a/test/unit/sources/nico_seiga_test.rb b/test/unit/sources/nico_seiga_test.rb index 8ff4d2eec..328706281 100644 --- a/test/unit/sources/nico_seiga_test.rb +++ b/test/unit/sources/nico_seiga_test.rb @@ -61,5 +61,24 @@ module Sources assert_match(full_image_url, site.canonical_url) end end + + context "normalizing for source" do + should "normalize correctly" do + source1 = "http://lohas.nicoseiga.jp/priv/3521156?e=1382558156&h=f2e089256abd1d453a455ec8f317a6c703e2cedf" + source2 = "http://lohas.nicoseiga.jp/priv/b80f86c0d8591b217e7513a9e175e94e00f3c7a1/1384936074/3583893" + source3 = "http://lohas.nicoseiga.jp/o/910aecf08e542285862954017f8a33a8c32a8aec/1433298801/4937663" + source4 = "http://seiga.nicovideo.jp/image/source?id=3312222" + + assert_equal("https://seiga.nicovideo.jp/seiga/im3521156", Sources::Strategies.normalize_source(source1)) + assert_equal("https://seiga.nicovideo.jp/seiga/im3583893", Sources::Strategies.normalize_source(source2)) + assert_equal("https://seiga.nicovideo.jp/seiga/im4937663", Sources::Strategies.normalize_source(source3)) + assert_equal("https://seiga.nicovideo.jp/seiga/im3312222", Sources::Strategies.normalize_source(source4)) + end + + should "avoid normalizing unnormalizable urls" do + bad_source = "https://seiga.nicovideo.jp" + assert_equal(bad_source, Sources::Strategies.normalize_source(bad_source)) + end + end end end diff --git a/test/unit/sources/nijie_test.rb b/test/unit/sources/nijie_test.rb index dee8882d1..1b4b2092e 100644 --- a/test/unit/sources/nijie_test.rb +++ b/test/unit/sources/nijie_test.rb @@ -274,5 +274,25 @@ module Sources end end end + + context "normalizing for source" do + should "normalize correctly" do + source1 = "https://pic01.nijie.info/nijie_picture/diff/main/218856_0_236014_20170620101329.png" + source2 = "https://pic04.nijie.info/nijie_picture/diff/main/287736_161475_20181112032855_1.png" + + assert_equal("https://nijie.info/view.php?id=218856", Sources::Strategies.normalize_source(source1)) + assert_equal("https://nijie.info/view.php?id=287736", Sources::Strategies.normalize_source(source2)) + end + + should "avoid normalizing unnormalizable urls" do + bad_source1 = "https://pic01.nijie.info/nijie_picture/20120211210359.jpg" + bad_source2 = "https://pic04.nijie.info/omata/4829_20161128012012.png" + bad_source3 = "https://pic03.nijie.info/nijie_picture/28310_20131101215959.jpg" + + assert_equal(bad_source1, Sources::Strategies.normalize_source(bad_source1)) + assert_equal(bad_source2, Sources::Strategies.normalize_source(bad_source2)) + assert_equal(bad_source3, Sources::Strategies.normalize_source(bad_source3)) + end + end end end diff --git a/test/unit/sources/null_test.rb b/test/unit/sources/null_test.rb index afa644069..b1ef6e3a1 100644 --- a/test/unit/sources/null_test.rb +++ b/test/unit/sources/null_test.rb @@ -23,10 +23,150 @@ module Sources should "find the artist" do a1 = FactoryBot.create(:artist, name: "test1", url_string: "http://oremuhax.x0.com") - a2 = FactoryBot.create(:artist, name: "test2", url_string: "http://google.com") assert_equal([a1], @site.artists) end end + + context "normalizing for source" do + should "normalize twitpic links" do + source = "http://d3j5vwomefv46c.cloudfront.net/photos/large/820960031.jpg?1384107199" + assert_equal("https://twitpic.com/dks0tb", Sources::Strategies.normalize_source(source)) + end + + should "normalize karabako links" do + source = "http://www.karabako.net/images/karabako_38835.jpg" + assert_equal("http://www.karabako.net/post/view/38835", Sources::Strategies.normalize_source(source)) + end + + should "normalize twipple links" do + source = "http://p.twpl.jp/show/orig/mI2c3" + assert_equal("http://p.twipple.jp/mI2c3", Sources::Strategies.normalize_source(source)) + end + + should "normalize fc2 links" do + source1 = "https://blog-imgs-41.fc2.com/t/u/y/tuyadasi/file.png" + source2 = "http://diary.fc2.com/user/kazuharoom/img/2020_1/29.jpg" + + assert_equal("http://tuyadasi.blog.fc2.com/img/file.png/", Sources::Strategies.normalize_source(source1)) + assert_equal("http://diary.fc2.com/cgi-sys/ed.cgi/kazuharoom?Y=2020&M=1&D=29", Sources::Strategies.normalize_source(source2)) + end + + should "normalize facebook links" do + source = "https://scontent-sin1-1.xx.fbcdn.net/hphotos-xtp1/t31.0-8/11254493_576443445841777_7716273903390212288_o.jpg" + assert_equal("https://www.facebook.com/photo.php?fbid=576443445841777", Sources::Strategies.normalize_source(source)) + end + + should "normalize sankaku links" do + source = "http://cs.sankakucomplex.com/data/sample/c2/d7/sample-c2d7270b84ac81326384d4eadd4d4746.jpg?2738848" + assert_equal("https://chan.sankakucomplex.com/en/post/show?md5=c2d7270b84ac81326384d4eadd4d4746", Sources::Strategies.normalize_source(source)) + end + + should "normalize zerochan links" do + source1 = "http://static.zerochan.net/full/23/15/183273.jpg" + source2 = "https://s4.zerochan.net/Victorique.de.Blois.full.411536.jpg" + source3 = "http://www.zerochan.net/full/1567893" + + assert_equal("https://www.zerochan.net/183273#full", Sources::Strategies.normalize_source(source1)) + assert_equal("https://www.zerochan.net/411536#full", Sources::Strategies.normalize_source(source2)) + assert_equal("https://www.zerochan.net/1567893#full", Sources::Strategies.normalize_source(source3)) + end + + should "normalize minitokyo links" do + source1 = "http://static.minitokyo.net/downloads/27/13/365677.jpg?433592448,Minitokyo.Eien.no.Aselia.Scans_365677.jpg" + source2 = "http://static.minitokyo.net/downloads/14/33/199164.jpg?928244019" + + assert_equal("http://gallery.minitokyo.net/download/365677", Sources::Strategies.normalize_source(source1)) + assert_equal("http://gallery.minitokyo.net/download/199164", Sources::Strategies.normalize_source(source2)) + end + + should "normalize gelbooru links" do + source1 = "https://gelbooru.com//images/ee/5c/ee5c9a69db9602c95debdb9b98fb3e3e.jpeg" + source2 = "http://simg.gelbooru.com//images/2003/edd1d2b3881cf70c3acf540780507531.png" + source3 = "https://simg3.gelbooru.com//samples/0b/3a/sample_0b3ae5e225072b8e391c827cb470d29c.jpg" + + assert_equal("https://gelbooru.com/index.php?page=post&s=list&md5=ee5c9a69db9602c95debdb9b98fb3e3e", Sources::Strategies.normalize_source(source1)) + assert_equal("https://gelbooru.com/index.php?page=post&s=list&md5=edd1d2b3881cf70c3acf540780507531", Sources::Strategies.normalize_source(source2)) + assert_equal("https://gelbooru.com/index.php?page=post&s=list&md5=0b3ae5e225072b8e391c827cb470d29c", Sources::Strategies.normalize_source(source3)) + end + + should "normalize wikia links" do + source = "https://vignette.wikia.nocookie.net/valkyriecrusade/images/c/c5/Crimson_Hatsune_H.png/revision/latest?cb=20180702031954" + assert_equal("https://valkyriecrusade.wikia.com/wiki/File:Crimson_Hatsune_H.png", Sources::Strategies.normalize_source(source)) + end + + should "normalize e-shuushuu links" do + source = "http://e-shuushuu.net/images/2014-07-22-662472.png" + assert_equal("https://e-shuushuu.net/image/662472", Sources::Strategies.normalize_source(source)) + end + + should "normalize nijigen-daiaru links" do + source = "http://jpg.nijigen-daiaru.com/19909/029.jpg" + assert_equal("http://nijigen-daiaru.com/book.php?idb=19909", Sources::Strategies.normalize_source(source)) + end + + should "normalize doujinantena links" do + source = "http://sozai.doujinantena.com/contents_jpg/d6c39f09d435e32c221e4ef866eceba4/015.jpg" + assert_equal("http://doujinantena.com/page.php?id=d6c39f09d435e32c221e4ef866eceba4", Sources::Strategies.normalize_source(source)) + end + + should "normalize paheal.net links" do + source = "http://rule34-data-010.paheal.net/_images/854806addcd3b1246424e7cea49afe31/852405%20-%20Darkstalkers%20Felicia.jpg" + assert_equal("https://rule34.paheal.net/post/list/md5:854806addcd3b1246424e7cea49afe31/1", Sources::Strategies.normalize_source(source)) + end + + should "normalize shimmie.katawa-shoujo.com links" do + source = "http://shimmie.katawa-shoujo.com/image/2740.png" + assert_equal("https://shimmie.katawa-shoujo.com/post/view/2740", Sources::Strategies.normalize_source(source)) + end + + should "normalize rule34.xxx links" do + source = "https://us.rule34.xxx//images/1802/0adc8fa0604dc445b4b47e6f4c436a08.jpeg?1949807" + assert_equal("https://rule34.xxx/index.php?page=post&s=list&md5=0adc8fa0604dc445b4b47e6f4c436a08", Sources::Strategies.normalize_source(source)) + end + + should "normalize diarypro links" do + source1 = "http://nekomataya.net/diarypro/data/upfile/216-1.jpg" + source2 = "http://akimbo.sakura.ne.jp/diarypro/diary.cgi?mode=image&upfile=716-3.jpg" + assert_equal("http://nekomataya.net/diarypro/diary.cgi?no=216", Sources::Strategies.normalize_source(source1)) + assert_equal("http://akimbo.sakura.ne.jp/diarypro/diary.cgi?no=716", Sources::Strategies.normalize_source(source2)) + end + + should "normalize minus.com links" do + source = "http://i1.minus.com/ibb0DuE2Ds0yE6.jpg" + assert_equal("http://minus.com/i/bb0DuE2Ds0yE6", Sources::Strategies.normalize_source(source)) + end + + should "normalize photozou links" do + source1 = "http://kura3.photozou.jp/pub/794/1481794/photo/161537258_org.v1364829097.jpg" + source2 = "http://art59.photozou.jp/pub/212/1986212/photo/118493247_org.v1534644005.jpg" + assert_equal("https://photozou.jp/photo/show/1481794/161537258", Sources::Strategies.normalize_source(source1)) + assert_equal("https://photozou.jp/photo/show/1986212/118493247", Sources::Strategies.normalize_source(source2)) + end + + should "normalize toranoana links" do + source1 = "http://img.toranoana.jp/popup_img/04/0030/09/76/040030097695-2p.jpg" + source2 = "https://ecdnimg.toranoana.jp/ec/img/04/0030/65/34/040030653417-6p.jpg" + assert_equal("https://ec.toranoana.jp/tora_r/ec/item/040030097695/", Sources::Strategies.normalize_source(source1)) + assert_equal("https://ec.toranoana.jp/tora_r/ec/item/040030653417/", Sources::Strategies.normalize_source(source2)) + end + + should "normalize hitomi.la links" do + source1 = "https://aa.hitomi.la/galleries/883451/t_rena1g.png" + source2 = "https://la.hitomi.la/galleries/1054851/001_main_image.jpg" + assert_equal("https://hitomi.la/galleries/883451.html", Sources::Strategies.normalize_source(source1)) + assert_equal("https://hitomi.la/reader/1054851.html#1", Sources::Strategies.normalize_source(source2)) + end + + should "leave unknown sources as they are" do + source1 = "https://google.com" + source2 = "a bad non-http source" + source3 = "https://example.com/Folder/中央大学.html" + + assert_equal(source1, Sources::Strategies.normalize_source(source1)) + assert_equal(source2, Sources::Strategies.normalize_source(source2)) + assert_equal(source3, Sources::Strategies.normalize_source(source3)) + end + end end end diff --git a/test/unit/sources/pawoo_test.rb b/test/unit/sources/pawoo_test.rb index 9458d7d66..a20485085 100644 --- a/test/unit/sources/pawoo_test.rb +++ b/test/unit/sources/pawoo_test.rb @@ -94,5 +94,23 @@ module Sources assert_equal("evazion", @site.artist_name) end end + + context "normalizing for source" do + should "normalize correctly" do + source1 = "https://pawoo.net/@evazion/19451018/" + source2 = "https://pawoo.net/web/statuses/19451018/favorites" + + assert_equal("https://pawoo.net/@evazion/19451018", Sources::Strategies.normalize_source(source1)) + assert_equal("https://pawoo.net/web/statuses/19451018", Sources::Strategies.normalize_source(source2)) + end + + should "avoid normalizing unnormalizable urls" do + bad_source1 = "https://img.pawoo.net/media_attachments/files/001/297/997/original/c4272a09570757c2.png" + bad_source2 = "https://pawoo.net/@evazion/media" + + assert_equal(bad_source1, Sources::Strategies.normalize_source(bad_source1)) + assert_equal(bad_source2, Sources::Strategies.normalize_source(bad_source2)) + end + end end end diff --git a/test/unit/sources/pixiv_test.rb b/test/unit/sources/pixiv_test.rb index 559202ce3..4ba9738e9 100644 --- a/test/unit/sources/pixiv_test.rb +++ b/test/unit/sources/pixiv_test.rb @@ -363,5 +363,21 @@ module Sources end end end + + context "normalizing for source" do + should "normalize correctly" do + source1 = "http://i2.pixiv.net/img12/img/zenze/39749565.png" + source2 = "http://i1.pixiv.net/img53/img/themare/39735353_big_p1.jpg" + source3 = "http://i1.pixiv.net/c/150x150/img-master/img/2010/11/30/08/39/58/14901720_p0_master1200.jpg" + source4 = "http://i1.pixiv.net/img-original/img/2010/11/30/08/39/58/14901720_p0.png" + source5 = "http://i2.pixiv.net/img-zip-ugoira/img/2014/08/05/06/01/10/44524589_ugoira1920x1080.zip" + + assert_equal("https://www.pixiv.net/artworks/39749565", Sources::Strategies.normalize_source(source1)) + assert_equal("https://www.pixiv.net/artworks/39735353", Sources::Strategies.normalize_source(source2)) + assert_equal("https://www.pixiv.net/artworks/14901720", Sources::Strategies.normalize_source(source3)) + assert_equal("https://www.pixiv.net/artworks/14901720", Sources::Strategies.normalize_source(source4)) + assert_equal("https://www.pixiv.net/artworks/44524589", Sources::Strategies.normalize_source(source5)) + end + end end end diff --git a/test/unit/sources/tumblr_test.rb b/test/unit/sources/tumblr_test.rb index fe29481c0..3b774f934 100644 --- a/test/unit/sources/tumblr_test.rb +++ b/test/unit/sources/tumblr_test.rb @@ -259,5 +259,24 @@ module Sources end end end + + context "normalizing for source" do + should "normalize correctly" do + source1 = "https://octrain1020.tumblr.com/post/190713122589" + source2 = "https://octrain1020.tumblr.com/image/190713122589" + source3 = "https://octrain1020.tumblr.com/image/190713122589#asd" + source4 = "https://superboin.tumblr.com/post/141169066579/photoset_iframe/superboin/tumblr_o45miiAOts1u6rxu8/500/false" + + assert_equal(source1, Sources::Strategies.normalize_source(source1)) + assert_equal(source1, Sources::Strategies.normalize_source(source2)) + assert_equal(source1, Sources::Strategies.normalize_source(source3)) + assert_equal("https://superboin.tumblr.com/post/141169066579", Sources::Strategies.normalize_source(source4)) + end + + should "avoid normalizing unnormalizable urls" do + bad_source = "https://octrain1020.tumblr.com/" + assert_equal(bad_source, Sources::Strategies.normalize_source(bad_source)) + end + end end end diff --git a/test/unit/sources/twitter_test.rb b/test/unit/sources/twitter_test.rb index 5d061d669..1f33d4ad2 100644 --- a/test/unit/sources/twitter_test.rb +++ b/test/unit/sources/twitter_test.rb @@ -298,5 +298,24 @@ module Sources assert_includes(site.translated_tags.map(&:name), "nishizumi_miho") end end + + context "normalizing for source" do + should "normalize correctly" do + source1 = "https://twitter.com/i/web/status/1261877313349640194" + source2 = "https://twitter.com/BOW999/status/1261877313349640194" + source3 = "https://twitter.com/BOW999/status/1261877313349640194/photo/1" + source4 = "https://twitter.com/BOW999/status/1261877313349640194?s=19" + + assert_equal(source1, Sources::Strategies.normalize_source(source1)) + assert_equal(source1, Sources::Strategies.normalize_source(source2)) + assert_equal(source1, Sources::Strategies.normalize_source(source3)) + assert_equal(source1, Sources::Strategies.normalize_source(source4)) + end + + should "normalize twimg twitpic correctly" do + source = "https://o.twimg.com/2/proxy.jpg?t=HBgpaHR0cHM6Ly90d2l0cGljLmNvbS9zaG93L2xhcmdlL2R0bnVydS5qcGcUsAkU0ggAFgASAA&s=dnN4DHCdnojC-iCJWdvZ-UZinrlWqAP7k7lmll2fTxs" + assert_equal("https://twitpic.com/dtnuru", Sources::Strategies.normalize_source(source)) + end + end end end