Refactor source normalization
* Move the source normalization logic out of the post model and into individual sources' strategies. * Rewrite normalization tests to be handled into each source's test, and expand them significantly. Previously we were only testing a very small subset of domains and variants. * Fix up normalization for several sites. * Normalize fav.me urls into normal deviantart urls.
This commit is contained in:
nonamethanks
@@ -1,25 +1,35 @@
|
||||
# Asset URLs:
|
||||
#
|
||||
# * http://orig12.deviantart.net/9b69/f/2017/023/7/c/illustration___tokyo_encount_oei__by_melisaongmiqin-dawi58s.png
|
||||
# * http://pre15.deviantart.net/81de/th/pre/f/2015/063/5/f/inha_by_inhaestudios-d8kfzm5.jpg
|
||||
# * http://th00.deviantart.net/fs71/PRE/f/2014/065/3/b/goruto_by_xyelkiltrox-d797tit.png
|
||||
## NORMALIZABLE
|
||||
## * http://orig12.deviantart.net/9b69/f/2017/023/7/c/illustration___tokyo_encount_oei__by_melisaongmiqin-dawi58s.png
|
||||
## * http://pre15.deviantart.net/81de/th/pre/f/2015/063/5/f/inha_by_inhaestudios-d8kfzm5.jpg
|
||||
## * http://th00.deviantart.net/fs71/PRE/f/2014/065/3/b/goruto_by_xyelkiltrox-d797tit.png
|
||||
##
|
||||
## * http://fc00.deviantart.net/fs71/f/2013/234/d/8/d84e05f26f0695b1153e9dab3a962f16-d6j8jl9.jpg
|
||||
## * http://th04.deviantart.net/fs71/PRE/f/2013/337/3/5/35081351f62b432f84eaeddeb4693caf-d6wlrqs.jpg
|
||||
##
|
||||
## * https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/52c4a3ad-d416-42f0-90f6-570983e36797/dczr28f-bd255304-01bf-4765-8cd3-e53983d3f78a.jpg
|
||||
## * https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/8b472d70-a0d6-41b5-9a66-c35687090acc/d23jbr4-8a06af02-70cb-46da-8a96-42a6ba73cdb4.jpg/v1/fill/w_786,h_1017,q_70,strp/silverhawks_quicksilver_by_edsfox_d23jbr4-pre.jpg
|
||||
## * https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/f/76098ac8-04ab-4784-b382-88ca082ba9b1/d9x7lmk-595099de-fe8f-48e5-9841-7254f9b2ab8d.png?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ1cm46YXBwOiIsImlzcyI6InVybjphcHA6Iiwib2JqIjpbW3sicGF0aCI6IlwvZlwvNzYwOThhYzgtMDRhYi00Nzg0LWIzODItODhjYTA4MmJhOWIxXC9kOXg3bG1rLTU5NTA5OWRlLWZlOGYtNDhlNS05ODQxLTcyNTRmOWIyYWI4ZC5wbmcifV1dLCJhdWQiOlsidXJuOnNlcnZpY2U6ZmlsZS5kb3dubG9hZCJdfQ.KFOVXAiF8MTlLb3oM-FlD0nnDvODmjqEhFYN5I2X5Bc
|
||||
## * https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/f/fe7ab27f-7530-4252-99ef-2baaf81b36fd/dddf6pe-1a4a091c-768c-4395-9465-5d33899be1eb.png/v1/fill/w_800,h_1130,q_80,strp/stay_hydrated_and_in_the_shade_by_raikoart_dddf6pe-fullview.jpg?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ1cm46YXBwOjdlMGQxODg5ODIyNjQzNzNhNWYwZDQxNWVhMGQyNmUwIiwiaXNzIjoidXJuOmFwcDo3ZTBkMTg4OTgyMjY0MzczYTVmMGQ0MTVlYTBkMjZlMCIsIm9iaiI6W1t7ImhlaWdodCI6Ijw9MTEzMCIsInBhdGgiOiJcL2ZcL2ZlN2FiMjdmLTc1MzAtNDI1Mi05OWVmLTJiYWFmODFiMzZmZFwvZGRkZjZwZS0xYTRhMDkxYy03NjhjLTQzOTUtOTQ2NS01ZDMzODk5YmUxZWIucG5nIiwid2lkdGgiOiI8PTgwMCJ9XV0sImF1ZCI6WyJ1cm46c2VydmljZTppbWFnZS5vcGVyYXRpb25zIl19.J0W4k-iV6Mg8Kt_5Lr_L_JbBq4lyr7aCausWWJ_Fsbw
|
||||
#
|
||||
# * http://th04.deviantart.net/fs70/300W/f/2009/364/4/d/Alphes_Mimic___Rika_by_Juriesute.png
|
||||
# * http://fc02.deviantart.net/fs48/f/2009/186/2/c/Animation_by_epe_tohri.swf
|
||||
# * http://fc08.deviantart.net/files/f/2007/120/c/9/Cool_Like_Me_by_47ness.jpg
|
||||
## * http://www.deviantart.com/download/135944599/Touhou___Suwako_Moriya_Colored_by_Turtle_Chibi.png
|
||||
## * https://www.deviantart.com/download/549677536/countdown_to_midnight_by_kawacy-d939hwg.jpg?token=92090cd3910d52089b566661e8c2f749755ed5f8&ts=1438535525
|
||||
#
|
||||
# * http://fc08.deviantart.net/images3/i/2004/088/8/f/Blackrose_for_MuzicFreq.jpg
|
||||
# * http://img04.deviantart.net/720b/i/2003/37/9/6/princess_peach.jpg
|
||||
## NOT NORMALIZABLE
|
||||
## * http://th04.deviantart.net/fs70/300W/f/2009/364/4/d/Alphes_Mimic___Rika_by_Juriesute.png
|
||||
## * http://fc02.deviantart.net/fs48/f/2009/186/2/c/Animation_by_epe_tohri.swf
|
||||
## * http://fc08.deviantart.net/files/f/2007/120/c/9/Cool_Like_Me_by_47ness.jpg
|
||||
##
|
||||
## * http://fc08.deviantart.net/images3/i/2004/088/8/f/Blackrose_for_MuzicFreq.jpg
|
||||
## * http://img04.deviantart.net/720b/i/2003/37/9/6/princess_peach.jpg
|
||||
##
|
||||
## * http://prnt00.deviantart.net/9b74/b/2016/101/4/468a9d89f52a835d4f6f1c8caca0dfb2-pnjfbh.jpg
|
||||
## * http://other00.deviantart.net/8863/o/2009/197/3/7/37ac79eaeef9fb32e6ae998e9a77d8dd.jpg
|
||||
## * http://fc09.deviantart.net/fs22/o/2009/197/3/7/37ac79eaeef9fb32e6ae998e9a77d8dd.jpg
|
||||
## * http://pre06.deviantart.net/8497/th/pre/f/2009/173/c/c/cc9686111dcffffffb5fcfaf0cf069fb.jpg
|
||||
#
|
||||
# * http://prnt00.deviantart.net/9b74/b/2016/101/4/468a9d89f52a835d4f6f1c8caca0dfb2-pnjfbh.jpg
|
||||
# * http://fc00.deviantart.net/fs71/f/2013/234/d/8/d84e05f26f0695b1153e9dab3a962f16-d6j8jl9.jpg
|
||||
# * http://th04.deviantart.net/fs71/PRE/f/2013/337/3/5/35081351f62b432f84eaeddeb4693caf-d6wlrqs.jpg
|
||||
#
|
||||
# * http://fc09.deviantart.net/fs22/o/2009/197/3/7/37ac79eaeef9fb32e6ae998e9a77d8dd.jpg
|
||||
# * http://pre06.deviantart.net/8497/th/pre/f/2009/173/c/c/cc9686111dcffffffb5fcfaf0cf069fb.jpg
|
||||
#
|
||||
# * https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/8b472d70-a0d6-41b5-9a66-c35687090acc/d23jbr4-8a06af02-70cb-46da-8a96-42a6ba73cdb4.jpg/v1/fill/w_786,h_1017,q_70,strp/silverhawks_quicksilver_by_edsfox_d23jbr4-pre.jpg
|
||||
# * https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/f/fe7ab27f-7530-4252-99ef-2baaf81b36fd/dddf6pe-1a4a091c-768c-4395-9465-5d33899be1eb.png/v1/fill/w_800,h_1130,q_80,strp/stay_hydrated_and_in_the_shade_by_raikoart_dddf6pe-fullview.jpg?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ1cm46YXBwOjdlMGQxODg5ODIyNjQzNzNhNWYwZDQxNWVhMGQyNmUwIiwiaXNzIjoidXJuOmFwcDo3ZTBkMTg4OTgyMjY0MzczYTVmMGQ0MTVlYTBkMjZlMCIsIm9iaiI6W1t7ImhlaWdodCI6Ijw9MTEzMCIsInBhdGgiOiJcL2ZcL2ZlN2FiMjdmLTc1MzAtNDI1Mi05OWVmLTJiYWFmODFiMzZmZFwvZGRkZjZwZS0xYTRhMDkxYy03NjhjLTQzOTUtOTQ2NS01ZDMzODk5YmUxZWIucG5nIiwid2lkdGgiOiI8PTgwMCJ9XV0sImF1ZCI6WyJ1cm46c2VydmljZTppbWFnZS5vcGVyYXRpb25zIl19.J0W4k-iV6Mg8Kt_5Lr_L_JbBq4lyr7aCausWWJ_Fsbw
|
||||
########################
|
||||
#
|
||||
# Page URLs:
|
||||
#
|
||||
@@ -39,26 +49,36 @@ module Sources
|
||||
class DeviantArt < Base
|
||||
ASSET_SUBDOMAINS = %r{(?:fc|th|pre|img|orig|origin-orig)\d*}i
|
||||
RESERVED_SUBDOMAINS = %r{\Ahttps?://(?:#{ASSET_SUBDOMAINS}|www)\.}i
|
||||
MAIN_DOMAIN = %r{\Ahttps?://(?:www\.)?deviantart.com}i
|
||||
|
||||
TITLE = %r{(?<title>[a-z0-9_-]+?)}i
|
||||
ARTIST = %r{(?<artist>[a-z0-9_-]+?)}i
|
||||
DEVIATION_ID = %r{(?<deviation_id>[0-9]+)}i
|
||||
|
||||
DA_FILENAME = %r{#{TITLE}(?:_by_#{ARTIST}(?:-d(?<base36_deviation_id>[a-z0-9]+))?)?\.}i
|
||||
WIX_FILENAME = %r{#{TITLE}_by_#{ARTIST}_d(?<base36_deviation_id>[a-z0-9]+)-[a-z0-9]+\.}i
|
||||
DA_FILENAME_1 = %r{[a-f0-9]{32}-d(?<base36_deviation_id>[a-z0-9]+)\.}i
|
||||
DA_FILENAME_2 = %r{#{TITLE}(?:_by_#{ARTIST}(?:-d(?<base36_deviation_id>[a-z0-9]+))?)?\.}i
|
||||
DA_FILENAME = Regexp.union(DA_FILENAME_1, DA_FILENAME_2)
|
||||
WIX_FILENAME = %r{d(?<base36_deviation_id>[a-z0-9]+)[0-9a-f-]+\.\w+(?:/\w+/\w+/[\w,]+/(?<title>[\w-]+)_by_(?<artist>[\w-]+)_d\w+-\w+\.\w+)?.+}i
|
||||
|
||||
NOT_NORMALIZABLE_ASSET = %r{\Ahttps?://#{ASSET_SUBDOMAINS}\.deviantart\.net/.+/[0-9a-f]{32}(?:-[^d]\w+)?\.}i
|
||||
|
||||
DA_ASSET = %r{\Ahttps?://#{ASSET_SUBDOMAINS}\.deviantart\.net/.+/#{DA_FILENAME}}i
|
||||
WIX_ASSET = %r{\Ahttps?://images-wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/.+/#{WIX_FILENAME}}i
|
||||
WIX_ASSET = %r{\Ahttps?://images-wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/(?:intermediary/)?\w/[0-9a-f-]+/#{WIX_FILENAME}}i
|
||||
ASSET = Regexp.union(DA_ASSET, WIX_ASSET)
|
||||
|
||||
PATH_ART = %r{\Ahttps?://www\.deviantart\.com/#{ARTIST}/art/#{TITLE}-#{DEVIATION_ID}\z}i
|
||||
DA_DOWNLOAD = %r{#{MAIN_DOMAIN}/download/#{DEVIATION_ID}/#{DA_FILENAME_2}?}i
|
||||
|
||||
DEVIATION_ART = %r{#{MAIN_DOMAIN}/deviation/#{DEVIATION_ID}\z}i
|
||||
PATH_ART = %r{#{MAIN_DOMAIN}/#{ARTIST}/art/#{TITLE}-#{DEVIATION_ID}\z}i
|
||||
SUBDOMAIN_ART = %r{\Ahttps?://#{ARTIST}\.deviantart\.com/art/#{TITLE}-#{DEVIATION_ID}\z}i
|
||||
|
||||
PATH_PROFILE = %r{\Ahttps?://(www\.)?deviantart\.com/#{ARTIST}/?\z}i
|
||||
PATH_PROFILE = %r{#{MAIN_DOMAIN}/#{ARTIST}/?\z}i
|
||||
SUBDOMAIN_PROFILE = %r{\Ahttps?://#{ARTIST}\.deviantart\.com/?\z}i
|
||||
|
||||
FAVME = %r{\Ahttps?://(www\.)?fav\.me/d(?<base36_deviation_id>[a-z0-9]+)\z}i
|
||||
|
||||
def domains
|
||||
["deviantart.net", "deviantart.com"]
|
||||
["deviantart.net", "deviantart.com", "fav.me"]
|
||||
end
|
||||
|
||||
def site_name
|
||||
@@ -70,14 +90,6 @@ module Sources
|
||||
parsed_url.domain.in?(domains) || parsed_url.host == "images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com"
|
||||
end
|
||||
|
||||
def canonical_url
|
||||
if self.class.deviation_id_from_url(image_url).present? || page_url.blank?
|
||||
image_url
|
||||
else
|
||||
page_url
|
||||
end
|
||||
end
|
||||
|
||||
def image_urls
|
||||
[image_url]
|
||||
end
|
||||
@@ -120,12 +132,16 @@ module Sources
|
||||
if artist.present? && title.present? && id.present?
|
||||
"https://www.deviantart.com/#{artist}/art/#{title}-#{id}"
|
||||
elsif id.present?
|
||||
"https://deviantart.com/deviation/#{id}"
|
||||
"https://www.deviantart.com/deviation/#{id}"
|
||||
else
|
||||
nil
|
||||
end
|
||||
end
|
||||
|
||||
def normalize_for_source
|
||||
page_url_from_image_url
|
||||
end
|
||||
|
||||
def profile_url
|
||||
return nil if artist_name.blank?
|
||||
"https://www.deviantart.com/#{artist_name.downcase}"
|
||||
@@ -204,9 +220,11 @@ module Sources
|
||||
end
|
||||
|
||||
def self.deviation_id_from_url(url)
|
||||
if url =~ ASSET
|
||||
if url =~ NOT_NORMALIZABLE_ASSET
|
||||
nil
|
||||
elsif url =~ ASSET || url =~ FAVME
|
||||
$~[:base36_deviation_id].try(:to_i, 36)
|
||||
elsif url =~ PATH_ART || (url !~ RESERVED_SUBDOMAINS && url =~ SUBDOMAIN_ART)
|
||||
elsif url =~ PATH_ART || (url !~ RESERVED_SUBDOMAINS && url =~ SUBDOMAIN_ART) || url =~ DA_DOWNLOAD || url =~ DEVIATION_ART
|
||||
$~[:deviation_id].to_i
|
||||
else
|
||||
nil
|
||||
@@ -214,7 +232,9 @@ module Sources
|
||||
end
|
||||
|
||||
def self.artist_name_from_url(url)
|
||||
if url =~ ASSET || url =~ PATH_ART || url =~ PATH_PROFILE
|
||||
if url =~ NOT_NORMALIZABLE_ASSET
|
||||
nil
|
||||
elsif url =~ ASSET || url =~ PATH_ART || url =~ PATH_PROFILE || url =~ DA_DOWNLOAD
|
||||
$~[:artist].try(:dasherize)
|
||||
elsif url !~ RESERVED_SUBDOMAINS && (url =~ SUBDOMAIN_ART || url =~ SUBDOMAIN_PROFILE)
|
||||
$~[:artist]
|
||||
@@ -224,9 +244,11 @@ module Sources
|
||||
end
|
||||
|
||||
def self.title_from_url(url)
|
||||
if url =~ ASSET || url =~ PATH_ART || url =~ PATH_PROFILE
|
||||
if url =~ NOT_NORMALIZABLE_ASSET
|
||||
nil
|
||||
elsif url =~ ASSET || url =~ PATH_ART || url =~ DA_DOWNLOAD
|
||||
$~[:title].to_s.titleize.strip.squeeze(" ").tr(" ", "-").presence
|
||||
elsif url !~ RESERVED_SUBDOMAINS && (url =~ SUBDOMAIN_ART || url =~ SUBDOMAIN_PROFILE)
|
||||
elsif url !~ RESERVED_SUBDOMAINS && url =~ SUBDOMAIN_ART
|
||||
$~[:title].to_s.titleize.strip.squeeze(" ").tr(" ", "-").presence
|
||||
else
|
||||
nil
|
||||
|
||||
Reference in New Issue
Block a user