Refactor source normalization

* Move the source normalization logic out of the post model and into individual sources' strategies. * Rewrite normalization tests to be handled into each source's test, and expand them significantly. Previously we were only testing a very small subset of domains and variants. * Fix up normalization for several sites. * Normalize fav.me urls into normal deviantart urls.
2020-05-16 23:03:09 +02:00
parent 364343453c
commit 307df3b3e4
26 changed files with 674 additions and 315 deletions
--- a/app/logical/sources/strategies.rb
+++ b/app/logical/sources/strategies.rb
@@ -24,5 +24,9 @@ module Sources
    def self.canonical(url, referer)
      find(url, referer).canonical_url
    end
+
+    def self.normalize_source(url)
+      find(url).normalize_for_source || url
+    end
  end
 end
--- a/app/logical/sources/strategies/art_station.rb
+++ b/app/logical/sources/strategies/art_station.rb
@@ -3,6 +3,7 @@
 # * https://www.artstation.com/artwork/04XA4
 # * https://www.artstation.com/artwork/cody-from-sf
 # * https://sa-dui.artstation.com/projects/DVERn
+# * https://dudeunderscore.artstation.com/projects/NoNmD?album_id=23041
 #
 # Profile URLs:
 #
@@ -20,7 +21,7 @@
 module Sources::Strategies
  class ArtStation < Base
    PROJECT1 = %r!\Ahttps?://www\.artstation\.com/artwork/(?<project_id>[a-z0-9-]+)/?\z!i
-    PROJECT2 = %r!\Ahttps?://(?<artist_name>[\w-]+)\.artstation\.com/projects/(?<project_id>[a-z0-9-]+)/?\z!i
+    PROJECT2 = %r!\Ahttps?://(?<artist_name>[\w-]+)\.artstation\.com/projects/(?<project_id>[a-z0-9-]+)(?:/|\?[\w=-]+)?\z!i
    PROJECT = Regexp.union(PROJECT1, PROJECT2)
    ARTIST1 = %r{\Ahttps?://(?<artist_name>[\w-]+)(?<!www)\.artstation\.com/?\z}i
    ARTIST2 = %r{\Ahttps?://www\.artstation\.com/artist/(?<artist_name>[\w-]+)/?\z}i
@@ -84,6 +85,16 @@ module Sources::Strategies
      profile_url.present? && url == profile_url
    end

+    def normalize_for_source
+      return if project_id.blank?
+
+      if artist_name_from_url.present?
+        "https://#{artist_name_from_url}.artstation.com/projects/#{project_id}"
+      else
+        "https://www.artstation.com/artwork/#{project_id}"
+      end
+    end
+
    def image_urls_sub
      if url.match?(ASSET)
        return [url]
--- a/app/logical/sources/strategies/base.rb
+++ b/app/logical/sources/strategies/base.rb
@@ -177,6 +177,12 @@ module Sources
        profile_url.presence || url
      end

+      # Given a post/image url, this is the normalized url that will be displayed in a post's page in its stead.
+      # This function should never make any network call, even indirectly. Return nil to never normalize.
+      def normalize_for_source
+        nil
+      end
+
      def artists
        ArtistFinder.find_artists(normalize_for_artist_finder.to_s)
      end
--- a/app/logical/sources/strategies/deviant_art.rb
+++ b/app/logical/sources/strategies/deviant_art.rb
@@ -1,25 +1,35 @@
 # Asset URLs:
 #
-# * http://orig12.deviantart.net/9b69/f/2017/023/7/c/illustration___tokyo_encount_oei__by_melisaongmiqin-dawi58s.png
-# * http://pre15.deviantart.net/81de/th/pre/f/2015/063/5/f/inha_by_inhaestudios-d8kfzm5.jpg
-# * http://th00.deviantart.net/fs71/PRE/f/2014/065/3/b/goruto_by_xyelkiltrox-d797tit.png
+## NORMALIZABLE
+## * http://orig12.deviantart.net/9b69/f/2017/023/7/c/illustration___tokyo_encount_oei__by_melisaongmiqin-dawi58s.png
+## * http://pre15.deviantart.net/81de/th/pre/f/2015/063/5/f/inha_by_inhaestudios-d8kfzm5.jpg
+## * http://th00.deviantart.net/fs71/PRE/f/2014/065/3/b/goruto_by_xyelkiltrox-d797tit.png
+##
+## * http://fc00.deviantart.net/fs71/f/2013/234/d/8/d84e05f26f0695b1153e9dab3a962f16-d6j8jl9.jpg
+## * http://th04.deviantart.net/fs71/PRE/f/2013/337/3/5/35081351f62b432f84eaeddeb4693caf-d6wlrqs.jpg
+##
+## * https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/52c4a3ad-d416-42f0-90f6-570983e36797/dczr28f-bd255304-01bf-4765-8cd3-e53983d3f78a.jpg
+## * https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/8b472d70-a0d6-41b5-9a66-c35687090acc/d23jbr4-8a06af02-70cb-46da-8a96-42a6ba73cdb4.jpg/v1/fill/w_786,h_1017,q_70,strp/silverhawks_quicksilver_by_edsfox_d23jbr4-pre.jpg
+## * https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/f/76098ac8-04ab-4784-b382-88ca082ba9b1/d9x7lmk-595099de-fe8f-48e5-9841-7254f9b2ab8d.png?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ1cm46YXBwOiIsImlzcyI6InVybjphcHA6Iiwib2JqIjpbW3sicGF0aCI6IlwvZlwvNzYwOThhYzgtMDRhYi00Nzg0LWIzODItODhjYTA4MmJhOWIxXC9kOXg3bG1rLTU5NTA5OWRlLWZlOGYtNDhlNS05ODQxLTcyNTRmOWIyYWI4ZC5wbmcifV1dLCJhdWQiOlsidXJuOnNlcnZpY2U6ZmlsZS5kb3dubG9hZCJdfQ.KFOVXAiF8MTlLb3oM-FlD0nnDvODmjqEhFYN5I2X5Bc
+## * https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/f/fe7ab27f-7530-4252-99ef-2baaf81b36fd/dddf6pe-1a4a091c-768c-4395-9465-5d33899be1eb.png/v1/fill/w_800,h_1130,q_80,strp/stay_hydrated_and_in_the_shade_by_raikoart_dddf6pe-fullview.jpg?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ1cm46YXBwOjdlMGQxODg5ODIyNjQzNzNhNWYwZDQxNWVhMGQyNmUwIiwiaXNzIjoidXJuOmFwcDo3ZTBkMTg4OTgyMjY0MzczYTVmMGQ0MTVlYTBkMjZlMCIsIm9iaiI6W1t7ImhlaWdodCI6Ijw9MTEzMCIsInBhdGgiOiJcL2ZcL2ZlN2FiMjdmLTc1MzAtNDI1Mi05OWVmLTJiYWFmODFiMzZmZFwvZGRkZjZwZS0xYTRhMDkxYy03NjhjLTQzOTUtOTQ2NS01ZDMzODk5YmUxZWIucG5nIiwid2lkdGgiOiI8PTgwMCJ9XV0sImF1ZCI6WyJ1cm46c2VydmljZTppbWFnZS5vcGVyYXRpb25zIl19.J0W4k-iV6Mg8Kt_5Lr_L_JbBq4lyr7aCausWWJ_Fsbw
 #
-# * http://th04.deviantart.net/fs70/300W/f/2009/364/4/d/Alphes_Mimic___Rika_by_Juriesute.png
-# * http://fc02.deviantart.net/fs48/f/2009/186/2/c/Animation_by_epe_tohri.swf
-# * http://fc08.deviantart.net/files/f/2007/120/c/9/Cool_Like_Me_by_47ness.jpg
+## * http://www.deviantart.com/download/135944599/Touhou___Suwako_Moriya_Colored_by_Turtle_Chibi.png
+## * https://www.deviantart.com/download/549677536/countdown_to_midnight_by_kawacy-d939hwg.jpg?token=92090cd3910d52089b566661e8c2f749755ed5f8&ts=1438535525
 #
-# * http://fc08.deviantart.net/images3/i/2004/088/8/f/Blackrose_for_MuzicFreq.jpg
-# * http://img04.deviantart.net/720b/i/2003/37/9/6/princess_peach.jpg
+## NOT NORMALIZABLE
+##  * http://th04.deviantart.net/fs70/300W/f/2009/364/4/d/Alphes_Mimic___Rika_by_Juriesute.png
+##  * http://fc02.deviantart.net/fs48/f/2009/186/2/c/Animation_by_epe_tohri.swf
+##  * http://fc08.deviantart.net/files/f/2007/120/c/9/Cool_Like_Me_by_47ness.jpg
+##
+##  * http://fc08.deviantart.net/images3/i/2004/088/8/f/Blackrose_for_MuzicFreq.jpg
+##  * http://img04.deviantart.net/720b/i/2003/37/9/6/princess_peach.jpg
+##
+##  * http://prnt00.deviantart.net/9b74/b/2016/101/4/468a9d89f52a835d4f6f1c8caca0dfb2-pnjfbh.jpg
+##  * http://other00.deviantart.net/8863/o/2009/197/3/7/37ac79eaeef9fb32e6ae998e9a77d8dd.jpg
+##  * http://fc09.deviantart.net/fs22/o/2009/197/3/7/37ac79eaeef9fb32e6ae998e9a77d8dd.jpg
+##  * http://pre06.deviantart.net/8497/th/pre/f/2009/173/c/c/cc9686111dcffffffb5fcfaf0cf069fb.jpg
 #
-# * http://prnt00.deviantart.net/9b74/b/2016/101/4/468a9d89f52a835d4f6f1c8caca0dfb2-pnjfbh.jpg
-# * http://fc00.deviantart.net/fs71/f/2013/234/d/8/d84e05f26f0695b1153e9dab3a962f16-d6j8jl9.jpg
-# * http://th04.deviantart.net/fs71/PRE/f/2013/337/3/5/35081351f62b432f84eaeddeb4693caf-d6wlrqs.jpg
-#
-# * http://fc09.deviantart.net/fs22/o/2009/197/3/7/37ac79eaeef9fb32e6ae998e9a77d8dd.jpg
-# * http://pre06.deviantart.net/8497/th/pre/f/2009/173/c/c/cc9686111dcffffffb5fcfaf0cf069fb.jpg
-#
-# * https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/8b472d70-a0d6-41b5-9a66-c35687090acc/d23jbr4-8a06af02-70cb-46da-8a96-42a6ba73cdb4.jpg/v1/fill/w_786,h_1017,q_70,strp/silverhawks_quicksilver_by_edsfox_d23jbr4-pre.jpg
-# * https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/f/fe7ab27f-7530-4252-99ef-2baaf81b36fd/dddf6pe-1a4a091c-768c-4395-9465-5d33899be1eb.png/v1/fill/w_800,h_1130,q_80,strp/stay_hydrated_and_in_the_shade_by_raikoart_dddf6pe-fullview.jpg?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ1cm46YXBwOjdlMGQxODg5ODIyNjQzNzNhNWYwZDQxNWVhMGQyNmUwIiwiaXNzIjoidXJuOmFwcDo3ZTBkMTg4OTgyMjY0MzczYTVmMGQ0MTVlYTBkMjZlMCIsIm9iaiI6W1t7ImhlaWdodCI6Ijw9MTEzMCIsInBhdGgiOiJcL2ZcL2ZlN2FiMjdmLTc1MzAtNDI1Mi05OWVmLTJiYWFmODFiMzZmZFwvZGRkZjZwZS0xYTRhMDkxYy03NjhjLTQzOTUtOTQ2NS01ZDMzODk5YmUxZWIucG5nIiwid2lkdGgiOiI8PTgwMCJ9XV0sImF1ZCI6WyJ1cm46c2VydmljZTppbWFnZS5vcGVyYXRpb25zIl19.J0W4k-iV6Mg8Kt_5Lr_L_JbBq4lyr7aCausWWJ_Fsbw
+########################
 #
 # Page URLs:
 #
@@ -39,26 +49,36 @@ module Sources
    class DeviantArt < Base
      ASSET_SUBDOMAINS = %r{(?:fc|th|pre|img|orig|origin-orig)\d*}i
      RESERVED_SUBDOMAINS = %r{\Ahttps?://(?:#{ASSET_SUBDOMAINS}|www)\.}i
+      MAIN_DOMAIN = %r{\Ahttps?://(?:www\.)?deviantart.com}i

      TITLE = %r{(?<title>[a-z0-9_-]+?)}i
      ARTIST = %r{(?<artist>[a-z0-9_-]+?)}i
      DEVIATION_ID = %r{(?<deviation_id>[0-9]+)}i

-      DA_FILENAME  = %r{#{TITLE}(?:_by_#{ARTIST}(?:-d(?<base36_deviation_id>[a-z0-9]+))?)?\.}i
-      WIX_FILENAME = %r{#{TITLE}_by_#{ARTIST}_d(?<base36_deviation_id>[a-z0-9]+)-[a-z0-9]+\.}i
+      DA_FILENAME_1 = %r{[a-f0-9]{32}-d(?<base36_deviation_id>[a-z0-9]+)\.}i
+      DA_FILENAME_2 = %r{#{TITLE}(?:_by_#{ARTIST}(?:-d(?<base36_deviation_id>[a-z0-9]+))?)?\.}i
+      DA_FILENAME = Regexp.union(DA_FILENAME_1, DA_FILENAME_2)
+      WIX_FILENAME  = %r{d(?<base36_deviation_id>[a-z0-9]+)[0-9a-f-]+\.\w+(?:/\w+/\w+/[\w,]+/(?<title>[\w-]+)_by_(?<artist>[\w-]+)_d\w+-\w+\.\w+)?.+}i
+
+      NOT_NORMALIZABLE_ASSET = %r{\Ahttps?://#{ASSET_SUBDOMAINS}\.deviantart\.net/.+/[0-9a-f]{32}(?:-[^d]\w+)?\.}i

      DA_ASSET = %r{\Ahttps?://#{ASSET_SUBDOMAINS}\.deviantart\.net/.+/#{DA_FILENAME}}i
-      WIX_ASSET = %r{\Ahttps?://images-wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/.+/#{WIX_FILENAME}}i
+      WIX_ASSET = %r{\Ahttps?://images-wixmp-ed30a86b8c4ca887773594c2\.wixmp\.com/(?:intermediary/)?\w/[0-9a-f-]+/#{WIX_FILENAME}}i
      ASSET = Regexp.union(DA_ASSET, WIX_ASSET)

-      PATH_ART = %r{\Ahttps?://www\.deviantart\.com/#{ARTIST}/art/#{TITLE}-#{DEVIATION_ID}\z}i
+      DA_DOWNLOAD = %r{#{MAIN_DOMAIN}/download/#{DEVIATION_ID}/#{DA_FILENAME_2}?}i
+
+      DEVIATION_ART = %r{#{MAIN_DOMAIN}/deviation/#{DEVIATION_ID}\z}i
+      PATH_ART = %r{#{MAIN_DOMAIN}/#{ARTIST}/art/#{TITLE}-#{DEVIATION_ID}\z}i
      SUBDOMAIN_ART = %r{\Ahttps?://#{ARTIST}\.deviantart\.com/art/#{TITLE}-#{DEVIATION_ID}\z}i

-      PATH_PROFILE = %r{\Ahttps?://(www\.)?deviantart\.com/#{ARTIST}/?\z}i
+      PATH_PROFILE = %r{#{MAIN_DOMAIN}/#{ARTIST}/?\z}i
      SUBDOMAIN_PROFILE = %r{\Ahttps?://#{ARTIST}\.deviantart\.com/?\z}i

+      FAVME = %r{\Ahttps?://(www\.)?fav\.me/d(?<base36_deviation_id>[a-z0-9]+)\z}i
+
      def domains
-        ["deviantart.net", "deviantart.com"]
+        ["deviantart.net", "deviantart.com", "fav.me"]
      end

      def site_name
@@ -70,14 +90,6 @@ module Sources
        parsed_url.domain.in?(domains) || parsed_url.host == "images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com"
      end

-      def canonical_url
-        if self.class.deviation_id_from_url(image_url).present? || page_url.blank?
-          image_url
-        else
-          page_url
-        end
-      end
-
      def image_urls
        [image_url]
      end
@@ -120,12 +132,16 @@ module Sources
        if artist.present? && title.present? && id.present?
          "https://www.deviantart.com/#{artist}/art/#{title}-#{id}"
        elsif id.present?
-          "https://deviantart.com/deviation/#{id}"
+          "https://www.deviantart.com/deviation/#{id}"
        else
          nil
        end
      end

+      def normalize_for_source
+        page_url_from_image_url
+      end
+
      def profile_url
        return nil if artist_name.blank?
        "https://www.deviantart.com/#{artist_name.downcase}"
@@ -204,9 +220,11 @@ module Sources
      end

      def self.deviation_id_from_url(url)
-        if url =~ ASSET
+        if url =~ NOT_NORMALIZABLE_ASSET
+          nil
+        elsif url =~ ASSET || url =~ FAVME
          $~[:base36_deviation_id].try(:to_i, 36)
-        elsif url =~ PATH_ART || (url !~ RESERVED_SUBDOMAINS && url =~ SUBDOMAIN_ART)
+        elsif url =~ PATH_ART || (url !~ RESERVED_SUBDOMAINS && url =~ SUBDOMAIN_ART) || url =~ DA_DOWNLOAD || url =~ DEVIATION_ART
          $~[:deviation_id].to_i
        else
          nil
@@ -214,7 +232,9 @@ module Sources
      end

      def self.artist_name_from_url(url)
-        if url =~ ASSET || url =~ PATH_ART || url =~ PATH_PROFILE
+        if url =~ NOT_NORMALIZABLE_ASSET
+          nil
+        elsif url =~ ASSET || url =~ PATH_ART || url =~ PATH_PROFILE || url =~ DA_DOWNLOAD
          $~[:artist].try(:dasherize)
        elsif url !~ RESERVED_SUBDOMAINS && (url =~ SUBDOMAIN_ART || url =~ SUBDOMAIN_PROFILE)
          $~[:artist]
@@ -224,9 +244,11 @@ module Sources
      end

      def self.title_from_url(url)
-        if url =~ ASSET || url =~ PATH_ART || url =~ PATH_PROFILE
+        if url =~ NOT_NORMALIZABLE_ASSET
+          nil
+        elsif url =~ ASSET || url =~ PATH_ART || url =~ DA_DOWNLOAD
          $~[:title].to_s.titleize.strip.squeeze(" ").tr(" ", "-").presence
-        elsif url !~ RESERVED_SUBDOMAINS && (url =~ SUBDOMAIN_ART || url =~ SUBDOMAIN_PROFILE)
+        elsif url !~ RESERVED_SUBDOMAINS && url =~ SUBDOMAIN_ART
          $~[:title].to_s.titleize.strip.squeeze(" ").tr(" ", "-").presence
        else
          nil
--- a/app/logical/sources/strategies/hentai_foundry.rb
+++ b/app/logical/sources/strategies/hentai_foundry.rb
@@ -5,7 +5,9 @@
 # Page URLs
 #
 # * https://www.hentai-foundry.com/pictures/user/Afrobull/795025/kuroeda
-# * https://www.hentai-foundry.com/pictures/user/Afrobull/795025o
+# * https://www.hentai-foundry.com/pictures/user/Afrobull/795025
+# * http://www.hentai-foundry.com/pic-795025
+# * http://www.hentai-foundry.com/pictures/user/Ganassa/457176/LOL-Swimsuit---Caitlyn-reworked-nude-ver.
 #
 # Preview URLs
 #
@@ -22,9 +24,10 @@ module Sources
  module Strategies
    class HentaiFoundry < Base
      BASE_URL =    %r!\Ahttps?://(?:www\.)?hentai-foundry\.com!i
-      PAGE_URL =    %r!#{BASE_URL}/pictures/user/(?<artist_name>[\w-]+)/(?<illust_id>\d+)(?:/[\w-]*)?(\?[\w=]*)?\z!i
+      PAGE_URL =    %r!#{BASE_URL}/pictures/user/(?<artist_name>[\w-]+)/(?<illust_id>\d+)(?:/[\w.-]*)?(\?[\w=]*)?\z!i
+      OLD_PAGE =    %r!#{BASE_URL}/pic-(?<illust_id>\d+)(?:\.html)?\z!i
      PROFILE_URL = %r!#{BASE_URL}/(?:pictures/)?user/(?<artist_name>[\w-]+)(?:/[a-z]*)?\z!i
-      IMAGE_URL =   %r!\Ahttps?://pictures\.hentai-foundry\.com/\w/(?<artist_name>[\w-]+)/(?<illust_id>\d+)/[\w-]+\.\w+\z!i
+      IMAGE_URL =   %r!\Ahttps?://pictures\.hentai-foundry\.com/+\w/(?<artist_name>[\w-]+)/(?<illust_id>\d+)(?:(?:/[\w.-]+)?\.\w+)?\z!i

      def domains
        ["hentai-foundry.com"]
@@ -49,8 +52,13 @@ module Sources
      end

      def page_url
-        return nil if illust_id.blank? || artist_name.blank?
-        "https://www.hentai-foundry.com/pictures/user/#{artist_name}/#{illust_id}"
+        return nil if illust_id.blank?
+
+        if artist_name.blank?
+          "https://www.hentai-foundry.com/pic-#{illust_id}"
+        else
+          "https://www.hentai-foundry.com/pictures/user/#{artist_name}/#{illust_id}"
+        end
      end

      def page
@@ -104,8 +112,12 @@ module Sources
        url =~ PROFILE_URL
      end

+      def normalize_for_source
+        page_url
+      end
+
      def illust_id
-        url[PAGE_URL, :illust_id] || url[IMAGE_URL, :illust_id]
+        url[PAGE_URL, :illust_id] || url[IMAGE_URL, :illust_id] || url[OLD_PAGE, :illust_id]
      end
    end
  end
--- a/app/logical/sources/strategies/moebooru.rb
+++ b/app/logical/sources/strategies/moebooru.rb
@@ -1,6 +1,7 @@
 # Original images:
 #
 # * https://yande.re/image/b4b1d11facd1700544554e4805d47bb6/.png
+# * https://files.yande.re/image/e4c2ba38de88ff1640aaebff84c84e81/469784.jpg
 # * https://files.yande.re/image/2a5d1d688f565cb08a69ecf4e35017ab/yande.re%20349790%20breast_hold%20kurashima_tomoyasu%20mahouka_koukou_no_rettousei%20naked%20nipples.jpg
 # * https://ayase.yande.re/image/2d0d229fd8465a325ee7686fcc7f75d2/yande.re%20192481%20animal_ears%20bunny_ears%20garter_belt%20headphones%20mitha%20stockings%20thighhighs.jpg
 # * https://yuno.yande.re/image/1764b95ae99e1562854791c232e3444b/yande.re%20281544%20cameltoe%20erect_nipples%20fundoshi%20horns%20loli%20miyama-zero%20sarashi%20sling_bikini%20swimsuits.jpg
@@ -33,7 +34,7 @@ module Sources
    class Moebooru < Base
      BASE_URL = %r!\Ahttps?://(?:[^.]+\.)?(?<domain>yande\.re|konachan\.com)!i
      POST_URL = %r!#{BASE_URL}/post/show/(?<id>\d+)!i
-      URL_SLUG = %r!/(?:yande\.re%20|Konachan\.com%20-%20)(?<id>\d+).*!i
+      URL_SLUG = %r!/(?:yande\.re%20|Konachan\.com%20-%20)?(?<id>\d+)?.*!i
      IMAGE_URL = %r!#{BASE_URL}/(?<type>image|jpeg|sample)/(?<md5>\h{32})#{URL_SLUG}?\.(?<ext>jpg|jpeg|png|gif)\z!i

      delegate :artist_name, :profile_url, :tag_name, :artist_commentary_title, :artist_commentary_desc, :dtext_artist_commentary_title, :dtext_artist_commentary_desc, to: :sub_strategy, allow_nil: true
@@ -75,6 +76,17 @@ module Sources
        image_url
      end

+      def normalize_for_source
+        id = post_id_from_url
+        md5 = post_md5_from_url
+
+        if id.present?
+          "https://#{site_name}/post/show/#{id}"
+        elsif md5.present?
+          "https://#{site_name}/post?tags=md5:#{md5}"
+        end
+      end
+
      def tags
        api_response[:tags].to_s.split.map do |tag|
          [tag, "https://#{site_name}/post?tags=#{CGI.escape(tag)}"]
--- a/app/logical/sources/strategies/nico_seiga.rb
+++ b/app/logical/sources/strategies/nico_seiga.rb
@@ -1,3 +1,15 @@
+# Image Direct URL
+# * https://lohas.nicoseiga.jp/o/971eb8af9bbcde5c2e51d5ef3a2f62d6d9ff5552/1589933964/3583893
+# * http://lohas.nicoseiga.jp/priv/3521156?e=1382558156&h=f2e089256abd1d453a455ec8f317a6c703e2cedf
+# * http://lohas.nicoseiga.jp/priv/b80f86c0d8591b217e7513a9e175e94e00f3c7a1/1384936074/3583893
+# * http://seiga.nicovideo.jp/image/source?id=3312222
+#
+# Image Page URL
+# * https://seiga.nicovideo.jp/seiga/im3521156
+#
+# Manga Page URL
+# * http://seiga.nicovideo.jp/watch/mg316708
+
 module Sources
  module Strategies
    class NicoSeiga < Base
@@ -77,6 +89,10 @@ module Sources
          if x =~ %r{/watch/mg\d+}
            return x
          end
+
+          if x =~ %r{/image/source\?id=(\d+)}
+            return "http://seiga.nicovideo.jp/seiga/im#{$1}"
+          end
        end

        return super
@@ -111,13 +127,21 @@ module Sources
      end

      def normalizable_for_artist_finder?
-        url =~ PAGE || url =~ MANGA_PAGE || url =~ PROFILE || url =~ DIRECT1 || url =~ DIRECT2
+        url =~ PAGE || url =~ MANGA_PAGE || url =~ PROFILE || url =~ DIRECT1 || url =~ DIRECT2 || url =~ PAGE2
      end

      def normalize_for_artist_finder
        "#{profile_url}/"
      end

+      def normalize_for_source
+        if illust_id.present?
+          "https://seiga.nicovideo.jp/seiga/im#{illust_id}"
+        elsif theme_id.present?
+          "http://seiga.nicovideo.jp/watch/mg#{theme_id}"
+        end
+      end
+
      def tag_name
        "nicoseiga#{api_client.user_id}"
      end
--- a/app/logical/sources/strategies/nijie.rb
+++ b/app/logical/sources/strategies/nijie.rb
@@ -162,6 +162,12 @@ module Sources
        artist_id_from_url || artist_id_from_page
      end

+      def normalize_for_source
+        return if illust_id.blank?
+
+        "https://nijie.info/view.php?id=#{illust_id}"
+      end
+
      def page
        return nil if page_url.blank?

--- a/app/logical/sources/strategies/null.rb
+++ b/app/logical/sources/strategies/null.rb
@@ -24,6 +24,117 @@ module Sources
      def normalize_for_artist_finder
        url
      end
+
+      def normalize_for_source
+        case url
+        when %r{\Ahttps?://(?:d3j5vwomefv46c|dn3pm25xmtlyu)\.cloudfront\.net/photos/large/(\d+)\.}i
+          base_10_id = $1.to_i
+          base_36_id = base_10_id.to_s(36)
+          "https://twitpic.com/#{base_36_id}"
+
+        when %r{\Ahttp://www\.karabako\.net/images(?:ub)?/karabako_(\d+)(?:_\d+)?\.}i
+          "http://www.karabako.net/post/view/#{$1}"
+
+        # XXX http://twipple.jp is defunct
+        # http://p.twpl.jp/show/orig/myRVs
+        when %r{\Ahttp://p\.twpl\.jp/show/(?:large|orig)/([a-z0-9]+)}i
+          "http://p.twipple.jp/#{$1}"
+
+        when %r{\Ahttps?://blog(?:(?:-imgs-)?\d*(?:-origin)?)?\.fc2\.com/(?:(?:[^/]/){3}|(?:[^/]/))([^/]+)/(?:file/)?([^\.]+\.[^\?]+)}i
+          username = $1
+          filename = $2
+          "http://#{username}.blog.fc2.com/img/#{filename}/"
+
+        when %r{\Ahttps?://diary(\d)?\.fc2\.com/user/([^/]+)/img/(\d+)_(\d+)/(\d+)\.}i
+          server_id = $1
+          username = $2
+          year = $3
+          month = $4
+          day = $5
+          "http://diary#{server_id}.fc2.com/cgi-sys/ed.cgi/#{username}?Y=#{year}&M=#{month}&D=#{day}"
+
+        when %r{\Ahttps?://(?:fbcdn-)?s(?:content|photos)-[^/]+\.(?:fbcdn|akamaihd)\.net/hphotos-.+/\d+_(\d+)_(?:\d+_){1,3}[no]\.}i
+          "https://www.facebook.com/photo.php?fbid=#{$1}"
+
+        when %r{\Ahttps?://c(?:s|han|[1-4])\.sankakucomplex\.com/data(?:/sample)?/(?:[a-f0-9]{2}/){2}(?:sample-|preview)?([a-f0-9]{32})}i
+          "https://chan.sankakucomplex.com/en/post/show?md5=#{$1}"
+
+        when %r{\Ahttps?://(?:www|s(?:tatic|[1-4]))\.zerochan\.net/.+(?:\.|\/)(\d+)(?:\.(?:jpe?g?))?\z}i
+          "https://www.zerochan.net/#{$1}#full"
+
+        when %r{\Ahttps?://static[1-6]?\.minitokyo\.net/(?:downloads|view)/(?:\d{2}/){2}(\d+)}i
+          "http://gallery.minitokyo.net/download/#{$1}"
+
+        # https://gelbooru.com//images/ee/5c/ee5c9a69db9602c95debdb9b98fb3e3e.jpeg
+        # http://simg.gelbooru.com//images/2003/edd1d2b3881cf70c3acf540780507531.png
+        # https://simg3.gelbooru.com//samples/0b/3a/sample_0b3ae5e225072b8e391c827cb470d29c.jpg
+        when %r{\Ahttps?://(?:\w+\.)?gelbooru\.com//?(?:images|samples)/(?:\d+|\h\h/\h\h)/(?:sample_)?(?<md5>\h{32})\.}i
+          "https://gelbooru.com/index.php?page=post&s=list&md5=#{$~[:md5]}"
+
+        when %r{\Ahttps?://(?:slot\d*\.)?im(?:g|ages)\d*\.wikia\.(?:nocookie\.net|com)/(?:_{2}cb\d{14}/)?([^/]+)(?:/[a-z]{2})?/images/(?:(?:thumb|archive)?/)?[a-f0-9]/[a-f0-9]{2}/(?:\d{14}(?:!|%21))?([^/]+)}i
+          subdomain = $1
+          filename = $2
+          "https://#{subdomain}.wikia.com/wiki/File:#{filename}"
+
+        when %r{\Ahttps?://vignette(?:\d*)\.wikia\.nocookie\.net/([^/]+)/images/[a-f0-9]/[a-f0-9]{2}/([^/]+)}i
+          subdomain = $1
+          filename = $2
+          "https://#{subdomain}.wikia.com/wiki/File:#{filename}"
+
+        when %r{\Ahttps?://e-shuushuu.net/images/\d{4}-(?:\d{2}-){2}(\d+)}i
+          "https://e-shuushuu.net/image/#{$1}"
+
+        when %r{\Ahttps?://jpg\.nijigen-daiaru\.com/(\d+)}i
+          "http://nijigen-daiaru.com/book.php?idb=#{$1}"
+
+        when %r{\Ahttps?://sozai\.doujinantena\.com/contents_jpg/([a-f0-9]{32})/}i
+          "http://doujinantena.com/page.php?id=#{$1}"
+
+        when %r{\Ahttps?://rule34-(?:data-\d{3}|images)\.paheal\.net/(?:_images/)?([a-f0-9]{32})}i
+          "https://rule34.paheal.net/post/list/md5:#{$1}/1"
+
+        when %r{\Ahttps?://shimmie\.katawa-shoujo\.com/image/(\d+)}i
+          "https://shimmie.katawa-shoujo.com/post/view/#{$1}"
+
+        when %r{\Ahttps://(?:(?:\w+\.)?rule34\.xxx|img\.booru\.org/(?:rule34|r34))(?:/(?:img/rule34|r34))?/{1,2}images/\d+/([a-f0-9]{32})\.}i
+          "https://rule34.xxx/index.php?page=post&s=list&md5=#{$1}"
+
+        when %r{(\Ahttps?://.+)/diarypro/d(?:ata/upfile/|iary\.cgi\?mode=image&upfile=)(\d+)}i
+          base_url = $1
+          entry_no = $2
+          "#{base_url}/diarypro/diary.cgi?no=#{entry_no}"
+
+        # XXX site is defunct
+        when %r{\Ahttps?://i(?:\d)?\.minus\.com/(?:i|j)([^\.]{12,})}i
+          "http://minus.com/i/#{$1}"
+
+        # http://art59.photozou.jp/pub/212/1986212/photo/118493247_org.v1534644005.jpg
+        # http://kura3.photozou.jp/pub/794/1481794/photo/161537258_org.v1364829097.jpg
+        when %r{\Ahttps?://\w+\.photozou\.jp/pub/\d+/(?<artist_id>\d+)/photo/(?<photo_id>\d+)_.*$}i
+          "https://photozou.jp/photo/show/#{$~[:artist_id]}/#{$~[:photo_id]}"
+
+        # http://img.toranoana.jp/popup_img/04/0030/09/76/040030097695-2p.jpg
+        # http://img.toranoana.jp/popup_img18/04/0010/22/87/040010228714-1p.jpg
+        # http://img.toranoana.jp/popup_blimg/04/0030/08/30/040030083068-1p.jpg
+        # https://ecdnimg.toranoana.jp/ec/img/04/0030/65/34/040030653417-6p.jpg
+        when %r{\Ahttps?://(\w+\.)?toranoana\.jp/(?:popup_(?:bl)?img\d*|ec/img)/\d{2}/\d{4}/\d{2}/\d{2}/(?<work_id>\d+)}i
+          "https://ec.toranoana.jp/tora_r/ec/item/#{$~[:work_id]}/"
+
+        # https://a.hitomi.la/galleries/907838/1.png
+        # https://0a.hitomi.la/galleries/1169701/23.png
+        # https://aa.hitomi.la/galleries/990722/003_01_002.jpg
+        # https://la.hitomi.la/galleries/1054851/001_main_image.jpg
+        when %r{\Ahttps?://\w+\.hitomi\.la/galleries/(?<gallery_id>\d+)/(?<image_id>\d+)\w*\.[a-z]+\z}i
+          "https://hitomi.la/reader/#{$~[:gallery_id]}.html##{$~[:image_id].to_i}"
+
+        # https://aa.hitomi.la/galleries/883451/t_rena1g.png
+        when %r{\Ahttps?://\w+\.hitomi\.la/galleries/(?<gallery_id>\d+)/\w*\.[a-z]+\z}i
+          "https://hitomi.la/galleries/#{$~[:gallery_id]}.html"
+
+        else
+          nil
+        end
+      end
    end
  end
 end
--- a/app/logical/sources/strategies/pawoo.rb
+++ b/app/logical/sources/strategies/pawoo.rb
@@ -1,6 +1,28 @@
+# Image URLS:
+# * https://img.pawoo.net/media_attachments/files/001/297/997/small/c4272a09570757c2.png
+# * https://img.pawoo.net/media_attachments/files/001/297/997/original/c4272a09570757c2.png
+# * https://pawoo.net/media/lU2uV7C1MMQSb1czwvg
+#
+# Page URLS:
+# * https://pawoo.net/@evazion/19451018
+# * https://pawoo.net/web/statuses/19451018
+#
+# Account URLS:
+# * https://pawoo.net/@evazion
+# * https://pawoo.net/web/accounts/47806
+#
+# OAUTH URLS: (NOTE: ID IS DIFFERENT FROM ACCOUNT URL ID)
+# * https://pawoo.net/oauth_authentications/17230064
+
 module Sources::Strategies
  class Pawoo < Base
+    HOST = %r!\Ahttps?://(www\.)?pawoo\.net!i
    IMAGE = %r!\Ahttps?://img\.pawoo\.net/media_attachments/files/(\d+/\d+/\d+)!
+    NAMED_PROFILE = %r!#{HOST}/@(?<artist_name>\w+)!i
+    ID_PROFILE = %r!#{HOST}/web/accounts/(?<artist_id>\d+)!
+
+    STATUS1 = %r!\A#{HOST}/web/statuses/(?<status_id>\d+)!
+    STATUS2 = %r!\A#{NAMED_PROFILE}/(?<status_id>\d+)!

    def domains
      ["pawoo.net"]
@@ -14,9 +36,6 @@ module Sources::Strategies
      image_urls.first
    end

-    # https://img.pawoo.net/media_attachments/files/001/297/997/small/c4272a09570757c2.png
-    # https://img.pawoo.net/media_attachments/files/001/297/997/original/c4272a09570757c2.png
-    # https://pawoo.net/media/lU2uV7C1MMQSb1czwvg
    def image_urls
      if url =~ %r!#{IMAGE}/small/([a-z0-9]+\.\w+)\z!i
        return ["https://img.pawoo.net/media_attachments/files/#{$1}/original/#{$2}"]
@@ -29,8 +48,6 @@ module Sources::Strategies
      return api_response.image_urls
    end

-    # https://pawoo.net/@evazion/19451018
-    # https://pawoo.net/web/statuses/19451018
    def page_url
      [url, referer_url].each do |x|
        if PawooApiClient::Status.is_match?(x)
@@ -41,8 +58,6 @@ module Sources::Strategies
      return super
    end

-    # https://pawoo.net/@evazion
-    # https://pawoo.net/web/accounts/47806
    def profile_url
      if url =~ PawooApiClient::PROFILE2
        return "https://pawoo.net/@#{$1}"
@@ -56,6 +71,22 @@ module Sources::Strategies
      api_response.account_name
    end

+    def artist_name_from_url
+      if url =~ NAMED_PROFILE
+        url[NAMED_PROFILE, :artist_name]
+      end
+    end
+
+    def artist_id_from_url
+      if url =~ ID_PROFILE
+        url[ID_PROFILE, :artist_name]
+      end
+    end
+
+    def status_id_from_url
+      urls.map { |url| url[STATUS1, :status_id] || url[STATUS2, :status_id] }.compact.first
+    end
+
    def artist_commentary_title
      nil
    end
@@ -76,6 +107,18 @@ module Sources::Strategies
      profile_url
    end

+    def normalize_for_source
+      artist_name = artist_name_from_url
+      status_id = status_id_from_url
+      return unless status_id.present?
+
+      if artist_name.present?
+        "https://pawoo.net/@#{artist_name}/#{status_id}"
+      else
+        "https://pawoo.net/web/statuses/#{status_id}"
+      end
+    end
+
    def dtext_artist_commentary_desc
      DText.from_html(artist_commentary_desc) do |element|
        if element.name == "a"
--- a/app/logical/sources/strategies/pixiv.rb
+++ b/app/logical/sources/strategies/pixiv.rb
@@ -213,6 +213,12 @@ module Sources
        illust_id.present? || novel_id.present? || fanbox_id.present? || fanbox_account_id.present?
      end

+      def normalize_for_source
+        return if illust_id.blank?
+
+        "https://www.pixiv.net/artworks/#{illust_id}"
+      end
+
      def tag_name
        moniker
      end
--- a/app/logical/sources/strategies/tumblr.rb
+++ b/app/logical/sources/strategies/tumblr.rb
@@ -5,6 +5,8 @@
 # https://emlan.tumblr.com/post/189469423572/kuro-attempts-to-buy-a-racy-book-at-comiket-but
 # https://66.media.tumblr.com/168dabd09d5ad69eb5fedcf94c45c31a/3dbfaec9b9e0c2e3-72/s640x960/bf33a1324f3f36d2dc64f011bfeab4867da62bc8.png
 # https://66.media.tumblr.com/5a2c3fe25c977e2281392752ab971c90/3dbfaec9b9e0c2e3-92/s500x750/4f92bbaaf95c0b4e7970e62b1d2e1415859dd659.png
+#
+# https://superboin.tumblr.com/post/141169066579/photoset_iframe/superboin/tumblr_o45miiAOts1u6rxu8/500/false

 module Sources::Strategies
  class Tumblr < Base
@@ -131,6 +133,12 @@ module Sources::Strategies
      super(tag)
    end

+    def normalize_for_source
+      return unless blog_name.present? && post_id.present?
+
+      "https://#{blog_name}.tumblr.com/post/#{post_id}"
+    end
+
    def dtext_artist_commentary_desc
      DText.from_html(artist_commentary_desc).strip
    end
--- a/app/logical/sources/strategies/twitter.rb
+++ b/app/logical/sources/strategies/twitter.rb
@@ -43,7 +43,7 @@ module Sources::Strategies
    # https://twitter.com/i/web/status/943446161586733056
    # https://twitter.com/motty08111213/status/943446161586733056
    def self.status_id_from_url(url)
-      if url =~ %r{\Ahttps?://(?:mobile\.)?twitter\.com/(?:i/web|\w+)/status/(\d+)}i
+      if url =~ %r{\Ahttps?://(?:(?:www|mobile)\.)?twitter\.com/(?:i/web|\w+)/status/(\d+)}i
        return $1
      end

@@ -145,6 +145,25 @@ module Sources::Strategies
      profile_url.try(:downcase).presence || url
    end

+    def normalize_for_source
+      status_id = self.class.status_id_from_url(url)
+      if status_id.present?
+        "https://twitter.com/i/web/status/#{status_id}"
+      elsif url =~ %r{\Ahttps?://(?:o|image-proxy-origin)\.twimg\.com/\d/proxy\.jpg\?t=(\w+)&}i
+        str = Base64.decode64($1)
+        source = URI.extract(str, ['http', 'https'])
+        if source.any?
+          source = source[0]
+          if source =~ %r{^https?://twitpic.com/show/large/[a-z0-9]+}i
+            source.gsub!(%r{show/large/}, "")
+            index = source.rindex('.')
+            source = source[0..index - 1]
+          end
+          source
+        end
+      end
+    end
+
    def tags
      api_response.dig(:entities, :hashtags).to_a.map do |hashtag|
        [hashtag[:text], "https://twitter.com/hashtag/#{hashtag[:text]}"]
--- a/app/models/post.rb
+++ b/app/models/post.rb
@@ -334,187 +334,7 @@ class Post < ApplicationRecord
    end

    def normalized_source
-      if pixiv_id.present?
-        return "https://www.pixiv.net/artworks/#{pixiv_id}"
-      end
-
-      case source
-      when %r{\Ahttps?://(?:mobile\.)?twitter.com/[^/]+/status/(\d+)(?:/(?:photo/\d)?|\?s=\d+)?\z}i
-        "https://twitter.com/i/web/status/#{$1}"
-
-      when %r{\Ahttps?://lohas\.nicoseiga\.jp/priv/(\d+)\?e=\d+&h=[a-f0-9]+}i,
-           %r{\Ahttps?://lohas\.nicoseiga\.jp/priv/[a-f0-9]+/\d+/(\d+)}i
-        "https://seiga.nicovideo.jp/seiga/im#{$1}"
-
-      when %r{\Ahttps?://(?:d3j5vwomefv46c|dn3pm25xmtlyu)\.cloudfront\.net/photos/large/(\d+)\.}i
-        base_10_id = $1.to_i
-        base_36_id = base_10_id.to_s(36)
-        "https://twitpic.com/#{base_36_id}"
-
-      # http://prnt00.deviantart.net/9b74/b/2016/101/4/468a9d89f52a835d4f6f1c8caca0dfb2-pnjfbh.jpg
-      # http://fc00.deviantart.net/fs71/f/2013/234/d/8/d84e05f26f0695b1153e9dab3a962f16-d6j8jl9.jpg
-      # http://th04.deviantart.net/fs71/PRE/f/2013/337/3/5/35081351f62b432f84eaeddeb4693caf-d6wlrqs.jpg
-      # http://fc09.deviantart.net/fs22/o/2009/197/3/7/37ac79eaeef9fb32e6ae998e9a77d8dd.jpg
-      when %r{\Ahttps?://(?:fc|th|pre|orig|img|prnt)\d{2}\.deviantart\.net/.+/[a-f0-9]{32}-d(?<id>[a-z0-9]+)\.}i
-        id = $~[:id].to_i(36)
-        "https://deviantart.com/deviation/#{id}"
-
-      when Sources::Strategies::DeviantArt::ASSET
-        Sources::Strategies::DeviantArt.new(source).page_url_from_image_url || source
-
-      when %r{\Ahttp://www\.karabako\.net/images(?:ub)?/karabako_(\d+)(?:_\d+)?\.}i
-        "http://www.karabako.net/post/view/#{$1}"
-
-      # XXX http://twipple.jp is defunct
-      # http://p.twpl.jp/show/orig/myRVs
-      when %r{\Ahttp://p\.twpl\.jp/show/(?:large|orig)/([a-z0-9]+)}i
-        "http://p.twipple.jp/#{$1}"
-
-      when %r{\Ahttps?://pictures\.hentai-foundry\.com//?[^/]/([^/]+)/(\d+)}i
-        "https://www.hentai-foundry.com/pictures/user/#{$1}/#{$2}"
-
-      when %r{\Ahttp://blog(?:(?:-imgs-)?\d*(?:-origin)?)?\.fc2\.com/(?:(?:[^/]/){3}|(?:[^/]/))([^/]+)/(?:file/)?([^\.]+\.[^\?]+)}i
-        username = $1
-        filename = $2
-        "http://#{username}.blog.fc2.com/img/#{filename}/"
-
-      when %r{\Ahttp://diary(\d)?\.fc2\.com/user/([^/]+)/img/(\d+)_(\d+)/(\d+)\.}i
-        server_id = $1
-        username = $2
-        year = $3
-        month = $4
-        day = $5
-        "http://diary#{server_id}.fc2.com/cgi-sys/ed.cgi/#{username}?Y=#{year}&M=#{month}&D=#{day}"
-
-      when %r{\Ahttps?://(?:fbcdn-)?s(?:content|photos)-[^/]+\.(?:fbcdn|akamaihd)\.net/hphotos-.+/\d+_(\d+)_(?:\d+_){1,3}[no]\.}i
-        "https://www.facebook.com/photo.php?fbid=#{$1}"
-
-      when %r{\Ahttps?://c(?:s|han|[1-4])\.sankakucomplex\.com/data(?:/sample)?/(?:[a-f0-9]{2}/){2}(?:sample-|preview)?([a-f0-9]{32})}i
-        "https://chan.sankakucomplex.com/en/post/show?md5=#{$1}"
-
-      when %r{\Ahttp://s(?:tatic|[1-4])\.zerochan\.net/.+(?:\.|\/)(\d+)\.(?:jpe?g?)\z}i
-        "https://www.zerochan.net/#{$1}#full"
-
-      when %r{\Ahttp://static[1-6]?\.minitokyo\.net/(?:downloads|view)/(?:\d{2}/){2}(\d+)}i
-        "http://gallery.minitokyo.net/download/#{$1}"
-
-      # https://gelbooru.com//images/ee/5c/ee5c9a69db9602c95debdb9b98fb3e3e.jpeg
-      # http://simg.gelbooru.com//images/2003/edd1d2b3881cf70c3acf540780507531.png
-      # https://simg3.gelbooru.com//samples/0b/3a/sample_0b3ae5e225072b8e391c827cb470d29c.jpg
-      when %r{\Ahttps?://(?:\w+\.)?gelbooru\.com//?(?:images|samples)/(?:\d+|\h\h/\h\h)/(?:sample_)?(?<md5>\h{32})\.}i
-        "https://gelbooru.com/index.php?page=post&s=list&md5=#{$~[:md5]}"
-
-      when %r{\Ahttps?://(?:slot\d*\.)?im(?:g|ages)\d*\.wikia\.(?:nocookie\.net|com)/(?:_{2}cb\d{14}/)?([^/]+)(?:/[a-z]{2})?/images/(?:(?:thumb|archive)?/)?[a-f0-9]/[a-f0-9]{2}/(?:\d{14}(?:!|%21))?([^/]+)}i
-        subdomain = $1
-        filename = $2
-        "http://#{subdomain}.wikia.com/wiki/File:#{filename}"
-
-      when %r{\Ahttps?://vignette(?:\d*)\.wikia\.nocookie\.net/([^/]+)/images/[a-f0-9]/[a-f0-9]{2}/([^/]+)}i
-        subdomain = $1
-        filename = $2
-        "http://#{subdomain}.wikia.com/wiki/File:#{filename}"
-
-      when %r{\Ahttp://(?:(?:\d{1,3}\.){3}\d{1,3}):(?:\d{1,5})/h/([a-f0-9]{40})-(?:\d+-){3}(?:png|gif|(?:jpe?g?))/keystamp=\d+-[a-f0-9]{10}/([^/]+)}i
-        sha1hash = $1
-        filename = $2
-        "http://g.e-hentai.org/?f_shash=#{sha1hash}&fs_from=#{filename}"
-
-      when %r{\Ahttp://e-shuushuu.net/images/\d{4}-(?:\d{2}-){2}(\d+)}i
-        "http://e-shuushuu.net/image/#{$1}"
-
-      when %r{\Ahttp://jpg\.nijigen-daiaru\.com/(\d+)}i
-        "http://nijigen-daiaru.com/book.php?idb=#{$1}"
-
-      when %r{\Ahttps?://sozai\.doujinantena\.com/contents_jpg/([a-f0-9]{32})/}i
-        "http://doujinantena.com/page.php?id=#{$1}"
-
-      when %r{\Ahttp://rule34-(?:data-\d{3}|images)\.paheal\.net/(?:_images/)?([a-f0-9]{32})}i
-        "https://rule34.paheal.net/post/list/md5:#{$1}/1"
-
-      when %r{\Ahttp://shimmie\.katawa-shoujo\.com/image/(\d+)}i
-        "https://shimmie.katawa-shoujo.com/post/view/#{$1}"
-
-      when %r{\Ahttp://(?:(?:(?:img\d?|cdn)\.)?rule34\.xxx|img\.booru\.org/(?:rule34|r34))(?:/(?:img/rule34|r34))?/{1,2}images/\d+/(?:[a-f0-9]{32}|[a-f0-9]{40})\.}i
-        "https://rule34.xxx/index.php?page=post&s=list&md5=#{md5}"
-
-      when %r{(\Ahttp://.+)/diarypro/d(?:ata/upfile/|iary\.cgi\?mode=image&upfile=)(\d+)}i
-        base_url = $1
-        entry_no = $2
-        "#{base_url}/diarypro/diary.cgi?no=#{entry_no}"
-
-      # XXX site is defunct
-      when %r{\Ahttp://i(?:\d)?\.minus\.com/(?:i|j)([^\.]{12,})}i
-        "http://minus.com/i/#{$1}"
-
-      when %r{\Ahttps?://pic0[1-4]\.nijie\.info/nijie_picture/(?:diff/main/)?\d+_(\d+)_(?:\d+{10}|\d+_\d+{14})}i
-        "https://nijie.info/view.php?id=#{$1}"
-
-      # http://ayase.yande.re/image/2d0d229fd8465a325ee7686fcc7f75d2/yande.re%20192481%20animal_ears%20bunny_ears%20garter_belt%20headphones%20mitha%20stockings%20thighhighs.jpg
-      # https://yuno.yande.re/image/1764b95ae99e1562854791c232e3444b/yande.re%20281544%20cameltoe%20erect_nipples%20fundoshi%20horns%20loli%20miyama-zero%20sarashi%20sling_bikini%20swimsuits.jpg
-      # https://files.yande.re/image/2a5d1d688f565cb08a69ecf4e35017ab/yande.re%20349790%20breast_hold%20kurashima_tomoyasu%20mahouka_koukou_no_rettousei%20naked%20nipples.jpg
-      # https://files.yande.re/sample/0d79447ce2c89138146f64ba93633568/yande.re%20290757%20sample%20seifuku%20thighhighs%20tsukudani_norio.jpg
-      when %r{\Ahttps?://(?:[^.]+\.)?yande\.re/(?:image|jpeg|sample)/\h{32}/yande\.re%20(?<post_id>\d+)}i
-        "https://yande.re/post/show/#{$~[:post_id]}"
-
-      # https://yande.re/jpeg/0c9ec0ffcaa40470093cb44c3fd40056/yande.re%2064649%20animal_ears%20cameltoe%20fixme%20nekomimi%20nipples%20ryohka%20school_swimsuit%20see_through%20shiraishi_nagomi%20suzuya%20swimsuits%20tail%20thighhighs.jpg
-      # https://yande.re/jpeg/22577d2344fe694cf47f80563031b3cd.jpg
-      # https://yande.re/image/b4b1d11facd1700544554e4805d47bb6/.png
-      # https://yande.re/sample/ceb6a12e87945413a95b90fada406f91/.jpg
-      when %r{\Ahttps?://(?:[^.]+\.)?yande\.re/(?:image|jpeg|sample)/(?<md5>\h{32})(?:/yande\.re.*|/?\.(?:jpg|png))\z}i
-        "https://yande.re/post?tags=md5:#{$~[:md5]}"
-
-      when %r{\Ahttps?://(?:[^.]+\.)?konachan\.com/(?:image|jpeg|sample)/\h{32}/Konachan\.com%20-%20(?<post_id>\d+)}i
-        "https://konachan.com/post/show/#{$~[:post_id]}"
-
-      when %r{\Ahttps?://(?:[^.]+\.)?konachan\.com/(?:image|jpeg|sample)/(?<md5>\h{32})(?:/Konachan\.com%20-%20.*|/?\.(?:jpg|png))\z}i
-        "https://konachan.com/post?tags=md5:#{$~[:md5]}"
-
-      # https://gfee_li.artstation.com/projects/XPGOD
-      # https://gfee_li.artstation.com/projects/asuka-7
-      when %r{\Ahttps?://\w+\.artstation.com/(?:artwork|projects)/(?<project_id>[a-z0-9-]+)\z/}i
-        "https://www.artstation.com/artwork/#{$~[:project_id]}"
-
-      when %r{\Ahttps?://(?:o|image-proxy-origin)\.twimg\.com/\d/proxy\.jpg\?t=(\w+)&}i
-        str = Base64.decode64($1)
-        url = URI.extract(str, ['http', 'https'])
-        if url.any?
-          url = url[0]
-          if url =~ /^https?:\/\/twitpic.com\/show\/large\/[a-z0-9]+/i
-            url.gsub!(/show\/large\//, "")
-            index = url.rindex('.')
-            url = url[0..index - 1]
-          end
-          url
-        else
-          source
-        end
-
-      # http://art59.photozou.jp/pub/212/1986212/photo/118493247_org.v1534644005.jpg
-      # http://kura3.photozou.jp/pub/794/1481794/photo/161537258_org.v1364829097.jpg
-      when %r{\Ahttps?://\w+\.photozou\.jp/pub/\d+/(?<artist_id>\d+)/photo/(?<photo_id>\d+)_.*$}i
-        "https://photozou.jp/photo/show/#{$~[:artist_id]}/#{$~[:photo_id]}"
-
-      # http://img.toranoana.jp/popup_img/04/0030/09/76/040030097695-2p.jpg
-      # http://img.toranoana.jp/popup_img18/04/0010/22/87/040010228714-1p.jpg
-      # http://img.toranoana.jp/popup_blimg/04/0030/08/30/040030083068-1p.jpg
-      # https://ecdnimg.toranoana.jp/ec/img/04/0030/65/34/040030653417-6p.jpg
-      when %r{\Ahttps?://(\w+\.)?toranoana\.jp/(?:popup_(?:bl)?img\d*|ec/img)/\d{2}/\d{4}/\d{2}/\d{2}/(?<work_id>\d+)}i
-        "https://ec.toranoana.jp/tora_r/ec/item/#{$~[:work_id]}/"
-
-      # https://a.hitomi.la/galleries/907838/1.png
-      # https://0a.hitomi.la/galleries/1169701/23.png
-      # https://aa.hitomi.la/galleries/990722/003_01_002.jpg
-      # https://la.hitomi.la/galleries/1054851/001_main_image.jpg
-      when %r{\Ahttps?://\w+\.hitomi\.la/galleries/(?<gallery_id>\d+)/(?<image_id>\d+)\w*\.[a-z]+\z}i
-        "https://hitomi.la/reader/#{$~[:gallery_id]}.html##{$~[:image_id].to_i}"
-
-      # https://aa.hitomi.la/galleries/883451/t_rena1g.png
-      when %r{\Ahttps?://\w+\.hitomi\.la/galleries/(?<gallery_id>\d+)/\w*\.[a-z]+\z}i
-        "https://hitomi.la/galleries/#{$~[:gallery_id]}.html"
-
-      else
-        source
-      end
+      Sources::Strategies.normalize_source(source)
    end

    def source_domain
--- a/test/unit/post_test.rb
+++ b/test/unit/post_test.rb
@@ -1531,60 +1531,6 @@ class PostTest < ActiveSupport::TestCase
            @post.pixiv_id = nil
          end
        end
-
-        should "normalize pixiv links" do
-          @post.update!(source: "http://i2.pixiv.net/img12/img/zenze/39749565.png")
-          assert_equal("https://www.pixiv.net/artworks/39749565", @post.normalized_source)
-
-          @post.update!(source: "http://i1.pixiv.net/img53/img/themare/39735353_big_p1.jpg")
-          assert_equal("https://www.pixiv.net/artworks/39735353", @post.normalized_source)
-
-          @post.update!(source: "http://i1.pixiv.net/c/150x150/img-master/img/2010/11/30/08/39/58/14901720_p0_master1200.jpg")
-          assert_equal("https://www.pixiv.net/artworks/14901720", @post.normalized_source)
-
-          @post.update!(source: "http://i1.pixiv.net/img-original/img/2010/11/30/08/39/58/14901720_p0.png")
-          assert_equal("https://www.pixiv.net/artworks/14901720", @post.normalized_source)
-
-          @post.update!(source: "http://i2.pixiv.net/img-zip-ugoira/img/2014/08/05/06/01/10/44524589_ugoira1920x1080.zip")
-          assert_equal("https://www.pixiv.net/artworks/44524589", @post.normalized_source)
-        end
-
-        should "normalize nicoseiga links" do
-          @post.source = "http://lohas.nicoseiga.jp/priv/3521156?e=1382558156&h=f2e089256abd1d453a455ec8f317a6c703e2cedf"
-          assert_equal("https://seiga.nicovideo.jp/seiga/im3521156", @post.normalized_source)
-          @post.source = "http://lohas.nicoseiga.jp/priv/b80f86c0d8591b217e7513a9e175e94e00f3c7a1/1384936074/3583893"
-          assert_equal("https://seiga.nicovideo.jp/seiga/im3583893", @post.normalized_source)
-        end
-
-        should "normalize twitpic links" do
-          @post.source = "http://d3j5vwomefv46c.cloudfront.net/photos/large/820960031.jpg?1384107199"
-          assert_equal("https://twitpic.com/dks0tb", @post.normalized_source)
-        end
-
-        should "normalize deviantart links" do
-          @post.source = "http://fc06.deviantart.net/fs71/f/2013/295/d/7/you_are_already_dead__by_mar11co-d6rgm0e.jpg"
-          assert_equal("https://www.deviantart.com/mar11co/art/You-Are-Already-Dead-408921710", @post.normalized_source)
-          @post.source = "http://fc00.deviantart.net/fs71/f/2013/337/3/5/35081351f62b432f84eaeddeb4693caf-d6wlrqs.jpg"
-          assert_equal("https://deviantart.com/deviation/417560500", @post.normalized_source)
-        end
-
-        should "normalize karabako links" do
-          @post.source = "http://www.karabako.net/images/karabako_38835.jpg"
-          assert_equal("http://www.karabako.net/post/view/38835", @post.normalized_source)
-        end
-
-        should "normalize twipple links" do
-          @post.source = "http://p.twpl.jp/show/orig/mI2c3"
-          assert_equal("http://p.twipple.jp/mI2c3", @post.normalized_source)
-        end
-
-        should "normalize hentai foundry links" do
-          @post.source = "http://pictures.hentai-foundry.com//a/AnimeFlux/219123.jpg"
-          assert_equal("https://www.hentai-foundry.com/pictures/user/AnimeFlux/219123", @post.normalized_source)
-
-          @post.source = "http://pictures.hentai-foundry.com/a/AnimeFlux/219123/Mobile-Suit-Equestria-rainbow-run.jpg"
-          assert_equal("https://www.hentai-foundry.com/pictures/user/AnimeFlux/219123", @post.normalized_source)
-        end
      end

      context "when validating tags" do
--- a/test/unit/sources/art_station_test.rb
+++ b/test/unit/sources/art_station_test.rb
@@ -160,5 +160,24 @@ module Sources
      site = Sources::Strategies.find("https://sa-dui.artstation.com/projects/DVERn")
      assert_equal("sa-dui", site.artist_name)
    end
+
+    context "normalizing for source" do
+      should "normalize correctly" do
+        source1 = "https://www.artstation.com/artwork/ghost-in-the-shell-fandom"
+        source2 = "https://anubis1982918.artstation.com/projects/qPVGP/"
+        source3 = "https://dudeunderscore.artstation.com/projects/NoNmD?album_id=23041"
+
+        assert_equal(source1, Sources::Strategies.normalize_source(source1))
+        assert_equal("https://anubis1982918.artstation.com/projects/qPVGP", Sources::Strategies.normalize_source(source2))
+        assert_equal("https://dudeunderscore.artstation.com/projects/NoNmD", Sources::Strategies.normalize_source(source3))
+      end
+
+      should "avoid normalizing unnormalizable urls" do
+        bad_source1 = "http://cdna.artstation.com/p/assets/images/images/005/804/224/large/titapa-khemakavat-sa-dui-srevere.jpg?1493887236"
+        bad_source2 = "https://www.artstation.com"
+        assert_equal(bad_source1, Sources::Strategies.normalize_source(bad_source1))
+        assert_equal(bad_source2, Sources::Strategies.normalize_source(bad_source2))
+      end
+    end
  end
 end
--- a/test/unit/sources/deviant_art_test.rb
+++ b/test/unit/sources/deviant_art_test.rb
@@ -32,7 +32,7 @@ module Sources
        @artist = create(:artist, name: "nickbeja", url_string: "https://nickbeja.deviantart.com")

        assert_equal("https://pre00.deviantart.net/423b/th/pre/i/2017/281/e/0/mindflayer_girl01_by_nickbeja-dbpxdt8.png", @site.image_url)
-        assert_equal(@site.image_url, @site.canonical_url)
+        assert_equal(@site.page_url, @site.canonical_url)
        assert_equal("nickbeja", @site.artist_name)
        assert_equal("https://www.deviantart.com/nickbeja", @site.profile_url)
        assert_equal("https://www.deviantart.com/nickbeja/art/Mindflayer-Girl01-708675884", @site.page_url_from_image_url)
@@ -362,5 +362,32 @@ module Sources
        assert_equal([@artist], @site.artists)
      end
    end
+
+    context "normalizing for source" do
+      should "normalize correctly" do
+        source1 = "http://fc06.deviantart.net/fs71/f/2013/295/d/7/you_are_already_dead__by_mar11co-d6rgm0e.jpg"
+        source2 = "https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/8b472d70-a0d6-41b5-9a66-c35687090acc/d23jbr4-8a06af02-70cb-46da-8a96-42a6ba73cdb4.jpg/v1/fill/w_786,h_1017,q_70,strp/silverhawks_quicksilver_by_edsfox_d23jbr4-pre.jpg"
+        source3 = "http://orig12.deviantart.net/9b69/f/2017/023/7/c/illustration___tokyo_encount_oei__by_melisaongmiqin-dawi58s.png"
+        source4 = "http://fc00.deviantart.net/fs71/f/2013/337/3/5/35081351f62b432f84eaeddeb4693caf-d6wlrqs.jpg"
+        source5 = "https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/f/76098ac8-04ab-4784-b382-88ca082ba9b1/d9x7lmk-595099de-fe8f-48e5-9841-7254f9b2ab8d.png?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ1cm46YXBwOiIsImlzcyI6InVybjphcHA6Iiwib2JqIjpbW3sicGF0aCI6IlwvZlwvNzYwOThhYzgtMDRhYi00Nzg0LWIzODItODhjYTA4MmJhOWIxXC9kOXg3bG1rLTU5NTA5OWRlLWZlOGYtNDhlNS05ODQxLTcyNTRmOWIyYWI4ZC5wbmcifV1dLCJhdWQiOlsidXJuOnNlcnZpY2U6ZmlsZS5kb3dubG9hZCJdfQ.KFOVXAiF8MTlLb3oM-FlD0nnDvODmjqEhFYN5I2X5Bc"
+        source6 = "https://fav.me/dbc3a48"
+
+        assert_equal("https://www.deviantart.com/mar11co/art/You-Are-Already-Dead-408921710", Sources::Strategies.normalize_source(source1))
+        assert_equal("https://www.deviantart.com/edsfox/art/Silverhawks-Quicksilver-126872896", Sources::Strategies.normalize_source(source2))
+        assert_equal("https://www.deviantart.com/melisaongmiqin/art/Illustration-Tokyo-Encount-Oei-659256076", Sources::Strategies.normalize_source(source3))
+        assert_equal("https://www.deviantart.com/deviation/417560500", Sources::Strategies.normalize_source(source4))
+        assert_equal("https://www.deviantart.com/deviation/599977532", Sources::Strategies.normalize_source(source5))
+        assert_equal("https://www.deviantart.com/deviation/685436408", Sources::Strategies.normalize_source(source6))
+      end
+
+      should "avoid normalizing unnormalizable urls" do
+        bad_source1 = "http://fc08.deviantart.net/images3/i/2004/088/8/f/Blackrose_for_MuzicFreq.jpg"
+        bad_source2 = "http://prnt00.deviantart.net/9b74/b/2016/101/4/468a9d89f52a835d4f6f1c8caca0dfb2-pnjfbh.jpg"
+        bad_source3 = "https://deviantart.net"
+        assert_equal(bad_source1, Sources::Strategies.normalize_source(bad_source1))
+        assert_equal(bad_source2, Sources::Strategies.normalize_source(bad_source2))
+        assert_equal(bad_source3, Sources::Strategies.normalize_source(bad_source3))
+      end
+    end
  end
 end
--- a/test/unit/sources/hentai_foundry_test.rb
+++ b/test/unit/sources/hentai_foundry_test.rb
@@ -85,5 +85,22 @@ module Sources
        assert_equal([@artist], @image.artists)
      end
    end
+
+    context "normalizing for source" do
+      should "normalize correctly" do
+        source1 = "http://pictures.hentai-foundry.com//a/AnimeFlux/219123.jpg"
+        source2 = "http://pictures.hentai-foundry.com/a/AnimeFlux/219123/Mobile-Suit-Equestria-rainbow-run.jpg"
+        source3 = "http://www.hentai-foundry.com/pictures/user/Ganassa/457176/LOL-Swimsuit---Caitlyn-reworked-nude-ver."
+
+        assert_equal("https://www.hentai-foundry.com/pictures/user/AnimeFlux/219123", Sources::Strategies.normalize_source(source1))
+        assert_equal("https://www.hentai-foundry.com/pictures/user/AnimeFlux/219123", Sources::Strategies.normalize_source(source2))
+        assert_equal("https://www.hentai-foundry.com/pictures/user/Ganassa/457176", Sources::Strategies.normalize_source(source3))
+      end
+
+      should "avoid normalizing unnormalizable urls" do
+        bad_source = "https://pictures.hentai-foundry.com/a/AnimeFlux"
+        assert_equal(bad_source, Sources::Strategies.normalize_source(bad_source))
+      end
+    end
  end
 end
--- a/test/unit/sources/moebooru_test.rb
+++ b/test/unit/sources/moebooru_test.rb
@@ -114,34 +114,29 @@ module Sources
      end
    end

-    context "Post#normalized_source" do
-      should "convert yande.re image urls to post urls" do
-        @post = FactoryBot.build(:post)
+    context "normalizing for source" do
+      should "normalize yande.re sources correctly" do
+        source1 = "https://files.yande.re/image/b66909b940e8d77accab7c9b25aa4dc3/yande.re%20377828.png"
+        source2 = "https://files.yande.re/image/2a5d1d688f565cb08a69ecf4e35017ab/yande.re%20349790%20breast_hold%20kurashima_tomoyasu%20mahouka_koukou_no_rettousei%20naked%20nipples.jpg"
+        source3 = "https://files.yande.re/image/e4c2ba38de88ff1640aaebff84c84e81/469784.jpg"
+        source4 = "https://yande.re/image/b4b1d11facd1700544554e4805d47bb6/.png"
+        source5 = "https://yande.re/jpeg/22577d2344fe694cf47f80563031b3cd.jpg"

-        @post.source = "https://files.yande.re/image/b66909b940e8d77accab7c9b25aa4dc3/yande.re%20377828.png"
-        assert_equal("https://yande.re/post/show/377828", @post.normalized_source)
-
-        @post.source = "https://files.yande.re/image/2a5d1d688f565cb08a69ecf4e35017ab/yande.re%20349790%20breast_hold%20kurashima_tomoyasu%20mahouka_koukou_no_rettousei%20naked%20nipples.jpg"
-        assert_equal("https://yande.re/post/show/349790", @post.normalized_source)
-
-        @post.source = "https://yande.re/image/b4b1d11facd1700544554e4805d47bb6/.png"
-        assert_equal("https://yande.re/post?tags=md5:b4b1d11facd1700544554e4805d47bb6", @post.normalized_source)
-
-        @post.source = "https://yande.re/jpeg/22577d2344fe694cf47f80563031b3cd.jpg"
-        assert_equal("https://yande.re/post?tags=md5:22577d2344fe694cf47f80563031b3cd", @post.normalized_source)
+        assert_equal("https://yande.re/post/show/377828", Sources::Strategies.normalize_source(source1))
+        assert_equal("https://yande.re/post/show/349790", Sources::Strategies.normalize_source(source2))
+        assert_equal("https://yande.re/post/show/469784", Sources::Strategies.normalize_source(source3))
+        assert_equal("https://yande.re/post?tags=md5:b4b1d11facd1700544554e4805d47bb6", Sources::Strategies.normalize_source(source4))
+        assert_equal("https://yande.re/post?tags=md5:22577d2344fe694cf47f80563031b3cd", Sources::Strategies.normalize_source(source5))
      end

-      should "convert konachan.com image urls to post urls" do
-        @post = FactoryBot.build(:post)
+      should "normalize konachan.com sources correctly" do
+        source1 = "https://konachan.com/image/5d633771614e4bf5c17df19a0f0f333f/Konachan.com%20-%20270807%20black_hair%20bokuden%20clouds%20grass%20landscape%20long_hair%20original%20phone%20rope%20scenic%20seifuku%20skirt%20sky%20summer%20torii%20tree.jpg"
+        source2 = "https://konachan.com/sample/e2e2994bae738ff52fff7f4f50b069d5/Konachan.com%20-%20270803%20sample.jpg"
+        source3 = "https://konachan.com/image/99a3c4f10c327d54486259a74173fc0b.jpg"

-        @post.source = "https://konachan.com/image/5d633771614e4bf5c17df19a0f0f333f/Konachan.com%20-%20270807%20black_hair%20bokuden%20clouds%20grass%20landscape%20long_hair%20original%20phone%20rope%20scenic%20seifuku%20skirt%20sky%20summer%20torii%20tree.jpg"
-        assert_equal("https://konachan.com/post/show/270807", @post.normalized_source)
-
-        @post.source = "https://konachan.com/sample/e2e2994bae738ff52fff7f4f50b069d5/Konachan.com%20-%20270803%20sample.jpg"
-        assert_equal("https://konachan.com/post/show/270803", @post.normalized_source)
-
-        @post.source = "https://konachan.com/image/99a3c4f10c327d54486259a74173fc0b.jpg"
-        assert_equal("https://konachan.com/post?tags=md5:99a3c4f10c327d54486259a74173fc0b", @post.normalized_source)
+        assert_equal("https://konachan.com/post/show/270807", Sources::Strategies.normalize_source(source1))
+        assert_equal("https://konachan.com/post/show/270803", Sources::Strategies.normalize_source(source2))
+        assert_equal("https://konachan.com/post?tags=md5:99a3c4f10c327d54486259a74173fc0b", Sources::Strategies.normalize_source(source3))
      end
    end
  end
--- a/test/unit/sources/nico_seiga_test.rb
+++ b/test/unit/sources/nico_seiga_test.rb
@@ -61,5 +61,24 @@ module Sources
        assert_match(full_image_url, site.canonical_url)
      end
    end
+
+    context "normalizing for source" do
+      should "normalize correctly" do
+        source1 = "http://lohas.nicoseiga.jp/priv/3521156?e=1382558156&h=f2e089256abd1d453a455ec8f317a6c703e2cedf"
+        source2 = "http://lohas.nicoseiga.jp/priv/b80f86c0d8591b217e7513a9e175e94e00f3c7a1/1384936074/3583893"
+        source3 = "http://lohas.nicoseiga.jp/o/910aecf08e542285862954017f8a33a8c32a8aec/1433298801/4937663"
+        source4 = "http://seiga.nicovideo.jp/image/source?id=3312222"
+
+        assert_equal("https://seiga.nicovideo.jp/seiga/im3521156", Sources::Strategies.normalize_source(source1))
+        assert_equal("https://seiga.nicovideo.jp/seiga/im3583893", Sources::Strategies.normalize_source(source2))
+        assert_equal("https://seiga.nicovideo.jp/seiga/im4937663", Sources::Strategies.normalize_source(source3))
+        assert_equal("https://seiga.nicovideo.jp/seiga/im3312222", Sources::Strategies.normalize_source(source4))
+      end
+
+      should "avoid normalizing unnormalizable urls" do
+        bad_source = "https://seiga.nicovideo.jp"
+        assert_equal(bad_source, Sources::Strategies.normalize_source(bad_source))
+      end
+    end
  end
 end
--- a/test/unit/sources/nijie_test.rb
+++ b/test/unit/sources/nijie_test.rb
@@ -274,5 +274,25 @@ module Sources
        end
      end
    end
+
+    context "normalizing for source" do
+      should "normalize correctly" do
+        source1 = "https://pic01.nijie.info/nijie_picture/diff/main/218856_0_236014_20170620101329.png"
+        source2 = "https://pic04.nijie.info/nijie_picture/diff/main/287736_161475_20181112032855_1.png"
+
+        assert_equal("https://nijie.info/view.php?id=218856", Sources::Strategies.normalize_source(source1))
+        assert_equal("https://nijie.info/view.php?id=287736", Sources::Strategies.normalize_source(source2))
+      end
+
+      should "avoid normalizing unnormalizable urls" do
+        bad_source1 = "https://pic01.nijie.info/nijie_picture/20120211210359.jpg"
+        bad_source2 = "https://pic04.nijie.info/omata/4829_20161128012012.png"
+        bad_source3 = "https://pic03.nijie.info/nijie_picture/28310_20131101215959.jpg"
+
+        assert_equal(bad_source1, Sources::Strategies.normalize_source(bad_source1))
+        assert_equal(bad_source2, Sources::Strategies.normalize_source(bad_source2))
+        assert_equal(bad_source3, Sources::Strategies.normalize_source(bad_source3))
+      end
+    end
  end
 end
--- a/test/unit/sources/null_test.rb
+++ b/test/unit/sources/null_test.rb
@@ -23,10 +23,150 @@ module Sources

      should "find the artist" do
        a1 = FactoryBot.create(:artist, name: "test1", url_string: "http://oremuhax.x0.com")
-        a2 = FactoryBot.create(:artist, name: "test2", url_string: "http://google.com")

        assert_equal([a1], @site.artists)
      end
    end
+
+    context "normalizing for source" do
+      should "normalize twitpic links" do
+        source = "http://d3j5vwomefv46c.cloudfront.net/photos/large/820960031.jpg?1384107199"
+        assert_equal("https://twitpic.com/dks0tb", Sources::Strategies.normalize_source(source))
+      end
+
+      should "normalize karabako links" do
+        source = "http://www.karabako.net/images/karabako_38835.jpg"
+        assert_equal("http://www.karabako.net/post/view/38835", Sources::Strategies.normalize_source(source))
+      end
+
+      should "normalize twipple links" do
+        source = "http://p.twpl.jp/show/orig/mI2c3"
+        assert_equal("http://p.twipple.jp/mI2c3", Sources::Strategies.normalize_source(source))
+      end
+
+      should "normalize fc2 links" do
+        source1 = "https://blog-imgs-41.fc2.com/t/u/y/tuyadasi/file.png"
+        source2 = "http://diary.fc2.com/user/kazuharoom/img/2020_1/29.jpg"
+
+        assert_equal("http://tuyadasi.blog.fc2.com/img/file.png/", Sources::Strategies.normalize_source(source1))
+        assert_equal("http://diary.fc2.com/cgi-sys/ed.cgi/kazuharoom?Y=2020&M=1&D=29", Sources::Strategies.normalize_source(source2))
+      end
+
+      should "normalize facebook links" do
+        source = "https://scontent-sin1-1.xx.fbcdn.net/hphotos-xtp1/t31.0-8/11254493_576443445841777_7716273903390212288_o.jpg"
+        assert_equal("https://www.facebook.com/photo.php?fbid=576443445841777", Sources::Strategies.normalize_source(source))
+      end
+
+      should "normalize sankaku links" do
+        source = "http://cs.sankakucomplex.com/data/sample/c2/d7/sample-c2d7270b84ac81326384d4eadd4d4746.jpg?2738848"
+        assert_equal("https://chan.sankakucomplex.com/en/post/show?md5=c2d7270b84ac81326384d4eadd4d4746", Sources::Strategies.normalize_source(source))
+      end
+
+      should "normalize zerochan links" do
+        source1 = "http://static.zerochan.net/full/23/15/183273.jpg"
+        source2 = "https://s4.zerochan.net/Victorique.de.Blois.full.411536.jpg"
+        source3 = "http://www.zerochan.net/full/1567893"
+
+        assert_equal("https://www.zerochan.net/183273#full", Sources::Strategies.normalize_source(source1))
+        assert_equal("https://www.zerochan.net/411536#full", Sources::Strategies.normalize_source(source2))
+        assert_equal("https://www.zerochan.net/1567893#full", Sources::Strategies.normalize_source(source3))
+      end
+
+      should "normalize minitokyo links" do
+        source1 = "http://static.minitokyo.net/downloads/27/13/365677.jpg?433592448,Minitokyo.Eien.no.Aselia.Scans_365677.jpg"
+        source2 = "http://static.minitokyo.net/downloads/14/33/199164.jpg?928244019"
+
+        assert_equal("http://gallery.minitokyo.net/download/365677", Sources::Strategies.normalize_source(source1))
+        assert_equal("http://gallery.minitokyo.net/download/199164", Sources::Strategies.normalize_source(source2))
+      end
+
+      should "normalize gelbooru links" do
+        source1 = "https://gelbooru.com//images/ee/5c/ee5c9a69db9602c95debdb9b98fb3e3e.jpeg"
+        source2 = "http://simg.gelbooru.com//images/2003/edd1d2b3881cf70c3acf540780507531.png"
+        source3 = "https://simg3.gelbooru.com//samples/0b/3a/sample_0b3ae5e225072b8e391c827cb470d29c.jpg"
+
+        assert_equal("https://gelbooru.com/index.php?page=post&s=list&md5=ee5c9a69db9602c95debdb9b98fb3e3e", Sources::Strategies.normalize_source(source1))
+        assert_equal("https://gelbooru.com/index.php?page=post&s=list&md5=edd1d2b3881cf70c3acf540780507531", Sources::Strategies.normalize_source(source2))
+        assert_equal("https://gelbooru.com/index.php?page=post&s=list&md5=0b3ae5e225072b8e391c827cb470d29c", Sources::Strategies.normalize_source(source3))
+      end
+
+      should "normalize wikia links" do
+        source = "https://vignette.wikia.nocookie.net/valkyriecrusade/images/c/c5/Crimson_Hatsune_H.png/revision/latest?cb=20180702031954"
+        assert_equal("https://valkyriecrusade.wikia.com/wiki/File:Crimson_Hatsune_H.png", Sources::Strategies.normalize_source(source))
+      end
+
+      should "normalize e-shuushuu links" do
+        source = "http://e-shuushuu.net/images/2014-07-22-662472.png"
+        assert_equal("https://e-shuushuu.net/image/662472", Sources::Strategies.normalize_source(source))
+      end
+
+      should "normalize nijigen-daiaru links" do
+        source = "http://jpg.nijigen-daiaru.com/19909/029.jpg"
+        assert_equal("http://nijigen-daiaru.com/book.php?idb=19909", Sources::Strategies.normalize_source(source))
+      end
+
+      should "normalize doujinantena links" do
+        source = "http://sozai.doujinantena.com/contents_jpg/d6c39f09d435e32c221e4ef866eceba4/015.jpg"
+        assert_equal("http://doujinantena.com/page.php?id=d6c39f09d435e32c221e4ef866eceba4", Sources::Strategies.normalize_source(source))
+      end
+
+      should "normalize paheal.net links" do
+        source = "http://rule34-data-010.paheal.net/_images/854806addcd3b1246424e7cea49afe31/852405%20-%20Darkstalkers%20Felicia.jpg"
+        assert_equal("https://rule34.paheal.net/post/list/md5:854806addcd3b1246424e7cea49afe31/1", Sources::Strategies.normalize_source(source))
+      end
+
+      should "normalize shimmie.katawa-shoujo.com links" do
+        source = "http://shimmie.katawa-shoujo.com/image/2740.png"
+        assert_equal("https://shimmie.katawa-shoujo.com/post/view/2740", Sources::Strategies.normalize_source(source))
+      end
+
+      should "normalize rule34.xxx links" do
+        source = "https://us.rule34.xxx//images/1802/0adc8fa0604dc445b4b47e6f4c436a08.jpeg?1949807"
+        assert_equal("https://rule34.xxx/index.php?page=post&s=list&md5=0adc8fa0604dc445b4b47e6f4c436a08", Sources::Strategies.normalize_source(source))
+      end
+
+      should "normalize diarypro links" do
+        source1 = "http://nekomataya.net/diarypro/data/upfile/216-1.jpg"
+        source2 = "http://akimbo.sakura.ne.jp/diarypro/diary.cgi?mode=image&upfile=716-3.jpg"
+        assert_equal("http://nekomataya.net/diarypro/diary.cgi?no=216", Sources::Strategies.normalize_source(source1))
+        assert_equal("http://akimbo.sakura.ne.jp/diarypro/diary.cgi?no=716", Sources::Strategies.normalize_source(source2))
+      end
+
+      should "normalize minus.com links" do
+        source = "http://i1.minus.com/ibb0DuE2Ds0yE6.jpg"
+        assert_equal("http://minus.com/i/bb0DuE2Ds0yE6", Sources::Strategies.normalize_source(source))
+      end
+
+      should "normalize photozou links" do
+        source1 = "http://kura3.photozou.jp/pub/794/1481794/photo/161537258_org.v1364829097.jpg"
+        source2 = "http://art59.photozou.jp/pub/212/1986212/photo/118493247_org.v1534644005.jpg"
+        assert_equal("https://photozou.jp/photo/show/1481794/161537258", Sources::Strategies.normalize_source(source1))
+        assert_equal("https://photozou.jp/photo/show/1986212/118493247", Sources::Strategies.normalize_source(source2))
+      end
+
+      should "normalize toranoana links" do
+        source1 = "http://img.toranoana.jp/popup_img/04/0030/09/76/040030097695-2p.jpg"
+        source2 = "https://ecdnimg.toranoana.jp/ec/img/04/0030/65/34/040030653417-6p.jpg"
+        assert_equal("https://ec.toranoana.jp/tora_r/ec/item/040030097695/", Sources::Strategies.normalize_source(source1))
+        assert_equal("https://ec.toranoana.jp/tora_r/ec/item/040030653417/", Sources::Strategies.normalize_source(source2))
+      end
+
+      should "normalize hitomi.la links" do
+        source1 = "https://aa.hitomi.la/galleries/883451/t_rena1g.png"
+        source2 = "https://la.hitomi.la/galleries/1054851/001_main_image.jpg"
+        assert_equal("https://hitomi.la/galleries/883451.html", Sources::Strategies.normalize_source(source1))
+        assert_equal("https://hitomi.la/reader/1054851.html#1", Sources::Strategies.normalize_source(source2))
+      end
+
+      should "leave unknown sources as they are" do
+        source1 = "https://google.com"
+        source2 = "a bad non-http source"
+        source3 = "https://example.com/Folder/中央大学.html"
+
+        assert_equal(source1, Sources::Strategies.normalize_source(source1))
+        assert_equal(source2, Sources::Strategies.normalize_source(source2))
+        assert_equal(source3, Sources::Strategies.normalize_source(source3))
+      end
+    end
  end
 end
--- a/test/unit/sources/pawoo_test.rb
+++ b/test/unit/sources/pawoo_test.rb
@@ -94,5 +94,23 @@ module Sources
        assert_equal("evazion", @site.artist_name)
      end
    end
+
+    context "normalizing for source" do
+      should "normalize correctly" do
+        source1 = "https://pawoo.net/@evazion/19451018/"
+        source2 = "https://pawoo.net/web/statuses/19451018/favorites"
+
+        assert_equal("https://pawoo.net/@evazion/19451018", Sources::Strategies.normalize_source(source1))
+        assert_equal("https://pawoo.net/web/statuses/19451018", Sources::Strategies.normalize_source(source2))
+      end
+
+      should "avoid normalizing unnormalizable urls" do
+        bad_source1 = "https://img.pawoo.net/media_attachments/files/001/297/997/original/c4272a09570757c2.png"
+        bad_source2 = "https://pawoo.net/@evazion/media"
+
+        assert_equal(bad_source1, Sources::Strategies.normalize_source(bad_source1))
+        assert_equal(bad_source2, Sources::Strategies.normalize_source(bad_source2))
+      end
+    end
  end
 end
--- a/test/unit/sources/pixiv_test.rb
+++ b/test/unit/sources/pixiv_test.rb
@@ -363,5 +363,21 @@ module Sources
        end
      end
    end
+
+    context "normalizing for source" do
+      should "normalize correctly" do
+        source1 = "http://i2.pixiv.net/img12/img/zenze/39749565.png"
+        source2 = "http://i1.pixiv.net/img53/img/themare/39735353_big_p1.jpg"
+        source3 = "http://i1.pixiv.net/c/150x150/img-master/img/2010/11/30/08/39/58/14901720_p0_master1200.jpg"
+        source4 = "http://i1.pixiv.net/img-original/img/2010/11/30/08/39/58/14901720_p0.png"
+        source5 = "http://i2.pixiv.net/img-zip-ugoira/img/2014/08/05/06/01/10/44524589_ugoira1920x1080.zip"
+
+        assert_equal("https://www.pixiv.net/artworks/39749565", Sources::Strategies.normalize_source(source1))
+        assert_equal("https://www.pixiv.net/artworks/39735353", Sources::Strategies.normalize_source(source2))
+        assert_equal("https://www.pixiv.net/artworks/14901720", Sources::Strategies.normalize_source(source3))
+        assert_equal("https://www.pixiv.net/artworks/14901720", Sources::Strategies.normalize_source(source4))
+        assert_equal("https://www.pixiv.net/artworks/44524589", Sources::Strategies.normalize_source(source5))
+      end
+    end
  end
 end
--- a/test/unit/sources/tumblr_test.rb
+++ b/test/unit/sources/tumblr_test.rb
@@ -259,5 +259,24 @@ module Sources
        end
      end
    end
+
+    context "normalizing for source" do
+      should "normalize correctly" do
+        source1 = "https://octrain1020.tumblr.com/post/190713122589"
+        source2 = "https://octrain1020.tumblr.com/image/190713122589"
+        source3 = "https://octrain1020.tumblr.com/image/190713122589#asd"
+        source4 = "https://superboin.tumblr.com/post/141169066579/photoset_iframe/superboin/tumblr_o45miiAOts1u6rxu8/500/false"
+
+        assert_equal(source1, Sources::Strategies.normalize_source(source1))
+        assert_equal(source1, Sources::Strategies.normalize_source(source2))
+        assert_equal(source1, Sources::Strategies.normalize_source(source3))
+        assert_equal("https://superboin.tumblr.com/post/141169066579", Sources::Strategies.normalize_source(source4))
+      end
+
+      should "avoid normalizing unnormalizable urls" do
+        bad_source = "https://octrain1020.tumblr.com/"
+        assert_equal(bad_source, Sources::Strategies.normalize_source(bad_source))
+      end
+    end
  end
 end
--- a/test/unit/sources/twitter_test.rb
+++ b/test/unit/sources/twitter_test.rb
@@ -298,5 +298,24 @@ module Sources
        assert_includes(site.translated_tags.map(&:name), "nishizumi_miho")
      end
    end
+
+    context "normalizing for source" do
+      should "normalize correctly" do
+        source1 = "https://twitter.com/i/web/status/1261877313349640194"
+        source2 = "https://twitter.com/BOW999/status/1261877313349640194"
+        source3 = "https://twitter.com/BOW999/status/1261877313349640194/photo/1"
+        source4 = "https://twitter.com/BOW999/status/1261877313349640194?s=19"
+
+        assert_equal(source1, Sources::Strategies.normalize_source(source1))
+        assert_equal(source1, Sources::Strategies.normalize_source(source2))
+        assert_equal(source1, Sources::Strategies.normalize_source(source3))
+        assert_equal(source1, Sources::Strategies.normalize_source(source4))
+      end
+
+      should "normalize twimg twitpic correctly" do
+        source = "https://o.twimg.com/2/proxy.jpg?t=HBgpaHR0cHM6Ly90d2l0cGljLmNvbS9zaG93L2xhcmdlL2R0bnVydS5qcGcUsAkU0ggAFgASAA&s=dnN4DHCdnojC-iCJWdvZ-UZinrlWqAP7k7lmll2fTxs"
+        assert_equal("https://twitpic.com/dtnuru", Sources::Strategies.normalize_source(source))
+      end
+    end
  end
 end