sources: add image_url?, page_url?, and profile_url? methods.

Add methods to Source::URL for determining whether a URL is an image
URL, a page URL, or a profile URL.

Also add more source URL tests and fix various URL parsing bugs.

app/logical/source/extractor/fantia.rb

@@ -11,14 +11,14 @@ class Source::Extractor
     end
 
     def image_urls
-      return [parsed_url.full_image_url] if parsed_url.image_url?
+      return [parsed_url.full_image_url] if parsed_url.full_image_url.present?
       return [image_from_downloadable(parsed_url)] if parsed_url.downloadable?
 
       images = images_for_post.presence || images_for_product.presence || []
 
       full_images = images.compact.map do |image|
         parsed = Source::URL.parse(image)
-        if parsed&.image_url?
+        if parsed&.full_image_url.present?
           parsed.full_image_url
         elsif parsed&.downloadable?
           image_from_downloadable(parsed)

app/logical/source/extractor/tumblr.rb

@@ -12,7 +12,7 @@ class Source::Extractor
     end
 
     def image_urls
-      return [find_largest(parsed_url)].compact if parsed_url.asset_url?
+      return [find_largest(parsed_url)].compact if parsed_url.image_url?
 
       assets = []
 

app/logical/source/url.rb

@@ -87,6 +87,43 @@ module Source
       self.class.name.demodulize.titleize
     end
 
+    # True if the URL is a direct image URL.
+    #
+    # Examples:
+    #
+    # * https://i.pximg.net/img-original/img/2014/10/03/18/10/20/46324488_p0.png
+    # * https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/8b472d70-a0d6-41b5-9a66-c35687090acc/d23jbr4-8a06af02-70cb-46da-8a96-42a6ba73cdb4.jpg/v1/fill/w_786,h_1017,q_70,strp/silverhawks_quicksilver_by_edsfox_d23jbr4-pre.jpg
+    # * https://pbs.twimg.com/media/EBGbJe_U8AA4Ekb.jpg:orig
+    #
+    # @return [Boolean]
+    def image_url?
+      file_ext.in?(%w[jpg jpeg png gif webp webm mp4 swf])
+    end
+
+    # True if the URL is a work page URL.
+    #
+    # Examples:
+    #
+    # * https://www.pixiv.net/artworks/46324488
+    # * https://twitter.com/motty08111213/status/943446161586733056
+    #
+    # @return [Boolean]
+    def page_url?
+      page_url.present? && !image_url?
+    end
+
+    # True if the URL is a profile page URL.
+    #
+    # Examples:
+    #
+    # * https://www.pixiv.net/users/9948
+    # * https://twitter.com/intent/user?user_id=889592953
+    #
+    # @return [Boolean]
+    def profile_url?
+      profile_url.present? && !page_url? && !image_url?
+    end
+
     # Convert an image URL to the URL of the page containing the image, or
     # return nil if it's not possible to convert the current URL to a page URL.
     #
@@ -133,6 +170,18 @@ module Source
       nil
     end
 
+    def self.image_url?(url)
+      Source::URL.parse(url)&.image_url?
+    end
+
+    def self.page_url?(url)
+      Source::URL.parse(url)&.page_url?
+    end
+
+    def self.profile_url?(url)
+      Source::URL.parse(url)&.profile_url?
+    end
+
     def self.page_url(url)
       Source::URL.parse(url)&.page_url
     end

app/logical/source/url/art_station.rb

@@ -65,7 +65,7 @@ class Source::URL::ArtStation < Source::URL
   end
 
   def image_url?
-    subdomain.starts_with?("cdn")
+    subdomain.to_s.starts_with?("cdn")
   end
 
   def full_image_url(size = "original")

app/logical/source/url/booth.rb

@@ -67,7 +67,7 @@ module Source
     end
 
     def image_url?
-      url.host.in?(["booth.pximg.net", "s2.booth.pm"])
+      host.in?(%w[booth.pximg.net s.booth.pm s2.booth.pm])
     end
 
     def full_image_url?

app/logical/source/url/deviant_art.rb

@@ -18,7 +18,7 @@ module Source
   class URL::DeviantArt < Source::URL
     RESERVED_SUBDOMAINS = %w[www]
 
-    attr_reader :username, :work_id, :stash_id, :title
+    attr_reader :username, :work_id, :stash_id, :title, :file
 
     def self.match?(url)
       url.domain.in?(%w[deviantart.net deviantart.com fav.me sta.sh]) || url.host.in?(%w[images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com wixmp-ed30a86b8c4ca887773594c2.wixmp.com api-da.wixmp.com])
@@ -106,6 +106,7 @@ module Source
       # https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/f/fe7ab27f-7530-4252-99ef-2baaf81b36fd/dddf6pe-1a4a091c-768c-4395-9465-5d33899be1eb.png/v1/fill/w_800,h_1130,q_80,strp/stay_hydrated_and_in_the_shade_by_raikoart_dddf6pe-fullview.jpg?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ1cm46YXBwOjdlMGQxODg5ODIyNjQzNzNhNWYwZDQxNWVhMGQyNmUwIiwiaXNzIjoidXJuOmFwcDo3ZTBkMTg4OTgyMjY0MzczYTVmMGQ0MTVlYTBkMjZlMCIsIm9iaiI6W1t7ImhlaWdodCI6Ijw9MTEzMCIsInBhdGgiOiJcL2ZcL2ZlN2FiMjdmLTc1MzAtNDI1Mi05OWVmLTJiYWFmODFiMzZmZFwvZGRkZjZwZS0xYTRhMDkxYy03NjhjLTQzOTUtOTQ2NS01ZDMzODk5YmUxZWIucG5nIiwid2lkdGgiOiI8PTgwMCJ9XV0sImF1ZCI6WyJ1cm46c2VydmljZTppbWFnZS5vcGVyYXRpb25zIl19.J0W4k-iV6Mg8Kt_5Lr_L_JbBq4lyr7aCausWWJ_Fsbw
       # https://www.deviantart.com/download/549677536/countdown_to_midnight_by_kawacy-d939hwg.jpg?token=92090cd3910d52089b566661e8c2f749755ed5f8&ts=1438535525
       when /^(.+)_by_(.+)[_-]d([a-z0-9]+)(?:-\w+)?$/i
+        @file = filename
         @title = $1
         @username = $2.dasherize
         @work_id = $3.to_i(36)
@@ -113,11 +114,13 @@ module Source
       # http://fc00.deviantart.net/fs71/f/2013/234/d/8/d84e05f26f0695b1153e9dab3a962f16-d6j8jl9.jpg
       # http://th04.deviantart.net/fs71/PRE/f/2013/337/3/5/35081351f62b432f84eaeddeb4693caf-d6wlrqs.jpg
       when /^[a-f0-9]{32}-d([a-z0-9]+)$/
+        @file = filename
         @work_id = $1.to_i(36)
 
       # https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/52c4a3ad-d416-42f0-90f6-570983e36797/dczr28f-bd255304-01bf-4765-8cd3-e53983d3f78a.jpg
       # https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/f/76098ac8-04ab-4784-b382-88ca082ba9b1/d9x7lmk-595099de-fe8f-48e5-9841-7254f9b2ab8d.png?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ1cm46YXBwOiIsImlzcyI6InVybjphcHA6Iiwib2JqIjpbW3sicGF0aCI6IlwvZlwvNzYwOThhYzgtMDRhYi00Nzg0LWIzODItODhjYTA4MmJhOWIxXC9kOXg3bG1rLTU5NTA5OWRlLWZlOGYtNDhlNS05ODQxLTcyNTRmOWIyYWI4ZC5wbmcifV1dLCJhdWQiOlsidXJuOnNlcnZpY2U6ZmlsZS5kb3dubG9hZCJdfQ.KFOVXAiF8MTlLb3oM-FlD0nnDvODmjqEhFYN5I2X5Bc
       when /^d([a-z0-9]{6})-\h{8}-\h{4}-\h{4}-\h{4}-\h{12}$/
+        @file = filename
         @work_id = $1.to_i(36)
 
       # http://www.deviantart.com/download/135944599/Touhou___Suwako_Moriya_Colored_by_Turtle_Chibi.png
@@ -125,14 +128,20 @@ module Source
       # http://fc02.deviantart.net/fs48/f/2009/186/2/c/Animation_by_epe_tohri.swf
       # http://fc08.deviantart.net/files/f/2007/120/c/9/Cool_Like_Me_by_47ness.jpg
       when /^(.+)_by_(.+)$/
+        @file = filename
         @title = $1
         @username = $2.dasherize
 
       else
-        nil
+        @file = filename
+
       end
     end
 
+    def image_url?
+      file.present?
+    end
+
     def page_url
       if stash_id.present?
         "https://sta.sh/#{stash_id}"

app/logical/source/url/fanbox.rb

@@ -48,7 +48,7 @@ class Source::URL::Fanbox < Source::URL
 
     # http://www.pixiv.net/fanbox/member.php?user_id=3410642
     in "www.pixiv.net", "fanbox", "member.php" if params[:user_id].present?
-      @user_id = user_id
+      @user_id = params[:user_id]
 
     # https://omu001.fanbox.cc/posts/39714
     # https://brllbrll.fanbox.cc/posts/626093 (R-18)

app/logical/source/url/fandom.rb

@@ -44,6 +44,10 @@ class Source::URL::Fandom < Source::URL
     end
   end
 
+  def image_url?
+    full_image_url.present?
+  end
+
   def wiki
     WIKI_DB_NAMES.fetch(wiki_db_name, wiki_db_name)
   end

app/logical/source/url/fantia.rb

@@ -70,7 +70,7 @@ class Source::URL::Fantia < Source::URL
   end
 
   def image_url?
-    @full_image_url.present?
+    @full_image_url.present? || downloadable?
   end
 
   def downloadable?

app/logical/source/url/fc2.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 class Source::URL::Fc2 < Source::URL
-  attr_reader :username, :profile_url, :page_url
+  attr_reader :username, :profile_url, :page_url, :file
 
   def self.match?(url)
     url.domain.in?(%w[fc2.com fc2blog.net fc2blog.us])
@@ -40,6 +40,7 @@ class Source::URL::Fc2 < Source::URL
     # http://blog23.fc2.com/m/mosha2/file/uru.jpg
     # http://blog.fc2.com/g/genshi/file/20070612a.jpg
     in /^blog\d*$/, "fc2", "com", /^\w$/, username, "file", file
+      @file = file
       @username = username
       @profile_url = "http://#{username}.blog.fc2.com"
 
@@ -47,6 +48,7 @@ class Source::URL::Fc2 < Source::URL
     # http://blog-imgs-61.fc2.com/o/m/o/omochi6262/20130402080220583.jpg
     # http://blog.fc2.com/g/b/o/gbot/20071023195141.jpg
     in (/^blog-imgs-\d+(-origin)?$/ | "blog"), "fc2", "com", /^\w$/, /^\w$/, /^\w$/, username, file
+      @file = file
       @username = username
       @page_url = "http://#{username}.blog.fc2.com/img/#{file}"
       @profile_url = "http://#{username}.blog.fc2.com"
@@ -55,6 +57,7 @@ class Source::URL::Fc2 < Source::URL
     # http://diary1.fc2.com/user/kou_48/img/2006_8/14.jpg
     # http://diary.fc2.com/user/kazuharoom/img/2015_5/22.jpg
     in /diary\d*$/, "fc2", "com", "user", username, "img", date, file
+      @file = file
       @username = username
       @year, @month = date.split("_")
       @day = filename

app/logical/source/url/foundation.rb

@@ -9,7 +9,7 @@
 # * https://foundation.app/collection/kgfgen
 
 class Source::URL::Foundation < Source::URL
-  attr_reader :username, :token_id, :work_id, :hash
+  attr_reader :username, :user_id, :token_id, :work_id, :hash
 
   IMAGE_HOSTS = %w[assets.foundation.app f8n-ipfs-production.imgix.net f8n-production-collection-assets.imgix.net d2ybmb80bbm9ts.cloudfront.net]
 
@@ -69,6 +69,10 @@ class Source::URL::Foundation < Source::URL
     end
   end
 
+  def image_url?
+    host.in?(IMAGE_HOSTS)
+  end
+
   def profile_url
     if username.present?
       "https://foundation.app/@#{username}"

app/logical/source/url/furaffinity.rb

@@ -12,7 +12,8 @@ class Source::URL::Furaffinity < Source::URL
 
     # https://www.furaffinity.net/view/46821705/
     # https://www.furaffinity.net/view/46802202/ (scrap)
-    in _, "view", /^\d+$/ => work_id
+    # https://www.furaffinity.net/full/46821705/
+    in _, ("view" | "full"), /^\d+$/ => work_id
       @work_id = work_id
 
     # https://d.furaffinity.net/art/iwbitu/1650222955/1650222955.iwbitu_yubi.jpg

app/logical/source/url/hentai_foundry.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 class Source::URL::HentaiFoundry < Source::URL
-  attr_reader :username, :work_id
+  attr_reader :username, :work_id, :file
 
   def self.match?(url)
     url.domain == "hentai-foundry.com"
@@ -12,17 +12,20 @@ class Source::URL::HentaiFoundry < Source::URL
 
     # https://pictures.hentai-foundry.com/a/Afrobull/795025/Afrobull-795025-kuroeda.png
     # https://pictures.hentai-foundry.com/_/-MadKaiser-/532792/-MadKaiser--532792-FFXIV_Miqote.png
-    in "pictures.hentai-foundry.com", _, username, /^\d+$/ => work_id, slug
+    in "pictures.hentai-foundry.com", _, username, /^\d+$/ => work_id, file
+      @file = file
       @username = username
       @work_id = work_id
 
     # http://pictures.hentai-foundry.com//s/soranamae/363663.jpg
-    in "pictures.hentai-foundry.com", _, username, /^(\d+)\.\w+$/
+    in "pictures.hentai-foundry.com", _, username, /^(\d+)\.\w+$/ => file
+      @file = file
       @username = username
       @work_id = $1
 
     # http://www.hentai-foundry.com/piccies/d/dmitrys/1183.jpg
-    in "www.hentai-foundry.com", "piccies", _, username, /^(\d+)\.\w+$/
+    in "www.hentai-foundry.com", "piccies", _, username, /^(\d+)\.\w+$/ => file
+      @file = file
       @username = username
       @work_id = $1
 
@@ -64,6 +67,10 @@ class Source::URL::HentaiFoundry < Source::URL
     end
   end
 
+  def image_url?
+    file.present? || host == "thumbs.hentai-foundry.com"
+  end
+
   def page_url
     if username.present? && work_id.present?
       "https://www.hentai-foundry.com/pictures/user/#{username}/#{work_id}"

app/logical/source/url/instagram.rb

@@ -9,7 +9,7 @@ class Source::URL::Instagram < Source::URL
   attr_reader :username, :work_id
 
   def self.match?(url)
-    url.domain.in?(%w[instagram.com])
+    url.domain.in?(%w[instagram.com cdninstagram.com]) || (url.domain == "fbcdn.net" && url.subdomain.include?("instagram"))
   end
 
   def parse
@@ -35,6 +35,10 @@ class Source::URL::Instagram < Source::URL
     end
   end
 
+  def image_url?
+    domain.in?(%w[cdninstagram.com fbcdn.net])
+  end
+
   def page_url
     "https://www.instagram.com/p/#{work_id}/" if work_id.present?
   end

app/logical/source/url/mastodon.rb

@@ -6,7 +6,7 @@
 # * https://pawoo.net/oauth_authentications/17230064
 
 class Source::URL::Mastodon < Source::URL
-  attr_reader :username, :user_id, :work_id, :full_image_url
+  attr_reader :username, :user_id, :work_id, :full_image_url, :media_hash
 
   def self.match?(url)
     url.domain.in?(%w[pawoo.net baraag.net])
@@ -61,6 +61,7 @@ class Source::URL::Mastodon < Source::URL
     # https://pawoo.net/media/lU2uV7C1MMQSb1czwvg
     in _, "pawoo.net", "media", media_hash
       @media_hash = media_hash
+      @full_image_url = "#{site}/media/#{media_hash}"
 
     else
       nil

app/logical/source/url/moebooru.rb

@@ -87,6 +87,10 @@ class Source::URL::Moebooru < Source::URL
     end
   end
 
+  def image_url?
+    md5.present?
+  end
+
   def page_url
     if work_id.present?
       "https://#{domain}/post/show/#{work_id}"

app/logical/source/url/nico_seiga.rb

@@ -157,6 +157,10 @@ module Source
       end
     end
 
+    def image_url?
+      host.in?(%w[lohas.nicoseiga.jp dcdn.cdn.nimg.jp drm.cdn.nicomanga.jp]) || (host == "seiga.nicovideo.jp" && path.starts_with?("/image/"))
+    end
+
     def page_url
       if illust_id.present?
         "https://seiga.nicovideo.jp/seiga/im#{illust_id}"

app/logical/source/url/tinami.rb

@@ -55,7 +55,7 @@ class Source::URL::Tinami < Source::URL
   end
 
   def image_url?
-    host == "img.tinami.com"
+    host == "img.tinami.com" || path.starts_with?("/view/tweet/card/")
   end
 
   def page_url

app/logical/source/url/tumblr.rb

@@ -12,7 +12,7 @@ class Source::URL::Tumblr < Source::URL
 
     # https://66.media.tumblr.com/168dabd09d5ad69eb5fedcf94c45c31a/3dbfaec9b9e0c2e3-72/s640x960/bf33a1324f3f36d2dc64f011bfeab4867da62bc8.png
     # https://66.media.tumblr.com/5a2c3fe25c977e2281392752ab971c90/3dbfaec9b9e0c2e3-92/s500x750/4f92bbaaf95c0b4e7970e62b1d2e1415859dd659.png
-    in _, *directories, /s\d+x\d+/ => dimensions, file if asset_url?
+    in _, *directories, /s\d+x\d+/ => dimensions, file if image_url?
       @directory = directories.first
       max_size = Integer.sqrt(Danbooru.config.max_image_resolution)
       @full_image_url = url.to_s.gsub(%r{/s\d+x\d+/\w+\.\w+\z}i, "/s#{max_size}x#{max_size}/#{file}")
@@ -29,7 +29,7 @@ class Source::URL::Tumblr < Source::URL
     # https://media.tumblr.com/0DNBGJovY5j3smfeQs8nB53z_500.jpg
     # https://media.tumblr.com/tumblr_m24kbxqKAX1rszquso1_1280.jpg
     # https://va.media.tumblr.com/tumblr_pgohk0TjhS1u7mrsl.mp4
-    in _, *directory, file if asset_url?
+    in _, *directory, file if image_url?
       @directory = directory.first
       @filename, @old_variant_size, @extension = file.match(/(\w+?)(?:_(\d+h?|raw))?\.(\w+)\z/).captures
 
@@ -65,7 +65,7 @@ class Source::URL::Tumblr < Source::URL
     # https://rosarrie.tumblr.com/archive
     # https://solisnotte.tumblr.com/about
     # http://whereisnovember.tumblr.com/tagged/art
-    in _, *rest unless asset_url? || subdomain == "www"
+    in _, *rest unless image_url? || subdomain == "www"
       @blog_name = subdomain
 
     else
@@ -73,7 +73,7 @@ class Source::URL::Tumblr < Source::URL
     end
   end
 
-  def asset_url?
+  def image_url?
     host.ends_with?("media.tumblr.com") || host == "data.tumblr.com"
   end
 

app/logical/source/url/twitter.rb

@@ -48,10 +48,6 @@ class Source::URL::Twitter < Source::URL
       @username = username
       @status_id = status_id
 
-    # https://twitter.com/motty08111213
-    in "twitter.com", username, *rest
-      @username = username unless username.in?(RESERVED_USERNAMES)
-
     # https://twitter.com/intent/user?user_id=1485229827984531457
     in "twitter.com", "intent", "user" if params[:user_id].present?
       @user_id = params[:user_id]
@@ -84,6 +80,12 @@ class Source::URL::Twitter < Source::URL
       # /media/EBGbJe_U8AA4Ekb.jpg
       # /ext_tw_video_thumb/1243725361986375680/pu/img/JDA7g7lcw7wK-PIv.jpg
       @file_path = File.join(media_type, subdirs.join("/"), "#{@file}.#{@file_ext}")
+
+    # https://twitter.com/motty08111213
+    # https://twitter.com/motty08111213/likes
+    in "twitter.com", username, *rest unless username.in?(RESERVED_USERNAMES)
+      @username = username
+
     else
       nil
     end

app/logical/source/url/weibo.rb

@@ -9,6 +9,8 @@
 # http://blog.sina.com.cn/u/1299088063
 
 class Source::URL::Weibo < Source::URL
+  RESERVED_USERNAMES = %w[u p profile status detail]
+
   attr_reader :full_image_url, :artist_short_id, :artist_long_id, :username
 
   def self.match?(url)
@@ -75,7 +77,7 @@ class Source::URL::Weibo < Source::URL
     # https://www.weibo.com/endlessnsmt (short id: https://www.weibo.com/u/1879370780)
     # https://www.weibo.cn/endlessnsmt
     # https://www.weibo.com/lvxiuzi0/home
-    in _, /^\w+$/ => artist_short_id, *rest
+    in _, /^\w+$/ => username, *rest unless username.in?(RESERVED_USERNAMES)
       @username = username
 
     else