jaculabilis/danbooru
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #4489 from nonamethanks/add_weibo_normalization

Browse Source 
Weibo: add source normalization
This commit is contained in:
evazion
2020-05-29 11:29:40 -05:00
committed by GitHub
parent fb19bf06ce d339947647
commit 9ba6d60ad3
2 changed files with 45 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
app/logical/sources/strategies/weibo.rb
View File

@@ -32,16 +32,16 @@
module Sources
module Strategies
class Weibo < Base
PROFILE_URL_1 = %r{https?://(?:(?:www|m)\.)?weibo\.c(?:om|n)/(?:(?:u|profile)/)?(?<artist_short_id>\d+)\z}i
PROFILE_URL_2 = %r{https?://photo\.weibo\.com/(?<artist_short_id>\d+)}i
PROFILE_URL_3 = %r{https?://(?:www\.)?weibo\.com/p/(?<artist_long_id>\d+)}i
PROFILE_URL_1 = %r{\Ahttps?://(?:(?:www|m)\.)?weibo\.c(?:om|n)/(?:(?:u|profile)/)?(?<artist_short_id>\d+)\z}i
PROFILE_URL_2 = %r{\Ahttps?://photo\.weibo\.com/(?<artist_short_id>\d+)}i
PROFILE_URL_3 = %r{\Ahttps?://(?:www\.)?weibo\.com/p/(?<artist_long_id>\d+)}i
PAGE_URL_1 = %r{https?://(?:www\.)?weibo\.com/(?<artist_short_id>\d+)/(?<illust_base62_id>\w+)(?:\?.*)?\z}i
PAGE_URL_1 = %r{\Ahttps?://(?:www\.)?weibo\.com/(?<artist_short_id>\d+)/(?<illust_base62_id>\w+)(?:\?.*)?\z}i
PAGE_URL_2 = %r{#{PROFILE_URL_2}/(?:wbphotos/large/mid|talbum/detail/photo_id)/(?<illust_long_id>\d+)(?:/pid/(?<image_id>\w{32}))?}i
PAGE_URL_3 = %r{https?://m\.weibo\.cn/(detail/(?<illust_long_id>\d+)|status/(?<illust_base62_id>\w+))}i
PAGE_URL_4 = %r{https?://tw\.weibo\.com/(?:(?<artist_short_id>\d+)|\w+)/(?<illust_long_id>\d+)}i
PAGE_URL_3 = %r{\Ahttps?://m\.weibo\.cn/(detail/(?<illust_long_id>\d+)|status/(?<illust_base62_id>\w+))}i
PAGE_URL_4 = %r{\Ahttps?://tw\.weibo\.com/(?:(?<artist_short_id>\d+)|\w+)/(?<illust_long_id>\d+)}i
IMAGE_URL = %r{https?://\w{3}\.sinaimg\.cn/\w+/(?<image_id>\w{32})\.}i
IMAGE_URL = %r{\Ahttps?://\w{3}\.sinaimg\.cn/\w+/(?<image_id>\w{32})\.}i
def domains
["weibo.com", "weibo.cn", "weibocdn.com", "sinaimg.cn"]
@@ -168,6 +168,21 @@ module Sources
profile_url || url
end
def normalize_for_source
return url if url =~ PAGE_URL_2
artist_id = artist_short_id_from_url
if artist_id.present?
if illust_base62_id.present?
"https://www.weibo.com/#{artist_id}/#{illust_base62_id}"
elsif illust_long_id.present?
"https://photo.weibo.com/#{artist_id}/talbum/detail/photo_id/#{illust_long_id}"
end
elsif mobile_url.present?
mobile_url
end
end
def self.convert_image_to_large(url)
url.gsub(%r{.cn/\w+/(\w+)}, '.cn/large/\1')
end
@@ -181,7 +196,7 @@ module Sources
end
def artist_short_id_from_url
[url, referer_url].compact.map { |x| x[PROFILE_URL_1, :artist_short_id] || x[PROFILE_URL_2, :artist_short_id] || x[PAGE_URL_4, :artist_short_id] }.compact.first
[url, referer_url].compact.map { |x| x[PROFILE_URL_1, :artist_short_id] || x[PROFILE_URL_2, :artist_short_id] || x[PAGE_URL_1, :artist_short_id] || x[PAGE_URL_4, :artist_short_id] }.compact.first
end
def artist_short_id

22
test/unit/sources/weibo_test.rb
View File

@@ -88,5 +88,27 @@ module Sources
assert_equal("https://www.weibo.com/2125874520/FDKGo4Lk0", site.canonical_url)
end
end
context "normalizing for source" do
should "normalize correctly" do
source1 = "https://www.weibo.com/3150932560/H4cFbeKKA?from=page_1005053150932560_profile&wvr=6&mod=weibotime"
source2 = "http://photo.weibo.com/2125874520/wbphotos/large/mid/4242129997905387/pid/7eb64558ly1friyzhj44lj20dw2qxe81"
source3 = "https://m.weibo.cn/status/4173757483008088?luicode=20000061&lfid=4170879204256635"
source4 = "https://tw.weibo.com/SEINEN/4098035921690224"
assert_equal("https://www.weibo.com/3150932560/H4cFbeKKA", Sources::Strategies.normalize_source(source1))
assert_equal(source2, Sources::Strategies.normalize_source(source2))
assert_equal("https://m.weibo.cn/status/4173757483008088", Sources::Strategies.normalize_source(source3))
assert_equal("https://m.weibo.cn/detail/4098035921690224", Sources::Strategies.normalize_source(source4))
end
should "avoid normalizing unnormalizable urls" do
bad_source1 = "https://weibo.com/u/"
bad_source2 = "https://www.weibo.com/4ubergine/photos"
assert_equal(bad_source1, Sources::Strategies.normalize_source(bad_source1))
assert_equal(bad_source2, Sources::Strategies.normalize_source(bad_source2))
end
end
end
end