diff --git a/app/logical/danbooru/url.rb b/app/logical/danbooru/url.rb index 7dce194a0..747cd7cfa 100644 --- a/app/logical/danbooru/url.rb +++ b/app/logical/danbooru/url.rb @@ -68,6 +68,17 @@ module Danbooru nil end + # Escape a string for use in an URL path or query parameter. Like `CGI.escape`, but leaves Unicode characters as Unicode. + # + # @example + # Danbooru::URL.escape("fate/stay_night") # => "fate%2Fstay_night" + # Danbooru::URL.escape("大丈夫?おっぱい揉む?") # => "大丈夫%3Fおっぱい揉む%3F" + # + # @return [String] The escaped string + def self.escape(string) + Addressable::URI.encode_component(string, /[\/?#&+%]/).force_encoding("UTF-8") + end + # @return [String] the URL in unnormalized form def to_s original_url diff --git a/app/logical/source/extractor/art_station.rb b/app/logical/source/extractor/art_station.rb index e6089438b..a9c0bfe7c 100644 --- a/app/logical/source/extractor/art_station.rb +++ b/app/logical/source/extractor/art_station.rb @@ -48,7 +48,7 @@ class Source::Extractor def tags api_response[:tags].to_a.map do |tag| - [tag, "https://www.artstation.com/search?q=#{CGI.escape(tag)}"] + [tag, "https://www.artstation.com/search?q=#{Danbooru::URL.escape(tag)}"] end end diff --git a/app/logical/source/extractor/gelbooru.rb b/app/logical/source/extractor/gelbooru.rb index 92933bf1f..54b0593a8 100644 --- a/app/logical/source/extractor/gelbooru.rb +++ b/app/logical/source/extractor/gelbooru.rb @@ -39,7 +39,7 @@ module Source tags = api_response[:tags].split + ["rating:#{api_response[:rating]}"] tags.map do |tag| - [tag, "https://#{domain}/index.php?page=post&s=list&tags=#{CGI.escape(tag)}"] + [tag, "https://#{domain}/index.php?page=post&s=list&tags=#{Danbooru::URL.escape(tag)}"] end end diff --git a/app/logical/source/extractor/hentai_foundry.rb b/app/logical/source/extractor/hentai_foundry.rb index a36aeeb99..ef7268815 100644 --- a/app/logical/source/extractor/hentai_foundry.rb +++ b/app/logical/source/extractor/hentai_foundry.rb @@ -39,7 +39,7 @@ module Source tags = page&.search(".boxbody [rel='tag']").to_a.map(&:text) tags.map do |tag| - [tag, "https://www.hentai-foundry.com/pictures/tagged/#{CGI.escape(tag)}"] + [tag, "https://www.hentai-foundry.com/pictures/tagged/#{Danbooru::URL.escape(tag)}"] end end diff --git a/app/logical/source/extractor/moebooru.rb b/app/logical/source/extractor/moebooru.rb index 30171da47..b6935efc3 100644 --- a/app/logical/source/extractor/moebooru.rb +++ b/app/logical/source/extractor/moebooru.rb @@ -23,7 +23,7 @@ module Source def tags api_response[:tags].to_s.split.map do |tag| - [tag, "https://#{domain}/post?tags=#{CGI.escape(tag)}"] + [tag, "https://#{domain}/post?tags=#{Danbooru::URL.escape(tag)}"] end end diff --git a/app/logical/source/extractor/nico_seiga.rb b/app/logical/source/extractor/nico_seiga.rb index cc9abe6dd..80494a05a 100644 --- a/app/logical/source/extractor/nico_seiga.rb +++ b/app/logical/source/extractor/nico_seiga.rb @@ -77,12 +77,8 @@ module Source def tags return [] if api_client.blank? - base_url = "https://seiga.nicovideo.jp/" - base_url += "manga/" if manga_id.present? - base_url += "tag/" - api_client.tags.map do |name| - [name, base_url + CGI.escape(name)] + [name, "https://seiga.nicovideo.jp/#{"manga/" if manga_id}tag/#{Danbooru::URL.escape(name)}"] end end diff --git a/app/logical/source/extractor/pixiv.rb b/app/logical/source/extractor/pixiv.rb index e8ae4eb52..e7bdc7cc5 100644 --- a/app/logical/source/extractor/pixiv.rb +++ b/app/logical/source/extractor/pixiv.rb @@ -105,7 +105,7 @@ module Source def tags tags = api_illust.dig(:tags, :tags).to_a.map do |item| - [item[:tag], "https://www.pixiv.net/tags/#{CGI.escape(item[:tag])}/artworks"] + [item[:tag], "https://www.pixiv.net/tags/#{Danbooru::URL.escape(item[:tag])}/artworks"] end if api_illust["aiType"] == 2 diff --git a/app/logical/source/extractor/rule34_dot_us.rb b/app/logical/source/extractor/rule34_dot_us.rb index d87597921..32d6ec47b 100644 --- a/app/logical/source/extractor/rule34_dot_us.rb +++ b/app/logical/source/extractor/rule34_dot_us.rb @@ -28,7 +28,7 @@ module Source def tags page&.css("meta[name='keywords']")&.attr("content")&.value.to_s.split(/, /).compact.map do |tag| - [tag.tr(" ", "_"), "https://rule34.us/index.php?r=posts/index&q=#{CGI.escape(tag)}"] + [tag.tr(" ", "_"), "https://rule34.us/index.php?r=posts/index&q=#{Danbooru::URL.escape(tag)}"] end end diff --git a/app/logical/source/extractor/tinami.rb b/app/logical/source/extractor/tinami.rb index 007c55382..8ba5844ac 100644 --- a/app/logical/source/extractor/tinami.rb +++ b/app/logical/source/extractor/tinami.rb @@ -47,7 +47,7 @@ module Source def tags page&.css("#view .tag a[href^='/search/list']").to_a.map do |tag| - [tag.text, "https://www.tinami.com/search/list?keyword=#{CGI.escape(tag.text)}"] + [tag.text, "https://www.tinami.com/search/list?keyword=#{Danbooru::URL.escape(tag.text)}"] end end diff --git a/app/logical/source/extractor/tumblr.rb b/app/logical/source/extractor/tumblr.rb index ea9a8a85c..fe05f8ea2 100644 --- a/app/logical/source/extractor/tumblr.rb +++ b/app/logical/source/extractor/tumblr.rb @@ -75,7 +75,7 @@ class Source::Extractor def tags post[:tags].to_a.map do |tag| - [tag, "https://tumblr.com/tagged/#{CGI.escape(tag)}"] + [tag, "https://tumblr.com/tagged/#{Danbooru::URL.escape(tag)}"] end.uniq end diff --git a/test/unit/sources/nico_seiga_test.rb b/test/unit/sources/nico_seiga_test.rb index 8e76951b5..c31339a96 100644 --- a/test/unit/sources/nico_seiga_test.rb +++ b/test/unit/sources/nico_seiga_test.rb @@ -8,12 +8,12 @@ module Sources context "A nicoseiga post url" do tags = [ - ["アニメ", "https://seiga.nicovideo.jp/tag/%E3%82%A2%E3%83%8B%E3%83%A1"], - ["コジコジ", "https://seiga.nicovideo.jp/tag/%E3%82%B3%E3%82%B8%E3%82%B3%E3%82%B8"], - ["さくらももこ", "https://seiga.nicovideo.jp/tag/%E3%81%95%E3%81%8F%E3%82%89%E3%82%82%E3%82%82%E3%81%93"], - ["ドット絵", "https://seiga.nicovideo.jp/tag/%E3%83%89%E3%83%83%E3%83%88%E7%B5%B5"], - ["ニコニコ大百科", "https://seiga.nicovideo.jp/tag/%E3%83%8B%E3%82%B3%E3%83%8B%E3%82%B3%E5%A4%A7%E7%99%BE%E7%A7%91"], - ["お絵カキコ", "https://seiga.nicovideo.jp/tag/%E3%81%8A%E7%B5%B5%E3%82%AB%E3%82%AD%E3%82%B3"], + ["アニメ", "https://seiga.nicovideo.jp/tag/アニメ"], + ["コジコジ", "https://seiga.nicovideo.jp/tag/コジコジ"], + ["さくらももこ", "https://seiga.nicovideo.jp/tag/さくらももこ"], + ["ドット絵", "https://seiga.nicovideo.jp/tag/ドット絵"], + ["ニコニコ大百科", "https://seiga.nicovideo.jp/tag/ニコニコ大百科"], + ["お絵カキコ", "https://seiga.nicovideo.jp/tag/お絵カキコ"], ] strategy_should_work( "http://seiga.nicovideo.jp/seiga/im4937663", @@ -32,12 +32,12 @@ module Sources context "A nicoseiga image url" do tags = [ - ["アニメ", "https://seiga.nicovideo.jp/tag/%E3%82%A2%E3%83%8B%E3%83%A1"], - ["コジコジ", "https://seiga.nicovideo.jp/tag/%E3%82%B3%E3%82%B8%E3%82%B3%E3%82%B8"], - ["さくらももこ", "https://seiga.nicovideo.jp/tag/%E3%81%95%E3%81%8F%E3%82%89%E3%82%82%E3%82%82%E3%81%93"], - ["ドット絵", "https://seiga.nicovideo.jp/tag/%E3%83%89%E3%83%83%E3%83%88%E7%B5%B5"], - ["ニコニコ大百科", "https://seiga.nicovideo.jp/tag/%E3%83%8B%E3%82%B3%E3%83%8B%E3%82%B3%E5%A4%A7%E7%99%BE%E7%A7%91"], - ["お絵カキコ", "https://seiga.nicovideo.jp/tag/%E3%81%8A%E7%B5%B5%E3%82%AB%E3%82%AD%E3%82%B3"], + ["アニメ", "https://seiga.nicovideo.jp/tag/アニメ"], + ["コジコジ", "https://seiga.nicovideo.jp/tag/コジコジ"], + ["さくらももこ", "https://seiga.nicovideo.jp/tag/さくらももこ"], + ["ドット絵", "https://seiga.nicovideo.jp/tag/ドット絵"], + ["ニコニコ大百科", "https://seiga.nicovideo.jp/tag/ニコニコ大百科"], + ["お絵カキコ", "https://seiga.nicovideo.jp/tag/お絵カキコ"], ] strategy_should_work( "http://lohas.nicoseiga.jp/o/910aecf08e542285862954017f8a33a8c32a8aec/1433298801/4937663", @@ -186,7 +186,7 @@ module Sources "https://seiga.nicovideo.jp/watch/mg302561", image_urls: image_urls, page_url: "https://seiga.nicovideo.jp/watch/mg302561", - tags: [["ロリ", "https://seiga.nicovideo.jp/manga/tag/%E3%83%AD%E3%83%AA"]], + tags: [["ロリ", "https://seiga.nicovideo.jp/manga/tag/ロリ"]], artist_name: "とろてい", other_names: ["とろてい"], tag_name: "nicoseiga_1848060" diff --git a/test/unit/sources/pixiv_test.rb b/test/unit/sources/pixiv_test.rb index df9973e5a..d3ebc55a3 100644 --- a/test/unit/sources/pixiv_test.rb +++ b/test/unit/sources/pixiv_test.rb @@ -94,10 +94,8 @@ module Sources should "get the tags" do pixiv_tags = @site.tags.map(&:first) - pixiv_links = @site.tags.map(&:last) assert_equal(%w[漫画 test], pixiv_tags) - assert_contains(pixiv_links, /search\.php/) end should "get the artist commentary" do