diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 69d4d8eaa..fc6a4f15f 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -55,6 +55,7 @@ jobs: DANBOORU_PAWOO_CLIENT_SECRET: ${{ secrets.DANBOORU_PAWOO_CLIENT_SECRET }} DANBOORU_BARAAG_CLIENT_ID: ${{ secrets.DANBOORU_BARAAG_CLIENT_ID }} DANBOORU_BARAAG_CLIENT_SECRET: ${{ secrets.DANBOORU_BARAAG_CLIENT_SECRET }} + DANBOORU_FANTIA_SESSION_ID: ${{ secrets.DANBOORU_FANTIA_SESSION_ID }} DANBOORU_DISCORD_WEBHOOK_ID: ${{ secrets.DANBOORU_DISCORD_WEBHOOK_ID }} DANBOORU_DISCORD_WEBHOOK_SECRET: ${{ secrets.DANBOORU_DISCORD_WEBHOOK_SECRET }} DANBOORU_RAKISMET_KEY: ${{ secrets.DANBOORU_RAKISMET_KEY }} diff --git a/app/logical/artist_finder.rb b/app/logical/artist_finder.rb index 74bf0e650..3e4db864a 100644 --- a/app/logical/artist_finder.rb +++ b/app/logical/artist_finder.rb @@ -46,6 +46,8 @@ module ArtistFinder "facebook.com", # https://www.facebook.com/LuutenantsLoot "fantia.jp", # http://fantia.jp/no100 "fantia.jp/fanclubs", # https://fantia.jp/fanclubs/1711 + "fantia.jp/posts", # https://fantia.jp/posts/20000 + "fantia.jp/products", # https://fantia.jp/products/10000 "fav.me", # http://fav.me/d9y1njg /blog-imgs-\d+(?:-origin)?\.fc2\.com/i, %r{blog\.fc2\.com(/\w)+/?}i, # http://blog71.fc2.com/a/abk00/file/20080220194219.jpg diff --git a/app/logical/source/url.rb b/app/logical/source/url.rb index 7cb054f08..a054dbe53 100644 --- a/app/logical/source/url.rb +++ b/app/logical/source/url.rb @@ -23,6 +23,7 @@ module Source Source::URL::ArtStation, Source::URL::DeviantArt, Source::URL::Fanbox, + Source::URL::Fantia, Source::URL::Foundation, Source::URL::HentaiFoundry, Source::URL::Lofter, diff --git a/app/logical/source/url/fantia.rb b/app/logical/source/url/fantia.rb new file mode 100644 index 000000000..1dc1540be --- /dev/null +++ b/app/logical/source/url/fantia.rb @@ -0,0 +1,90 @@ +# frozen_string_literal: true + +# Unparsed: +# https://fantia.jp/asanagi + +class Source::URL::Fantia < Source::URL + attr_reader :full_image_url + + def self.match?(url) + url.domain == "fantia.jp" + end + + def parse + case [host, *path_segments] + + # posts: + # https://c.fantia.jp/uploads/post/file/1070093/main_16faf0b1-58d8-4aac-9e86-b243063eaaf1.jpeg (sample) + # https://c.fantia.jp/uploads/post/file/1070093/16faf0b1-58d8-4aac-9e86-b243063eaaf1.jpeg + # https://cc.fantia.jp/uploads/post_content_photo/file/4563389/main_a9763427-3ccd-4e51-bcde-ff5e1ce0aa56.jpg?Key-Pair-Id=APKAIOCKYZS7WKBB6G7A&Policy=eyJTdGF0ZW1lbnQiOlt7IlJlc291cmNlIjoiaHR0cHM6Ly9jYy5mYW50aWEuanAvdXBsb2Fkcy9wb3N0X2NvbnRlbnRfcGhvdG8vZmlsZS80NTYzMzg5L21haW5fYTk3NjM0MjctM2NjZC00ZTUxLWJjZGUtZmY1ZTFjZTBhYTU2LmpwZyIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTY0NjkxMzk3OH19fV19&Signature=jyW5ankfO9uCHlKkozYU9RPpO3jzKTW2HuyXgS81i~cRgrXcI9orYU0IXuiit~0TznIyXbB7F~6Z790t7lX948PYAb9luYIREJC2u7pRMP3OBbsANbbFE0o4VR-6O3ZKbYQ4aG~ofVEZfiFVGoKoVtdJxj0bBNQV29eeFylGQATkFmywne1YMtJMqDirRBFMIatqNuunGsiWCQHqLYNHCeS4dZXlOnV8JQq0u1rPkeAQBmDCStFMA5ywjnWTfSZK7RN6RXKCAsMTXTl5X~I6EZASUPoGQy2vHUj5I-veffACg46jpvqTv6mLjQEw8JG~JLIOrZazKZR9O2kIoLNVGQ__ + # from file download: https://cc.fantia.jp/uploads/post_content/file/1830956/cbcdfcbe_20220224_120_040_100.png?Key-Pair-Id=APKAIOCKYZS7WKBB6G7A&Policy=eyJTdGF0ZW1lbnQiOlt7IlJlc291cmNlIjoiaHR0cHM6Ly9jYy5mYW50aWEuanAvdXBsb2Fkcy9wb3N0X2NvbnRlbnQvZmlsZS8xODMwOTU2L2NiY2RmY2JlXzIwMjIwMjI0XzEyMF8wNDBfMTAwLnBuZyIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTY0NjkxNDU4Nn19fV19&Signature=d1nw8gs9vcshIAeEH4oESm9-7z6y4A7MfoIRRvtUtV9iqTNA8KM0ORuCI7NwEoYc1VHsxy9ByeuSBpNaJoknnc3TOmHFhVRcLn~OWpnWqiHEPpMcSEG7uGlorysjEPmYYRGHjE7LJYcWiiJxjZ~fSBbYzxxwsjroPm-fyGUtNhdJWEMNp52vHe5P9KErb7M8tP01toekGdOqO-pkWm1t9xm2Tp5P7RWcbtQPOixgG4UgOhE0f3LVwHGHYJV~-lB5RjrDbTTO3ezVi7I7ybZjjHotVUK5MbHHmXzC1NqI-VN3vHddTwTbTK9xEnPMR27NHSlho3-O18WcNs1YgKD48w__ + # + # products: + # https://c.fantia.jp/uploads/product/image/249638/main_fd5aef8f-c217-49d0-83e8-289efb33dfc4.jpg + # https://c.fantia.jp/uploads/product_image/file/219407/main_bd7419c2-2450-4c53-a28a-90101fa466ab.jpg (sample) + # https://c.fantia.jp/uploads/product_image/file/219407/bd7419c2-2450-4c53-a28a-90101fa466ab.jpg + in _, "uploads", image_type, ("file" | "image"), image_id, /(?:\w+_)?([\w-]+\.\w+)/ => file + # post_id/product_id == image_id only for the first image in a post/product + case image_type + when "post" + @post_id = image_id + @full_image_url = "https://c.fantia.jp/uploads/post/file/#{@post_id}/#{$1}" + when "product" + @product_id = image_id + @full_image_url = "https://c.fantia.jp/uploads/product/image/#{@product_id}/#{$1}" + when "product_image" + @full_image_url = "https://c.fantia.jp/uploads/product_image/file/#{image_id}/#{$1}" + else + @full_image_url = original_url + end + + # https://fantia.jp/posts/1143951/download/1830956 + in _, "posts", post_id, "download", image_id + @post_id = post_id + @download_id = image_id + + # https://fantia.jp/posts/1148334 + in _, "posts", /\d+/ => post_id + @post_id = post_id + + # https://fantia.jp/products/249638 + in _, "products", /\d+/ => product_id + @product_id = product_id + + # https://fantia.jp/fanclubs/64496 + # https://fantia.jp/fanclubs/1654/posts + in _, "fanclubs", /\d+/ => fanclub_id, *rest + @fanclub_id = fanclub_id + + else + end + end + + def image_url? + @full_image_url.present? + end + + def downloadable? + @download_id.present? + end + + def page_url + if @post_id.present? + "https://fantia.jp/posts/#{@post_id}" + elsif @product_id.present? + "https://fantia.jp/products/#{@product_id}" + end + end + + def work_id + @post_id || @product_id + end + + def work_type + if @post_id.present? + "post" + elsif @product_id.present? + "product" + end + end +end diff --git a/app/logical/sources/strategies.rb b/app/logical/sources/strategies.rb index 2f949c1fc..3ced69356 100644 --- a/app/logical/sources/strategies.rb +++ b/app/logical/sources/strategies.rb @@ -24,6 +24,7 @@ module Sources Strategies::Foundation, Strategies::Plurk, Strategies::TwitPic, + Strategies::Fantia, ] end diff --git a/app/logical/sources/strategies/fantia.rb b/app/logical/sources/strategies/fantia.rb new file mode 100644 index 000000000..824ce9ae2 --- /dev/null +++ b/app/logical/sources/strategies/fantia.rb @@ -0,0 +1,171 @@ +# frozen_string_literal: true + +module Sources::Strategies + class Fantia < Base + def self.enabled? + Danbooru.config.fantia_session_id.present? + end + + def match? + Source::URL::Fantia === parsed_url + end + + def site_name + parsed_url.site_name + end + + def image_urls + return [parsed_url.full_image_url] if parsed_url.image_url? + return [image_from_downloadable(parsed_url)] if parsed_url.downloadable? + + images = images_for_post.presence || images_for_product.presence || [] + + full_images = images.compact.map do |image| + parsed = Source::URL.parse(image) + if parsed&.image_url? + parsed.full_image_url + elsif parsed&.downloadable? + image_from_downloadable(parsed) + else + image + end + end + full_images.compact.uniq + end + + def image_from_downloadable(url) + resp = http.head(url) + return url if resp.status != 200 + resp.uri.to_s + end + + def images_for_post + return [] unless api_response.present? + images = [api_response.dig("post", "thumb_micro")] + api_response.dig("post", "post_contents").to_a.map do |content| + next if content["visible_status"] != "visible" + + case content["category"] + when "photo_gallery" + content["post_content_photos"].to_a.map { |i| images << i.dig("url", "original") } + when "file" + images << image_from_downloadable("https://www.fantia.jp/#{content["download_uri"]}") + when "blog" + begin + sub_json = JSON.parse(content["comment"]) + rescue Json::ParserError + sub_json = {} + end + sub_json["ops"].to_a.map { |js| images << js.dig("insert", "fantiaImage", "url") } + end + end + images + end + + def images_for_product + html_response&.css(".product-gallery-item .img-fluid").to_a.map do |element| + element["src"] unless element["src"] =~ %r{/fallback/} + end.compact + end + + def page_url + parsed_url.page_url || parsed_referer&.page_url + end + + def tags + case work_type + when "post" + api_response&.dig("post", "tags").to_a.map do |tag| + [tag["name"], "https://fantia.jp/posts?tag=#{tag["name"]}"] + end + when "product" + html_response&.css(".product-category a").to_a.map do |element| + tag_name = element.text.delete_prefix("#") + [tag_name, "https://fantia.jp/products?product_category=##{tag_name}"] + end + else + [] + end + end + + def other_names + case work_type + when "post" + [api_response&.dig("post", "fanclub", "creator_name")].compact + when "product" + [html_response&.at(".fanclub-name a")&.text].compact + end + end + + def profile_url + case work_type + when "post" + fanclub_id = api_response&.dig("post", "fanclub", "id") + return unless fanclub_id.present? + "https://fantia.jp/fanclubs/#{fanclub_id}" + when "product" + href = html_response&.at(".fanclub-name a")&.[]("href") + return unless href.present? + URI.join("https://fantia.jp/", href).to_s + end + end + + def artist_commentary_title + case work_type + when "post" + api_response&.dig("post", "title") + when "product" + html_response&.at(".product-title")&.text + end + end + + def artist_commentary_desc + case work_type + when "post" + api_response&.dig("post", "comment") + when "product" + html_response&.at(".product-description")&.text + end + end + + def dtext_artist_commentary_desc + DText.from_html(artist_commentary_desc) + end + + def normalize_for_source + page_url + end + + def work_type + parsed_url.work_type || parsed_referer&.work_type + end + + def work_id + parsed_url.work_id || parsed_referer&.work_id + end + + def api_response + return {} unless work_type == "post" + api_url = "https://fantia.jp/api/v1/posts/#{work_id}" + + response = http.cache(1.minute).get(api_url) + return {} unless response.status == 200 + + JSON.parse(response) + rescue JSON::ParserError + {} + end + + def html_response + return nil unless work_type == "product" + response = http.cache(1.minute).get("https://fantia.jp/products/#{work_id}") + + return nil unless response.status == 200 + response.parse + end + + def http + Danbooru::Http.new.cookies(_session_id: Danbooru.config.fantia_session_id) + end + end +end diff --git a/app/models/artist_url.rb b/app/models/artist_url.rb index a28e8acb7..6a14df295 100644 --- a/app/models/artist_url.rb +++ b/app/models/artist_url.rb @@ -109,8 +109,8 @@ class ArtistURL < ApplicationRecord def priority sites = %w[ Pixiv Twitter - ArtStation baraag.net BCY Deviant\ Art Hentai\ Foundry Foundation Nico\ Seiga Nijie pawoo.net Pixiv\ Fanbox Pixiv\ Sketch Plurk Tinami Tumblr - Ask.fm Booth.pm Facebook Fantia FC2 Gumroad Instagram Ko-fi Livedoor Lofter Mihuashi Mixi.jp Patreon Piapro.jp Picarto Privatter Sakura.ne.jp Stickam Skeb Twitch Weibo Youtube + ArtStation Baraag BCY Deviant\ Art Hentai\ Foundry Fantia Foundation Lofter Nico\ Seiga Nijie Pawoo Pixiv\ Fanbox Pixiv\ Sketch Plurk Tinami Tumblr Weibo + Ask.fm Booth.pm Facebook FC2 Gumroad Instagram Ko-fi Livedoor Mihuashi Mixi.jp Patreon Piapro.jp Picarto Privatter Sakura.ne.jp Stickam Skeb Twitch Youtube Amazon Circle.ms DLSite Doujinshi.org Erogamescape Mangaupdates Melonbooks Toranoana Wikipedia ] diff --git a/config/danbooru_default_config.rb b/config/danbooru_default_config.rb index d10141f32..3e8388f90 100644 --- a/config/danbooru_default_config.rb +++ b/config/danbooru_default_config.rb @@ -311,6 +311,11 @@ module Danbooru nil end + # Your Fantia "_session_id" cookie. Login to Fantia then use the + # devtools to find the "_session_id" cookie. + def fantia_session_id + end + # A list of tags that should be removed when a post is replaced. Regexes allowed. def post_replacement_tag_removals %w[replaceme .*_sample resized upscaled downscaled md5_mismatch diff --git a/test/functional/uploads_controller_test.rb b/test/functional/uploads_controller_test.rb index 01900ef92..f0c14ed47 100644 --- a/test/functional/uploads_controller_test.rb +++ b/test/functional/uploads_controller_test.rb @@ -352,6 +352,10 @@ class UploadsControllerTest < ActionDispatch::IntegrationTest should_upload_successfully("https://www.plurk.com/p/om6zv4") should_upload_successfully("https://gengar563.lofter.com/post/1e82da8c_1c98dae1b") + + should_upload_successfully("https://c.fantia.jp/uploads/post/file/1070093/16faf0b1-58d8-4aac-9e86-b243063eaaf1.jpeg") + should_upload_successfully("https://fantia.jp/posts/1132267") + should_upload_successfully("https://fantia.jp/products/249638") end end end diff --git a/test/unit/sources/fantia_test.rb b/test/unit/sources/fantia_test.rb new file mode 100644 index 000000000..c9ed40d8a --- /dev/null +++ b/test/unit/sources/fantia_test.rb @@ -0,0 +1,154 @@ +require "test_helper" + +module Sources + class FantiaTest < ActiveSupport::TestCase + def setup + super + skip "session_id cookie not set" unless Danbooru.config.fantia_session_id.present? + end + + context "A c.fantia.jp/uploads/post/file/ url" do + should "work" do + url = "https://c.fantia.jp/uploads/post/file/1070093/16faf0b1-58d8-4aac-9e86-b243063eaaf1.jpeg" + source = Sources::Strategies.find(url) + + assert_equal([url], source.image_urls) + assert_equal("豆ラッコ", source.other_names.first) + assert_equal("https://fantia.jp/fanclubs/27264", source.profile_url) + assert_equal("https://fantia.jp/posts/1070093", source.page_url) + assert_equal([], source.tags) + assert_equal("大きく育った心春ちゃん1", source.artist_commentary_title) + assert_equal("色々やります", source.artist_commentary_desc) + assert_downloaded(3_692_131, url) + assert_nothing_raised { source.to_h } + end + end + + context "A c.fantia.jp/uploads/post_content_photo/ url" do + should "work" do + url = "https://cc.fantia.jp/uploads/post_content_photo/file/7087182/main_7f04ff3c-1f08-450f-bd98-796c290fc2d1.jpg?Key-Pair-Id=APKAIOCKYZS7WKBB6G7A&Policy=eyJTdGF0ZW1lbnQiOlt7IlJlc291cmNlIjoiaHR0cHM6Ly9jYy5mYW50aWEuanAvdXBsb2Fkcy9wb3N0X2NvbnRlbnRfcGhvdG8vZmlsZS83MDg3MTgyL21haW5fN2YwNGZmM2MtMWYwOC00NTBmLWJkOTgtNzk2YzI5MGZjMmQxLmpwZyIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTY0NjkyODAzN319fV19&Signature=wl2Nr9i1O5R5dDc7FB-8CKtRvyZPS6ZEFXn7Q74rBh9R2PZkpKuQUDDsJubgkYaHrqHEapcOdZczzZaM5kbRLXGPOnVFUE7vHKnXZTO~Z1-Z8Cqt823NKCR-AXBjYPhQoGP0pITLYkjhofy0FXg6RYJ0oNJPdKkdjcnwzr-nZfyaFgkrrQ5~LRDhW5HOgSNfvhJleMRLRgLtXbbgNnVwHmpFWNkFSwwmDcUTXTh4hrhQrOJ~xJmiQesSP1wPAE5ZZSBGsbUstOa5Y1nVu540wItR4VWLm-jjuMk9OIr-Nvxg0ocoP9WU13WrRbeMeL5X0xhxBYSxgVIKXko2BqMf5w__" + ref = "https://fantia.jp/posts/1132267" + source = Sources::Strategies.find(url, ref) + + assert_equal("稲光伸二", source.other_names.first) + assert_equal("https://fantia.jp/fanclubs/1096", source.profile_url) + assert_equal(ref, source.page_url) + assert_equal(["オリジナル", "漫画"], source.tags.map(&:first)) + assert_equal("黒い歴史(5)", source.artist_commentary_title) + assert_match(/^この回から絵はほとんど今と/, source.artist_commentary_desc) + assert_nothing_raised { source.to_h } + end + end + + context "A c.fantia.jp/uploads/product/image/ url" do + should "work" do + url = "https://c.fantia.jp/uploads/product/image/249638/fd5aef8f-c217-49d0-83e8-289efb33dfc4.jpg" + source = Sources::Strategies.find(url) + tags = ["イラスト集", "CG集", "PNG", "オリジナル", "宮前詩帆", "春川朱璃愛", "夏川黒羽", "ASMR", "音声", "原神", "シニョーラ"] + + assert_equal([url], source.image_urls) + assert_match(/電波暗室/, source.other_names.first) + assert_equal("https://fantia.jp/fanclubs/7", source.profile_url) + assert_equal("https://fantia.jp/products/249638", source.page_url) + assert_equal(tags, source.tags.map(&:first)) + assert_equal("2021年9月更新分[PNG] - September 2021", source.artist_commentary_title) + assert_match(/This is the same as the image data updated in September 2021/, source.artist_commentary_desc) + assert_downloaded(288_801, url) + assert_nothing_raised { source.to_h } + end + end + + context "A c.fantia.jp/uploads/product_image/file sample url" do + should "work" do + url = "https://c.fantia.jp/uploads/product_image/file/219407/main_bd7419c2-2450-4c53-a28a-90101fa466ab.jpg" + ref = "https://fantia.jp/products/249638" + source = Sources::Strategies.find(url, ref) + + assert_equal(["https://c.fantia.jp/uploads/product_image/file/219407/bd7419c2-2450-4c53-a28a-90101fa466ab.jpg"], source.image_urls) + assert_equal("https://fantia.jp/fanclubs/7", source.profile_url) + assert_equal("https://fantia.jp/products/249638", source.page_url) + assert_equal("2021年9月更新分[PNG] - September 2021", source.artist_commentary_title) + assert_downloaded(613_103, url) + assert_nothing_raised { source.to_h } + end + end + + context "A fantia.jp/posts/$id/download url" do + should "work" do + url = "https://fantia.jp/posts/1143951/download/1830956" + source = Sources::Strategies.find(url) + + assert_match(%r{1830956/cbcdfcbe_20220224_120_040_100.png}, source.image_url) + assert_equal("松永紅葉", source.other_names.first) + assert_equal("https://fantia.jp/fanclubs/322", source.profile_url) + assert_equal("https://fantia.jp/posts/1143951", source.page_url) + assert_equal([], source.tags) + assert_equal("今日の一枚3186 (1:20+0:40+1:00)", source.artist_commentary_title) + assert_equal("今日の一枚3186 (1:20+0:40+1:00)", source.artist_commentary_desc) + assert_downloaded(14_371_816, url) + assert_nothing_raised { source.to_h } + end + end + + context "A fantia.jp/posts/$id url" do + should "work" do + url = "https://fantia.jp/posts/1143951" + source = Sources::Strategies.find(url) + + assert_equal("https://c.fantia.jp/uploads/post/file/1143951/47491020-a6c6-47db-b09e-815b0530c0bc.png", source.image_url) + assert_match(%r{1830956/cbcdfcbe_20220224_120_040_100.png}, source.image_urls.second) + assert_equal("松永紅葉", source.other_names.first) + assert_equal("https://fantia.jp/fanclubs/322", source.profile_url) + assert_equal("https://fantia.jp/posts/1143951", source.page_url) + assert_equal([], source.tags) + assert_equal("今日の一枚3186 (1:20+0:40+1:00)", source.artist_commentary_title) + assert_equal("今日の一枚3186 (1:20+0:40+1:00)", source.artist_commentary_desc) + assert_downloaded(1_157_953, url) + assert_nothing_raised { source.to_h } + end + end + + context "A fantia.jp/products/$id url" do + should "work" do + url = "https://fantia.jp/products/249638" + source = Sources::Strategies.find(url) + image_urls = %w[ + https://c.fantia.jp/uploads/product/image/249638/fd5aef8f-c217-49d0-83e8-289efb33dfc4.jpg + https://c.fantia.jp/uploads/product_image/file/219406/c73bd7f9-a13a-48f7-9ac7-35309faa88c3.jpg + https://c.fantia.jp/uploads/product_image/file/219407/bd7419c2-2450-4c53-a28a-90101fa466ab.jpg + https://c.fantia.jp/uploads/product_image/file/219408/50aae3fd-11c5-4679-a584-e4276617d4b9.jpg + https://c.fantia.jp/uploads/product_image/file/219409/1e777b93-2672-4a5d-8076-91b3766d3664.jpg + ] + + assert_equal(image_urls, source.image_urls) + assert_equal("https://fantia.jp/fanclubs/7", source.profile_url) + assert_equal("https://fantia.jp/products/249638", source.page_url) + assert_downloaded(288_801, url) + assert_nothing_raised { source.to_h } + end + end + + context "A product url with no images" do + should "not get placeholder images" do + source = Sources::Strategies.find("https://fantia.jp/products/10000") + assert_equal([], source.image_urls) + assert_nothing_raised { source.to_h } + end + end + + context "A deleted or non-existing fantia url" do + should "work" do + url1 = "https://fantia.jp/posts/12345678901234567890" + url2 = "https://fantia.jp/products/12345678901234567890" + + source1 = Sources::Strategies.find(url1) + source2 = Sources::Strategies.find(url2) + + assert_equal([], source1.image_urls) + assert_equal([], source2.image_urls) + assert_nothing_raised { source1.to_h } + assert_nothing_raised { source2.to_h } + end + end + end +end