jaculabilis/danbooru
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #5043 from nonamethanks/fantia-support

Browse Source 
Add Fantia support
This commit is contained in:
evazion
2022-03-10 15:21:03 -06:00
committed by GitHub
parent 8bba1b0a54 a6549bc6fe
commit 5016d9ad26
10 changed files with 431 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.github/workflows/test.yaml vendored
View File

@@ -55,6 +55,7 @@ jobs:
DANBOORU_PAWOO_CLIENT_SECRET: ${{ secrets.DANBOORU_PAWOO_CLIENT_SECRET }}
DANBOORU_BARAAG_CLIENT_ID: ${{ secrets.DANBOORU_BARAAG_CLIENT_ID }}
DANBOORU_BARAAG_CLIENT_SECRET: ${{ secrets.DANBOORU_BARAAG_CLIENT_SECRET }}
DANBOORU_FANTIA_SESSION_ID: ${{ secrets.DANBOORU_FANTIA_SESSION_ID }}
DANBOORU_DISCORD_WEBHOOK_ID: ${{ secrets.DANBOORU_DISCORD_WEBHOOK_ID }}
DANBOORU_DISCORD_WEBHOOK_SECRET: ${{ secrets.DANBOORU_DISCORD_WEBHOOK_SECRET }}
DANBOORU_RAKISMET_KEY: ${{ secrets.DANBOORU_RAKISMET_KEY }}

2
app/logical/artist_finder.rb
View File

@@ -46,6 +46,8 @@ module ArtistFinder
"facebook.com", # https://www.facebook.com/LuutenantsLoot
"fantia.jp", # http://fantia.jp/no100
"fantia.jp/fanclubs", # https://fantia.jp/fanclubs/1711
"fantia.jp/posts", # https://fantia.jp/posts/20000
"fantia.jp/products", # https://fantia.jp/products/10000
"fav.me", # http://fav.me/d9y1njg
/blog-imgs-\d+(?:-origin)?\.fc2\.com/i,
%r{blog\.fc2\.com(/\w)+/?}i, # http://blog71.fc2.com/a/abk00/file/20080220194219.jpg

1
app/logical/source/url.rb
View File

@@ -23,6 +23,7 @@ module Source
Source::URL::ArtStation,
Source::URL::DeviantArt,
Source::URL::Fanbox,
Source::URL::Fantia,
Source::URL::Foundation,
Source::URL::HentaiFoundry,
Source::URL::Lofter,

90
app/logical/source/url/fantia.rb Normal file
View File

@@ -0,0 +1,90 @@
# frozen_string_literal: true
# Unparsed:
# https://fantia.jp/asanagi
class Source::URL::Fantia < Source::URL
attr_reader :full_image_url
def self.match?(url)
url.domain == "fantia.jp"
end
def parse
case [host, *path_segments]
# posts:
# https://c.fantia.jp/uploads/post/file/1070093/main_16faf0b1-58d8-4aac-9e86-b243063eaaf1.jpeg (sample)
# https://c.fantia.jp/uploads/post/file/1070093/16faf0b1-58d8-4aac-9e86-b243063eaaf1.jpeg
# https://cc.fantia.jp/uploads/post_content_photo/file/4563389/main_a9763427-3ccd-4e51-bcde-ff5e1ce0aa56.jpg?Key-Pair-Id=APKAIOCKYZS7WKBB6G7A&Policy=eyJTdGF0ZW1lbnQiOlt7IlJlc291cmNlIjoiaHR0cHM6Ly9jYy5mYW50aWEuanAvdXBsb2Fkcy9wb3N0X2NvbnRlbnRfcGhvdG8vZmlsZS80NTYzMzg5L21haW5fYTk3NjM0MjctM2NjZC00ZTUxLWJjZGUtZmY1ZTFjZTBhYTU2LmpwZyIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTY0NjkxMzk3OH19fV19&Signature=jyW5ankfO9uCHlKkozYU9RPpO3jzKTW2HuyXgS81i~cRgrXcI9orYU0IXuiit~0TznIyXbB7F~6Z790t7lX948PYAb9luYIREJC2u7pRMP3OBbsANbbFE0o4VR-6O3ZKbYQ4aG~ofVEZfiFVGoKoVtdJxj0bBNQV29eeFylGQATkFmywne1YMtJMqDirRBFMIatqNuunGsiWCQHqLYNHCeS4dZXlOnV8JQq0u1rPkeAQBmDCStFMA5ywjnWTfSZK7RN6RXKCAsMTXTl5X~I6EZASUPoGQy2vHUj5I-veffACg46jpvqTv6mLjQEw8JG~JLIOrZazKZR9O2kIoLNVGQ__
# from file download: https://cc.fantia.jp/uploads/post_content/file/1830956/cbcdfcbe_20220224_120_040_100.png?Key-Pair-Id=APKAIOCKYZS7WKBB6G7A&Policy=eyJTdGF0ZW1lbnQiOlt7IlJlc291cmNlIjoiaHR0cHM6Ly9jYy5mYW50aWEuanAvdXBsb2Fkcy9wb3N0X2NvbnRlbnQvZmlsZS8xODMwOTU2L2NiY2RmY2JlXzIwMjIwMjI0XzEyMF8wNDBfMTAwLnBuZyIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTY0NjkxNDU4Nn19fV19&Signature=d1nw8gs9vcshIAeEH4oESm9-7z6y4A7MfoIRRvtUtV9iqTNA8KM0ORuCI7NwEoYc1VHsxy9ByeuSBpNaJoknnc3TOmHFhVRcLn~OWpnWqiHEPpMcSEG7uGlorysjEPmYYRGHjE7LJYcWiiJxjZ~fSBbYzxxwsjroPm-fyGUtNhdJWEMNp52vHe5P9KErb7M8tP01toekGdOqO-pkWm1t9xm2Tp5P7RWcbtQPOixgG4UgOhE0f3LVwHGHYJV~-lB5RjrDbTTO3ezVi7I7ybZjjHotVUK5MbHHmXzC1NqI-VN3vHddTwTbTK9xEnPMR27NHSlho3-O18WcNs1YgKD48w__
#
# products:
# https://c.fantia.jp/uploads/product/image/249638/main_fd5aef8f-c217-49d0-83e8-289efb33dfc4.jpg
# https://c.fantia.jp/uploads/product_image/file/219407/main_bd7419c2-2450-4c53-a28a-90101fa466ab.jpg (sample)
# https://c.fantia.jp/uploads/product_image/file/219407/bd7419c2-2450-4c53-a28a-90101fa466ab.jpg
in _, "uploads", image_type, ("file" | "image"), image_id, /(?:\w+_)?([\w-]+\.\w+)/ => file
# post_id/product_id == image_id only for the first image in a post/product
case image_type
when "post"
@post_id = image_id
@full_image_url = "https://c.fantia.jp/uploads/post/file/#{@post_id}/#{$1}"
when "product"
@product_id = image_id
@full_image_url = "https://c.fantia.jp/uploads/product/image/#{@product_id}/#{$1}"
when "product_image"
@full_image_url = "https://c.fantia.jp/uploads/product_image/file/#{image_id}/#{$1}"
else
@full_image_url = original_url
end
# https://fantia.jp/posts/1143951/download/1830956
in _, "posts", post_id, "download", image_id
@post_id = post_id
@download_id = image_id
# https://fantia.jp/posts/1148334
in _, "posts", /\d+/ => post_id
@post_id = post_id
# https://fantia.jp/products/249638
in _, "products", /\d+/ => product_id
@product_id = product_id
# https://fantia.jp/fanclubs/64496
# https://fantia.jp/fanclubs/1654/posts
in _, "fanclubs", /\d+/ => fanclub_id, *rest
@fanclub_id = fanclub_id
else
end
end
def image_url?
@full_image_url.present?
end
def downloadable?
@download_id.present?
end
def page_url
if @post_id.present?
"https://fantia.jp/posts/#{@post_id}"
elsif @product_id.present?
"https://fantia.jp/products/#{@product_id}"
end
end
def work_id
@post_id || @product_id
end
def work_type
if @post_id.present?
"post"
elsif @product_id.present?
"product"
end
end
end

1
app/logical/sources/strategies.rb
View File

@@ -24,6 +24,7 @@ module Sources
Strategies::Foundation,
Strategies::Plurk,
Strategies::TwitPic,
Strategies::Fantia,
]
end

171
app/logical/sources/strategies/fantia.rb Normal file
View File

@@ -0,0 +1,171 @@
# frozen_string_literal: true
module Sources::Strategies
class Fantia < Base
def self.enabled?
Danbooru.config.fantia_session_id.present?
end
def match?
Source::URL::Fantia === parsed_url
end
def site_name
parsed_url.site_name
end
def image_urls
return [parsed_url.full_image_url] if parsed_url.image_url?
return [image_from_downloadable(parsed_url)] if parsed_url.downloadable?
images = images_for_post.presence || images_for_product.presence || []
full_images = images.compact.map do |image|
parsed = Source::URL.parse(image)
if parsed&.image_url?
parsed.full_image_url
elsif parsed&.downloadable?
image_from_downloadable(parsed)
else
image
end
end
full_images.compact.uniq
end
def image_from_downloadable(url)
resp = http.head(url)
return url if resp.status != 200
resp.uri.to_s
end
def images_for_post
return [] unless api_response.present?
images = [api_response.dig("post", "thumb_micro")]
api_response.dig("post", "post_contents").to_a.map do |content|
next if content["visible_status"] != "visible"
case content["category"]
when "photo_gallery"
content["post_content_photos"].to_a.map { |i| images << i.dig("url", "original") }
when "file"
images << image_from_downloadable("https://www.fantia.jp/#{content["download_uri"]}")
when "blog"
begin
sub_json = JSON.parse(content["comment"])
rescue Json::ParserError
sub_json = {}
end
sub_json["ops"].to_a.map { |js| images << js.dig("insert", "fantiaImage", "url") }
end
end
images
end
def images_for_product
html_response&.css(".product-gallery-item .img-fluid").to_a.map do |element|
element["src"] unless element["src"] =~ %r{/fallback/}
end.compact
end
def page_url
parsed_url.page_url || parsed_referer&.page_url
end
def tags
case work_type
when "post"
api_response&.dig("post", "tags").to_a.map do |tag|
[tag["name"], "https://fantia.jp/posts?tag=#{tag["name"]}"]
end
when "product"
html_response&.css(".product-category a").to_a.map do |element|
tag_name = element.text.delete_prefix("#")
[tag_name, "https://fantia.jp/products?product_category=##{tag_name}"]
end
else
[]
end
end
def other_names
case work_type
when "post"
[api_response&.dig("post", "fanclub", "creator_name")].compact
when "product"
[html_response&.at(".fanclub-name a")&.text].compact
end
end
def profile_url
case work_type
when "post"
fanclub_id = api_response&.dig("post", "fanclub", "id")
return unless fanclub_id.present?
"https://fantia.jp/fanclubs/#{fanclub_id}"
when "product"
href = html_response&.at(".fanclub-name a")&.[]("href")
return unless href.present?
URI.join("https://fantia.jp/", href).to_s
end
end
def artist_commentary_title
case work_type
when "post"
api_response&.dig("post", "title")
when "product"
html_response&.at(".product-title")&.text
end
end
def artist_commentary_desc
case work_type
when "post"
api_response&.dig("post", "comment")
when "product"
html_response&.at(".product-description")&.text
end
end
def dtext_artist_commentary_desc
DText.from_html(artist_commentary_desc)
end
def normalize_for_source
page_url
end
def work_type
parsed_url.work_type || parsed_referer&.work_type
end
def work_id
parsed_url.work_id || parsed_referer&.work_id
end
def api_response
return {} unless work_type == "post"
api_url = "https://fantia.jp/api/v1/posts/#{work_id}"
response = http.cache(1.minute).get(api_url)
return {} unless response.status == 200
JSON.parse(response)
rescue JSON::ParserError
{}
end
def html_response
return nil unless work_type == "product"
response = http.cache(1.minute).get("https://fantia.jp/products/#{work_id}")
return nil unless response.status == 200
response.parse
end
def http
Danbooru::Http.new.cookies(_session_id: Danbooru.config.fantia_session_id)
end
end
end

4
app/models/artist_url.rb
View File

@@ -109,8 +109,8 @@ class ArtistURL < ApplicationRecord
def priority
sites = %w[
Pixiv Twitter
ArtStation baraag.net BCY Deviant\ Art Hentai\ Foundry Foundation Nico\ Seiga Nijie pawoo.net Pixiv\ Fanbox Pixiv\ Sketch Plurk Tinami Tumblr
Ask.fm Booth.pm Facebook Fantia FC2 Gumroad Instagram Ko-fi Livedoor Lofter Mihuashi Mixi.jp Patreon Piapro.jp Picarto Privatter Sakura.ne.jp Stickam Skeb Twitch Weibo Youtube
ArtStation Baraag BCY Deviant\ Art Hentai\ Foundry Fantia Foundation Lofter Nico\ Seiga Nijie Pawoo Pixiv\ Fanbox Pixiv\ Sketch Plurk Tinami Tumblr Weibo
Ask.fm Booth.pm Facebook FC2 Gumroad Instagram Ko-fi Livedoor Mihuashi Mixi.jp Patreon Piapro.jp Picarto Privatter Sakura.ne.jp Stickam Skeb Twitch Youtube
Amazon Circle.ms DLSite Doujinshi.org Erogamescape Mangaupdates Melonbooks Toranoana Wikipedia
]

5
config/danbooru_default_config.rb
View File

@@ -311,6 +311,11 @@ module Danbooru
nil
end
# Your Fantia "_session_id" cookie. Login to Fantia then use the
# devtools to find the "_session_id" cookie.
def fantia_session_id
end
# A list of tags that should be removed when a post is replaced. Regexes allowed.
def post_replacement_tag_removals
%w[replaceme .*_sample resized upscaled downscaled md5_mismatch

4
test/functional/uploads_controller_test.rb
View File

@@ -352,6 +352,10 @@ class UploadsControllerTest < ActionDispatch::IntegrationTest
should_upload_successfully("https://www.plurk.com/p/om6zv4")
should_upload_successfully("https://gengar563.lofter.com/post/1e82da8c_1c98dae1b")
should_upload_successfully("https://c.fantia.jp/uploads/post/file/1070093/16faf0b1-58d8-4aac-9e86-b243063eaaf1.jpeg")
should_upload_successfully("https://fantia.jp/posts/1132267")
should_upload_successfully("https://fantia.jp/products/249638")
end
end
end

154
test/unit/sources/fantia_test.rb Normal file
View File

@@ -0,0 +1,154 @@
require "test_helper"
module Sources
class FantiaTest < ActiveSupport::TestCase
def setup
super
skip "session_id cookie not set" unless Danbooru.config.fantia_session_id.present?
end
context "A c.fantia.jp/uploads/post/file/ url" do
should "work" do
url = "https://c.fantia.jp/uploads/post/file/1070093/16faf0b1-58d8-4aac-9e86-b243063eaaf1.jpeg"
source = Sources::Strategies.find(url)
assert_equal([url], source.image_urls)
assert_equal("豆ラッコ", source.other_names.first)
assert_equal("https://fantia.jp/fanclubs/27264", source.profile_url)
assert_equal("https://fantia.jp/posts/1070093", source.page_url)
assert_equal([], source.tags)
assert_equal("大きく育った心春ちゃん1", source.artist_commentary_title)
assert_equal("色々やります", source.artist_commentary_desc)
assert_downloaded(3_692_131, url)
assert_nothing_raised { source.to_h }
end
end
context "A c.fantia.jp/uploads/post_content_photo/ url" do
should "work" do
url = "https://cc.fantia.jp/uploads/post_content_photo/file/7087182/main_7f04ff3c-1f08-450f-bd98-796c290fc2d1.jpg?Key-Pair-Id=APKAIOCKYZS7WKBB6G7A&Policy=eyJTdGF0ZW1lbnQiOlt7IlJlc291cmNlIjoiaHR0cHM6Ly9jYy5mYW50aWEuanAvdXBsb2Fkcy9wb3N0X2NvbnRlbnRfcGhvdG8vZmlsZS83MDg3MTgyL21haW5fN2YwNGZmM2MtMWYwOC00NTBmLWJkOTgtNzk2YzI5MGZjMmQxLmpwZyIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTY0NjkyODAzN319fV19&Signature=wl2Nr9i1O5R5dDc7FB-8CKtRvyZPS6ZEFXn7Q74rBh9R2PZkpKuQUDDsJubgkYaHrqHEapcOdZczzZaM5kbRLXGPOnVFUE7vHKnXZTO~Z1-Z8Cqt823NKCR-AXBjYPhQoGP0pITLYkjhofy0FXg6RYJ0oNJPdKkdjcnwzr-nZfyaFgkrrQ5~LRDhW5HOgSNfvhJleMRLRgLtXbbgNnVwHmpFWNkFSwwmDcUTXTh4hrhQrOJ~xJmiQesSP1wPAE5ZZSBGsbUstOa5Y1nVu540wItR4VWLm-jjuMk9OIr-Nvxg0ocoP9WU13WrRbeMeL5X0xhxBYSxgVIKXko2BqMf5w__"
ref = "https://fantia.jp/posts/1132267"
source = Sources::Strategies.find(url, ref)
assert_equal("稲光伸二", source.other_names.first)
assert_equal("https://fantia.jp/fanclubs/1096", source.profile_url)
assert_equal(ref, source.page_url)
assert_equal(["オリジナル", "漫画"], source.tags.map(&:first))
assert_equal("黒い歴史(5)", source.artist_commentary_title)
assert_match(/^この回から絵はほとんど今と/, source.artist_commentary_desc)
assert_nothing_raised { source.to_h }
end
end
context "A c.fantia.jp/uploads/product/image/ url" do
should "work" do
url = "https://c.fantia.jp/uploads/product/image/249638/fd5aef8f-c217-49d0-83e8-289efb33dfc4.jpg"
source = Sources::Strategies.find(url)
tags = ["イラスト集", "CG集", "PNG", "オリジナル", "宮前詩帆", "春川朱璃愛", "夏川黒羽", "ASMR", "音声", "原神", "シニョーラ"]
assert_equal([url], source.image_urls)
assert_match(/電波暗室/, source.other_names.first)
assert_equal("https://fantia.jp/fanclubs/7", source.profile_url)
assert_equal("https://fantia.jp/products/249638", source.page_url)
assert_equal(tags, source.tags.map(&:first))
assert_equal("2021年9月更新分[PNG] - September 2021", source.artist_commentary_title)
assert_match(/This is the same as the image data updated in September 2021/, source.artist_commentary_desc)
assert_downloaded(288_801, url)
assert_nothing_raised { source.to_h }
end
end
context "A c.fantia.jp/uploads/product_image/file sample url" do
should "work" do
url = "https://c.fantia.jp/uploads/product_image/file/219407/main_bd7419c2-2450-4c53-a28a-90101fa466ab.jpg"
ref = "https://fantia.jp/products/249638"
source = Sources::Strategies.find(url, ref)
assert_equal(["https://c.fantia.jp/uploads/product_image/file/219407/bd7419c2-2450-4c53-a28a-90101fa466ab.jpg"], source.image_urls)
assert_equal("https://fantia.jp/fanclubs/7", source.profile_url)
assert_equal("https://fantia.jp/products/249638", source.page_url)
assert_equal("2021年9月更新分[PNG] - September 2021", source.artist_commentary_title)
assert_downloaded(613_103, url)
assert_nothing_raised { source.to_h }
end
end
context "A fantia.jp/posts/$id/download url" do
should "work" do
url = "https://fantia.jp/posts/1143951/download/1830956"
source = Sources::Strategies.find(url)
assert_match(%r{1830956/cbcdfcbe_20220224_120_040_100.png}, source.image_url)
assert_equal("松永紅葉", source.other_names.first)
assert_equal("https://fantia.jp/fanclubs/322", source.profile_url)
assert_equal("https://fantia.jp/posts/1143951", source.page_url)
assert_equal([], source.tags)
assert_equal("今日の一枚3186 (1:20+0:40+1:00)", source.artist_commentary_title)
assert_equal("今日の一枚3186 (1:20+0:40+1:00)", source.artist_commentary_desc)
assert_downloaded(14_371_816, url)
assert_nothing_raised { source.to_h }
end
end
context "A fantia.jp/posts/$id url" do
should "work" do
url = "https://fantia.jp/posts/1143951"
source = Sources::Strategies.find(url)
assert_equal("https://c.fantia.jp/uploads/post/file/1143951/47491020-a6c6-47db-b09e-815b0530c0bc.png", source.image_url)
assert_match(%r{1830956/cbcdfcbe_20220224_120_040_100.png}, source.image_urls.second)
assert_equal("松永紅葉", source.other_names.first)
assert_equal("https://fantia.jp/fanclubs/322", source.profile_url)
assert_equal("https://fantia.jp/posts/1143951", source.page_url)
assert_equal([], source.tags)
assert_equal("今日の一枚3186 (1:20+0:40+1:00)", source.artist_commentary_title)
assert_equal("今日の一枚3186 (1:20+0:40+1:00)", source.artist_commentary_desc)
assert_downloaded(1_157_953, url)
assert_nothing_raised { source.to_h }
end
end
context "A fantia.jp/products/$id url" do
should "work" do
url = "https://fantia.jp/products/249638"
source = Sources::Strategies.find(url)
image_urls = %w[
https://c.fantia.jp/uploads/product/image/249638/fd5aef8f-c217-49d0-83e8-289efb33dfc4.jpg
https://c.fantia.jp/uploads/product_image/file/219406/c73bd7f9-a13a-48f7-9ac7-35309faa88c3.jpg
https://c.fantia.jp/uploads/product_image/file/219407/bd7419c2-2450-4c53-a28a-90101fa466ab.jpg
https://c.fantia.jp/uploads/product_image/file/219408/50aae3fd-11c5-4679-a584-e4276617d4b9.jpg
https://c.fantia.jp/uploads/product_image/file/219409/1e777b93-2672-4a5d-8076-91b3766d3664.jpg
]
assert_equal(image_urls, source.image_urls)
assert_equal("https://fantia.jp/fanclubs/7", source.profile_url)
assert_equal("https://fantia.jp/products/249638", source.page_url)
assert_downloaded(288_801, url)
assert_nothing_raised { source.to_h }
end
end
context "A product url with no images" do
should "not get placeholder images" do
source = Sources::Strategies.find("https://fantia.jp/products/10000")
assert_equal([], source.image_urls)
assert_nothing_raised { source.to_h }
end
end
context "A deleted or non-existing fantia url" do
should "work" do
url1 = "https://fantia.jp/posts/12345678901234567890"
url2 = "https://fantia.jp/products/12345678901234567890"
source1 = Sources::Strategies.find(url1)
source2 = Sources::Strategies.find(url2)
assert_equal([], source1.image_urls)
assert_equal([], source2.image_urls)
assert_nothing_raised { source1.to_h }
assert_nothing_raised { source2.to_h }
end
end
end
end