From 8edd5dd81094c73bc32c463a6b7a5e3e3defbf23 Mon Sep 17 00:00:00 2001 From: nonamethanks Date: Wed, 27 Apr 2022 03:42:10 +0200 Subject: [PATCH] Add furaffinity support --- .github/workflows/test.yaml | 2 + app/logical/source/extractor.rb | 1 + app/logical/source/extractor/furaffinity.rb | 73 +++++++++++++++++++++ app/logical/source/url.rb | 1 + app/logical/source/url/furaffinity.rb | 45 +++++++++++++ app/models/artist_url.rb | 2 +- config/danbooru_default_config.rb | 11 ++++ test/unit/sources/furaffinity_test.rb | 45 +++++++++++++ 8 files changed, 179 insertions(+), 1 deletion(-) create mode 100644 app/logical/source/extractor/furaffinity.rb create mode 100644 app/logical/source/url/furaffinity.rb create mode 100644 test/unit/sources/furaffinity_test.rb diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 82ba30b8c..6a059cd47 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -56,6 +56,8 @@ jobs: DANBOORU_BARAAG_CLIENT_ID: ${{ secrets.DANBOORU_BARAAG_CLIENT_ID }} DANBOORU_BARAAG_CLIENT_SECRET: ${{ secrets.DANBOORU_BARAAG_CLIENT_SECRET }} DANBOORU_FANTIA_SESSION_ID: ${{ secrets.DANBOORU_FANTIA_SESSION_ID }} + DANBOORU_FURAFFINITY_COOKIE_A: ${{ secrets.DANBOORU_FURAFFINITY_COOKIE_A }} + DANBOORU_FURAFFINITY_COOKIE_B: ${{ secrets.DANBOORU_FURAFFINITY_COOKIE_B }} DANBOORU_TINAMI_SESSION_ID: ${{ secrets.DANBOORU_TINAMI_SESSION_ID }} DANBOORU_DISCORD_WEBHOOK_ID: ${{ secrets.DANBOORU_DISCORD_WEBHOOK_ID }} DANBOORU_DISCORD_WEBHOOK_SECRET: ${{ secrets.DANBOORU_DISCORD_WEBHOOK_SECRET }} diff --git a/app/logical/source/extractor.rb b/app/logical/source/extractor.rb index b7b1237e6..e79daa769 100644 --- a/app/logical/source/extractor.rb +++ b/app/logical/source/extractor.rb @@ -53,6 +53,7 @@ module Source Source::Extractor::Fantia, Source::Extractor::Booth, Source::Extractor::Anifty, + Source::Extractor::Furaffinity, ] # Should return true if the extractor is configured correctly. Return false diff --git a/app/logical/source/extractor/furaffinity.rb b/app/logical/source/extractor/furaffinity.rb new file mode 100644 index 000000000..f0a50ffc6 --- /dev/null +++ b/app/logical/source/extractor/furaffinity.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +class Source::Extractor + class Furaffinity < Source::Extractor + def self.enabled? + # https://www.furaffinity.net/controls/settings/ + # For this strategy to work properly, in the above settings "Enable Adult Artwork" must be set to "General, Mature, Adult". + Danbooru.config.furaffinity_cookie_a.present? && Danbooru.config.furaffinity_cookie_b.present? + end + + def match? + Source::URL::Furaffinity === parsed_url + end + + def image_urls + if parsed_url.image_url? + [parsed_url.to_s] + else + download_button = html_response&.css(".submission-content .auto_link .button").to_a.find { |el| el.text == "Download" } + partial_image = download_button&.[]("href") + return [] unless partial_image.present? + [URI.join("https://d.furaffinity.net", partial_image).to_s].compact + end + end + + def page_url + parsed_url.page_url || parsed_referer&.page_url + end + + def tags + tags = html_response&.css(".tags").to_a.map!(&:text).compact.uniq + tags.map {|tag| [tag, "https://www.furaffinity.net/search/@keywords #{tag}"] } + end + + def artist_name + html_response&.at(".submission-id-sub-container a")&.text || parsed_url.username || parsed_referer&.username + end + + def profile_url + parsed_url.profile_url || parsed_referer&.profile_url || profile_url_from_page + end + + def profile_url_from_page + slug = html_response&.at(".submission-id-avatar a")&.[](:href) + return unless slug.present? + Source::URL.parse(URI.join("https://www.furaffinity.net/", slug)).profile_url + end + + def artist_commentary_title + html_response&.at(".submission-title")&.text&.strip + end + + def artist_commentary_desc + html_response&.at(".submission-content .section-body")&.to_html + end + + def dtext_artist_commentary_desc + DText.from_html(artist_commentary_desc)&.strip + end + + def html_response + return nil unless page_url.present? + response = http.cache(1.minute).get(page_url) + + return nil unless response.status == 200 + response.parse + end + + def http + Danbooru::Http.new.cookies(a: Danbooru.config.furaffinity_cookie_a, b: Danbooru.config.furaffinity_cookie_b, sfw: 0) + end + end +end diff --git a/app/logical/source/url.rb b/app/logical/source/url.rb index f8adb9cf9..0ef9e1f58 100644 --- a/app/logical/source/url.rb +++ b/app/logical/source/url.rb @@ -49,6 +49,7 @@ module Source Source::URL::TwitPic, Source::URL::Weibo, Source::URL::Anifty, + Source::URL::Furaffinity, ] # Parse a URL into a subclass of Source::URL, or raise an exception if the URL is not a valid HTTP or HTTPS URL. diff --git a/app/logical/source/url/furaffinity.rb b/app/logical/source/url/furaffinity.rb new file mode 100644 index 000000000..d40d3ce88 --- /dev/null +++ b/app/logical/source/url/furaffinity.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +class Source::URL::Furaffinity < Source::URL + attr_reader :work_id, :username, :filename + + def self.match?(url) + url.domain == "furaffinity.net" + end + + def parse + case [host, *path_segments] + + # https://www.furaffinity.net/view/46821705/ + # https://www.furaffinity.net/view/46802202/ (scrap) + in _, "view", /^\d+$/ => work_id + @work_id = work_id + + # https://d.furaffinity.net/art/iwbitu/1650222955/1650222955.iwbitu_yubi.jpg + in _, "art", username, subdir, filename + @username = username + @filename = filename + + # https://www.furaffinity.net/gallery/iwbitu + # https://www.furaffinity.net/scraps/iwbitu/2/? + # https://www.furaffinity.net/gallery/iwbitu/folder/133763/Regular-commissions + in _, ("gallery" | "user" | "favorites" | "scraps" | "journals"), username, *pages + @username = username + + else + nil + end + end + + def image_url? + @filename.present? + end + + def page_url + "https://www.furaffinity.net/view/#{work_id}" if work_id.present? + end + + def profile_url + "https://www.furaffinity.net/user/#{username}" if username.present? + end +end diff --git a/app/models/artist_url.rb b/app/models/artist_url.rb index 036ba482f..d0e49d1ef 100644 --- a/app/models/artist_url.rb +++ b/app/models/artist_url.rb @@ -95,7 +95,7 @@ class ArtistURL < ApplicationRecord def priority sites = %w[ Pixiv Twitter - Anifty ArtStation Baraag BCY Booth Deviant\ Art Hentai\ Foundry Fantia Foundation Lofter Nico\ Seiga Nijie Pawoo Fanbox Pixiv\ Sketch Plurk Tinami Tumblr Weibo + Anifty ArtStation Baraag BCY Booth Deviant\ Art Hentai\ Foundry Fantia Furaffinity Foundation Lofter Nico\ Seiga Nijie Pawoo Fanbox Pixiv\ Sketch Plurk Tinami Tumblr Weibo Ask.fm Facebook FC2 Gumroad Instagram Ko-fi Livedoor Mihuashi Mixi.jp Patreon Piapro.jp Picarto Privatter Sakura.ne.jp Stickam Skeb Twitch Youtube Amazon Circle.ms DLSite Doujinshi.org Erogamescape Mangaupdates Melonbooks Toranoana Wikipedia ] diff --git a/config/danbooru_default_config.rb b/config/danbooru_default_config.rb index c37d225ac..b338dc1db 100644 --- a/config/danbooru_default_config.rb +++ b/config/danbooru_default_config.rb @@ -333,6 +333,17 @@ module Danbooru def fantia_session_id end + # Your Furaffinity "a" cookie. Login to Furaffinity then use the + # devtools to find the "a" cookie. + # !!WARNING!! logging out of furaffinity will expire this cookie too! + def furaffinity_cookie_a + end + + # Your Furaffinity "b" cookie. Login to Furaffinity then use the + # devtools to find the "b" cookie. + def furaffinity_cookie_b + end + # A list of tags that should be removed when a post is replaced. Regexes allowed. def post_replacement_tag_removals %w[replaceme .*_sample resized upscaled downscaled md5_mismatch diff --git a/test/unit/sources/furaffinity_test.rb b/test/unit/sources/furaffinity_test.rb new file mode 100644 index 000000000..31962a290 --- /dev/null +++ b/test/unit/sources/furaffinity_test.rb @@ -0,0 +1,45 @@ +require "test_helper" + +module Sources + class FuraffinityTest < ActiveSupport::TestCase + context "A furaffinity post" do + strategy_should_work( + "https://www.furaffinity.net/view/46821705/", + image_urls: ["https://d.furaffinity.net/art/iwbitu/1650222955/1650222955.iwbitu_yubi.jpg"], + profile_url: "https://www.furaffinity.net/user/iwbitu", + page_url: "https://www.furaffinity.net/view/46821705", + artist_name: "iwbitu", + artist_commentary_title: "Yubi", + artist_commentary_desc: /little gift doodle for/ + ) + end + + context "A furaffinity image" do + strategy_should_work( + "https://d.furaffinity.net/art/iwbitu/1650222955/1650222955.iwbitu_yubi.jpg", + image_urls: ["https://d.furaffinity.net/art/iwbitu/1650222955/1650222955.iwbitu_yubi.jpg"], + profile_url: "https://www.furaffinity.net/user/iwbitu", + artist_name: "iwbitu", + page_url: nil, + artist_commentary_title: nil + ) + end + + context "An adult age-restricted furaffinity post" do + strategy_should_work( + "https://www.furaffinity.net/view/46590097/", + image_urls: ["https://d.furaffinity.net/art/iwbitu/1648803766/1648803766.iwbitu_nyopu_tori.jpg"], + profile_url: "https://www.furaffinity.net/user/iwbitu", + page_url: "https://www.furaffinity.net/view/46590097", + artist_name: "iwbitu", + tags: [], + artist_commentary_title: "Nyopu and Tori", + artist_commentary_desc: /UwU/ + ) + end + + context "A deleted or non-existing furaffinity post" do + strategy_should_work("https://www.furaffinity.net/view/3404111", deleted: true, profile_url: nil) + end + end +end