jaculabilis/danbooru
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'master' into mobile-mode-default-image-size

Browse Source
This commit is contained in:
evazion
2020-07-23 16:22:37 -05:00
committed by GitHub
parent 131e526a15 c4fb43a5b4
commit da9c2a26ce
267 changed files with 5414 additions and 4398 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
app/controllers/application_controller.rb
View File

@@ -88,6 +88,8 @@ class ApplicationController < ActionController::Base
def rescue_exception(exception)
case exception
when ActionView::Template::Error
rescue_exception(exception.cause)
when ActiveRecord::QueryCanceled
render_error_page(500, exception, template: "static/search_timeout", message: "The database timed out running your query.")
when ActionController::BadRequest

16
app/controllers/autocomplete_controller.rb Normal file
View File

@@ -0,0 +1,16 @@
class AutocompleteController < ApplicationController
respond_to :xml, :json
def index
@tags = Tag.names_matches_with_aliases(params[:query], params.fetch(:limit, 10).to_i)
if request.variant.opensearch?
expires_in 1.hour
results = [params[:query], @tags.map(&:pretty_name)]
respond_with(results)
else
# XXX
respond_with(@tags.map(&:attributes))
end
end
end

1
app/controllers/comments_controller.rb
View File

@@ -97,6 +97,7 @@ class CommentsController < ApplicationController
if request.format.atom?
@comments = @comments.includes(:creator, :post)
@comments = @comments.select { |comment| comment.post.visible? }
elsif request.format.html?
@comments = @comments.includes(:creator, :updater, post: :uploader)
@comments = @comments.includes(:votes) if CurrentUser.is_member?

10
app/controllers/explore/posts_controller.rb
View File

@@ -22,23 +22,25 @@ module Explore
def viewed
@date, @scale, @min_date, @max_date = parse_date(params)
@posts = PostViewCountService.new.popular_posts(@date)
@posts = ReportbooruService.new.popular_posts(@date)
respond_with(@posts)
end
def searches
@date, @scale, @min_date, @max_date = parse_date(params)
@search_service = PopularSearchService.new(@date)
@searches = ReportbooruService.new.post_search_rankings(@date)
respond_with(@searches)
end
def missed_searches
@search_service = MissedSearchService.new
@missed_searches = ReportbooruService.new.missed_search_rankings
respond_with(@missed_searches)
end
private
def parse_date(params)
date = params[:date].present? ? Date.parse(params[:date]) : Time.zone.today
date = params[:date].present? ? Date.parse(params[:date]) : Date.today
scale = params[:scale].in?(["day", "week", "month"]) ? params[:scale] : "day"
min_date = date.send("beginning_of_#{scale}")
max_date = date.send("next_#{scale}").send("beginning_of_#{scale}")

2
app/controllers/iqdb_queries_controller.rb
View File

@@ -5,7 +5,7 @@ class IqdbQueriesController < ApplicationController
# XXX allow bare search params for backwards compatibility.
search_params.merge!(params.slice(:url, :image_url, :file_url, :post_id, :limit, :similarity, :high_similarity).permit!)
@high_similarity_matches, @low_similarity_matches, @matches = IqdbProxy.search(search_params)
@high_similarity_matches, @low_similarity_matches, @matches = IqdbProxy.new.search(search_params)
respond_with(@matches, template: "iqdb_queries/show")
end

48
app/controllers/mock_services_controller.rb Normal file
View File

@@ -0,0 +1,48 @@
class MockServicesController < ApplicationController
skip_forgery_protection
respond_to :json
before_action do
raise User::PrivilegeError if Rails.env.production?
end
def recommender_recommend
@data = posts.map { |post| [post.id, rand(0.0..1.0)] }
render json: @data
end
def recommender_similar
@data = posts.map { |post| [post.id, rand(0.0..1.0)] }
render json: @data
end
def reportbooru_missed_searches
@data = tags.map { |tag| "#{tag.name} #{rand(1.0..1000.0)}" }.join("\n")
render json: @data
end
def reportbooru_post_searches
@data = tags.map { |tag| [tag.name, rand(1..1000)] }
render json: @data
end
def reportbooru_post_views
@data = posts.map { |post| [post.id, rand(1..1000)] }
render json: @data
end
def iqdbs_similar
@data = posts.map { |post| { post_id: post.id, score: rand(0..100)} }
render json: @data
end
private
def posts(limit = 10)
Post.last(limit)
end
def tags(limit = 10)
Tag.order(post_count: :desc).limit(limit)
end
end

40
app/controllers/static_controller.rb
View File

@@ -1,4 +1,7 @@
class StaticController < ApplicationController
def privacy_policy
end
def terms_of_service
end
@@ -13,13 +16,40 @@ class StaticController < ApplicationController
redirect_to wiki_page_path("help:dtext") unless request.format.js?
end
def opensearch
end
def site_map
end
def sitemap
@popular_search_service = PopularSearchService.new(Date.yesterday)
@posts = Post.where("created_at > ?", 1.week.ago).order(score: :desc).limit(200)
@posts = @posts.select(&:visible?)
render layout: false
def sitemap_index
@sitemap = params[:sitemap]
@limit = params.fetch(:limit, 10000).to_i
case @sitemap
when "artists"
@relation = Artist.undeleted
@search = { is_deleted: "false" }
when "forum_topics"
@relation = ForumTopic.undeleted
@search = { is_deleted: "false" }
when "pools"
@relation = Pool.undeleted
@search = { is_deleted: "false" }
when "posts"
@relation = Post.order(id: :asc)
@serach = {}
when "tags"
@relation = Tag.nonempty
@search = {}
when "users"
@relation = User.all
@search = {}
when "wiki_pages"
@relation = WikiPage.undeleted
@search = { is_deleted: "false" }
else
raise NotImplementedError
end
end
end

2
app/controllers/uploads_controller.rb
View File

@@ -21,7 +21,7 @@ class UploadsController < ApplicationController
def image_proxy
authorize Upload
resp = ImageProxy.get_image(params[:url])
send_data resp.body, :type => resp.content_type, :disposition => "inline"
send_data resp.body, type: resp.mime_type, disposition: "inline"
end
def index

2
app/controllers/user_name_change_requests_controller.rb
View File

@@ -19,7 +19,7 @@ class UserNameChangeRequestsController < ApplicationController
end
def index
@change_requests = authorize UserNameChangeRequest.visible(CurrentUser.user).order("id desc").paginate(params[:page], :limit => params[:limit])
@change_requests = authorize UserNameChangeRequest.visible(CurrentUser.user).paginated_search(params)
respond_with(@change_requests)
end
end

6
app/controllers/users_controller.rb
View File

@@ -27,7 +27,7 @@ class UsersController < ApplicationController
def index
if params[:name].present?
@user = User.find_by_name!(params[:name])
redirect_to user_path(@user)
redirect_to user_path(@user, variant: params[:variant])
return
end
@@ -42,7 +42,9 @@ class UsersController < ApplicationController
def show
@user = authorize User.find(params[:id])
respond_with(@user, methods: @user.full_attributes)
respond_with(@user, methods: @user.full_attributes) do |format|
format.html.tooltip { render layout: false }
end
end
def profile

49
app/helpers/application_helper.rb
View File

@@ -121,6 +121,10 @@ module ApplicationHelper
raw content_tag(:time, duration, datetime: datetime, title: title)
end
def humanized_number(number)
number_to_human number, units: { thousand: "k", million: "m" }, format: "%n%u"
end
def time_ago_in_words_tagged(time, compact: false)
if time.nil?
tag.em(tag.time("unknown"))
@@ -162,8 +166,10 @@ module ApplicationHelper
end
end
def link_to_ip(ip)
link_to ip, ip_addresses_path(search: { ip_addr: ip, group_by: "user" })
def link_to_ip(ip, shorten: false, **options)
ip_addr = IPAddr.new(ip.to_s)
ip_addr.prefix = 64 if ip_addr.ipv6? && shorten
link_to ip_addr.to_s, ip_addresses_path(search: { ip_addr: ip, group_by: "user" }), **options
end
def link_to_search(search)
@@ -186,12 +192,13 @@ module ApplicationHelper
def link_to_user(user)
return "anonymous" if user.blank?
user_class = "user-#{user.level_string.downcase}"
user_class = "user user-#{user.level_string.downcase}"
user_class += " user-post-approver" if user.can_approve_posts?
user_class += " user-post-uploader" if user.can_upload_free?
user_class += " user-banned" if user.is_banned?
user_class += " with-style" if CurrentUser.user.style_usernames?
link_to(user.pretty_name, user_path(user), :class => user_class)
data = { "user-id": user.id, "user-name": user.name, "user-level": user.level }
link_to(user.pretty_name, user_path(user), class: user_class, data: data)
end
def mod_link_to_user(user, positive_or_negative)
@@ -217,21 +224,8 @@ module ApplicationHelper
tag.div(text, class: "prose", **options)
end
def dtext_field(object, name, options = {})
options[:name] ||= name.capitalize
options[:input_id] ||= "#{object}_#{name}"
options[:input_name] ||= "#{object}[#{name}]"
options[:value] ||= instance_variable_get("@#{object}").try(name)
options[:preview_id] ||= "dtext-preview"
options[:classes] ||= ""
options[:hint] ||= ""
options[:type] ||= "text"
render "dtext/form", options
end
def dtext_preview_button(object, name, input_id: "#{object}_#{name}", preview_id: "dtext-preview")
tag.input value: "Preview", type: "button", class: "dtext-preview-button", "data-input-id": input_id, "data-preview-id": preview_id
def dtext_preview_button(preview_field)
tag.input value: "Preview", type: "button", class: "dtext-preview-button", "data-preview-field": preview_field
end
def quick_search_form_for(attribute, url, name, autocomplete: nil, redirect: false, &block)
@@ -266,7 +260,7 @@ module ApplicationHelper
def body_attributes(user, params, current_item = nil)
current_user_data_attributes = data_attributes_for(user, "current-user", current_user_attributes)
if current_item.present? && current_item.respond_to?(:html_data_attributes) && current_item.respond_to?(:model_name)
if !current_item.nil? && current_item.respond_to?(:html_data_attributes) && current_item.respond_to?(:model_name)
model_name = current_item.model_name.singular.dasherize
model_attributes = current_item.html_data_attributes
current_item_data_attributes = data_attributes_for(current_item, model_name, model_attributes)
@@ -353,6 +347,19 @@ module ApplicationHelper
end
end
def canonical_url(url = nil)
if url.present?
content_for(:canonical_url) { url }
elsif content_for(:canonical_url).present?
content_for(:canonical_url)
else
request_params = request.params.sort.to_h.with_indifferent_access
request_params.delete(:page) if request_params[:page].to_i == 1
request_params.delete(:limit)
url_for(**request_params, host: Danbooru.config.hostname, only_path: false)
end
end
def atom_feed_tag(title, url = {})
content_for(:html_header, auto_discovery_link_tag(:atom, url, title: title))
end

4
app/helpers/pagination_helper.rb
View File

@@ -20,8 +20,8 @@ module PaginationHelper
params[:page] =~ /[ab]/ || records.current_page >= Danbooru.config.max_numbered_pages
end
def numbered_paginator(records, switch_to_sequential = true)
if use_sequential_paginator?(records) && switch_to_sequential
def numbered_paginator(records)
if use_sequential_paginator?(records)
return sequential_paginator(records)
end

65
app/helpers/seo_helper.rb Normal file
View File

@@ -0,0 +1,65 @@
# https://yoast.com/structured-data-schema-ultimate-guide/
# https://technicalseo.com/tools/schema-markup-generator/
# https://developers.google.com/search/docs/data-types/sitelinks-searchbox
# https://developers.google.com/search/docs/data-types/logo
# https://search.google.com/structured-data/testing-tool/u/0/
# https://search.google.com/test/rich-results
# https://schema.org/Organization
# https://schema.org/WebSite
module SeoHelper
def site_description
"#{Danbooru.config.canonical_app_name} is the original anime image booru. Search millions of anime pictures categorized by thousands of tags."
end
# https://developers.google.com/search/docs/data-types/video#video-object
def json_ld_video_data(post)
json_ld_tag({
"@context": "https://schema.org",
"@type": "VideoObject",
"name": page_title,
"description": meta_description,
"uploadDate": post.created_at.iso8601,
"thumbnailUrl": post.preview_file_url,
"contentUrl": post.file_url,
})
end
def json_ld_website_data
urls = [
Danbooru.config.twitter_url,
Danbooru.config.discord_server_url,
Danbooru.config.source_code_url,
"https://en.wikipedia.org/wiki/Danbooru"
].compact
json_ld_tag({
"@context": "https://schema.org",
"@graph": [
{
"@type": "Organization",
url: root_url(host: Danbooru.config.hostname),
name: Danbooru.config.app_name,
logo: "#{root_url(host: Danbooru.config.hostname)}images/danbooru-logo-500x500.png",
sameAs: urls
},
{
"@type": "WebSite",
"@id": root_url(anchor: "website", host: Danbooru.config.hostname),
"url": root_url(host: Danbooru.config.hostname),
"name": Danbooru.config.app_name,
"description": site_description,
"potentialAction": [{
"@type": "SearchAction",
"target": "#{posts_url(host: Danbooru.config.hostname)}?tags={search_term_string}",
"query-input": "required name=search_term_string"
}]
}
]
})
end
def json_ld_tag(data)
tag.script(data.to_json.html_safe, type: "application/ld+json")
end
end

4
app/javascript/packs/application.js
View File

@@ -6,12 +6,12 @@ function importAll(r) {
require('@rails/ujs').start();
require('hammerjs');
require('stupid-table-plugin');
require('jquery-hotkeys');
// should start looking for nodejs replacements
importAll(require.context('../vendor', true, /\.js$/));
require('jquery');
require("jquery-ui/ui/effects/effect-shake");
require("jquery-ui/ui/widgets/autocomplete");
require("jquery-ui/ui/widgets/button");
@@ -33,6 +33,7 @@ require("@fortawesome/fontawesome-free/css/regular.css");
importAll(require.context('../src/javascripts', true, /\.js(\.erb)?$/));
importAll(require.context('../src/styles', true, /\.s?css(?:\.erb)?$/));
export { default as jQuery } from "jquery";
export { default as Autocomplete } from '../src/javascripts/autocomplete.js.erb';
export { default as Blacklist } from '../src/javascripts/blacklists.js';
export { default as Comment } from '../src/javascripts/comments.js';
@@ -47,5 +48,6 @@ export { default as PostVersion } from '../src/javascripts/post_version.js';
export { default as RelatedTag } from '../src/javascripts/related_tag.js';
export { default as Shortcuts } from '../src/javascripts/shortcuts.js';
export { default as Upload } from '../src/javascripts/uploads.js.erb';
export { default as UserTooltip } from '../src/javascripts/user_tooltips.js';
export { default as Utility } from '../src/javascripts/utility.js';
export { default as Ugoira } from '../src/javascripts/ugoira.js';

18
app/javascript/src/javascripts/autocomplete.js.erb
View File

@@ -9,6 +9,7 @@ Autocomplete.ORDER_METATAGS = <%= PostQueryBuilder::ORDER_METATAGS.to_json.html_
Autocomplete.DISAPPROVAL_REASONS = <%= PostDisapproval::REASONS.to_json.html_safe %>;
/* eslint-enable */
Autocomplete.MISC_STATUSES = ["deleted", "active", "pending", "flagged", "banned", "modqueue", "unmoderated"];
Autocomplete.TAG_PREFIXES = "-|~|" + Object.keys(Autocomplete.TAG_CATEGORIES).map(category => category + ":").join("|");
Autocomplete.METATAGS_REGEX = Autocomplete.METATAGS.concat(Object.keys(Autocomplete.TAG_CATEGORIES)).join("|");
Autocomplete.TERM_REGEX = new RegExp(`([-~]*)(?:(${Autocomplete.METATAGS_REGEX}):)?(\\S*)$`, "i");
@@ -37,7 +38,7 @@ Autocomplete.initialize_all = function() {
});
this.initialize_tag_autocomplete();
this.initialize_mention_autocomplete($(".autocomplete-mentions textarea"));
this.initialize_mention_autocomplete($("form div.input.dtext textarea"));
this.initialize_fields($('[data-autocomplete="tag"]'), Autocomplete.tag_source);
this.initialize_fields($('[data-autocomplete="artist"]'), Autocomplete.artist_source);
this.initialize_fields($('[data-autocomplete="pool"]'), Autocomplete.pool_source);
@@ -248,9 +249,6 @@ Autocomplete.render_item = function(list, item) {
} else if (item.type === "user") {
var level_class = "user-" + item.level.toLowerCase();
$link.addClass(level_class);
if (CurrentUser.data("style-usernames")) {
$link.addClass("with-style");
}
} else if (item.type === "pool") {
$link.addClass("pool-category-" + item.category);
}
@@ -268,9 +266,7 @@ Autocomplete.render_item = function(list, item) {
Autocomplete.static_metatags = {
order: Autocomplete.ORDER_METATAGS,
status: [
"any", "deleted", "active", "pending", "flagged", "banned", "modqueue", "unmoderated"
],
status: ["any"].concat(Autocomplete.MISC_STATUSES),
rating: [
"safe", "questionable", "explicit"
],
@@ -280,12 +276,8 @@ Autocomplete.static_metatags = {
embedded: [
"true", "false"
],
child: [
"any", "none"
],
parent: [
"any", "none"
],
child: ["any", "none"].concat(Autocomplete.MISC_STATUSES),
parent: ["any", "none"].concat(Autocomplete.MISC_STATUSES),
filetype: [
"jpg", "png", "gif", "swf", "zip", "webm", "mp4"
],

7
app/javascript/src/javascripts/dtext.js
View File

@@ -39,8 +39,11 @@ Dtext.call_edit = function(e, $button, $input, $preview) {
Dtext.click_button = function(e) {
var $button = $(e.target);
var $input = $("#" + $button.data("input-id"));
var $preview = $("#" + $button.data("preview-id"));
var $form = $button.parents("form");
var fieldName = $button.data("preview-field");
var $inputContainer = $form.find(`div.input.${fieldName} .dtext-previewable`);
var $input = $inputContainer.find("> input, > textarea");
var $preview = $inputContainer.find("div.dtext-preview");
if ($button.val().match(/preview/i)) {
Dtext.call_preview(e, $button, $input, $preview);

14
app/javascript/src/javascripts/forum_posts.js
View File

@@ -7,17 +7,15 @@ ForumPost.initialize_all = function() {
}
ForumPost.initialize_edit_links = function() {
$(".edit_forum_post_link").on("click.danbooru", function(e) {
var link_id = $(this).attr("id");
var forum_post_id = link_id.match(/^edit_forum_post_link_(\d+)$/)[1];
$("#edit_forum_post_" + forum_post_id).fadeToggle("fast");
$(document).on("click.danbooru", ".edit_forum_post_link", function(e) {
let $form = $(this).parents("article.forum-post").find("form.edit_forum_post");
$form.fadeToggle("fast");
e.preventDefault();
});
$(".edit_forum_topic_link").on("click.danbooru", function(e) {
var link_id = $(this).attr("id");
var forum_topic_id = link_id.match(/^edit_forum_topic_link_(\d+)$/)[1];
$("#edit_forum_topic_" + forum_topic_id).fadeToggle("fast");
$(document).on("click.danbooru", ".edit_forum_topic_link", function(e) {
let $form = $(this).parents("article.forum-post").find("form.edit_forum_topic");
$form.fadeToggle("fast");
e.preventDefault();
});

5
app/javascript/src/javascripts/pools.js
View File

@@ -20,11 +20,6 @@ Pool.initialize_add_to_pool_link = function() {
e.preventDefault();
$("#add-to-pool-dialog").dialog("open");
});
$("#recent-pools li").on("click.danbooru", function(e) {
e.preventDefault();
$("#pool_name").val($(this).attr("data-value"));
});
}
Pool.initialize_simple_edit = function() {

2
app/javascript/src/javascripts/post_mode_menu.js
View File

@@ -70,7 +70,7 @@ PostModeMenu.initialize_edit_form = function() {
$(document).on("click.danbooru", "#quick-edit-form input[type=submit]", async function(e) {
e.preventDefault();
let post_id = $("#quick-edit-form").data("post-id");
let post_id = $("#quick-edit-form").attr("data-post-id");
await Post.update(post_id, "quick-edit", { post: { tag_string: $("#post_tag_string").val() }});
});
}

165
app/javascript/src/javascripts/post_tooltips.js
View File

@@ -1,28 +1,76 @@
import CurrentUser from './current_user'
import Utility from './utility'
require('qtip2');
require('qtip2/dist/jquery.qtip.css');
import CurrentUser from './current_user';
import Utility from './utility';
import { delegate, hideAll } from 'tippy.js';
import 'tippy.js/dist/tippy.css';
let PostTooltip = {};
PostTooltip.render_tooltip = async function (event, qtip) {
PostTooltip.POST_SELECTOR = "*:not(.ui-sortable-handle) > .post-preview img, .dtext-post-id-link";
PostTooltip.SHOW_DELAY = 500;
PostTooltip.HIDE_DELAY = 125;
PostTooltip.DURATION = 250;
PostTooltip.initialize = function () {
if (PostTooltip.disabled()) {
return;
}
delegate("body", {
allowHTML: true,
appendTo: document.body,
delay: [PostTooltip.SHOW_DELAY, PostTooltip.HIDE_DELAY],
duration: PostTooltip.DURATION,
interactive: true,
maxWidth: "none",
target: PostTooltip.POST_SELECTOR,
theme: "common-tooltip post-tooltip",
touch: false,
onCreate: PostTooltip.on_create,
onShow: PostTooltip.on_show,
onHide: PostTooltip.on_hide,
});
$(document).on("click.danbooru.postTooltip", ".post-tooltip-disable", PostTooltip.on_disable_tooltips);
};
PostTooltip.on_create = function (instance) {
let title = instance.reference.getAttribute("title");
if (title) {
instance.reference.setAttribute("data-title", title);
instance.reference.setAttribute("title", "");
}
};
PostTooltip.on_show = async function (instance) {
let post_id = null;
let preview = false;
let $target = $(instance.reference);
let $tooltip = $(instance.popper);
if ($(this).is(".dtext-post-id-link")) {
hideAll({ exclude: instance });
// skip if tooltip has already been rendered.
if ($tooltip.has(".post-tooltip-body").length) {
return;
}
if ($target.is(".dtext-post-id-link")) {
preview = true;
post_id = /\/posts\/(\d+)/.exec($(this).attr("href"))[1];
post_id = /\/posts\/(\d+)/.exec($target.attr("href"))[1];
} else {
post_id = $(this).parents("[data-id]").data("id");
post_id = $target.parents("[data-id]").data("id");
}
try {
qtip.cache.request = $.get(`/posts/${post_id}`, { variant: "tooltip", preview: preview });
let html = await qtip.cache.request;
$tooltip.addClass("tooltip-loading");
qtip.set("content.text", html);
qtip.elements.tooltip.removeClass("post-tooltip-loading");
instance._request = $.get(`/posts/${post_id}`, { variant: "tooltip", preview: preview });
let html = await instance._request;
instance.setContent(html);
$tooltip.removeClass("tooltip-loading");
} catch (error) {
if (error.status !== 0 && error.statusText !== "abort") {
Utility.error(`Error displaying tooltip for post #${post_id} (error: ${error.status} ${error.statusText})`);
@@ -30,98 +78,19 @@ PostTooltip.render_tooltip = async function (event, qtip) {
}
};
// Hide the tooltip the first time it is shown, while we wait on the ajax call to complete.
PostTooltip.on_show = function (event, qtip) {
if (!qtip.cache.hasBeenShown) {
qtip.elements.tooltip.addClass("post-tooltip-loading");
qtip.cache.hasBeenShown = true;
PostTooltip.on_hide = function (instance) {
if (instance._request?.state() === "pending") {
instance._request.abort();
}
};
PostTooltip.POST_SELECTOR = "*:not(.ui-sortable-handle) > .post-preview img, .dtext-post-id-link";
// http://qtip2.com/options
PostTooltip.QTIP_OPTIONS = {
style: {
classes: "qtip-light post-tooltip",
tip: false
},
content: PostTooltip.render_tooltip,
overwrite: false,
position: {
viewport: true,
my: "bottom left",
at: "top left",
effect: false,
adjust: {
y: -2,
method: "shift",
},
},
show: {
solo: true,
delay: 750,
effect: false,
ready: true,
event: "mouseenter",
},
hide: {
delay: 250,
fixed: true,
effect: false,
event: "unfocus click mouseleave",
},
events: {
show: PostTooltip.on_show,
},
};
PostTooltip.initialize = function () {
$(document).on("mouseenter.danbooru.postTooltip", PostTooltip.POST_SELECTOR, function (event) {
if (PostTooltip.disabled()) {
$(this).qtip("disable");
} else {
$(this).qtip(PostTooltip.QTIP_OPTIONS, event);
}
});
// Cancel pending ajax requests when we mouse out of the thumbnail.
$(document).on("mouseleave.danbooru.postTooltip", PostTooltip.POST_SELECTOR, function (event) {
let qtip = $(event.target).qtip("api");
if (qtip && qtip.cache && qtip.cache.request && qtip.cache.request.state() === "pending") {
qtip.cache.request.abort();
}
});
$(document).on("click.danbooru.postTooltip", ".post-tooltip-disable", PostTooltip.on_disable_tooltips);
// Hide tooltips when pressing keys or clicking thumbnails.
$(document).on("keydown.danbooru.postTooltip", PostTooltip.hide);
$(document).on("click.danbooru.postTooltip", PostTooltip.POST_SELECTOR, PostTooltip.hide);
// Disable tooltips on touch devices. https://developer.mozilla.org/en-US/docs/Web/API/Touch_events/Supporting_both_TouchEvent_and_MouseEvent
PostTooltip.isTouching = false;
$(document).on("touchstart.danbooru.postTooltip", function (event) { PostTooltip.isTouching = true; });
$(document).on("touchend.danbooru.postTooltip", function (event) { PostTooltip.isTouching = false; });
};
PostTooltip.hide = function (event) {
// Hide on any key except control (user may be control-clicking link inside tooltip).
if (event.type === "keydown" && event.ctrlKey === true) {
return;
}
$(".post-tooltip:visible").qtip("hide");
};
}
PostTooltip.disabled = function (event) {
return PostTooltip.isTouching || CurrentUser.data("disable-post-tooltips");
return CurrentUser.data("disable-post-tooltips");
};
PostTooltip.on_disable_tooltips = async function (event) {
event.preventDefault();
$(event.target).parents(".qtip").qtip("hide");
hideAll();
if (CurrentUser.data("is-anonymous")) {
Utility.notice('You must <a href="/session/new">login</a> to disable tooltips');

12
app/javascript/src/javascripts/posts.js.erb
View File

@@ -300,10 +300,10 @@ Post.initialize_favlist = function() {
});
}
Post.view_original = function(e) {
Post.view_original = function(e = null) {
if (Utility.test_max_width(660)) {
// Do the default behavior (navigate to image)
return false;
return;
}
var $image = $("#image");
@@ -316,13 +316,13 @@ Post.view_original = function(e) {
});
Note.Box.scale_all();
$("body").attr("data-post-current-image-size", "original");
return false;
e?.preventDefault();
}
Post.view_large = function(e) {
Post.view_large = function(e = null) {
if (Utility.test_max_width(660)) {
// Do the default behavior (navigate to image)
return false;
return;
}
var $image = $("#image");
@@ -335,7 +335,7 @@ Post.view_large = function(e) {
});
Note.Box.scale_all();
$("body").attr("data-post-current-image-size", "large");
return false;
e?.preventDefault();
}
Post.toggle_fit_window = function(e) {

11
app/javascript/src/javascripts/saved_searches.js
View File

@@ -1,11 +0,0 @@
let SavedSearch = {};
SavedSearch.initialize_all = function() {
if ($("#c-saved-searches").length) {
$("#c-saved-searches table").stupidtable();
}
}
$(SavedSearch.initialize_all);
export default SavedSearch

85
app/javascript/src/javascripts/user_tooltips.js Normal file
View File

@@ -0,0 +1,85 @@
import Utility from './utility';
import { delegate } from 'tippy.js';
import 'tippy.js/dist/tippy.css';
let UserTooltip = {};
UserTooltip.SELECTOR = "*:not(.user-tooltip-name) > a.user, a.dtext-user-id-link, a.dtext-user-mention-link";
UserTooltip.SHOW_DELAY = 500;
UserTooltip.HIDE_DELAY = 125;
UserTooltip.DURATION = 250;
UserTooltip.MAX_WIDTH = 600;
UserTooltip.initialize = function () {
delegate("body", {
allowHTML: true,
appendTo: document.body,
delay: [UserTooltip.SHOW_DELAY, UserTooltip.HIDE_DELAY],
duration: UserTooltip.DURATION,
interactive: true,
maxWidth: UserTooltip.MAX_WIDTH,
target: UserTooltip.SELECTOR,
theme: "common-tooltip user-tooltip",
touch: false,
onShow: UserTooltip.on_show,
onHide: UserTooltip.on_hide,
});
delegate("body", {
allowHTML: true,
interactive: true,
theme: "common-tooltip",
target: ".user-tooltip-menu-button",
placement: "bottom",
touch: false,
trigger: "click",
content: (element) => {
return $(element).parents(".user-tooltip").find(".user-tooltip-menu").get(0);
}
});
};
UserTooltip.on_show = async function (instance) {
let $target = $(instance.reference);
let $tooltip = $(instance.popper);
// skip if tooltip has already been rendered.
if ($tooltip.has(".user-tooltip-body").length) {
return;
}
try {
$tooltip.addClass("tooltip-loading");
if ($target.is("a.dtext-user-id-link")) {
let user_id = /\/users\/(\d+)/.exec($target.attr("href"))[1];
instance._request = $.get(`/users/${user_id}`, { variant: "tooltip" });
} else if ($target.is("a.user")) {
let user_id = $target.attr("data-user-id");
instance._request = $.get(`/users/${user_id}`, { variant: "tooltip" });
} else if ($target.is("a.dtext-user-mention-link")) {
let user_name = $target.attr("data-user-name");
instance._request = $.get(`/users`, { name: user_name, variant: "tooltip" });
}
let html = await instance._request;
instance.setContent(html);
$tooltip.removeClass("tooltip-loading");
} catch (error) {
if (error.status !== 0 && error.statusText !== "abort") {
Utility.error(`Error displaying tooltip (error: ${error.status} ${error.statusText})`);
}
}
};
UserTooltip.on_hide = function (instance) {
if (instance._request?.state() === "pending") {
instance._request.abort();
}
}
$(document).ready(UserTooltip.initialize);
export default UserTooltip

8
app/javascript/src/styles/base/020_base.scss
View File

@@ -129,6 +129,14 @@ table tfoot {
font-size: 0.9em;
}
a.link-plain {
color: unset;
&:hover {
text-decoration: underline;
}
}
.fixed-width-container {
max-width: 70em;
}

13
app/javascript/src/styles/base/040_colors.css
View File

@@ -2,6 +2,7 @@
--body-background-color: white;
--text-color: hsl(0, 0%, 15%);
--inverse-text-color: white;
--muted-text-color: hsl(0, 0%, 55%);
--header-color: hsl(0, 0%, 15%);
@@ -72,7 +73,8 @@
--comment-sticky-background-color: var(--subnav-menu-background-color);
--post-tooltip-background-color: var(--body-background-color);
--post-tooltip-border-color: #767676;
--post-tooltip-border-color: hsla(210, 100%, 3%, 0.15);
--post-tooltip-box-shadow: 0 4px 14px -2px hsla(210, 100%, 3%, 0.10);
--post-tooltip-header-background-color: var(--subnav-menu-background-color);
--post-tooltip-info-color: #555;
--post-tooltip-scrollbar-background: #999999;
@@ -81,6 +83,9 @@
--post-tooltip-scrollbar-track-background: #EEEEEE;
--post-tooltip-scrollbar-track-border: 0 none white;
--user-tooltip-positive-feedback-color: orange;
--user-tooltip-negative-feedback-color: red;
--preview-current-post-background: rgba(0, 0, 0, 0.1);
--autocomplete-selected-background-color: var(--subnav-menu-background-color);
@@ -200,6 +205,7 @@
--user-platinum-color: gray;
--user-gold-color: #00F;
--user-member-color: var(--link-color);
--user-banned-color: black;
--news-updates-background: #EEE;
--news-updates-border: 2px solid #666;
@@ -260,6 +266,7 @@ body[data-current-user-theme="dark"] {
/* main text colors */
--text-color: var(--grey-5);
--inverse-text-color: white;
--muted-text-color: var(--grey-4);
--header-color: var(--grey-6);
@@ -282,6 +289,7 @@ body[data-current-user-theme="dark"] {
--collection-pool-color: var(--general-tag-color);
--collection-pool-hover-color: var(--general-tag-hover-color);
--user-banned-color: var(--grey-1);
--user-member-color: var(--blue-1);
--user-gold-color: var(--yellow-1);
--user-platinum-color: var(--grey-4);
@@ -394,6 +402,9 @@ body[data-current-user-theme="dark"] {
--post-tooltip-scrollbar-track-background: var(--grey-1);
--post-tooltip-scrollbar-track-border: none;
--user-tooltip-positive-feedback-color: var(--yellow-1);
--user-tooltip-negative-feedback-color: var(--red-1);
--preview-pending-color: var(--blue-1);
--preview-flagged-color: var(--red-1);
--preview-deleted-color: var(--grey-5);

3
app/javascript/src/styles/common/messages.scss
View File

@@ -17,7 +17,8 @@ div.list-of-messages {
a.message-timestamp {
font-style: italic;
color: var(--text-color);
font-size: 0.90em;
color: var(--muted-text-color);
&:hover { text-decoration: underline; }
}
}

2
app/javascript/src/styles/common/simple_form.scss
View File

@@ -39,7 +39,7 @@ form.simple_form {
padding-left: 1em;
}
&.text {
&.text, &.dtext {
.hint {
padding-left: 0;
display: block;

36
app/javascript/src/styles/common/user_styles.scss
View File

@@ -1,23 +1,25 @@
a.user-admin.with-style {
color: var(--user-admin-color);
}
body[data-current-user-style-usernames="true"] {
a.user-admin {
color: var(--user-admin-color);
}
a.user-moderator.with-style {
color: var(--user-moderator-color);
}
a.user-moderator {
color: var(--user-moderator-color);
}
a.user-builder.with-style {
color: var(--user-builder-color);
}
a.user-builder {
color: var(--user-builder-color);
}
a.user-platinum.with-style {
color: var(--user-platinum-color);
}
a.user-platinum {
color: var(--user-platinum-color);
}
a.user-gold.with-style {
color: var(--user-gold-color);
}
a.user-gold {
color: var(--user-gold-color);
}
a.user-member.with-style {
color: var(--user-member-color);
a.user-member {
color: var(--user-member-color);
}
}

51
app/javascript/src/styles/specific/common_tooltips.scss Normal file
View File

@@ -0,0 +1,51 @@
div[data-tippy-root].tooltip-loading {
visibility: hidden !important;
}
.tippy-box[data-theme~="common-tooltip"] {
box-sizing: border-box;
border: 1px solid var(--post-tooltip-border-color);
border-radius: 4px;
color: var(--text-color);
background-color: var(--post-tooltip-background-color);
background-clip: padding-box;
box-shadow: var(--post-tooltip-box-shadow);
/* bordered arrow styling; see https://github.com/atomiks/tippyjs/blob/master/src/scss/themes/light-border.scss */
&[data-placement^=bottom] {
> .tippy-arrow:before {
border-bottom-color: var(--post-tooltip-background-color);
bottom: 16px;
}
> .tippy-arrow:after {
border-bottom-color: var(--post-tooltip-border-color);
border-width: 0 7px 7px;
top: -8px;
left: 1px;
}
}
&[data-placement^=top] {
> .tippy-arrow:before {
border-top-color: var(--post-tooltip-background-color);
}
> .tippy-arrow:after {
border-top-color: var(--post-tooltip-border-color);
border-width: 7px 7px 0;
top: 17px;
left: 1px;
}
}
> .tippy-arrow:after {
border-color: transparent;
border-style: solid;
content: "";
position: absolute;
z-index: -1;
}
}

54
app/javascript/src/styles/specific/post_tooltips.scss
View File

@@ -1,6 +1,5 @@
$tooltip-line-height: 16px;
$tooltip-body-height: $tooltip-line-height * 6; // 6 lines high.
$tooltip-width: 164px * 3 - 10; // 3 thumbnails wide.
$tooltip-body-height: $tooltip-line-height * 4; // 4 lines high.
@mixin thin-scrollbar {
&::-webkit-scrollbar {
@@ -46,16 +45,13 @@ $tooltip-width: 164px * 3 - 10; // 3 thumbnails wide.
}
}
.post-tooltip {
max-width: $tooltip-width;
min-width: $tooltip-width;
box-sizing: border-box;
.tippy-box[data-theme~="post-tooltip"] {
min-width: 20em;
max-width: 40em !important;
font-size: 11px;
line-height: $tooltip-line-height;
border-color: var(--post-tooltip-border-color);
background-color: var(--post-tooltip-background-color);
.qtip-content {
.tippy-content {
padding: 0;
> * {
@@ -85,38 +81,32 @@ $tooltip-width: 164px * 3 - 10; // 3 thumbnails wide.
.post-tooltip-body-right { flex: 1; }
}
.post-tooltip-header {
div.post-tooltip-header {
background-color: var(--post-tooltip-header-background-color);
display: flex;
white-space: nowrap;
overflow: hidden;
align-items: center;
.post-tooltip-header-left {
flex: 1;
.post-tooltip-info {
margin-right: 0.5em;
color: var(--post-tooltip-info-color);
font-size: 10px;
flex: 0;
}
.post-tooltip-header-right {
a.user {
margin-right: 0.5em;
overflow: hidden;
white-space: nowrap;
text-overflow: ellipsis;
min-width: 0;
max-width: 11em;
}
.post-tooltip-source {
flex: 1;
text-align: right;
}
.fa-xs {
vertical-align: baseline;
}
.post-tooltip-disable {
margin-left: 0.5em;
}
}
.post-tooltip-info {
margin-left: 0.5em;
color: var(--post-tooltip-info-color);
font-size: 10px;
}
}
&.post-tooltip-loading {
visibility: hidden;
}
}

5
app/javascript/src/styles/specific/privacy_policy.scss Normal file
View File

@@ -0,0 +1,5 @@
#c-static #a-privacy-policy {
.summary {
font-style: italic;
}
}

103
app/javascript/src/styles/specific/user_tooltips.scss Normal file
View File

@@ -0,0 +1,103 @@
.tippy-box[data-theme~="user-tooltip"] {
line-height: 1.25em;
min-width: 350px;
.user-tooltip-header {
margin-bottom: 1em;
display: grid;
grid:
"avatar header-top menu"
"avatar header-bottom menu" /
32px 1fr 32px;
column-gap: 0.25em;
.user-tooltip-avatar {
font-size: 32px;
grid-area: avatar;
align-self: center;
}
.user-tooltip-header-top {
grid-area: header-top;
.user-tooltip-badge {
color: var(--inverse-text-color);
font-size: 0.70em;
padding: 2px 4px;
margin-right: 0.25em;
border-radius: 3px;
&.user-tooltip-badge-admin { background-color: var(--user-admin-color); }
&.user-tooltip-badge-moderator { background-color: var(--user-moderator-color); }
&.user-tooltip-badge-approver { background-color: var(--user-builder-color); }
&.user-tooltip-badge-contributor { background-color: var(--user-builder-color); }
&.user-tooltip-badge-builder { background-color: var(--user-builder-color); }
&.user-tooltip-badge-platinum { background-color: var(--user-platinum-color); }
&.user-tooltip-badge-gold { background-color: var(--user-gold-color); }
&.user-tooltip-badge-member { background-color: var(--user-member-color); }
&.user-tooltip-badge-banned { background-color: var(--user-banned-color); }
&.user-tooltip-badge-positive-feedback {
color: var(--user-tooltip-positive-feedback-color);
border: 1px solid;
}
&.user-tooltip-badge-negative-feedback {
color: var(--user-tooltip-negative-feedback-color);
border: 1px solid;
}
}
}
.user-tooltip-header-bottom {
grid-area: header-bottom;
color: var(--muted-text-color);
font-size: 0.75em;
}
a.user-tooltip-menu-button {
color: var(--muted-text-color);
grid-area: menu;
align-self: center;
text-align: center;
width: 28px;
height: 28px;
line-height: 28px;
border-radius: 50%;
&:hover {
background-color: var(--subnav-menu-background-color);
}
}
> ul.user-tooltip-menu {
display: none;
}
ul.user-tooltip-menu {
.icon {
width: 1.5em;
}
}
}
.user-tooltip-stats {
display: grid;
grid: auto / repeat(3, 1fr);
column-gap: 1em;
row-gap: 0.5em;
.user-tooltip-stat-item {
text-align: center;
.user-tooltip-stat-value {
font-weight: bold;
}
.user-tooltip-stat-name {
font-size: 0.90em;
color: var(--muted-text-color);
}
}
}
}

8
app/logical/apng_inspector.rb
View File

@@ -57,7 +57,7 @@ class APNGInspector
# if we did, file is probably maliciously formed
# fail gracefully without marking the file as corrupt
chunks += 1
if chunks > 100000
if chunks > 100_000
iend_reached = true
break
end
@@ -66,7 +66,8 @@ class APNGInspector
file.seek(current_pos + chunk_len + 4, IO::SEEK_SET)
end
end
return iend_reached
iend_reached
end
def inspect!
@@ -105,6 +106,7 @@ class APNGInspector
if framedata.nil? || framedata.length != 4
return -1
end
return framedata.unpack1("N".freeze)
framedata.unpack1("N".freeze)
end
end

12
app/logical/artist_finder.rb
View File

@@ -6,7 +6,7 @@ module ArtistFinder
SITE_BLACKLIST = [
"artstation.com/artist", # http://www.artstation.com/artist/serafleur/
"www.artstation.com", # http://www.artstation.com/serafleur/
%r!cdn[ab]?\.artstation\.com/p/assets/images/images!i, # https://cdna.artstation.com/p/assets/images/images/001/658/068/large/yang-waterkuma-b402.jpg?1450269769
%r{cdn[ab]?\.artstation\.com/p/assets/images/images}i, # https://cdna.artstation.com/p/assets/images/images/001/658/068/large/yang-waterkuma-b402.jpg?1450269769
"ask.fm", # http://ask.fm/mikuroko_396
"bcyimg.com",
"bcyimg.com/drawer", # https://img9.bcyimg.com/drawer/32360/post/178vu/46229ec06e8111e79558c1b725ebc9e6.jpg
@@ -52,7 +52,7 @@ module ArtistFinder
"hentai-foundry.com",
"hentai-foundry.com/pictures/user", # http://www.hentai-foundry.com/pictures/user/aaaninja/
"hentai-foundry.com/user", # http://www.hentai-foundry.com/user/aaaninja/profile
%r!pictures\.hentai-foundry\.com(?:/\w)?!i, # http://pictures.hentai-foundry.com/a/aaaninja/
%r{pictures\.hentai-foundry\.com(?:/\w)?}i, # http://pictures.hentai-foundry.com/a/aaaninja/
"i.imgur.com", # http://i.imgur.com/Ic9q3.jpg
"instagram.com", # http://www.instagram.com/serafleur.art/
"iwara.tv",
@@ -62,13 +62,15 @@ module ArtistFinder
"monappy.jp",
"monappy.jp/u", # https://monappy.jp/u/abara_bone
"mstdn.jp", # https://mstdn.jp/@oneb
"www.newgrounds.com", # https://jessxjess.newgrounds.com/
"newgrounds.com/art/view/", # https://www.newgrounds.com/art/view/jessxjess/avatar-korra
"nicoseiga.jp",
"nicoseiga.jp/priv", # http://lohas.nicoseiga.jp/priv/2017365fb6cfbdf47ad26c7b6039feb218c5e2d4/1498430264/6820259
"nicovideo.jp",
"nicovideo.jp/user", # http://www.nicovideo.jp/user/317609
"nicovideo.jp/user/illust", # http://seiga.nicovideo.jp/user/illust/29075429
"nijie.info", # http://nijie.info/members.php?id=15235
%r!nijie\.info/nijie_picture!i, # http://pic03.nijie.info/nijie_picture/32243_20150609224803_0.png
%r{nijie\.info/nijie_picture}i, # http://pic03.nijie.info/nijie_picture/32243_20150609224803_0.png
"patreon.com", # http://patreon.com/serafleur
"pawoo.net", # https://pawoo.net/@148nasuka
"pawoo.net/web/accounts", # https://pawoo.net/web/accounts/228341
@@ -120,7 +122,7 @@ module ArtistFinder
SITE_BLACKLIST_REGEXP = Regexp.union(SITE_BLACKLIST.map do |domain|
domain = Regexp.escape(domain) if domain.is_a?(String)
%r!\Ahttps?://(?:[a-zA-Z0-9_-]+\.)*#{domain}/\z!i
%r{\Ahttps?://(?:[a-zA-Z0-9_-]+\.)*#{domain}/\z}i
end)
def find_artists(url)
@@ -128,7 +130,7 @@ module ArtistFinder
artists = []
while artists.empty? && url.size > 10
u = url.sub(/\/+$/, "") + "/"
u = url.sub(%r{/+$}, "") + "/"
u = u.to_escaped_for_sql_like.gsub(/\*/, '%') + '%'
artists += Artist.joins(:urls).where(["artists.is_deleted = FALSE AND artist_urls.normalized_url LIKE ? ESCAPE E'\\\\'", u]).limit(10).order("artists.name").all
url = File.dirname(url) + "/"

2
app/logical/bulk_update_request_processor.rb
View File

@@ -148,8 +148,6 @@ class BulkUpdateRequestProcessor
end.join("\n")
end
private
def self.is_tag_move_allowed?(antecedent_name, consequent_name)
antecedent_tag = Tag.find_by_name(Tag.normalize_name(antecedent_name))
consequent_tag = Tag.find_by_name(Tag.normalize_name(consequent_name))

9
app/logical/cloudflare_service.rb
View File

@@ -9,15 +9,6 @@ class CloudflareService
api_token.present? && zone.present?
end
def ips(expiry: 24.hours)
response = Danbooru::Http.cache(expiry).get("https://api.cloudflare.com/client/v4/ips")
return [] if response.code != 200
result = response.parse["result"]
ips = result["ipv4_cidrs"] + result["ipv6_cidrs"]
ips.map { |ip| IPAddr.new(ip) }
end
def purge_cache(urls)
return unless enabled?

2
app/logical/concerns/has_bit_flags.rb
View File

@@ -16,7 +16,7 @@ module HasBitFlags
end
define_method("#{attribute}=") do |val|
if val.to_s =~ /t|1|y/
if val.to_s =~ /[t1y]/
send("#{field}=", send(field) | bit_flag)
else
send("#{field}=", send(field) & ~bit_flag)

2
app/logical/concerns/mentionable.rb
View File

@@ -11,8 +11,6 @@ module Mentionable
# - user_field
def mentionable(options = {})
@mentionable_options = options
message_field = mentionable_option(:message_field)
after_save :queue_mention_messages
end

7
app/logical/concerns/searchable.rb
View File

@@ -89,7 +89,7 @@ module Searchable
def where_array_count(attr, value)
qualified_column = "cardinality(#{qualified_column_for(attr)})"
range = PostQueryBuilder.new(nil).parse_range(value, :integer)
where_operator("cardinality(#{qualified_column_for(attr)})", *range)
where_operator(qualified_column, *range)
end
def search_boolean_attribute(attribute, params)
@@ -170,7 +170,7 @@ module Searchable
end
end
def search_text_attribute(attr, params, **options)
def search_text_attribute(attr, params)
if params[attr].present?
where(attr => params[attr])
elsif params[:"#{attr}_eq"].present?
@@ -279,7 +279,8 @@ module Searchable
return find_ordered(parse_ids[1])
end
end
return default_order
default_order
end
def default_order

9
app/logical/current_user.rb
View File

@@ -24,15 +24,6 @@ class CurrentUser
scoped(user, &block)
end
def self.as_system(&block)
if block_given?
scoped(::User.system, "127.0.0.1", &block)
else
self.user = User.system
self.ip_addr = "127.0.0.1"
end
end
def self.user
RequestStore[:current_user]
end

8
app/logical/d_text.rb
View File

@@ -11,7 +11,7 @@ class DText
html = DTextRagel.parse(text, **options)
html = postprocess(html, *data)
html
rescue DTextRagel::Error => e
rescue DTextRagel::Error
""
end
@@ -135,7 +135,7 @@ class DText
fragment = Nokogiri::HTML.fragment(html)
titles = fragment.css("a.dtext-wiki-link").map do |node|
title = node["href"][%r!\A/wiki_pages/(.*)\z!i, 1]
title = node["href"][%r{\A/wiki_pages/(.*)\z}i, 1]
title = CGI.unescape(title)
title = WikiPage.normalize_title(title)
title
@@ -163,7 +163,7 @@ class DText
string = string.dup
string.gsub!(/\s*\[#{tag}\](?!\])\s*/mi, "\n\n[#{tag}]\n\n")
string.gsub!(/\s*\[\/#{tag}\]\s*/mi, "\n\n[/#{tag}]\n\n")
string.gsub!(%r{\s*\[/#{tag}\]\s*}mi, "\n\n[/#{tag}]\n\n")
string.gsub!(/(?:\r?\n){3,}/, "\n\n")
string.strip!
@@ -203,7 +203,7 @@ class DText
end
end
text = text.gsub(/\A[[:space:]]+|[[:space:]]+\z/, "")
text.gsub(/\A[[:space:]]+|[[:space:]]+\z/, "")
end
def self.from_html(text, inline: false, &block)

128
app/logical/danbooru/http.rb
View File

@@ -1,17 +1,49 @@
require "danbooru/http/html_adapter"
require "danbooru/http/xml_adapter"
require "danbooru/http/cache"
require "danbooru/http/redirector"
require "danbooru/http/retriable"
require "danbooru/http/session"
require "danbooru/http/spoof_referrer"
require "danbooru/http/unpolish_cloudflare"
module Danbooru
class Http
DEFAULT_TIMEOUT = 3
class DownloadError < StandardError; end
class FileTooLargeError < StandardError; end
attr_writer :cache, :http
DEFAULT_TIMEOUT = 10
MAX_REDIRECTS = 5
attr_accessor :max_size, :http
class << self
delegate :get, :post, :delete, :cache, :auth, :basic_auth, :headers, to: :new
delegate :get, :head, :put, :post, :delete, :cache, :follow, :max_size, :timeout, :auth, :basic_auth, :headers, :cookies, :use, :public_only, :download_media, to: :new
end
def initialize
@http ||=
::Danbooru::Http::ApplicationClient.new
.timeout(DEFAULT_TIMEOUT)
.headers("Accept-Encoding" => "gzip")
.headers("User-Agent": "#{Danbooru.config.canonical_app_name}/#{Rails.application.config.x.git_hash}")
.use(:auto_inflate)
.use(redirector: { max_redirects: MAX_REDIRECTS })
.use(:session)
end
def get(url, **options)
request(:get, url, **options)
end
def head(url, **options)
request(:head, url, **options)
end
def put(url, **options)
request(:get, url, **options)
end
def post(url, **options)
request(:post, url, **options)
end
@@ -20,8 +52,16 @@ module Danbooru
request(:delete, url, **options)
end
def cache(expiry)
dup.tap { |o| o.cache = expiry.to_i }
def follow(*args)
dup.tap { |o| o.http = o.http.follow(*args) }
end
def max_size(size)
dup.tap { |o| o.max_size = size }
end
def timeout(*args)
dup.tap { |o| o.http = o.http.timeout(*args) }
end
def auth(*args)
@@ -36,36 +76,66 @@ module Danbooru
dup.tap { |o| o.http = o.http.headers(*args) }
end
def cookies(*args)
dup.tap { |o| o.http = o.http.cookies(*args) }
end
def use(*args)
dup.tap { |o| o.http = o.http.use(*args) }
end
def cache(expires_in)
use(cache: { expires_in: expires_in })
end
# allow requests only to public IPs, not to local or private networks.
def public_only
dup.tap do |o|
o.http = o.http.dup.tap do |http|
http.default_options = http.default_options.with_socket_class(ValidatingSocket)
end
end
end
concerning :DownloadMethods do
def download_media(url, file: Tempfile.new("danbooru-download-", binmode: true))
response = get(url)
raise DownloadError, "Downloading #{response.uri} failed with code #{response.status}" if response.status != 200
raise FileTooLargeError, response if @max_size && response.content_length.to_i > @max_size
size = 0
response.body.each do |chunk|
size += chunk.size
raise FileTooLargeError if @max_size && size > @max_size
file.write(chunk)
end
file.rewind
[response, MediaFile.open(file)]
end
end
protected
def request(method, url, **options)
if @cache.present?
cached_request(method, url, **options)
else
raw_request(method, url, **options)
end
rescue HTTP::TimeoutError
# return a synthetic http error on connection timeouts
::HTTP::Response.new(status: 522, body: "", version: "1.1")
end
def cached_request(method, url, **options)
key = Cache.hash({ method: method, url: url, headers: http.default_options.headers.to_h, **options }.to_json)
cached_response = Cache.get(key, @cache) do
response = raw_request(method, url, **options)
{ status: response.status, body: response.to_s, headers: response.headers.to_h, version: "1.1" }
end
::HTTP::Response.new(**cached_response)
end
def raw_request(method, url, **options)
http.send(method, url, **options)
rescue OpenSSL::SSL::SSLError
fake_response(590, "")
rescue ValidatingSocket::ProhibitedIpError
fake_response(591, "")
rescue HTTP::Redirector::TooManyRedirectsError
fake_response(596, "")
rescue HTTP::TimeoutError
fake_response(597, "")
rescue HTTP::ConnectionError
fake_response(598, "")
rescue HTTP::Error
fake_response(599, "")
end
def http
@http ||= ::HTTP.timeout(DEFAULT_TIMEOUT).use(:auto_inflate).headers(Danbooru.config.http_headers).headers("Accept-Encoding" => "gzip")
def fake_response(status, body)
::HTTP::Response.new(status: status, version: "1.1", body: ::HTTP::Response::Body.new(body))
end
end
end

31
app/logical/danbooru/http/application_client.rb Normal file
View File

@@ -0,0 +1,31 @@
# An extension to HTTP::Client that lets us write Rack-style middlewares that
# hook into the request/response cycle and override how requests are made. This
# works by extending http.rb's concept of features (HTTP::Feature) to give them
# a `perform` method that takes a http request and returns a http response.
# This can be used to intercept and modify requests and return arbitrary responses.
module Danbooru
class Http
class ApplicationClient < HTTP::Client
# Override `perform` to call the `perform` method on features first.
def perform(request, options)
features = options.features.values.reverse.select do |feature|
feature.respond_to?(:perform)
end
perform = proc { |req| super(req, options) }
callback_chain = features.reduce(perform) do |callback_chain, feature|
proc { |req| feature.perform(req, &callback_chain) }
end
callback_chain.call(request)
end
# Override `branch` to return an ApplicationClient instead of a
# HTTP::Client so that chaining works.
def branch(...)
ApplicationClient.new(...)
end
end
end
end

30
app/logical/danbooru/http/cache.rb Normal file
View File

@@ -0,0 +1,30 @@
module Danbooru
class Http
class Cache < HTTP::Feature
HTTP::Options.register_feature :cache, self
attr_reader :expires_in
def initialize(expires_in:)
@expires_in = expires_in
end
def perform(request, &block)
::Cache.get(cache_key(request), expires_in) do
response = yield request
# XXX hack to remove connection state from response body so we can serialize it for caching.
response.flush
response.body.instance_variable_set(:@connection, nil)
response.body.instance_variable_set(:@stream, nil)
response
end
end
def cache_key(request)
"http:" + ::Cache.hash({ method: request.verb, url: request.uri.to_s, headers: request.headers.sort }.to_json)
end
end
end
end

12
app/logical/danbooru/http/html_adapter.rb Normal file
View File

@@ -0,0 +1,12 @@
module Danbooru
class Http
class HtmlAdapter < HTTP::MimeType::Adapter
HTTP::MimeType.register_adapter "text/html", self
HTTP::MimeType.register_alias "text/html", :html
def decode(str)
Nokogiri::HTML5(str)
end
end
end
end

40
app/logical/danbooru/http/redirector.rb Normal file
View File

@@ -0,0 +1,40 @@
# A HTTP::Feature that automatically follows HTTP redirects.
# https://developer.mozilla.org/en-US/docs/Web/HTTP/Redirections
module Danbooru
class Http
class Redirector < HTTP::Feature
HTTP::Options.register_feature :redirector, self
attr_reader :max_redirects
def initialize(max_redirects: 5)
@max_redirects = max_redirects
end
def perform(request, &block)
response = yield request
redirects = max_redirects
while response.status.redirect?
raise HTTP::Redirector::TooManyRedirectsError if redirects <= 0
response = yield build_redirect(request, response)
redirects -= 1
end
response
end
def build_redirect(request, response)
location = response.headers["Location"].to_s
uri = HTTP::URI.parse(location)
verb = request.verb
verb = :get if response.status == 303 && !request.verb.in?([:get, :head])
request.redirect(uri, verb)
end
end
end
end

54
app/logical/danbooru/http/retriable.rb Normal file
View File

@@ -0,0 +1,54 @@
# A HTTP::Feature that automatically retries requests that return a 429 error
# or a Retry-After header. Usage: `Danbooru::Http.use(:retriable).get(url)`.
#
# https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/429
# https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
module Danbooru
class Http
class Retriable < HTTP::Feature
HTTP::Options.register_feature :retriable, self
attr_reader :max_retries, :max_delay
def initialize(max_retries: 2, max_delay: 5.seconds)
@max_retries = max_retries
@max_delay = max_delay
end
def perform(request, &block)
response = yield request
retries = max_retries
while retriable?(response) && retries > 0 && retry_delay(response) <= max_delay
DanbooruLogger.info "Retrying url=#{request.uri} status=#{response.status} retries=#{retries} delay=#{retry_delay(response)}"
retries -= 1
sleep(retry_delay(response))
response = yield request
end
response
end
def retriable?(response)
response.status == 429 || response.headers["Retry-After"].present?
end
def retry_delay(response, current_time: Time.zone.now)
retry_after = response.headers["Retry-After"]
if retry_after.blank?
0.seconds
elsif retry_after =~ /\A\d+\z/
retry_after.to_i.seconds
else
retry_at = Time.zone.parse(retry_after)
return 0.seconds if retry_at.blank?
[retry_at - current_time, 0].max.seconds
end
end
end
end
end

37
app/logical/danbooru/http/session.rb Normal file
View File

@@ -0,0 +1,37 @@
module Danbooru
class Http
class Session < HTTP::Feature
HTTP::Options.register_feature :session, self
attr_reader :cookie_jar
def initialize(cookie_jar: HTTP::CookieJar.new)
@cookie_jar = cookie_jar
end
def perform(request)
add_cookies(request)
response = yield request
save_cookies(response)
response
end
def add_cookies(request)
cookies = cookies_for_request(request)
request.headers["Cookie"] = cookies if cookies.present?
end
def cookies_for_request(request)
saved_cookies = cookie_jar.each(request.uri).map { |c| [c.name, c.value] }.to_h
request_cookies = HTTP::Cookie.cookie_value_to_hash(request.headers["Cookie"].to_s)
saved_cookies.merge(request_cookies).map { |name, value| "#{name}=#{value}" }.join("; ")
end
def save_cookies(response)
response.cookies.each do |cookie|
cookie_jar.add(cookie)
end
end
end
end
end

13
app/logical/danbooru/http/spoof_referrer.rb Normal file
View File

@@ -0,0 +1,13 @@
module Danbooru
class Http
class SpoofReferrer < HTTP::Feature
HTTP::Options.register_feature :spoof_referrer, self
def perform(request, &block)
request.headers["Referer"] = request.uri.origin unless request.headers["Referer"].present?
response = yield request
response
end
end
end
end

20
app/logical/danbooru/http/unpolish_cloudflare.rb Normal file
View File

@@ -0,0 +1,20 @@
# Bypass Cloudflare Polish (https://support.cloudflare.com/hc/en-us/articles/360000607372-Using-Cloudflare-Polish-to-compress-images)
module Danbooru
class Http
class UnpolishCloudflare < HTTP::Feature
HTTP::Options.register_feature :unpolish_cloudflare, self
def perform(request, &block)
response = yield request
if response.headers["CF-Polished"].present?
request.uri.query_values = request.uri.query_values.to_h.merge(danbooru_no_polish: SecureRandom.uuid)
response = yield request
end
response
end
end
end
end

12
app/logical/danbooru/http/xml_adapter.rb Normal file
View File

@@ -0,0 +1,12 @@
module Danbooru
class Http
class XmlAdapter < HTTP::MimeType::Adapter
HTTP::MimeType.register_adapter "application/xml", self
HTTP::MimeType.register_alias "application/xml", :xml
def decode(str)
Hash.from_xml(str).with_indifferent_access
end
end
end
end

3
app/logical/danbooru_maintenance.rb
View File

@@ -2,14 +2,13 @@ module DanbooruMaintenance
module_function
def hourly
safely { Upload.prune! }
end
def daily
safely { PostPruner.new.prune! }
safely { Upload.prune! }
safely { Delayed::Job.where('created_at < ?', 45.days.ago).delete_all }
safely { PostDisapproval.prune! }
safely { PostDisapproval.dmail_messages! }
safely { regenerate_post_counts! }
safely { TokenBucket.prune! }
safely { BulkUpdateRequestPruner.warn_old }

123
app/logical/downloads/file.rb
View File

@@ -1,123 +0,0 @@
require 'resolv'
module Downloads
class File
include ActiveModel::Validations
class Error < StandardError; end
RETRIABLE_ERRORS = [Errno::ECONNRESET, Errno::ETIMEDOUT, Errno::EIO, Errno::EHOSTUNREACH, Errno::ECONNREFUSED, Timeout::Error, IOError]
delegate :data, to: :strategy
attr_reader :url, :referer
validate :validate_url
def initialize(url, referer = nil)
@url = Addressable::URI.parse(url) rescue nil
@referer = referer
validate!
end
def size
res = HTTParty.head(uncached_url, **httparty_options, timeout: 3)
if res.success?
res.content_length
else
raise HTTParty::ResponseError.new(res)
end
end
def download!(url: uncached_url, tries: 3, **options)
Retriable.retriable(on: RETRIABLE_ERRORS, tries: tries, base_interval: 0) do
file = http_get_streaming(url, headers: strategy.headers, **options)
return [file, strategy]
end
end
def validate_url
errors[:base] << "URL must not be blank" if url.blank?
errors[:base] << "'#{url}' is not a valid url" if !url.host.present?
errors[:base] << "'#{url}' is not a valid url. Did you mean 'http://#{url}'?" if !url.scheme.in?(%w[http https])
end
def http_get_streaming(url, file: Tempfile.new(binmode: true), headers: {}, max_size: Danbooru.config.max_file_size)
size = 0
res = HTTParty.get(url, httparty_options) do |chunk|
next if chunk.code == 302
size += chunk.size
raise Error.new("File is too large (max size: #{max_size})") if size > max_size && max_size > 0
file.write(chunk)
end
if res.success?
file.rewind
return file
else
raise Error.new("HTTP error code: #{res.code} #{res.message}")
end
end
# Prevent Cloudflare from potentially mangling the image. See issue #3528.
def uncached_url
return file_url unless is_cloudflare?(file_url)
url = file_url.dup
url.query_values = url.query_values.to_h.merge(danbooru_no_cache: SecureRandom.uuid)
url
end
def preview_url
@preview_url ||= Addressable::URI.parse(strategy.preview_url)
end
def file_url
@file_url ||= Addressable::URI.parse(strategy.image_url)
end
def strategy
@strategy ||= Sources::Strategies.find(url.to_s, referer)
end
def httparty_options
{
timeout: 10,
stream_body: true,
headers: strategy.headers,
connection_adapter: ValidatingConnectionAdapter
}.deep_merge(Danbooru.config.httparty_options)
end
def is_cloudflare?(url)
ip_addr = IPAddr.new(Resolv.getaddress(url.hostname))
CloudflareService.new.ips.any? { |subnet| subnet.include?(ip_addr) }
end
def self.banned_ip?(ip)
ip = IPAddress.parse(ip.to_s) unless ip.is_a?(IPAddress)
if ip.ipv4?
ip.loopback? || ip.link_local? || ip.multicast? || ip.private?
elsif ip.ipv6?
ip.loopback? || ip.link_local? || ip.unique_local? || ip.unspecified?
end
end
end
# Hook into HTTParty to validate the IP before following redirects.
# https://www.rubydoc.info/github/jnunemaker/httparty/HTTParty/ConnectionAdapter
class ValidatingConnectionAdapter < HTTParty::ConnectionAdapter
def self.call(uri, options)
ip_addr = IPAddress.parse(::Resolv.getaddress(uri.hostname))
if Downloads::File.banned_ip?(ip_addr)
raise Downloads::File::Error, "Downloads from #{ip_addr} are not allowed"
end
super(uri, options)
end
end
end

30
app/logical/dtext_input.rb Normal file
View File

@@ -0,0 +1,30 @@
# A custom SimpleForm input for DText fields.
#
# Usage:
#
# <%= f.input :body, as: :dtext %>
# <%= f.input :reason, as: :dtext, inline: true %>
#
# https://github.com/heartcombo/simple_form/wiki/Custom-inputs-examples
# https://github.com/heartcombo/simple_form/blob/master/lib/simple_form/inputs/string_input.rb
# https://github.com/heartcombo/simple_form/blob/master/lib/simple_form/inputs/text_input.rb
class DtextInput < SimpleForm::Inputs::Base
enable :placeholder, :maxlength, :minlength
def input(wrapper_options)
t = template
merged_input_options = merge_wrapper_options(input_html_options, wrapper_options)
t.tag.div(class: "dtext-previewable") do
if options[:inline]
t.concat @builder.text_field(attribute_name, merged_input_options)
else
t.concat @builder.text_area(attribute_name, { rows: 20, cols: 30 }.merge(merged_input_options))
end
t.concat t.tag.div(id: "dtext-preview", class: "dtext-preview prose")
t.concat t.tag.span(t.link_to("Formatting help", t.dtext_help_path, remote: true, method: :get), class: "hint dtext-hint")
end
end
end

20
app/logical/image_proxy.rb
View File

@@ -1,4 +1,6 @@
class ImageProxy
class Error < StandardError; end
def self.needs_proxy?(url)
fake_referer_for(url).present?
end
@@ -8,19 +10,13 @@ class ImageProxy
end
def self.get_image(url)
if url.blank?
raise "Must specify url"
end
raise Error, "URL not present" unless url.present?
raise Error, "Proxy not allowed for this url (url=#{url})" unless needs_proxy?(url)
if !needs_proxy?(url)
raise "Proxy not allowed for this site"
end
referer = fake_referer_for(url)
response = Danbooru::Http.timeout(30).headers(Referer: referer).get(url)
raise Error, "Couldn't proxy image (code=#{response.status}, url=#{url})" unless response.status.success?
response = HTTParty.get(url, Danbooru.config.httparty_options.deep_merge(headers: {"Referer" => fake_referer_for(url)}))
if response.success?
return response
else
raise "HTTP error code: #{response.code} #{response.message}"
end
response
end
end

2
app/logical/ip_lookup.rb
View File

@@ -16,7 +16,7 @@ class IpLookup
end
def info
return {} unless api_key.present?
return {} if api_key.blank?
response = Danbooru::Http.cache(cache_duration).get("https://api.ipregistry.co/#{ip_addr}?key=#{api_key}")
return {} if response.status != 200
json = response.parse.deep_symbolize_keys.with_indifferent_access

43
app/logical/iqdb_proxy.rb
View File

@@ -1,19 +1,24 @@
class IqdbProxy
class Error < StandardError; end
attr_reader :http, :iqdbs_server
def self.enabled?
Danbooru.config.iqdbs_server.present?
def initialize(http: Danbooru::Http.new, iqdbs_server: Danbooru.config.iqdbs_server)
@iqdbs_server = iqdbs_server
@http = http
end
def self.download(url, type)
download = Downloads::File.new(url)
file, strategy = download.download!(url: download.send(type))
def enabled?
iqdbs_server.present?
end
def download(url, type)
strategy = Sources::Strategies.find(url)
download_url = strategy.send(type)
file = strategy.download_file!(download_url)
file
end
def self.search(params)
raise NotImplementedError, "the IQDBs service isn't configured" unless enabled?
def search(params)
limit = params[:limit]&.to_i&.clamp(1, 1000) || 20
similarity = params[:similarity]&.to_f&.clamp(0.0, 100.0) || 0.0
high_similarity = params[:high_similarity]&.to_f&.clamp(0.0, 100.0) || 65.0
@@ -28,7 +33,7 @@ class IqdbProxy
file = download(params[:image_url], :url)
results = query(file: file, limit: limit)
elsif params[:file_url].present?
file = download(params[:file_url], :file_url)
file = download(params[:file_url], :image_url)
results = query(file: file, limit: limit)
elsif params[:post_id].present?
url = Post.find(params[:post_id]).preview_file_url
@@ -46,15 +51,21 @@ class IqdbProxy
file.try(:close)
end
def self.query(params)
response = HTTParty.post("#{Danbooru.config.iqdbs_server}/similar", body: params, **Danbooru.config.httparty_options)
raise Error, "IQDB error: #{response.code} #{response.message}" unless response.success?
raise Error, "IQDB error: #{response.parsed_response["error"]}" if response.parsed_response.is_a?(Hash)
raise Error, "IQDB error: #{response.parsed_response.first}" if response.parsed_response.try(:first).is_a?(String)
response.parsed_response
def query(file: nil, url: nil, limit: 20)
raise NotImplementedError, "the IQDBs service isn't configured" unless enabled?
file = HTTP::FormData::File.new(file) if file
form = { file: file, url: url, limit: limit }.compact
response = http.timeout(30).post("#{iqdbs_server}/similar", form: form)
raise Error, "IQDB error: #{response.status}" if response.status != 200
raise Error, "IQDB error: #{response.parse["error"]}" if response.parse.is_a?(Hash)
raise Error, "IQDB error: #{response.parse.first}" if response.parse.try(:first).is_a?(String)
response.parse
end
def self.decorate_posts(json)
def decorate_posts(json)
post_ids = json.map { |match| match["post_id"] }
posts = Post.where(id: post_ids).group_by(&:id).transform_values(&:first)

2
app/logical/media_file.rb
View File

@@ -43,6 +43,8 @@ class MediaFile
else
:bin
end
rescue EOFError
:bin
end
def self.videos_enabled?

12
app/logical/media_file/flash.rb
View File

@@ -42,7 +42,7 @@ class MediaFile::Flash < MediaFile
signature = contents[0..2]
# SWF version
version = contents[3].unpack('C').join.to_i
_version = contents[3].unpack('C').join.to_i
# Determine the length of the uncompressed stream
length = contents[4..7].unpack('V').join.to_i
@@ -50,7 +50,7 @@ class MediaFile::Flash < MediaFile
# If we do, in fact, have compression
if signature == 'CWS'
# Decompress the body of the SWF
body = Zlib::Inflate.inflate( contents[8..length] )
body = Zlib::Inflate.inflate(contents[8..length])
# And reconstruct the stream contents to the first 8 bytes (header)
# Plus our decompressed body
@@ -58,10 +58,10 @@ class MediaFile::Flash < MediaFile
end
# Determine the nbits of our dimensions rectangle
nbits = contents.unpack('C'*contents.length)[8] >> 3
nbits = contents.unpack('C' * contents.length)[8] >> 3
# Determine how many bits long this entire RECT structure is
rectbits = 5 + nbits * 4 # 5 bits for nbits, as well as nbits * number of fields (4)
rectbits = 5 + nbits * 4 # 5 bits for nbits, as well as nbits * number of fields (4)
# Determine how many bytes rectbits composes (ceil(rectbits/8))
rectbytes = (rectbits.to_f / 8).ceil
@@ -70,11 +70,11 @@ class MediaFile::Flash < MediaFile
rect = contents[8..(8 + rectbytes)].unpack("#{'B8' * rectbytes}").join
# Read in nbits incremenets starting from 5
dimensions = Array.new
dimensions = []
4.times do |n|
s = 5 + (n * nbits) # Calculate our start index
e = s + (nbits - 1) # Calculate our end index
dimensions[n] = rect[s..e].to_i(2) # Read that range (binary) and convert it to an integer
dimensions[n] = rect[s..e].to_i(2) # Read that range (binary) and convert it to an integer
end
# The values we have here are in "twips"

29
app/logical/missed_search_service.rb
View File

@@ -1,29 +0,0 @@
# queries reportbooru to find missed post searches
class MissedSearchService
def self.enabled?
Danbooru.config.reportbooru_server.present?
end
def initialize
if !MissedSearchService.enabled?
raise NotImplementedError.new("the Reportbooru service isn't configured. Missed searches are not available.")
end
end
def each_search(&block)
fetch_data.scan(/(.+?) (\d+)\.0\n/).each(&block)
end
def fetch_data
Cache.get("ms", 1.minute) do
url = URI.parse("#{Danbooru.config.reportbooru_server}/missed_searches")
response = HTTParty.get(url, Danbooru.config.httparty_options.reverse_merge(timeout: 6))
if response.success?
response = response.body
else
response = ""
end
response.force_encoding("utf-8")
end
end
end

137
app/logical/nico_seiga_api_client.rb
View File

@@ -1,83 +1,102 @@
class NicoSeigaApiClient
extend Memoist
BASE_URL = "http://seiga.nicovideo.jp/api"
attr_reader :illust_id
XML_API = "https://seiga.nicovideo.jp/api"
def self.agent
mech = Mechanize.new
mech.redirect_ok = false
mech.keep_alive = false
attr_reader :http
session = Cache.get("nico-seiga-session")
if session
cookie = Mechanize::Cookie.new("user_session", session)
cookie.domain = ".nicovideo.jp"
cookie.path = "/"
mech.cookie_jar.add(cookie)
else
mech.get("https://account.nicovideo.jp/login") do |page|
page.form_with(:id => "login_form") do |form|
form["mail_tel"] = Danbooru.config.nico_seiga_login
form["password"] = Danbooru.config.nico_seiga_password
end.click_button
end
session = mech.cookie_jar.cookies.select {|c| c.name == "user_session"}.first
if session
Cache.put("nico-seiga-session", session.value, 1.week)
else
raise "Session not found"
def initialize(work_id:, type:, http: Danbooru::Http.new)
@work_id = work_id
@work_type = type
@http = http
end
def image_ids
if @work_type == "illust"
[api_response["id"]]
elsif @work_type == "manga"
manga_api_response.map do |x|
case x["meta"]["source_url"]
when %r{/thumb/(\d+)\w}i then Regexp.last_match(1)
when %r{nicoseiga\.cdn\.nimg\.jp/drm/image/\w+/(\d+)\w}i then Regexp.last_match(1)
end
end
end
# This cookie needs to be set to allow viewing of adult works
cookie = Mechanize::Cookie.new("skip_fetish_warning", "1")
cookie.domain = "seiga.nicovideo.jp"
cookie.path = "/"
mech.cookie_jar.add(cookie)
mech.redirect_ok = true
mech
end
def initialize(illust_id:, user_id: nil)
@illust_id = illust_id
@user_id = user_id
end
def image_id
illust_xml["response"]["image"]["id"].to_i
end
def user_id
@user_id || illust_xml["response"]["image"]["user_id"].to_i
end
def title
illust_xml["response"]["image"]["title"]
api_response["title"]
end
def desc
illust_xml["response"]["image"]["description"]
def description
api_response["description"]
end
def moniker
artist_xml["response"]["user"]["nickname"]
def tags
api_response.dig("tag_list", "tag").to_a.map { |t| t["name"] }.compact
end
def illust_xml
get("#{BASE_URL}/illust/info?id=#{illust_id}")
def user_id
api_response["user_id"]
end
def artist_xml
get("#{BASE_URL}/user/info?id=#{user_id}")
def user_name
if @work_type == "illust"
api_response["nickname"]
elsif @work_type == "manga"
user_api_response(user_id)["nickname"]
end
end
def api_response
if @work_type == "illust"
resp = get("https://sp.seiga.nicovideo.jp/ajax/seiga/im#{@work_id}")
return {} if resp.blank? || resp.code.to_i == 404
api_response = JSON.parse(resp)["target_image"]
elsif @work_type == "manga"
resp = http.cache(1.minute).get("#{XML_API}/theme/info?id=#{@work_id}")
return {} if resp.blank? || resp.code.to_i == 404
api_response = Hash.from_xml(resp.to_s)["response"]["theme"]
end
api_response || {}
rescue JSON::ParserError
{}
end
def manga_api_response
resp = get("https://ssl.seiga.nicovideo.jp/api/v1/app/manga/episodes/#{@work_id}/frames")
return {} if resp.blank? || resp.code.to_i == 404
JSON.parse(resp)["data"]["result"]
rescue JSON::ParserError
{}
end
def user_api_response(user_id)
resp = http.cache(1.minute).get("#{XML_API}/user/info?id=#{user_id}")
return {} if resp.blank? || resp.code.to_i == 404
Hash.from_xml(resp.to_s)["response"]["user"]
end
def login
form = {
mail_tel: Danbooru.config.nico_seiga_login,
password: Danbooru.config.nico_seiga_password
}
# XXX should fail gracefully instead of raising exception
resp = http.cache(1.hour).post("https://account.nicovideo.jp/login/redirector?site=seiga", form: form)
raise RuntimeError, "NicoSeiga login failed (status=#{resp.status} url=#{url})" if resp.status != 200
http
end
def get(url)
response = Danbooru::Http.cache(1.minute).get(url)
raise "nico seiga api call failed (code=#{response.code}, body=#{response.body})" if response.code != 200
resp = login.cache(1.minute).get(url)
#raise RuntimeError, "NicoSeiga get failed (status=#{resp.status} url=#{url})" if resp.status != 200
Hash.from_xml(response.to_s)
resp
end
memoize :artist_xml, :illust_xml
memoize :api_response, :manga_api_response, :user_api_response
end

60
app/logical/nico_seiga_manga_api_client.rb
View File

@@ -1,60 +0,0 @@
class NicoSeigaMangaApiClient
extend Memoist
BASE_URL = "https://seiga.nicovideo.jp/api"
attr_reader :theme_id
def initialize(theme_id)
@theme_id = theme_id
end
def user_id
theme_info_xml["response"]["theme"]["user_id"].to_i
end
def title
theme_info_xml["response"]["theme"]["title"]
end
def desc
theme_info_xml["response"]["theme"]["description"]
end
def moniker
artist_xml["response"]["user"]["nickname"]
end
def image_ids
images = theme_data_xml["response"]["image_list"]["image"]
images = [images] unless images.is_a?(Array)
images.map {|x| x["id"]}
end
def tags
theme_info_xml["response"]["theme"]["tag_list"]["tag"].map {|x| x["name"]}
end
def theme_data_xml
uri = "#{BASE_URL}/theme/data?theme_id=#{theme_id}"
body = NicoSeigaApiClient.agent.get(uri).body
Hash.from_xml(body)
end
def theme_info_xml
uri = "#{BASE_URL}/theme/info?id=#{theme_id}"
body = NicoSeigaApiClient.agent.get(uri).body
Hash.from_xml(body)
end
def artist_xml
get("#{BASE_URL}/user/info?id=#{user_id}")
end
def get(url)
response = Danbooru::Http.cache(1.minute).get(url)
raise "nico seiga api call failed (code=#{response.code}, body=#{response.body})" if response.code != 200
Hash.from_xml(response.to_s)
end
memoize :theme_data_xml, :theme_info_xml, :artist_xml
end

39
app/logical/pagination_extension.rb
View File

@@ -3,9 +3,9 @@ module PaginationExtension
attr_accessor :current_page, :records_per_page, :paginator_count, :paginator_mode
def paginate(page, limit: nil, count: nil, search_count: nil)
def paginate(page, limit: nil, max_limit: 1000, count: nil, search_count: nil)
@records_per_page = limit || Danbooru.config.posts_per_page
@records_per_page = @records_per_page.to_i.clamp(1, 1000)
@records_per_page = @records_per_page.to_i.clamp(1, max_limit)
if count.present?
@paginator_count = count
@@ -61,27 +61,35 @@ module PaginationExtension
end
def prev_page
return nil if is_first_page?
if paginator_mode == :numbered
if is_first_page?
nil
elsif paginator_mode == :numbered
current_page - 1
elsif paginator_mode == :sequential_before
elsif paginator_mode == :sequential_before && records.present?
"a#{records.first.id}"
elsif paginator_mode == :sequential_after
elsif paginator_mode == :sequential_after && records.present?
"b#{records.last.id}"
else
nil
end
rescue ActiveRecord::QueryCanceled
nil
end
def next_page
return nil if is_last_page?
if paginator_mode == :numbered
if is_last_page?
nil
elsif paginator_mode == :numbered
current_page + 1
elsif paginator_mode == :sequential_before
elsif paginator_mode == :sequential_before && records.present?
"b#{records.last.id}"
elsif paginator_mode == :sequential_after
elsif paginator_mode == :sequential_after && records.present?
"a#{records.first.id}"
else
nil
end
rescue ActiveRecord::QueryCanceled
nil
end
# XXX Hack: in sequential pagination we fetch one more record than we
@@ -106,10 +114,7 @@ module PaginationExtension
def total_count
@paginator_count ||= unscoped.from(except(:offset, :limit, :order).reorder(nil)).count
rescue ActiveRecord::StatementInvalid => e
if e.to_s =~ /statement timeout/
@paginator_count ||= 1_000_000
else
raise
end
raise unless e.to_s =~ /statement timeout/
@paginator_count ||= 1_000_000
end
end

6
app/logical/pawoo_api_client.rb
View File

@@ -128,15 +128,15 @@ class PawooApiClient
rescue
data = {}
end
return Account.new(data)
Account.new(data)
end
end
private
def fetch_access_token
raise MissingConfigurationError.new("missing pawoo client id") if Danbooru.config.pawoo_client_id.nil?
raise MissingConfigurationError.new("missing pawoo client secret") if Danbooru.config.pawoo_client_secret.nil?
raise MissingConfigurationError, "missing pawoo client id" if Danbooru.config.pawoo_client_id.nil?
raise MissingConfigurationError, "missing pawoo client secret" if Danbooru.config.pawoo_client_secret.nil?
Cache.get("pawoo-token") do
result = client.client_credentials.get_token

148
app/logical/pixiv_api_client.rb
View File

@@ -1,5 +1,3 @@
require 'resolv-replace'
class PixivApiClient
extend Memoist
@@ -114,66 +112,13 @@ class PixivApiClient
end
end
class FanboxResponse
attr_reader :json
def initialize(json)
@json = json
end
def name
json["body"]["user"]["name"]
end
def user_id
json["body"]["user"]["userId"]
end
def moniker
""
end
def page_count
json["body"]["body"]["images"].size
end
def artist_commentary_title
json["body"]["title"]
end
def artist_commentary_desc
json["body"]["body"]["text"]
end
def tags
[]
end
def pages
if json["body"]["body"]
json["body"]["body"]["images"].map {|x| x["originalUrl"]}
else
[]
end
end
end
def work(illust_id)
headers = Danbooru.config.http_headers.merge(
"Referer" => "http://www.pixiv.net",
"Content-Type" => "application/x-www-form-urlencoded",
"Authorization" => "Bearer #{access_token}"
)
params = {
"image_sizes" => "large",
"include_stats" => "true"
}
params = { image_sizes: "large", include_stats: "true" }
url = "https://public-api.secure.pixiv.net/v#{API_VERSION}/works/#{illust_id.to_i}.json"
response = Danbooru::Http.cache(1.minute).headers(headers).get(url, params: params)
response = api_client.cache(1.minute).get(url, params: params)
json = response.parse
if response.code == 200
if response.status == 200
WorkResponse.new(json["response"][0])
elsif json["status"] == "failure" && json.dig("errors", "system", "message") =~ /対象のイラストは見つかりませんでした。/
raise BadIDError.new("Pixiv ##{illust_id} not found: work was deleted, made private, or ID is invalid.")
@@ -184,32 +129,12 @@ class PixivApiClient
raise Error.new("Pixiv API call failed (status=#{response.code} body=#{response.body})")
end
def fanbox(fanbox_id)
url = "https://www.pixiv.net/ajax/fanbox/post?postId=#{fanbox_id.to_i}"
resp = agent.get(url)
json = JSON.parse(resp.body)
if resp.code == "200"
FanboxResponse.new(json)
elsif json["status"] == "failure"
raise Error.new("Pixiv API call failed (status=#{resp.code} body=#{body})")
end
rescue JSON::ParserError
raise Error.new("Pixiv API call failed (status=#{resp.code} body=#{body})")
end
def novel(novel_id)
headers = Danbooru.config.http_headers.merge(
"Referer" => "http://www.pixiv.net",
"Content-Type" => "application/x-www-form-urlencoded",
"Authorization" => "Bearer #{access_token}"
)
url = "https://public-api.secure.pixiv.net/v#{API_VERSION}/novels/#{novel_id.to_i}.json"
resp = HTTParty.get(url, Danbooru.config.httparty_options.deep_merge(headers: headers))
body = resp.body.force_encoding("utf-8")
json = JSON.parse(body)
resp = api_client.cache(1.minute).get(url)
json = resp.parse
if resp.success?
if resp.status == 200
NovelResponse.new(json["response"][0])
elsif json["status"] == "failure" && json.dig("errors", "system", "message") =~ /対象のイラストは見つかりませんでした。/
raise Error.new("Pixiv API call failed (status=#{resp.code} body=#{body})")
@@ -219,42 +144,41 @@ class PixivApiClient
end
def access_token
Cache.get("pixiv-papi-access-token", 3000) do
access_token = nil
# truncate timestamp to 1-hour resolution so that it doesn't break caching.
client_time = Time.zone.now.utc.change(min: 0).rfc3339
client_hash = Digest::MD5.hexdigest(client_time + CLIENT_HASH_SALT)
client_time = Time.now.rfc3339
client_hash = Digest::MD5.hexdigest(client_time + CLIENT_HASH_SALT)
headers = {
"Referer": "http://www.pixiv.net",
"X-Client-Time": client_time,
"X-Client-Hash": client_hash
}
headers = {
"Referer": "http://www.pixiv.net",
"X-Client-Time": client_time,
"X-Client-Hash": client_hash
}
params = {
username: Danbooru.config.pixiv_login,
password: Danbooru.config.pixiv_password,
grant_type: "password",
client_id: CLIENT_ID,
client_secret: CLIENT_SECRET
}
url = "https://oauth.secure.pixiv.net/auth/token"
params = {
username: Danbooru.config.pixiv_login,
password: Danbooru.config.pixiv_password,
grant_type: "password",
client_id: CLIENT_ID,
client_secret: CLIENT_SECRET
}
resp = HTTParty.post(url, Danbooru.config.httparty_options.deep_merge(body: params, headers: headers))
body = resp.body.force_encoding("utf-8")
resp = http.headers(headers).cache(1.hour).post("https://oauth.secure.pixiv.net/auth/token", form: params)
return nil unless resp.status == 200
if resp.success?
json = JSON.parse(body)
access_token = json["response"]["access_token"]
else
raise Error.new("Pixiv API access token call failed (status=#{resp.code} body=#{body})")
end
access_token
end
resp.parse.dig("response", "access_token")
end
def agent
PixivWebAgent.build
def api_client
http.headers(
"Referer": "http://www.pixiv.net",
"Content-Type": "application/x-www-form-urlencoded",
"Authorization": "Bearer #{access_token}"
)
end
memoize :agent
def http
Danbooru::Http.new
end
memoize :access_token, :api_client, :http
end

74
app/logical/pixiv_web_agent.rb
View File

@@ -1,74 +0,0 @@
class PixivWebAgent
SESSION_CACHE_KEY = "pixiv-phpsessid"
COMIC_SESSION_CACHE_KEY = "pixiv-comicsessid"
SESSION_COOKIE_KEY = "PHPSESSID"
COMIC_SESSION_COOKIE_KEY = "_pixiv-comic_session"
def self.phpsessid(agent)
agent.cookies.select { |cookie| cookie.name == SESSION_COOKIE_KEY }.first.try(:value)
end
def self.build
mech = Mechanize.new
mech.keep_alive = false
phpsessid = Cache.get(SESSION_CACHE_KEY)
comicsessid = Cache.get(COMIC_SESSION_CACHE_KEY)
if phpsessid
cookie = Mechanize::Cookie.new(SESSION_COOKIE_KEY, phpsessid)
cookie.domain = ".pixiv.net"
cookie.path = "/"
mech.cookie_jar.add(cookie)
if comicsessid
cookie = Mechanize::Cookie.new(COMIC_SESSION_COOKIE_KEY, comicsessid)
cookie.domain = ".pixiv.net"
cookie.path = "/"
mech.cookie_jar.add(cookie)
end
else
headers = {
"Origin" => "https://accounts.pixiv.net",
"Referer" => "https://accounts.pixiv.net/login?lang=en^source=pc&view_type=page&ref=wwwtop_accounts_index"
}
params = {
pixiv_id: Danbooru.config.pixiv_login,
password: Danbooru.config.pixiv_password,
captcha: nil,
g_captcha_response: nil,
source: "pc",
post_key: nil
}
mech.get("https://accounts.pixiv.net/login?lang=en&source=pc&view_type=page&ref=wwwtop_accounts_index") do |page|
json = page.search("input#init-config").first.attr("value")
if json =~ /pixivAccount\.postKey":"([a-f0-9]+)/
params[:post_key] = $1
end
end
mech.post("https://accounts.pixiv.net/api/login?lang=en", params, headers)
if mech.current_page.body =~ /"error":false/
cookie = mech.cookies.select {|x| x.name == SESSION_COOKIE_KEY}.first
if cookie
Cache.put(SESSION_CACHE_KEY, cookie.value, 1.week)
end
end
begin
mech.get("https://comic.pixiv.net") do |page|
cookie = mech.cookies.select {|x| x.name == COMIC_SESSION_COOKIE_KEY}.first
if cookie
Cache.put(COMIC_SESSION_CACHE_KEY, cookie.value, 1.week)
end
end
rescue Net::HTTPServiceUnavailable
# ignore
end
end
mech
end
end

61
app/logical/popular_search_service.rb
View File

@@ -1,61 +0,0 @@
# queries reportbooru to find popular post searches
class PopularSearchService
attr_reader :date
def self.enabled?
Danbooru.config.reportbooru_server.present?
end
def initialize(date)
if !PopularSearchService.enabled?
raise NotImplementedError.new("the Reportbooru service isn't configured. Popular searches are not available.")
end
@date = date
end
def each_search(limit = 100, &block)
JSON.parse(fetch_data.to_s).slice(0, limit).each(&block)
end
def tags
JSON.parse(fetch_data.to_s).map {|x| x[0]}
end
def fetch_data
return [] unless self.class.enabled?
dates = date.strftime("%Y-%m-%d")
data = Cache.get("ps-day-#{dates}", 1.minute) do
url = "#{Danbooru.config.reportbooru_server}/post_searches/rank?date=#{dates}"
response = HTTParty.get(url, Danbooru.config.httparty_options.reverse_merge(timeout: 3))
if response.success?
response = response.body
else
response = "[]"
end
response
end.to_s.force_encoding("utf-8")
if data.blank? || data == "[]"
dates = date.yesterday.strftime("%Y-%m-%d")
data = Cache.get("ps-day-#{dates}", 1.minute) do
url = "#{Danbooru.config.reportbooru_server}/post_searches/rank?date=#{dates}"
response = HTTParty.get(url, Danbooru.config.httparty_options.reverse_merge(timeout: 3))
if response.success?
response = response.body
else
response = "[]"
end
response
end.to_s.force_encoding("utf-8")
end
data
rescue StandardError => e
DanbooruLogger.log(e)
return []
end
end

4
app/logical/post_query_builder.rb
View File

@@ -307,6 +307,8 @@ class PostQueryBuilder
Post.where(parent: nil)
when "any"
Post.where.not(parent: nil)
when /pending|flagged|modqueue|deleted|banned|active|unmoderated/
Post.where.not(parent: nil).where(parent: status_matches(parent))
when /\A\d+\z/
Post.where(id: parent).or(Post.where(parent: parent))
else
@@ -320,6 +322,8 @@ class PostQueryBuilder
Post.where(has_children: false)
when "any"
Post.where(has_children: true)
when /pending|flagged|modqueue|deleted|banned|active|unmoderated/
Post.where(has_children: true).where(children: status_matches(child))
else
Post.none
end

29
app/logical/post_sets/post.rb
View File

@@ -24,9 +24,8 @@ module PostSets
end
def wiki_page
return nil unless tag.present? && tag.wiki_page.present?
return nil unless !tag.wiki_page.is_deleted?
tag.wiki_page
return nil unless normalized_query.has_single_tag?
@wiki_page ||= WikiPage.undeleted.find_by(title: normalized_query.tags.first.name)
end
def tag
@@ -77,7 +76,11 @@ module PostSets
end
def per_page
(@per_page || query.find_metatag(:limit) || CurrentUser.user.per_page).to_i.clamp(0, MAX_PER_PAGE)
(@per_page || query.find_metatag(:limit) || CurrentUser.user.per_page).to_i.clamp(0, max_per_page)
end
def max_per_page
(format == "sitemap") ? 10_000 : MAX_PER_PAGE
end
def is_random?
@@ -94,7 +97,7 @@ module PostSets
end
def get_random_posts
per_page.times.inject([]) do |all, x|
per_page.times.inject([]) do |all, _|
all << ::Post.user_tag_match(tag_string).random
end.compact.uniq
end
@@ -104,15 +107,15 @@ module PostSets
@post_count = get_post_count
if is_random?
temp = get_random_posts
get_random_posts
else
temp = normalized_query.build.paginate(page, count: post_count, search_count: !post_count.nil?, limit: per_page)
normalized_query.build.paginate(page, count: post_count, search_count: !post_count.nil?, limit: per_page, max_limit: max_per_page).load
end
end
end
def hide_from_crawler?
return true if current_page > 1
return true if current_page > 50
return false if query.is_empty_search? || query.is_simple_tag? || query.is_metatag?(:order, :rank)
true
end
@@ -160,20 +163,16 @@ module PostSets
elsif query.is_metatag?(:search)
saved_search_tags
elsif query.is_empty_search? || query.is_metatag?(:order, :rank)
popular_tags
popular_tags.presence || frequent_tags
elsif query.is_single_term?
similar_tags
similar_tags.presence || frequent_tags
else
frequent_tags
end
end
def popular_tags
if PopularSearchService.enabled?
PopularSearchService.new(Date.today).tags
else
frequent_tags
end
ReportbooruService.new.popular_searches(Date.today, limit: MAX_SIDEBAR_TAGS)
end
def similar_tags

38
app/logical/post_view_count_service.rb
View File

@@ -1,38 +0,0 @@
class PostViewCountService
def self.enabled?
Danbooru.config.reportbooru_server.present?
end
def initialize
if !PostViewCountService.enabled?
raise NotImplementedError.new("the Reportbooru service isn't configured. Post views are not available.")
end
end
def fetch_count(post_id)
url = URI.parse("#{Danbooru.config.reportbooru_server}/post_views/#{post_id}")
response = HTTParty.get(url, Danbooru.config.httparty_options.reverse_merge(timeout: 6))
if response.success?
return JSON.parse(response.body)
else
return nil
end
end
def fetch_rank(date = Date.today)
url = URI.parse("#{Danbooru.config.reportbooru_server}/post_views/rank?date=#{date}")
response = HTTParty.get(url, Danbooru.config.httparty_options.reverse_merge(timeout: 6))
if response.success?
return JSON.parse(response.body)
else
return nil
end
rescue JSON::ParserError
nil
end
def popular_posts(date = Date.today)
ranking = fetch_rank(date) || []
ranking.slice(0, 50).map {|x| Post.find(x[0])}
end
end

50
app/logical/reportbooru_service.rb Normal file
View File

@@ -0,0 +1,50 @@
class ReportbooruService
attr_reader :http, :reportbooru_server
def initialize(http: Danbooru::Http.new, reportbooru_server: Danbooru.config.reportbooru_server)
@reportbooru_server = reportbooru_server
@http = http
end
def enabled?
reportbooru_server.present?
end
def missed_search_rankings(expires_in: 1.minutes)
return [] unless enabled?
response = http.cache(expires_in).get("#{reportbooru_server}/missed_searches")
return [] if response.status != 200
body = response.to_s.force_encoding("utf-8")
body.lines.map(&:split).map { [_1, _2.to_i] }
end
def post_search_rankings(date, expires_in: 1.minutes)
request("#{reportbooru_server}/post_searches/rank?date=#{date}", expires_in)
end
def post_view_rankings(date, expires_in: 1.minutes)
request("#{reportbooru_server}/post_views/rank?date=#{date}", expires_in)
end
def popular_searches(date, limit: 100)
ranking = post_search_rankings(date)
ranking = post_search_rankings(date.yesterday) if ranking.blank?
ranking.take(limit).map(&:first)
end
def popular_posts(date, limit: 100)
ranking = post_view_rankings(date)
ranking = post_view_rankings(date.yesterday) if ranking.blank?
ranking.take(limit).map { |x| Post.find(x[0]) }
end
def request(url, expires_in)
return [] unless enabled?
response = http.cache(expires_in).get(url)
return [] if response.status != 200
JSON.parse(response.to_s.force_encoding("utf-8"))
end
end

5
app/logical/sources/strategies.rb
View File

@@ -1,7 +1,7 @@
module Sources
module Strategies
def self.all
return [
[
Strategies::Pixiv,
Strategies::NicoSeiga,
Strategies::Twitter,
@@ -13,7 +13,8 @@ module Sources
Strategies::Pawoo,
Strategies::Moebooru,
Strategies::HentaiFoundry,
Strategies::Weibo
Strategies::Weibo,
Strategies::Newgrounds
]
end

10
app/logical/sources/strategies/art_station.rb
View File

@@ -22,15 +22,15 @@
module Sources::Strategies
class ArtStation < Base
PROJECT1 = %r!\Ahttps?://www\.artstation\.com/artwork/(?<project_id>[a-z0-9-]+)/?\z!i
PROJECT2 = %r!\Ahttps?://(?<artist_name>[\w-]+)\.artstation\.com/projects/(?<project_id>[a-z0-9-]+)(?:/|\?[\w=-]+)?\z!i
PROJECT1 = %r{\Ahttps?://www\.artstation\.com/artwork/(?<project_id>[a-z0-9-]+)/?\z}i
PROJECT2 = %r{\Ahttps?://(?<artist_name>[\w-]+)\.artstation\.com/projects/(?<project_id>[a-z0-9-]+)(?:/|\?[\w=-]+)?\z}i
PROJECT = Regexp.union(PROJECT1, PROJECT2)
ARTIST1 = %r{\Ahttps?://(?<artist_name>[\w-]+)(?<!www)\.artstation\.com/?\z}i
ARTIST2 = %r{\Ahttps?://www\.artstation\.com/artist/(?<artist_name>[\w-]+)/?\z}i
ARTIST3 = %r{\Ahttps?://www\.artstation\.com/(?<artist_name>[\w-]+)/?\z}i
ARTIST = Regexp.union(ARTIST1, ARTIST2, ARTIST3)
ASSET = %r!\Ahttps?://cdn\w*\.artstation\.com/p/assets/(?<type>images|covers)/images/(?<id>\d+/\d+/\d+)/(?<size>[^/]+)/(?<filename>.+)\z!i
ASSET = %r{\Ahttps?://cdn\w*\.artstation\.com/p/assets/(?<type>images|covers)/images/(?<id>\d+/\d+/\d+)/(?<size>[^/]+)/(?<filename>.+)\z}i
attr_reader :json
@@ -144,10 +144,10 @@ module Sources::Strategies
urls = image_url_sizes($~[:type], $~[:id], $~[:filename])
if size == :smallest
urls = urls.reverse()
urls = urls.reverse
end
chosen_url = urls.find { |url| http_exists?(url, headers) }
chosen_url = urls.find { |url| http_exists?(url) }
chosen_url || url
end
end

56
app/logical/sources/strategies/base.rb
View File

@@ -14,6 +14,8 @@
module Sources
module Strategies
class Base
class DownloadError < StandardError; end
attr_reader :url, :referer_url, :urls, :parsed_url, :parsed_referer, :parsed_urls
extend Memoist
@@ -35,9 +37,9 @@ module Sources
# <tt>referrer_url</tt> so the strategy can discover the HTML
# page and other information.
def initialize(url, referer_url = nil)
@url = url
@referer_url = referer_url
@urls = [url, referer_url].select(&:present?)
@url = url.to_s
@referer_url = referer_url&.to_s
@urls = [@url, @referer_url].select(&:present?)
@parsed_url = Addressable::URI.heuristic_parse(url) rescue nil
@parsed_referer = Addressable::URI.heuristic_parse(referer_url) rescue nil
@@ -58,8 +60,8 @@ module Sources
end
def site_name
Addressable::URI.heuristic_parse(url).host
rescue Addressable::URI::InvalidURIError => e
Addressable::URI.heuristic_parse(url)&.host
rescue Addressable::URI::InvalidURIError
nil
end
@@ -90,9 +92,7 @@ module Sources
# eventually be assigned as the source for the post, but it does not
# represent what the downloader will fetch.
def page_url
Rails.logger.warn "Valid page url for (#{url}, #{referer_url}) not found"
return nil
nil
end
# This will be the url stored in posts. Typically this is the page
@@ -141,14 +141,37 @@ module Sources
# Subclasses should merge in any required headers needed to access resources
# on the site.
def headers
return Danbooru.config.http_headers
{}
end
# Returns the size of the image resource without actually downloading the file.
def size
Downloads::File.new(image_url).size
def remote_size
response = http_downloader.head(image_url)
return nil unless response.status == 200 && response.content_length.present?
response.content_length.to_i
end
memoize :size
memoize :remote_size
# Download the file at the given url, or at the main image url by default.
def download_file!(download_url = image_url)
raise DownloadError, "Download failed: couldn't find download url for #{url}" if download_url.blank?
response, file = http_downloader.download_media(download_url)
raise DownloadError, "Download failed: #{download_url} returned error #{response.status}" if response.status != 200
file
end
# A http client for API requests.
def http
Danbooru::Http.new.public_only
end
memoize :http
# A http client for downloading files.
def http_downloader
http.timeout(30).max_size(Danbooru.config.max_file_size).use(:spoof_referrer).use(:unpolish_cloudflare)
end
memoize :http_downloader
# The url to use for artist finding purposes. This will be stored in the
# artist entry. Normally this will be the profile url.
@@ -189,7 +212,7 @@ module Sources
end
def normalized_tags
tags.map { |tag, url| normalize_tag(tag) }.sort.uniq
tags.map { |tag, _url| normalize_tag(tag) }.sort.uniq
end
def normalize_tag(tag)
@@ -243,7 +266,7 @@ module Sources
end
def to_h
return {
{
:artist => {
:name => artist_name,
:tag_name => tag_name,
@@ -276,9 +299,8 @@ module Sources
to_h.to_json
end
def http_exists?(url, headers)
res = HTTParty.head(url, Danbooru.config.httparty_options.deep_merge(headers: headers))
res.success?
def http_exists?(url)
http_downloader.head(url).status.success?
end
# Convert commentary to dtext by stripping html tags. Sites can override

32
app/logical/sources/strategies/deviant_art.rb
View File

@@ -47,18 +47,18 @@
module Sources
module Strategies
class DeviantArt < Base
ASSET_SUBDOMAINS = %r{(?:fc|th|pre|img|orig|origin-orig)\d*}i
ASSET_SUBDOMAINS = /(?:fc|th|pre|img|orig|origin-orig)\d*/i
RESERVED_SUBDOMAINS = %r{\Ahttps?://(?:#{ASSET_SUBDOMAINS}|www)\.}i
MAIN_DOMAIN = %r{\Ahttps?://(?:www\.)?deviantart.com}i
TITLE = %r{(?<title>[a-z0-9_-]+?)}i
ARTIST = %r{(?<artist>[a-z0-9_-]+?)}i
DEVIATION_ID = %r{(?<deviation_id>[0-9]+)}i
TITLE = /(?<title>[a-z0-9_-]+?)/i
ARTIST = /(?<artist>[a-z0-9_-]+?)/i
DEVIATION_ID = /(?<deviation_id>[0-9]+)/i
DA_FILENAME_1 = %r{[a-f0-9]{32}-d(?<base36_deviation_id>[a-z0-9]+)\.}i
DA_FILENAME_2 = %r{#{TITLE}(?:_by_#{ARTIST}(?:-d(?<base36_deviation_id>[a-z0-9]+))?)?\.}i
DA_FILENAME_1 = /[a-f0-9]{32}-d(?<base36_deviation_id>[a-z0-9]+)\./i
DA_FILENAME_2 = /#{TITLE}(?:_by_#{ARTIST}(?:-d(?<base36_deviation_id>[a-z0-9]+))?)?\./i
DA_FILENAME = Regexp.union(DA_FILENAME_1, DA_FILENAME_2)
WIX_FILENAME = %r{d(?<base36_deviation_id>[a-z0-9]+)[0-9a-f-]+\.\w+(?:/\w+/\w+/[\w,]+/(?<title>[\w-]+)_by_(?<artist>[\w-]+)_d\w+-\w+\.\w+)?.+}i
WIX_FILENAME = %r{d(?<base36_deviation_id>[a-z0-9]+)[0-9a-f-]+\.\w+(?:/\w+/\w+/[\w,]+/(?<title>[\w-]+)_by_(?<artist>[\w-]+)_d\w+-\w+\.\w+)?.+}i
NOT_NORMALIZABLE_ASSET = %r{\Ahttps?://#{ASSET_SUBDOMAINS}\.deviantart\.net/.+/[0-9a-f]{32}(?:-[^d]\w+)?\.}i
@@ -75,7 +75,7 @@ module Sources
PATH_PROFILE = %r{#{MAIN_DOMAIN}/#{ARTIST}/?\z}i
SUBDOMAIN_PROFILE = %r{\Ahttps?://#{ARTIST}\.deviantart\.com/?\z}i
FAVME = %r{\Ahttps?://(www\.)?fav\.me/d(?<base36_deviation_id>[a-z0-9]+)\z}i
FAVME = %r{\Ahttps?://(?:www\.)?fav\.me/d(?<base36_deviation_id>[a-z0-9]+)\z}i
def domains
["deviantart.net", "deviantart.com", "fav.me"]
@@ -110,12 +110,12 @@ module Sources
api_deviation[:videos].max_by { |x| x[:filesize] }[:src]
else
src = api_deviation.dig(:content, :src)
if deviation_id && deviation_id.to_i <= 790677560 && src =~ /^https:\/\/images-wixmp-/ && src !~ /\.gif\?/
src = src.sub(%r!(/f/[a-f0-9-]+/[a-f0-9-]+)!, '/intermediary\1')
src = src.sub(%r!/v1/(fit|fill)/.*\z!i, "")
if deviation_id && deviation_id.to_i <= 790_677_560 && src =~ %r{\Ahttps://images-wixmp-} && src !~ /\.gif\?/
src = src.sub(%r{(/f/[a-f0-9-]+/[a-f0-9-]+)}, '/intermediary\1')
src = src.sub(%r{/v1/(fit|fill)/.*\z}i, "")
end
src = src.sub(%r!\Ahttps?://orig\d+\.deviantart\.net!i, "http://origin-orig.deviantart.net")
src = src.gsub(%r!q_\d+,strp!, "q_100")
src = src.sub(%r{\Ahttps?://orig\d+\.deviantart\.net}i, "http://origin-orig.deviantart.net")
src = src.gsub(/q_\d+,strp/, "q_100")
src
end
end
@@ -191,7 +191,7 @@ module Sources
# <a href="https://sa-dui.deviantart.com/journal/About-Commissions-223178193" data-sigil="thumb" class="thumb lit" ...>
if element["class"].split.include?("lit")
deviation_id = element["href"][%r!-(\d+)\z!, 1].to_i
deviation_id = element["href"][/-(\d+)\z/, 1].to_i
element.content = "deviantart ##{deviation_id}"
else
element.content = ""
@@ -199,7 +199,7 @@ module Sources
end
if element.name == "a" && element["href"].present?
element["href"] = element["href"].gsub(%r!\Ahttps?://www\.deviantart\.com/users/outgoing\?!i, "")
element["href"] = element["href"].gsub(%r{\Ahttps?://www\.deviantart\.com/users/outgoing\?}i, "")
# href may be missing the `http://` bit (ex: `inprnt.com`, `//inprnt.com`). Add it if missing.
uri = Addressable::URI.heuristic_parse(element["href"]) rescue nil
@@ -283,7 +283,7 @@ module Sources
return nil if meta.nil?
appurl = meta["content"]
uuid = appurl[%r!\ADeviantArt://deviation/(.*)\z!, 1]
uuid = appurl[%r{\ADeviantArt://deviation/(.*)\z}, 1]
uuid
end
memoize :uuid

17
app/logical/sources/strategies/hentai_foundry.rb
View File

@@ -23,11 +23,11 @@
module Sources
module Strategies
class HentaiFoundry < Base
BASE_URL = %r!\Ahttps?://(?:www\.)?hentai-foundry\.com!i
PAGE_URL = %r!#{BASE_URL}/pictures/user/(?<artist_name>[\w-]+)/(?<illust_id>\d+)(?:/[\w.-]*)?(\?[\w=]*)?\z!i
OLD_PAGE = %r!#{BASE_URL}/pic-(?<illust_id>\d+)(?:\.html)?\z!i
PROFILE_URL = %r!#{BASE_URL}/(?:pictures/)?user/(?<artist_name>[\w-]+)(?:/[a-z]*)?\z!i
IMAGE_URL = %r!\Ahttps?://pictures\.hentai-foundry\.com/+\w/(?<artist_name>[\w-]+)/(?<illust_id>\d+)(?:(?:/[\w.-]+)?\.\w+)?\z!i
BASE_URL = %r{\Ahttps?://(?:www\.)?hentai-foundry\.com}i
PAGE_URL = %r{#{BASE_URL}/pictures/user/(?<artist_name>[\w-]+)/(?<illust_id>\d+)(?:/[\w.-]*)?(\?[\w=]*)?\z}i
OLD_PAGE = %r{#{BASE_URL}/pic-(?<illust_id>\d+)(?:\.html)?\z}i
PROFILE_URL = %r{#{BASE_URL}/(?:pictures/)?user/(?<artist_name>[\w-]+)(?:/[a-z]*)?\z}i
IMAGE_URL = %r{\Ahttps?://pictures\.hentai-foundry\.com/+\w/(?<artist_name>[\w-]+)/(?<illust_id>\d+)(?:(?:/[\w.-]+)?\.\w+)?\z}i
def domains
["hentai-foundry.com"]
@@ -64,11 +64,10 @@ module Sources
def page
return nil if page_url.blank?
doc = Cache.get("hentai-foundry:#{page_url}", 1.minute) do
HTTParty.get("#{page_url}?enterAgree=1").body
end
response = Danbooru::Http.new.cache(1.minute).get("#{page_url}?enterAgree=1")
return nil unless response.status == 200
Nokogiri::HTML(doc)
response.parse
end
def tags

12
app/logical/sources/strategies/moebooru.rb
View File

@@ -32,10 +32,10 @@
module Sources
module Strategies
class Moebooru < Base
BASE_URL = %r!\Ahttps?://(?:[^.]+\.)?(?<domain>yande\.re|konachan\.com)!i
POST_URL = %r!#{BASE_URL}/post/show/(?<id>\d+)!i
URL_SLUG = %r!/(?:yande\.re%20|Konachan\.com%20-%20)?(?<id>\d+)?.*!i
IMAGE_URL = %r!#{BASE_URL}/(?<type>image|jpeg|sample)/(?<md5>\h{32})#{URL_SLUG}?\.(?<ext>jpg|jpeg|png|gif)\z!i
BASE_URL = %r{\Ahttps?://(?:[^.]+\.)?(?<domain>yande\.re|konachan\.com)}i
POST_URL = %r{#{BASE_URL}/post/show/(?<id>\d+)}i
URL_SLUG = %r{/(?:yande\.re%20|Konachan\.com%20-%20)?(?<id>\d+)?.*}i
IMAGE_URL = %r{#{BASE_URL}/(?<type>image|jpeg|sample)/(?<md5>\h{32})#{URL_SLUG}?\.(?<ext>jpg|jpeg|png|gif)\z}i
delegate :artist_name, :profile_url, :tag_name, :artist_commentary_title, :artist_commentary_desc, :dtext_artist_commentary_title, :dtext_artist_commentary_desc, to: :sub_strategy, allow_nil: true
@@ -63,7 +63,7 @@ module Sources
end
def preview_urls
return image_urls unless post_md5.present?
return image_urls if post_md5.blank?
["https://#{file_host}/data/preview/#{post_md5[0..1]}/#{post_md5[2..3]}/#{post_md5}.jpg"]
end
@@ -155,7 +155,7 @@ module Sources
# the api_response wasn't available because it's a deleted post.
elsif post_md5.present?
%w[jpg png gif].find { |ext| http_exists?("https://#{site_name}/image/#{post_md5}.#{ext}", headers) }
%w[jpg png gif].find { |ext| http_exists?("https://#{site_name}/image/#{post_md5}.#{ext}") }
else
nil

111
app/logical/sources/strategies/newgrounds.rb Normal file
View File

@@ -0,0 +1,111 @@
# Image Urls
# * https://art.ngfiles.com/images/1254000/1254722_natthelich_pandora.jpg
# * https://art.ngfiles.com/images/1033000/1033622_natthelich_fire-emblem-marth-plus-progress-pic.png?f1569487181
# * https://art.ngfiles.com/comments/57000/iu_57615_7115981.jpg
#
# Page URLs
# * https://www.newgrounds.com/art/view/puddbytes/costanza-at-bat
# * https://www.newgrounds.com/art/view/natthelich/fire-emblem-marth-plus-progress-pic (multiple)
#
# Profile URLs
# * https://natthelich.newgrounds.com/
module Sources
module Strategies
class Newgrounds < Base
IMAGE_URL = %r{\Ahttps?://art\.ngfiles\.com/images/\d+/\d+_(?<user_name>[0-9a-z-]+)_(?<illust_title>[0-9a-z-]+)\.\w+}i
COMMENT_URL = %r{\Ahttps?://art\.ngfiles\.com/comments/\d+/\w+\.\w+}i
PAGE_URL = %r{\Ahttps?://(?:www\.)?newgrounds\.com/art/view/(?<user_name>[0-9a-z-]+)/(?<illust_title>[0-9a-z-]+)(?:\?.*)?}i
PROFILE_URL = %r{\Ahttps?://(?<artist_name>(?!www)[0-9a-z-]+)\.newgrounds\.com(?:/.*)?}i
def domains
["newgrounds.com", "ngfiles.com"]
end
def site_name
"NewGrounds"
end
def image_urls
if url =~ COMMENT_URL || url =~ IMAGE_URL
[url]
else
urls = []
urls += page&.css(".image img").to_a.map { |img| img["src"] }
urls += page&.css("#author_comments img[data-user-image='1']").to_a.map { |img| img["data-smartload-src"] || img["src"] }
urls.compact
end
end
def page_url
return nil if illust_title.blank? || user_name.blank?
"https://www.newgrounds.com/art/view/#{user_name}/#{illust_title}"
end
def page
return nil if page_url.blank?
response = Danbooru::Http.cache(1.minute).get(page_url)
return nil if response.status == 404
response.parse
end
memoize :page
def tags
page&.css("#sidestats .tags a").to_a.map do |tag|
[tag.text, "https://www.newgrounds.com/search/conduct/art?match=tags&tags=" + tag.text]
end
end
def normalize_tag(tag)
tag = tag.tr("-", "_")
super(tag)
end
def artist_name
name = page&.css(".item-user .item-details h4 a")&.text&.strip || user_name
name&.downcase
end
def other_names
[artist_name, user_name].compact.uniq
end
def profile_url
# user names are not mutable, artist names are.
# However we need the latest name for normalization
"https://#{artist_name}.newgrounds.com"
end
def artist_commentary_title
page&.css(".pod-head > [itemprop='name']")&.text
end
def artist_commentary_desc
page&.css("#author_comments")&.to_html
end
def dtext_artist_commentary_desc
DText.from_html(artist_commentary_desc)
end
def normalize_for_source
page_url
end
def user_name
urls.map { |u| url[PROFILE_URL, :artist_name] || u[IMAGE_URL, :user_name] || u[PAGE_URL, :user_name] }.compact.first
end
def illust_title
urls.map { |u| u[IMAGE_URL, :illust_title] || u[PAGE_URL, :illust_title] }.compact.first
end
end
end
end

247
app/logical/sources/strategies/nico_seiga.rb
View File

@@ -1,25 +1,54 @@
# Image Direct URL
# Direct URL
# * https://lohas.nicoseiga.jp/o/971eb8af9bbcde5c2e51d5ef3a2f62d6d9ff5552/1589933964/3583893
# * http://lohas.nicoseiga.jp/priv/3521156?e=1382558156&h=f2e089256abd1d453a455ec8f317a6c703e2cedf
# * http://lohas.nicoseiga.jp/priv/b80f86c0d8591b217e7513a9e175e94e00f3c7a1/1384936074/3583893
# * https://dcdn.cdn.nimg.jp/priv/62a56a7f67d3d3746ae5712db9cac7d465f4a339/1592186183/10466669
# * https://dcdn.cdn.nimg.jp/nicoseiga/lohas/o/8ba0a9b2ea34e1ef3b5cc50785bd10cd63ec7e4a/1592187477/10466669
#
# * http://lohas.nicoseiga.jp/material/5746c5/4459092
#
# (Manga direct url)
# * https://lohas.nicoseiga.jp/priv/f5b8966fd53bf7e06cccff9fbb2c4eef62877538/1590752727/8947170
#
# Samples
# * http://lohas.nicoseiga.jp/thumb/2163478i?
# * https://lohas.nicoseiga.jp/thumb/8947170p
#
## The direct urls and samples above can belong to both illust and manga.
## There's two ways to tell them apart:
## * visit the /source/ equivalent: illusts redirect to the /o/ intermediary page, manga redirect to /priv/ directly
## * try an api call: illusts will succeed, manga will fail
#
# Source Link
# * http://seiga.nicovideo.jp/image/source?id=3312222
#
# Image Page URL
# Illust Page URL
# * https://seiga.nicovideo.jp/seiga/im3521156
# * https://seiga.nicovideo.jp/seiga/im520647 (anonymous artist)
#
# Manga Page URL
# * http://seiga.nicovideo.jp/watch/mg316708
#
# Video Page URL (not supported)
# * https://www.nicovideo.jp/watch/sm36465441
#
# Oekaki
# * https://dic.nicovideo.jp/oekaki/52833.png
module Sources
module Strategies
class NicoSeiga < Base
URL = %r!\Ahttps?://(?:\w+\.)?nico(?:seiga|video)\.jp!
DIRECT1 = %r!\Ahttps?://lohas\.nicoseiga\.jp/priv/[0-9a-f]+!
DIRECT2 = %r!\Ahttps?://lohas\.nicoseiga\.jp/o/[0-9a-f]+/\d+/\d+!
DIRECT3 = %r!\Ahttps?://seiga\.nicovideo\.jp/images/source/\d+!
PAGE = %r!\Ahttps?://seiga\.nicovideo\.jp/seiga/im(\d+)!i
PROFILE = %r!\Ahttps?://seiga\.nicovideo\.jp/user/illust/(\d+)!i
MANGA_PAGE = %r!\Ahttps?://seiga\.nicovideo\.jp/watch/mg(\d+)!i
DIRECT = %r{\Ahttps?://lohas\.nicoseiga\.jp/(?:priv|o)/(?:\w+/\d+/)?(?<image_id>\d+)(?:\?.+)?}i
CDN_DIRECT = %r{\Ahttps?://dcdn\.cdn\.nimg\.jp/.+/\w+/\d+/(?<image_id>\d+)}i
SOURCE = %r{\Ahttps?://seiga\.nicovideo\.jp/image/source(?:/|\?id=)(?<image_id>\d+)}i
ILLUST_THUMB = %r{\Ahttps?://lohas\.nicoseiga\.jp/thumb/(?<illust_id>\d+)i}i
MANGA_THUMB = %r{\Ahttps?://lohas\.nicoseiga\.jp/thumb/(?<image_id>\d+)p}i
ILLUST_PAGE = %r{\Ahttps?://(?:sp\.)?seiga\.nicovideo\.jp/seiga/im(?<illust_id>\d+)}i
MANGA_PAGE = %r{\Ahttps?://(?:sp\.)?seiga\.nicovideo\.jp/watch/mg(?<manga_id>\d+)}i
PROFILE_PAGE = %r{\Ahttps?://seiga\.nicovideo\.jp/user/illust/(?<artist_id>\d+)}i
def domains
["nicoseiga.jp", "nicovideo.jp"]
@@ -30,160 +59,136 @@ module Sources
end
def image_urls
if url =~ DIRECT1
return [url]
urls = []
return urls if api_client&.api_response.blank?
if image_id.present?
urls << "https://seiga.nicovideo.jp/image/source/#{image_id}"
elsif illust_id.present?
urls << "https://seiga.nicovideo.jp/image/source/#{illust_id}"
elsif manga_id.present? && api_client.image_ids.present?
urls += api_client.image_ids.map { |id| "https://seiga.nicovideo.jp/image/source/#{id}" }
end
urls
end
def image_url
return url if image_urls.blank? || api_client.blank?
img = case url
when DIRECT || CDN_DIRECT then "https://seiga.nicovideo.jp/image/source/#{image_id_from_url(url)}"
when SOURCE then url
else image_urls.first
end
if theme_id
return api_client.image_ids.map do |image_id|
"https://seiga.nicovideo.jp/image/source/#{image_id}"
end
resp = api_client.login.head(img)
if resp.uri.to_s =~ %r{https?://.+/(\w+/\d+/\d+)\z}i
"https://lohas.nicoseiga.jp/priv/#{$1}"
else
img
end
end
link = page.search("a#illust_link")
if link.any?
image_url = "http://seiga.nicovideo.jp" + link[0]["href"]
page = agent.get(image_url) # need to follow this redirect while logged in or it won't work
if page.is_a?(Mechanize::Image)
return [page.uri.to_s]
end
images = page.search("div.illust_view_big").select {|x| x["data-src"] =~ /\/priv\//}
if images.any?
return ["http://lohas.nicoseiga.jp" + images[0]["data-src"]]
end
def preview_urls
if illust_id.present?
["https://lohas.nicoseiga.jp/thumb/#{illust_id}i"]
else
image_urls
end
raise "image url not found for (#{url}, #{referer_url})"
end
def page_url
[url, referer_url].each do |x|
if x =~ %r!\Ahttps?://lohas\.nicoseiga\.jp/o/[a-f0-9]+/\d+/(\d+)!
return "http://seiga.nicovideo.jp/seiga/im#{$1}"
end
if x =~ %r{\Ahttps?://lohas\.nicoseiga\.jp/priv/(\d+)\?e=\d+&h=[a-f0-9]+}i
return "http://seiga.nicovideo.jp/seiga/im#{$1}"
end
if x =~ %r{\Ahttps?://lohas\.nicoseiga\.jp/priv/[a-f0-9]+/\d+/(\d+)}i
return "http://seiga.nicovideo.jp/seiga/im#{$1}"
end
if x =~ %r{\Ahttps?://lohas\.nicoseiga\.jp/priv/(\d+)}i
return "http://seiga.nicovideo.jp/seiga/im#{$1}"
end
if x =~ %r{\Ahttps?://lohas\.nicoseiga\.jp//?thumb/(\d+)i?}i
return "http://seiga.nicovideo.jp/seiga/im#{$1}"
end
if x =~ %r{/seiga/im\d+}
return x
end
if x =~ %r{/watch/mg\d+}
return x
end
if x =~ %r{/image/source\?id=(\d+)}
return "http://seiga.nicovideo.jp/seiga/im#{$1}"
end
if illust_id.present?
"https://seiga.nicovideo.jp/seiga/im#{illust_id}"
elsif manga_id.present?
"https://seiga.nicovideo.jp/watch/mg#{manga_id}"
elsif image_id.present?
"https://seiga.nicovideo.jp/image/source/#{image_id}"
end
return super
end
def canonical_url
image_url
end
def profile_url
if url =~ PROFILE
return url
end
user_id = api_client&.user_id
return if user_id.blank? # artists can be anonymous
"http://seiga.nicovideo.jp/user/illust/#{api_client.user_id}"
end
def artist_name
api_client.moniker
return if api_client.blank?
api_client.user_name
end
def artist_commentary_title
return if api_client.blank?
api_client.title
end
def artist_commentary_desc
api_client.desc
return if api_client.blank?
api_client.description
end
def dtext_artist_commentary_desc
DText.from_html(artist_commentary_desc).gsub(/[^\w]im(\d+)/, ' seiga #\1 ')
end
def normalize_for_source
if illust_id.present?
"https://seiga.nicovideo.jp/seiga/im#{illust_id}"
elsif theme_id.present?
"http://seiga.nicovideo.jp/watch/mg#{theme_id}"
# There's no way to tell apart illust from manga from the direct image url alone. What's worse,
# nicoseiga itself doesn't know how to normalize back to manga, so if it's not an illust type then
# it's impossible to get the original manga page back from the image url alone.
# /source/ links on the other hand correctly redirect, hence we use them to normalize saved direct sources.
if url =~ DIRECT
"https://seiga.nicovideo.jp/image/source/#{image_id}"
else
page_url
end
end
def tag_name
return if api_client&.user_id.blank?
"nicoseiga#{api_client.user_id}"
end
def tags
string = page.at("meta[name=keywords]").try(:[], "content") || ""
string.split(/,/).map do |name|
[name, "https://seiga.nicovideo.jp/tag/#{CGI.escape(name)}"]
return [] if api_client.blank?
base_url = "https://seiga.nicovideo.jp/"
base_url += "manga/" if manga_id.present?
base_url += "tag/"
api_client.tags.map do |name|
[name, base_url + CGI.escape(name)]
end
end
memoize :tags
def image_id
image_id_from_url(url)
end
def image_id_from_url(url)
url[DIRECT, :image_id] || url[SOURCE, :image_id] || url[MANGA_THUMB, :image_id] || url[CDN_DIRECT, :image_id]
end
def illust_id
urls.map { |u| u[ILLUST_PAGE, :illust_id] || u[ILLUST_THUMB, :illust_id] }.compact.first
end
def manga_id
urls.compact.map { |u| u[MANGA_PAGE, :manga_id] }.compact.first
end
def api_client
if illust_id
NicoSeigaApiClient.new(illust_id: illust_id)
elsif theme_id
NicoSeigaMangaApiClient.new(theme_id)
if illust_id.present?
NicoSeigaApiClient.new(work_id: illust_id, type: "illust", http: http)
elsif manga_id.present?
NicoSeigaApiClient.new(work_id: manga_id, type: "manga", http: http)
elsif image_id.present?
# We default to illust to attempt getting the api anyway
NicoSeigaApiClient.new(work_id: image_id, type: "illust", http: http)
end
end
memoize :api_client
def illust_id
if page_url =~ PAGE
return $1.to_i
end
return nil
end
def theme_id
if page_url =~ MANGA_PAGE
return $1.to_i
end
return nil
end
def page
doc = agent.get(page_url)
if doc.search("a#link_btn_login").any?
# Session cache is invalid, clear it and log in normally.
Cache.delete("nico-seiga-session")
doc = agent.get(page_url)
end
doc
end
memoize :page
def agent
NicoSeigaApiClient.agent
end
memoize :agent
end
end
end

78
app/logical/sources/strategies/nijie.rb
View File

@@ -44,25 +44,25 @@
module Sources
module Strategies
class Nijie < Base
BASE_URL = %r!\Ahttps?://(?:[^.]+\.)?nijie\.info!i
PAGE_URL = %r!#{BASE_URL}/view(?:_popup)?\.php\?id=(?<illust_id>\d+)!i
PROFILE_URL = %r!#{BASE_URL}/members(?:_illust)?\.php\?id=(?<artist_id>\d+)\z!i
BASE_URL = %r{\Ahttps?://(?:[^.]+\.)?nijie\.info}i
PAGE_URL = %r{#{BASE_URL}/view(?:_popup)?\.php\?id=(?<illust_id>\d+)}i
PROFILE_URL = %r{#{BASE_URL}/members(?:_illust)?\.php\?id=(?<artist_id>\d+)\z}i
# https://pic03.nijie.info/nijie_picture/28310_20131101215959.jpg
# https://pic03.nijie.info/nijie_picture/236014_20170620101426_0.png
# http://pic.nijie.net/03/nijie_picture/829001_20190620004513_0.mp4
# https://pic05.nijie.info/nijie_picture/diff/main/559053_20180604023346_1.png
FILENAME1 = %r!(?<artist_id>\d+)_(?<timestamp>\d{14})(?:_\d+)?!i
FILENAME1 = /(?<artist_id>\d+)_(?<timestamp>\d{14})(?:_\d+)?/i
# https://pic01.nijie.info/nijie_picture/diff/main/218856_0_236014_20170620101329.png
FILENAME2 = %r!(?<illust_id>\d+)_\d+_(?<artist_id>\d+)_(?<timestamp>\d{14})!i
FILENAME2 = /(?<illust_id>\d+)_\d+_(?<artist_id>\d+)_(?<timestamp>\d{14})/i
# https://pic04.nijie.info/nijie_picture/diff/main/287736_161475_20181112032855_1.png
FILENAME3 = %r!(?<illust_id>\d+)_(?<artist_id>\d+)_(?<timestamp>\d{14})_\d+!i
FILENAME3 = /(?<illust_id>\d+)_(?<artist_id>\d+)_(?<timestamp>\d{14})_\d+/i
IMAGE_BASE_URL = %r!\Ahttps?://(?:pic\d+\.nijie\.info|pic\.nijie\.net)!i
DIR = %r!(?:\d+/)?(?:__rs_\w+/)?nijie_picture(?:/diff/main)?!
IMAGE_URL = %r!#{IMAGE_BASE_URL}/#{DIR}/#{Regexp.union(FILENAME1, FILENAME2, FILENAME3)}\.\w+\z!i
IMAGE_BASE_URL = %r{\Ahttps?://(?:pic\d+\.nijie\.info|pic\.nijie\.net)}i
DIR = %r{(?:\d+/)?(?:__rs_\w+/)?nijie_picture(?:/diff/main)?}
IMAGE_URL = %r{#{IMAGE_BASE_URL}/#{DIR}/#{Regexp.union(FILENAME1, FILENAME2, FILENAME3)}\.\w+\z}i
def domains
["nijie.info", "nijie.net"]
@@ -146,7 +146,7 @@ module Sources
end
def to_full_image_url(x)
x.gsub(%r!__rs_\w+/!i, "").gsub(/\Ahttp:/, "https:")
x.gsub(%r{__rs_\w+/}i, "").gsub(/\Ahttp:/, "https:")
end
def to_preview_url(url)
@@ -178,57 +178,21 @@ module Sources
def page
return nil if page_url.blank?
doc = agent.get(page_url)
http = Danbooru::Http.new
form = { email: Danbooru.config.nijie_login, password: Danbooru.config.nijie_password }
if doc.search("div#header-login-container").any?
# Session cache is invalid, clear it and log in normally.
Cache.delete("nijie-session")
doc = agent.get(page_url)
end
# XXX `retriable` must come after `cache` so that retries don't return cached error responses.
response = http.cache(1.hour).use(retriable: { max_retries: 20 }).post("https://nijie.info/login_int.php", form: form)
DanbooruLogger.info "Nijie login failed (#{url}, #{response.status})" if response.status != 200
return nil unless response.status == 200
return doc
rescue Mechanize::ResponseCodeError => e
return nil if e.response_code.to_i == 404
raise
response = http.cookies(R18: 1).cache(1.minute).get(page_url)
return nil unless response.status == 200
response&.parse
end
memoize :page
def agent
mech = Mechanize.new
session = Cache.get("nijie-session")
if session
cookie = Mechanize::Cookie.new("NIJIEIJIEID", session)
cookie.domain = ".nijie.info"
cookie.path = "/"
mech.cookie_jar.add(cookie)
else
mech.get("https://nijie.info/login.php") do |page|
page.form_with(:action => "/login_int.php") do |form|
form['email'] = Danbooru.config.nijie_login
form['password'] = Danbooru.config.nijie_password
end.click_button
end
session = mech.cookie_jar.cookies.select {|c| c.name == "NIJIEIJIEID"}.first
Cache.put("nijie-session", session.value, 1.day) if session
end
# This cookie needs to be set to allow viewing of adult works while anonymous
cookie = Mechanize::Cookie.new("R18", "1")
cookie.domain = ".nijie.info"
cookie.path = "/"
mech.cookie_jar.add(cookie)
mech
rescue Mechanize::ResponseCodeError => x
if x.response_code.to_i == 429
sleep(5)
retry
else
raise
end
end
memoize :agent
end
end
end

6
app/logical/sources/strategies/null.rb
View File

@@ -28,7 +28,7 @@ module Sources
when %r{\Ahttp://p\.twpl\.jp/show/(?:large|orig)/([a-z0-9]+)}i
"http://p.twipple.jp/#{$1}"
when %r{\Ahttps?://blog(?:(?:-imgs-)?\d*(?:-origin)?)?\.fc2\.com/(?:(?:[^/]/){3}|(?:[^/]/))([^/]+)/(?:file/)?([^\.]+\.[^\?]+)}i
when %r{\Ahttps?://blog(?:(?:-imgs-)?\d*(?:-origin)?)?\.fc2\.com/(?:(?:[^/]/){3}|(?:[^/]/))([^/]+)/(?:file/)?([^.]+\.[^?]+)}i
username = $1
filename = $2
"http://#{username}.blog.fc2.com/img/#{filename}/"
@@ -47,7 +47,7 @@ module Sources
when %r{\Ahttps?://c(?:s|han|[1-4])\.sankakucomplex\.com/data(?:/sample)?/(?:[a-f0-9]{2}/){2}(?:sample-|preview)?([a-f0-9]{32})}i
"https://chan.sankakucomplex.com/en/post/show?md5=#{$1}"
when %r{\Ahttps?://(?:www|s(?:tatic|[1-4]))\.zerochan\.net/.+(?:\.|\/)(\d+)(?:\.(?:jpe?g?))?\z}i
when %r{\Ahttps?://(?:www|s(?:tatic|[1-4]))\.zerochan\.net/.+(?:\.|\/)(\d+)(?:\.(?:jpe?g?|png))?\z}i
"https://www.zerochan.net/#{$1}#full"
when %r{\Ahttps?://static[1-6]?\.minitokyo\.net/(?:downloads|view)/(?:\d{2}/){2}(\d+)}i
@@ -105,7 +105,7 @@ module Sources
# http://img.toranoana.jp/popup_img18/04/0010/22/87/040010228714-1p.jpg
# http://img.toranoana.jp/popup_blimg/04/0030/08/30/040030083068-1p.jpg
# https://ecdnimg.toranoana.jp/ec/img/04/0030/65/34/040030653417-6p.jpg
when %r{\Ahttps?://(\w+\.)?toranoana\.jp/(?:popup_(?:bl)?img\d*|ec/img)/\d{2}/\d{4}/\d{2}/\d{2}/(?<work_id>\d+)}i
when %r{\Ahttps?://(?:\w+\.)?toranoana\.jp/(?:popup_(?:bl)?img\d*|ec/img)/\d{2}/\d{4}/\d{2}/\d{2}/(?<work_id>\d+)}i
"https://ec.toranoana.jp/tora_r/ec/item/#{$~[:work_id]}/"
# https://a.hitomi.la/galleries/907838/1.png

53
app/logical/sources/strategies/pawoo.rb
View File

@@ -16,13 +16,13 @@
module Sources::Strategies
class Pawoo < Base
HOST = %r!\Ahttps?://(www\.)?pawoo\.net!i
IMAGE = %r!\Ahttps?://img\.pawoo\.net/media_attachments/files/(\d+/\d+/\d+)!
NAMED_PROFILE = %r!#{HOST}/@(?<artist_name>\w+)!i
ID_PROFILE = %r!#{HOST}/web/accounts/(?<artist_id>\d+)!
HOST = %r{\Ahttps?://(www\.)?pawoo\.net}i
IMAGE = %r{\Ahttps?://img\.pawoo\.net/media_attachments/files/(\d+/\d+/\d+)}
NAMED_PROFILE = %r{#{HOST}/@(?<artist_name>\w+)}i
ID_PROFILE = %r{#{HOST}/web/accounts/(?<artist_id>\d+)}
STATUS1 = %r!\A#{HOST}/web/statuses/(?<status_id>\d+)!
STATUS2 = %r!\A#{NAMED_PROFILE}/(?<status_id>\d+)!
STATUS1 = %r{\A#{HOST}/web/statuses/(?<status_id>\d+)}
STATUS2 = %r{\A#{NAMED_PROFILE}/(?<status_id>\d+)}
def domains
["pawoo.net"]
@@ -37,15 +37,13 @@ module Sources::Strategies
end
def image_urls
if url =~ %r!#{IMAGE}/small/([a-z0-9]+\.\w+)\z!i
return ["https://img.pawoo.net/media_attachments/files/#{$1}/original/#{$2}"]
if url =~ %r{#{IMAGE}/small/([a-z0-9]+\.\w+)\z}i
["https://img.pawoo.net/media_attachments/files/#{$1}/original/#{$2}"]
elsif url =~ %r{#{IMAGE}/original/([a-z0-9]+\.\w+)\z}i
[url]
else
api_response.image_urls
end
if url =~ %r!#{IMAGE}/original/([a-z0-9]+\.\w+)\z!i
return [url]
end
return api_response.image_urls
end
def page_url
@@ -55,16 +53,17 @@ module Sources::Strategies
end
end
return super
super
end
def profile_url
if url =~ PawooApiClient::PROFILE2
return "https://pawoo.net/@#{$1}"
"https://pawoo.net/@#{$1}"
elsif api_response.profile_url.blank?
url
else
api_response.profile_url
end
return url if api_response.profile_url.blank?
api_response.profile_url
end
def artist_name
@@ -87,10 +86,6 @@ module Sources::Strategies
urls.map { |url| url[STATUS1, :status_id] || url[STATUS2, :status_id] }.compact.first
end
def artist_commentary_title
nil
end
def artist_commentary_desc
api_response.commentary
end
@@ -99,18 +94,10 @@ module Sources::Strategies
api_response.tags
end
def normalizable_for_artist_finder?
true
end
def normalize_for_artist_finder
profile_url
end
def normalize_for_source
artist_name = artist_name_from_url
status_id = status_id_from_url
return unless status_id.present?
return if status_id.blank?
if artist_name.present?
"https://pawoo.net/@#{artist_name}/#{status_id}"
@@ -131,7 +118,7 @@ module Sources::Strategies
def api_response
[url, referer_url].each do |x|
if client = PawooApiClient.new.get(x)
if (client = PawooApiClient.new.get(x))
return client
end
end

168
app/logical/sources/strategies/pixiv.rb
View File

@@ -50,37 +50,34 @@
module Sources
module Strategies
class Pixiv < Base
MONIKER = %r!(?:[a-zA-Z0-9_-]+)!
PROFILE = %r!\Ahttps?://www\.pixiv\.net/member\.php\?id=[0-9]+\z!
DATE = %r!(?<date>\d{4}/\d{2}/\d{2}/\d{2}/\d{2}/\d{2})!i
EXT = %r!(?:jpg|jpeg|png|gif)!i
MONIKER = /(?:[a-zA-Z0-9_-]+)/
PROFILE = %r{\Ahttps?://www\.pixiv\.net/member\.php\?id=[0-9]+\z}
DATE = %r{(?<date>\d{4}/\d{2}/\d{2}/\d{2}/\d{2}/\d{2})}i
EXT = /(?:jpg|jpeg|png|gif)/i
WEB = %r!(?:\A(?:https?://)?www\.pixiv\.net)!
I12 = %r!(?:\A(?:https?://)?i[0-9]+\.pixiv\.net)!
IMG = %r!(?:\A(?:https?://)?img[0-9]*\.pixiv\.net)!
PXIMG = %r!(?:\A(?:https?://)?[^.]+\.pximg\.net)!
TOUCH = %r!(?:\A(?:https?://)?touch\.pixiv\.net)!
UGOIRA = %r!#{PXIMG}/img-zip-ugoira/img/#{DATE}/(?<illust_id>\d+)_ugoira1920x1080\.zip\z!i
ORIG_IMAGE = %r!#{PXIMG}/img-original/img/#{DATE}/(?<illust_id>\d+)_p(?<page>\d+)\.#{EXT}\z!i
STACC_PAGE = %r!\A#{WEB}/stacc/#{MONIKER}/?\z!i
NOVEL_PAGE = %r!(?:\Ahttps?://www\.pixiv\.net/novel/show\.php\?id=(\d+))!
FANBOX_ACCOUNT = %r!(?:\Ahttps?://www\.pixiv\.net/fanbox/creator/\d+\z)!
FANBOX_IMAGE = %r!(?:\Ahttps?://fanbox\.pixiv\.net/images/post/(\d+))!
FANBOX_PAGE = %r!(?:\Ahttps?://www\.pixiv\.net/fanbox/creator/\d+/post/(\d+))!
WEB = %r{(?:\A(?:https?://)?www\.pixiv\.net)}
I12 = %r{(?:\A(?:https?://)?i[0-9]+\.pixiv\.net)}
IMG = %r{(?:\A(?:https?://)?img[0-9]*\.pixiv\.net)}
PXIMG = %r{(?:\A(?:https?://)?[^.]+\.pximg\.net)}
TOUCH = %r{(?:\A(?:https?://)?touch\.pixiv\.net)}
UGOIRA = %r{#{PXIMG}/img-zip-ugoira/img/#{DATE}/(?<illust_id>\d+)_ugoira1920x1080\.zip\z}i
ORIG_IMAGE = %r{#{PXIMG}/img-original/img/#{DATE}/(?<illust_id>\d+)_p(?<page>\d+)\.#{EXT}\z}i
STACC_PAGE = %r{\A#{WEB}/stacc/#{MONIKER}/?\z}i
NOVEL_PAGE = %r{(?:\Ahttps?://www\.pixiv\.net/novel/show\.php\?id=(\d+))}
def self.to_dtext(text)
if text.nil?
return nil
end
text = text.gsub(%r!https?://www\.pixiv\.net/member_illust\.php\?mode=medium&illust_id=([0-9]+)!i) do |match|
text = text.gsub(%r{https?://www\.pixiv\.net/member_illust\.php\?mode=medium&illust_id=([0-9]+)}i) do |_match|
pixiv_id = $1
%(pixiv ##{pixiv_id} "»":[/posts?tags=pixiv:#{pixiv_id}])
end
text = text.gsub(%r!https?://www\.pixiv\.net/member\.php\?id=([0-9]+)!i) do |match|
text = text.gsub(%r{https?://www\.pixiv\.net/member\.php\?id=([0-9]+)}i) do |_match|
member_id = $1
profile_url = "https://www.pixiv.net/member.php?id=#{member_id}"
profile_url = "https://www.pixiv.net/users/#{member_id}"
search_params = {"search[url_matches]" => profile_url}.to_param
%("user/#{member_id}":[#{profile_url}] "»":[/artists?#{search_params}])
@@ -127,25 +124,17 @@ module Sources
return "https://www.pixiv.net/novel/show.php?id=#{novel_id}&mode=cover"
end
if fanbox_id.present?
return "https://www.pixiv.net/fanbox/creator/#{metadata.user_id}/post/#{fanbox_id}"
end
if fanbox_account_id.present?
return "https://www.pixiv.net/fanbox/creator/#{fanbox_account_id}"
end
if illust_id.present?
return "https://www.pixiv.net/artworks/#{illust_id}"
end
return url
url
rescue PixivApiClient::BadIDError
nil
end
def canonical_url
return image_url
image_url
end
def profile_url
@@ -155,7 +144,7 @@ module Sources
end
end
"https://www.pixiv.net/member.php?id=#{metadata.user_id}"
"https://www.pixiv.net/users/#{metadata.user_id}"
rescue PixivApiClient::BadIDError
nil
end
@@ -192,17 +181,7 @@ module Sources
end
def headers
if fanbox_id.present?
# need the session to download fanbox images
return {
"Referer" => "https://www.pixiv.net/fanbox",
"Cookie" => HTTP::Cookie.cookie_value(agent.cookies)
}
end
return {
"Referer" => "https://www.pixiv.net"
}
{ "Referer" => "https://www.pixiv.net" }
end
def normalize_for_source
@@ -231,7 +210,7 @@ module Sources
translated_tags = super(tag)
if translated_tags.empty? && tag.include?("/")
translated_tags = tag.split("/").flat_map { |tag| super(tag) }
translated_tags = tag.split("/").flat_map { |translated_tag| super(translated_tag) }
end
translated_tags
@@ -242,10 +221,6 @@ module Sources
end
def image_urls_sub
if url =~ FANBOX_IMAGE
return [url]
end
# there's too much normalization bullshit we have to deal with
# raw urls, so just fetch the canonical url from the api every
# time.
@@ -257,7 +232,7 @@ module Sources
return [ugoira_zip_url]
end
return metadata.pages
metadata.pages
end
# in order to prevent recursive loops, this method should not make any
@@ -265,7 +240,7 @@ module Sources
# even though it makes sense to reference page_url here, it will only look
# at (url, referer_url).
def illust_id
return nil if novel_id.present? || fanbox_id.present?
return nil if novel_id.present?
parsed_urls.each do |url|
# http://www.pixiv.net/member_illust.php?mode=medium&illust_id=18557054
@@ -276,11 +251,11 @@ module Sources
return url.query_values["illust_id"].to_i
# http://www.pixiv.net/en/artworks/46324488
elsif url.host == "www.pixiv.net" && url.path =~ %r!\A/(?:en/)?artworks/(?<illust_id>\d+)!i
elsif url.host == "www.pixiv.net" && url.path =~ %r{\A/(?:en/)?artworks/(?<illust_id>\d+)}i
return $~[:illust_id].to_i
# http://www.pixiv.net/i/18557054
elsif url.host == "www.pixiv.net" && url.path =~ %r!\A/i/(?<illust_id>\d+)\z!i
elsif url.host == "www.pixiv.net" && url.path =~ %r{\A/i/(?<illust_id>\d+)\z}i
return $~[:illust_id].to_i
# http://img18.pixiv.net/img/evazion/14901720.png
@@ -289,8 +264,8 @@ module Sources
# http://i2.pixiv.net/img18/img/evazion/14901720_s.png
# http://i1.pixiv.net/img07/img/pasirism/18557054_p1.png
# http://i1.pixiv.net/img07/img/pasirism/18557054_big_p1.png
elsif url.host =~ %r!\A(?:i\d+|img\d+)\.pixiv\.net\z!i &&
url.path =~ %r!\A(?:/img\d+)?/img/#{MONIKER}/(?<illust_id>\d+)(?:_\w+)?\.(?:jpg|jpeg|png|gif|zip)!i
elsif url.host =~ /\A(?:i\d+|img\d+)\.pixiv\.net\z/i &&
url.path =~ %r{\A(?:/img\d+)?/img/#{MONIKER}/(?<illust_id>\d+)(?:_\w+)?\.(?:jpg|jpeg|png|gif|zip)}i
return $~[:illust_id].to_i
# http://i1.pixiv.net/img-inf/img/2011/05/01/23/28/04/18557054_64x64.jpg
@@ -307,13 +282,13 @@ module Sources
#
# https://i.pximg.net/novel-cover-original/img/2019/01/14/01/15/05/10617324_d84daae89092d96bbe66efafec136e42.jpg
# https://img-sketch.pixiv.net/uploads/medium/file/4463372/8906921629213362989.jpg
elsif url.host =~ %r!\A(?:[^.]+\.pximg\.net|i\d+\.pixiv\.net|tc-pximg01\.techorus-cdn\.com)\z!i &&
url.path =~ %r!\A(/c/\w+)?/img-[a-z-]+/img/#{DATE}/(?<illust_id>\d+)(?:_\w+)?\.(?:jpg|jpeg|png|gif|zip)!i
elsif url.host =~ /\A(?:[^.]+\.pximg\.net|i\d+\.pixiv\.net|tc-pximg01\.techorus-cdn\.com)\z/i &&
url.path =~ %r{\A(/c/\w+)?/img-[a-z-]+/img/#{DATE}/(?<illust_id>\d+)(?:_\w+)?\.(?:jpg|jpeg|png|gif|zip)}i
return $~[:illust_id].to_i
end
end
return nil
nil
end
memoize :illust_id
@@ -324,89 +299,48 @@ module Sources
end
end
return nil
nil
end
memoize :novel_id
def fanbox_id
[url, referer_url].each do |x|
if x =~ FANBOX_PAGE
return $1
end
if x =~ FANBOX_IMAGE
return $1
end
end
return nil
end
memoize :fanbox_id
def fanbox_account_id
[url, referer_url].each do |x|
if x =~ FANBOX_ACCOUNT
return x
end
end
return nil
end
memoize :fanbox_account_id
def agent
PixivWebAgent.build
end
memoize :agent
def metadata
if novel_id.present?
return PixivApiClient.new.novel(novel_id)
end
if fanbox_id.present?
return PixivApiClient.new.fanbox(fanbox_id)
end
return PixivApiClient.new.work(illust_id)
PixivApiClient.new.work(illust_id)
end
memoize :metadata
def moniker
# we can sometimes get the moniker from the url
if url =~ %r!#{IMG}/img/(#{MONIKER})!i
return $1
if url =~ %r{#{IMG}/img/(#{MONIKER})}i
$1
elsif url =~ %r{#{I12}/img[0-9]+/img/(#{MONIKER})}i
$1
elsif url =~ %r{#{WEB}/stacc/(#{MONIKER})/?$}i
$1
else
metadata.moniker
end
if url =~ %r!#{I12}/img[0-9]+/img/(#{MONIKER})!i
return $1
end
if url =~ %r!#{WEB}/stacc/(#{MONIKER})/?$!i
return $1
end
return metadata.moniker
rescue PixivApiClient::BadIDError
nil
end
memoize :moniker
def data
return {
ugoira_frame_data: ugoira_frame_data
}
{ ugoira_frame_data: ugoira_frame_data }
end
def ugoira_zip_url
if metadata.pages.is_a?(Hash) && metadata.pages["ugoira600x600"]
return metadata.pages["ugoira600x600"].sub("_ugoira600x600.zip", "_ugoira1920x1080.zip")
metadata.pages["ugoira600x600"].sub("_ugoira600x600.zip", "_ugoira1920x1080.zip")
end
end
memoize :ugoira_zip_url
def ugoira_frame_data
return metadata.json.dig("metadata", "frames")
metadata.json.dig("metadata", "frames")
rescue PixivApiClient::BadIDError
nil
end
@@ -415,16 +349,14 @@ module Sources
def ugoira_content_type
case metadata.json["image_urls"].to_s
when /\.jpg/
return "image/jpeg"
"image/jpeg"
when /\.png/
return "image/png"
"image/png"
when /\.gif/
return "image/gif"
"image/gif"
else
raise Sources::Error, "content type not found for (#{url}, #{referer_url})"
end
raise Sources::Error.new("content type not found for (#{url}, #{referer_url})")
end
memoize :ugoira_content_type
@@ -434,7 +366,7 @@ module Sources
# http://i2.pixiv.net/img04/img/syounen_no_uta/46170939_p0.jpg
# http://i1.pixiv.net/c/600x600/img-master/img/2014/09/24/23/25/08/46168376_p0_master1200.jpg
# http://i1.pixiv.net/img-original/img/2014/09/25/23/09/29/46183440_p0.jpg
if url =~ %r!/\d+_p(\d+)(?:_\w+)?\.#{EXT}!i
if url =~ %r{/\d+_p(\d+)(?:_\w+)?\.#{EXT}}i
return $1.to_i
end
@@ -445,7 +377,7 @@ module Sources
end
end
return nil
nil
end
memoize :manga_page
end

22
app/logical/sources/strategies/tumblr.rb
View File

@@ -12,19 +12,19 @@ module Sources::Strategies
class Tumblr < Base
SIZES = %w[1280 640 540 500h 500 400 250 100]
BASE_URL = %r!\Ahttps?://(?:[^/]+\.)*tumblr\.com!i
DOMAIN = %r{(data|(\d+\.)?media)\.tumblr\.com}
MD5 = %r{(?<md5>[0-9a-f]{32})}i
FILENAME = %r{(?<filename>(tumblr_(inline_)?)?[a-z0-9]+(_r[0-9]+)?)}i
EXT = %r{(?<ext>\w+)}
BASE_URL = %r{\Ahttps?://(?:[^/]+\.)*tumblr\.com}i
DOMAIN = /(data|(?:\d+\.)?media)\.tumblr\.com/i
MD5 = /(?<md5>[0-9a-f]{32})/i
FILENAME = /(?<filename>(?:tumblr_(?:inline_)?)?[a-z0-9]+(?:_r[0-9]+)?)/i
EXT = /(?<ext>\w+)/
# old: https://66.media.tumblr.com/2c6f55531618b4335c67e29157f5c1fc/tumblr_pz4a44xdVj1ssucdno1_1280.png
# new: https://66.media.tumblr.com/168dabd09d5ad69eb5fedcf94c45c31a/3dbfaec9b9e0c2e3-72/s640x960/bf33a1324f3f36d2dc64f011bfeab4867da62bc8.png
OLD_IMAGE = %r!\Ahttps?://#{DOMAIN}/(?<dir>#{MD5}/)?#{FILENAME}_(?<size>\w+)\.#{EXT}\z!i
OLD_IMAGE = %r{\Ahttps?://#{DOMAIN}/(?<dir>#{MD5}/)?#{FILENAME}_(?<size>\w+)\.#{EXT}\z}i
IMAGE = %r!\Ahttps?://#{DOMAIN}/!i
VIDEO = %r!\Ahttps?://(?:vtt|ve\.media)\.tumblr\.com/!i
POST = %r!\Ahttps?://(?<blog_name>[^.]+)\.tumblr\.com/(?:post|image)/(?<post_id>\d+)!i
IMAGE = %r{\Ahttps?://#{DOMAIN}/}i
VIDEO = %r{\Ahttps?://(?:vtt|ve|va\.media)\.tumblr\.com/}i
POST = %r{\Ahttps?://(?<blog_name>[^.]+)\.tumblr\.com/(?:post|image)/(?<post_id>\d+)}i
def self.enabled?
Danbooru.config.tumblr_consumer_key.present?
@@ -68,7 +68,7 @@ module Sources::Strategies
def preview_urls
image_urls.map do |x|
x.sub(%r!_1280\.(jpg|png|gif|jpeg)\z!, '_250.\1')
x.sub(/_1280\.(jpg|png|gif|jpeg)\z/, '_250.\1')
end
end
@@ -168,7 +168,7 @@ module Sources::Strategies
end
candidates.find do |candidate|
http_exists?(candidate, headers)
http_exists?(candidate)
end
end

34
app/logical/sources/strategies/twitter.rb
View File

@@ -1,20 +1,20 @@
module Sources::Strategies
class Twitter < Base
PAGE = %r!\Ahttps?://(?:mobile\.)?twitter\.com!i
PROFILE = %r!\Ahttps?://(?:mobile\.)?twitter.com/(?<username>[a-z0-9_]+)!i
PAGE = %r{\Ahttps?://(?:mobile\.)?twitter\.com}i
PROFILE = %r{\Ahttps?://(?:mobile\.)?twitter.com/(?<username>[a-z0-9_]+)}i
# https://pbs.twimg.com/media/EBGbJe_U8AA4Ekb.jpg
# https://pbs.twimg.com/media/EBGbJe_U8AA4Ekb?format=jpg&name=900x900
# https://pbs.twimg.com/tweet_video_thumb/ETkN_L3X0AMy1aT.jpg
# https://pbs.twimg.com/ext_tw_video_thumb/1243725361986375680/pu/img/JDA7g7lcw7wK-PIv.jpg
# https://pbs.twimg.com/amplify_video_thumb/1215590775364259840/img/lolCkEEioFZTb5dl.jpg
BASE_IMAGE_URL = %r!\Ahttps?://pbs\.twimg\.com/(?<media_type>media|tweet_video_thumb|ext_tw_video_thumb|amplify_video_thumb)!i
FILENAME1 = %r!(?<file_name>[a-zA-Z0-9_-]+)\.(?<file_ext>\w+)!i
FILENAME2 = %r!(?<file_name>[a-zA-Z0-9_-]+)\?.*format=(?<file_ext>\w+)!i
FILEPATH1 = %r!(?<file_path>\d+/[\w_-]+/img)!i
FILEPATH2 = %r!(?<file_path>\d+/img)!i
IMAGE_URL1 = %r!#{BASE_IMAGE_URL}/#{Regexp.union(FILENAME1, FILENAME2)}!i
IMAGE_URL2 = %r!#{BASE_IMAGE_URL}/#{Regexp.union(FILEPATH1, FILEPATH2)}/#{FILENAME1}!i
BASE_IMAGE_URL = %r{\Ahttps?://pbs\.twimg\.com/(?<media_type>media|tweet_video_thumb|ext_tw_video_thumb|amplify_video_thumb)}i
FILENAME1 = /(?<file_name>[a-zA-Z0-9_-]+)\.(?<file_ext>\w+)/i
FILENAME2 = /(?<file_name>[a-zA-Z0-9_-]+)\?.*format=(?<file_ext>\w+)/i
FILEPATH1 = %r{(?<file_path>\d+/[\w_-]+/img)}i
FILEPATH2 = %r{(?<file_path>\d+/img)}i
IMAGE_URL1 = %r{#{BASE_IMAGE_URL}/#{Regexp.union(FILENAME1, FILENAME2)}}i
IMAGE_URL2 = %r{#{BASE_IMAGE_URL}/#{Regexp.union(FILEPATH1, FILEPATH2)}/#{FILENAME1}}i
# Twitter provides a list but it's inaccurate; some names ('intent') aren't
# included and other names in the list aren't actually reserved.
@@ -47,7 +47,7 @@ module Sources::Strategies
return $1
end
return nil
nil
end
def self.artist_name_from_url(url)
@@ -78,7 +78,7 @@ module Sources::Strategies
elsif media[:type].in?(["video", "animated_gif"])
variants = media.dig(:video_info, :variants)
videos = variants.select { |variant| variant[:content_type] == "video/mp4" }
video = videos.max_by { |video| video[:bitrate].to_i }
video = videos.max_by { |v| v[:bitrate].to_i }
video[:url]
end
end
@@ -137,10 +137,6 @@ module Sources::Strategies
api_response[:full_text].to_s
end
def normalizable_for_artist_finder?
url =~ PAGE
end
def normalize_for_artist_finder
profile_url.try(:downcase).presence || url
end
@@ -193,9 +189,9 @@ module Sources::Strategies
desc = artist_commentary_desc.unicode_normalize(:nfkc)
desc = CGI.unescapeHTML(desc)
desc = desc.gsub(%r!https?://t\.co/[a-zA-Z0-9]+!i, url_replacements)
desc = desc.gsub(%r!#([^[:space:]]+)!, '"#\\1":[https://twitter.com/hashtag/\\1]')
desc = desc.gsub(%r!@([a-zA-Z0-9_]+)!, '"@\\1":[https://twitter.com/\\1]')
desc = desc.gsub(%r{https?://t\.co/[a-zA-Z0-9]+}i, url_replacements)
desc = desc.gsub(/#([^[:space:]]+)/, '"#\\1":[https://twitter.com/hashtag/\\1]')
desc = desc.gsub(/@([a-zA-Z0-9_]+)/, '"@\\1":[https://twitter.com/\\1]')
desc.strip
end
@@ -204,7 +200,7 @@ module Sources::Strategies
end
def api_response
return {} if !self.class.enabled?
return {} unless self.class.enabled? && status_id.present?
api_client.status(status_id)
end

6
app/logical/sources/strategies/weibo.rb
View File

@@ -38,7 +38,7 @@ module Sources
PAGE_URL_1 = %r{\Ahttps?://(?:www\.)?weibo\.com/(?<artist_short_id>\d+)/(?<illust_base62_id>\w+)(?:\?.*)?\z}i
PAGE_URL_2 = %r{#{PROFILE_URL_2}/(?:wbphotos/large/mid|talbum/detail/photo_id)/(?<illust_long_id>\d+)(?:/pid/(?<image_id>\w{32}))?}i
PAGE_URL_3 = %r{\Ahttps?://m\.weibo\.cn/(detail/(?<illust_long_id>\d+)|status/(?<illust_base62_id>\w+))}i
PAGE_URL_3 = %r{\Ahttps?://m\.weibo\.cn/(?:detail/(?<illust_long_id>\d+)|status/(?<illust_base62_id>\w+))}i
PAGE_URL_4 = %r{\Ahttps?://tw\.weibo\.com/(?:(?<artist_short_id>\d+)|\w+)/(?<illust_long_id>\d+)}i
IMAGE_URL = %r{\Ahttps?://\w{3}\.sinaimg\.cn/\w+/(?<image_id>\w{32})\.}i
@@ -203,12 +203,12 @@ module Sources
end
def api_response
return nil if mobile_url.blank?
return {} if mobile_url.blank?
resp = Danbooru::Http.cache(1.minute).get(mobile_url)
json_string = resp.to_s[/var \$render_data = \[(.*)\]\[0\]/m, 1]
return nil if json_string.blank?
return {} if json_string.blank?
JSON.parse(json_string)["status"]
end

9
app/logical/spam_detector.rb
View File

@@ -7,10 +7,11 @@ class SpamDetector
# if a person receives more than 10 automatic spam reports within a 1 hour
# window, automatically ban them forever.
AUTOBAN_THRESHOLD = 10
AUTOBAN_WINDOW = 1.hours
AUTOBAN_DURATION = 999999
AUTOBAN_WINDOW = 1.hour
AUTOBAN_DURATION = 999_999
attr_accessor :record, :user, :user_ip, :content, :comment_type
rakismet_attrs author: proc { user.name },
author_email: proc { user.email_address&.address },
blog_lang: "en",
@@ -84,8 +85,8 @@ class SpamDetector
end
is_spam
rescue StandardError => exception
DanbooruLogger.log(exception)
rescue StandardError => e
DanbooruLogger.log(e)
false
end
end

4
app/logical/storage_manager/sftp.rb
View File

@@ -21,7 +21,7 @@ class StorageManager::SFTP < StorageManager
temp_upload_path = dest_path + "-" + SecureRandom.uuid + ".tmp"
dest_backup_path = dest_path + "-" + SecureRandom.uuid + ".bak"
each_host do |host, sftp|
each_host do |_host, sftp|
sftp.upload!(file.path, temp_upload_path)
sftp.setstat!(temp_upload_path, permissions: DEFAULT_PERMISSIONS)
@@ -40,7 +40,7 @@ class StorageManager::SFTP < StorageManager
end
def delete(dest_path)
each_host do |host, sftp|
each_host do |_host, sftp|
force { sftp.remove!(dest_path) }
end
end

5
app/logical/tag_autocomplete.rb
View File

@@ -25,8 +25,7 @@ module TagAutocomplete
def search(query)
query = Tag.normalize_name(query)
candidates = count_sort(
query,
count_sort(
search_exact(query, 8) +
search_prefix(query, 4) +
search_correct(query, 2) +
@@ -34,7 +33,7 @@ module TagAutocomplete
)
end
def count_sort(query, words)
def count_sort(words)
words.uniq(&:name).sort_by do |x|
x.post_count * x.weight
end.reverse.slice(0, LIMIT)

24
app/logical/tag_relationship_retirement_service.rb
View File

@@ -4,19 +4,11 @@ module TagRelationshipRetirementService
THRESHOLD = 2.years
def forum_topic_title
return "Retired tag aliases & implications"
"Retired tag aliases & implications"
end
def forum_topic_body
return "This topic deals with tag relationships created two or more years ago that have not been used since. They will be retired. This topic will be updated as an automated system retires expired relationships."
end
def dry_run
[TagAlias, TagImplication].each do |model|
each_candidate(model) do |rel|
puts "#{rel.relationship} #{rel.antecedent_name} -> #{rel.consequent_name} retired"
end
end
"This topic deals with tag relationships created two or more years ago that have not been used since. They will be retired. This topic will be updated as an automated system retires expired relationships."
end
def forum_topic
@@ -27,7 +19,7 @@ module TagRelationshipRetirementService
forum_post = ForumPost.create!(creator: User.system, body: forum_topic_body, topic: topic)
end
end
return topic
topic
end
def find_and_retire!
@@ -50,16 +42,6 @@ module TagRelationshipRetirementService
yield(rel)
end
end
# model.active.where("created_at < ?", SMALL_THRESHOLD.ago).find_each do |rel|
# if is_underused?(rel.consequent_name)
# yield(rel)
# end
# end
end
def is_underused?(name)
(Tag.find_by_name(name).try(:post_count) || 0) < COUNT_THRESHOLD
end
def is_unused?(name)

4
app/logical/upload_limit.rb
View File

@@ -2,7 +2,7 @@ class UploadLimit
extend Memoist
INITIAL_POINTS = 1000
MAXIMUM_POINTS = 10000
MAXIMUM_POINTS = 10_000
attr_reader :user
@@ -75,7 +75,7 @@ class UploadLimit
points += upload_value(points, is_deleted)
points = points.clamp(0, MAXIMUM_POINTS)
#warn "slots: %2d, points: %3d, value: %2d" % [UploadLimit.points_to_level(points) + 5, points, UploadLimit.upload_value(level, is_deleted)]
# warn "slots: %2d, points: %3d, value: %2d" % [UploadLimit.points_to_level(points) + 5, points, UploadLimit.upload_value(level, is_deleted)]
end
points

13
app/logical/upload_service.rb
View File

@@ -15,7 +15,6 @@ class UploadService
start!
end
rescue ActiveRecord::RecordNotUnique
return
end
def start!
@@ -31,8 +30,8 @@ class UploadService
begin
create_post_from_upload(@upload)
rescue Exception => x
@upload.update(status: "error: #{x.class} - #{x.message}", backtrace: x.backtrace.join("\n"))
rescue Exception => e
@upload.update(status: "error: #{e.class} - #{e.message}", backtrace: e.backtrace.join("\n"))
end
return @upload
end
@@ -53,16 +52,16 @@ class UploadService
@upload.save!
@post = create_post_from_upload(@upload)
return @upload
rescue Exception => x
@upload.update(status: "error: #{x.class} - #{x.message}", backtrace: x.backtrace.join("\n"))
@upload
rescue Exception => e
@upload.update(status: "error: #{e.class} - #{e.message}", backtrace: e.backtrace.join("\n"))
@upload
end
end
def warnings
return [] if @post.nil?
return @post.warnings.full_messages
@post.warnings.full_messages
end
def create_post_from_upload(upload)

9
app/logical/upload_service/controller_helper.rb
View File

@@ -7,11 +7,8 @@ class UploadService
# this gets called from UploadsController#new so we need to preprocess async
UploadPreprocessorDelayedStartJob.perform_later(url, ref, CurrentUser.user)
begin
download = Downloads::File.new(url, ref)
remote_size = download.size
rescue Exception
end
strategy = Sources::Strategies.find(url, ref)
remote_size = strategy.remote_size
return [upload, remote_size]
end
@@ -21,7 +18,7 @@ class UploadService
Preprocessor.new(file: file).delayed_start(CurrentUser.id)
end
return [upload]
[upload]
end
end
end

27
app/logical/upload_service/preprocessor.rb
View File

@@ -46,11 +46,9 @@ class UploadService
def predecessor
if md5.present?
return Upload.where(status: ["preprocessed", "preprocessing"], md5: md5).first
end
if Utils.is_downloadable?(source)
return Upload.where(status: ["preprocessed", "preprocessing"], source: source).first
Upload.where(status: ["preprocessed", "preprocessing"], md5: md5).first
elsif Utils.is_downloadable?(source)
Upload.where(status: ["preprocessed", "preprocessing"], source: source).first
end
end
@@ -63,21 +61,20 @@ class UploadService
start!
end
rescue ActiveRecord::RecordNotUnique
return
end
def start!
if Utils.is_downloadable?(source)
if Post.system_tag_match("source:#{canonical_source}").where.not(id: original_post_id).exists?
raise ActiveRecord::RecordNotUnique.new("A post with source #{canonical_source} already exists")
raise ActiveRecord::RecordNotUnique, "A post with source #{canonical_source} already exists"
end
if Upload.where(source: source, status: "completed").exists?
raise ActiveRecord::RecordNotUnique.new("A completed upload with source #{source} already exists")
raise ActiveRecord::RecordNotUnique, "A completed upload with source #{source} already exists"
end
if Upload.where(source: source).where("status like ?", "error%").exists?
raise ActiveRecord::RecordNotUnique.new("An errored upload with source #{source} already exists")
raise ActiveRecord::RecordNotUnique, "An errored upload with source #{source} already exists"
end
end
@@ -95,21 +92,21 @@ class UploadService
upload.tag_string = params[:tag_string]
upload.status = "preprocessed"
upload.save!
rescue Exception => x
upload.update(file_ext: nil, status: "error: #{x.class} - #{x.message}", backtrace: x.backtrace.join("\n"))
rescue Exception => e
upload.update(file_ext: nil, status: "error: #{e.class} - #{e.message}", backtrace: e.backtrace.join("\n"))
end
return upload
upload
end
def finish!(upload = nil)
pred = upload || self.predecessor
pred = upload || predecessor
# regardless of who initialized the upload, credit should
# goto whoever submitted the form
pred.initialize_attributes
pred.attributes = self.params
pred.attributes = params
# if a file was uploaded after the preprocessing occurred,
# then process the file and overwrite whatever the preprocessor
@@ -118,7 +115,7 @@ class UploadService
pred.status = "completed"
pred.save
return pred
pred
end
end
end

4
app/logical/upload_service/replacer.rb
View File

@@ -62,7 +62,7 @@ class UploadService
end
def source_strategy(upload)
return Sources::Strategies.find(upload.source, upload.referer_url)
Sources::Strategies.find(upload.source, upload.referer_url)
end
def find_replacement_url(repl, upload)
@@ -78,7 +78,7 @@ class UploadService
return source_strategy(upload).canonical_url
end
return upload.source
upload.source
end
def process!

10
app/logical/upload_service/utils.rb
View File

@@ -71,19 +71,19 @@ class UploadService
return file if file.present?
raise "No file or source URL provided" if upload.source_url.blank?
download = Downloads::File.new(upload.source_url, upload.referer_url)
file, strategy = download.download!
strategy = Sources::Strategies.find(upload.source_url, upload.referer_url)
file = strategy.download_file!
if download.data[:ugoira_frame_data].present?
if strategy.data[:ugoira_frame_data].present?
upload.context = {
"ugoira" => {
"frame_data" => download.data[:ugoira_frame_data],
"frame_data" => strategy.data[:ugoira_frame_data],
"content_type" => "image/jpeg"
}
}
end
return file
file
end
end
end

1
app/logical/user_deletion.rb
View File

@@ -2,6 +2,7 @@ class UserDeletion
include ActiveModel::Validations
attr_reader :user, :password
validate :validate_deletion
def initialize(user, password)

6
app/logical/user_promotion.rb
View File

@@ -88,11 +88,7 @@ class UserPromotion
end
def create_dmail
Dmail.create_automated(
:to_id => user.id,
:title => "You have been promoted",
:body => build_messages
)
Dmail.create_automated(to_id: user.id, title: "Your account has been updated", body: build_messages)
end
def create_user_feedback

27
app/logical/validating_socket.rb Normal file
View File

@@ -0,0 +1,27 @@
# A TCPSocket wrapper that disallows connections to local or private IPs. Used for SSRF protection.
# https://owasp.org/www-community/attacks/Server_Side_Request_Forgery
require "resolv"
class ValidatingSocket < TCPSocket
class ProhibitedIpError < StandardError; end
def initialize(hostname, port)
ip = validate_hostname!(hostname)
super(ip, port)
end
def validate_hostname!(hostname)
ip = IPAddress.parse(::Resolv.getaddress(hostname))
raise ProhibitedIpError, "Connection to #{hostname} failed; #{ip} is a prohibited IP" if prohibited_ip?(ip)
ip.to_s
end
def prohibited_ip?(ip)
if ip.ipv4?
ip.loopback? || ip.link_local? || ip.multicast? || ip.private?
elsif ip.ipv6?
ip.loopback? || ip.link_local? || ip.unique_local? || ip.unspecified?
end
end
end

3
app/models/application_record.rb
View File

@@ -16,7 +16,8 @@ class ApplicationRecord < ActiveRecord::Base
search_params = params.fetch(:search, {}).permit!
search_params = defaults.merge(search_params).with_indifferent_access
search(search_params).paginate(params[:page], limit: params[:limit], search_count: count_pages)
max_limit = (params[:format] == "sitemap") ? 10_000 : 1_000
search(search_params).paginate(params[:page], limit: params[:limit], max_limit: max_limit, search_count: count_pages)
end
end
end

Some files were not shown because too many files have changed in this diff Show More