jaculabilis/danbooru
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'master' into mobile-mode-default-image-size

Browse Source
This commit is contained in:
evazion
2020-07-23 16:22:37 -05:00
committed by GitHub
parent 131e526a15 c4fb43a5b4
commit da9c2a26ce
267 changed files with 5414 additions and 4398 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.bundle/config Normal file
View File

@@ -0,0 +1,3 @@
---
BUNDLE_BUILD__NOKOGIRI: "--use-system-libraries"
BUNDLE_BUILD__NOKOGUMBO: "--without-libxml2"

18
.codeclimate.yml
View File

@@ -1,17 +1,25 @@
version: 2
checks:
argument-count:
enabled: false
complex-logic:
config:
threshold: 4
threshold: 8
file-lines:
config:
threshold: 500
threshold: 1000
method-complexity:
config:
threshold: 15
method-count:
config:
threshold: 40
enabled: false
method-lines:
enabled: false
nested-control-flow:
config:
threshold: 100
threshold: 4
return-statements:
enabled: false
plugins:
eslint:
enabled: true

1
.codecov.yml Normal file
View File

@@ -0,0 +1 @@
comment: false

14
.editorconfig Normal file
View File

@@ -0,0 +1,14 @@
root = true
[**.{js,rb,css,erb,md,json,yml}]
indent_style = space
indent_size = 2
end_of_line = lf
insert_final_newline = true
trim_trailing_whitespace = true
[**.html.erb]
indent_size = unset
[app/javascript/vendor/**]
indent_size = unset

6
.eslintrc.yml
View File

@@ -8,9 +8,13 @@ parserOptions:
globals:
$: false
require: false
parser: babel-eslint
plugins:
- babel
rules:
# https://eslint.org/docs/rules/
array-callback-return: error
babel/no-unused-expressions: error
block-scoped-var: error
consistent-return: error
default-case: error
@@ -32,7 +36,7 @@ rules:
no-sequences: error
no-shadow: error
no-shadow-restricted-names: error
no-unused-expressions: error
#no-unused-expressions: error
no-unused-vars:
- error
- argsIgnorePattern: "^_"

59
.github/workflows/test.yaml vendored
View File

@@ -1,20 +1,40 @@
name: Test
name: Github
on: [push, pull_request]
# Trigger on pushes to master or pull requests to master, but not both.
on:
push:
branches:
- master
pull_request:
branches:
- master
jobs:
# https://github.com/Marr11317/ConflictAdviser
notify-merge-conflicts:
runs-on: ubuntu-latest
steps:
- uses: Marr11317/ConflictAdviser@v1
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
conflict_label: "merge conflict"
comment: 'Rebase needed'
test:
runs-on: ubuntu-latest
container: ubuntu:20.04
defaults:
run:
shell: bash
env:
DEBIAN_FRONTEND: noninteractive
PARALLEL_WORKERS: 8 # number of parallel tests to run
RUBYOPT: -W0 # silence ruby warnings
VIPS_WARNING: 0 # silence libvips warnings
# Code Climate configuration. https://docs.codeclimate.com/docs/finding-your-test-coverage-token
CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }}
GIT_COMMIT_SHA: ${{ github.sha }}
GIT_BRANCH: ${{ github.ref }}
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
DATABASE_URL: postgresql://danbooru:danbooru@postgres/danbooru
ARCHIVE_DATABASE_URL: postgresql://danbooru:danbooru@postgres/danbooru
@@ -45,32 +65,25 @@ jobs:
POSTGRES_PASSWORD: danbooru
steps:
- name: Check out code
uses: actions/checkout@v2
# - name: Save environment
# run: env | egrep "DANBOORU|DATABASE_URL" > ~/.env
# - name: Install docker-compose
# run: sudo apt-get update && sudo apt-get -y install docker-compose
# - name: Run tests
# run: docker-compose --env-file ~/.env -f config/docker/docker-compose.test.yaml -p danbooru up
- name: Install OS dependencies
run: |
apt-get update
apt-get -y install --no-install-recommends build-essential ruby ruby-dev ruby-bundler git nodejs yarnpkg webpack ffmpeg mkvtoolnix libvips-dev libxml2-dev postgresql-server-dev-all wget
apt-get -y install --no-install-recommends build-essential ruby ruby-dev ruby-bundler git nodejs yarnpkg webpack ffmpeg mkvtoolnix libvips-dev libxml2-dev libxslt-dev zlib1g-dev postgresql-server-dev-all wget curl git
ln -sf /usr/bin/yarnpkg /usr/bin/yarn
- name: Check out code
uses: actions/checkout@v2
with:
fetch-depth: 0
- name: Install Ruby dependencies
run: BUNDLE_DEPLOYMENT=true bundle install --jobs 4
- name: Install Javascript dependencies
run: yarn install
- name: Prepare database
run: config/docker/prepare-tests.sh
# https://docs.codeclimate.com/docs/configuring-test-coverage
- name: Prepare test coverage for Code Climate
run: |
wget https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64
chmod +x test-reporter-latest-linux-amd64
./test-reporter-latest-linux-amd64 before-build
- name: Run tests
run: bin/rails test
- name: Upload test coverage to Code Climate
run: ./test-reporter-latest-linux-amd64 after-build

1
.gitignore vendored
View File

@@ -2,7 +2,6 @@
.yarn-integrity
.gitconfig
.git/
.bundle/
config/database.yml
config/danbooru_local_config.rb
config/deploy/*.rb

19
.rubocop.yml
View File

@@ -1,8 +1,9 @@
# some of these settings are overriden in also test/.rubocop.yml
require:
- rubocop-rails
AllCops:
TargetRubyVersion: 2.7.0
NewCops: enable
Exclude:
- "bin/*"
@@ -41,14 +42,22 @@ Layout/SpaceInsideHashLiteralBraces:
Metrics/AbcSize:
Enabled: false
Metrics/BlockLength:
Max: 50
ExcludedMethods:
- concerning
- context
- should
Metrics/BlockNesting:
CountBlocks: false
Max: 4
Metrics/ClassLength:
Max: 500
Metrics/CyclomaticComplexity:
Max: 10
Enabled: false
Metrics/MethodLength:
Max: 100
@@ -60,7 +69,7 @@ Metrics/ParameterLists:
Max: 4
Metrics/PerceivedComplexity:
Enabled: false
Max: 20
Lint/InheritException:
EnforcedStyle: standard_error
@@ -110,7 +119,9 @@ Style/NumericPredicate:
Style/PercentLiteralDelimiters:
PreferredDelimiters:
"default": "[]"
"%r": "!!"
Style/ParallelAssignment:
Enabled: false
Style/PerlBackrefs:
Enabled: false

12
.simplecov
View File

@@ -1,8 +1,12 @@
SimpleCov.start "rails" do
add_group "Libraries", ["app/logical", "lib"]
add_group "Presenters", "app/presenters"
#enable_coverage :branch
#minimum_coverage line: 85, branch: 75
#minimum_coverage_by_file 50
#coverage_dir "tmp/coverage"
add_group "Policies", "app/policies"
enable_coverage :branch
# https://github.com/codecov/codecov-ruby#submit-only-in-ci-example
if ENV["CODECOV_TOKEN"]
require "codecov"
SimpleCov.formatter = SimpleCov::Formatter::Codecov
end
end

13
Gemfile
View File

@@ -7,7 +7,6 @@ gem "pg"
gem "delayed_job"
gem "delayed_job_active_record"
gem "simple_form"
gem "mechanize"
gem "whenever", :require => false
gem "sanitize"
gem 'ruby-vips'
@@ -28,14 +27,11 @@ gem 'daemons'
gem 'oauth2'
gem 'bootsnap'
gem 'addressable'
gem 'httparty'
gem 'rakismet'
gem 'recaptcha', require: "recaptcha/rails"
gem 'activemodel-serializers-xml'
gem 'jquery-rails'
gem 'webpacker', '>= 4.0.x'
gem 'rake'
gem 'retriable'
gem 'redis'
gem 'request_store'
gem 'builder'
@@ -47,9 +43,7 @@ gem 'http'
gem 'activerecord-hierarchical_query'
gem 'pundit'
gem 'mail'
# locked to 1.10.9 to workaround an incompatibility with nokogumbo 2.0.2.
gem 'nokogiri', '~> 1.10.9'
gem 'nokogiri'
group :production, :staging do
gem 'unicorn', :platforms => :ruby
@@ -65,7 +59,6 @@ end
group :development do
gem 'rubocop'
gem 'rubocop-rails'
gem 'sinatra'
gem 'meta_request'
gem 'rack-mini-profiler'
gem 'stackprof'
@@ -85,11 +78,11 @@ group :test do
gem "factory_bot"
gem "mocha", require: "mocha/minitest"
gem "ffaker"
gem "simplecov", "~> 0.17.0", require: false
gem "webmock", require: "webmock/minitest"
gem "simplecov", require: false
gem "minitest-ci"
gem "minitest-reporters", require: "minitest/reporters"
gem "mock_redis"
gem "capybara"
gem "selenium-webdriver"
gem "codecov", require: false
end

261
Gemfile.lock
View File

@@ -1,70 +1,70 @@
GIT
remote: https://github.com/evazion/dtext_rb.git
revision: 507e97e0963822c20351c82620c28cc8e23423d5
revision: a95bf1d537cbdba4585adb8e123f03f001f56fd7
specs:
dtext_rb (1.10.5)
dtext_rb (1.10.6)
nokogiri (~> 1.8)
GEM
remote: https://rubygems.org/
specs:
actioncable (6.0.3.1)
actionpack (= 6.0.3.1)
actioncable (6.0.3.2)
actionpack (= 6.0.3.2)
nio4r (~> 2.0)
websocket-driver (>= 0.6.1)
actionmailbox (6.0.3.1)
actionpack (= 6.0.3.1)
activejob (= 6.0.3.1)
activerecord (= 6.0.3.1)
activestorage (= 6.0.3.1)
activesupport (= 6.0.3.1)
actionmailbox (6.0.3.2)
actionpack (= 6.0.3.2)
activejob (= 6.0.3.2)
activerecord (= 6.0.3.2)
activestorage (= 6.0.3.2)
activesupport (= 6.0.3.2)
mail (>= 2.7.1)
actionmailer (6.0.3.1)
actionpack (= 6.0.3.1)
actionview (= 6.0.3.1)
activejob (= 6.0.3.1)
actionmailer (6.0.3.2)
actionpack (= 6.0.3.2)
actionview (= 6.0.3.2)
activejob (= 6.0.3.2)
mail (~> 2.5, >= 2.5.4)
rails-dom-testing (~> 2.0)
actionpack (6.0.3.1)
actionview (= 6.0.3.1)
activesupport (= 6.0.3.1)
actionpack (6.0.3.2)
actionview (= 6.0.3.2)
activesupport (= 6.0.3.2)
rack (~> 2.0, >= 2.0.8)
rack-test (>= 0.6.3)
rails-dom-testing (~> 2.0)
rails-html-sanitizer (~> 1.0, >= 1.2.0)
actiontext (6.0.3.1)
actionpack (= 6.0.3.1)
activerecord (= 6.0.3.1)
activestorage (= 6.0.3.1)
activesupport (= 6.0.3.1)
actiontext (6.0.3.2)
actionpack (= 6.0.3.2)
activerecord (= 6.0.3.2)
activestorage (= 6.0.3.2)
activesupport (= 6.0.3.2)
nokogiri (>= 1.8.5)
actionview (6.0.3.1)
activesupport (= 6.0.3.1)
actionview (6.0.3.2)
activesupport (= 6.0.3.2)
builder (~> 3.1)
erubi (~> 1.4)
rails-dom-testing (~> 2.0)
rails-html-sanitizer (~> 1.1, >= 1.2.0)
activejob (6.0.3.1)
activesupport (= 6.0.3.1)
activejob (6.0.3.2)
activesupport (= 6.0.3.2)
globalid (>= 0.3.6)
activemodel (6.0.3.1)
activesupport (= 6.0.3.1)
activemodel (6.0.3.2)
activesupport (= 6.0.3.2)
activemodel-serializers-xml (1.0.2)
activemodel (> 5.x)
activesupport (> 5.x)
builder (~> 3.1)
activerecord (6.0.3.1)
activemodel (= 6.0.3.1)
activesupport (= 6.0.3.1)
activerecord (6.0.3.2)
activemodel (= 6.0.3.2)
activesupport (= 6.0.3.2)
activerecord-hierarchical_query (1.2.3)
activerecord (>= 5.0, < 6.1)
pg (>= 0.21, < 1.3)
activestorage (6.0.3.1)
actionpack (= 6.0.3.1)
activejob (= 6.0.3.1)
activerecord (= 6.0.3.1)
activestorage (6.0.3.2)
actionpack (= 6.0.3.2)
activejob (= 6.0.3.2)
activerecord (= 6.0.3.2)
marcel (~> 0.3.1)
activesupport (6.0.3.1)
activesupport (6.0.3.2)
concurrent-ruby (~> 1.0, >= 1.0.2)
i18n (>= 0.7, < 2)
minitest (~> 5.1)
@@ -75,42 +75,42 @@ GEM
airbrussh (1.4.0)
sshkit (>= 1.6.1, != 1.7.0)
ansi (1.5.0)
ast (2.4.0)
ast (2.4.1)
aws-eventstream (1.1.0)
aws-partitions (1.326.0)
aws-sdk-core (3.98.0)
aws-partitions (1.341.0)
aws-sdk-core (3.103.0)
aws-eventstream (~> 1, >= 1.0.2)
aws-partitions (~> 1, >= 1.239.0)
aws-sigv4 (~> 1.1)
jmespath (~> 1.0)
aws-sdk-sqs (1.26.0)
aws-sdk-core (~> 3, >= 3.71.0)
aws-sdk-sqs (1.30.0)
aws-sdk-core (~> 3, >= 3.99.0)
aws-sigv4 (~> 1.1)
aws-sigv4 (1.1.4)
aws-eventstream (~> 1.0, >= 1.0.2)
aws-sigv4 (1.2.1)
aws-eventstream (~> 1, >= 1.0.2)
bcrypt (3.1.13)
bootsnap (1.4.6)
msgpack (~> 1.0)
builder (3.2.4)
byebug (11.1.3)
capistrano (3.14.0)
capistrano (3.14.1)
airbrussh (>= 1.0.0)
i18n
rake (>= 10.0.0)
sshkit (>= 1.9.0)
capistrano-bundler (1.6.0)
capistrano-bundler (2.0.0)
capistrano (~> 3.1)
capistrano-deploytags (1.0.7)
capistrano (>= 3.7.0)
capistrano-rails (1.5.0)
capistrano-rails (1.6.1)
capistrano (~> 3.1)
capistrano-bundler (~> 1.1)
capistrano-rbenv (2.1.6)
capistrano-bundler (>= 1.1, < 3)
capistrano-rbenv (2.2.0)
capistrano (~> 3.1)
sshkit (~> 1.3)
capistrano3-unicorn (0.2.1)
capistrano (~> 3.1, >= 3.1.0)
capybara (3.32.2)
capybara (3.33.0)
addressable
mini_mime (>= 0.1.3)
nokogiri (~> 1.8)
@@ -120,11 +120,13 @@ GEM
xpath (~> 3.2)
childprocess (3.0.0)
chronic (0.10.2)
codecov (0.2.0)
colorize
json
simplecov
coderay (1.1.3)
colorize (0.8.1)
concurrent-ruby (1.1.6)
connection_pool (2.2.3)
crack (0.4.3)
safe_yaml (~> 1.0.0)
crass (1.0.6)
daemons (1.3.1)
delayed_job (4.1.8)
@@ -132,22 +134,21 @@ GEM
delayed_job_active_record (4.1.4)
activerecord (>= 3.0, < 6.1)
delayed_job (>= 3.0, < 5)
diff-lcs (1.3)
diff-lcs (1.4.4)
docile (1.3.2)
domain_name (0.5.20190701)
unf (>= 0.0.5, < 1.0.0)
dotenv (2.7.5)
dotenv-rails (2.7.5)
dotenv (= 2.7.5)
railties (>= 3.2, < 6.1)
dotenv (2.7.6)
dotenv-rails (2.7.6)
dotenv (= 2.7.6)
railties (>= 3.2)
erubi (1.9.0)
factory_bot (5.2.0)
activesupport (>= 4.2.0)
factory_bot (6.1.0)
activesupport (>= 5.0.0)
faraday (1.0.1)
multipart-post (>= 1.2, < 3)
ffaker (2.15.0)
ffi (1.13.0)
ffi (1.13.0-x64-mingw32)
ffi (1.13.1)
ffi-compiler (1.0.1)
ffi (>= 1.0.0)
rake
@@ -156,7 +157,6 @@ GEM
ffi (~> 1.0)
globalid (0.4.2)
activesupport (>= 4.2.0)
hashdiff (1.0.1)
http (4.4.1)
addressable (~> 2.3)
http-cookie (~> 1.0)
@@ -167,48 +167,29 @@ GEM
http-form_data (2.3.0)
http-parser (1.2.1)
ffi-compiler (>= 1.0, < 2.0)
httparty (0.18.0)
mime-types (~> 3.0)
multi_xml (>= 0.5.2)
i18n (1.8.3)
concurrent-ruby (~> 1.0)
ipaddress_2 (0.13.0)
jmespath (1.4.0)
jquery-rails (4.3.5)
rails-dom-testing (>= 1, < 3)
railties (>= 4.2.0)
thor (>= 0.14, < 2.0)
json (2.3.0)
json (2.3.1)
jwt (2.2.1)
kgio (2.11.3)
listen (3.2.1)
rb-fsevent (~> 0.10, >= 0.10.3)
rb-inotify (~> 0.9, >= 0.9.10)
loofah (2.5.0)
loofah (2.6.0)
crass (~> 1.0.2)
nokogiri (>= 1.5.9)
mail (2.7.1)
mini_mime (>= 0.1.1)
marcel (0.3.3)
mimemagic (~> 0.3.2)
mechanize (2.7.6)
domain_name (~> 0.5, >= 0.5.1)
http-cookie (~> 1.0)
mime-types (>= 1.17.2)
net-http-digest_auth (~> 1.1, >= 1.1.1)
net-http-persistent (>= 2.5.2)
nokogiri (~> 1.6)
ntlm-http (~> 0.1, >= 0.1.1)
webrobots (>= 0.0.9, < 0.2)
memoist (0.16.2)
memory_profiler (0.9.14)
meta_request (0.7.2)
rack-contrib (>= 1.1, < 3)
railties (>= 3.0.0, < 7)
method_source (1.0.0)
mime-types (3.3.1)
mime-types-data (~> 3.2015)
mime-types-data (3.2020.0512)
mimemagic (0.3.5)
mini_mime (1.0.2)
mini_portile2 (2.4.0)
@@ -221,42 +202,32 @@ GEM
minitest (>= 5.0)
ruby-progressbar
mocha (1.11.2)
mock_redis (0.23.0)
mock_redis (0.25.0)
msgpack (1.3.3)
msgpack (1.3.3-x64-mingw32)
multi_json (1.14.1)
multi_json (1.15.0)
multi_xml (0.6.0)
multipart-post (2.1.1)
mustermann (1.1.1)
ruby2_keywords (~> 0.0.1)
net-http-digest_auth (1.4.1)
net-http-persistent (4.0.0)
connection_pool (~> 2.2)
net-scp (3.0.0)
net-ssh (>= 2.6.5, < 7.0.0)
net-sftp (3.0.0)
net-ssh (>= 5.0.0, < 7.0.0)
net-ssh (6.0.2)
net-ssh (6.1.0)
newrelic_rpm (6.11.0.365)
nio4r (2.5.2)
nokogiri (1.10.9)
mini_portile2 (~> 2.4.0)
nokogiri (1.10.9-x64-mingw32)
nokogiri (1.10.10)
mini_portile2 (~> 2.4.0)
nokogumbo (2.0.2)
nokogiri (~> 1.8, >= 1.8.4)
ntlm-http (0.1.1)
oauth2 (1.4.4)
faraday (>= 0.8, < 2.0)
jwt (>= 1.0, < 3.0)
multi_json (~> 1.3)
multi_xml (~> 0.5)
rack (>= 1.2, < 3)
parallel (1.19.1)
parser (2.7.1.3)
ast (~> 2.4.0)
parallel (1.19.2)
parser (2.7.1.4)
ast (~> 2.4.1)
pg (1.2.3)
pg (1.2.3-x64-mingw32)
pry (0.13.1)
coderay (~> 1.1)
method_source (~> 1.0)
@@ -270,40 +241,38 @@ GEM
nio4r (~> 2.0)
pundit (2.1.0)
activesupport (>= 3.0.0)
rack (2.2.2)
rack (2.2.3)
rack-contrib (2.2.0)
rack (~> 2.0)
rack-mini-profiler (2.0.2)
rack (>= 1.2.0)
rack-protection (2.0.8.1)
rack
rack-proxy (0.6.5)
rack
rack-test (1.1.0)
rack (>= 1.0, < 3)
rails (6.0.3.1)
actioncable (= 6.0.3.1)
actionmailbox (= 6.0.3.1)
actionmailer (= 6.0.3.1)
actionpack (= 6.0.3.1)
actiontext (= 6.0.3.1)
actionview (= 6.0.3.1)
activejob (= 6.0.3.1)
activemodel (= 6.0.3.1)
activerecord (= 6.0.3.1)
activestorage (= 6.0.3.1)
activesupport (= 6.0.3.1)
rails (6.0.3.2)
actioncable (= 6.0.3.2)
actionmailbox (= 6.0.3.2)
actionmailer (= 6.0.3.2)
actionpack (= 6.0.3.2)
actiontext (= 6.0.3.2)
actionview (= 6.0.3.2)
activejob (= 6.0.3.2)
activemodel (= 6.0.3.2)
activerecord (= 6.0.3.2)
activestorage (= 6.0.3.2)
activesupport (= 6.0.3.2)
bundler (>= 1.3.0)
railties (= 6.0.3.1)
railties (= 6.0.3.2)
sprockets-rails (>= 2.0.0)
rails-dom-testing (2.0.3)
activesupport (>= 4.2.0)
nokogiri (>= 1.6)
rails-html-sanitizer (1.3.0)
loofah (~> 2.3)
railties (6.0.3.1)
actionpack (= 6.0.3.1)
activesupport (= 6.0.3.1)
railties (6.0.3.2)
actionpack (= 6.0.3.2)
activesupport (= 6.0.3.2)
method_source
rake (>= 0.8.7)
thor (>= 0.20.3, < 2.0)
@@ -316,25 +285,24 @@ GEM
ffi (~> 1.0)
recaptcha (5.5.0)
json
redis (4.1.4)
redis (4.2.1)
regexp_parser (1.7.1)
request_store (1.5.0)
rack (>= 1.4)
responders (3.0.1)
actionpack (>= 5.0)
railties (>= 5.0)
retriable (3.1.2)
rexml (3.2.4)
rubocop (0.85.1)
rubocop (0.88.0)
parallel (~> 1.10)
parser (>= 2.7.0.1)
parser (>= 2.7.1.1)
rainbow (>= 2.2.2, < 4.0)
regexp_parser (>= 1.7)
rexml
rubocop-ast (>= 0.0.3)
rubocop-ast (>= 0.1.0, < 1.0)
ruby-progressbar (~> 1.7)
unicode-display_width (>= 1.4.0, < 2.0)
rubocop-ast (0.0.3)
rubocop-ast (0.1.0)
parser (>= 2.7.0.1)
rubocop-rails (2.6.0)
activesupport (>= 4.2.0)
@@ -343,10 +311,8 @@ GEM
ruby-progressbar (1.10.1)
ruby-vips (2.0.17)
ffi (~> 1.9)
ruby2_keywords (0.0.2)
rubyzip (2.3.0)
safe_yaml (1.0.5)
sanitize (5.2.0)
sanitize (5.2.1)
crass (~> 1.0.2)
nokogiri (>= 1.8.0)
nokogumbo (~> 2.0)
@@ -357,23 +323,17 @@ GEM
childprocess (>= 0.5, < 4.0)
rubyzip (>= 1.2.2)
semantic_range (2.3.0)
shoulda-context (1.2.2)
shoulda-context (2.0.0)
shoulda-matchers (4.3.0)
activesupport (>= 4.2.0)
simple_form (5.0.2)
actionpack (>= 5.0)
activemodel (>= 5.0)
simplecov (0.17.1)
simplecov (0.18.5)
docile (~> 1.1)
json (>= 1.8, < 3)
simplecov-html (~> 0.10.0)
simplecov-html (0.10.2)
sinatra (2.0.8.1)
mustermann (~> 1.0)
rack (~> 2.0)
rack-protection (= 2.0.8.1)
tilt (~> 2.0)
sprockets (4.0.0)
simplecov-html (~> 0.11)
simplecov-html (0.12.2)
sprockets (4.0.2)
concurrent-ruby (~> 1.0)
rack (> 1, < 3)
sprockets-rails (3.2.1)
@@ -389,13 +349,11 @@ GEM
stripe (5.22.0)
thor (1.0.1)
thread_safe (0.3.6)
tilt (2.0.10)
tzinfo (1.2.7)
thread_safe (~> 0.1)
unf (0.1.4)
unf_ext
unf_ext (0.0.7.7)
unf_ext (0.0.7.7-x64-mingw32)
unicode-display_width (1.7.0)
unicorn (5.5.5)
kgio (~> 2.6)
@@ -403,28 +361,22 @@ GEM
unicorn-worker-killer (0.4.4)
get_process_mem (~> 0)
unicorn (>= 4, < 6)
webmock (3.8.3)
addressable (>= 2.3.6)
crack (>= 0.3.2)
hashdiff (>= 0.4.0, < 2.0.0)
webpacker (5.1.1)
activesupport (>= 5.2)
rack-proxy (>= 0.6.1)
railties (>= 5.2)
semantic_range (>= 2.3.0)
webrobots (0.1.2)
websocket-driver (0.7.2)
websocket-driver (0.7.3)
websocket-extensions (>= 0.1.0)
websocket-extensions (0.1.5)
whenever (1.0.0)
chronic (>= 0.6.3)
xpath (3.2.0)
nokogiri (~> 1.8)
zeitwerk (2.3.0)
zeitwerk (2.3.1)
PLATFORMS
ruby
x64-mingw32
DEPENDENCIES
activemodel-serializers-xml
@@ -440,6 +392,7 @@ DEPENDENCIES
capistrano-rbenv
capistrano3-unicorn
capybara
codecov
daemons
delayed_job
delayed_job_active_record
@@ -450,12 +403,9 @@ DEPENDENCIES
ffaker
flamegraph
http
httparty
ipaddress_2
jquery-rails
listen
mail
mechanize
memoist
memory_profiler
meta_request
@@ -465,7 +415,7 @@ DEPENDENCIES
mock_redis
net-sftp
newrelic_rpm
nokogiri (~> 1.10.9)
nokogiri
oauth2
pg
pry-byebug
@@ -480,7 +430,6 @@ DEPENDENCIES
redis
request_store
responders
retriable
rubocop
rubocop-rails
ruby-vips
@@ -491,14 +440,12 @@ DEPENDENCIES
shoulda-context
shoulda-matchers
simple_form
simplecov (~> 0.17.0)
sinatra
simplecov
stackprof
streamio-ffmpeg
stripe
unicorn
unicorn-worker-killer
webmock
webpacker (>= 4.0.x)
whenever

14
INSTALL.debian
View File

@@ -32,9 +32,6 @@ if [[ -z "$HOSTNAME" ]] ; then
exit 1
fi
echo -n "* Enter the VLAN IP address for this server (ex: 172.16.0.1, enter nothing to skip): "
read VLAN_IP_ADDR
# Install packages
echo "* Installing packages..."
@@ -52,17 +49,6 @@ apt-get -y install $LIBSSL_DEV_PKG build-essential automake libxml2-dev libxslt-
apt-get -y install libpq-dev postgresql-client
apt-get -y install liblcms2-dev $LIBJPEG_TURBO_DEV_PKG libexpat1-dev libgif-dev libpng-dev libexif-dev
# vrack specific stuff
if [ -n "$VLAN_IP_ADDR" ] ; then
apt-get -y install vlan
modprobe 8021q
echo "8021q" >> /etc/modules
vconfig add eno2 99
ip addr add $VLAN_IP_ADDR/24 dev eno2.99
ip link set up eno2.99
curl -L -s $GITHUB_INSTALL_SCRIPTS/vrack-cfg.yaml -o /etc/netplan/01-netcfg.yaml
fi
curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -
echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list
curl -sSL https://deb.nodesource.com/setup_10.x | sudo -E bash -

2
README.md
View File

@@ -1,4 +1,4 @@
[![Test Coverage](https://api.codeclimate.com/v1/badges/a51f46fb460a35104d82/test_coverage)](https://codeclimate.com/github/danbooru/danbooru/test_coverage) [![Discord](https://img.shields.io/discord/310432830138089472?label=Discord)](https://discord.gg/eSVKkUF)
[![codecov](https://codecov.io/gh/danbooru/danbooru/branch/master/graph/badge.svg)](https://codecov.io/gh/danbooru/danbooru) [![Discord](https://img.shields.io/discord/310432830138089472?label=Discord)](https://discord.gg/eSVKkUF)
## Installation

2
app/controllers/application_controller.rb
View File

@@ -88,6 +88,8 @@ class ApplicationController < ActionController::Base
def rescue_exception(exception)
case exception
when ActionView::Template::Error
rescue_exception(exception.cause)
when ActiveRecord::QueryCanceled
render_error_page(500, exception, template: "static/search_timeout", message: "The database timed out running your query.")
when ActionController::BadRequest

16
app/controllers/autocomplete_controller.rb Normal file
View File

@@ -0,0 +1,16 @@
class AutocompleteController < ApplicationController
respond_to :xml, :json
def index
@tags = Tag.names_matches_with_aliases(params[:query], params.fetch(:limit, 10).to_i)
if request.variant.opensearch?
expires_in 1.hour
results = [params[:query], @tags.map(&:pretty_name)]
respond_with(results)
else
# XXX
respond_with(@tags.map(&:attributes))
end
end
end

1
app/controllers/comments_controller.rb
View File

@@ -97,6 +97,7 @@ class CommentsController < ApplicationController
if request.format.atom?
@comments = @comments.includes(:creator, :post)
@comments = @comments.select { |comment| comment.post.visible? }
elsif request.format.html?
@comments = @comments.includes(:creator, :updater, post: :uploader)
@comments = @comments.includes(:votes) if CurrentUser.is_member?

10
app/controllers/explore/posts_controller.rb
View File

@@ -22,23 +22,25 @@ module Explore
def viewed
@date, @scale, @min_date, @max_date = parse_date(params)
@posts = PostViewCountService.new.popular_posts(@date)
@posts = ReportbooruService.new.popular_posts(@date)
respond_with(@posts)
end
def searches
@date, @scale, @min_date, @max_date = parse_date(params)
@search_service = PopularSearchService.new(@date)
@searches = ReportbooruService.new.post_search_rankings(@date)
respond_with(@searches)
end
def missed_searches
@search_service = MissedSearchService.new
@missed_searches = ReportbooruService.new.missed_search_rankings
respond_with(@missed_searches)
end
private
def parse_date(params)
date = params[:date].present? ? Date.parse(params[:date]) : Time.zone.today
date = params[:date].present? ? Date.parse(params[:date]) : Date.today
scale = params[:scale].in?(["day", "week", "month"]) ? params[:scale] : "day"
min_date = date.send("beginning_of_#{scale}")
max_date = date.send("next_#{scale}").send("beginning_of_#{scale}")

2
app/controllers/iqdb_queries_controller.rb
View File

@@ -5,7 +5,7 @@ class IqdbQueriesController < ApplicationController
# XXX allow bare search params for backwards compatibility.
search_params.merge!(params.slice(:url, :image_url, :file_url, :post_id, :limit, :similarity, :high_similarity).permit!)
@high_similarity_matches, @low_similarity_matches, @matches = IqdbProxy.search(search_params)
@high_similarity_matches, @low_similarity_matches, @matches = IqdbProxy.new.search(search_params)
respond_with(@matches, template: "iqdb_queries/show")
end

48
app/controllers/mock_services_controller.rb Normal file
View File

@@ -0,0 +1,48 @@
class MockServicesController < ApplicationController
skip_forgery_protection
respond_to :json
before_action do
raise User::PrivilegeError if Rails.env.production?
end
def recommender_recommend
@data = posts.map { |post| [post.id, rand(0.0..1.0)] }
render json: @data
end
def recommender_similar
@data = posts.map { |post| [post.id, rand(0.0..1.0)] }
render json: @data
end
def reportbooru_missed_searches
@data = tags.map { |tag| "#{tag.name} #{rand(1.0..1000.0)}" }.join("\n")
render json: @data
end
def reportbooru_post_searches
@data = tags.map { |tag| [tag.name, rand(1..1000)] }
render json: @data
end
def reportbooru_post_views
@data = posts.map { |post| [post.id, rand(1..1000)] }
render json: @data
end
def iqdbs_similar
@data = posts.map { |post| { post_id: post.id, score: rand(0..100)} }
render json: @data
end
private
def posts(limit = 10)
Post.last(limit)
end
def tags(limit = 10)
Tag.order(post_count: :desc).limit(limit)
end
end

40
app/controllers/static_controller.rb
View File

@@ -1,4 +1,7 @@
class StaticController < ApplicationController
def privacy_policy
end
def terms_of_service
end
@@ -13,13 +16,40 @@ class StaticController < ApplicationController
redirect_to wiki_page_path("help:dtext") unless request.format.js?
end
def opensearch
end
def site_map
end
def sitemap
@popular_search_service = PopularSearchService.new(Date.yesterday)
@posts = Post.where("created_at > ?", 1.week.ago).order(score: :desc).limit(200)
@posts = @posts.select(&:visible?)
render layout: false
def sitemap_index
@sitemap = params[:sitemap]
@limit = params.fetch(:limit, 10000).to_i
case @sitemap
when "artists"
@relation = Artist.undeleted
@search = { is_deleted: "false" }
when "forum_topics"
@relation = ForumTopic.undeleted
@search = { is_deleted: "false" }
when "pools"
@relation = Pool.undeleted
@search = { is_deleted: "false" }
when "posts"
@relation = Post.order(id: :asc)
@serach = {}
when "tags"
@relation = Tag.nonempty
@search = {}
when "users"
@relation = User.all
@search = {}
when "wiki_pages"
@relation = WikiPage.undeleted
@search = { is_deleted: "false" }
else
raise NotImplementedError
end
end
end

2
app/controllers/uploads_controller.rb
View File

@@ -21,7 +21,7 @@ class UploadsController < ApplicationController
def image_proxy
authorize Upload
resp = ImageProxy.get_image(params[:url])
send_data resp.body, :type => resp.content_type, :disposition => "inline"
send_data resp.body, type: resp.mime_type, disposition: "inline"
end
def index

2
app/controllers/user_name_change_requests_controller.rb
View File

@@ -19,7 +19,7 @@ class UserNameChangeRequestsController < ApplicationController
end
def index
@change_requests = authorize UserNameChangeRequest.visible(CurrentUser.user).order("id desc").paginate(params[:page], :limit => params[:limit])
@change_requests = authorize UserNameChangeRequest.visible(CurrentUser.user).paginated_search(params)
respond_with(@change_requests)
end
end

6
app/controllers/users_controller.rb
View File

@@ -27,7 +27,7 @@ class UsersController < ApplicationController
def index
if params[:name].present?
@user = User.find_by_name!(params[:name])
redirect_to user_path(@user)
redirect_to user_path(@user, variant: params[:variant])
return
end
@@ -42,7 +42,9 @@ class UsersController < ApplicationController
def show
@user = authorize User.find(params[:id])
respond_with(@user, methods: @user.full_attributes)
respond_with(@user, methods: @user.full_attributes) do |format|
format.html.tooltip { render layout: false }
end
end
def profile

49
app/helpers/application_helper.rb
View File

@@ -121,6 +121,10 @@ module ApplicationHelper
raw content_tag(:time, duration, datetime: datetime, title: title)
end
def humanized_number(number)
number_to_human number, units: { thousand: "k", million: "m" }, format: "%n%u"
end
def time_ago_in_words_tagged(time, compact: false)
if time.nil?
tag.em(tag.time("unknown"))
@@ -162,8 +166,10 @@ module ApplicationHelper
end
end
def link_to_ip(ip)
link_to ip, ip_addresses_path(search: { ip_addr: ip, group_by: "user" })
def link_to_ip(ip, shorten: false, **options)
ip_addr = IPAddr.new(ip.to_s)
ip_addr.prefix = 64 if ip_addr.ipv6? && shorten
link_to ip_addr.to_s, ip_addresses_path(search: { ip_addr: ip, group_by: "user" }), **options
end
def link_to_search(search)
@@ -186,12 +192,13 @@ module ApplicationHelper
def link_to_user(user)
return "anonymous" if user.blank?
user_class = "user-#{user.level_string.downcase}"
user_class = "user user-#{user.level_string.downcase}"
user_class += " user-post-approver" if user.can_approve_posts?
user_class += " user-post-uploader" if user.can_upload_free?
user_class += " user-banned" if user.is_banned?
user_class += " with-style" if CurrentUser.user.style_usernames?
link_to(user.pretty_name, user_path(user), :class => user_class)
data = { "user-id": user.id, "user-name": user.name, "user-level": user.level }
link_to(user.pretty_name, user_path(user), class: user_class, data: data)
end
def mod_link_to_user(user, positive_or_negative)
@@ -217,21 +224,8 @@ module ApplicationHelper
tag.div(text, class: "prose", **options)
end
def dtext_field(object, name, options = {})
options[:name] ||= name.capitalize
options[:input_id] ||= "#{object}_#{name}"
options[:input_name] ||= "#{object}[#{name}]"
options[:value] ||= instance_variable_get("@#{object}").try(name)
options[:preview_id] ||= "dtext-preview"
options[:classes] ||= ""
options[:hint] ||= ""
options[:type] ||= "text"
render "dtext/form", options
end
def dtext_preview_button(object, name, input_id: "#{object}_#{name}", preview_id: "dtext-preview")
tag.input value: "Preview", type: "button", class: "dtext-preview-button", "data-input-id": input_id, "data-preview-id": preview_id
def dtext_preview_button(preview_field)
tag.input value: "Preview", type: "button", class: "dtext-preview-button", "data-preview-field": preview_field
end
def quick_search_form_for(attribute, url, name, autocomplete: nil, redirect: false, &block)
@@ -266,7 +260,7 @@ module ApplicationHelper
def body_attributes(user, params, current_item = nil)
current_user_data_attributes = data_attributes_for(user, "current-user", current_user_attributes)
if current_item.present? && current_item.respond_to?(:html_data_attributes) && current_item.respond_to?(:model_name)
if !current_item.nil? && current_item.respond_to?(:html_data_attributes) && current_item.respond_to?(:model_name)
model_name = current_item.model_name.singular.dasherize
model_attributes = current_item.html_data_attributes
current_item_data_attributes = data_attributes_for(current_item, model_name, model_attributes)
@@ -353,6 +347,19 @@ module ApplicationHelper
end
end
def canonical_url(url = nil)
if url.present?
content_for(:canonical_url) { url }
elsif content_for(:canonical_url).present?
content_for(:canonical_url)
else
request_params = request.params.sort.to_h.with_indifferent_access
request_params.delete(:page) if request_params[:page].to_i == 1
request_params.delete(:limit)
url_for(**request_params, host: Danbooru.config.hostname, only_path: false)
end
end
def atom_feed_tag(title, url = {})
content_for(:html_header, auto_discovery_link_tag(:atom, url, title: title))
end

4
app/helpers/pagination_helper.rb
View File

@@ -20,8 +20,8 @@ module PaginationHelper
params[:page] =~ /[ab]/ || records.current_page >= Danbooru.config.max_numbered_pages
end
def numbered_paginator(records, switch_to_sequential = true)
if use_sequential_paginator?(records) && switch_to_sequential
def numbered_paginator(records)
if use_sequential_paginator?(records)
return sequential_paginator(records)
end

65
app/helpers/seo_helper.rb Normal file
View File

@@ -0,0 +1,65 @@
# https://yoast.com/structured-data-schema-ultimate-guide/
# https://technicalseo.com/tools/schema-markup-generator/
# https://developers.google.com/search/docs/data-types/sitelinks-searchbox
# https://developers.google.com/search/docs/data-types/logo
# https://search.google.com/structured-data/testing-tool/u/0/
# https://search.google.com/test/rich-results
# https://schema.org/Organization
# https://schema.org/WebSite
module SeoHelper
def site_description
"#{Danbooru.config.canonical_app_name} is the original anime image booru. Search millions of anime pictures categorized by thousands of tags."
end
# https://developers.google.com/search/docs/data-types/video#video-object
def json_ld_video_data(post)
json_ld_tag({
"@context": "https://schema.org",
"@type": "VideoObject",
"name": page_title,
"description": meta_description,
"uploadDate": post.created_at.iso8601,
"thumbnailUrl": post.preview_file_url,
"contentUrl": post.file_url,
})
end
def json_ld_website_data
urls = [
Danbooru.config.twitter_url,
Danbooru.config.discord_server_url,
Danbooru.config.source_code_url,
"https://en.wikipedia.org/wiki/Danbooru"
].compact
json_ld_tag({
"@context": "https://schema.org",
"@graph": [
{
"@type": "Organization",
url: root_url(host: Danbooru.config.hostname),
name: Danbooru.config.app_name,
logo: "#{root_url(host: Danbooru.config.hostname)}images/danbooru-logo-500x500.png",
sameAs: urls
},
{
"@type": "WebSite",
"@id": root_url(anchor: "website", host: Danbooru.config.hostname),
"url": root_url(host: Danbooru.config.hostname),
"name": Danbooru.config.app_name,
"description": site_description,
"potentialAction": [{
"@type": "SearchAction",
"target": "#{posts_url(host: Danbooru.config.hostname)}?tags={search_term_string}",
"query-input": "required name=search_term_string"
}]
}
]
})
end
def json_ld_tag(data)
tag.script(data.to_json.html_safe, type: "application/ld+json")
end
end

4
app/javascript/packs/application.js
View File

@@ -6,12 +6,12 @@ function importAll(r) {
require('@rails/ujs').start();
require('hammerjs');
require('stupid-table-plugin');
require('jquery-hotkeys');
// should start looking for nodejs replacements
importAll(require.context('../vendor', true, /\.js$/));
require('jquery');
require("jquery-ui/ui/effects/effect-shake");
require("jquery-ui/ui/widgets/autocomplete");
require("jquery-ui/ui/widgets/button");
@@ -33,6 +33,7 @@ require("@fortawesome/fontawesome-free/css/regular.css");
importAll(require.context('../src/javascripts', true, /\.js(\.erb)?$/));
importAll(require.context('../src/styles', true, /\.s?css(?:\.erb)?$/));
export { default as jQuery } from "jquery";
export { default as Autocomplete } from '../src/javascripts/autocomplete.js.erb';
export { default as Blacklist } from '../src/javascripts/blacklists.js';
export { default as Comment } from '../src/javascripts/comments.js';
@@ -47,5 +48,6 @@ export { default as PostVersion } from '../src/javascripts/post_version.js';
export { default as RelatedTag } from '../src/javascripts/related_tag.js';
export { default as Shortcuts } from '../src/javascripts/shortcuts.js';
export { default as Upload } from '../src/javascripts/uploads.js.erb';
export { default as UserTooltip } from '../src/javascripts/user_tooltips.js';
export { default as Utility } from '../src/javascripts/utility.js';
export { default as Ugoira } from '../src/javascripts/ugoira.js';

18
app/javascript/src/javascripts/autocomplete.js.erb
View File

@@ -9,6 +9,7 @@ Autocomplete.ORDER_METATAGS = <%= PostQueryBuilder::ORDER_METATAGS.to_json.html_
Autocomplete.DISAPPROVAL_REASONS = <%= PostDisapproval::REASONS.to_json.html_safe %>;
/* eslint-enable */
Autocomplete.MISC_STATUSES = ["deleted", "active", "pending", "flagged", "banned", "modqueue", "unmoderated"];
Autocomplete.TAG_PREFIXES = "-|~|" + Object.keys(Autocomplete.TAG_CATEGORIES).map(category => category + ":").join("|");
Autocomplete.METATAGS_REGEX = Autocomplete.METATAGS.concat(Object.keys(Autocomplete.TAG_CATEGORIES)).join("|");
Autocomplete.TERM_REGEX = new RegExp(`([-~]*)(?:(${Autocomplete.METATAGS_REGEX}):)?(\\S*)$`, "i");
@@ -37,7 +38,7 @@ Autocomplete.initialize_all = function() {
});
this.initialize_tag_autocomplete();
this.initialize_mention_autocomplete($(".autocomplete-mentions textarea"));
this.initialize_mention_autocomplete($("form div.input.dtext textarea"));
this.initialize_fields($('[data-autocomplete="tag"]'), Autocomplete.tag_source);
this.initialize_fields($('[data-autocomplete="artist"]'), Autocomplete.artist_source);
this.initialize_fields($('[data-autocomplete="pool"]'), Autocomplete.pool_source);
@@ -248,9 +249,6 @@ Autocomplete.render_item = function(list, item) {
} else if (item.type === "user") {
var level_class = "user-" + item.level.toLowerCase();
$link.addClass(level_class);
if (CurrentUser.data("style-usernames")) {
$link.addClass("with-style");
}
} else if (item.type === "pool") {
$link.addClass("pool-category-" + item.category);
}
@@ -268,9 +266,7 @@ Autocomplete.render_item = function(list, item) {
Autocomplete.static_metatags = {
order: Autocomplete.ORDER_METATAGS,
status: [
"any", "deleted", "active", "pending", "flagged", "banned", "modqueue", "unmoderated"
],
status: ["any"].concat(Autocomplete.MISC_STATUSES),
rating: [
"safe", "questionable", "explicit"
],
@@ -280,12 +276,8 @@ Autocomplete.static_metatags = {
embedded: [
"true", "false"
],
child: [
"any", "none"
],
parent: [
"any", "none"
],
child: ["any", "none"].concat(Autocomplete.MISC_STATUSES),
parent: ["any", "none"].concat(Autocomplete.MISC_STATUSES),
filetype: [
"jpg", "png", "gif", "swf", "zip", "webm", "mp4"
],

7
app/javascript/src/javascripts/dtext.js
View File

@@ -39,8 +39,11 @@ Dtext.call_edit = function(e, $button, $input, $preview) {
Dtext.click_button = function(e) {
var $button = $(e.target);
var $input = $("#" + $button.data("input-id"));
var $preview = $("#" + $button.data("preview-id"));
var $form = $button.parents("form");
var fieldName = $button.data("preview-field");
var $inputContainer = $form.find(`div.input.${fieldName} .dtext-previewable`);
var $input = $inputContainer.find("> input, > textarea");
var $preview = $inputContainer.find("div.dtext-preview");
if ($button.val().match(/preview/i)) {
Dtext.call_preview(e, $button, $input, $preview);

14
app/javascript/src/javascripts/forum_posts.js
View File

@@ -7,17 +7,15 @@ ForumPost.initialize_all = function() {
}
ForumPost.initialize_edit_links = function() {
$(".edit_forum_post_link").on("click.danbooru", function(e) {
var link_id = $(this).attr("id");
var forum_post_id = link_id.match(/^edit_forum_post_link_(\d+)$/)[1];
$("#edit_forum_post_" + forum_post_id).fadeToggle("fast");
$(document).on("click.danbooru", ".edit_forum_post_link", function(e) {
let $form = $(this).parents("article.forum-post").find("form.edit_forum_post");
$form.fadeToggle("fast");
e.preventDefault();
});
$(".edit_forum_topic_link").on("click.danbooru", function(e) {
var link_id = $(this).attr("id");
var forum_topic_id = link_id.match(/^edit_forum_topic_link_(\d+)$/)[1];
$("#edit_forum_topic_" + forum_topic_id).fadeToggle("fast");
$(document).on("click.danbooru", ".edit_forum_topic_link", function(e) {
let $form = $(this).parents("article.forum-post").find("form.edit_forum_topic");
$form.fadeToggle("fast");
e.preventDefault();
});

5
app/javascript/src/javascripts/pools.js
View File

@@ -20,11 +20,6 @@ Pool.initialize_add_to_pool_link = function() {
e.preventDefault();
$("#add-to-pool-dialog").dialog("open");
});
$("#recent-pools li").on("click.danbooru", function(e) {
e.preventDefault();
$("#pool_name").val($(this).attr("data-value"));
});
}
Pool.initialize_simple_edit = function() {

2
app/javascript/src/javascripts/post_mode_menu.js
View File

@@ -70,7 +70,7 @@ PostModeMenu.initialize_edit_form = function() {
$(document).on("click.danbooru", "#quick-edit-form input[type=submit]", async function(e) {
e.preventDefault();
let post_id = $("#quick-edit-form").data("post-id");
let post_id = $("#quick-edit-form").attr("data-post-id");
await Post.update(post_id, "quick-edit", { post: { tag_string: $("#post_tag_string").val() }});
});
}

163
app/javascript/src/javascripts/post_tooltips.js
View File

@@ -1,28 +1,76 @@
import CurrentUser from './current_user'
import Utility from './utility'
require('qtip2');
require('qtip2/dist/jquery.qtip.css');
import CurrentUser from './current_user';
import Utility from './utility';
import { delegate, hideAll } from 'tippy.js';
import 'tippy.js/dist/tippy.css';
let PostTooltip = {};
PostTooltip.render_tooltip = async function (event, qtip) {
PostTooltip.POST_SELECTOR = "*:not(.ui-sortable-handle) > .post-preview img, .dtext-post-id-link";
PostTooltip.SHOW_DELAY = 500;
PostTooltip.HIDE_DELAY = 125;
PostTooltip.DURATION = 250;
PostTooltip.initialize = function () {
if (PostTooltip.disabled()) {
return;
}
delegate("body", {
allowHTML: true,
appendTo: document.body,
delay: [PostTooltip.SHOW_DELAY, PostTooltip.HIDE_DELAY],
duration: PostTooltip.DURATION,
interactive: true,
maxWidth: "none",
target: PostTooltip.POST_SELECTOR,
theme: "common-tooltip post-tooltip",
touch: false,
onCreate: PostTooltip.on_create,
onShow: PostTooltip.on_show,
onHide: PostTooltip.on_hide,
});
$(document).on("click.danbooru.postTooltip", ".post-tooltip-disable", PostTooltip.on_disable_tooltips);
};
PostTooltip.on_create = function (instance) {
let title = instance.reference.getAttribute("title");
if (title) {
instance.reference.setAttribute("data-title", title);
instance.reference.setAttribute("title", "");
}
};
PostTooltip.on_show = async function (instance) {
let post_id = null;
let preview = false;
let $target = $(instance.reference);
let $tooltip = $(instance.popper);
if ($(this).is(".dtext-post-id-link")) {
hideAll({ exclude: instance });
// skip if tooltip has already been rendered.
if ($tooltip.has(".post-tooltip-body").length) {
return;
}
if ($target.is(".dtext-post-id-link")) {
preview = true;
post_id = /\/posts\/(\d+)/.exec($(this).attr("href"))[1];
post_id = /\/posts\/(\d+)/.exec($target.attr("href"))[1];
} else {
post_id = $(this).parents("[data-id]").data("id");
post_id = $target.parents("[data-id]").data("id");
}
try {
qtip.cache.request = $.get(`/posts/${post_id}`, { variant: "tooltip", preview: preview });
let html = await qtip.cache.request;
$tooltip.addClass("tooltip-loading");
qtip.set("content.text", html);
qtip.elements.tooltip.removeClass("post-tooltip-loading");
instance._request = $.get(`/posts/${post_id}`, { variant: "tooltip", preview: preview });
let html = await instance._request;
instance.setContent(html);
$tooltip.removeClass("tooltip-loading");
} catch (error) {
if (error.status !== 0 && error.statusText !== "abort") {
Utility.error(`Error displaying tooltip for post #${post_id} (error: ${error.status} ${error.statusText})`);
@@ -30,98 +78,19 @@ PostTooltip.render_tooltip = async function (event, qtip) {
}
};
// Hide the tooltip the first time it is shown, while we wait on the ajax call to complete.
PostTooltip.on_show = function (event, qtip) {
if (!qtip.cache.hasBeenShown) {
qtip.elements.tooltip.addClass("post-tooltip-loading");
qtip.cache.hasBeenShown = true;
PostTooltip.on_hide = function (instance) {
if (instance._request?.state() === "pending") {
instance._request.abort();
}
};
PostTooltip.POST_SELECTOR = "*:not(.ui-sortable-handle) > .post-preview img, .dtext-post-id-link";
// http://qtip2.com/options
PostTooltip.QTIP_OPTIONS = {
style: {
classes: "qtip-light post-tooltip",
tip: false
},
content: PostTooltip.render_tooltip,
overwrite: false,
position: {
viewport: true,
my: "bottom left",
at: "top left",
effect: false,
adjust: {
y: -2,
method: "shift",
},
},
show: {
solo: true,
delay: 750,
effect: false,
ready: true,
event: "mouseenter",
},
hide: {
delay: 250,
fixed: true,
effect: false,
event: "unfocus click mouseleave",
},
events: {
show: PostTooltip.on_show,
},
};
PostTooltip.initialize = function () {
$(document).on("mouseenter.danbooru.postTooltip", PostTooltip.POST_SELECTOR, function (event) {
if (PostTooltip.disabled()) {
$(this).qtip("disable");
} else {
$(this).qtip(PostTooltip.QTIP_OPTIONS, event);
}
});
// Cancel pending ajax requests when we mouse out of the thumbnail.
$(document).on("mouseleave.danbooru.postTooltip", PostTooltip.POST_SELECTOR, function (event) {
let qtip = $(event.target).qtip("api");
if (qtip && qtip.cache && qtip.cache.request && qtip.cache.request.state() === "pending") {
qtip.cache.request.abort();
}
});
$(document).on("click.danbooru.postTooltip", ".post-tooltip-disable", PostTooltip.on_disable_tooltips);
// Hide tooltips when pressing keys or clicking thumbnails.
$(document).on("keydown.danbooru.postTooltip", PostTooltip.hide);
$(document).on("click.danbooru.postTooltip", PostTooltip.POST_SELECTOR, PostTooltip.hide);
// Disable tooltips on touch devices. https://developer.mozilla.org/en-US/docs/Web/API/Touch_events/Supporting_both_TouchEvent_and_MouseEvent
PostTooltip.isTouching = false;
$(document).on("touchstart.danbooru.postTooltip", function (event) { PostTooltip.isTouching = true; });
$(document).on("touchend.danbooru.postTooltip", function (event) { PostTooltip.isTouching = false; });
};
PostTooltip.hide = function (event) {
// Hide on any key except control (user may be control-clicking link inside tooltip).
if (event.type === "keydown" && event.ctrlKey === true) {
return;
}
$(".post-tooltip:visible").qtip("hide");
};
PostTooltip.disabled = function (event) {
return PostTooltip.isTouching || CurrentUser.data("disable-post-tooltips");
return CurrentUser.data("disable-post-tooltips");
};
PostTooltip.on_disable_tooltips = async function (event) {
event.preventDefault();
$(event.target).parents(".qtip").qtip("hide");
hideAll();
if (CurrentUser.data("is-anonymous")) {
Utility.notice('You must <a href="/session/new">login</a> to disable tooltips');

12
app/javascript/src/javascripts/posts.js.erb
View File

@@ -300,10 +300,10 @@ Post.initialize_favlist = function() {
});
}
Post.view_original = function(e) {
Post.view_original = function(e = null) {
if (Utility.test_max_width(660)) {
// Do the default behavior (navigate to image)
return false;
return;
}
var $image = $("#image");
@@ -316,13 +316,13 @@ Post.view_original = function(e) {
});
Note.Box.scale_all();
$("body").attr("data-post-current-image-size", "original");
return false;
e?.preventDefault();
}
Post.view_large = function(e) {
Post.view_large = function(e = null) {
if (Utility.test_max_width(660)) {
// Do the default behavior (navigate to image)
return false;
return;
}
var $image = $("#image");
@@ -335,7 +335,7 @@ Post.view_large = function(e) {
});
Note.Box.scale_all();
$("body").attr("data-post-current-image-size", "large");
return false;
e?.preventDefault();
}
Post.toggle_fit_window = function(e) {

11
app/javascript/src/javascripts/saved_searches.js
View File

@@ -1,11 +0,0 @@
let SavedSearch = {};
SavedSearch.initialize_all = function() {
if ($("#c-saved-searches").length) {
$("#c-saved-searches table").stupidtable();
}
}
$(SavedSearch.initialize_all);
export default SavedSearch

85
app/javascript/src/javascripts/user_tooltips.js Normal file
View File

@@ -0,0 +1,85 @@
import Utility from './utility';
import { delegate } from 'tippy.js';
import 'tippy.js/dist/tippy.css';
let UserTooltip = {};
UserTooltip.SELECTOR = "*:not(.user-tooltip-name) > a.user, a.dtext-user-id-link, a.dtext-user-mention-link";
UserTooltip.SHOW_DELAY = 500;
UserTooltip.HIDE_DELAY = 125;
UserTooltip.DURATION = 250;
UserTooltip.MAX_WIDTH = 600;
UserTooltip.initialize = function () {
delegate("body", {
allowHTML: true,
appendTo: document.body,
delay: [UserTooltip.SHOW_DELAY, UserTooltip.HIDE_DELAY],
duration: UserTooltip.DURATION,
interactive: true,
maxWidth: UserTooltip.MAX_WIDTH,
target: UserTooltip.SELECTOR,
theme: "common-tooltip user-tooltip",
touch: false,
onShow: UserTooltip.on_show,
onHide: UserTooltip.on_hide,
});
delegate("body", {
allowHTML: true,
interactive: true,
theme: "common-tooltip",
target: ".user-tooltip-menu-button",
placement: "bottom",
touch: false,
trigger: "click",
content: (element) => {
return $(element).parents(".user-tooltip").find(".user-tooltip-menu").get(0);
}
});
};
UserTooltip.on_show = async function (instance) {
let $target = $(instance.reference);
let $tooltip = $(instance.popper);
// skip if tooltip has already been rendered.
if ($tooltip.has(".user-tooltip-body").length) {
return;
}
try {
$tooltip.addClass("tooltip-loading");
if ($target.is("a.dtext-user-id-link")) {
let user_id = /\/users\/(\d+)/.exec($target.attr("href"))[1];
instance._request = $.get(`/users/${user_id}`, { variant: "tooltip" });
} else if ($target.is("a.user")) {
let user_id = $target.attr("data-user-id");
instance._request = $.get(`/users/${user_id}`, { variant: "tooltip" });
} else if ($target.is("a.dtext-user-mention-link")) {
let user_name = $target.attr("data-user-name");
instance._request = $.get(`/users`, { name: user_name, variant: "tooltip" });
}
let html = await instance._request;
instance.setContent(html);
$tooltip.removeClass("tooltip-loading");
} catch (error) {
if (error.status !== 0 && error.statusText !== "abort") {
Utility.error(`Error displaying tooltip (error: ${error.status} ${error.statusText})`);
}
}
};
UserTooltip.on_hide = function (instance) {
if (instance._request?.state() === "pending") {
instance._request.abort();
}
}
$(document).ready(UserTooltip.initialize);
export default UserTooltip

8
app/javascript/src/styles/base/020_base.scss
View File

@@ -129,6 +129,14 @@ table tfoot {
font-size: 0.9em;
}
a.link-plain {
color: unset;
&:hover {
text-decoration: underline;
}
}
.fixed-width-container {
max-width: 70em;
}

13
app/javascript/src/styles/base/040_colors.css
View File

@@ -2,6 +2,7 @@
--body-background-color: white;
--text-color: hsl(0, 0%, 15%);
--inverse-text-color: white;
--muted-text-color: hsl(0, 0%, 55%);
--header-color: hsl(0, 0%, 15%);
@@ -72,7 +73,8 @@
--comment-sticky-background-color: var(--subnav-menu-background-color);
--post-tooltip-background-color: var(--body-background-color);
--post-tooltip-border-color: #767676;
--post-tooltip-border-color: hsla(210, 100%, 3%, 0.15);
--post-tooltip-box-shadow: 0 4px 14px -2px hsla(210, 100%, 3%, 0.10);
--post-tooltip-header-background-color: var(--subnav-menu-background-color);
--post-tooltip-info-color: #555;
--post-tooltip-scrollbar-background: #999999;
@@ -81,6 +83,9 @@
--post-tooltip-scrollbar-track-background: #EEEEEE;
--post-tooltip-scrollbar-track-border: 0 none white;
--user-tooltip-positive-feedback-color: orange;
--user-tooltip-negative-feedback-color: red;
--preview-current-post-background: rgba(0, 0, 0, 0.1);
--autocomplete-selected-background-color: var(--subnav-menu-background-color);
@@ -200,6 +205,7 @@
--user-platinum-color: gray;
--user-gold-color: #00F;
--user-member-color: var(--link-color);
--user-banned-color: black;
--news-updates-background: #EEE;
--news-updates-border: 2px solid #666;
@@ -260,6 +266,7 @@ body[data-current-user-theme="dark"] {
/* main text colors */
--text-color: var(--grey-5);
--inverse-text-color: white;
--muted-text-color: var(--grey-4);
--header-color: var(--grey-6);
@@ -282,6 +289,7 @@ body[data-current-user-theme="dark"] {
--collection-pool-color: var(--general-tag-color);
--collection-pool-hover-color: var(--general-tag-hover-color);
--user-banned-color: var(--grey-1);
--user-member-color: var(--blue-1);
--user-gold-color: var(--yellow-1);
--user-platinum-color: var(--grey-4);
@@ -394,6 +402,9 @@ body[data-current-user-theme="dark"] {
--post-tooltip-scrollbar-track-background: var(--grey-1);
--post-tooltip-scrollbar-track-border: none;
--user-tooltip-positive-feedback-color: var(--yellow-1);
--user-tooltip-negative-feedback-color: var(--red-1);
--preview-pending-color: var(--blue-1);
--preview-flagged-color: var(--red-1);
--preview-deleted-color: var(--grey-5);

3
app/javascript/src/styles/common/messages.scss
View File

@@ -17,7 +17,8 @@ div.list-of-messages {
a.message-timestamp {
font-style: italic;
color: var(--text-color);
font-size: 0.90em;
color: var(--muted-text-color);
&:hover { text-decoration: underline; }
}
}

2
app/javascript/src/styles/common/simple_form.scss
View File

@@ -39,7 +39,7 @@ form.simple_form {
padding-left: 1em;
}
&.text {
&.text, &.dtext {
.hint {
padding-left: 0;
display: block;

14
app/javascript/src/styles/common/user_styles.scss
View File

@@ -1,23 +1,25 @@
a.user-admin.with-style {
body[data-current-user-style-usernames="true"] {
a.user-admin {
color: var(--user-admin-color);
}
a.user-moderator.with-style {
a.user-moderator {
color: var(--user-moderator-color);
}
a.user-builder.with-style {
a.user-builder {
color: var(--user-builder-color);
}
a.user-platinum.with-style {
a.user-platinum {
color: var(--user-platinum-color);
}
a.user-gold.with-style {
a.user-gold {
color: var(--user-gold-color);
}
a.user-member.with-style {
a.user-member {
color: var(--user-member-color);
}
}

51
app/javascript/src/styles/specific/common_tooltips.scss Normal file
View File

@@ -0,0 +1,51 @@
div[data-tippy-root].tooltip-loading {
visibility: hidden !important;
}
.tippy-box[data-theme~="common-tooltip"] {
box-sizing: border-box;
border: 1px solid var(--post-tooltip-border-color);
border-radius: 4px;
color: var(--text-color);
background-color: var(--post-tooltip-background-color);
background-clip: padding-box;
box-shadow: var(--post-tooltip-box-shadow);
/* bordered arrow styling; see https://github.com/atomiks/tippyjs/blob/master/src/scss/themes/light-border.scss */
&[data-placement^=bottom] {
> .tippy-arrow:before {
border-bottom-color: var(--post-tooltip-background-color);
bottom: 16px;
}
> .tippy-arrow:after {
border-bottom-color: var(--post-tooltip-border-color);
border-width: 0 7px 7px;
top: -8px;
left: 1px;
}
}
&[data-placement^=top] {
> .tippy-arrow:before {
border-top-color: var(--post-tooltip-background-color);
}
> .tippy-arrow:after {
border-top-color: var(--post-tooltip-border-color);
border-width: 7px 7px 0;
top: 17px;
left: 1px;
}
}
> .tippy-arrow:after {
border-color: transparent;
border-style: solid;
content: "";
position: absolute;
z-index: -1;
}
}

54
app/javascript/src/styles/specific/post_tooltips.scss
View File

@@ -1,6 +1,5 @@
$tooltip-line-height: 16px;
$tooltip-body-height: $tooltip-line-height * 6; // 6 lines high.
$tooltip-width: 164px * 3 - 10; // 3 thumbnails wide.
$tooltip-body-height: $tooltip-line-height * 4; // 4 lines high.
@mixin thin-scrollbar {
&::-webkit-scrollbar {
@@ -46,16 +45,13 @@ $tooltip-width: 164px * 3 - 10; // 3 thumbnails wide.
}
}
.post-tooltip {
max-width: $tooltip-width;
min-width: $tooltip-width;
box-sizing: border-box;
.tippy-box[data-theme~="post-tooltip"] {
min-width: 20em;
max-width: 40em !important;
font-size: 11px;
line-height: $tooltip-line-height;
border-color: var(--post-tooltip-border-color);
background-color: var(--post-tooltip-background-color);
.qtip-content {
.tippy-content {
padding: 0;
> * {
@@ -85,38 +81,32 @@ $tooltip-width: 164px * 3 - 10; // 3 thumbnails wide.
.post-tooltip-body-right { flex: 1; }
}
.post-tooltip-header {
div.post-tooltip-header {
background-color: var(--post-tooltip-header-background-color);
display: flex;
white-space: nowrap;
overflow: hidden;
align-items: center;
.post-tooltip-header-left {
flex: 1;
.post-tooltip-info {
margin-right: 0.5em;
color: var(--post-tooltip-info-color);
font-size: 10px;
flex: 0;
}
.post-tooltip-header-right {
a.user {
margin-right: 0.5em;
overflow: hidden;
white-space: nowrap;
text-overflow: ellipsis;
min-width: 0;
max-width: 11em;
}
.post-tooltip-source {
flex: 1;
text-align: right;
}
.fa-xs {
vertical-align: baseline;
}
.post-tooltip-disable {
margin-left: 0.5em;
}
}
.post-tooltip-info {
margin-left: 0.5em;
color: var(--post-tooltip-info-color);
font-size: 10px;
}
}
&.post-tooltip-loading {
visibility: hidden;
}
}

5
app/javascript/src/styles/specific/privacy_policy.scss Normal file
View File

@@ -0,0 +1,5 @@
#c-static #a-privacy-policy {
.summary {
font-style: italic;
}
}

103
app/javascript/src/styles/specific/user_tooltips.scss Normal file
View File

@@ -0,0 +1,103 @@
.tippy-box[data-theme~="user-tooltip"] {
line-height: 1.25em;
min-width: 350px;
.user-tooltip-header {
margin-bottom: 1em;
display: grid;
grid:
"avatar header-top menu"
"avatar header-bottom menu" /
32px 1fr 32px;
column-gap: 0.25em;
.user-tooltip-avatar {
font-size: 32px;
grid-area: avatar;
align-self: center;
}
.user-tooltip-header-top {
grid-area: header-top;
.user-tooltip-badge {
color: var(--inverse-text-color);
font-size: 0.70em;
padding: 2px 4px;
margin-right: 0.25em;
border-radius: 3px;
&.user-tooltip-badge-admin { background-color: var(--user-admin-color); }
&.user-tooltip-badge-moderator { background-color: var(--user-moderator-color); }
&.user-tooltip-badge-approver { background-color: var(--user-builder-color); }
&.user-tooltip-badge-contributor { background-color: var(--user-builder-color); }
&.user-tooltip-badge-builder { background-color: var(--user-builder-color); }
&.user-tooltip-badge-platinum { background-color: var(--user-platinum-color); }
&.user-tooltip-badge-gold { background-color: var(--user-gold-color); }
&.user-tooltip-badge-member { background-color: var(--user-member-color); }
&.user-tooltip-badge-banned { background-color: var(--user-banned-color); }
&.user-tooltip-badge-positive-feedback {
color: var(--user-tooltip-positive-feedback-color);
border: 1px solid;
}
&.user-tooltip-badge-negative-feedback {
color: var(--user-tooltip-negative-feedback-color);
border: 1px solid;
}
}
}
.user-tooltip-header-bottom {
grid-area: header-bottom;
color: var(--muted-text-color);
font-size: 0.75em;
}
a.user-tooltip-menu-button {
color: var(--muted-text-color);
grid-area: menu;
align-self: center;
text-align: center;
width: 28px;
height: 28px;
line-height: 28px;
border-radius: 50%;
&:hover {
background-color: var(--subnav-menu-background-color);
}
}
> ul.user-tooltip-menu {
display: none;
}
ul.user-tooltip-menu {
.icon {
width: 1.5em;
}
}
}
.user-tooltip-stats {
display: grid;
grid: auto / repeat(3, 1fr);
column-gap: 1em;
row-gap: 0.5em;
.user-tooltip-stat-item {
text-align: center;
.user-tooltip-stat-value {
font-weight: bold;
}
.user-tooltip-stat-name {
font-size: 0.90em;
color: var(--muted-text-color);
}
}
}
}

8
app/logical/apng_inspector.rb
View File

@@ -57,7 +57,7 @@ class APNGInspector
# if we did, file is probably maliciously formed
# fail gracefully without marking the file as corrupt
chunks += 1
if chunks > 100000
if chunks > 100_000
iend_reached = true
break
end
@@ -66,7 +66,8 @@ class APNGInspector
file.seek(current_pos + chunk_len + 4, IO::SEEK_SET)
end
end
return iend_reached
iend_reached
end
def inspect!
@@ -105,6 +106,7 @@ class APNGInspector
if framedata.nil? || framedata.length != 4
return -1
end
return framedata.unpack1("N".freeze)
framedata.unpack1("N".freeze)
end
end

12
app/logical/artist_finder.rb
View File

@@ -6,7 +6,7 @@ module ArtistFinder
SITE_BLACKLIST = [
"artstation.com/artist", # http://www.artstation.com/artist/serafleur/
"www.artstation.com", # http://www.artstation.com/serafleur/
%r!cdn[ab]?\.artstation\.com/p/assets/images/images!i, # https://cdna.artstation.com/p/assets/images/images/001/658/068/large/yang-waterkuma-b402.jpg?1450269769
%r{cdn[ab]?\.artstation\.com/p/assets/images/images}i, # https://cdna.artstation.com/p/assets/images/images/001/658/068/large/yang-waterkuma-b402.jpg?1450269769
"ask.fm", # http://ask.fm/mikuroko_396
"bcyimg.com",
"bcyimg.com/drawer", # https://img9.bcyimg.com/drawer/32360/post/178vu/46229ec06e8111e79558c1b725ebc9e6.jpg
@@ -52,7 +52,7 @@ module ArtistFinder
"hentai-foundry.com",
"hentai-foundry.com/pictures/user", # http://www.hentai-foundry.com/pictures/user/aaaninja/
"hentai-foundry.com/user", # http://www.hentai-foundry.com/user/aaaninja/profile
%r!pictures\.hentai-foundry\.com(?:/\w)?!i, # http://pictures.hentai-foundry.com/a/aaaninja/
%r{pictures\.hentai-foundry\.com(?:/\w)?}i, # http://pictures.hentai-foundry.com/a/aaaninja/
"i.imgur.com", # http://i.imgur.com/Ic9q3.jpg
"instagram.com", # http://www.instagram.com/serafleur.art/
"iwara.tv",
@@ -62,13 +62,15 @@ module ArtistFinder
"monappy.jp",
"monappy.jp/u", # https://monappy.jp/u/abara_bone
"mstdn.jp", # https://mstdn.jp/@oneb
"www.newgrounds.com", # https://jessxjess.newgrounds.com/
"newgrounds.com/art/view/", # https://www.newgrounds.com/art/view/jessxjess/avatar-korra
"nicoseiga.jp",
"nicoseiga.jp/priv", # http://lohas.nicoseiga.jp/priv/2017365fb6cfbdf47ad26c7b6039feb218c5e2d4/1498430264/6820259
"nicovideo.jp",
"nicovideo.jp/user", # http://www.nicovideo.jp/user/317609
"nicovideo.jp/user/illust", # http://seiga.nicovideo.jp/user/illust/29075429
"nijie.info", # http://nijie.info/members.php?id=15235
%r!nijie\.info/nijie_picture!i, # http://pic03.nijie.info/nijie_picture/32243_20150609224803_0.png
%r{nijie\.info/nijie_picture}i, # http://pic03.nijie.info/nijie_picture/32243_20150609224803_0.png
"patreon.com", # http://patreon.com/serafleur
"pawoo.net", # https://pawoo.net/@148nasuka
"pawoo.net/web/accounts", # https://pawoo.net/web/accounts/228341
@@ -120,7 +122,7 @@ module ArtistFinder
SITE_BLACKLIST_REGEXP = Regexp.union(SITE_BLACKLIST.map do |domain|
domain = Regexp.escape(domain) if domain.is_a?(String)
%r!\Ahttps?://(?:[a-zA-Z0-9_-]+\.)*#{domain}/\z!i
%r{\Ahttps?://(?:[a-zA-Z0-9_-]+\.)*#{domain}/\z}i
end)
def find_artists(url)
@@ -128,7 +130,7 @@ module ArtistFinder
artists = []
while artists.empty? && url.size > 10
u = url.sub(/\/+$/, "") + "/"
u = url.sub(%r{/+$}, "") + "/"
u = u.to_escaped_for_sql_like.gsub(/\*/, '%') + '%'
artists += Artist.joins(:urls).where(["artists.is_deleted = FALSE AND artist_urls.normalized_url LIKE ? ESCAPE E'\\\\'", u]).limit(10).order("artists.name").all
url = File.dirname(url) + "/"

2
app/logical/bulk_update_request_processor.rb
View File

@@ -148,8 +148,6 @@ class BulkUpdateRequestProcessor
end.join("\n")
end
private
def self.is_tag_move_allowed?(antecedent_name, consequent_name)
antecedent_tag = Tag.find_by_name(Tag.normalize_name(antecedent_name))
consequent_tag = Tag.find_by_name(Tag.normalize_name(consequent_name))

9
app/logical/cloudflare_service.rb
View File

@@ -9,15 +9,6 @@ class CloudflareService
api_token.present? && zone.present?
end
def ips(expiry: 24.hours)
response = Danbooru::Http.cache(expiry).get("https://api.cloudflare.com/client/v4/ips")
return [] if response.code != 200
result = response.parse["result"]
ips = result["ipv4_cidrs"] + result["ipv6_cidrs"]
ips.map { |ip| IPAddr.new(ip) }
end
def purge_cache(urls)
return unless enabled?

2
app/logical/concerns/has_bit_flags.rb
View File

@@ -16,7 +16,7 @@ module HasBitFlags
end
define_method("#{attribute}=") do |val|
if val.to_s =~ /t|1|y/
if val.to_s =~ /[t1y]/
send("#{field}=", send(field) | bit_flag)
else
send("#{field}=", send(field) & ~bit_flag)

2
app/logical/concerns/mentionable.rb
View File

@@ -11,8 +11,6 @@ module Mentionable
# - user_field
def mentionable(options = {})
@mentionable_options = options
message_field = mentionable_option(:message_field)
after_save :queue_mention_messages
end

7
app/logical/concerns/searchable.rb
View File

@@ -89,7 +89,7 @@ module Searchable
def where_array_count(attr, value)
qualified_column = "cardinality(#{qualified_column_for(attr)})"
range = PostQueryBuilder.new(nil).parse_range(value, :integer)
where_operator("cardinality(#{qualified_column_for(attr)})", *range)
where_operator(qualified_column, *range)
end
def search_boolean_attribute(attribute, params)
@@ -170,7 +170,7 @@ module Searchable
end
end
def search_text_attribute(attr, params, **options)
def search_text_attribute(attr, params)
if params[attr].present?
where(attr => params[attr])
elsif params[:"#{attr}_eq"].present?
@@ -279,7 +279,8 @@ module Searchable
return find_ordered(parse_ids[1])
end
end
return default_order
default_order
end
def default_order

9
app/logical/current_user.rb
View File

@@ -24,15 +24,6 @@ class CurrentUser
scoped(user, &block)
end
def self.as_system(&block)
if block_given?
scoped(::User.system, "127.0.0.1", &block)
else
self.user = User.system
self.ip_addr = "127.0.0.1"
end
end
def self.user
RequestStore[:current_user]
end

8
app/logical/d_text.rb
View File

@@ -11,7 +11,7 @@ class DText
html = DTextRagel.parse(text, **options)
html = postprocess(html, *data)
html
rescue DTextRagel::Error => e
rescue DTextRagel::Error
""
end
@@ -135,7 +135,7 @@ class DText
fragment = Nokogiri::HTML.fragment(html)
titles = fragment.css("a.dtext-wiki-link").map do |node|
title = node["href"][%r!\A/wiki_pages/(.*)\z!i, 1]
title = node["href"][%r{\A/wiki_pages/(.*)\z}i, 1]
title = CGI.unescape(title)
title = WikiPage.normalize_title(title)
title
@@ -163,7 +163,7 @@ class DText
string = string.dup
string.gsub!(/\s*\[#{tag}\](?!\])\s*/mi, "\n\n[#{tag}]\n\n")
string.gsub!(/\s*\[\/#{tag}\]\s*/mi, "\n\n[/#{tag}]\n\n")
string.gsub!(%r{\s*\[/#{tag}\]\s*}mi, "\n\n[/#{tag}]\n\n")
string.gsub!(/(?:\r?\n){3,}/, "\n\n")
string.strip!
@@ -203,7 +203,7 @@ class DText
end
end
text = text.gsub(/\A[[:space:]]+|[[:space:]]+\z/, "")
text.gsub(/\A[[:space:]]+|[[:space:]]+\z/, "")
end
def self.from_html(text, inline: false, &block)

128
app/logical/danbooru/http.rb
View File

@@ -1,17 +1,49 @@
require "danbooru/http/html_adapter"
require "danbooru/http/xml_adapter"
require "danbooru/http/cache"
require "danbooru/http/redirector"
require "danbooru/http/retriable"
require "danbooru/http/session"
require "danbooru/http/spoof_referrer"
require "danbooru/http/unpolish_cloudflare"
module Danbooru
class Http
DEFAULT_TIMEOUT = 3
class DownloadError < StandardError; end
class FileTooLargeError < StandardError; end
attr_writer :cache, :http
DEFAULT_TIMEOUT = 10
MAX_REDIRECTS = 5
attr_accessor :max_size, :http
class << self
delegate :get, :post, :delete, :cache, :auth, :basic_auth, :headers, to: :new
delegate :get, :head, :put, :post, :delete, :cache, :follow, :max_size, :timeout, :auth, :basic_auth, :headers, :cookies, :use, :public_only, :download_media, to: :new
end
def initialize
@http ||=
::Danbooru::Http::ApplicationClient.new
.timeout(DEFAULT_TIMEOUT)
.headers("Accept-Encoding" => "gzip")
.headers("User-Agent": "#{Danbooru.config.canonical_app_name}/#{Rails.application.config.x.git_hash}")
.use(:auto_inflate)
.use(redirector: { max_redirects: MAX_REDIRECTS })
.use(:session)
end
def get(url, **options)
request(:get, url, **options)
end
def head(url, **options)
request(:head, url, **options)
end
def put(url, **options)
request(:get, url, **options)
end
def post(url, **options)
request(:post, url, **options)
end
@@ -20,8 +52,16 @@ module Danbooru
request(:delete, url, **options)
end
def cache(expiry)
dup.tap { |o| o.cache = expiry.to_i }
def follow(*args)
dup.tap { |o| o.http = o.http.follow(*args) }
end
def max_size(size)
dup.tap { |o| o.max_size = size }
end
def timeout(*args)
dup.tap { |o| o.http = o.http.timeout(*args) }
end
def auth(*args)
@@ -36,36 +76,66 @@ module Danbooru
dup.tap { |o| o.http = o.http.headers(*args) }
end
def cookies(*args)
dup.tap { |o| o.http = o.http.cookies(*args) }
end
def use(*args)
dup.tap { |o| o.http = o.http.use(*args) }
end
def cache(expires_in)
use(cache: { expires_in: expires_in })
end
# allow requests only to public IPs, not to local or private networks.
def public_only
dup.tap do |o|
o.http = o.http.dup.tap do |http|
http.default_options = http.default_options.with_socket_class(ValidatingSocket)
end
end
end
concerning :DownloadMethods do
def download_media(url, file: Tempfile.new("danbooru-download-", binmode: true))
response = get(url)
raise DownloadError, "Downloading #{response.uri} failed with code #{response.status}" if response.status != 200
raise FileTooLargeError, response if @max_size && response.content_length.to_i > @max_size
size = 0
response.body.each do |chunk|
size += chunk.size
raise FileTooLargeError if @max_size && size > @max_size
file.write(chunk)
end
file.rewind
[response, MediaFile.open(file)]
end
end
protected
def request(method, url, **options)
if @cache.present?
cached_request(method, url, **options)
else
raw_request(method, url, **options)
end
rescue HTTP::TimeoutError
# return a synthetic http error on connection timeouts
::HTTP::Response.new(status: 522, body: "", version: "1.1")
end
def cached_request(method, url, **options)
key = Cache.hash({ method: method, url: url, headers: http.default_options.headers.to_h, **options }.to_json)
cached_response = Cache.get(key, @cache) do
response = raw_request(method, url, **options)
{ status: response.status, body: response.to_s, headers: response.headers.to_h, version: "1.1" }
end
::HTTP::Response.new(**cached_response)
end
def raw_request(method, url, **options)
http.send(method, url, **options)
rescue OpenSSL::SSL::SSLError
fake_response(590, "")
rescue ValidatingSocket::ProhibitedIpError
fake_response(591, "")
rescue HTTP::Redirector::TooManyRedirectsError
fake_response(596, "")
rescue HTTP::TimeoutError
fake_response(597, "")
rescue HTTP::ConnectionError
fake_response(598, "")
rescue HTTP::Error
fake_response(599, "")
end
def http
@http ||= ::HTTP.timeout(DEFAULT_TIMEOUT).use(:auto_inflate).headers(Danbooru.config.http_headers).headers("Accept-Encoding" => "gzip")
def fake_response(status, body)
::HTTP::Response.new(status: status, version: "1.1", body: ::HTTP::Response::Body.new(body))
end
end
end

31
app/logical/danbooru/http/application_client.rb Normal file
View File

@@ -0,0 +1,31 @@
# An extension to HTTP::Client that lets us write Rack-style middlewares that
# hook into the request/response cycle and override how requests are made. This
# works by extending http.rb's concept of features (HTTP::Feature) to give them
# a `perform` method that takes a http request and returns a http response.
# This can be used to intercept and modify requests and return arbitrary responses.
module Danbooru
class Http
class ApplicationClient < HTTP::Client
# Override `perform` to call the `perform` method on features first.
def perform(request, options)
features = options.features.values.reverse.select do |feature|
feature.respond_to?(:perform)
end
perform = proc { |req| super(req, options) }
callback_chain = features.reduce(perform) do |callback_chain, feature|
proc { |req| feature.perform(req, &callback_chain) }
end
callback_chain.call(request)
end
# Override `branch` to return an ApplicationClient instead of a
# HTTP::Client so that chaining works.
def branch(...)
ApplicationClient.new(...)
end
end
end
end

30
app/logical/danbooru/http/cache.rb Normal file
View File

@@ -0,0 +1,30 @@
module Danbooru
class Http
class Cache < HTTP::Feature
HTTP::Options.register_feature :cache, self
attr_reader :expires_in
def initialize(expires_in:)
@expires_in = expires_in
end
def perform(request, &block)
::Cache.get(cache_key(request), expires_in) do
response = yield request
# XXX hack to remove connection state from response body so we can serialize it for caching.
response.flush
response.body.instance_variable_set(:@connection, nil)
response.body.instance_variable_set(:@stream, nil)
response
end
end
def cache_key(request)
"http:" + ::Cache.hash({ method: request.verb, url: request.uri.to_s, headers: request.headers.sort }.to_json)
end
end
end
end

12
app/logical/danbooru/http/html_adapter.rb Normal file
View File

@@ -0,0 +1,12 @@
module Danbooru
class Http
class HtmlAdapter < HTTP::MimeType::Adapter
HTTP::MimeType.register_adapter "text/html", self
HTTP::MimeType.register_alias "text/html", :html
def decode(str)
Nokogiri::HTML5(str)
end
end
end
end

40
app/logical/danbooru/http/redirector.rb Normal file
View File

@@ -0,0 +1,40 @@
# A HTTP::Feature that automatically follows HTTP redirects.
# https://developer.mozilla.org/en-US/docs/Web/HTTP/Redirections
module Danbooru
class Http
class Redirector < HTTP::Feature
HTTP::Options.register_feature :redirector, self
attr_reader :max_redirects
def initialize(max_redirects: 5)
@max_redirects = max_redirects
end
def perform(request, &block)
response = yield request
redirects = max_redirects
while response.status.redirect?
raise HTTP::Redirector::TooManyRedirectsError if redirects <= 0
response = yield build_redirect(request, response)
redirects -= 1
end
response
end
def build_redirect(request, response)
location = response.headers["Location"].to_s
uri = HTTP::URI.parse(location)
verb = request.verb
verb = :get if response.status == 303 && !request.verb.in?([:get, :head])
request.redirect(uri, verb)
end
end
end
end

54
app/logical/danbooru/http/retriable.rb Normal file
View File

@@ -0,0 +1,54 @@
# A HTTP::Feature that automatically retries requests that return a 429 error
# or a Retry-After header. Usage: `Danbooru::Http.use(:retriable).get(url)`.
#
# https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/429
# https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
module Danbooru
class Http
class Retriable < HTTP::Feature
HTTP::Options.register_feature :retriable, self
attr_reader :max_retries, :max_delay
def initialize(max_retries: 2, max_delay: 5.seconds)
@max_retries = max_retries
@max_delay = max_delay
end
def perform(request, &block)
response = yield request
retries = max_retries
while retriable?(response) && retries > 0 && retry_delay(response) <= max_delay
DanbooruLogger.info "Retrying url=#{request.uri} status=#{response.status} retries=#{retries} delay=#{retry_delay(response)}"
retries -= 1
sleep(retry_delay(response))
response = yield request
end
response
end
def retriable?(response)
response.status == 429 || response.headers["Retry-After"].present?
end
def retry_delay(response, current_time: Time.zone.now)
retry_after = response.headers["Retry-After"]
if retry_after.blank?
0.seconds
elsif retry_after =~ /\A\d+\z/
retry_after.to_i.seconds
else
retry_at = Time.zone.parse(retry_after)
return 0.seconds if retry_at.blank?
[retry_at - current_time, 0].max.seconds
end
end
end
end
end

37
app/logical/danbooru/http/session.rb Normal file
View File

@@ -0,0 +1,37 @@
module Danbooru
class Http
class Session < HTTP::Feature
HTTP::Options.register_feature :session, self
attr_reader :cookie_jar
def initialize(cookie_jar: HTTP::CookieJar.new)
@cookie_jar = cookie_jar
end
def perform(request)
add_cookies(request)
response = yield request
save_cookies(response)
response
end
def add_cookies(request)
cookies = cookies_for_request(request)
request.headers["Cookie"] = cookies if cookies.present?
end
def cookies_for_request(request)
saved_cookies = cookie_jar.each(request.uri).map { |c| [c.name, c.value] }.to_h
request_cookies = HTTP::Cookie.cookie_value_to_hash(request.headers["Cookie"].to_s)
saved_cookies.merge(request_cookies).map { |name, value| "#{name}=#{value}" }.join("; ")
end
def save_cookies(response)
response.cookies.each do |cookie|
cookie_jar.add(cookie)
end
end
end
end
end

13
app/logical/danbooru/http/spoof_referrer.rb Normal file
View File

@@ -0,0 +1,13 @@
module Danbooru
class Http
class SpoofReferrer < HTTP::Feature
HTTP::Options.register_feature :spoof_referrer, self
def perform(request, &block)
request.headers["Referer"] = request.uri.origin unless request.headers["Referer"].present?
response = yield request
response
end
end
end
end

20
app/logical/danbooru/http/unpolish_cloudflare.rb Normal file
View File

@@ -0,0 +1,20 @@
# Bypass Cloudflare Polish (https://support.cloudflare.com/hc/en-us/articles/360000607372-Using-Cloudflare-Polish-to-compress-images)
module Danbooru
class Http
class UnpolishCloudflare < HTTP::Feature
HTTP::Options.register_feature :unpolish_cloudflare, self
def perform(request, &block)
response = yield request
if response.headers["CF-Polished"].present?
request.uri.query_values = request.uri.query_values.to_h.merge(danbooru_no_polish: SecureRandom.uuid)
response = yield request
end
response
end
end
end
end

12
app/logical/danbooru/http/xml_adapter.rb Normal file
View File

@@ -0,0 +1,12 @@
module Danbooru
class Http
class XmlAdapter < HTTP::MimeType::Adapter
HTTP::MimeType.register_adapter "application/xml", self
HTTP::MimeType.register_alias "application/xml", :xml
def decode(str)
Hash.from_xml(str).with_indifferent_access
end
end
end
end

3
app/logical/danbooru_maintenance.rb
View File

@@ -2,14 +2,13 @@ module DanbooruMaintenance
module_function
def hourly
safely { Upload.prune! }
end
def daily
safely { PostPruner.new.prune! }
safely { Upload.prune! }
safely { Delayed::Job.where('created_at < ?', 45.days.ago).delete_all }
safely { PostDisapproval.prune! }
safely { PostDisapproval.dmail_messages! }
safely { regenerate_post_counts! }
safely { TokenBucket.prune! }
safely { BulkUpdateRequestPruner.warn_old }

123
app/logical/downloads/file.rb
View File

@@ -1,123 +0,0 @@
require 'resolv'
module Downloads
class File
include ActiveModel::Validations
class Error < StandardError; end
RETRIABLE_ERRORS = [Errno::ECONNRESET, Errno::ETIMEDOUT, Errno::EIO, Errno::EHOSTUNREACH, Errno::ECONNREFUSED, Timeout::Error, IOError]
delegate :data, to: :strategy
attr_reader :url, :referer
validate :validate_url
def initialize(url, referer = nil)
@url = Addressable::URI.parse(url) rescue nil
@referer = referer
validate!
end
def size
res = HTTParty.head(uncached_url, **httparty_options, timeout: 3)
if res.success?
res.content_length
else
raise HTTParty::ResponseError.new(res)
end
end
def download!(url: uncached_url, tries: 3, **options)
Retriable.retriable(on: RETRIABLE_ERRORS, tries: tries, base_interval: 0) do
file = http_get_streaming(url, headers: strategy.headers, **options)
return [file, strategy]
end
end
def validate_url
errors[:base] << "URL must not be blank" if url.blank?
errors[:base] << "'#{url}' is not a valid url" if !url.host.present?
errors[:base] << "'#{url}' is not a valid url. Did you mean 'http://#{url}'?" if !url.scheme.in?(%w[http https])
end
def http_get_streaming(url, file: Tempfile.new(binmode: true), headers: {}, max_size: Danbooru.config.max_file_size)
size = 0
res = HTTParty.get(url, httparty_options) do |chunk|
next if chunk.code == 302
size += chunk.size
raise Error.new("File is too large (max size: #{max_size})") if size > max_size && max_size > 0
file.write(chunk)
end
if res.success?
file.rewind
return file
else
raise Error.new("HTTP error code: #{res.code} #{res.message}")
end
end
# Prevent Cloudflare from potentially mangling the image. See issue #3528.
def uncached_url
return file_url unless is_cloudflare?(file_url)
url = file_url.dup
url.query_values = url.query_values.to_h.merge(danbooru_no_cache: SecureRandom.uuid)
url
end
def preview_url
@preview_url ||= Addressable::URI.parse(strategy.preview_url)
end
def file_url
@file_url ||= Addressable::URI.parse(strategy.image_url)
end
def strategy
@strategy ||= Sources::Strategies.find(url.to_s, referer)
end
def httparty_options
{
timeout: 10,
stream_body: true,
headers: strategy.headers,
connection_adapter: ValidatingConnectionAdapter
}.deep_merge(Danbooru.config.httparty_options)
end
def is_cloudflare?(url)
ip_addr = IPAddr.new(Resolv.getaddress(url.hostname))
CloudflareService.new.ips.any? { |subnet| subnet.include?(ip_addr) }
end
def self.banned_ip?(ip)
ip = IPAddress.parse(ip.to_s) unless ip.is_a?(IPAddress)
if ip.ipv4?
ip.loopback? || ip.link_local? || ip.multicast? || ip.private?
elsif ip.ipv6?
ip.loopback? || ip.link_local? || ip.unique_local? || ip.unspecified?
end
end
end
# Hook into HTTParty to validate the IP before following redirects.
# https://www.rubydoc.info/github/jnunemaker/httparty/HTTParty/ConnectionAdapter
class ValidatingConnectionAdapter < HTTParty::ConnectionAdapter
def self.call(uri, options)
ip_addr = IPAddress.parse(::Resolv.getaddress(uri.hostname))
if Downloads::File.banned_ip?(ip_addr)
raise Downloads::File::Error, "Downloads from #{ip_addr} are not allowed"
end
super(uri, options)
end
end
end

30
app/logical/dtext_input.rb Normal file
View File

@@ -0,0 +1,30 @@
# A custom SimpleForm input for DText fields.
#
# Usage:
#
# <%= f.input :body, as: :dtext %>
# <%= f.input :reason, as: :dtext, inline: true %>
#
# https://github.com/heartcombo/simple_form/wiki/Custom-inputs-examples
# https://github.com/heartcombo/simple_form/blob/master/lib/simple_form/inputs/string_input.rb
# https://github.com/heartcombo/simple_form/blob/master/lib/simple_form/inputs/text_input.rb
class DtextInput < SimpleForm::Inputs::Base
enable :placeholder, :maxlength, :minlength
def input(wrapper_options)
t = template
merged_input_options = merge_wrapper_options(input_html_options, wrapper_options)
t.tag.div(class: "dtext-previewable") do
if options[:inline]
t.concat @builder.text_field(attribute_name, merged_input_options)
else
t.concat @builder.text_area(attribute_name, { rows: 20, cols: 30 }.merge(merged_input_options))
end
t.concat t.tag.div(id: "dtext-preview", class: "dtext-preview prose")
t.concat t.tag.span(t.link_to("Formatting help", t.dtext_help_path, remote: true, method: :get), class: "hint dtext-hint")
end
end
end

20
app/logical/image_proxy.rb
View File

@@ -1,4 +1,6 @@
class ImageProxy
class Error < StandardError; end
def self.needs_proxy?(url)
fake_referer_for(url).present?
end
@@ -8,19 +10,13 @@ class ImageProxy
end
def self.get_image(url)
if url.blank?
raise "Must specify url"
end
raise Error, "URL not present" unless url.present?
raise Error, "Proxy not allowed for this url (url=#{url})" unless needs_proxy?(url)
if !needs_proxy?(url)
raise "Proxy not allowed for this site"
end
referer = fake_referer_for(url)
response = Danbooru::Http.timeout(30).headers(Referer: referer).get(url)
raise Error, "Couldn't proxy image (code=#{response.status}, url=#{url})" unless response.status.success?
response = HTTParty.get(url, Danbooru.config.httparty_options.deep_merge(headers: {"Referer" => fake_referer_for(url)}))
if response.success?
return response
else
raise "HTTP error code: #{response.code} #{response.message}"
end
response
end
end

2
app/logical/ip_lookup.rb
View File

@@ -16,7 +16,7 @@ class IpLookup
end
def info
return {} unless api_key.present?
return {} if api_key.blank?
response = Danbooru::Http.cache(cache_duration).get("https://api.ipregistry.co/#{ip_addr}?key=#{api_key}")
return {} if response.status != 200
json = response.parse.deep_symbolize_keys.with_indifferent_access

43
app/logical/iqdb_proxy.rb
View File

@@ -1,19 +1,24 @@
class IqdbProxy
class Error < StandardError; end
attr_reader :http, :iqdbs_server
def self.enabled?
Danbooru.config.iqdbs_server.present?
def initialize(http: Danbooru::Http.new, iqdbs_server: Danbooru.config.iqdbs_server)
@iqdbs_server = iqdbs_server
@http = http
end
def self.download(url, type)
download = Downloads::File.new(url)
file, strategy = download.download!(url: download.send(type))
def enabled?
iqdbs_server.present?
end
def download(url, type)
strategy = Sources::Strategies.find(url)
download_url = strategy.send(type)
file = strategy.download_file!(download_url)
file
end
def self.search(params)
raise NotImplementedError, "the IQDBs service isn't configured" unless enabled?
def search(params)
limit = params[:limit]&.to_i&.clamp(1, 1000) || 20
similarity = params[:similarity]&.to_f&.clamp(0.0, 100.0) || 0.0
high_similarity = params[:high_similarity]&.to_f&.clamp(0.0, 100.0) || 65.0
@@ -28,7 +33,7 @@ class IqdbProxy
file = download(params[:image_url], :url)
results = query(file: file, limit: limit)
elsif params[:file_url].present?
file = download(params[:file_url], :file_url)
file = download(params[:file_url], :image_url)
results = query(file: file, limit: limit)
elsif params[:post_id].present?
url = Post.find(params[:post_id]).preview_file_url
@@ -46,15 +51,21 @@ class IqdbProxy
file.try(:close)
end
def self.query(params)
response = HTTParty.post("#{Danbooru.config.iqdbs_server}/similar", body: params, **Danbooru.config.httparty_options)
raise Error, "IQDB error: #{response.code} #{response.message}" unless response.success?
raise Error, "IQDB error: #{response.parsed_response["error"]}" if response.parsed_response.is_a?(Hash)
raise Error, "IQDB error: #{response.parsed_response.first}" if response.parsed_response.try(:first).is_a?(String)
response.parsed_response
def query(file: nil, url: nil, limit: 20)
raise NotImplementedError, "the IQDBs service isn't configured" unless enabled?
file = HTTP::FormData::File.new(file) if file
form = { file: file, url: url, limit: limit }.compact
response = http.timeout(30).post("#{iqdbs_server}/similar", form: form)
raise Error, "IQDB error: #{response.status}" if response.status != 200
raise Error, "IQDB error: #{response.parse["error"]}" if response.parse.is_a?(Hash)
raise Error, "IQDB error: #{response.parse.first}" if response.parse.try(:first).is_a?(String)
response.parse
end
def self.decorate_posts(json)
def decorate_posts(json)
post_ids = json.map { |match| match["post_id"] }
posts = Post.where(id: post_ids).group_by(&:id).transform_values(&:first)

2
app/logical/media_file.rb
View File

@@ -43,6 +43,8 @@ class MediaFile
else
:bin
end
rescue EOFError
:bin
end
def self.videos_enabled?

4
app/logical/media_file/flash.rb
View File

@@ -42,7 +42,7 @@ class MediaFile::Flash < MediaFile
signature = contents[0..2]
# SWF version
version = contents[3].unpack('C').join.to_i
_version = contents[3].unpack('C').join.to_i
# Determine the length of the uncompressed stream
length = contents[4..7].unpack('V').join.to_i
@@ -70,7 +70,7 @@ class MediaFile::Flash < MediaFile
rect = contents[8..(8 + rectbytes)].unpack("#{'B8' * rectbytes}").join
# Read in nbits incremenets starting from 5
dimensions = Array.new
dimensions = []
4.times do |n|
s = 5 + (n * nbits) # Calculate our start index
e = s + (nbits - 1) # Calculate our end index

29
app/logical/missed_search_service.rb
View File

@@ -1,29 +0,0 @@
# queries reportbooru to find missed post searches
class MissedSearchService
def self.enabled?
Danbooru.config.reportbooru_server.present?
end
def initialize
if !MissedSearchService.enabled?
raise NotImplementedError.new("the Reportbooru service isn't configured. Missed searches are not available.")
end
end
def each_search(&block)
fetch_data.scan(/(.+?) (\d+)\.0\n/).each(&block)
end
def fetch_data
Cache.get("ms", 1.minute) do
url = URI.parse("#{Danbooru.config.reportbooru_server}/missed_searches")
response = HTTParty.get(url, Danbooru.config.httparty_options.reverse_merge(timeout: 6))
if response.success?
response = response.body
else
response = ""
end
response.force_encoding("utf-8")
end
end
end

131
app/logical/nico_seiga_api_client.rb
View File

@@ -1,83 +1,102 @@
class NicoSeigaApiClient
extend Memoist
BASE_URL = "http://seiga.nicovideo.jp/api"
attr_reader :illust_id
XML_API = "https://seiga.nicovideo.jp/api"
def self.agent
mech = Mechanize.new
mech.redirect_ok = false
mech.keep_alive = false
attr_reader :http
session = Cache.get("nico-seiga-session")
if session
cookie = Mechanize::Cookie.new("user_session", session)
cookie.domain = ".nicovideo.jp"
cookie.path = "/"
mech.cookie_jar.add(cookie)
else
mech.get("https://account.nicovideo.jp/login") do |page|
page.form_with(:id => "login_form") do |form|
form["mail_tel"] = Danbooru.config.nico_seiga_login
form["password"] = Danbooru.config.nico_seiga_password
end.click_button
end
session = mech.cookie_jar.cookies.select {|c| c.name == "user_session"}.first
if session
Cache.put("nico-seiga-session", session.value, 1.week)
else
raise "Session not found"
end
def initialize(work_id:, type:, http: Danbooru::Http.new)
@work_id = work_id
@work_type = type
@http = http
end
# This cookie needs to be set to allow viewing of adult works
cookie = Mechanize::Cookie.new("skip_fetish_warning", "1")
cookie.domain = "seiga.nicovideo.jp"
cookie.path = "/"
mech.cookie_jar.add(cookie)
mech.redirect_ok = true
mech
def image_ids
if @work_type == "illust"
[api_response["id"]]
elsif @work_type == "manga"
manga_api_response.map do |x|
case x["meta"]["source_url"]
when %r{/thumb/(\d+)\w}i then Regexp.last_match(1)
when %r{nicoseiga\.cdn\.nimg\.jp/drm/image/\w+/(\d+)\w}i then Regexp.last_match(1)
end
def initialize(illust_id:, user_id: nil)
@illust_id = illust_id
@user_id = user_id
end
def image_id
illust_xml["response"]["image"]["id"].to_i
end
def user_id
@user_id || illust_xml["response"]["image"]["user_id"].to_i
end
def title
illust_xml["response"]["image"]["title"]
api_response["title"]
end
def desc
illust_xml["response"]["image"]["description"]
def description
api_response["description"]
end
def moniker
artist_xml["response"]["user"]["nickname"]
def tags
api_response.dig("tag_list", "tag").to_a.map { |t| t["name"] }.compact
end
def illust_xml
get("#{BASE_URL}/illust/info?id=#{illust_id}")
def user_id
api_response["user_id"]
end
def artist_xml
get("#{BASE_URL}/user/info?id=#{user_id}")
def user_name
if @work_type == "illust"
api_response["nickname"]
elsif @work_type == "manga"
user_api_response(user_id)["nickname"]
end
end
def api_response
if @work_type == "illust"
resp = get("https://sp.seiga.nicovideo.jp/ajax/seiga/im#{@work_id}")
return {} if resp.blank? || resp.code.to_i == 404
api_response = JSON.parse(resp)["target_image"]
elsif @work_type == "manga"
resp = http.cache(1.minute).get("#{XML_API}/theme/info?id=#{@work_id}")
return {} if resp.blank? || resp.code.to_i == 404
api_response = Hash.from_xml(resp.to_s)["response"]["theme"]
end
api_response || {}
rescue JSON::ParserError
{}
end
def manga_api_response
resp = get("https://ssl.seiga.nicovideo.jp/api/v1/app/manga/episodes/#{@work_id}/frames")
return {} if resp.blank? || resp.code.to_i == 404
JSON.parse(resp)["data"]["result"]
rescue JSON::ParserError
{}
end
def user_api_response(user_id)
resp = http.cache(1.minute).get("#{XML_API}/user/info?id=#{user_id}")
return {} if resp.blank? || resp.code.to_i == 404
Hash.from_xml(resp.to_s)["response"]["user"]
end
def login
form = {
mail_tel: Danbooru.config.nico_seiga_login,
password: Danbooru.config.nico_seiga_password
}
# XXX should fail gracefully instead of raising exception
resp = http.cache(1.hour).post("https://account.nicovideo.jp/login/redirector?site=seiga", form: form)
raise RuntimeError, "NicoSeiga login failed (status=#{resp.status} url=#{url})" if resp.status != 200
http
end
def get(url)
response = Danbooru::Http.cache(1.minute).get(url)
raise "nico seiga api call failed (code=#{response.code}, body=#{response.body})" if response.code != 200
resp = login.cache(1.minute).get(url)
#raise RuntimeError, "NicoSeiga get failed (status=#{resp.status} url=#{url})" if resp.status != 200
Hash.from_xml(response.to_s)
resp
end
memoize :artist_xml, :illust_xml
memoize :api_response, :manga_api_response, :user_api_response
end

60
app/logical/nico_seiga_manga_api_client.rb
View File

@@ -1,60 +0,0 @@
class NicoSeigaMangaApiClient
extend Memoist
BASE_URL = "https://seiga.nicovideo.jp/api"
attr_reader :theme_id
def initialize(theme_id)
@theme_id = theme_id
end
def user_id
theme_info_xml["response"]["theme"]["user_id"].to_i
end
def title
theme_info_xml["response"]["theme"]["title"]
end
def desc
theme_info_xml["response"]["theme"]["description"]
end
def moniker
artist_xml["response"]["user"]["nickname"]
end
def image_ids
images = theme_data_xml["response"]["image_list"]["image"]
images = [images] unless images.is_a?(Array)
images.map {|x| x["id"]}
end
def tags
theme_info_xml["response"]["theme"]["tag_list"]["tag"].map {|x| x["name"]}
end
def theme_data_xml
uri = "#{BASE_URL}/theme/data?theme_id=#{theme_id}"
body = NicoSeigaApiClient.agent.get(uri).body
Hash.from_xml(body)
end
def theme_info_xml
uri = "#{BASE_URL}/theme/info?id=#{theme_id}"
body = NicoSeigaApiClient.agent.get(uri).body
Hash.from_xml(body)
end
def artist_xml
get("#{BASE_URL}/user/info?id=#{user_id}")
end
def get(url)
response = Danbooru::Http.cache(1.minute).get(url)
raise "nico seiga api call failed (code=#{response.code}, body=#{response.body})" if response.code != 200
Hash.from_xml(response.to_s)
end
memoize :theme_data_xml, :theme_info_xml, :artist_xml
end

37
app/logical/pagination_extension.rb
View File

@@ -3,9 +3,9 @@ module PaginationExtension
attr_accessor :current_page, :records_per_page, :paginator_count, :paginator_mode
def paginate(page, limit: nil, count: nil, search_count: nil)
def paginate(page, limit: nil, max_limit: 1000, count: nil, search_count: nil)
@records_per_page = limit || Danbooru.config.posts_per_page
@records_per_page = @records_per_page.to_i.clamp(1, 1000)
@records_per_page = @records_per_page.to_i.clamp(1, max_limit)
if count.present?
@paginator_count = count
@@ -61,27 +61,35 @@ module PaginationExtension
end
def prev_page
return nil if is_first_page?
if paginator_mode == :numbered
if is_first_page?
nil
elsif paginator_mode == :numbered
current_page - 1
elsif paginator_mode == :sequential_before
elsif paginator_mode == :sequential_before && records.present?
"a#{records.first.id}"
elsif paginator_mode == :sequential_after
elsif paginator_mode == :sequential_after && records.present?
"b#{records.last.id}"
else
nil
end
rescue ActiveRecord::QueryCanceled
nil
end
def next_page
return nil if is_last_page?
if paginator_mode == :numbered
if is_last_page?
nil
elsif paginator_mode == :numbered
current_page + 1
elsif paginator_mode == :sequential_before
elsif paginator_mode == :sequential_before && records.present?
"b#{records.last.id}"
elsif paginator_mode == :sequential_after
elsif paginator_mode == :sequential_after && records.present?
"a#{records.first.id}"
else
nil
end
rescue ActiveRecord::QueryCanceled
nil
end
# XXX Hack: in sequential pagination we fetch one more record than we
@@ -106,10 +114,7 @@ module PaginationExtension
def total_count
@paginator_count ||= unscoped.from(except(:offset, :limit, :order).reorder(nil)).count
rescue ActiveRecord::StatementInvalid => e
if e.to_s =~ /statement timeout/
raise unless e.to_s =~ /statement timeout/
@paginator_count ||= 1_000_000
else
raise
end
end
end

6
app/logical/pawoo_api_client.rb
View File

@@ -128,15 +128,15 @@ class PawooApiClient
rescue
data = {}
end
return Account.new(data)
Account.new(data)
end
end
private
def fetch_access_token
raise MissingConfigurationError.new("missing pawoo client id") if Danbooru.config.pawoo_client_id.nil?
raise MissingConfigurationError.new("missing pawoo client secret") if Danbooru.config.pawoo_client_secret.nil?
raise MissingConfigurationError, "missing pawoo client id" if Danbooru.config.pawoo_client_id.nil?
raise MissingConfigurationError, "missing pawoo client secret" if Danbooru.config.pawoo_client_secret.nil?
Cache.get("pawoo-token") do
result = client.client_credentials.get_token

120
app/logical/pixiv_api_client.rb
View File

@@ -1,5 +1,3 @@
require 'resolv-replace'
class PixivApiClient
extend Memoist
@@ -114,66 +112,13 @@ class PixivApiClient
end
end
class FanboxResponse
attr_reader :json
def initialize(json)
@json = json
end
def name
json["body"]["user"]["name"]
end
def user_id
json["body"]["user"]["userId"]
end
def moniker
""
end
def page_count
json["body"]["body"]["images"].size
end
def artist_commentary_title
json["body"]["title"]
end
def artist_commentary_desc
json["body"]["body"]["text"]
end
def tags
[]
end
def pages
if json["body"]["body"]
json["body"]["body"]["images"].map {|x| x["originalUrl"]}
else
[]
end
end
end
def work(illust_id)
headers = Danbooru.config.http_headers.merge(
"Referer" => "http://www.pixiv.net",
"Content-Type" => "application/x-www-form-urlencoded",
"Authorization" => "Bearer #{access_token}"
)
params = {
"image_sizes" => "large",
"include_stats" => "true"
}
params = { image_sizes: "large", include_stats: "true" }
url = "https://public-api.secure.pixiv.net/v#{API_VERSION}/works/#{illust_id.to_i}.json"
response = Danbooru::Http.cache(1.minute).headers(headers).get(url, params: params)
response = api_client.cache(1.minute).get(url, params: params)
json = response.parse
if response.code == 200
if response.status == 200
WorkResponse.new(json["response"][0])
elsif json["status"] == "failure" && json.dig("errors", "system", "message") =~ /対象のイラストは見つかりませんでした。/
raise BadIDError.new("Pixiv ##{illust_id} not found: work was deleted, made private, or ID is invalid.")
@@ -184,32 +129,12 @@ class PixivApiClient
raise Error.new("Pixiv API call failed (status=#{response.code} body=#{response.body})")
end
def fanbox(fanbox_id)
url = "https://www.pixiv.net/ajax/fanbox/post?postId=#{fanbox_id.to_i}"
resp = agent.get(url)
json = JSON.parse(resp.body)
if resp.code == "200"
FanboxResponse.new(json)
elsif json["status"] == "failure"
raise Error.new("Pixiv API call failed (status=#{resp.code} body=#{body})")
end
rescue JSON::ParserError
raise Error.new("Pixiv API call failed (status=#{resp.code} body=#{body})")
end
def novel(novel_id)
headers = Danbooru.config.http_headers.merge(
"Referer" => "http://www.pixiv.net",
"Content-Type" => "application/x-www-form-urlencoded",
"Authorization" => "Bearer #{access_token}"
)
url = "https://public-api.secure.pixiv.net/v#{API_VERSION}/novels/#{novel_id.to_i}.json"
resp = HTTParty.get(url, Danbooru.config.httparty_options.deep_merge(headers: headers))
body = resp.body.force_encoding("utf-8")
json = JSON.parse(body)
resp = api_client.cache(1.minute).get(url)
json = resp.parse
if resp.success?
if resp.status == 200
NovelResponse.new(json["response"][0])
elsif json["status"] == "failure" && json.dig("errors", "system", "message") =~ /対象のイラストは見つかりませんでした。/
raise Error.new("Pixiv API call failed (status=#{resp.code} body=#{body})")
@@ -219,10 +144,8 @@ class PixivApiClient
end
def access_token
Cache.get("pixiv-papi-access-token", 3000) do
access_token = nil
client_time = Time.now.rfc3339
# truncate timestamp to 1-hour resolution so that it doesn't break caching.
client_time = Time.zone.now.utc.change(min: 0).rfc3339
client_hash = Digest::MD5.hexdigest(client_time + CLIENT_HASH_SALT)
headers = {
@@ -230,6 +153,7 @@ class PixivApiClient
"X-Client-Time": client_time,
"X-Client-Hash": client_hash
}
params = {
username: Danbooru.config.pixiv_login,
password: Danbooru.config.pixiv_password,
@@ -237,24 +161,24 @@ class PixivApiClient
client_id: CLIENT_ID,
client_secret: CLIENT_SECRET
}
url = "https://oauth.secure.pixiv.net/auth/token"
resp = HTTParty.post(url, Danbooru.config.httparty_options.deep_merge(body: params, headers: headers))
body = resp.body.force_encoding("utf-8")
resp = http.headers(headers).cache(1.hour).post("https://oauth.secure.pixiv.net/auth/token", form: params)
return nil unless resp.status == 200
if resp.success?
json = JSON.parse(body)
access_token = json["response"]["access_token"]
else
raise Error.new("Pixiv API access token call failed (status=#{resp.code} body=#{body})")
resp.parse.dig("response", "access_token")
end
access_token
end
def api_client
http.headers(
"Referer": "http://www.pixiv.net",
"Content-Type": "application/x-www-form-urlencoded",
"Authorization": "Bearer #{access_token}"
)
end
def agent
PixivWebAgent.build
def http
Danbooru::Http.new
end
memoize :agent
memoize :access_token, :api_client, :http
end

74
app/logical/pixiv_web_agent.rb
View File

@@ -1,74 +0,0 @@
class PixivWebAgent
SESSION_CACHE_KEY = "pixiv-phpsessid"
COMIC_SESSION_CACHE_KEY = "pixiv-comicsessid"
SESSION_COOKIE_KEY = "PHPSESSID"
COMIC_SESSION_COOKIE_KEY = "_pixiv-comic_session"
def self.phpsessid(agent)
agent.cookies.select { |cookie| cookie.name == SESSION_COOKIE_KEY }.first.try(:value)
end
def self.build
mech = Mechanize.new
mech.keep_alive = false
phpsessid = Cache.get(SESSION_CACHE_KEY)
comicsessid = Cache.get(COMIC_SESSION_CACHE_KEY)
if phpsessid
cookie = Mechanize::Cookie.new(SESSION_COOKIE_KEY, phpsessid)
cookie.domain = ".pixiv.net"
cookie.path = "/"
mech.cookie_jar.add(cookie)
if comicsessid
cookie = Mechanize::Cookie.new(COMIC_SESSION_COOKIE_KEY, comicsessid)
cookie.domain = ".pixiv.net"
cookie.path = "/"
mech.cookie_jar.add(cookie)
end
else
headers = {
"Origin" => "https://accounts.pixiv.net",
"Referer" => "https://accounts.pixiv.net/login?lang=en^source=pc&view_type=page&ref=wwwtop_accounts_index"
}
params = {
pixiv_id: Danbooru.config.pixiv_login,
password: Danbooru.config.pixiv_password,
captcha: nil,
g_captcha_response: nil,
source: "pc",
post_key: nil
}
mech.get("https://accounts.pixiv.net/login?lang=en&source=pc&view_type=page&ref=wwwtop_accounts_index") do |page|
json = page.search("input#init-config").first.attr("value")
if json =~ /pixivAccount\.postKey":"([a-f0-9]+)/
params[:post_key] = $1
end
end
mech.post("https://accounts.pixiv.net/api/login?lang=en", params, headers)
if mech.current_page.body =~ /"error":false/
cookie = mech.cookies.select {|x| x.name == SESSION_COOKIE_KEY}.first
if cookie
Cache.put(SESSION_CACHE_KEY, cookie.value, 1.week)
end
end
begin
mech.get("https://comic.pixiv.net") do |page|
cookie = mech.cookies.select {|x| x.name == COMIC_SESSION_COOKIE_KEY}.first
if cookie
Cache.put(COMIC_SESSION_CACHE_KEY, cookie.value, 1.week)
end
end
rescue Net::HTTPServiceUnavailable
# ignore
end
end
mech
end
end

61
app/logical/popular_search_service.rb
View File

@@ -1,61 +0,0 @@
# queries reportbooru to find popular post searches
class PopularSearchService
attr_reader :date
def self.enabled?
Danbooru.config.reportbooru_server.present?
end
def initialize(date)
if !PopularSearchService.enabled?
raise NotImplementedError.new("the Reportbooru service isn't configured. Popular searches are not available.")
end
@date = date
end
def each_search(limit = 100, &block)
JSON.parse(fetch_data.to_s).slice(0, limit).each(&block)
end
def tags
JSON.parse(fetch_data.to_s).map {|x| x[0]}
end
def fetch_data
return [] unless self.class.enabled?
dates = date.strftime("%Y-%m-%d")
data = Cache.get("ps-day-#{dates}", 1.minute) do
url = "#{Danbooru.config.reportbooru_server}/post_searches/rank?date=#{dates}"
response = HTTParty.get(url, Danbooru.config.httparty_options.reverse_merge(timeout: 3))
if response.success?
response = response.body
else
response = "[]"
end
response
end.to_s.force_encoding("utf-8")
if data.blank? || data == "[]"
dates = date.yesterday.strftime("%Y-%m-%d")
data = Cache.get("ps-day-#{dates}", 1.minute) do
url = "#{Danbooru.config.reportbooru_server}/post_searches/rank?date=#{dates}"
response = HTTParty.get(url, Danbooru.config.httparty_options.reverse_merge(timeout: 3))
if response.success?
response = response.body
else
response = "[]"
end
response
end.to_s.force_encoding("utf-8")
end
data
rescue StandardError => e
DanbooruLogger.log(e)
return []
end
end

4
app/logical/post_query_builder.rb
View File

@@ -307,6 +307,8 @@ class PostQueryBuilder
Post.where(parent: nil)
when "any"
Post.where.not(parent: nil)
when /pending|flagged|modqueue|deleted|banned|active|unmoderated/
Post.where.not(parent: nil).where(parent: status_matches(parent))
when /\A\d+\z/
Post.where(id: parent).or(Post.where(parent: parent))
else
@@ -320,6 +322,8 @@ class PostQueryBuilder
Post.where(has_children: false)
when "any"
Post.where(has_children: true)
when /pending|flagged|modqueue|deleted|banned|active|unmoderated/
Post.where(has_children: true).where(children: status_matches(child))
else
Post.none
end

29
app/logical/post_sets/post.rb
View File

@@ -24,9 +24,8 @@ module PostSets
end
def wiki_page
return nil unless tag.present? && tag.wiki_page.present?
return nil unless !tag.wiki_page.is_deleted?
tag.wiki_page
return nil unless normalized_query.has_single_tag?
@wiki_page ||= WikiPage.undeleted.find_by(title: normalized_query.tags.first.name)
end
def tag
@@ -77,7 +76,11 @@ module PostSets
end
def per_page
(@per_page || query.find_metatag(:limit) || CurrentUser.user.per_page).to_i.clamp(0, MAX_PER_PAGE)
(@per_page || query.find_metatag(:limit) || CurrentUser.user.per_page).to_i.clamp(0, max_per_page)
end
def max_per_page
(format == "sitemap") ? 10_000 : MAX_PER_PAGE
end
def is_random?
@@ -94,7 +97,7 @@ module PostSets
end
def get_random_posts
per_page.times.inject([]) do |all, x|
per_page.times.inject([]) do |all, _|
all << ::Post.user_tag_match(tag_string).random
end.compact.uniq
end
@@ -104,15 +107,15 @@ module PostSets
@post_count = get_post_count
if is_random?
temp = get_random_posts
get_random_posts
else
temp = normalized_query.build.paginate(page, count: post_count, search_count: !post_count.nil?, limit: per_page)
normalized_query.build.paginate(page, count: post_count, search_count: !post_count.nil?, limit: per_page, max_limit: max_per_page).load
end
end
end
def hide_from_crawler?
return true if current_page > 1
return true if current_page > 50
return false if query.is_empty_search? || query.is_simple_tag? || query.is_metatag?(:order, :rank)
true
end
@@ -160,20 +163,16 @@ module PostSets
elsif query.is_metatag?(:search)
saved_search_tags
elsif query.is_empty_search? || query.is_metatag?(:order, :rank)
popular_tags
popular_tags.presence || frequent_tags
elsif query.is_single_term?
similar_tags
similar_tags.presence || frequent_tags
else
frequent_tags
end
end
def popular_tags
if PopularSearchService.enabled?
PopularSearchService.new(Date.today).tags
else
frequent_tags
end
ReportbooruService.new.popular_searches(Date.today, limit: MAX_SIDEBAR_TAGS)
end
def similar_tags

38
app/logical/post_view_count_service.rb
View File

@@ -1,38 +0,0 @@
class PostViewCountService
def self.enabled?
Danbooru.config.reportbooru_server.present?
end
def initialize
if !PostViewCountService.enabled?
raise NotImplementedError.new("the Reportbooru service isn't configured. Post views are not available.")
end
end
def fetch_count(post_id)
url = URI.parse("#{Danbooru.config.reportbooru_server}/post_views/#{post_id}")
response = HTTParty.get(url, Danbooru.config.httparty_options.reverse_merge(timeout: 6))
if response.success?
return JSON.parse(response.body)
else
return nil
end
end
def fetch_rank(date = Date.today)
url = URI.parse("#{Danbooru.config.reportbooru_server}/post_views/rank?date=#{date}")
response = HTTParty.get(url, Danbooru.config.httparty_options.reverse_merge(timeout: 6))
if response.success?
return JSON.parse(response.body)
else
return nil
end
rescue JSON::ParserError
nil
end
def popular_posts(date = Date.today)
ranking = fetch_rank(date) || []
ranking.slice(0, 50).map {|x| Post.find(x[0])}
end
end

50
app/logical/reportbooru_service.rb Normal file
View File

@@ -0,0 +1,50 @@
class ReportbooruService
attr_reader :http, :reportbooru_server
def initialize(http: Danbooru::Http.new, reportbooru_server: Danbooru.config.reportbooru_server)
@reportbooru_server = reportbooru_server
@http = http
end
def enabled?
reportbooru_server.present?
end
def missed_search_rankings(expires_in: 1.minutes)
return [] unless enabled?
response = http.cache(expires_in).get("#{reportbooru_server}/missed_searches")
return [] if response.status != 200
body = response.to_s.force_encoding("utf-8")
body.lines.map(&:split).map { [_1, _2.to_i] }
end
def post_search_rankings(date, expires_in: 1.minutes)
request("#{reportbooru_server}/post_searches/rank?date=#{date}", expires_in)
end
def post_view_rankings(date, expires_in: 1.minutes)
request("#{reportbooru_server}/post_views/rank?date=#{date}", expires_in)
end
def popular_searches(date, limit: 100)
ranking = post_search_rankings(date)
ranking = post_search_rankings(date.yesterday) if ranking.blank?
ranking.take(limit).map(&:first)
end
def popular_posts(date, limit: 100)
ranking = post_view_rankings(date)
ranking = post_view_rankings(date.yesterday) if ranking.blank?
ranking.take(limit).map { |x| Post.find(x[0]) }
end
def request(url, expires_in)
return [] unless enabled?
response = http.cache(expires_in).get(url)
return [] if response.status != 200
JSON.parse(response.to_s.force_encoding("utf-8"))
end
end

5
app/logical/sources/strategies.rb
View File

@@ -1,7 +1,7 @@
module Sources
module Strategies
def self.all
return [
[
Strategies::Pixiv,
Strategies::NicoSeiga,
Strategies::Twitter,
@@ -13,7 +13,8 @@ module Sources
Strategies::Pawoo,
Strategies::Moebooru,
Strategies::HentaiFoundry,
Strategies::Weibo
Strategies::Weibo,
Strategies::Newgrounds
]
end

10
app/logical/sources/strategies/art_station.rb
View File

@@ -22,15 +22,15 @@
module Sources::Strategies
class ArtStation < Base
PROJECT1 = %r!\Ahttps?://www\.artstation\.com/artwork/(?<project_id>[a-z0-9-]+)/?\z!i
PROJECT2 = %r!\Ahttps?://(?<artist_name>[\w-]+)\.artstation\.com/projects/(?<project_id>[a-z0-9-]+)(?:/|\?[\w=-]+)?\z!i
PROJECT1 = %r{\Ahttps?://www\.artstation\.com/artwork/(?<project_id>[a-z0-9-]+)/?\z}i
PROJECT2 = %r{\Ahttps?://(?<artist_name>[\w-]+)\.artstation\.com/projects/(?<project_id>[a-z0-9-]+)(?:/|\?[\w=-]+)?\z}i
PROJECT = Regexp.union(PROJECT1, PROJECT2)
ARTIST1 = %r{\Ahttps?://(?<artist_name>[\w-]+)(?<!www)\.artstation\.com/?\z}i
ARTIST2 = %r{\Ahttps?://www\.artstation\.com/artist/(?<artist_name>[\w-]+)/?\z}i
ARTIST3 = %r{\Ahttps?://www\.artstation\.com/(?<artist_name>[\w-]+)/?\z}i
ARTIST = Regexp.union(ARTIST1, ARTIST2, ARTIST3)
ASSET = %r!\Ahttps?://cdn\w*\.artstation\.com/p/assets/(?<type>images|covers)/images/(?<id>\d+/\d+/\d+)/(?<size>[^/]+)/(?<filename>.+)\z!i
ASSET = %r{\Ahttps?://cdn\w*\.artstation\.com/p/assets/(?<type>images|covers)/images/(?<id>\d+/\d+/\d+)/(?<size>[^/]+)/(?<filename>.+)\z}i
attr_reader :json
@@ -144,10 +144,10 @@ module Sources::Strategies
urls = image_url_sizes($~[:type], $~[:id], $~[:filename])
if size == :smallest
urls = urls.reverse()
urls = urls.reverse
end
chosen_url = urls.find { |url| http_exists?(url, headers) }
chosen_url = urls.find { |url| http_exists?(url) }
chosen_url || url
end
end

56
app/logical/sources/strategies/base.rb
View File

@@ -14,6 +14,8 @@
module Sources
module Strategies
class Base
class DownloadError < StandardError; end
attr_reader :url, :referer_url, :urls, :parsed_url, :parsed_referer, :parsed_urls
extend Memoist
@@ -35,9 +37,9 @@ module Sources
# <tt>referrer_url</tt> so the strategy can discover the HTML
# page and other information.
def initialize(url, referer_url = nil)
@url = url
@referer_url = referer_url
@urls = [url, referer_url].select(&:present?)
@url = url.to_s
@referer_url = referer_url&.to_s
@urls = [@url, @referer_url].select(&:present?)
@parsed_url = Addressable::URI.heuristic_parse(url) rescue nil
@parsed_referer = Addressable::URI.heuristic_parse(referer_url) rescue nil
@@ -58,8 +60,8 @@ module Sources
end
def site_name
Addressable::URI.heuristic_parse(url).host
rescue Addressable::URI::InvalidURIError => e
Addressable::URI.heuristic_parse(url)&.host
rescue Addressable::URI::InvalidURIError
nil
end
@@ -90,9 +92,7 @@ module Sources
# eventually be assigned as the source for the post, but it does not
# represent what the downloader will fetch.
def page_url
Rails.logger.warn "Valid page url for (#{url}, #{referer_url}) not found"
return nil
nil
end
# This will be the url stored in posts. Typically this is the page
@@ -141,14 +141,37 @@ module Sources
# Subclasses should merge in any required headers needed to access resources
# on the site.
def headers
return Danbooru.config.http_headers
{}
end
# Returns the size of the image resource without actually downloading the file.
def size
Downloads::File.new(image_url).size
def remote_size
response = http_downloader.head(image_url)
return nil unless response.status == 200 && response.content_length.present?
response.content_length.to_i
end
memoize :size
memoize :remote_size
# Download the file at the given url, or at the main image url by default.
def download_file!(download_url = image_url)
raise DownloadError, "Download failed: couldn't find download url for #{url}" if download_url.blank?
response, file = http_downloader.download_media(download_url)
raise DownloadError, "Download failed: #{download_url} returned error #{response.status}" if response.status != 200
file
end
# A http client for API requests.
def http
Danbooru::Http.new.public_only
end
memoize :http
# A http client for downloading files.
def http_downloader
http.timeout(30).max_size(Danbooru.config.max_file_size).use(:spoof_referrer).use(:unpolish_cloudflare)
end
memoize :http_downloader
# The url to use for artist finding purposes. This will be stored in the
# artist entry. Normally this will be the profile url.
@@ -189,7 +212,7 @@ module Sources
end
def normalized_tags
tags.map { |tag, url| normalize_tag(tag) }.sort.uniq
tags.map { |tag, _url| normalize_tag(tag) }.sort.uniq
end
def normalize_tag(tag)
@@ -243,7 +266,7 @@ module Sources
end
def to_h
return {
{
:artist => {
:name => artist_name,
:tag_name => tag_name,
@@ -276,9 +299,8 @@ module Sources
to_h.to_json
end
def http_exists?(url, headers)
res = HTTParty.head(url, Danbooru.config.httparty_options.deep_merge(headers: headers))
res.success?
def http_exists?(url)
http_downloader.head(url).status.success?
end
# Convert commentary to dtext by stripping html tags. Sites can override

30
app/logical/sources/strategies/deviant_art.rb
View File

@@ -47,16 +47,16 @@
module Sources
module Strategies
class DeviantArt < Base
ASSET_SUBDOMAINS = %r{(?:fc|th|pre|img|orig|origin-orig)\d*}i
ASSET_SUBDOMAINS = /(?:fc|th|pre|img|orig|origin-orig)\d*/i
RESERVED_SUBDOMAINS = %r{\Ahttps?://(?:#{ASSET_SUBDOMAINS}|www)\.}i
MAIN_DOMAIN = %r{\Ahttps?://(?:www\.)?deviantart.com}i
TITLE = %r{(?<title>[a-z0-9_-]+?)}i
ARTIST = %r{(?<artist>[a-z0-9_-]+?)}i
DEVIATION_ID = %r{(?<deviation_id>[0-9]+)}i
TITLE = /(?<title>[a-z0-9_-]+?)/i
ARTIST = /(?<artist>[a-z0-9_-]+?)/i
DEVIATION_ID = /(?<deviation_id>[0-9]+)/i
DA_FILENAME_1 = %r{[a-f0-9]{32}-d(?<base36_deviation_id>[a-z0-9]+)\.}i
DA_FILENAME_2 = %r{#{TITLE}(?:_by_#{ARTIST}(?:-d(?<base36_deviation_id>[a-z0-9]+))?)?\.}i
DA_FILENAME_1 = /[a-f0-9]{32}-d(?<base36_deviation_id>[a-z0-9]+)\./i
DA_FILENAME_2 = /#{TITLE}(?:_by_#{ARTIST}(?:-d(?<base36_deviation_id>[a-z0-9]+))?)?\./i
DA_FILENAME = Regexp.union(DA_FILENAME_1, DA_FILENAME_2)
WIX_FILENAME = %r{d(?<base36_deviation_id>[a-z0-9]+)[0-9a-f-]+\.\w+(?:/\w+/\w+/[\w,]+/(?<title>[\w-]+)_by_(?<artist>[\w-]+)_d\w+-\w+\.\w+)?.+}i
@@ -75,7 +75,7 @@ module Sources
PATH_PROFILE = %r{#{MAIN_DOMAIN}/#{ARTIST}/?\z}i
SUBDOMAIN_PROFILE = %r{\Ahttps?://#{ARTIST}\.deviantart\.com/?\z}i
FAVME = %r{\Ahttps?://(www\.)?fav\.me/d(?<base36_deviation_id>[a-z0-9]+)\z}i
FAVME = %r{\Ahttps?://(?:www\.)?fav\.me/d(?<base36_deviation_id>[a-z0-9]+)\z}i
def domains
["deviantart.net", "deviantart.com", "fav.me"]
@@ -110,12 +110,12 @@ module Sources
api_deviation[:videos].max_by { |x| x[:filesize] }[:src]
else
src = api_deviation.dig(:content, :src)
if deviation_id && deviation_id.to_i <= 790677560 && src =~ /^https:\/\/images-wixmp-/ && src !~ /\.gif\?/
src = src.sub(%r!(/f/[a-f0-9-]+/[a-f0-9-]+)!, '/intermediary\1')
src = src.sub(%r!/v1/(fit|fill)/.*\z!i, "")
if deviation_id && deviation_id.to_i <= 790_677_560 && src =~ %r{\Ahttps://images-wixmp-} && src !~ /\.gif\?/
src = src.sub(%r{(/f/[a-f0-9-]+/[a-f0-9-]+)}, '/intermediary\1')
src = src.sub(%r{/v1/(fit|fill)/.*\z}i, "")
end
src = src.sub(%r!\Ahttps?://orig\d+\.deviantart\.net!i, "http://origin-orig.deviantart.net")
src = src.gsub(%r!q_\d+,strp!, "q_100")
src = src.sub(%r{\Ahttps?://orig\d+\.deviantart\.net}i, "http://origin-orig.deviantart.net")
src = src.gsub(/q_\d+,strp/, "q_100")
src
end
end
@@ -191,7 +191,7 @@ module Sources
# <a href="https://sa-dui.deviantart.com/journal/About-Commissions-223178193" data-sigil="thumb" class="thumb lit" ...>
if element["class"].split.include?("lit")
deviation_id = element["href"][%r!-(\d+)\z!, 1].to_i
deviation_id = element["href"][/-(\d+)\z/, 1].to_i
element.content = "deviantart ##{deviation_id}"
else
element.content = ""
@@ -199,7 +199,7 @@ module Sources
end
if element.name == "a" && element["href"].present?
element["href"] = element["href"].gsub(%r!\Ahttps?://www\.deviantart\.com/users/outgoing\?!i, "")
element["href"] = element["href"].gsub(%r{\Ahttps?://www\.deviantart\.com/users/outgoing\?}i, "")
# href may be missing the `http://` bit (ex: `inprnt.com`, `//inprnt.com`). Add it if missing.
uri = Addressable::URI.heuristic_parse(element["href"]) rescue nil
@@ -283,7 +283,7 @@ module Sources
return nil if meta.nil?
appurl = meta["content"]
uuid = appurl[%r!\ADeviantArt://deviation/(.*)\z!, 1]
uuid = appurl[%r{\ADeviantArt://deviation/(.*)\z}, 1]
uuid
end
memoize :uuid

17
app/logical/sources/strategies/hentai_foundry.rb
View File

@@ -23,11 +23,11 @@
module Sources
module Strategies
class HentaiFoundry < Base
BASE_URL = %r!\Ahttps?://(?:www\.)?hentai-foundry\.com!i
PAGE_URL = %r!#{BASE_URL}/pictures/user/(?<artist_name>[\w-]+)/(?<illust_id>\d+)(?:/[\w.-]*)?(\?[\w=]*)?\z!i
OLD_PAGE = %r!#{BASE_URL}/pic-(?<illust_id>\d+)(?:\.html)?\z!i
PROFILE_URL = %r!#{BASE_URL}/(?:pictures/)?user/(?<artist_name>[\w-]+)(?:/[a-z]*)?\z!i
IMAGE_URL = %r!\Ahttps?://pictures\.hentai-foundry\.com/+\w/(?<artist_name>[\w-]+)/(?<illust_id>\d+)(?:(?:/[\w.-]+)?\.\w+)?\z!i
BASE_URL = %r{\Ahttps?://(?:www\.)?hentai-foundry\.com}i
PAGE_URL = %r{#{BASE_URL}/pictures/user/(?<artist_name>[\w-]+)/(?<illust_id>\d+)(?:/[\w.-]*)?(\?[\w=]*)?\z}i
OLD_PAGE = %r{#{BASE_URL}/pic-(?<illust_id>\d+)(?:\.html)?\z}i
PROFILE_URL = %r{#{BASE_URL}/(?:pictures/)?user/(?<artist_name>[\w-]+)(?:/[a-z]*)?\z}i
IMAGE_URL = %r{\Ahttps?://pictures\.hentai-foundry\.com/+\w/(?<artist_name>[\w-]+)/(?<illust_id>\d+)(?:(?:/[\w.-]+)?\.\w+)?\z}i
def domains
["hentai-foundry.com"]
@@ -64,11 +64,10 @@ module Sources
def page
return nil if page_url.blank?
doc = Cache.get("hentai-foundry:#{page_url}", 1.minute) do
HTTParty.get("#{page_url}?enterAgree=1").body
end
response = Danbooru::Http.new.cache(1.minute).get("#{page_url}?enterAgree=1")
return nil unless response.status == 200
Nokogiri::HTML(doc)
response.parse
end
def tags

12
app/logical/sources/strategies/moebooru.rb
View File

@@ -32,10 +32,10 @@
module Sources
module Strategies
class Moebooru < Base
BASE_URL = %r!\Ahttps?://(?:[^.]+\.)?(?<domain>yande\.re|konachan\.com)!i
POST_URL = %r!#{BASE_URL}/post/show/(?<id>\d+)!i
URL_SLUG = %r!/(?:yande\.re%20|Konachan\.com%20-%20)?(?<id>\d+)?.*!i
IMAGE_URL = %r!#{BASE_URL}/(?<type>image|jpeg|sample)/(?<md5>\h{32})#{URL_SLUG}?\.(?<ext>jpg|jpeg|png|gif)\z!i
BASE_URL = %r{\Ahttps?://(?:[^.]+\.)?(?<domain>yande\.re|konachan\.com)}i
POST_URL = %r{#{BASE_URL}/post/show/(?<id>\d+)}i
URL_SLUG = %r{/(?:yande\.re%20|Konachan\.com%20-%20)?(?<id>\d+)?.*}i
IMAGE_URL = %r{#{BASE_URL}/(?<type>image|jpeg|sample)/(?<md5>\h{32})#{URL_SLUG}?\.(?<ext>jpg|jpeg|png|gif)\z}i
delegate :artist_name, :profile_url, :tag_name, :artist_commentary_title, :artist_commentary_desc, :dtext_artist_commentary_title, :dtext_artist_commentary_desc, to: :sub_strategy, allow_nil: true
@@ -63,7 +63,7 @@ module Sources
end
def preview_urls
return image_urls unless post_md5.present?
return image_urls if post_md5.blank?
["https://#{file_host}/data/preview/#{post_md5[0..1]}/#{post_md5[2..3]}/#{post_md5}.jpg"]
end
@@ -155,7 +155,7 @@ module Sources
# the api_response wasn't available because it's a deleted post.
elsif post_md5.present?
%w[jpg png gif].find { |ext| http_exists?("https://#{site_name}/image/#{post_md5}.#{ext}", headers) }
%w[jpg png gif].find { |ext| http_exists?("https://#{site_name}/image/#{post_md5}.#{ext}") }
else
nil

111
app/logical/sources/strategies/newgrounds.rb Normal file
View File

@@ -0,0 +1,111 @@
# Image Urls
# * https://art.ngfiles.com/images/1254000/1254722_natthelich_pandora.jpg
# * https://art.ngfiles.com/images/1033000/1033622_natthelich_fire-emblem-marth-plus-progress-pic.png?f1569487181
# * https://art.ngfiles.com/comments/57000/iu_57615_7115981.jpg
#
# Page URLs
# * https://www.newgrounds.com/art/view/puddbytes/costanza-at-bat
# * https://www.newgrounds.com/art/view/natthelich/fire-emblem-marth-plus-progress-pic (multiple)
#
# Profile URLs
# * https://natthelich.newgrounds.com/
module Sources
module Strategies
class Newgrounds < Base
IMAGE_URL = %r{\Ahttps?://art\.ngfiles\.com/images/\d+/\d+_(?<user_name>[0-9a-z-]+)_(?<illust_title>[0-9a-z-]+)\.\w+}i
COMMENT_URL = %r{\Ahttps?://art\.ngfiles\.com/comments/\d+/\w+\.\w+}i
PAGE_URL = %r{\Ahttps?://(?:www\.)?newgrounds\.com/art/view/(?<user_name>[0-9a-z-]+)/(?<illust_title>[0-9a-z-]+)(?:\?.*)?}i
PROFILE_URL = %r{\Ahttps?://(?<artist_name>(?!www)[0-9a-z-]+)\.newgrounds\.com(?:/.*)?}i
def domains
["newgrounds.com", "ngfiles.com"]
end
def site_name
"NewGrounds"
end
def image_urls
if url =~ COMMENT_URL || url =~ IMAGE_URL
[url]
else
urls = []
urls += page&.css(".image img").to_a.map { |img| img["src"] }
urls += page&.css("#author_comments img[data-user-image='1']").to_a.map { |img| img["data-smartload-src"] || img["src"] }
urls.compact
end
end
def page_url
return nil if illust_title.blank? || user_name.blank?
"https://www.newgrounds.com/art/view/#{user_name}/#{illust_title}"
end
def page
return nil if page_url.blank?
response = Danbooru::Http.cache(1.minute).get(page_url)
return nil if response.status == 404
response.parse
end
memoize :page
def tags
page&.css("#sidestats .tags a").to_a.map do |tag|
[tag.text, "https://www.newgrounds.com/search/conduct/art?match=tags&tags=" + tag.text]
end
end
def normalize_tag(tag)
tag = tag.tr("-", "_")
super(tag)
end
def artist_name
name = page&.css(".item-user .item-details h4 a")&.text&.strip || user_name
name&.downcase
end
def other_names
[artist_name, user_name].compact.uniq
end
def profile_url
# user names are not mutable, artist names are.
# However we need the latest name for normalization
"https://#{artist_name}.newgrounds.com"
end
def artist_commentary_title
page&.css(".pod-head > [itemprop='name']")&.text
end
def artist_commentary_desc
page&.css("#author_comments")&.to_html
end
def dtext_artist_commentary_desc
DText.from_html(artist_commentary_desc)
end
def normalize_for_source
page_url
end
def user_name
urls.map { |u| url[PROFILE_URL, :artist_name] || u[IMAGE_URL, :user_name] || u[PAGE_URL, :user_name] }.compact.first
end
def illust_title
urls.map { |u| u[IMAGE_URL, :illust_title] || u[PAGE_URL, :illust_title] }.compact.first
end
end
end
end

257
app/logical/sources/strategies/nico_seiga.rb
View File

@@ -1,25 +1,54 @@
# Image Direct URL
# Direct URL
# * https://lohas.nicoseiga.jp/o/971eb8af9bbcde5c2e51d5ef3a2f62d6d9ff5552/1589933964/3583893
# * http://lohas.nicoseiga.jp/priv/3521156?e=1382558156&h=f2e089256abd1d453a455ec8f317a6c703e2cedf
# * http://lohas.nicoseiga.jp/priv/b80f86c0d8591b217e7513a9e175e94e00f3c7a1/1384936074/3583893
# * https://dcdn.cdn.nimg.jp/priv/62a56a7f67d3d3746ae5712db9cac7d465f4a339/1592186183/10466669
# * https://dcdn.cdn.nimg.jp/nicoseiga/lohas/o/8ba0a9b2ea34e1ef3b5cc50785bd10cd63ec7e4a/1592187477/10466669
#
# * http://lohas.nicoseiga.jp/material/5746c5/4459092
#
# (Manga direct url)
# * https://lohas.nicoseiga.jp/priv/f5b8966fd53bf7e06cccff9fbb2c4eef62877538/1590752727/8947170
#
# Samples
# * http://lohas.nicoseiga.jp/thumb/2163478i?
# * https://lohas.nicoseiga.jp/thumb/8947170p
#
## The direct urls and samples above can belong to both illust and manga.
## There's two ways to tell them apart:
## * visit the /source/ equivalent: illusts redirect to the /o/ intermediary page, manga redirect to /priv/ directly
## * try an api call: illusts will succeed, manga will fail
#
# Source Link
# * http://seiga.nicovideo.jp/image/source?id=3312222
#
# Image Page URL
# Illust Page URL
# * https://seiga.nicovideo.jp/seiga/im3521156
# * https://seiga.nicovideo.jp/seiga/im520647 (anonymous artist)
#
# Manga Page URL
# * http://seiga.nicovideo.jp/watch/mg316708
#
# Video Page URL (not supported)
# * https://www.nicovideo.jp/watch/sm36465441
#
# Oekaki
# * https://dic.nicovideo.jp/oekaki/52833.png
module Sources
module Strategies
class NicoSeiga < Base
URL = %r!\Ahttps?://(?:\w+\.)?nico(?:seiga|video)\.jp!
DIRECT1 = %r!\Ahttps?://lohas\.nicoseiga\.jp/priv/[0-9a-f]+!
DIRECT2 = %r!\Ahttps?://lohas\.nicoseiga\.jp/o/[0-9a-f]+/\d+/\d+!
DIRECT3 = %r!\Ahttps?://seiga\.nicovideo\.jp/images/source/\d+!
PAGE = %r!\Ahttps?://seiga\.nicovideo\.jp/seiga/im(\d+)!i
PROFILE = %r!\Ahttps?://seiga\.nicovideo\.jp/user/illust/(\d+)!i
MANGA_PAGE = %r!\Ahttps?://seiga\.nicovideo\.jp/watch/mg(\d+)!i
DIRECT = %r{\Ahttps?://lohas\.nicoseiga\.jp/(?:priv|o)/(?:\w+/\d+/)?(?<image_id>\d+)(?:\?.+)?}i
CDN_DIRECT = %r{\Ahttps?://dcdn\.cdn\.nimg\.jp/.+/\w+/\d+/(?<image_id>\d+)}i
SOURCE = %r{\Ahttps?://seiga\.nicovideo\.jp/image/source(?:/|\?id=)(?<image_id>\d+)}i
ILLUST_THUMB = %r{\Ahttps?://lohas\.nicoseiga\.jp/thumb/(?<illust_id>\d+)i}i
MANGA_THUMB = %r{\Ahttps?://lohas\.nicoseiga\.jp/thumb/(?<image_id>\d+)p}i
ILLUST_PAGE = %r{\Ahttps?://(?:sp\.)?seiga\.nicovideo\.jp/seiga/im(?<illust_id>\d+)}i
MANGA_PAGE = %r{\Ahttps?://(?:sp\.)?seiga\.nicovideo\.jp/watch/mg(?<manga_id>\d+)}i
PROFILE_PAGE = %r{\Ahttps?://seiga\.nicovideo\.jp/user/illust/(?<artist_id>\d+)}i
def domains
["nicoseiga.jp", "nicovideo.jp"]
@@ -30,160 +59,136 @@ module Sources
end
def image_urls
if url =~ DIRECT1
return [url]
urls = []
return urls if api_client&.api_response.blank?
if image_id.present?
urls << "https://seiga.nicovideo.jp/image/source/#{image_id}"
elsif illust_id.present?
urls << "https://seiga.nicovideo.jp/image/source/#{illust_id}"
elsif manga_id.present? && api_client.image_ids.present?
urls += api_client.image_ids.map { |id| "https://seiga.nicovideo.jp/image/source/#{id}" }
end
urls
end
if theme_id
return api_client.image_ids.map do |image_id|
def image_url
return url if image_urls.blank? || api_client.blank?
img = case url
when DIRECT || CDN_DIRECT then "https://seiga.nicovideo.jp/image/source/#{image_id_from_url(url)}"
when SOURCE then url
else image_urls.first
end
resp = api_client.login.head(img)
if resp.uri.to_s =~ %r{https?://.+/(\w+/\d+/\d+)\z}i
"https://lohas.nicoseiga.jp/priv/#{$1}"
else
img
end
end
def preview_urls
if illust_id.present?
["https://lohas.nicoseiga.jp/thumb/#{illust_id}i"]
else
image_urls
end
end
def page_url
if illust_id.present?
"https://seiga.nicovideo.jp/seiga/im#{illust_id}"
elsif manga_id.present?
"https://seiga.nicovideo.jp/watch/mg#{manga_id}"
elsif image_id.present?
"https://seiga.nicovideo.jp/image/source/#{image_id}"
end
end
link = page.search("a#illust_link")
if link.any?
image_url = "http://seiga.nicovideo.jp" + link[0]["href"]
page = agent.get(image_url) # need to follow this redirect while logged in or it won't work
if page.is_a?(Mechanize::Image)
return [page.uri.to_s]
end
images = page.search("div.illust_view_big").select {|x| x["data-src"] =~ /\/priv\//}
if images.any?
return ["http://lohas.nicoseiga.jp" + images[0]["data-src"]]
end
end
raise "image url not found for (#{url}, #{referer_url})"
end
def page_url
[url, referer_url].each do |x|
if x =~ %r!\Ahttps?://lohas\.nicoseiga\.jp/o/[a-f0-9]+/\d+/(\d+)!
return "http://seiga.nicovideo.jp/seiga/im#{$1}"
end
if x =~ %r{\Ahttps?://lohas\.nicoseiga\.jp/priv/(\d+)\?e=\d+&h=[a-f0-9]+}i
return "http://seiga.nicovideo.jp/seiga/im#{$1}"
end
if x =~ %r{\Ahttps?://lohas\.nicoseiga\.jp/priv/[a-f0-9]+/\d+/(\d+)}i
return "http://seiga.nicovideo.jp/seiga/im#{$1}"
end
if x =~ %r{\Ahttps?://lohas\.nicoseiga\.jp/priv/(\d+)}i
return "http://seiga.nicovideo.jp/seiga/im#{$1}"
end
if x =~ %r{\Ahttps?://lohas\.nicoseiga\.jp//?thumb/(\d+)i?}i
return "http://seiga.nicovideo.jp/seiga/im#{$1}"
end
if x =~ %r{/seiga/im\d+}
return x
end
if x =~ %r{/watch/mg\d+}
return x
end
if x =~ %r{/image/source\?id=(\d+)}
return "http://seiga.nicovideo.jp/seiga/im#{$1}"
end
end
return super
end
def canonical_url
image_url
end
def profile_url
if url =~ PROFILE
return url
end
user_id = api_client&.user_id
return if user_id.blank? # artists can be anonymous
"http://seiga.nicovideo.jp/user/illust/#{api_client.user_id}"
end
def artist_name
api_client.moniker
return if api_client.blank?
api_client.user_name
end
def artist_commentary_title
return if api_client.blank?
api_client.title
end
def artist_commentary_desc
api_client.desc
return if api_client.blank?
api_client.description
end
def dtext_artist_commentary_desc
DText.from_html(artist_commentary_desc).gsub(/[^\w]im(\d+)/, ' seiga #\1 ')
end
def normalize_for_source
if illust_id.present?
"https://seiga.nicovideo.jp/seiga/im#{illust_id}"
elsif theme_id.present?
"http://seiga.nicovideo.jp/watch/mg#{theme_id}"
# There's no way to tell apart illust from manga from the direct image url alone. What's worse,
# nicoseiga itself doesn't know how to normalize back to manga, so if it's not an illust type then
# it's impossible to get the original manga page back from the image url alone.
# /source/ links on the other hand correctly redirect, hence we use them to normalize saved direct sources.
if url =~ DIRECT
"https://seiga.nicovideo.jp/image/source/#{image_id}"
else
page_url
end
end
def tag_name
return if api_client&.user_id.blank?
"nicoseiga#{api_client.user_id}"
end
def tags
string = page.at("meta[name=keywords]").try(:[], "content") || ""
string.split(/,/).map do |name|
[name, "https://seiga.nicovideo.jp/tag/#{CGI.escape(name)}"]
return [] if api_client.blank?
base_url = "https://seiga.nicovideo.jp/"
base_url += "manga/" if manga_id.present?
base_url += "tag/"
api_client.tags.map do |name|
[name, base_url + CGI.escape(name)]
end
end
memoize :tags
def image_id
image_id_from_url(url)
end
def image_id_from_url(url)
url[DIRECT, :image_id] || url[SOURCE, :image_id] || url[MANGA_THUMB, :image_id] || url[CDN_DIRECT, :image_id]
end
def illust_id
urls.map { |u| u[ILLUST_PAGE, :illust_id] || u[ILLUST_THUMB, :illust_id] }.compact.first
end
def manga_id
urls.compact.map { |u| u[MANGA_PAGE, :manga_id] }.compact.first
end
def api_client
if illust_id
NicoSeigaApiClient.new(illust_id: illust_id)
elsif theme_id
NicoSeigaMangaApiClient.new(theme_id)
if illust_id.present?
NicoSeigaApiClient.new(work_id: illust_id, type: "illust", http: http)
elsif manga_id.present?
NicoSeigaApiClient.new(work_id: manga_id, type: "manga", http: http)
elsif image_id.present?
# We default to illust to attempt getting the api anyway
NicoSeigaApiClient.new(work_id: image_id, type: "illust", http: http)
end
end
memoize :api_client
def illust_id
if page_url =~ PAGE
return $1.to_i
end
return nil
end
def theme_id
if page_url =~ MANGA_PAGE
return $1.to_i
end
return nil
end
def page
doc = agent.get(page_url)
if doc.search("a#link_btn_login").any?
# Session cache is invalid, clear it and log in normally.
Cache.delete("nico-seiga-session")
doc = agent.get(page_url)
end
doc
end
memoize :page
def agent
NicoSeigaApiClient.agent
end
memoize :agent
end
end
end

78
app/logical/sources/strategies/nijie.rb
View File

@@ -44,25 +44,25 @@
module Sources
module Strategies
class Nijie < Base
BASE_URL = %r!\Ahttps?://(?:[^.]+\.)?nijie\.info!i
PAGE_URL = %r!#{BASE_URL}/view(?:_popup)?\.php\?id=(?<illust_id>\d+)!i
PROFILE_URL = %r!#{BASE_URL}/members(?:_illust)?\.php\?id=(?<artist_id>\d+)\z!i
BASE_URL = %r{\Ahttps?://(?:[^.]+\.)?nijie\.info}i
PAGE_URL = %r{#{BASE_URL}/view(?:_popup)?\.php\?id=(?<illust_id>\d+)}i
PROFILE_URL = %r{#{BASE_URL}/members(?:_illust)?\.php\?id=(?<artist_id>\d+)\z}i
# https://pic03.nijie.info/nijie_picture/28310_20131101215959.jpg
# https://pic03.nijie.info/nijie_picture/236014_20170620101426_0.png
# http://pic.nijie.net/03/nijie_picture/829001_20190620004513_0.mp4
# https://pic05.nijie.info/nijie_picture/diff/main/559053_20180604023346_1.png
FILENAME1 = %r!(?<artist_id>\d+)_(?<timestamp>\d{14})(?:_\d+)?!i
FILENAME1 = /(?<artist_id>\d+)_(?<timestamp>\d{14})(?:_\d+)?/i
# https://pic01.nijie.info/nijie_picture/diff/main/218856_0_236014_20170620101329.png
FILENAME2 = %r!(?<illust_id>\d+)_\d+_(?<artist_id>\d+)_(?<timestamp>\d{14})!i
FILENAME2 = /(?<illust_id>\d+)_\d+_(?<artist_id>\d+)_(?<timestamp>\d{14})/i
# https://pic04.nijie.info/nijie_picture/diff/main/287736_161475_20181112032855_1.png
FILENAME3 = %r!(?<illust_id>\d+)_(?<artist_id>\d+)_(?<timestamp>\d{14})_\d+!i
FILENAME3 = /(?<illust_id>\d+)_(?<artist_id>\d+)_(?<timestamp>\d{14})_\d+/i
IMAGE_BASE_URL = %r!\Ahttps?://(?:pic\d+\.nijie\.info|pic\.nijie\.net)!i
DIR = %r!(?:\d+/)?(?:__rs_\w+/)?nijie_picture(?:/diff/main)?!
IMAGE_URL = %r!#{IMAGE_BASE_URL}/#{DIR}/#{Regexp.union(FILENAME1, FILENAME2, FILENAME3)}\.\w+\z!i
IMAGE_BASE_URL = %r{\Ahttps?://(?:pic\d+\.nijie\.info|pic\.nijie\.net)}i
DIR = %r{(?:\d+/)?(?:__rs_\w+/)?nijie_picture(?:/diff/main)?}
IMAGE_URL = %r{#{IMAGE_BASE_URL}/#{DIR}/#{Regexp.union(FILENAME1, FILENAME2, FILENAME3)}\.\w+\z}i
def domains
["nijie.info", "nijie.net"]
@@ -146,7 +146,7 @@ module Sources
end
def to_full_image_url(x)
x.gsub(%r!__rs_\w+/!i, "").gsub(/\Ahttp:/, "https:")
x.gsub(%r{__rs_\w+/}i, "").gsub(/\Ahttp:/, "https:")
end
def to_preview_url(url)
@@ -178,57 +178,21 @@ module Sources
def page
return nil if page_url.blank?
doc = agent.get(page_url)
http = Danbooru::Http.new
form = { email: Danbooru.config.nijie_login, password: Danbooru.config.nijie_password }
if doc.search("div#header-login-container").any?
# Session cache is invalid, clear it and log in normally.
Cache.delete("nijie-session")
doc = agent.get(page_url)
# XXX `retriable` must come after `cache` so that retries don't return cached error responses.
response = http.cache(1.hour).use(retriable: { max_retries: 20 }).post("https://nijie.info/login_int.php", form: form)
DanbooruLogger.info "Nijie login failed (#{url}, #{response.status})" if response.status != 200
return nil unless response.status == 200
response = http.cookies(R18: 1).cache(1.minute).get(page_url)
return nil unless response.status == 200
response&.parse
end
return doc
rescue Mechanize::ResponseCodeError => e
return nil if e.response_code.to_i == 404
raise
end
memoize :page
def agent
mech = Mechanize.new
session = Cache.get("nijie-session")
if session
cookie = Mechanize::Cookie.new("NIJIEIJIEID", session)
cookie.domain = ".nijie.info"
cookie.path = "/"
mech.cookie_jar.add(cookie)
else
mech.get("https://nijie.info/login.php") do |page|
page.form_with(:action => "/login_int.php") do |form|
form['email'] = Danbooru.config.nijie_login
form['password'] = Danbooru.config.nijie_password
end.click_button
end
session = mech.cookie_jar.cookies.select {|c| c.name == "NIJIEIJIEID"}.first
Cache.put("nijie-session", session.value, 1.day) if session
end
# This cookie needs to be set to allow viewing of adult works while anonymous
cookie = Mechanize::Cookie.new("R18", "1")
cookie.domain = ".nijie.info"
cookie.path = "/"
mech.cookie_jar.add(cookie)
mech
rescue Mechanize::ResponseCodeError => x
if x.response_code.to_i == 429
sleep(5)
retry
else
raise
end
end
memoize :agent
end
end
end

6
app/logical/sources/strategies/null.rb
View File

@@ -28,7 +28,7 @@ module Sources
when %r{\Ahttp://p\.twpl\.jp/show/(?:large|orig)/([a-z0-9]+)}i
"http://p.twipple.jp/#{$1}"
when %r{\Ahttps?://blog(?:(?:-imgs-)?\d*(?:-origin)?)?\.fc2\.com/(?:(?:[^/]/){3}|(?:[^/]/))([^/]+)/(?:file/)?([^\.]+\.[^\?]+)}i
when %r{\Ahttps?://blog(?:(?:-imgs-)?\d*(?:-origin)?)?\.fc2\.com/(?:(?:[^/]/){3}|(?:[^/]/))([^/]+)/(?:file/)?([^.]+\.[^?]+)}i
username = $1
filename = $2
"http://#{username}.blog.fc2.com/img/#{filename}/"
@@ -47,7 +47,7 @@ module Sources
when %r{\Ahttps?://c(?:s|han|[1-4])\.sankakucomplex\.com/data(?:/sample)?/(?:[a-f0-9]{2}/){2}(?:sample-|preview)?([a-f0-9]{32})}i
"https://chan.sankakucomplex.com/en/post/show?md5=#{$1}"
when %r{\Ahttps?://(?:www|s(?:tatic|[1-4]))\.zerochan\.net/.+(?:\.|\/)(\d+)(?:\.(?:jpe?g?))?\z}i
when %r{\Ahttps?://(?:www|s(?:tatic|[1-4]))\.zerochan\.net/.+(?:\.|\/)(\d+)(?:\.(?:jpe?g?|png))?\z}i
"https://www.zerochan.net/#{$1}#full"
when %r{\Ahttps?://static[1-6]?\.minitokyo\.net/(?:downloads|view)/(?:\d{2}/){2}(\d+)}i
@@ -105,7 +105,7 @@ module Sources
# http://img.toranoana.jp/popup_img18/04/0010/22/87/040010228714-1p.jpg
# http://img.toranoana.jp/popup_blimg/04/0030/08/30/040030083068-1p.jpg
# https://ecdnimg.toranoana.jp/ec/img/04/0030/65/34/040030653417-6p.jpg
when %r{\Ahttps?://(\w+\.)?toranoana\.jp/(?:popup_(?:bl)?img\d*|ec/img)/\d{2}/\d{4}/\d{2}/\d{2}/(?<work_id>\d+)}i
when %r{\Ahttps?://(?:\w+\.)?toranoana\.jp/(?:popup_(?:bl)?img\d*|ec/img)/\d{2}/\d{4}/\d{2}/\d{2}/(?<work_id>\d+)}i
"https://ec.toranoana.jp/tora_r/ec/item/#{$~[:work_id]}/"
# https://a.hitomi.la/galleries/907838/1.png

53
app/logical/sources/strategies/pawoo.rb
View File

@@ -16,13 +16,13 @@
module Sources::Strategies
class Pawoo < Base
HOST = %r!\Ahttps?://(www\.)?pawoo\.net!i
IMAGE = %r!\Ahttps?://img\.pawoo\.net/media_attachments/files/(\d+/\d+/\d+)!
NAMED_PROFILE = %r!#{HOST}/@(?<artist_name>\w+)!i
ID_PROFILE = %r!#{HOST}/web/accounts/(?<artist_id>\d+)!
HOST = %r{\Ahttps?://(www\.)?pawoo\.net}i
IMAGE = %r{\Ahttps?://img\.pawoo\.net/media_attachments/files/(\d+/\d+/\d+)}
NAMED_PROFILE = %r{#{HOST}/@(?<artist_name>\w+)}i
ID_PROFILE = %r{#{HOST}/web/accounts/(?<artist_id>\d+)}
STATUS1 = %r!\A#{HOST}/web/statuses/(?<status_id>\d+)!
STATUS2 = %r!\A#{NAMED_PROFILE}/(?<status_id>\d+)!
STATUS1 = %r{\A#{HOST}/web/statuses/(?<status_id>\d+)}
STATUS2 = %r{\A#{NAMED_PROFILE}/(?<status_id>\d+)}
def domains
["pawoo.net"]
@@ -37,15 +37,13 @@ module Sources::Strategies
end
def image_urls
if url =~ %r!#{IMAGE}/small/([a-z0-9]+\.\w+)\z!i
return ["https://img.pawoo.net/media_attachments/files/#{$1}/original/#{$2}"]
if url =~ %r{#{IMAGE}/small/([a-z0-9]+\.\w+)\z}i
["https://img.pawoo.net/media_attachments/files/#{$1}/original/#{$2}"]
elsif url =~ %r{#{IMAGE}/original/([a-z0-9]+\.\w+)\z}i
[url]
else
api_response.image_urls
end
if url =~ %r!#{IMAGE}/original/([a-z0-9]+\.\w+)\z!i
return [url]
end
return api_response.image_urls
end
def page_url
@@ -55,17 +53,18 @@ module Sources::Strategies
end
end
return super
super
end
def profile_url
if url =~ PawooApiClient::PROFILE2
return "https://pawoo.net/@#{$1}"
end
return url if api_response.profile_url.blank?
"https://pawoo.net/@#{$1}"
elsif api_response.profile_url.blank?
url
else
api_response.profile_url
end
end
def artist_name
api_response.account_name
@@ -87,10 +86,6 @@ module Sources::Strategies
urls.map { |url| url[STATUS1, :status_id] || url[STATUS2, :status_id] }.compact.first
end
def artist_commentary_title
nil
end
def artist_commentary_desc
api_response.commentary
end
@@ -99,18 +94,10 @@ module Sources::Strategies
api_response.tags
end
def normalizable_for_artist_finder?
true
end
def normalize_for_artist_finder
profile_url
end
def normalize_for_source
artist_name = artist_name_from_url
status_id = status_id_from_url
return unless status_id.present?
return if status_id.blank?
if artist_name.present?
"https://pawoo.net/@#{artist_name}/#{status_id}"
@@ -131,7 +118,7 @@ module Sources::Strategies
def api_response
[url, referer_url].each do |x|
if client = PawooApiClient.new.get(x)
if (client = PawooApiClient.new.get(x))
return client
end
end

168
app/logical/sources/strategies/pixiv.rb
View File

@@ -50,37 +50,34 @@
module Sources
module Strategies
class Pixiv < Base
MONIKER = %r!(?:[a-zA-Z0-9_-]+)!
PROFILE = %r!\Ahttps?://www\.pixiv\.net/member\.php\?id=[0-9]+\z!
DATE = %r!(?<date>\d{4}/\d{2}/\d{2}/\d{2}/\d{2}/\d{2})!i
EXT = %r!(?:jpg|jpeg|png|gif)!i
MONIKER = /(?:[a-zA-Z0-9_-]+)/
PROFILE = %r{\Ahttps?://www\.pixiv\.net/member\.php\?id=[0-9]+\z}
DATE = %r{(?<date>\d{4}/\d{2}/\d{2}/\d{2}/\d{2}/\d{2})}i
EXT = /(?:jpg|jpeg|png|gif)/i
WEB = %r!(?:\A(?:https?://)?www\.pixiv\.net)!
I12 = %r!(?:\A(?:https?://)?i[0-9]+\.pixiv\.net)!
IMG = %r!(?:\A(?:https?://)?img[0-9]*\.pixiv\.net)!
PXIMG = %r!(?:\A(?:https?://)?[^.]+\.pximg\.net)!
TOUCH = %r!(?:\A(?:https?://)?touch\.pixiv\.net)!
UGOIRA = %r!#{PXIMG}/img-zip-ugoira/img/#{DATE}/(?<illust_id>\d+)_ugoira1920x1080\.zip\z!i
ORIG_IMAGE = %r!#{PXIMG}/img-original/img/#{DATE}/(?<illust_id>\d+)_p(?<page>\d+)\.#{EXT}\z!i
STACC_PAGE = %r!\A#{WEB}/stacc/#{MONIKER}/?\z!i
NOVEL_PAGE = %r!(?:\Ahttps?://www\.pixiv\.net/novel/show\.php\?id=(\d+))!
FANBOX_ACCOUNT = %r!(?:\Ahttps?://www\.pixiv\.net/fanbox/creator/\d+\z)!
FANBOX_IMAGE = %r!(?:\Ahttps?://fanbox\.pixiv\.net/images/post/(\d+))!
FANBOX_PAGE = %r!(?:\Ahttps?://www\.pixiv\.net/fanbox/creator/\d+/post/(\d+))!
WEB = %r{(?:\A(?:https?://)?www\.pixiv\.net)}
I12 = %r{(?:\A(?:https?://)?i[0-9]+\.pixiv\.net)}
IMG = %r{(?:\A(?:https?://)?img[0-9]*\.pixiv\.net)}
PXIMG = %r{(?:\A(?:https?://)?[^.]+\.pximg\.net)}
TOUCH = %r{(?:\A(?:https?://)?touch\.pixiv\.net)}
UGOIRA = %r{#{PXIMG}/img-zip-ugoira/img/#{DATE}/(?<illust_id>\d+)_ugoira1920x1080\.zip\z}i
ORIG_IMAGE = %r{#{PXIMG}/img-original/img/#{DATE}/(?<illust_id>\d+)_p(?<page>\d+)\.#{EXT}\z}i
STACC_PAGE = %r{\A#{WEB}/stacc/#{MONIKER}/?\z}i
NOVEL_PAGE = %r{(?:\Ahttps?://www\.pixiv\.net/novel/show\.php\?id=(\d+))}
def self.to_dtext(text)
if text.nil?
return nil
end
text = text.gsub(%r!https?://www\.pixiv\.net/member_illust\.php\?mode=medium&illust_id=([0-9]+)!i) do |match|
text = text.gsub(%r{https?://www\.pixiv\.net/member_illust\.php\?mode=medium&illust_id=([0-9]+)}i) do |_match|
pixiv_id = $1
%(pixiv ##{pixiv_id} "»":[/posts?tags=pixiv:#{pixiv_id}])
end
text = text.gsub(%r!https?://www\.pixiv\.net/member\.php\?id=([0-9]+)!i) do |match|
text = text.gsub(%r{https?://www\.pixiv\.net/member\.php\?id=([0-9]+)}i) do |_match|
member_id = $1
profile_url = "https://www.pixiv.net/member.php?id=#{member_id}"
profile_url = "https://www.pixiv.net/users/#{member_id}"
search_params = {"search[url_matches]" => profile_url}.to_param
%("user/#{member_id}":[#{profile_url}] "»":[/artists?#{search_params}])
@@ -127,25 +124,17 @@ module Sources
return "https://www.pixiv.net/novel/show.php?id=#{novel_id}&mode=cover"
end
if fanbox_id.present?
return "https://www.pixiv.net/fanbox/creator/#{metadata.user_id}/post/#{fanbox_id}"
end
if fanbox_account_id.present?
return "https://www.pixiv.net/fanbox/creator/#{fanbox_account_id}"
end
if illust_id.present?
return "https://www.pixiv.net/artworks/#{illust_id}"
end
return url
url
rescue PixivApiClient::BadIDError
nil
end
def canonical_url
return image_url
image_url
end
def profile_url
@@ -155,7 +144,7 @@ module Sources
end
end
"https://www.pixiv.net/member.php?id=#{metadata.user_id}"
"https://www.pixiv.net/users/#{metadata.user_id}"
rescue PixivApiClient::BadIDError
nil
end
@@ -192,17 +181,7 @@ module Sources
end
def headers
if fanbox_id.present?
# need the session to download fanbox images
return {
"Referer" => "https://www.pixiv.net/fanbox",
"Cookie" => HTTP::Cookie.cookie_value(agent.cookies)
}
end
return {
"Referer" => "https://www.pixiv.net"
}
{ "Referer" => "https://www.pixiv.net" }
end
def normalize_for_source
@@ -231,7 +210,7 @@ module Sources
translated_tags = super(tag)
if translated_tags.empty? && tag.include?("/")
translated_tags = tag.split("/").flat_map { |tag| super(tag) }
translated_tags = tag.split("/").flat_map { |translated_tag| super(translated_tag) }
end
translated_tags
@@ -242,10 +221,6 @@ module Sources
end
def image_urls_sub
if url =~ FANBOX_IMAGE
return [url]
end
# there's too much normalization bullshit we have to deal with
# raw urls, so just fetch the canonical url from the api every
# time.
@@ -257,7 +232,7 @@ module Sources
return [ugoira_zip_url]
end
return metadata.pages
metadata.pages
end
# in order to prevent recursive loops, this method should not make any
@@ -265,7 +240,7 @@ module Sources
# even though it makes sense to reference page_url here, it will only look
# at (url, referer_url).
def illust_id
return nil if novel_id.present? || fanbox_id.present?
return nil if novel_id.present?
parsed_urls.each do |url|
# http://www.pixiv.net/member_illust.php?mode=medium&illust_id=18557054
@@ -276,11 +251,11 @@ module Sources
return url.query_values["illust_id"].to_i
# http://www.pixiv.net/en/artworks/46324488
elsif url.host == "www.pixiv.net" && url.path =~ %r!\A/(?:en/)?artworks/(?<illust_id>\d+)!i
elsif url.host == "www.pixiv.net" && url.path =~ %r{\A/(?:en/)?artworks/(?<illust_id>\d+)}i
return $~[:illust_id].to_i
# http://www.pixiv.net/i/18557054
elsif url.host == "www.pixiv.net" && url.path =~ %r!\A/i/(?<illust_id>\d+)\z!i
elsif url.host == "www.pixiv.net" && url.path =~ %r{\A/i/(?<illust_id>\d+)\z}i
return $~[:illust_id].to_i
# http://img18.pixiv.net/img/evazion/14901720.png
@@ -289,8 +264,8 @@ module Sources
# http://i2.pixiv.net/img18/img/evazion/14901720_s.png
# http://i1.pixiv.net/img07/img/pasirism/18557054_p1.png
# http://i1.pixiv.net/img07/img/pasirism/18557054_big_p1.png
elsif url.host =~ %r!\A(?:i\d+|img\d+)\.pixiv\.net\z!i &&
url.path =~ %r!\A(?:/img\d+)?/img/#{MONIKER}/(?<illust_id>\d+)(?:_\w+)?\.(?:jpg|jpeg|png|gif|zip)!i
elsif url.host =~ /\A(?:i\d+|img\d+)\.pixiv\.net\z/i &&
url.path =~ %r{\A(?:/img\d+)?/img/#{MONIKER}/(?<illust_id>\d+)(?:_\w+)?\.(?:jpg|jpeg|png|gif|zip)}i
return $~[:illust_id].to_i
# http://i1.pixiv.net/img-inf/img/2011/05/01/23/28/04/18557054_64x64.jpg
@@ -307,13 +282,13 @@ module Sources
#
# https://i.pximg.net/novel-cover-original/img/2019/01/14/01/15/05/10617324_d84daae89092d96bbe66efafec136e42.jpg
# https://img-sketch.pixiv.net/uploads/medium/file/4463372/8906921629213362989.jpg
elsif url.host =~ %r!\A(?:[^.]+\.pximg\.net|i\d+\.pixiv\.net|tc-pximg01\.techorus-cdn\.com)\z!i &&
url.path =~ %r!\A(/c/\w+)?/img-[a-z-]+/img/#{DATE}/(?<illust_id>\d+)(?:_\w+)?\.(?:jpg|jpeg|png|gif|zip)!i
elsif url.host =~ /\A(?:[^.]+\.pximg\.net|i\d+\.pixiv\.net|tc-pximg01\.techorus-cdn\.com)\z/i &&
url.path =~ %r{\A(/c/\w+)?/img-[a-z-]+/img/#{DATE}/(?<illust_id>\d+)(?:_\w+)?\.(?:jpg|jpeg|png|gif|zip)}i
return $~[:illust_id].to_i
end
end
return nil
nil
end
memoize :illust_id
@@ -324,89 +299,48 @@ module Sources
end
end
return nil
nil
end
memoize :novel_id
def fanbox_id
[url, referer_url].each do |x|
if x =~ FANBOX_PAGE
return $1
end
if x =~ FANBOX_IMAGE
return $1
end
end
return nil
end
memoize :fanbox_id
def fanbox_account_id
[url, referer_url].each do |x|
if x =~ FANBOX_ACCOUNT
return x
end
end
return nil
end
memoize :fanbox_account_id
def agent
PixivWebAgent.build
end
memoize :agent
def metadata
if novel_id.present?
return PixivApiClient.new.novel(novel_id)
end
if fanbox_id.present?
return PixivApiClient.new.fanbox(fanbox_id)
end
return PixivApiClient.new.work(illust_id)
PixivApiClient.new.work(illust_id)
end
memoize :metadata
def moniker
# we can sometimes get the moniker from the url
if url =~ %r!#{IMG}/img/(#{MONIKER})!i
return $1
if url =~ %r{#{IMG}/img/(#{MONIKER})}i
$1
elsif url =~ %r{#{I12}/img[0-9]+/img/(#{MONIKER})}i
$1
elsif url =~ %r{#{WEB}/stacc/(#{MONIKER})/?$}i
$1
else
metadata.moniker
end
if url =~ %r!#{I12}/img[0-9]+/img/(#{MONIKER})!i
return $1
end
if url =~ %r!#{WEB}/stacc/(#{MONIKER})/?$!i
return $1
end
return metadata.moniker
rescue PixivApiClient::BadIDError
nil
end
memoize :moniker
def data
return {
ugoira_frame_data: ugoira_frame_data
}
{ ugoira_frame_data: ugoira_frame_data }
end
def ugoira_zip_url
if metadata.pages.is_a?(Hash) && metadata.pages["ugoira600x600"]
return metadata.pages["ugoira600x600"].sub("_ugoira600x600.zip", "_ugoira1920x1080.zip")
metadata.pages["ugoira600x600"].sub("_ugoira600x600.zip", "_ugoira1920x1080.zip")
end
end
memoize :ugoira_zip_url
def ugoira_frame_data
return metadata.json.dig("metadata", "frames")
metadata.json.dig("metadata", "frames")
rescue PixivApiClient::BadIDError
nil
end
@@ -415,16 +349,14 @@ module Sources
def ugoira_content_type
case metadata.json["image_urls"].to_s
when /\.jpg/
return "image/jpeg"
"image/jpeg"
when /\.png/
return "image/png"
"image/png"
when /\.gif/
return "image/gif"
"image/gif"
else
raise Sources::Error, "content type not found for (#{url}, #{referer_url})"
end
raise Sources::Error.new("content type not found for (#{url}, #{referer_url})")
end
memoize :ugoira_content_type
@@ -434,7 +366,7 @@ module Sources
# http://i2.pixiv.net/img04/img/syounen_no_uta/46170939_p0.jpg
# http://i1.pixiv.net/c/600x600/img-master/img/2014/09/24/23/25/08/46168376_p0_master1200.jpg
# http://i1.pixiv.net/img-original/img/2014/09/25/23/09/29/46183440_p0.jpg
if url =~ %r!/\d+_p(\d+)(?:_\w+)?\.#{EXT}!i
if url =~ %r{/\d+_p(\d+)(?:_\w+)?\.#{EXT}}i
return $1.to_i
end
@@ -445,7 +377,7 @@ module Sources
end
end
return nil
nil
end
memoize :manga_page
end

22
app/logical/sources/strategies/tumblr.rb
View File

@@ -12,19 +12,19 @@ module Sources::Strategies
class Tumblr < Base
SIZES = %w[1280 640 540 500h 500 400 250 100]
BASE_URL = %r!\Ahttps?://(?:[^/]+\.)*tumblr\.com!i
DOMAIN = %r{(data|(\d+\.)?media)\.tumblr\.com}
MD5 = %r{(?<md5>[0-9a-f]{32})}i
FILENAME = %r{(?<filename>(tumblr_(inline_)?)?[a-z0-9]+(_r[0-9]+)?)}i
EXT = %r{(?<ext>\w+)}
BASE_URL = %r{\Ahttps?://(?:[^/]+\.)*tumblr\.com}i
DOMAIN = /(data|(?:\d+\.)?media)\.tumblr\.com/i
MD5 = /(?<md5>[0-9a-f]{32})/i
FILENAME = /(?<filename>(?:tumblr_(?:inline_)?)?[a-z0-9]+(?:_r[0-9]+)?)/i
EXT = /(?<ext>\w+)/
# old: https://66.media.tumblr.com/2c6f55531618b4335c67e29157f5c1fc/tumblr_pz4a44xdVj1ssucdno1_1280.png
# new: https://66.media.tumblr.com/168dabd09d5ad69eb5fedcf94c45c31a/3dbfaec9b9e0c2e3-72/s640x960/bf33a1324f3f36d2dc64f011bfeab4867da62bc8.png
OLD_IMAGE = %r!\Ahttps?://#{DOMAIN}/(?<dir>#{MD5}/)?#{FILENAME}_(?<size>\w+)\.#{EXT}\z!i
OLD_IMAGE = %r{\Ahttps?://#{DOMAIN}/(?<dir>#{MD5}/)?#{FILENAME}_(?<size>\w+)\.#{EXT}\z}i
IMAGE = %r!\Ahttps?://#{DOMAIN}/!i
VIDEO = %r!\Ahttps?://(?:vtt|ve\.media)\.tumblr\.com/!i
POST = %r!\Ahttps?://(?<blog_name>[^.]+)\.tumblr\.com/(?:post|image)/(?<post_id>\d+)!i
IMAGE = %r{\Ahttps?://#{DOMAIN}/}i
VIDEO = %r{\Ahttps?://(?:vtt|ve|va\.media)\.tumblr\.com/}i
POST = %r{\Ahttps?://(?<blog_name>[^.]+)\.tumblr\.com/(?:post|image)/(?<post_id>\d+)}i
def self.enabled?
Danbooru.config.tumblr_consumer_key.present?
@@ -68,7 +68,7 @@ module Sources::Strategies
def preview_urls
image_urls.map do |x|
x.sub(%r!_1280\.(jpg|png|gif|jpeg)\z!, '_250.\1')
x.sub(/_1280\.(jpg|png|gif|jpeg)\z/, '_250.\1')
end
end
@@ -168,7 +168,7 @@ module Sources::Strategies
end
candidates.find do |candidate|
http_exists?(candidate, headers)
http_exists?(candidate)
end
end

34
app/logical/sources/strategies/twitter.rb
View File

@@ -1,20 +1,20 @@
module Sources::Strategies
class Twitter < Base
PAGE = %r!\Ahttps?://(?:mobile\.)?twitter\.com!i
PROFILE = %r!\Ahttps?://(?:mobile\.)?twitter.com/(?<username>[a-z0-9_]+)!i
PAGE = %r{\Ahttps?://(?:mobile\.)?twitter\.com}i
PROFILE = %r{\Ahttps?://(?:mobile\.)?twitter.com/(?<username>[a-z0-9_]+)}i
# https://pbs.twimg.com/media/EBGbJe_U8AA4Ekb.jpg
# https://pbs.twimg.com/media/EBGbJe_U8AA4Ekb?format=jpg&name=900x900
# https://pbs.twimg.com/tweet_video_thumb/ETkN_L3X0AMy1aT.jpg
# https://pbs.twimg.com/ext_tw_video_thumb/1243725361986375680/pu/img/JDA7g7lcw7wK-PIv.jpg
# https://pbs.twimg.com/amplify_video_thumb/1215590775364259840/img/lolCkEEioFZTb5dl.jpg
BASE_IMAGE_URL = %r!\Ahttps?://pbs\.twimg\.com/(?<media_type>media|tweet_video_thumb|ext_tw_video_thumb|amplify_video_thumb)!i
FILENAME1 = %r!(?<file_name>[a-zA-Z0-9_-]+)\.(?<file_ext>\w+)!i
FILENAME2 = %r!(?<file_name>[a-zA-Z0-9_-]+)\?.*format=(?<file_ext>\w+)!i
FILEPATH1 = %r!(?<file_path>\d+/[\w_-]+/img)!i
FILEPATH2 = %r!(?<file_path>\d+/img)!i
IMAGE_URL1 = %r!#{BASE_IMAGE_URL}/#{Regexp.union(FILENAME1, FILENAME2)}!i
IMAGE_URL2 = %r!#{BASE_IMAGE_URL}/#{Regexp.union(FILEPATH1, FILEPATH2)}/#{FILENAME1}!i
BASE_IMAGE_URL = %r{\Ahttps?://pbs\.twimg\.com/(?<media_type>media|tweet_video_thumb|ext_tw_video_thumb|amplify_video_thumb)}i
FILENAME1 = /(?<file_name>[a-zA-Z0-9_-]+)\.(?<file_ext>\w+)/i
FILENAME2 = /(?<file_name>[a-zA-Z0-9_-]+)\?.*format=(?<file_ext>\w+)/i
FILEPATH1 = %r{(?<file_path>\d+/[\w_-]+/img)}i
FILEPATH2 = %r{(?<file_path>\d+/img)}i
IMAGE_URL1 = %r{#{BASE_IMAGE_URL}/#{Regexp.union(FILENAME1, FILENAME2)}}i
IMAGE_URL2 = %r{#{BASE_IMAGE_URL}/#{Regexp.union(FILEPATH1, FILEPATH2)}/#{FILENAME1}}i
# Twitter provides a list but it's inaccurate; some names ('intent') aren't
# included and other names in the list aren't actually reserved.
@@ -47,7 +47,7 @@ module Sources::Strategies
return $1
end
return nil
nil
end
def self.artist_name_from_url(url)
@@ -78,7 +78,7 @@ module Sources::Strategies
elsif media[:type].in?(["video", "animated_gif"])
variants = media.dig(:video_info, :variants)
videos = variants.select { |variant| variant[:content_type] == "video/mp4" }
video = videos.max_by { |video| video[:bitrate].to_i }
video = videos.max_by { |v| v[:bitrate].to_i }
video[:url]
end
end
@@ -137,10 +137,6 @@ module Sources::Strategies
api_response[:full_text].to_s
end
def normalizable_for_artist_finder?
url =~ PAGE
end
def normalize_for_artist_finder
profile_url.try(:downcase).presence || url
end
@@ -193,9 +189,9 @@ module Sources::Strategies
desc = artist_commentary_desc.unicode_normalize(:nfkc)
desc = CGI.unescapeHTML(desc)
desc = desc.gsub(%r!https?://t\.co/[a-zA-Z0-9]+!i, url_replacements)
desc = desc.gsub(%r!#([^[:space:]]+)!, '"#\\1":[https://twitter.com/hashtag/\\1]')
desc = desc.gsub(%r!@([a-zA-Z0-9_]+)!, '"@\\1":[https://twitter.com/\\1]')
desc = desc.gsub(%r{https?://t\.co/[a-zA-Z0-9]+}i, url_replacements)
desc = desc.gsub(/#([^[:space:]]+)/, '"#\\1":[https://twitter.com/hashtag/\\1]')
desc = desc.gsub(/@([a-zA-Z0-9_]+)/, '"@\\1":[https://twitter.com/\\1]')
desc.strip
end
@@ -204,7 +200,7 @@ module Sources::Strategies
end
def api_response
return {} if !self.class.enabled?
return {} unless self.class.enabled? && status_id.present?
api_client.status(status_id)
end

6
app/logical/sources/strategies/weibo.rb
View File

@@ -38,7 +38,7 @@ module Sources
PAGE_URL_1 = %r{\Ahttps?://(?:www\.)?weibo\.com/(?<artist_short_id>\d+)/(?<illust_base62_id>\w+)(?:\?.*)?\z}i
PAGE_URL_2 = %r{#{PROFILE_URL_2}/(?:wbphotos/large/mid|talbum/detail/photo_id)/(?<illust_long_id>\d+)(?:/pid/(?<image_id>\w{32}))?}i
PAGE_URL_3 = %r{\Ahttps?://m\.weibo\.cn/(detail/(?<illust_long_id>\d+)|status/(?<illust_base62_id>\w+))}i
PAGE_URL_3 = %r{\Ahttps?://m\.weibo\.cn/(?:detail/(?<illust_long_id>\d+)|status/(?<illust_base62_id>\w+))}i
PAGE_URL_4 = %r{\Ahttps?://tw\.weibo\.com/(?:(?<artist_short_id>\d+)|\w+)/(?<illust_long_id>\d+)}i
IMAGE_URL = %r{\Ahttps?://\w{3}\.sinaimg\.cn/\w+/(?<image_id>\w{32})\.}i
@@ -203,12 +203,12 @@ module Sources
end
def api_response
return nil if mobile_url.blank?
return {} if mobile_url.blank?
resp = Danbooru::Http.cache(1.minute).get(mobile_url)
json_string = resp.to_s[/var \$render_data = \[(.*)\]\[0\]/m, 1]
return nil if json_string.blank?
return {} if json_string.blank?
JSON.parse(json_string)["status"]
end

9
app/logical/spam_detector.rb
View File

@@ -7,10 +7,11 @@ class SpamDetector
# if a person receives more than 10 automatic spam reports within a 1 hour
# window, automatically ban them forever.
AUTOBAN_THRESHOLD = 10
AUTOBAN_WINDOW = 1.hours
AUTOBAN_DURATION = 999999
AUTOBAN_WINDOW = 1.hour
AUTOBAN_DURATION = 999_999
attr_accessor :record, :user, :user_ip, :content, :comment_type
rakismet_attrs author: proc { user.name },
author_email: proc { user.email_address&.address },
blog_lang: "en",
@@ -84,8 +85,8 @@ class SpamDetector
end
is_spam
rescue StandardError => exception
DanbooruLogger.log(exception)
rescue StandardError => e
DanbooruLogger.log(e)
false
end
end

Some files were not shown because too many files have changed in this diff Show More