Merge branch 'master' into fix-pixiv-profile-url
This commit is contained in:
evazion
GitHub
3
.bundle/config
Normal file
3
.bundle/config
Normal file
@@ -0,0 +1,3 @@
|
||||
---
|
||||
BUNDLE_BUILD__NOKOGIRI: "--use-system-libraries"
|
||||
BUNDLE_BUILD__NOKOGUMBO: "--without-libxml2"
|
||||
14
.editorconfig
Normal file
14
.editorconfig
Normal file
@@ -0,0 +1,14 @@
|
||||
root = true
|
||||
|
||||
[**.{js,rb,css,erb,md,json,yml}]
|
||||
indent_style = space
|
||||
indent_size = 2
|
||||
end_of_line = lf
|
||||
insert_final_newline = true
|
||||
trim_trailing_whitespace = true
|
||||
|
||||
[**.html.erb]
|
||||
indent_size = unset
|
||||
|
||||
[app/javascript/vendor/**]
|
||||
indent_size = unset
|
||||
@@ -8,9 +8,13 @@ parserOptions:
|
||||
globals:
|
||||
$: false
|
||||
require: false
|
||||
parser: babel-eslint
|
||||
plugins:
|
||||
- babel
|
||||
rules:
|
||||
# https://eslint.org/docs/rules/
|
||||
array-callback-return: error
|
||||
babel/no-unused-expressions: error
|
||||
block-scoped-var: error
|
||||
consistent-return: error
|
||||
default-case: error
|
||||
@@ -32,7 +36,7 @@ rules:
|
||||
no-sequences: error
|
||||
no-shadow: error
|
||||
no-shadow-restricted-names: error
|
||||
no-unused-expressions: error
|
||||
#no-unused-expressions: error
|
||||
no-unused-vars:
|
||||
- error
|
||||
- argsIgnorePattern: "^_"
|
||||
|
||||
2
.github/workflows/test.yaml
vendored
2
.github/workflows/test.yaml
vendored
@@ -64,7 +64,7 @@ jobs:
|
||||
- name: Install OS dependencies
|
||||
run: |
|
||||
apt-get update
|
||||
apt-get -y install --no-install-recommends build-essential ruby ruby-dev ruby-bundler git nodejs yarnpkg webpack ffmpeg mkvtoolnix libvips-dev libxml2-dev postgresql-server-dev-all wget
|
||||
apt-get -y install --no-install-recommends build-essential ruby ruby-dev ruby-bundler git nodejs yarnpkg webpack ffmpeg mkvtoolnix libvips-dev libxml2-dev libxslt-dev zlib1g-dev postgresql-server-dev-all wget
|
||||
ln -sf /usr/bin/yarnpkg /usr/bin/yarn
|
||||
|
||||
- name: Install Ruby dependencies
|
||||
|
||||
1
.gitignore
vendored
1
.gitignore
vendored
@@ -2,7 +2,6 @@
|
||||
.yarn-integrity
|
||||
.gitconfig
|
||||
.git/
|
||||
.bundle/
|
||||
config/database.yml
|
||||
config/danbooru_local_config.rb
|
||||
config/deploy/*.rb
|
||||
|
||||
8
Gemfile
8
Gemfile
@@ -7,7 +7,6 @@ gem "pg"
|
||||
gem "delayed_job"
|
||||
gem "delayed_job_active_record"
|
||||
gem "simple_form"
|
||||
gem "mechanize"
|
||||
gem "whenever", :require => false
|
||||
gem "sanitize"
|
||||
gem 'ruby-vips'
|
||||
@@ -28,7 +27,6 @@ gem 'daemons'
|
||||
gem 'oauth2'
|
||||
gem 'bootsnap'
|
||||
gem 'addressable'
|
||||
gem 'httparty'
|
||||
gem 'rakismet'
|
||||
gem 'recaptcha', require: "recaptcha/rails"
|
||||
gem 'activemodel-serializers-xml'
|
||||
@@ -47,9 +45,7 @@ gem 'http'
|
||||
gem 'activerecord-hierarchical_query'
|
||||
gem 'pundit'
|
||||
gem 'mail'
|
||||
|
||||
# locked to 1.10.9 to workaround an incompatibility with nokogumbo 2.0.2.
|
||||
gem 'nokogiri', '~> 1.10.9'
|
||||
gem 'nokogiri'
|
||||
|
||||
group :production, :staging do
|
||||
gem 'unicorn', :platforms => :ruby
|
||||
@@ -65,7 +61,6 @@ end
|
||||
group :development do
|
||||
gem 'rubocop'
|
||||
gem 'rubocop-rails'
|
||||
gem 'sinatra'
|
||||
gem 'meta_request'
|
||||
gem 'rack-mini-profiler'
|
||||
gem 'stackprof'
|
||||
@@ -86,7 +81,6 @@ group :test do
|
||||
gem "mocha", require: "mocha/minitest"
|
||||
gem "ffaker"
|
||||
gem "simplecov", "~> 0.17.0", require: false
|
||||
gem "webmock", require: "webmock/minitest"
|
||||
gem "minitest-ci"
|
||||
gem "minitest-reporters", require: "minitest/reporters"
|
||||
gem "mock_redis"
|
||||
|
||||
177
Gemfile.lock
177
Gemfile.lock
@@ -8,63 +8,63 @@ GIT
|
||||
GEM
|
||||
remote: https://rubygems.org/
|
||||
specs:
|
||||
actioncable (6.0.3.1)
|
||||
actionpack (= 6.0.3.1)
|
||||
actioncable (6.0.3.2)
|
||||
actionpack (= 6.0.3.2)
|
||||
nio4r (~> 2.0)
|
||||
websocket-driver (>= 0.6.1)
|
||||
actionmailbox (6.0.3.1)
|
||||
actionpack (= 6.0.3.1)
|
||||
activejob (= 6.0.3.1)
|
||||
activerecord (= 6.0.3.1)
|
||||
activestorage (= 6.0.3.1)
|
||||
activesupport (= 6.0.3.1)
|
||||
actionmailbox (6.0.3.2)
|
||||
actionpack (= 6.0.3.2)
|
||||
activejob (= 6.0.3.2)
|
||||
activerecord (= 6.0.3.2)
|
||||
activestorage (= 6.0.3.2)
|
||||
activesupport (= 6.0.3.2)
|
||||
mail (>= 2.7.1)
|
||||
actionmailer (6.0.3.1)
|
||||
actionpack (= 6.0.3.1)
|
||||
actionview (= 6.0.3.1)
|
||||
activejob (= 6.0.3.1)
|
||||
actionmailer (6.0.3.2)
|
||||
actionpack (= 6.0.3.2)
|
||||
actionview (= 6.0.3.2)
|
||||
activejob (= 6.0.3.2)
|
||||
mail (~> 2.5, >= 2.5.4)
|
||||
rails-dom-testing (~> 2.0)
|
||||
actionpack (6.0.3.1)
|
||||
actionview (= 6.0.3.1)
|
||||
activesupport (= 6.0.3.1)
|
||||
actionpack (6.0.3.2)
|
||||
actionview (= 6.0.3.2)
|
||||
activesupport (= 6.0.3.2)
|
||||
rack (~> 2.0, >= 2.0.8)
|
||||
rack-test (>= 0.6.3)
|
||||
rails-dom-testing (~> 2.0)
|
||||
rails-html-sanitizer (~> 1.0, >= 1.2.0)
|
||||
actiontext (6.0.3.1)
|
||||
actionpack (= 6.0.3.1)
|
||||
activerecord (= 6.0.3.1)
|
||||
activestorage (= 6.0.3.1)
|
||||
activesupport (= 6.0.3.1)
|
||||
actiontext (6.0.3.2)
|
||||
actionpack (= 6.0.3.2)
|
||||
activerecord (= 6.0.3.2)
|
||||
activestorage (= 6.0.3.2)
|
||||
activesupport (= 6.0.3.2)
|
||||
nokogiri (>= 1.8.5)
|
||||
actionview (6.0.3.1)
|
||||
activesupport (= 6.0.3.1)
|
||||
actionview (6.0.3.2)
|
||||
activesupport (= 6.0.3.2)
|
||||
builder (~> 3.1)
|
||||
erubi (~> 1.4)
|
||||
rails-dom-testing (~> 2.0)
|
||||
rails-html-sanitizer (~> 1.1, >= 1.2.0)
|
||||
activejob (6.0.3.1)
|
||||
activesupport (= 6.0.3.1)
|
||||
activejob (6.0.3.2)
|
||||
activesupport (= 6.0.3.2)
|
||||
globalid (>= 0.3.6)
|
||||
activemodel (6.0.3.1)
|
||||
activesupport (= 6.0.3.1)
|
||||
activemodel (6.0.3.2)
|
||||
activesupport (= 6.0.3.2)
|
||||
activemodel-serializers-xml (1.0.2)
|
||||
activemodel (> 5.x)
|
||||
activesupport (> 5.x)
|
||||
builder (~> 3.1)
|
||||
activerecord (6.0.3.1)
|
||||
activemodel (= 6.0.3.1)
|
||||
activesupport (= 6.0.3.1)
|
||||
activerecord (6.0.3.2)
|
||||
activemodel (= 6.0.3.2)
|
||||
activesupport (= 6.0.3.2)
|
||||
activerecord-hierarchical_query (1.2.3)
|
||||
activerecord (>= 5.0, < 6.1)
|
||||
pg (>= 0.21, < 1.3)
|
||||
activestorage (6.0.3.1)
|
||||
actionpack (= 6.0.3.1)
|
||||
activejob (= 6.0.3.1)
|
||||
activerecord (= 6.0.3.1)
|
||||
activestorage (6.0.3.2)
|
||||
actionpack (= 6.0.3.2)
|
||||
activejob (= 6.0.3.2)
|
||||
activerecord (= 6.0.3.2)
|
||||
marcel (~> 0.3.1)
|
||||
activesupport (6.0.3.1)
|
||||
activesupport (6.0.3.2)
|
||||
concurrent-ruby (~> 1.0, >= 1.0.2)
|
||||
i18n (>= 0.7, < 2)
|
||||
minitest (~> 5.1)
|
||||
@@ -77,7 +77,7 @@ GEM
|
||||
ansi (1.5.0)
|
||||
ast (2.4.1)
|
||||
aws-eventstream (1.1.0)
|
||||
aws-partitions (1.329.0)
|
||||
aws-partitions (1.332.0)
|
||||
aws-sdk-core (3.100.0)
|
||||
aws-eventstream (~> 1, >= 1.0.2)
|
||||
aws-partitions (~> 1, >= 1.239.0)
|
||||
@@ -86,8 +86,8 @@ GEM
|
||||
aws-sdk-sqs (1.27.1)
|
||||
aws-sdk-core (~> 3, >= 3.99.0)
|
||||
aws-sigv4 (~> 1.1)
|
||||
aws-sigv4 (1.1.4)
|
||||
aws-eventstream (~> 1.0, >= 1.0.2)
|
||||
aws-sigv4 (1.2.0)
|
||||
aws-eventstream (~> 1, >= 1.0.2)
|
||||
bcrypt (3.1.13)
|
||||
bootsnap (1.4.6)
|
||||
msgpack (~> 1.0)
|
||||
@@ -110,7 +110,7 @@ GEM
|
||||
sshkit (~> 1.3)
|
||||
capistrano3-unicorn (0.2.1)
|
||||
capistrano (~> 3.1, >= 3.1.0)
|
||||
capybara (3.32.2)
|
||||
capybara (3.33.0)
|
||||
addressable
|
||||
mini_mime (>= 0.1.3)
|
||||
nokogiri (~> 1.8)
|
||||
@@ -122,9 +122,6 @@ GEM
|
||||
chronic (0.10.2)
|
||||
coderay (1.1.3)
|
||||
concurrent-ruby (1.1.6)
|
||||
connection_pool (2.2.3)
|
||||
crack (0.4.3)
|
||||
safe_yaml (~> 1.0.0)
|
||||
crass (1.0.6)
|
||||
daemons (1.3.1)
|
||||
delayed_job (4.1.8)
|
||||
@@ -141,8 +138,8 @@ GEM
|
||||
dotenv (= 2.7.5)
|
||||
railties (>= 3.2, < 6.1)
|
||||
erubi (1.9.0)
|
||||
factory_bot (5.2.0)
|
||||
activesupport (>= 4.2.0)
|
||||
factory_bot (6.0.2)
|
||||
activesupport (>= 5.0.0)
|
||||
faraday (1.0.1)
|
||||
multipart-post (>= 1.2, < 3)
|
||||
ffaker (2.15.0)
|
||||
@@ -156,7 +153,6 @@ GEM
|
||||
ffi (~> 1.0)
|
||||
globalid (0.4.2)
|
||||
activesupport (>= 4.2.0)
|
||||
hashdiff (1.0.1)
|
||||
http (4.4.1)
|
||||
addressable (~> 2.3)
|
||||
http-cookie (~> 1.0)
|
||||
@@ -167,9 +163,6 @@ GEM
|
||||
http-form_data (2.3.0)
|
||||
http-parser (1.2.1)
|
||||
ffi-compiler (>= 1.0, < 2.0)
|
||||
httparty (0.18.1)
|
||||
mime-types (~> 3.0)
|
||||
multi_xml (>= 0.5.2)
|
||||
i18n (1.8.3)
|
||||
concurrent-ruby (~> 1.0)
|
||||
ipaddress_2 (0.13.0)
|
||||
@@ -184,34 +177,22 @@ GEM
|
||||
listen (3.2.1)
|
||||
rb-fsevent (~> 0.10, >= 0.10.3)
|
||||
rb-inotify (~> 0.9, >= 0.9.10)
|
||||
loofah (2.5.0)
|
||||
loofah (2.6.0)
|
||||
crass (~> 1.0.2)
|
||||
nokogiri (>= 1.5.9)
|
||||
mail (2.7.1)
|
||||
mini_mime (>= 0.1.1)
|
||||
marcel (0.3.3)
|
||||
mimemagic (~> 0.3.2)
|
||||
mechanize (2.7.6)
|
||||
domain_name (~> 0.5, >= 0.5.1)
|
||||
http-cookie (~> 1.0)
|
||||
mime-types (>= 1.17.2)
|
||||
net-http-digest_auth (~> 1.1, >= 1.1.1)
|
||||
net-http-persistent (>= 2.5.2)
|
||||
nokogiri (~> 1.6)
|
||||
ntlm-http (~> 0.1, >= 0.1.1)
|
||||
webrobots (>= 0.0.9, < 0.2)
|
||||
memoist (0.16.2)
|
||||
memory_profiler (0.9.14)
|
||||
meta_request (0.7.2)
|
||||
rack-contrib (>= 1.1, < 3)
|
||||
railties (>= 3.0.0, < 7)
|
||||
method_source (1.0.0)
|
||||
mime-types (3.3.1)
|
||||
mime-types-data (~> 3.2015)
|
||||
mime-types-data (3.2020.0512)
|
||||
mimemagic (0.3.5)
|
||||
mini_mime (1.0.2)
|
||||
mini_portile2 (2.4.0)
|
||||
mini_portile2 (2.5.0)
|
||||
minitest (5.14.1)
|
||||
minitest-ci (3.4.0)
|
||||
minitest (>= 5.0.6)
|
||||
@@ -221,17 +202,12 @@ GEM
|
||||
minitest (>= 5.0)
|
||||
ruby-progressbar
|
||||
mocha (1.11.2)
|
||||
mock_redis (0.23.0)
|
||||
mock_redis (0.24.0)
|
||||
msgpack (1.3.3)
|
||||
msgpack (1.3.3-x64-mingw32)
|
||||
multi_json (1.14.1)
|
||||
multi_xml (0.6.0)
|
||||
multipart-post (2.1.1)
|
||||
mustermann (1.1.1)
|
||||
ruby2_keywords (~> 0.0.1)
|
||||
net-http-digest_auth (1.4.1)
|
||||
net-http-persistent (4.0.0)
|
||||
connection_pool (~> 2.2)
|
||||
net-scp (3.0.0)
|
||||
net-ssh (>= 2.6.5, < 7.0.0)
|
||||
net-sftp (3.0.0)
|
||||
@@ -239,22 +215,20 @@ GEM
|
||||
net-ssh (6.1.0)
|
||||
newrelic_rpm (6.11.0.365)
|
||||
nio4r (2.5.2)
|
||||
nokogiri (1.10.9)
|
||||
mini_portile2 (~> 2.4.0)
|
||||
nokogiri (1.10.9-x64-mingw32)
|
||||
mini_portile2 (~> 2.4.0)
|
||||
nokogiri (1.11.0.rc2)
|
||||
mini_portile2 (~> 2.5.0)
|
||||
nokogiri (1.11.0.rc2-x64-mingw32)
|
||||
nokogumbo (2.0.2)
|
||||
nokogiri (~> 1.8, >= 1.8.4)
|
||||
ntlm-http (0.1.1)
|
||||
oauth2 (1.4.4)
|
||||
faraday (>= 0.8, < 2.0)
|
||||
jwt (>= 1.0, < 3.0)
|
||||
multi_json (~> 1.3)
|
||||
multi_xml (~> 0.5)
|
||||
rack (>= 1.2, < 3)
|
||||
parallel (1.19.1)
|
||||
parser (2.7.1.3)
|
||||
ast (~> 2.4.0)
|
||||
parallel (1.19.2)
|
||||
parser (2.7.1.4)
|
||||
ast (~> 2.4.1)
|
||||
pg (1.2.3)
|
||||
pg (1.2.3-x64-mingw32)
|
||||
pry (0.13.1)
|
||||
@@ -275,35 +249,33 @@ GEM
|
||||
rack (~> 2.0)
|
||||
rack-mini-profiler (2.0.2)
|
||||
rack (>= 1.2.0)
|
||||
rack-protection (2.0.8.1)
|
||||
rack
|
||||
rack-proxy (0.6.5)
|
||||
rack
|
||||
rack-test (1.1.0)
|
||||
rack (>= 1.0, < 3)
|
||||
rails (6.0.3.1)
|
||||
actioncable (= 6.0.3.1)
|
||||
actionmailbox (= 6.0.3.1)
|
||||
actionmailer (= 6.0.3.1)
|
||||
actionpack (= 6.0.3.1)
|
||||
actiontext (= 6.0.3.1)
|
||||
actionview (= 6.0.3.1)
|
||||
activejob (= 6.0.3.1)
|
||||
activemodel (= 6.0.3.1)
|
||||
activerecord (= 6.0.3.1)
|
||||
activestorage (= 6.0.3.1)
|
||||
activesupport (= 6.0.3.1)
|
||||
rails (6.0.3.2)
|
||||
actioncable (= 6.0.3.2)
|
||||
actionmailbox (= 6.0.3.2)
|
||||
actionmailer (= 6.0.3.2)
|
||||
actionpack (= 6.0.3.2)
|
||||
actiontext (= 6.0.3.2)
|
||||
actionview (= 6.0.3.2)
|
||||
activejob (= 6.0.3.2)
|
||||
activemodel (= 6.0.3.2)
|
||||
activerecord (= 6.0.3.2)
|
||||
activestorage (= 6.0.3.2)
|
||||
activesupport (= 6.0.3.2)
|
||||
bundler (>= 1.3.0)
|
||||
railties (= 6.0.3.1)
|
||||
railties (= 6.0.3.2)
|
||||
sprockets-rails (>= 2.0.0)
|
||||
rails-dom-testing (2.0.3)
|
||||
activesupport (>= 4.2.0)
|
||||
nokogiri (>= 1.6)
|
||||
rails-html-sanitizer (1.3.0)
|
||||
loofah (~> 2.3)
|
||||
railties (6.0.3.1)
|
||||
actionpack (= 6.0.3.1)
|
||||
activesupport (= 6.0.3.1)
|
||||
railties (6.0.3.2)
|
||||
actionpack (= 6.0.3.2)
|
||||
activesupport (= 6.0.3.2)
|
||||
method_source
|
||||
rake (>= 0.8.7)
|
||||
thor (>= 0.20.3, < 2.0)
|
||||
@@ -343,9 +315,7 @@ GEM
|
||||
ruby-progressbar (1.10.1)
|
||||
ruby-vips (2.0.17)
|
||||
ffi (~> 1.9)
|
||||
ruby2_keywords (0.0.2)
|
||||
rubyzip (2.3.0)
|
||||
safe_yaml (1.0.5)
|
||||
sanitize (5.2.1)
|
||||
crass (~> 1.0.2)
|
||||
nokogiri (>= 1.8.0)
|
||||
@@ -368,11 +338,6 @@ GEM
|
||||
json (>= 1.8, < 3)
|
||||
simplecov-html (~> 0.10.0)
|
||||
simplecov-html (0.10.2)
|
||||
sinatra (2.0.8.1)
|
||||
mustermann (~> 1.0)
|
||||
rack (~> 2.0)
|
||||
rack-protection (= 2.0.8.1)
|
||||
tilt (~> 2.0)
|
||||
sprockets (4.0.2)
|
||||
concurrent-ruby (~> 1.0)
|
||||
rack (> 1, < 3)
|
||||
@@ -389,7 +354,6 @@ GEM
|
||||
stripe (5.22.0)
|
||||
thor (1.0.1)
|
||||
thread_safe (0.3.6)
|
||||
tilt (2.0.10)
|
||||
tzinfo (1.2.7)
|
||||
thread_safe (~> 0.1)
|
||||
unf (0.1.4)
|
||||
@@ -403,16 +367,11 @@ GEM
|
||||
unicorn-worker-killer (0.4.4)
|
||||
get_process_mem (~> 0)
|
||||
unicorn (>= 4, < 6)
|
||||
webmock (3.8.3)
|
||||
addressable (>= 2.3.6)
|
||||
crack (>= 0.3.2)
|
||||
hashdiff (>= 0.4.0, < 2.0.0)
|
||||
webpacker (5.1.1)
|
||||
activesupport (>= 5.2)
|
||||
rack-proxy (>= 0.6.1)
|
||||
railties (>= 5.2)
|
||||
semantic_range (>= 2.3.0)
|
||||
webrobots (0.1.2)
|
||||
websocket-driver (0.7.2)
|
||||
websocket-extensions (>= 0.1.0)
|
||||
websocket-extensions (0.1.5)
|
||||
@@ -450,12 +409,10 @@ DEPENDENCIES
|
||||
ffaker
|
||||
flamegraph
|
||||
http
|
||||
httparty
|
||||
ipaddress_2
|
||||
jquery-rails
|
||||
listen
|
||||
mail
|
||||
mechanize
|
||||
memoist
|
||||
memory_profiler
|
||||
meta_request
|
||||
@@ -465,7 +422,7 @@ DEPENDENCIES
|
||||
mock_redis
|
||||
net-sftp
|
||||
newrelic_rpm
|
||||
nokogiri (~> 1.10.9)
|
||||
nokogiri
|
||||
oauth2
|
||||
pg
|
||||
pry-byebug
|
||||
@@ -492,13 +449,11 @@ DEPENDENCIES
|
||||
shoulda-matchers
|
||||
simple_form
|
||||
simplecov (~> 0.17.0)
|
||||
sinatra
|
||||
stackprof
|
||||
streamio-ffmpeg
|
||||
stripe
|
||||
unicorn
|
||||
unicorn-worker-killer
|
||||
webmock
|
||||
webpacker (>= 4.0.x)
|
||||
whenever
|
||||
|
||||
|
||||
@@ -32,9 +32,6 @@ if [[ -z "$HOSTNAME" ]] ; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo -n "* Enter the VLAN IP address for this server (ex: 172.16.0.1, enter nothing to skip): "
|
||||
read VLAN_IP_ADDR
|
||||
|
||||
# Install packages
|
||||
echo "* Installing packages..."
|
||||
|
||||
@@ -52,17 +49,6 @@ apt-get -y install $LIBSSL_DEV_PKG build-essential automake libxml2-dev libxslt-
|
||||
apt-get -y install libpq-dev postgresql-client
|
||||
apt-get -y install liblcms2-dev $LIBJPEG_TURBO_DEV_PKG libexpat1-dev libgif-dev libpng-dev libexif-dev
|
||||
|
||||
# vrack specific stuff
|
||||
if [ -n "$VLAN_IP_ADDR" ] ; then
|
||||
apt-get -y install vlan
|
||||
modprobe 8021q
|
||||
echo "8021q" >> /etc/modules
|
||||
vconfig add eno2 99
|
||||
ip addr add $VLAN_IP_ADDR/24 dev eno2.99
|
||||
ip link set up eno2.99
|
||||
curl -L -s $GITHUB_INSTALL_SCRIPTS/vrack-cfg.yaml -o /etc/netplan/01-netcfg.yaml
|
||||
fi
|
||||
|
||||
curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -
|
||||
echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list
|
||||
curl -sSL https://deb.nodesource.com/setup_10.x | sudo -E bash -
|
||||
|
||||
@@ -28,11 +28,11 @@ module Explore
|
||||
|
||||
def searches
|
||||
@date, @scale, @min_date, @max_date = parse_date(params)
|
||||
@search_service = ReportbooruService.new
|
||||
@searches = ReportbooruService.new.popular_searches(@date)
|
||||
end
|
||||
|
||||
def missed_searches
|
||||
@search_service = ReportbooruService.new
|
||||
@missed_searches = ReportbooruService.new.missed_search_rankings
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
48
app/controllers/mock_services_controller.rb
Normal file
48
app/controllers/mock_services_controller.rb
Normal file
@@ -0,0 +1,48 @@
|
||||
class MockServicesController < ApplicationController
|
||||
skip_forgery_protection
|
||||
respond_to :json
|
||||
|
||||
before_action do
|
||||
raise User::PrivilegeError if Rails.env.production?
|
||||
end
|
||||
|
||||
def recommender_recommend
|
||||
@data = posts.map { |post| [post.id, rand(0.0..1.0)] }
|
||||
render json: @data
|
||||
end
|
||||
|
||||
def recommender_similar
|
||||
@data = posts.map { |post| [post.id, rand(0.0..1.0)] }
|
||||
render json: @data
|
||||
end
|
||||
|
||||
def reportbooru_missed_searches
|
||||
@data = tags.map { |tag| "#{tag.name} #{rand(1.0..1000.0)}" }.join("\n")
|
||||
render json: @data
|
||||
end
|
||||
|
||||
def reportbooru_post_searches
|
||||
@data = tags.map { |tag| [tag.name, rand(1..1000)] }
|
||||
render json: @data
|
||||
end
|
||||
|
||||
def reportbooru_post_views
|
||||
@data = posts.map { |post| [post.id, rand(1..1000)] }
|
||||
render json: @data
|
||||
end
|
||||
|
||||
def iqdbs_similar
|
||||
@data = posts.map { |post| { post_id: post.id, score: rand(0..100)} }
|
||||
render json: @data
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def posts(limit = 10)
|
||||
Post.last(limit)
|
||||
end
|
||||
|
||||
def tags(limit = 10)
|
||||
Tag.order(post_count: :desc).limit(limit)
|
||||
end
|
||||
end
|
||||
@@ -21,7 +21,7 @@ class UploadsController < ApplicationController
|
||||
def image_proxy
|
||||
authorize Upload
|
||||
resp = ImageProxy.get_image(params[:url])
|
||||
send_data resp.body, :type => resp.content_type, :disposition => "inline"
|
||||
send_data resp.body, type: resp.mime_type, disposition: "inline"
|
||||
end
|
||||
|
||||
def index
|
||||
|
||||
@@ -9,6 +9,7 @@ Autocomplete.ORDER_METATAGS = <%= PostQueryBuilder::ORDER_METATAGS.to_json.html_
|
||||
Autocomplete.DISAPPROVAL_REASONS = <%= PostDisapproval::REASONS.to_json.html_safe %>;
|
||||
/* eslint-enable */
|
||||
|
||||
Autocomplete.MISC_STATUSES = ["deleted", "active", "pending", "flagged", "banned", "modqueue", "unmoderated"];
|
||||
Autocomplete.TAG_PREFIXES = "-|~|" + Object.keys(Autocomplete.TAG_CATEGORIES).map(category => category + ":").join("|");
|
||||
Autocomplete.METATAGS_REGEX = Autocomplete.METATAGS.concat(Object.keys(Autocomplete.TAG_CATEGORIES)).join("|");
|
||||
Autocomplete.TERM_REGEX = new RegExp(`([-~]*)(?:(${Autocomplete.METATAGS_REGEX}):)?(\\S*)$`, "i");
|
||||
@@ -268,9 +269,7 @@ Autocomplete.render_item = function(list, item) {
|
||||
|
||||
Autocomplete.static_metatags = {
|
||||
order: Autocomplete.ORDER_METATAGS,
|
||||
status: [
|
||||
"any", "deleted", "active", "pending", "flagged", "banned", "modqueue", "unmoderated"
|
||||
],
|
||||
status: ["any"].concat(Autocomplete.MISC_STATUSES),
|
||||
rating: [
|
||||
"safe", "questionable", "explicit"
|
||||
],
|
||||
@@ -280,12 +279,8 @@ Autocomplete.static_metatags = {
|
||||
embedded: [
|
||||
"true", "false"
|
||||
],
|
||||
child: [
|
||||
"any", "none"
|
||||
],
|
||||
parent: [
|
||||
"any", "none"
|
||||
],
|
||||
child: ["any", "none"].concat(Autocomplete.MISC_STATUSES),
|
||||
parent: ["any", "none"].concat(Autocomplete.MISC_STATUSES),
|
||||
filetype: [
|
||||
"jpg", "png", "gif", "swf", "zip", "webm", "mp4"
|
||||
],
|
||||
|
||||
@@ -300,10 +300,10 @@ Post.initialize_favlist = function() {
|
||||
});
|
||||
}
|
||||
|
||||
Post.view_original = function(e) {
|
||||
Post.view_original = function(e = null) {
|
||||
if (Utility.test_max_width(660)) {
|
||||
// Do the default behavior (navigate to image)
|
||||
return false;
|
||||
return;
|
||||
}
|
||||
|
||||
var $image = $("#image");
|
||||
@@ -316,13 +316,13 @@ Post.view_original = function(e) {
|
||||
});
|
||||
Note.Box.scale_all();
|
||||
$("body").attr("data-post-current-image-size", "original");
|
||||
return false;
|
||||
e?.preventDefault();
|
||||
}
|
||||
|
||||
Post.view_large = function(e) {
|
||||
Post.view_large = function(e = null) {
|
||||
if (Utility.test_max_width(660)) {
|
||||
// Do the default behavior (navigate to image)
|
||||
return false;
|
||||
return;
|
||||
}
|
||||
|
||||
var $image = $("#image");
|
||||
@@ -335,7 +335,7 @@ Post.view_large = function(e) {
|
||||
});
|
||||
Note.Box.scale_all();
|
||||
$("body").attr("data-post-current-image-size", "large");
|
||||
return false;
|
||||
e?.preventDefault();
|
||||
}
|
||||
|
||||
Post.toggle_fit_window = function(e) {
|
||||
|
||||
@@ -9,15 +9,6 @@ class CloudflareService
|
||||
api_token.present? && zone.present?
|
||||
end
|
||||
|
||||
def ips(expiry: 24.hours)
|
||||
response = Danbooru::Http.cache(expiry).get("https://api.cloudflare.com/client/v4/ips")
|
||||
return [] if response.code != 200
|
||||
|
||||
result = response.parse["result"]
|
||||
ips = result["ipv4_cidrs"] + result["ipv6_cidrs"]
|
||||
ips.map { |ip| IPAddr.new(ip) }
|
||||
end
|
||||
|
||||
def purge_cache(urls)
|
||||
return unless enabled?
|
||||
|
||||
|
||||
@@ -24,15 +24,6 @@ class CurrentUser
|
||||
scoped(user, &block)
|
||||
end
|
||||
|
||||
def self.as_system(&block)
|
||||
if block_given?
|
||||
scoped(::User.system, "127.0.0.1", &block)
|
||||
else
|
||||
self.user = User.system
|
||||
self.ip_addr = "127.0.0.1"
|
||||
end
|
||||
end
|
||||
|
||||
def self.user
|
||||
RequestStore[:current_user]
|
||||
end
|
||||
|
||||
@@ -1,18 +1,43 @@
|
||||
require "danbooru/http/html_adapter"
|
||||
require "danbooru/http/xml_adapter"
|
||||
require "danbooru/http/cache"
|
||||
require "danbooru/http/redirector"
|
||||
require "danbooru/http/retriable"
|
||||
require "danbooru/http/session"
|
||||
|
||||
module Danbooru
|
||||
class Http
|
||||
DEFAULT_TIMEOUT = 3
|
||||
class DownloadError < StandardError; end
|
||||
class FileTooLargeError < StandardError; end
|
||||
|
||||
DEFAULT_TIMEOUT = 10
|
||||
MAX_REDIRECTS = 5
|
||||
|
||||
attr_writer :cache, :http
|
||||
attr_accessor :max_size, :http
|
||||
|
||||
class << self
|
||||
delegate :get, :put, :post, :delete, :cache, :follow, :timeout, :auth, :basic_auth, :headers, to: :new
|
||||
delegate :get, :head, :put, :post, :delete, :cache, :follow, :max_size, :timeout, :auth, :basic_auth, :headers, :cookies, :use, :public_only, :download_media, to: :new
|
||||
end
|
||||
|
||||
def initialize
|
||||
@http ||=
|
||||
::Danbooru::Http::ApplicationClient.new
|
||||
.timeout(DEFAULT_TIMEOUT)
|
||||
.headers("Accept-Encoding" => "gzip")
|
||||
.headers("User-Agent": "#{Danbooru.config.canonical_app_name}/#{Rails.application.config.x.git_hash}")
|
||||
.use(:auto_inflate)
|
||||
.use(redirector: { max_redirects: MAX_REDIRECTS })
|
||||
.use(:session)
|
||||
end
|
||||
|
||||
def get(url, **options)
|
||||
request(:get, url, **options)
|
||||
end
|
||||
|
||||
def head(url, **options)
|
||||
request(:head, url, **options)
|
||||
end
|
||||
|
||||
def put(url, **options)
|
||||
request(:get, url, **options)
|
||||
end
|
||||
@@ -25,14 +50,14 @@ module Danbooru
|
||||
request(:delete, url, **options)
|
||||
end
|
||||
|
||||
def cache(expiry)
|
||||
dup.tap { |o| o.cache = expiry.to_i }
|
||||
end
|
||||
|
||||
def follow(*args)
|
||||
dup.tap { |o| o.http = o.http.follow(*args) }
|
||||
end
|
||||
|
||||
def max_size(size)
|
||||
dup.tap { |o| o.max_size = size }
|
||||
end
|
||||
|
||||
def timeout(*args)
|
||||
dup.tap { |o| o.http = o.http.timeout(*args) }
|
||||
end
|
||||
@@ -49,43 +74,72 @@ module Danbooru
|
||||
dup.tap { |o| o.http = o.http.headers(*args) }
|
||||
end
|
||||
|
||||
def cookies(*args)
|
||||
dup.tap { |o| o.http = o.http.cookies(*args) }
|
||||
end
|
||||
|
||||
def use(*args)
|
||||
dup.tap { |o| o.http = o.http.use(*args) }
|
||||
end
|
||||
|
||||
def cache(expires_in)
|
||||
use(cache: { expires_in: expires_in })
|
||||
end
|
||||
|
||||
# allow requests only to public IPs, not to local or private networks.
|
||||
def public_only
|
||||
dup.tap do |o|
|
||||
o.http = o.http.dup.tap do |http|
|
||||
http.default_options = http.default_options.with_socket_class(ValidatingSocket)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
concerning :DownloadMethods do
|
||||
def download_media(url, no_polish: true, **options)
|
||||
url = Addressable::URI.heuristic_parse(url)
|
||||
response = headers(Referer: url.origin).get(url)
|
||||
|
||||
# prevent Cloudflare Polish from modifying images.
|
||||
if no_polish && response.headers["CF-Polished"].present?
|
||||
url.query_values = url.query_values.to_h.merge(danbooru_no_polish: SecureRandom.uuid)
|
||||
return download_media(url, no_polish: false)
|
||||
end
|
||||
|
||||
file = download_response(response, **options)
|
||||
[response, MediaFile.open(file)]
|
||||
end
|
||||
|
||||
def download_response(response, file: Tempfile.new("danbooru-download-", binmode: true))
|
||||
raise DownloadError, "Downloading #{response.uri} failed with code #{response.status}" if response.status != 200
|
||||
raise FileTooLargeError, response if @max_size && response.content_length.to_i > @max_size
|
||||
|
||||
size = 0
|
||||
response.body.each do |chunk|
|
||||
size += chunk.size
|
||||
raise FileTooLargeError if @max_size && size > @max_size
|
||||
file.write(chunk)
|
||||
end
|
||||
|
||||
file.rewind
|
||||
file
|
||||
end
|
||||
end
|
||||
|
||||
protected
|
||||
|
||||
def request(method, url, **options)
|
||||
if @cache.present?
|
||||
cached_request(method, url, **options)
|
||||
else
|
||||
raw_request(method, url, **options)
|
||||
end
|
||||
rescue HTTP::Redirector::TooManyRedirectsError
|
||||
::HTTP::Response.new(status: 598, body: "", version: "1.1")
|
||||
rescue HTTP::TimeoutError
|
||||
# return a synthetic http error on connection timeouts
|
||||
::HTTP::Response.new(status: 599, body: "", version: "1.1")
|
||||
end
|
||||
|
||||
def cached_request(method, url, **options)
|
||||
key = Cache.hash({ method: method, url: url, headers: http.default_options.headers.to_h, **options }.to_json)
|
||||
|
||||
cached_response = Cache.get(key, @cache) do
|
||||
response = raw_request(method, url, **options)
|
||||
{ status: response.status, body: response.to_s, headers: response.headers.to_h, version: "1.1" }
|
||||
end
|
||||
|
||||
::HTTP::Response.new(**cached_response)
|
||||
end
|
||||
|
||||
def raw_request(method, url, **options)
|
||||
http.send(method, url, **options)
|
||||
rescue ValidatingSocket::ProhibitedIpError
|
||||
fake_response(597, "")
|
||||
rescue HTTP::Redirector::TooManyRedirectsError
|
||||
fake_response(598, "")
|
||||
rescue HTTP::TimeoutError
|
||||
fake_response(599, "")
|
||||
end
|
||||
|
||||
def http
|
||||
@http ||= ::HTTP.
|
||||
follow(strict: false, max_hops: MAX_REDIRECTS).
|
||||
timeout(DEFAULT_TIMEOUT).
|
||||
use(:auto_inflate).
|
||||
headers(Danbooru.config.http_headers).
|
||||
headers("Accept-Encoding" => "gzip")
|
||||
def fake_response(status, body)
|
||||
::HTTP::Response.new(status: status, version: "1.1", body: ::HTTP::Response::Body.new(body))
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
31
app/logical/danbooru/http/application_client.rb
Normal file
31
app/logical/danbooru/http/application_client.rb
Normal file
@@ -0,0 +1,31 @@
|
||||
# An extension to HTTP::Client that lets us write Rack-style middlewares that
|
||||
# hook into the request/response cycle and override how requests are made. This
|
||||
# works by extending http.rb's concept of features (HTTP::Feature) to give them
|
||||
# a `perform` method that takes a http request and returns a http response.
|
||||
# This can be used to intercept and modify requests and return arbitrary responses.
|
||||
|
||||
module Danbooru
|
||||
class Http
|
||||
class ApplicationClient < HTTP::Client
|
||||
# Override `perform` to call the `perform` method on features first.
|
||||
def perform(request, options)
|
||||
features = options.features.values.reverse.select do |feature|
|
||||
feature.respond_to?(:perform)
|
||||
end
|
||||
|
||||
perform = proc { |req| super(req, options) }
|
||||
callback_chain = features.reduce(perform) do |callback_chain, feature|
|
||||
proc { |req| feature.perform(req, &callback_chain) }
|
||||
end
|
||||
|
||||
callback_chain.call(request)
|
||||
end
|
||||
|
||||
# Override `branch` to return an ApplicationClient instead of a
|
||||
# HTTP::Client so that chaining works.
|
||||
def branch(...)
|
||||
ApplicationClient.new(...)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
30
app/logical/danbooru/http/cache.rb
Normal file
30
app/logical/danbooru/http/cache.rb
Normal file
@@ -0,0 +1,30 @@
|
||||
module Danbooru
|
||||
class Http
|
||||
class Cache < HTTP::Feature
|
||||
HTTP::Options.register_feature :cache, self
|
||||
|
||||
attr_reader :expires_in
|
||||
|
||||
def initialize(expires_in:)
|
||||
@expires_in = expires_in
|
||||
end
|
||||
|
||||
def perform(request, &block)
|
||||
::Cache.get(cache_key(request), expires_in) do
|
||||
response = yield request
|
||||
|
||||
# XXX hack to remove connection state from response body so we can serialize it for caching.
|
||||
response.flush
|
||||
response.body.instance_variable_set(:@connection, nil)
|
||||
response.body.instance_variable_set(:@stream, nil)
|
||||
|
||||
response
|
||||
end
|
||||
end
|
||||
|
||||
def cache_key(request)
|
||||
"http:" + ::Cache.hash({ method: request.verb, url: request.uri.to_s, headers: request.headers.sort }.to_json)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
12
app/logical/danbooru/http/html_adapter.rb
Normal file
12
app/logical/danbooru/http/html_adapter.rb
Normal file
@@ -0,0 +1,12 @@
|
||||
module Danbooru
|
||||
class Http
|
||||
class HtmlAdapter < HTTP::MimeType::Adapter
|
||||
HTTP::MimeType.register_adapter "text/html", self
|
||||
HTTP::MimeType.register_alias "text/html", :html
|
||||
|
||||
def decode(str)
|
||||
Nokogiri::HTML5(str)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
40
app/logical/danbooru/http/redirector.rb
Normal file
40
app/logical/danbooru/http/redirector.rb
Normal file
@@ -0,0 +1,40 @@
|
||||
# A HTTP::Feature that automatically follows HTTP redirects.
|
||||
# https://developer.mozilla.org/en-US/docs/Web/HTTP/Redirections
|
||||
|
||||
module Danbooru
|
||||
class Http
|
||||
class Redirector < HTTP::Feature
|
||||
HTTP::Options.register_feature :redirector, self
|
||||
|
||||
attr_reader :max_redirects
|
||||
|
||||
def initialize(max_redirects: 5)
|
||||
@max_redirects = max_redirects
|
||||
end
|
||||
|
||||
def perform(request, &block)
|
||||
response = yield request
|
||||
|
||||
redirects = max_redirects
|
||||
while response.status.redirect?
|
||||
raise HTTP::Redirector::TooManyRedirectsError if redirects <= 0
|
||||
|
||||
response = yield build_redirect(request, response)
|
||||
redirects -= 1
|
||||
end
|
||||
|
||||
response
|
||||
end
|
||||
|
||||
def build_redirect(request, response)
|
||||
location = response.headers["Location"].to_s
|
||||
uri = HTTP::URI.parse(location)
|
||||
|
||||
verb = request.verb
|
||||
verb = :get if response.status == 303 && !request.verb.in?([:get, :head])
|
||||
|
||||
request.redirect(uri, verb)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
54
app/logical/danbooru/http/retriable.rb
Normal file
54
app/logical/danbooru/http/retriable.rb
Normal file
@@ -0,0 +1,54 @@
|
||||
# A HTTP::Feature that automatically retries requests that return a 429 error
|
||||
# or a Retry-After header. Usage: `Danbooru::Http.use(:retriable).get(url)`.
|
||||
#
|
||||
# https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/429
|
||||
# https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
|
||||
|
||||
module Danbooru
|
||||
class Http
|
||||
class Retriable < HTTP::Feature
|
||||
HTTP::Options.register_feature :retriable, self
|
||||
|
||||
attr_reader :max_retries, :max_delay
|
||||
|
||||
def initialize(max_retries: 2, max_delay: 5.seconds)
|
||||
@max_retries = max_retries
|
||||
@max_delay = max_delay
|
||||
end
|
||||
|
||||
def perform(request, &block)
|
||||
response = yield request
|
||||
|
||||
retries = max_retries
|
||||
while retriable?(response) && retries > 0 && retry_delay(response) <= max_delay
|
||||
DanbooruLogger.info "Retrying url=#{request.uri} status=#{response.status} retries=#{retries} delay=#{retry_delay(response)}"
|
||||
|
||||
retries -= 1
|
||||
sleep(retry_delay(response))
|
||||
response = yield request
|
||||
end
|
||||
|
||||
response
|
||||
end
|
||||
|
||||
def retriable?(response)
|
||||
response.status == 429 || response.headers["Retry-After"].present?
|
||||
end
|
||||
|
||||
def retry_delay(response, current_time: Time.zone.now)
|
||||
retry_after = response.headers["Retry-After"]
|
||||
|
||||
if retry_after.blank?
|
||||
0.seconds
|
||||
elsif retry_after =~ /\A\d+\z/
|
||||
retry_after.to_i.seconds
|
||||
else
|
||||
retry_at = Time.zone.parse(retry_after)
|
||||
return 0.seconds if retry_at.blank?
|
||||
|
||||
[retry_at - current_time, 0].max.seconds
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
37
app/logical/danbooru/http/session.rb
Normal file
37
app/logical/danbooru/http/session.rb
Normal file
@@ -0,0 +1,37 @@
|
||||
module Danbooru
|
||||
class Http
|
||||
class Session < HTTP::Feature
|
||||
HTTP::Options.register_feature :session, self
|
||||
|
||||
attr_reader :cookie_jar
|
||||
|
||||
def initialize(cookie_jar: HTTP::CookieJar.new)
|
||||
@cookie_jar = cookie_jar
|
||||
end
|
||||
|
||||
def perform(request)
|
||||
add_cookies(request)
|
||||
response = yield request
|
||||
save_cookies(response)
|
||||
response
|
||||
end
|
||||
|
||||
def add_cookies(request)
|
||||
cookies = cookies_for_request(request)
|
||||
request.headers["Cookie"] = cookies if cookies.present?
|
||||
end
|
||||
|
||||
def cookies_for_request(request)
|
||||
saved_cookies = cookie_jar.each(request.uri).map { |c| [c.name, c.value] }.to_h
|
||||
request_cookies = HTTP::Cookie.cookie_value_to_hash(request.headers["Cookie"].to_s)
|
||||
saved_cookies.merge(request_cookies).map { |name, value| "#{name}=#{value}" }.join("; ")
|
||||
end
|
||||
|
||||
def save_cookies(response)
|
||||
response.cookies.each do |cookie|
|
||||
cookie_jar.add(cookie)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
12
app/logical/danbooru/http/xml_adapter.rb
Normal file
12
app/logical/danbooru/http/xml_adapter.rb
Normal file
@@ -0,0 +1,12 @@
|
||||
module Danbooru
|
||||
class Http
|
||||
class XmlAdapter < HTTP::MimeType::Adapter
|
||||
HTTP::MimeType.register_adapter "application/xml", self
|
||||
HTTP::MimeType.register_alias "application/xml", :xml
|
||||
|
||||
def decode(str)
|
||||
Hash.from_xml(str).with_indifferent_access
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,123 +0,0 @@
|
||||
require 'resolv'
|
||||
|
||||
module Downloads
|
||||
class File
|
||||
include ActiveModel::Validations
|
||||
class Error < StandardError; end
|
||||
|
||||
RETRIABLE_ERRORS = [Errno::ECONNRESET, Errno::ETIMEDOUT, Errno::EIO, Errno::EHOSTUNREACH, Errno::ECONNREFUSED, Timeout::Error, IOError]
|
||||
|
||||
delegate :data, to: :strategy
|
||||
attr_reader :url, :referer
|
||||
|
||||
validate :validate_url
|
||||
|
||||
def initialize(url, referer = nil)
|
||||
@url = Addressable::URI.parse(url) rescue nil
|
||||
@referer = referer
|
||||
validate!
|
||||
end
|
||||
|
||||
def size
|
||||
res = HTTParty.head(uncached_url, **httparty_options, timeout: 3)
|
||||
|
||||
if res.success?
|
||||
res.content_length
|
||||
else
|
||||
raise HTTParty::ResponseError.new(res)
|
||||
end
|
||||
end
|
||||
|
||||
def download!(url: uncached_url, tries: 3, **options)
|
||||
Retriable.retriable(on: RETRIABLE_ERRORS, tries: tries, base_interval: 0) do
|
||||
file = http_get_streaming(url, headers: strategy.headers, **options)
|
||||
return [file, strategy]
|
||||
end
|
||||
end
|
||||
|
||||
def validate_url
|
||||
errors[:base] << "URL must not be blank" if url.blank?
|
||||
errors[:base] << "'#{url}' is not a valid url" if !url.host.present?
|
||||
errors[:base] << "'#{url}' is not a valid url. Did you mean 'http://#{url}'?" if !url.scheme.in?(%w[http https])
|
||||
end
|
||||
|
||||
def http_get_streaming(url, file: Tempfile.new(binmode: true), headers: {}, max_size: Danbooru.config.max_file_size)
|
||||
size = 0
|
||||
|
||||
res = HTTParty.get(url, httparty_options) do |chunk|
|
||||
next if chunk.code == 302
|
||||
|
||||
size += chunk.size
|
||||
raise Error.new("File is too large (max size: #{max_size})") if size > max_size && max_size > 0
|
||||
|
||||
file.write(chunk)
|
||||
end
|
||||
|
||||
if res.success?
|
||||
file.rewind
|
||||
return file
|
||||
else
|
||||
raise Error.new("HTTP error code: #{res.code} #{res.message}")
|
||||
end
|
||||
end
|
||||
|
||||
# Prevent Cloudflare from potentially mangling the image. See issue #3528.
|
||||
def uncached_url
|
||||
return file_url unless is_cloudflare?(file_url)
|
||||
|
||||
url = file_url.dup
|
||||
url.query_values = url.query_values.to_h.merge(danbooru_no_cache: SecureRandom.uuid)
|
||||
url
|
||||
end
|
||||
|
||||
def preview_url
|
||||
@preview_url ||= Addressable::URI.parse(strategy.preview_url)
|
||||
end
|
||||
|
||||
def file_url
|
||||
@file_url ||= Addressable::URI.parse(strategy.image_url)
|
||||
end
|
||||
|
||||
def strategy
|
||||
@strategy ||= Sources::Strategies.find(url.to_s, referer)
|
||||
end
|
||||
|
||||
def httparty_options
|
||||
{
|
||||
timeout: 10,
|
||||
stream_body: true,
|
||||
headers: strategy.headers,
|
||||
connection_adapter: ValidatingConnectionAdapter
|
||||
}.deep_merge(Danbooru.config.httparty_options)
|
||||
end
|
||||
|
||||
def is_cloudflare?(url)
|
||||
ip_addr = IPAddr.new(Resolv.getaddress(url.hostname))
|
||||
CloudflareService.new.ips.any? { |subnet| subnet.include?(ip_addr) }
|
||||
end
|
||||
|
||||
def self.banned_ip?(ip)
|
||||
ip = IPAddress.parse(ip.to_s) unless ip.is_a?(IPAddress)
|
||||
|
||||
if ip.ipv4?
|
||||
ip.loopback? || ip.link_local? || ip.multicast? || ip.private?
|
||||
elsif ip.ipv6?
|
||||
ip.loopback? || ip.link_local? || ip.unique_local? || ip.unspecified?
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# Hook into HTTParty to validate the IP before following redirects.
|
||||
# https://www.rubydoc.info/github/jnunemaker/httparty/HTTParty/ConnectionAdapter
|
||||
class ValidatingConnectionAdapter < HTTParty::ConnectionAdapter
|
||||
def self.call(uri, options)
|
||||
ip_addr = IPAddress.parse(::Resolv.getaddress(uri.hostname))
|
||||
|
||||
if Downloads::File.banned_ip?(ip_addr)
|
||||
raise Downloads::File::Error, "Downloads from #{ip_addr} are not allowed"
|
||||
end
|
||||
|
||||
super(uri, options)
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,4 +1,6 @@
|
||||
class ImageProxy
|
||||
class Error < StandardError; end
|
||||
|
||||
def self.needs_proxy?(url)
|
||||
fake_referer_for(url).present?
|
||||
end
|
||||
@@ -8,16 +10,13 @@ class ImageProxy
|
||||
end
|
||||
|
||||
def self.get_image(url)
|
||||
if url.blank?
|
||||
raise "Must specify url"
|
||||
end
|
||||
raise Error, "URL not present" unless url.present?
|
||||
raise Error, "Proxy not allowed for this url (url=#{url})" unless needs_proxy?(url)
|
||||
|
||||
if !needs_proxy?(url)
|
||||
raise "Proxy not allowed for this site"
|
||||
end
|
||||
referer = fake_referer_for(url)
|
||||
response = Danbooru::Http.headers(Referer: referer).get(url)
|
||||
raise Error, "Couldn't proxy image (code=#{response.status}, url=#{url})" unless response.status.success?
|
||||
|
||||
response = HTTParty.get(url, Danbooru.config.httparty_options.deep_merge(headers: {"Referer" => fake_referer_for(url)}))
|
||||
raise "HTTP error code: #{response.code} #{response.message}" unless response.success?
|
||||
response
|
||||
end
|
||||
end
|
||||
|
||||
@@ -12,8 +12,9 @@ class IqdbProxy
|
||||
end
|
||||
|
||||
def download(url, type)
|
||||
download = Downloads::File.new(url)
|
||||
file, strategy = download.download!(url: download.send(type))
|
||||
strategy = Sources::Strategies.find(url)
|
||||
download_url = strategy.send(type)
|
||||
file = strategy.download_file!(download_url)
|
||||
file
|
||||
end
|
||||
|
||||
@@ -32,7 +33,7 @@ class IqdbProxy
|
||||
file = download(params[:image_url], :url)
|
||||
results = query(file: file, limit: limit)
|
||||
elsif params[:file_url].present?
|
||||
file = download(params[:file_url], :file_url)
|
||||
file = download(params[:file_url], :image_url)
|
||||
results = query(file: file, limit: limit)
|
||||
elsif params[:post_id].present?
|
||||
url = Post.find(params[:post_id]).preview_file_url
|
||||
@@ -50,9 +51,12 @@ class IqdbProxy
|
||||
file.try(:close)
|
||||
end
|
||||
|
||||
def query(params)
|
||||
def query(file: nil, url: nil, limit: 20)
|
||||
raise NotImplementedError, "the IQDBs service isn't configured" unless enabled?
|
||||
response = http.post("#{iqdbs_server}/similar", body: params)
|
||||
|
||||
file = HTTP::FormData::File.new(file) if file
|
||||
form = { file: file, url: url, limit: limit }.compact
|
||||
response = http.timeout(30).post("#{iqdbs_server}/similar", form: form)
|
||||
|
||||
raise Error, "IQDB error: #{response.status}" if response.status != 200
|
||||
raise Error, "IQDB error: #{response.parse["error"]}" if response.parse.is_a?(Hash)
|
||||
|
||||
@@ -43,6 +43,8 @@ class MediaFile
|
||||
else
|
||||
:bin
|
||||
end
|
||||
rescue EOFError
|
||||
:bin
|
||||
end
|
||||
|
||||
def self.videos_enabled?
|
||||
|
||||
@@ -4,8 +4,7 @@ class NicoSeigaApiClient
|
||||
|
||||
attr_reader :http
|
||||
|
||||
# XXX temp disable following redirects.
|
||||
def initialize(work_id:, type:, http: Danbooru::Http.follow(nil))
|
||||
def initialize(work_id:, type:, http: Danbooru::Http.new)
|
||||
@work_id = work_id
|
||||
@work_type = type
|
||||
@http = http
|
||||
@@ -80,28 +79,19 @@ class NicoSeigaApiClient
|
||||
end
|
||||
|
||||
def get(url)
|
||||
cookie_header = Cache.get("nicoseiga-cookie-header") || regenerate_cookie_header
|
||||
|
||||
resp = http.headers({Cookie: cookie_header}).cache(1.minute).get(url)
|
||||
|
||||
if resp.headers["Location"] =~ %r{seiga\.nicovideo\.jp/login/}i
|
||||
cookie_header = regenerate_cookie_header
|
||||
resp = http.headers({Cookie: cookie_header}).cache(1.minute).get(url)
|
||||
end
|
||||
|
||||
resp
|
||||
end
|
||||
|
||||
def regenerate_cookie_header
|
||||
form = {
|
||||
mail_tel: Danbooru.config.nico_seiga_login,
|
||||
password: Danbooru.config.nico_seiga_password
|
||||
}
|
||||
resp = http.post("https://account.nicovideo.jp/api/v1/login", form: form)
|
||||
cookies = resp.cookies.map { |c| c.name + "=" + c.value }
|
||||
cookies << "accept_fetish_warning=2"
|
||||
|
||||
Cache.put("nicoseiga-cookie-header", cookies.join(";"), 1.week)
|
||||
# XXX should fail gracefully instead of raising exception
|
||||
resp = http.cache(1.hour).post("https://account.nicovideo.jp/login/redirector?site=seiga", form: form)
|
||||
raise RuntimeError, "NicoSeiga login failed (status=#{resp.status} url=#{url})" if resp.status != 200
|
||||
|
||||
resp = http.cache(1.minute).get(url)
|
||||
#raise RuntimeError, "NicoSeiga get failed (status=#{resp.status} url=#{url})" if resp.status != 200
|
||||
|
||||
resp
|
||||
end
|
||||
|
||||
memoize :api_response, :manga_api_response, :user_api_response
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
require 'resolv-replace'
|
||||
|
||||
class PixivApiClient
|
||||
extend Memoist
|
||||
|
||||
@@ -8,6 +6,21 @@ class PixivApiClient
|
||||
CLIENT_SECRET = "HP3RmkgAmEGro0gn1x9ioawQE8WMfvLXDz3ZqxpK"
|
||||
CLIENT_HASH_SALT = "28c1fdd170a5204386cb1313c7077b34f83e4aaf4aa829ce78c231e05b0bae2c"
|
||||
|
||||
# Tools to not include in the tags list. We don't tag digital media, so
|
||||
# including these results in bad translated tags suggestions.
|
||||
TOOLS_BLACKLIST = %w[
|
||||
Photoshop Illustrator Fireworks Flash Painter PaintShopPro pixiv\ Sketch
|
||||
CLIP\ STUDIO\ PAINT IllustStudio ComicStudio RETAS\ STUDIO SAI PhotoStudio
|
||||
Pixia NekoPaint PictBear openCanvas ArtRage Expression Inkscape GIMP
|
||||
CGillust COMICWORKS MS_Paint EDGE AzPainter AzPainter2 AzDrawing
|
||||
PicturePublisher SketchBookPro Processing 4thPaint GraphicsGale mdiapp
|
||||
Paintgraphic AfterEffects drawr CLIP\ PAINT\ Lab FireAlpaca Pixelmator
|
||||
AzDrawing2 MediBang\ Paint Krita ibisPaint Procreate Live2D
|
||||
Lightwave3D Shade Poser STRATA AnimationMaster XSI CARRARA CINEMA4D Maya
|
||||
3dsMax Blender ZBrush Metasequoia Sunny3D Bryce Vue Hexagon\ King SketchUp
|
||||
VistaPro Sculptris Comi\ Po! modo DAZ\ Studio 3D-Coat
|
||||
]
|
||||
|
||||
class Error < StandardError; end
|
||||
class BadIDError < Error; end
|
||||
|
||||
@@ -24,7 +37,7 @@ class PixivApiClient
|
||||
@artist_commentary_title = json["title"].to_s
|
||||
@artist_commentary_desc = json["caption"].to_s
|
||||
@tags = json["tags"].reject {|x| x =~ /^http:/}
|
||||
@tags += json["tools"]
|
||||
@tags += json["tools"] - TOOLS_BLACKLIST
|
||||
|
||||
if json["metadata"]
|
||||
if json["metadata"]["zip_urls"]
|
||||
@@ -99,66 +112,13 @@ class PixivApiClient
|
||||
end
|
||||
end
|
||||
|
||||
class FanboxResponse
|
||||
attr_reader :json
|
||||
|
||||
def initialize(json)
|
||||
@json = json
|
||||
end
|
||||
|
||||
def name
|
||||
json["body"]["user"]["name"]
|
||||
end
|
||||
|
||||
def user_id
|
||||
json["body"]["user"]["userId"]
|
||||
end
|
||||
|
||||
def moniker
|
||||
""
|
||||
end
|
||||
|
||||
def page_count
|
||||
json["body"]["body"]["images"].size
|
||||
end
|
||||
|
||||
def artist_commentary_title
|
||||
json["body"]["title"]
|
||||
end
|
||||
|
||||
def artist_commentary_desc
|
||||
json["body"]["body"]["text"]
|
||||
end
|
||||
|
||||
def tags
|
||||
[]
|
||||
end
|
||||
|
||||
def pages
|
||||
if json["body"]["body"]
|
||||
json["body"]["body"]["images"].map {|x| x["originalUrl"]}
|
||||
else
|
||||
[]
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def work(illust_id)
|
||||
headers = Danbooru.config.http_headers.merge(
|
||||
"Referer" => "http://www.pixiv.net",
|
||||
"Content-Type" => "application/x-www-form-urlencoded",
|
||||
"Authorization" => "Bearer #{access_token}"
|
||||
)
|
||||
params = {
|
||||
"image_sizes" => "large",
|
||||
"include_stats" => "true"
|
||||
}
|
||||
|
||||
params = { image_sizes: "large", include_stats: "true" }
|
||||
url = "https://public-api.secure.pixiv.net/v#{API_VERSION}/works/#{illust_id.to_i}.json"
|
||||
response = Danbooru::Http.cache(1.minute).headers(headers).get(url, params: params)
|
||||
response = api_client.cache(1.minute).get(url, params: params)
|
||||
json = response.parse
|
||||
|
||||
if response.code == 200
|
||||
if response.status == 200
|
||||
WorkResponse.new(json["response"][0])
|
||||
elsif json["status"] == "failure" && json.dig("errors", "system", "message") =~ /対象のイラストは見つかりませんでした。/
|
||||
raise BadIDError.new("Pixiv ##{illust_id} not found: work was deleted, made private, or ID is invalid.")
|
||||
@@ -169,32 +129,12 @@ class PixivApiClient
|
||||
raise Error.new("Pixiv API call failed (status=#{response.code} body=#{response.body})")
|
||||
end
|
||||
|
||||
def fanbox(fanbox_id)
|
||||
url = "https://www.pixiv.net/ajax/fanbox/post?postId=#{fanbox_id.to_i}"
|
||||
resp = agent.get(url)
|
||||
json = JSON.parse(resp.body)
|
||||
if resp.code == "200"
|
||||
FanboxResponse.new(json)
|
||||
elsif json["status"] == "failure"
|
||||
raise Error.new("Pixiv API call failed (status=#{resp.code} body=#{body})")
|
||||
end
|
||||
rescue JSON::ParserError
|
||||
raise Error.new("Pixiv API call failed (status=#{resp.code} body=#{body})")
|
||||
end
|
||||
|
||||
def novel(novel_id)
|
||||
headers = Danbooru.config.http_headers.merge(
|
||||
"Referer" => "http://www.pixiv.net",
|
||||
"Content-Type" => "application/x-www-form-urlencoded",
|
||||
"Authorization" => "Bearer #{access_token}"
|
||||
)
|
||||
|
||||
url = "https://public-api.secure.pixiv.net/v#{API_VERSION}/novels/#{novel_id.to_i}.json"
|
||||
resp = HTTParty.get(url, Danbooru.config.httparty_options.deep_merge(headers: headers))
|
||||
body = resp.body.force_encoding("utf-8")
|
||||
json = JSON.parse(body)
|
||||
resp = api_client.cache(1.minute).get(url)
|
||||
json = resp.parse
|
||||
|
||||
if resp.success?
|
||||
if resp.status == 200
|
||||
NovelResponse.new(json["response"][0])
|
||||
elsif json["status"] == "failure" && json.dig("errors", "system", "message") =~ /対象のイラストは見つかりませんでした。/
|
||||
raise Error.new("Pixiv API call failed (status=#{resp.code} body=#{body})")
|
||||
@@ -204,42 +144,41 @@ class PixivApiClient
|
||||
end
|
||||
|
||||
def access_token
|
||||
Cache.get("pixiv-papi-access-token", 3000) do
|
||||
access_token = nil
|
||||
# truncate timestamp to 1-hour resolution so that it doesn't break caching.
|
||||
client_time = Time.zone.now.utc.change(min: 0).rfc3339
|
||||
client_hash = Digest::MD5.hexdigest(client_time + CLIENT_HASH_SALT)
|
||||
|
||||
client_time = Time.now.rfc3339
|
||||
client_hash = Digest::MD5.hexdigest(client_time + CLIENT_HASH_SALT)
|
||||
headers = {
|
||||
"Referer": "http://www.pixiv.net",
|
||||
"X-Client-Time": client_time,
|
||||
"X-Client-Hash": client_hash
|
||||
}
|
||||
|
||||
headers = {
|
||||
"Referer": "http://www.pixiv.net",
|
||||
"X-Client-Time": client_time,
|
||||
"X-Client-Hash": client_hash
|
||||
}
|
||||
params = {
|
||||
username: Danbooru.config.pixiv_login,
|
||||
password: Danbooru.config.pixiv_password,
|
||||
grant_type: "password",
|
||||
client_id: CLIENT_ID,
|
||||
client_secret: CLIENT_SECRET
|
||||
}
|
||||
url = "https://oauth.secure.pixiv.net/auth/token"
|
||||
params = {
|
||||
username: Danbooru.config.pixiv_login,
|
||||
password: Danbooru.config.pixiv_password,
|
||||
grant_type: "password",
|
||||
client_id: CLIENT_ID,
|
||||
client_secret: CLIENT_SECRET
|
||||
}
|
||||
|
||||
resp = HTTParty.post(url, Danbooru.config.httparty_options.deep_merge(body: params, headers: headers))
|
||||
body = resp.body.force_encoding("utf-8")
|
||||
resp = http.headers(headers).cache(1.hour).post("https://oauth.secure.pixiv.net/auth/token", form: params)
|
||||
return nil unless resp.status == 200
|
||||
|
||||
if resp.success?
|
||||
json = JSON.parse(body)
|
||||
access_token = json["response"]["access_token"]
|
||||
else
|
||||
raise Error.new("Pixiv API access token call failed (status=#{resp.code} body=#{body})")
|
||||
end
|
||||
|
||||
access_token
|
||||
end
|
||||
resp.parse.dig("response", "access_token")
|
||||
end
|
||||
|
||||
def agent
|
||||
PixivWebAgent.build
|
||||
def api_client
|
||||
http.headers(
|
||||
"Referer": "http://www.pixiv.net",
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
"Authorization": "Bearer #{access_token}"
|
||||
)
|
||||
end
|
||||
memoize :agent
|
||||
|
||||
def http
|
||||
Danbooru::Http.new
|
||||
end
|
||||
|
||||
memoize :access_token, :api_client, :http
|
||||
end
|
||||
|
||||
@@ -1,74 +0,0 @@
|
||||
class PixivWebAgent
|
||||
SESSION_CACHE_KEY = "pixiv-phpsessid"
|
||||
COMIC_SESSION_CACHE_KEY = "pixiv-comicsessid"
|
||||
SESSION_COOKIE_KEY = "PHPSESSID"
|
||||
COMIC_SESSION_COOKIE_KEY = "_pixiv-comic_session"
|
||||
|
||||
def self.phpsessid(agent)
|
||||
agent.cookies.select { |cookie| cookie.name == SESSION_COOKIE_KEY }.first.try(:value)
|
||||
end
|
||||
|
||||
def self.build
|
||||
mech = Mechanize.new
|
||||
mech.keep_alive = false
|
||||
|
||||
phpsessid = Cache.get(SESSION_CACHE_KEY)
|
||||
comicsessid = Cache.get(COMIC_SESSION_CACHE_KEY)
|
||||
|
||||
if phpsessid
|
||||
cookie = Mechanize::Cookie.new(SESSION_COOKIE_KEY, phpsessid)
|
||||
cookie.domain = ".pixiv.net"
|
||||
cookie.path = "/"
|
||||
mech.cookie_jar.add(cookie)
|
||||
|
||||
if comicsessid
|
||||
cookie = Mechanize::Cookie.new(COMIC_SESSION_COOKIE_KEY, comicsessid)
|
||||
cookie.domain = ".pixiv.net"
|
||||
cookie.path = "/"
|
||||
mech.cookie_jar.add(cookie)
|
||||
end
|
||||
else
|
||||
headers = {
|
||||
"Origin" => "https://accounts.pixiv.net",
|
||||
"Referer" => "https://accounts.pixiv.net/login?lang=en^source=pc&view_type=page&ref=wwwtop_accounts_index"
|
||||
}
|
||||
|
||||
params = {
|
||||
pixiv_id: Danbooru.config.pixiv_login,
|
||||
password: Danbooru.config.pixiv_password,
|
||||
captcha: nil,
|
||||
g_captcha_response: nil,
|
||||
source: "pc",
|
||||
post_key: nil
|
||||
}
|
||||
|
||||
mech.get("https://accounts.pixiv.net/login?lang=en&source=pc&view_type=page&ref=wwwtop_accounts_index") do |page|
|
||||
json = page.search("input#init-config").first.attr("value")
|
||||
if json =~ /pixivAccount\.postKey":"([a-f0-9]+)/
|
||||
params[:post_key] = $1
|
||||
end
|
||||
end
|
||||
|
||||
mech.post("https://accounts.pixiv.net/api/login?lang=en", params, headers)
|
||||
if mech.current_page.body =~ /"error":false/
|
||||
cookie = mech.cookies.select {|x| x.name == SESSION_COOKIE_KEY}.first
|
||||
if cookie
|
||||
Cache.put(SESSION_CACHE_KEY, cookie.value, 1.week)
|
||||
end
|
||||
end
|
||||
|
||||
begin
|
||||
mech.get("https://comic.pixiv.net") do
|
||||
cookie = mech.cookies.select {|x| x.name == COMIC_SESSION_COOKIE_KEY}.first
|
||||
if cookie
|
||||
Cache.put(COMIC_SESSION_CACHE_KEY, cookie.value, 1.week)
|
||||
end
|
||||
end
|
||||
rescue Net::HTTPServiceUnavailable
|
||||
# ignore
|
||||
end
|
||||
end
|
||||
|
||||
mech
|
||||
end
|
||||
end
|
||||
@@ -307,6 +307,8 @@ class PostQueryBuilder
|
||||
Post.where(parent: nil)
|
||||
when "any"
|
||||
Post.where.not(parent: nil)
|
||||
when /pending|flagged|modqueue|deleted|banned|active|unmoderated/
|
||||
Post.where.not(parent: nil).where(parent: status_matches(parent))
|
||||
when /\A\d+\z/
|
||||
Post.where(id: parent).or(Post.where(parent: parent))
|
||||
else
|
||||
@@ -320,6 +322,8 @@ class PostQueryBuilder
|
||||
Post.where(has_children: false)
|
||||
when "any"
|
||||
Post.where(has_children: true)
|
||||
when /pending|flagged|modqueue|deleted|banned|active|unmoderated/
|
||||
Post.where(has_children: true).where(children: status_matches(child))
|
||||
else
|
||||
Post.none
|
||||
end
|
||||
|
||||
@@ -20,29 +20,30 @@ class ReportbooruService
|
||||
body.lines.map(&:split).map { [_1, _2.to_i] }
|
||||
end
|
||||
|
||||
def post_search_rankings(date = Date.today, expires_in: 1.minutes)
|
||||
return [] unless enabled?
|
||||
|
||||
response = http.cache(expires_in).get("#{reportbooru_server}/post_searches/rank?date=#{date}")
|
||||
return [] if response.status != 200
|
||||
JSON.parse(response.to_s.force_encoding("utf-8"))
|
||||
def post_search_rankings(date, expires_in: 1.minutes)
|
||||
request("#{reportbooru_server}/post_searches/rank?date=#{date}", expires_in)
|
||||
end
|
||||
|
||||
def post_view_rankings(date = Date.today, expires_in: 1.minutes)
|
||||
return [] unless enabled?
|
||||
|
||||
response = http.get("#{reportbooru_server}/post_views/rank?date=#{date}")
|
||||
return [] if response.status != 200
|
||||
JSON.parse(response.to_s.force_encoding("utf-8"))
|
||||
def post_view_rankings(date, expires_in: 1.minutes)
|
||||
request("#{reportbooru_server}/post_views/rank?date=#{date}", expires_in)
|
||||
end
|
||||
|
||||
def popular_searches(date = Date.today, limit: 100)
|
||||
def popular_searches(date, limit: 100)
|
||||
ranking = post_search_rankings(date)
|
||||
ranking.take(limit).map(&:first)
|
||||
end
|
||||
|
||||
def popular_posts(date = Date.today, limit: 100)
|
||||
def popular_posts(date, limit: 100)
|
||||
ranking = post_view_rankings(date)
|
||||
ranking = post_view_rankings(date.yesterday) if ranking.blank?
|
||||
ranking.take(limit).map { |x| Post.find(x[0]) }
|
||||
end
|
||||
|
||||
def request(url, expires_in)
|
||||
return [] unless enabled?
|
||||
|
||||
response = http.cache(expires_in).get(url)
|
||||
return [] if response.status != 200
|
||||
JSON.parse(response.to_s.force_encoding("utf-8"))
|
||||
end
|
||||
end
|
||||
|
||||
@@ -147,7 +147,7 @@ module Sources::Strategies
|
||||
urls = urls.reverse
|
||||
end
|
||||
|
||||
chosen_url = urls.find { |url| http_exists?(url, headers) }
|
||||
chosen_url = urls.find { |url| http_exists?(url) }
|
||||
chosen_url || url
|
||||
end
|
||||
end
|
||||
|
||||
@@ -14,6 +14,8 @@
|
||||
module Sources
|
||||
module Strategies
|
||||
class Base
|
||||
class DownloadError < StandardError; end
|
||||
|
||||
attr_reader :url, :referer_url, :urls, :parsed_url, :parsed_referer, :parsed_urls
|
||||
|
||||
extend Memoist
|
||||
@@ -35,9 +37,9 @@ module Sources
|
||||
# <tt>referrer_url</tt> so the strategy can discover the HTML
|
||||
# page and other information.
|
||||
def initialize(url, referer_url = nil)
|
||||
@url = url
|
||||
@referer_url = referer_url
|
||||
@urls = [url, referer_url].select(&:present?)
|
||||
@url = url.to_s
|
||||
@referer_url = referer_url&.to_s
|
||||
@urls = [@url, @referer_url].select(&:present?)
|
||||
|
||||
@parsed_url = Addressable::URI.heuristic_parse(url) rescue nil
|
||||
@parsed_referer = Addressable::URI.heuristic_parse(referer_url) rescue nil
|
||||
@@ -139,15 +141,28 @@ module Sources
|
||||
# Subclasses should merge in any required headers needed to access resources
|
||||
# on the site.
|
||||
def headers
|
||||
Danbooru.config.http_headers
|
||||
{}
|
||||
end
|
||||
|
||||
# Returns the size of the image resource without actually downloading the file.
|
||||
def size
|
||||
Downloads::File.new(image_url).size
|
||||
http.head(image_url).content_length.to_i
|
||||
end
|
||||
memoize :size
|
||||
|
||||
# Download the file at the given url, or at the main image url by default.
|
||||
def download_file!(download_url = image_url)
|
||||
raise DownloadError, "Download failed: couldn't find download url for #{url}" if download_url.blank?
|
||||
response, file = http.download_media(download_url)
|
||||
raise DownloadError, "Download failed: #{download_url} returned error #{response.status}" if response.status != 200
|
||||
file
|
||||
end
|
||||
|
||||
def http
|
||||
Danbooru::Http.public_only.timeout(30).max_size(Danbooru.config.max_file_size)
|
||||
end
|
||||
memoize :http
|
||||
|
||||
# The url to use for artist finding purposes. This will be stored in the
|
||||
# artist entry. Normally this will be the profile url.
|
||||
def normalize_for_artist_finder
|
||||
@@ -274,9 +289,8 @@ module Sources
|
||||
to_h.to_json
|
||||
end
|
||||
|
||||
def http_exists?(url, headers)
|
||||
res = HTTParty.head(url, Danbooru.config.httparty_options.deep_merge(headers: headers))
|
||||
res.success?
|
||||
def http_exists?(url, headers = {})
|
||||
http.headers(headers).head(url).status.success?
|
||||
end
|
||||
|
||||
# Convert commentary to dtext by stripping html tags. Sites can override
|
||||
|
||||
@@ -64,11 +64,10 @@ module Sources
|
||||
def page
|
||||
return nil if page_url.blank?
|
||||
|
||||
doc = Cache.get("hentai-foundry:#{page_url}", 1.minute) do
|
||||
HTTParty.get("#{page_url}?enterAgree=1").body
|
||||
end
|
||||
response = Danbooru::Http.new.cache(1.minute).get("#{page_url}?enterAgree=1")
|
||||
return nil unless response.status == 200
|
||||
|
||||
Nokogiri::HTML(doc)
|
||||
response.parse
|
||||
end
|
||||
|
||||
def tags
|
||||
|
||||
@@ -73,8 +73,7 @@ module Sources
|
||||
end
|
||||
|
||||
def image_url
|
||||
return if image_urls.blank?
|
||||
return url if api_client.blank?
|
||||
return url if image_urls.blank? || api_client.blank?
|
||||
|
||||
img = case url
|
||||
when DIRECT || CDN_DIRECT then "https://seiga.nicovideo.jp/image/source/#{image_id_from_url(url)}"
|
||||
@@ -83,7 +82,7 @@ module Sources
|
||||
end
|
||||
|
||||
resp = api_client.get(img)
|
||||
if resp.headers["Location"] =~ %r{https?://.+/(\w+/\d+/\d+)\z}i
|
||||
if resp.uri.to_s =~ %r{https?://.+/(\w+/\d+/\d+)\z}i
|
||||
"https://lohas.nicoseiga.jp/priv/#{$1}"
|
||||
else
|
||||
img
|
||||
@@ -181,12 +180,12 @@ module Sources
|
||||
|
||||
def api_client
|
||||
if illust_id.present?
|
||||
NicoSeigaApiClient.new(work_id: illust_id, type: "illust")
|
||||
NicoSeigaApiClient.new(work_id: illust_id, type: "illust", http: http)
|
||||
elsif manga_id.present?
|
||||
NicoSeigaApiClient.new(work_id: manga_id, type: "manga")
|
||||
NicoSeigaApiClient.new(work_id: manga_id, type: "manga", http: http)
|
||||
elsif image_id.present?
|
||||
# We default to illust to attempt getting the api anyway
|
||||
NicoSeigaApiClient.new(work_id: image_id, type: "illust")
|
||||
NicoSeigaApiClient.new(work_id: image_id, type: "illust", http: http)
|
||||
end
|
||||
end
|
||||
memoize :api_client
|
||||
|
||||
@@ -178,54 +178,21 @@ module Sources
|
||||
def page
|
||||
return nil if page_url.blank?
|
||||
|
||||
doc = agent.get(page_url)
|
||||
http = Danbooru::Http.new
|
||||
form = { email: Danbooru.config.nijie_login, password: Danbooru.config.nijie_password }
|
||||
|
||||
if doc.search("div#header-login-container").any?
|
||||
# Session cache is invalid, clear it and log in normally.
|
||||
Cache.delete("nijie-session")
|
||||
doc = agent.get(page_url)
|
||||
end
|
||||
# XXX `retriable` must come after `cache` so that retries don't return cached error responses.
|
||||
response = http.cache(1.hour).use(retriable: { max_retries: 20 }).post("https://nijie.info/login_int.php", form: form)
|
||||
DanbooruLogger.info "Nijie login failed (#{url}, #{response.status})" if response.status != 200
|
||||
return nil unless response.status == 200
|
||||
|
||||
doc
|
||||
rescue Mechanize::ResponseCodeError => e
|
||||
return nil if e.response_code.to_i == 404
|
||||
raise
|
||||
response = http.cookies(R18: 1).cache(1.minute).get(page_url)
|
||||
return nil unless response.status == 200
|
||||
|
||||
response&.parse
|
||||
end
|
||||
|
||||
memoize :page
|
||||
|
||||
def agent
|
||||
mech = Mechanize.new
|
||||
|
||||
session = Cache.get("nijie-session")
|
||||
if session
|
||||
cookie = Mechanize::Cookie.new("NIJIEIJIEID", session)
|
||||
cookie.domain = ".nijie.info"
|
||||
cookie.path = "/"
|
||||
mech.cookie_jar.add(cookie)
|
||||
else
|
||||
mech.get("https://nijie.info/login.php") do |page|
|
||||
page.form_with(:action => "/login_int.php") do |form|
|
||||
form['email'] = Danbooru.config.nijie_login
|
||||
form['password'] = Danbooru.config.nijie_password
|
||||
end.click_button
|
||||
end
|
||||
session = mech.cookie_jar.cookies.select {|c| c.name == "NIJIEIJIEID"}.first
|
||||
Cache.put("nijie-session", session.value, 1.day) if session
|
||||
end
|
||||
|
||||
# This cookie needs to be set to allow viewing of adult works while anonymous
|
||||
cookie = Mechanize::Cookie.new("R18", "1")
|
||||
cookie.domain = ".nijie.info"
|
||||
cookie.path = "/"
|
||||
mech.cookie_jar.add(cookie)
|
||||
|
||||
mech
|
||||
rescue Mechanize::ResponseCodeError => e
|
||||
raise unless e.response_code.to_i == 429
|
||||
sleep(5)
|
||||
retry
|
||||
end
|
||||
memoize :agent
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@@ -47,7 +47,7 @@ module Sources
|
||||
when %r{\Ahttps?://c(?:s|han|[1-4])\.sankakucomplex\.com/data(?:/sample)?/(?:[a-f0-9]{2}/){2}(?:sample-|preview)?([a-f0-9]{32})}i
|
||||
"https://chan.sankakucomplex.com/en/post/show?md5=#{$1}"
|
||||
|
||||
when %r{\Ahttps?://(?:www|s(?:tatic|[1-4]))\.zerochan\.net/.+(?:\.|\/)(\d+)(?:\.(?:jpe?g?))?\z}i
|
||||
when %r{\Ahttps?://(?:www|s(?:tatic|[1-4]))\.zerochan\.net/.+(?:\.|\/)(\d+)(?:\.(?:jpe?g?|png))?\z}i
|
||||
"https://www.zerochan.net/#{$1}#full"
|
||||
|
||||
when %r{\Ahttps?://static[1-6]?\.minitokyo\.net/(?:downloads|view)/(?:\d{2}/){2}(\d+)}i
|
||||
|
||||
@@ -64,9 +64,6 @@ module Sources
|
||||
ORIG_IMAGE = %r{#{PXIMG}/img-original/img/#{DATE}/(?<illust_id>\d+)_p(?<page>\d+)\.#{EXT}\z}i
|
||||
STACC_PAGE = %r{\A#{WEB}/stacc/#{MONIKER}/?\z}i
|
||||
NOVEL_PAGE = %r{(?:\Ahttps?://www\.pixiv\.net/novel/show\.php\?id=(\d+))}
|
||||
FANBOX_ACCOUNT = %r{(?:\Ahttps?://www\.pixiv\.net/fanbox/creator/\d+\z)}
|
||||
FANBOX_IMAGE = %r{(?:\Ahttps?://fanbox\.pixiv\.net/images/post/(\d+))}
|
||||
FANBOX_PAGE = %r{(?:\Ahttps?://www\.pixiv\.net/fanbox/creator/\d+/post/(\d+))}
|
||||
|
||||
def self.to_dtext(text)
|
||||
if text.nil?
|
||||
@@ -127,14 +124,6 @@ module Sources
|
||||
return "https://www.pixiv.net/novel/show.php?id=#{novel_id}&mode=cover"
|
||||
end
|
||||
|
||||
if fanbox_id.present?
|
||||
return "https://www.pixiv.net/fanbox/creator/#{metadata.user_id}/post/#{fanbox_id}"
|
||||
end
|
||||
|
||||
if fanbox_account_id.present?
|
||||
return "https://www.pixiv.net/fanbox/creator/#{fanbox_account_id}"
|
||||
end
|
||||
|
||||
if illust_id.present?
|
||||
return "https://www.pixiv.net/artworks/#{illust_id}"
|
||||
end
|
||||
@@ -192,17 +181,7 @@ module Sources
|
||||
end
|
||||
|
||||
def headers
|
||||
if fanbox_id.present?
|
||||
# need the session to download fanbox images
|
||||
return {
|
||||
"Referer" => "https://www.pixiv.net/fanbox",
|
||||
"Cookie" => HTTP::Cookie.cookie_value(agent.cookies)
|
||||
}
|
||||
end
|
||||
|
||||
{
|
||||
"Referer" => "https://www.pixiv.net"
|
||||
}
|
||||
{ "Referer" => "https://www.pixiv.net" }
|
||||
end
|
||||
|
||||
def normalize_for_source
|
||||
@@ -242,10 +221,6 @@ module Sources
|
||||
end
|
||||
|
||||
def image_urls_sub
|
||||
if url =~ FANBOX_IMAGE
|
||||
return [url]
|
||||
end
|
||||
|
||||
# there's too much normalization bullshit we have to deal with
|
||||
# raw urls, so just fetch the canonical url from the api every
|
||||
# time.
|
||||
@@ -265,7 +240,7 @@ module Sources
|
||||
# even though it makes sense to reference page_url here, it will only look
|
||||
# at (url, referer_url).
|
||||
def illust_id
|
||||
return nil if novel_id.present? || fanbox_id.present?
|
||||
return nil if novel_id.present?
|
||||
|
||||
parsed_urls.each do |url|
|
||||
# http://www.pixiv.net/member_illust.php?mode=medium&illust_id=18557054
|
||||
@@ -328,46 +303,11 @@ module Sources
|
||||
end
|
||||
memoize :novel_id
|
||||
|
||||
def fanbox_id
|
||||
[url, referer_url].each do |x|
|
||||
if x =~ FANBOX_PAGE
|
||||
return $1
|
||||
end
|
||||
|
||||
if x =~ FANBOX_IMAGE
|
||||
return $1
|
||||
end
|
||||
end
|
||||
|
||||
nil
|
||||
end
|
||||
memoize :fanbox_id
|
||||
|
||||
def fanbox_account_id
|
||||
[url, referer_url].each do |x|
|
||||
if x =~ FANBOX_ACCOUNT
|
||||
return x
|
||||
end
|
||||
end
|
||||
|
||||
nil
|
||||
end
|
||||
memoize :fanbox_account_id
|
||||
|
||||
def agent
|
||||
PixivWebAgent.build
|
||||
end
|
||||
memoize :agent
|
||||
|
||||
def metadata
|
||||
if novel_id.present?
|
||||
return PixivApiClient.new.novel(novel_id)
|
||||
end
|
||||
|
||||
if fanbox_id.present?
|
||||
return PixivApiClient.new.fanbox(fanbox_id)
|
||||
end
|
||||
|
||||
PixivApiClient.new.work(illust_id)
|
||||
end
|
||||
memoize :metadata
|
||||
|
||||
@@ -23,7 +23,7 @@ module Sources::Strategies
|
||||
OLD_IMAGE = %r{\Ahttps?://#{DOMAIN}/(?<dir>#{MD5}/)?#{FILENAME}_(?<size>\w+)\.#{EXT}\z}i
|
||||
|
||||
IMAGE = %r{\Ahttps?://#{DOMAIN}/}i
|
||||
VIDEO = %r{\Ahttps?://(?:vtt|ve\.media)\.tumblr\.com/}i
|
||||
VIDEO = %r{\Ahttps?://(?:vtt|ve|va\.media)\.tumblr\.com/}i
|
||||
POST = %r{\Ahttps?://(?<blog_name>[^.]+)\.tumblr\.com/(?:post|image)/(?<post_id>\d+)}i
|
||||
|
||||
def self.enabled?
|
||||
@@ -168,7 +168,7 @@ module Sources::Strategies
|
||||
end
|
||||
|
||||
candidates.find do |candidate|
|
||||
http_exists?(candidate, headers)
|
||||
http_exists?(candidate)
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
@@ -200,7 +200,7 @@ module Sources::Strategies
|
||||
end
|
||||
|
||||
def api_response
|
||||
return {} unless self.class.enabled?
|
||||
return {} unless self.class.enabled? && status_id.present?
|
||||
api_client.status(status_id)
|
||||
end
|
||||
|
||||
|
||||
@@ -11,14 +11,6 @@ module TagRelationshipRetirementService
|
||||
"This topic deals with tag relationships created two or more years ago that have not been used since. They will be retired. This topic will be updated as an automated system retires expired relationships."
|
||||
end
|
||||
|
||||
def dry_run
|
||||
[TagAlias, TagImplication].each do |model|
|
||||
each_candidate(model) do |rel|
|
||||
puts "#{rel.relationship} #{rel.antecedent_name} -> #{rel.consequent_name} retired"
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def forum_topic
|
||||
topic = ForumTopic.where(title: forum_topic_title).first
|
||||
if topic.nil?
|
||||
|
||||
@@ -7,13 +7,10 @@ class UploadService
|
||||
# this gets called from UploadsController#new so we need to preprocess async
|
||||
UploadPreprocessorDelayedStartJob.perform_later(url, ref, CurrentUser.user)
|
||||
|
||||
begin
|
||||
download = Downloads::File.new(url, ref)
|
||||
remote_size = download.size
|
||||
rescue Exception
|
||||
end
|
||||
strategy = Sources::Strategies.find(url, ref)
|
||||
remote_size = strategy.size
|
||||
|
||||
[upload, remote_size]
|
||||
return [upload, remote_size]
|
||||
end
|
||||
|
||||
if file
|
||||
|
||||
@@ -71,13 +71,13 @@ class UploadService
|
||||
return file if file.present?
|
||||
raise "No file or source URL provided" if upload.source_url.blank?
|
||||
|
||||
download = Downloads::File.new(upload.source_url, upload.referer_url)
|
||||
file, strategy = download.download!
|
||||
strategy = Sources::Strategies.find(upload.source_url, upload.referer_url)
|
||||
file = strategy.download_file!
|
||||
|
||||
if download.data[:ugoira_frame_data].present?
|
||||
if strategy.data[:ugoira_frame_data].present?
|
||||
upload.context = {
|
||||
"ugoira" => {
|
||||
"frame_data" => download.data[:ugoira_frame_data],
|
||||
"frame_data" => strategy.data[:ugoira_frame_data],
|
||||
"content_type" => "image/jpeg"
|
||||
}
|
||||
}
|
||||
|
||||
27
app/logical/validating_socket.rb
Normal file
27
app/logical/validating_socket.rb
Normal file
@@ -0,0 +1,27 @@
|
||||
# A TCPSocket wrapper that disallows connections to local or private IPs. Used for SSRF protection.
|
||||
# https://owasp.org/www-community/attacks/Server_Side_Request_Forgery
|
||||
|
||||
require "resolv"
|
||||
|
||||
class ValidatingSocket < TCPSocket
|
||||
class ProhibitedIpError < StandardError; end
|
||||
|
||||
def initialize(hostname, port)
|
||||
ip = validate_hostname!(hostname)
|
||||
super(ip, port)
|
||||
end
|
||||
|
||||
def validate_hostname!(hostname)
|
||||
ip = IPAddress.parse(::Resolv.getaddress(hostname))
|
||||
raise ProhibitedIpError, "Connection to #{hostname} failed; #{ip} is a prohibited IP" if prohibited_ip?(ip)
|
||||
ip.to_s
|
||||
end
|
||||
|
||||
def prohibited_ip?(ip)
|
||||
if ip.ipv4?
|
||||
ip.loopback? || ip.link_local? || ip.multicast? || ip.private?
|
||||
elsif ip.ipv6?
|
||||
ip.loopback? || ip.link_local? || ip.unique_local? || ip.unspecified?
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -34,7 +34,7 @@ class ModerationReport < ApplicationRecord
|
||||
def forum_topic
|
||||
topic = ForumTopic.find_by_title(forum_topic_title)
|
||||
if topic.nil?
|
||||
CurrentUser.as_system do
|
||||
CurrentUser.scoped(User.system) do
|
||||
topic = ForumTopic.create!(creator: User.system, title: forum_topic_title, category_id: 0, min_level: User::Levels::MODERATOR)
|
||||
forum_post = ForumPost.create!(creator: User.system, body: forum_topic_body, topic: topic)
|
||||
end
|
||||
|
||||
@@ -33,7 +33,7 @@ class PostVersion < ApplicationRecord
|
||||
end
|
||||
|
||||
def tag_matches(string)
|
||||
tag = string.split(/\S+/)[0]
|
||||
tag = string.match(/\S+/)[0]
|
||||
return all if tag.nil?
|
||||
tag = "*#{tag}*" unless tag =~ /\*/
|
||||
where_ilike(:tags, tag)
|
||||
|
||||
@@ -18,8 +18,7 @@ class SavedSearch < ApplicationRecord
|
||||
post_ids = Set.new
|
||||
queries.each do |query|
|
||||
redis_key = "search:#{query}"
|
||||
# XXX change to `exists?` (ref: https://github.com-sds/mock_redis/pull/188
|
||||
if redis.exists(redis_key)
|
||||
if redis.exists?(redis_key)
|
||||
sub_ids = redis.smembers(redis_key).map(&:to_i)
|
||||
post_ids.merge(sub_ids)
|
||||
else
|
||||
@@ -116,7 +115,7 @@ class SavedSearch < ApplicationRecord
|
||||
|
||||
def populate(query, timeout: 10_000)
|
||||
redis_key = "search:#{query}"
|
||||
return if redis.exists(redis_key)
|
||||
return if redis.exists?(redis_key)
|
||||
|
||||
post_ids = Post.with_timeout(timeout, [], query: query) do
|
||||
Post.system_tag_match(query).limit(QUERY_LIMIT).pluck(:id)
|
||||
|
||||
@@ -53,7 +53,11 @@ class WikiPage < ApplicationRecord
|
||||
end
|
||||
|
||||
def linked_to(title)
|
||||
where(id: DtextLink.wiki_page.wiki_link.where(link_target: title).select(:model_id))
|
||||
where(dtext_links: DtextLink.wiki_page.wiki_link.where(link_target: normalize_title(title)))
|
||||
end
|
||||
|
||||
def not_linked_to(title)
|
||||
where.not(dtext_links: DtextLink.wiki_page.wiki_link.where(link_target: normalize_title(title)))
|
||||
end
|
||||
|
||||
def default_order
|
||||
@@ -82,6 +86,10 @@ class WikiPage < ApplicationRecord
|
||||
q = q.linked_to(params[:linked_to])
|
||||
end
|
||||
|
||||
if params[:not_linked_to].present?
|
||||
q = q.not_linked_to(params[:not_linked_to])
|
||||
end
|
||||
|
||||
if params[:hide_deleted].to_s.truthy?
|
||||
q = q.where("is_deleted = false")
|
||||
end
|
||||
@@ -146,6 +154,7 @@ class WikiPage < ApplicationRecord
|
||||
end
|
||||
|
||||
def self.normalize_title(title)
|
||||
return if title.blank?
|
||||
title.downcase.delete_prefix("~").gsub(/[[:space:]]+/, "_").gsub(/__/, "_").gsub(/\A_|_\z/, "")
|
||||
end
|
||||
|
||||
|
||||
@@ -24,7 +24,7 @@ class ForumPostPolicy < ApplicationPolicy
|
||||
end
|
||||
|
||||
def votable?
|
||||
unbanned? && show? && record.bulk_update_request.present? && record.bulk_update_request.is_pending?
|
||||
unbanned? && show? && record.bulk_update_request.present? && record.bulk_update_request.is_pending? && record.bulk_update_request.user_id != user.id
|
||||
end
|
||||
|
||||
def reportable?
|
||||
|
||||
@@ -8,12 +8,14 @@
|
||||
<% if @artist.is_banned? && !policy(@artist).can_view_banned? %>
|
||||
<p>The artist requested removal of this page.</p>
|
||||
<% else %>
|
||||
<% if @artist.wiki_page.present? %>
|
||||
<div class="prose">
|
||||
<%= format_text(@artist.wiki_page.body, :disable_mentions => true) %>
|
||||
</div>
|
||||
<% if @artist.wiki_page.present? && !@artist.wiki_page.is_deleted? %>
|
||||
<div class="artist-wiki">
|
||||
<div class="prose">
|
||||
<%= format_text(@artist.wiki_page.body, :disable_mentions => true) %>
|
||||
</div>
|
||||
|
||||
<p><%= link_to "View wiki page", @artist.wiki_page %></p>
|
||||
<p><%= link_to "View wiki page", @artist.wiki_page %></p>
|
||||
</div>
|
||||
<% end %>
|
||||
|
||||
<%= yield %>
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<% @search_service.missed_search_rankings.each do |tags, count| %>
|
||||
<% @missed_searches.each do |tags, count| %>
|
||||
<tr class="tag-type-<%= Tag.category_for(tags) %>">
|
||||
<td><%= link_to tags, posts_path(:tags => tags) %></td>
|
||||
<td>
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<% @search_service.post_search_rankings(@date).each do |tags, count| %>
|
||||
<% @searches.each do |tags, count| %>
|
||||
<tr class="tag-type-<%= Tag.category_for(tags) %>">
|
||||
<td><%= link_to tags, posts_path(:tags => tags) %></td>
|
||||
<td style="text-align: right;"><%= count.to_i %></td>
|
||||
|
||||
@@ -4,6 +4,8 @@
|
||||
<%= f.input :title_normalize, label: "Title", hint: "Use * for wildcard searches", input_html: { "data-autocomplete": "wiki-page" } %>
|
||||
<%= f.input :other_names_match, label: "Other names", hint: "Use * for wildcard searches" %>
|
||||
<%= f.input :body_matches, label: "Body" %>
|
||||
<%= f.input :linked_to, hint: "Which wikis link to the specified wiki.", input_html: { "data-autocomplete": "wiki-page" } %>
|
||||
<%= f.input :not_linked_to, hint: "Which wikis do not link to the specified wiki.", input_html: { "data-autocomplete": "wiki-page" } %>
|
||||
<%= f.input :other_names_present, as: :select %>
|
||||
<%= f.input :hide_deleted, as: :select, include_blank: false %>
|
||||
<%= f.input :order, collection: [%w[Name title], %w[Date time], %w[Posts post_count]], include_blank: false %>
|
||||
|
||||
@@ -28,7 +28,7 @@
|
||||
<%= format_text(@wiki_page.body) %>
|
||||
<% end %>
|
||||
|
||||
<% if @wiki_page.artist %>
|
||||
<% if @wiki_page.artist.present? && !@wiki_page.artist.is_deleted? %>
|
||||
<p><%= link_to "View artist", @wiki_page.artist %></p>
|
||||
<% end %>
|
||||
|
||||
|
||||
@@ -1,10 +0,0 @@
|
||||
development:
|
||||
adapter: async
|
||||
|
||||
test:
|
||||
adapter: test
|
||||
|
||||
production:
|
||||
adapter: redis
|
||||
url: <%= ENV.fetch("REDIS_URL") { "redis://localhost:6379/1" } %>
|
||||
channel_prefix: danbooru_production
|
||||
@@ -317,13 +317,15 @@ module Danbooru
|
||||
# A list of tags that should be removed when a post is replaced. Regexes allowed.
|
||||
def post_replacement_tag_removals
|
||||
%w[replaceme .*_sample resized upscaled downscaled md5_mismatch
|
||||
jpeg_artifacts corrupted_image source_request non-web_source]
|
||||
jpeg_artifacts corrupted_image missing_image missing_sample missing_thumbnail
|
||||
resolution_mismatch source_larger source_smaller source_request non-web_source]
|
||||
end
|
||||
|
||||
# Posts with these tags will be highlighted in the modqueue.
|
||||
def modqueue_warning_tags
|
||||
%w[hard_translated self_upload nude_filter third-party_edit screencap
|
||||
duplicate image_sample md5_mismatch resized upscaled downscaled]
|
||||
duplicate image_sample md5_mismatch resized upscaled downscaled
|
||||
resolution_mismatch source_larger source_smaller]
|
||||
end
|
||||
|
||||
def stripe_secret_key
|
||||
@@ -338,22 +340,6 @@ module Danbooru
|
||||
def twitter_api_secret
|
||||
end
|
||||
|
||||
# The default headers to be sent with outgoing http requests. Some external
|
||||
# services will fail if you don't set a valid User-Agent.
|
||||
def http_headers
|
||||
{
|
||||
"User-Agent" => "#{Danbooru.config.canonical_app_name}/#{Rails.application.config.x.git_hash}"
|
||||
}
|
||||
end
|
||||
|
||||
def httparty_options
|
||||
# proxy example:
|
||||
# {http_proxyaddr: "", http_proxyport: "", http_proxyuser: nil, http_proxypass: nil}
|
||||
{
|
||||
headers: Danbooru.config.http_headers
|
||||
}
|
||||
end
|
||||
|
||||
# you should override this
|
||||
def email_key
|
||||
"zDMSATq0W3hmA5p3rKTgD"
|
||||
@@ -374,14 +360,20 @@ module Danbooru
|
||||
false
|
||||
end
|
||||
|
||||
# reportbooru options - see https://github.com/r888888888/reportbooru
|
||||
# The URL for the Reportbooru server (https://github.com/evazion/reportbooru).
|
||||
# Optional. Used for tracking post views, popular searches, and missed searches.
|
||||
# Set to http://localhost/mock/reportbooru to enable a fake reportbooru
|
||||
# server for development purposes.
|
||||
def reportbooru_server
|
||||
end
|
||||
|
||||
def reportbooru_key
|
||||
end
|
||||
|
||||
# iqdbs options - see https://github.com/r888888888/iqdbs
|
||||
# The URL for the IQDBs server (https://github.com/evazion/iqdbs).
|
||||
# Optional. Used for dupe detection and reverse image searches.
|
||||
# Set to http://localhost/mock/iqdbs to enable a fake iqdb server for
|
||||
# development purposes.
|
||||
def iqdbs_server
|
||||
end
|
||||
|
||||
@@ -459,6 +451,10 @@ module Danbooru
|
||||
def cloudflare_zone
|
||||
end
|
||||
|
||||
# The URL for the recommender server (https://github.com/evazion/recommender).
|
||||
# Optional. Used to generate post recommendations.
|
||||
# Set to http://localhost/mock/recommender to enable a fake recommender
|
||||
# server for development purposes.
|
||||
def recommender_server
|
||||
end
|
||||
|
||||
|
||||
@@ -13,14 +13,20 @@ RUN \
|
||||
webpack \
|
||||
libvips-dev \
|
||||
libxml2-dev \
|
||||
libxslt-dev \
|
||||
zlib1g-dev \
|
||||
postgresql-server-dev-all && \
|
||||
# webpacker expects the binary to be called `yarn`, but debian/ubuntu installs it as `yarnpkg`.
|
||||
ln -sf /usr/bin/yarnpkg /usr/bin/yarn
|
||||
|
||||
WORKDIR /build
|
||||
|
||||
COPY .bundle .bundle
|
||||
COPY Gemfile Gemfile.lock ./
|
||||
RUN BUNDLE_DEPLOYMENT=true bundle install --jobs 4
|
||||
RUN \
|
||||
bundle config set deployment true --local && \
|
||||
bundle config set path vendor/bundle && \
|
||||
bundle install --jobs 4
|
||||
|
||||
COPY package.json yarn.lock ./
|
||||
RUN yarn install
|
||||
@@ -44,6 +50,8 @@ RUN \
|
||||
mkvtoolnix \
|
||||
libvips \
|
||||
libxml2 \
|
||||
libxslt1.1 \
|
||||
zlib1g \
|
||||
postgresql-client
|
||||
|
||||
USER danbooru
|
||||
|
||||
@@ -1,64 +0,0 @@
|
||||
require 'mechanize'
|
||||
|
||||
if Rails.env.test?
|
||||
# something about the root certs on the travis ci image causes Mechanize
|
||||
# to intermittently fail. this is a monkey patch to reset the connection
|
||||
# after every request to avoid dealing wtiht he issue.
|
||||
#
|
||||
# from http://scottwb.com/blog/2013/11/09/defeating-the-infamous-mechanize-too-many-connection-resets-bug/
|
||||
class Mechanize::HTTP::Agent
|
||||
MAX_RESET_RETRIES = 10
|
||||
|
||||
# We need to replace the core Mechanize HTTP method:
|
||||
#
|
||||
# Mechanize::HTTP::Agent#fetch
|
||||
#
|
||||
# with a wrapper that handles the infamous "too many connection resets"
|
||||
# Mechanize bug that is described here:
|
||||
#
|
||||
# https://github.com/sparklemotion/mechanize/issues/123
|
||||
#
|
||||
# The wrapper shuts down the persistent HTTP connection when it fails with
|
||||
# this error, and simply tries again. In practice, this only ever needs to
|
||||
# be retried once, but I am going to let it retry a few times
|
||||
# (MAX_RESET_RETRIES), just in case.
|
||||
#
|
||||
def fetch_with_retry(
|
||||
uri,
|
||||
method = :get,
|
||||
headers = {},
|
||||
params = [],
|
||||
referer = current_page,
|
||||
redirects = 0
|
||||
)
|
||||
action = "#{method.to_s.upcase} #{uri}"
|
||||
retry_count = 0
|
||||
|
||||
begin
|
||||
fetch_without_retry(uri, method, headers, params, referer, redirects)
|
||||
rescue Net::HTTP::Persistent::Error => e
|
||||
# Pass on any other type of error.
|
||||
raise unless e.message =~ /too many connection resets/
|
||||
|
||||
# Pass on the error if we've tried too many times.
|
||||
if retry_count >= MAX_RESET_RETRIES
|
||||
print "R"
|
||||
# puts "**** WARN: Mechanize retried connection reset #{MAX_RESET_RETRIES} times and never succeeded: #{action}"
|
||||
raise
|
||||
end
|
||||
|
||||
# Otherwise, shutdown the persistent HTTP connection and try again.
|
||||
print "R"
|
||||
# puts "**** WARN: Mechanize retrying connection reset error: #{action}"
|
||||
retry_count += 1
|
||||
self.http.shutdown
|
||||
retry
|
||||
end
|
||||
end
|
||||
|
||||
# Alias so #fetch actually uses our new #fetch_with_retry to wrap the
|
||||
# old one aliased as #fetch_without_retry.
|
||||
alias fetch_without_retry fetch
|
||||
alias fetch fetch_with_retry
|
||||
end
|
||||
end
|
||||
@@ -372,6 +372,14 @@ Rails.application.routes.draw do
|
||||
get "/static/contact" => "static#contact", :as => "contact"
|
||||
get "/static/dtext_help" => "static#dtext_help", :as => "dtext_help"
|
||||
|
||||
get "/mock/recommender/recommend/:user_id" => "mock_services#recommender_recommend", as: "mock_recommender_recommend"
|
||||
get "/mock/recommender/similiar/:post_id" => "mock_services#recommender_similar", as: "mock_recommender_similar"
|
||||
get "/mock/reportbooru/missed_searches" => "mock_services#reportbooru_missed_searches", as: "mock_reportbooru_missed_searches"
|
||||
get "/mock/reportbooru/post_searches/rank" => "mock_services#reportbooru_post_searches", as: "mock_reportbooru_post_searches"
|
||||
get "/mock/reportbooru/post_views/rank" => "mock_services#reportbooru_post_views", as: "mock_reportbooru_post_views"
|
||||
get "/mock/iqdbs/similar" => "mock_services#iqdbs_similar", as: "mock_iqdbs_similar"
|
||||
post "/mock/iqdbs/similar" => "mock_services#iqdbs_similar"
|
||||
|
||||
root :to => "posts#index"
|
||||
|
||||
get "*other", :to => "static#not_found"
|
||||
|
||||
@@ -1,26 +0,0 @@
|
||||
# Be sure to restart your server when you modify this file.
|
||||
|
||||
# Your secret key is used for verifying the integrity of signed cookies.
|
||||
# If you change this key, all old signed cookies will become invalid!
|
||||
|
||||
# Make sure the secret is at least 30 characters and all random,
|
||||
# no regular words or you'll be exposed to dictionary attacks.
|
||||
# You can use `rake secret` to generate a secure secret key.
|
||||
|
||||
# Make sure the secrets in this file are kept private
|
||||
# if you're sharing your code publicly.
|
||||
|
||||
development:
|
||||
secret_key_base: bcc62a512b9c055c292c17742f1e65bd6d88fa37f4d01c8475103809f3ac4c03e3e98605c47d55cd8801333010ea98920a61b722770629926759624bce732539
|
||||
|
||||
test:
|
||||
secret_key_base: 60e32a818af77bdfc40bca866e3b4d7b88d7ba767057ffc9e4532279358af8c67d42f2b99c084b700727303ce25b812a592b52723ebc1e3b812fd09a1f969435
|
||||
|
||||
# Do not keep production secrets in the repository,
|
||||
# instead read values from the environment.
|
||||
production:
|
||||
secret_key_base: <%= ENV["SECRET_KEY_BASE"] %>
|
||||
|
||||
staging:
|
||||
secret_key_base: <%= ENV["SECRET_KEY_BASE"] %>
|
||||
|
||||
@@ -1,34 +0,0 @@
|
||||
test:
|
||||
service: Disk
|
||||
root: <%= Rails.root.join("tmp/storage") %>
|
||||
|
||||
local:
|
||||
service: Disk
|
||||
root: <%= Rails.root.join("storage") %>
|
||||
|
||||
# Use rails credentials:edit to set the AWS secrets (as aws:access_key_id|secret_access_key)
|
||||
# amazon:
|
||||
# service: S3
|
||||
# access_key_id: <%= Rails.application.credentials.dig(:aws, :access_key_id) %>
|
||||
# secret_access_key: <%= Rails.application.credentials.dig(:aws, :secret_access_key) %>
|
||||
# region: us-east-1
|
||||
# bucket: your_own_bucket
|
||||
|
||||
# Remember not to checkin your GCS keyfile to a repository
|
||||
# google:
|
||||
# service: GCS
|
||||
# project: your_project
|
||||
# credentials: <%= Rails.root.join("path/to/gcs.keyfile") %>
|
||||
# bucket: your_own_bucket
|
||||
|
||||
# Use rails credentials:edit to set the Azure Storage secret (as azure_storage:storage_access_key)
|
||||
# microsoft:
|
||||
# service: AzureStorage
|
||||
# storage_account_name: your_account_name
|
||||
# storage_access_key: <%= Rails.application.credentials.dig(:azure_storage, :storage_access_key) %>
|
||||
# container: your_container_name
|
||||
|
||||
# mirror:
|
||||
# service: Mirror
|
||||
# primary: local
|
||||
# mirrors: [ amazon, google, microsoft ]
|
||||
@@ -6,7 +6,7 @@ worker_processes 20
|
||||
|
||||
timeout 180
|
||||
# listen "127.0.0.1:9000", :tcp_nopush => true
|
||||
listen "/tmp/.unicorn.sock", :backlog => 512
|
||||
listen "/tmp/.unicorn.sock", backlog: 1024
|
||||
|
||||
# Spawn unicorn master worker for user apps (group: apps)
|
||||
user 'danbooru', 'danbooru'
|
||||
|
||||
@@ -27,8 +27,10 @@
|
||||
"webpack-cli": "^3.3.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"eslint": "^6.0.0",
|
||||
"babel-eslint": "^10.1.0",
|
||||
"eslint": "^7.0.0",
|
||||
"eslint-loader": "^4.0.0",
|
||||
"eslint-plugin-babel": "^5.3.0",
|
||||
"eslint-plugin-ignore-erb": "^0.1.1",
|
||||
"stylelint": "^13.0.0",
|
||||
"stylelint-config-standard": "^20.0.0",
|
||||
|
||||
@@ -1,101 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# this is a version of the install script designed to be run on
|
||||
# app servers (that is, they won't install PostgreSQL server).
|
||||
#
|
||||
# Run: curl -L -s https://raw.githubusercontent.com/r888888888/danbooru/master/script/install/app_server.sh | sh
|
||||
|
||||
export RUBY_VERSION=2.6.3
|
||||
export GITHUB_INSTALL_SCRIPTS=https://raw.githubusercontent.com/r888888888/danbooru/master/script/install
|
||||
export VIPS_VERSION=8.7.0
|
||||
|
||||
if [[ "$(whoami)" != "root" ]] ; then
|
||||
echo "You must run this script as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "* DANBOORU INSTALLATION SCRIPT"
|
||||
echo "*"
|
||||
echo "* This script will install all the necessary packages to run Danbooru on an"
|
||||
echo "* Ubuntu server."
|
||||
echo
|
||||
|
||||
echo -n "* Enter the VLAN IP address for this server: "
|
||||
read VLAN_IP_ADDR
|
||||
|
||||
# Install packages
|
||||
echo "* Installing packages..."
|
||||
|
||||
apt-get update
|
||||
apt-get -y install libssl-dev build-essential automake libxml2-dev libxslt-dev ncurses-dev sudo libreadline-dev flex bison ragel redis git curl libcurl4-openssl-dev sendmail-bin sendmail nginx ssh coreutils ffmpeg mkvtoolnix
|
||||
apt-get -y install libpq-dev postgresql-client
|
||||
apt-get -y install liblcms2-dev libjpeg-turbo8-dev libexpat1-dev libgif-dev libpng-dev libexif-dev
|
||||
|
||||
# vrack specific stuff
|
||||
apt-get -y install vlan
|
||||
modprobe 8021q
|
||||
echo "8021q" >> /etc/modules
|
||||
vconfig add eno2 99
|
||||
ip addr add $VLAN_IP_ADDR/24 dev eno2.99
|
||||
ip link set up eno2.99
|
||||
curl -L -s $GITHUB_INSTALL_SCRIPTS/vrack-cfg.yaml -o /etc/netplan/01-netcfg.yaml
|
||||
|
||||
curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -
|
||||
echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list
|
||||
curl -sSL https://deb.nodesource.com/setup_10.x | sudo -E bash -
|
||||
apt-get update
|
||||
apt-get -y install nodejs yarn
|
||||
apt-get remove cmdtest
|
||||
|
||||
# compile and install libvips (the version in apt is too old)
|
||||
cd /tmp
|
||||
wget -q https://github.com/libvips/libvips/releases/download/v$VIPS_VERSION/vips-$VIPS_VERSION.tar.gz
|
||||
tar xzf vips-$VIPS_VERSION.tar.gz
|
||||
cd vips-$VIPS_VERSION
|
||||
./configure --prefix=/usr
|
||||
make install
|
||||
ldconfig
|
||||
|
||||
# Create user account
|
||||
useradd -m danbooru
|
||||
chsh -s /bin/bash danbooru
|
||||
usermod -G danbooru,sudo danbooru
|
||||
|
||||
# Set up Postgres
|
||||
git clone https://github.com/r888888888/test_parser.git /tmp/test_parser
|
||||
cd /tmp/test_parser
|
||||
make install
|
||||
|
||||
# Install rbenv
|
||||
echo "* Installing rbenv..."
|
||||
cd /tmp
|
||||
sudo -u danbooru git clone git://github.com/sstephenson/rbenv.git ~danbooru/.rbenv
|
||||
sudo -u danbooru touch ~danbooru/.bash_profile
|
||||
echo 'export PATH="$HOME/.rbenv/bin:$PATH"' >> ~danbooru/.bash_profile
|
||||
echo 'eval "$(rbenv init -)"' >> ~danbooru/.bash_profile
|
||||
sudo -u danbooru mkdir -p ~danbooru/.rbenv/plugins
|
||||
sudo -u danbooru git clone git://github.com/sstephenson/ruby-build.git ~danbooru/.rbenv/plugins/ruby-build
|
||||
sudo -u danbooru bash -l -c "rbenv install $RUBY_VERSION"
|
||||
sudo -u danbooru bash -l -c "rbenv global $RUBY_VERSION"
|
||||
|
||||
# Install gems
|
||||
echo "* Installing gems..."
|
||||
sudo -u danbooru bash -l -c 'gem install --no-ri --no-rdoc bundler'
|
||||
|
||||
# Setup danbooru account
|
||||
echo "* Enter a new password for the danbooru account"
|
||||
passwd danbooru
|
||||
|
||||
echo "* Setting up SSH keys for the danbooru account"
|
||||
sudo -u danbooru ssh-keygen
|
||||
sudo -u danbooru cat ~danbooru/.ssh/id_rsa.pub >> ~danbooru/.ssh/authorized_keys
|
||||
|
||||
echo "* TODO:"
|
||||
echo "on kagamihara:"
|
||||
echo "script/install/distribute_new_pubkey.sh"
|
||||
echo
|
||||
echo "on this server:"
|
||||
echo "rsync -av kagamihara:/etc/nginx/nginx.conf /etc/nginx"
|
||||
echo "rsync -av kagamihara:/etc/nginx/conf.d /etc/nginx"
|
||||
echo "rsync -av kagamihara:/etc/nginx/sites-enabled /etc/nginx"
|
||||
echo "rsync -av kagamihara:/etc/logrotate.d /etc/logrotate.d"
|
||||
@@ -1,10 +0,0 @@
|
||||
#!/bin/sh
|
||||
|
||||
HOSTS="kagamihara shima saitou"
|
||||
|
||||
echo "Enter new SSH pubkey: "
|
||||
read $key
|
||||
|
||||
for host in $HOSTS ; do
|
||||
ssh danbooru@$host echo $key >> .ssh/authorized_keys
|
||||
done
|
||||
@@ -1,2 +0,0 @@
|
||||
[Service]
|
||||
LimitNOFILE=10000
|
||||
@@ -1,92 +0,0 @@
|
||||
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
|
||||
ssl_ciphers 'ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS';
|
||||
ssl_prefer_server_ciphers on;
|
||||
ssl_session_timeout 4h;
|
||||
ssl_session_cache shared:SSL:20m;
|
||||
ssl_session_tickets off;
|
||||
ssl_stapling on;
|
||||
ssl_stapling_verify on;
|
||||
ssl_dhparam /etc/nginx/ssl/dhparam.pem;
|
||||
resolver 8.8.8.8 8.8.4.4;
|
||||
|
||||
root /var/www/danbooru/current/public;
|
||||
index index.html;
|
||||
access_log off;
|
||||
error_log /var/www/danbooru/shared/log/server.error.log;
|
||||
try_files $uri/index.html $uri.html $uri @app;
|
||||
client_max_body_size 35m;
|
||||
error_page 503 @maintenance;
|
||||
error_page 404 /404.html;
|
||||
error_page 500 502 503 504 /500.html;
|
||||
|
||||
location /assets {
|
||||
expires max;
|
||||
break;
|
||||
}
|
||||
|
||||
location /data/preview {
|
||||
expires max;
|
||||
break;
|
||||
}
|
||||
|
||||
location /posts/mobile {
|
||||
return 404;
|
||||
}
|
||||
|
||||
location /users {
|
||||
limit_req zone=users burst=5;
|
||||
limit_req_status 429;
|
||||
try_files $uri @app_server;
|
||||
}
|
||||
|
||||
location /posts {
|
||||
limit_req zone=posts burst=20;
|
||||
limit_req_status 429;
|
||||
try_files $uri @app_server;
|
||||
}
|
||||
|
||||
location /data {
|
||||
valid_referers none *.donmai.us donmai.us ~\.google\. ~\.bing\. ~\.yahoo\.;
|
||||
|
||||
if ($invalid_referer) {
|
||||
return 403;
|
||||
}
|
||||
|
||||
rewrite ^/data/sample/__.+?__(.+) /data/sample/$1 last;
|
||||
rewrite ^/data/__.+?__(.+) /data/$1 last;
|
||||
|
||||
expires max;
|
||||
break;
|
||||
}
|
||||
|
||||
location /maintenance.html {
|
||||
expires 10;
|
||||
}
|
||||
|
||||
if (-f $document_root/maintenance.html) {
|
||||
return 503;
|
||||
}
|
||||
|
||||
if ($http_user_agent ~ (WinHttp\.WinHttpRequest\.5) ) {
|
||||
return 403;
|
||||
}
|
||||
|
||||
location @maintenance {
|
||||
rewrite ^(.*)$ /maintenance.html last;
|
||||
}
|
||||
|
||||
location @app_server {
|
||||
proxy_pass http://app_server;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_redirect off;
|
||||
proxy_set_header Host $http_host;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_buffer_size 128k;
|
||||
proxy_buffers 4 256k;
|
||||
proxy_busy_buffers_size 256k;
|
||||
}
|
||||
|
||||
location / {
|
||||
try_files $uri @app_server;
|
||||
}
|
||||
@@ -1,82 +0,0 @@
|
||||
user www-data;
|
||||
worker_processes auto;
|
||||
pid /var/run/nginx.pid;
|
||||
|
||||
events {
|
||||
use epoll;
|
||||
worker_connections 10000;
|
||||
multi_accept on;
|
||||
accept_mutex on;
|
||||
}
|
||||
|
||||
http {
|
||||
limit_req_zone $binary_remote_addr zone=users:10m rate=5r/s;
|
||||
limit_req_zone $binary_remote_addr zone=posts:100m rate=10r/s;
|
||||
|
||||
##
|
||||
# Basic Settings
|
||||
##
|
||||
|
||||
sendfile on;
|
||||
tcp_nopush on;
|
||||
tcp_nodelay on;
|
||||
keepalive_timeout 5;
|
||||
types_hash_max_size 2048;
|
||||
# server_tokens off;
|
||||
|
||||
server_names_hash_bucket_size 128;
|
||||
# server_name_in_redirect off;
|
||||
|
||||
include /etc/nginx/mime.types;
|
||||
default_type application/octet-stream;
|
||||
|
||||
##
|
||||
# Logging Settings
|
||||
##
|
||||
|
||||
access_log off;
|
||||
error_log /var/log/nginx/error.log;
|
||||
|
||||
##
|
||||
# Gzip Settings
|
||||
##
|
||||
|
||||
gzip on;
|
||||
gzip_disable "msie6";
|
||||
|
||||
gzip_http_version 1.1;
|
||||
gzip_vary on;
|
||||
gzip_comp_level 5;
|
||||
gzip_proxied any;
|
||||
gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/rss+xml text/javascript application/atom+xml;
|
||||
|
||||
# curl https://www.cloudflare.com/ips-v4 | sort
|
||||
set_real_ip_from 103.21.244.0/22;
|
||||
set_real_ip_from 103.22.200.0/22;
|
||||
set_real_ip_from 103.31.4.0/22;
|
||||
set_real_ip_from 104.16.0.0/12;
|
||||
set_real_ip_from 108.162.192.0/18;
|
||||
set_real_ip_from 131.0.72.0/22;
|
||||
set_real_ip_from 141.101.64.0/18;
|
||||
set_real_ip_from 162.158.0.0/15;
|
||||
set_real_ip_from 172.64.0.0/13;
|
||||
set_real_ip_from 173.245.48.0/20;
|
||||
set_real_ip_from 188.114.96.0/20;
|
||||
set_real_ip_from 190.93.240.0/20;
|
||||
set_real_ip_from 197.234.240.0/22;
|
||||
set_real_ip_from 198.41.128.0/17;
|
||||
set_real_ip_from 199.27.128.0/21;
|
||||
|
||||
# curl https://www.cloudflare.com/ips-v4 | sort
|
||||
set_real_ip_from 2400:cb00::/32;
|
||||
set_real_ip_from 2606:4700::/32;
|
||||
set_real_ip_from 2803:f800::/32;
|
||||
set_real_ip_from 2405:b500::/32;
|
||||
set_real_ip_from 2405:8100::/32;
|
||||
set_real_ip_from 2a06:98c0::/29;
|
||||
set_real_ip_from 2c0f:f248::/32;
|
||||
|
||||
real_ip_header CF-Connecting-IP;
|
||||
|
||||
include /etc/nginx/sites-enabled/*.conf;
|
||||
}
|
||||
@@ -1,68 +0,0 @@
|
||||
server {
|
||||
listen 443 ssl http2 default_server;
|
||||
server_name danbooru.donmai.us;
|
||||
|
||||
ssl_certificate /etc/nginx/ssl/danbooru.chain.pem;
|
||||
ssl_certificate_key /etc/nginx/ssl/danbooru.key;
|
||||
|
||||
include /etc/nginx/conf.d/common.conf;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 443 ssl http2;
|
||||
server_name safebooru.donmai.us;
|
||||
|
||||
ssl_certificate /etc/nginx/ssl/safebooru.chain.pem;
|
||||
ssl_certificate_key /etc/nginx/ssl/safebooru.key;
|
||||
|
||||
include /etc/nginx/conf.d/common.conf;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 443 ssl http2;
|
||||
server_name kagamihara.donmai.us;
|
||||
|
||||
ssl_certificate /etc/letsencrypt/live/kagamihara.donmai.us/fullchain.pem; # managed by Certbot
|
||||
ssl_certificate_key /etc/letsencrypt/live/kagamihara.donmai.us/privkey.pem; # managed by Certbot
|
||||
|
||||
include /etc/nginx/conf.d/common.conf;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 443 ssl http2;
|
||||
server_name saitou.donmai.us;
|
||||
|
||||
ssl_certificate /etc/letsencrypt/live/saitou.donmai.us/fullchain.pem; # managed by Certbot
|
||||
ssl_certificate_key /etc/letsencrypt/live/saitou.donmai.us/privkey.pem; # managed by Certbot
|
||||
|
||||
include /etc/nginx/conf.d/common.conf;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 443 ssl http2;
|
||||
server_name shima.donmai.us;
|
||||
|
||||
ssl_certificate /etc/letsencrypt/live/shima.donmai.us/fullchain.pem; # managed by Certbot
|
||||
ssl_certificate_key /etc/letsencrypt/live/shima.donmai.us/privkey.pem; # managed by Certbot
|
||||
|
||||
include /etc/nginx/conf.d/common.conf;
|
||||
}
|
||||
|
||||
# redirect HTTP to HTTPS.
|
||||
server {
|
||||
listen 80;
|
||||
server_name safebooru.donmai.us danbooru.donmai.us kagamihara.donmai.us saitou.donmai.us shima.donmai.us;
|
||||
return 301 https://$host$request_uri;
|
||||
}
|
||||
|
||||
# redirect donmai.us and www.donmai.us to danbooru.donmai.us.
|
||||
server {
|
||||
listen 80;
|
||||
listen 443 ssl;
|
||||
server_name donmai.us www.donmai.us;
|
||||
return 301 https://danbooru.donmai.us$request_uri;
|
||||
}
|
||||
|
||||
upstream app_server {
|
||||
server unix:/tmp/.unicorn.sock fail_timeout=0;
|
||||
}
|
||||
@@ -1,10 +0,0 @@
|
||||
network:
|
||||
version: 2
|
||||
renderer: networkd
|
||||
ethernets:
|
||||
eno2: {}
|
||||
vlans:
|
||||
eno2.99:
|
||||
id: 99
|
||||
link: eno2
|
||||
addresses: [172.16.0.1]
|
||||
@@ -1,7 +0,0 @@
|
||||
These are mocked services to be used for development purposes.
|
||||
|
||||
- danbooru: port 3000
|
||||
- recommender: port 3001
|
||||
- iqdbs: port 3002
|
||||
- reportbooru: port 3003
|
||||
|
||||
@@ -1,14 +0,0 @@
|
||||
require 'sinatra'
|
||||
require 'json'
|
||||
require_relative './mock_service_helper'
|
||||
|
||||
set :port, 3002
|
||||
|
||||
configure do
|
||||
POST_IDS = MockServiceHelper.fetch_post_ids
|
||||
end
|
||||
|
||||
get '/similar' do
|
||||
content_type :json
|
||||
POST_IDS[0..10].map {|x| {post_id: x}}.to_json
|
||||
end
|
||||
@@ -1,22 +0,0 @@
|
||||
require 'socket'
|
||||
require 'timeout'
|
||||
require 'httparty'
|
||||
|
||||
module MockServiceHelper
|
||||
module_function
|
||||
|
||||
DANBOORU_PORT = 3000
|
||||
|
||||
def fetch_post_ids
|
||||
begin
|
||||
s = TCPSocket.new("localhost", DANBOORU_PORT)
|
||||
s.close
|
||||
rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH
|
||||
sleep 1
|
||||
retry
|
||||
end
|
||||
|
||||
json = HTTParty.get("http://localhost:#{DANBOORU_PORT}/posts.json?random=true&limit=10").body
|
||||
return JSON.parse(json).map {|x| x["id"]}
|
||||
end
|
||||
end
|
||||
@@ -1,19 +0,0 @@
|
||||
require 'sinatra'
|
||||
require 'json'
|
||||
require_relative './mock_service_helper'
|
||||
|
||||
set :port, 3001
|
||||
|
||||
configure do
|
||||
POST_IDS = MockServiceHelper.fetch_post_ids
|
||||
end
|
||||
|
||||
get '/recommend/:user_id' do
|
||||
content_type :json
|
||||
POST_IDS[0..10].map {|x| [x, "1.000"]}.to_json
|
||||
end
|
||||
|
||||
get '/similar/:post_id' do
|
||||
content_type :json
|
||||
POST_IDS[0..6].map {|x| [x, "1.000"]}.to_json
|
||||
end
|
||||
@@ -1,22 +0,0 @@
|
||||
require 'sinatra'
|
||||
require 'json'
|
||||
|
||||
set :port, 3003
|
||||
|
||||
get '/missed_searches' do
|
||||
content_type :text
|
||||
return "abcdefg 10.0\nblahblahblah 20.0\n"
|
||||
end
|
||||
|
||||
get '/post_searches/rank' do
|
||||
content_type :json
|
||||
return [["abc", 100], ["def", 200]].to_json
|
||||
end
|
||||
|
||||
get '/reports/user_similarity' do
|
||||
# todo
|
||||
end
|
||||
|
||||
post '/post_views' do
|
||||
# todo
|
||||
end
|
||||
0
test/files/test-empty.bin
Normal file
0
test/files/test-empty.bin
Normal file
@@ -4,11 +4,8 @@ class ArtistsControllerTest < ActionDispatch::IntegrationTest
|
||||
def assert_artist_found(expected_artist, source_url = nil)
|
||||
if source_url
|
||||
get_auth artists_path(format: "json", search: { url_matches: source_url }), @user
|
||||
if response.body =~ /Net::OpenTimeout/
|
||||
skip "Remote connection to #{source_url} failed"
|
||||
return
|
||||
end
|
||||
end
|
||||
|
||||
assert_response :success
|
||||
json = JSON.parse(response.body)
|
||||
assert_equal(1, json.size, "Testing URL: #{source_url}")
|
||||
@@ -17,10 +14,6 @@ class ArtistsControllerTest < ActionDispatch::IntegrationTest
|
||||
|
||||
def assert_artist_not_found(source_url)
|
||||
get_auth artists_path(format: "json", search: { url_matches: source_url }), @user
|
||||
if response.body =~ /Net::OpenTimeout/
|
||||
skip "Remote connection to #{source_url} failed"
|
||||
return
|
||||
end
|
||||
|
||||
assert_response :success
|
||||
json = JSON.parse(response.body)
|
||||
@@ -54,6 +47,22 @@ class ArtistsControllerTest < ActionDispatch::IntegrationTest
|
||||
get artist_path(@artist.id)
|
||||
assert_response :success
|
||||
end
|
||||
|
||||
should "show active wikis" do
|
||||
as(@user) { create(:wiki_page, title: @artist.name) }
|
||||
get artist_path(@artist.id)
|
||||
|
||||
assert_response :success
|
||||
assert_select ".artist-wiki", count: 1
|
||||
end
|
||||
|
||||
should "not show deleted wikis" do
|
||||
as(@user) { create(:wiki_page, title: @artist.name, is_deleted: true) }
|
||||
get artist_path(@artist.id)
|
||||
|
||||
assert_response :success
|
||||
assert_select ".artist-wiki", count: 0
|
||||
end
|
||||
end
|
||||
|
||||
context "new action" do
|
||||
|
||||
@@ -36,6 +36,13 @@ class ForumPostVotesControllerTest < ActionDispatch::IntegrationTest
|
||||
assert_response 403
|
||||
end
|
||||
end
|
||||
|
||||
should "not allow creators to vote on their own BURs" do
|
||||
assert_difference("ForumPostVote.count", 0) do
|
||||
post_auth forum_post_votes_path(format: :js), @bulk_update_request.user, params: { forum_post_id: @forum_post.id, forum_post_vote: { score: 1 }}
|
||||
assert_response 403
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
context "destroy action" do
|
||||
|
||||
28
test/functional/mock_services_controller_test.rb
Normal file
28
test/functional/mock_services_controller_test.rb
Normal file
@@ -0,0 +1,28 @@
|
||||
require 'test_helper'
|
||||
|
||||
class MockServicesControllerTest < ActionDispatch::IntegrationTest
|
||||
context "The mock services controller" do
|
||||
setup do
|
||||
create(:post)
|
||||
create(:tag)
|
||||
end
|
||||
|
||||
context "for all actions" do
|
||||
should "work" do
|
||||
paths = [
|
||||
mock_recommender_recommend_path(42),
|
||||
mock_recommender_similar_path(42),
|
||||
mock_reportbooru_missed_searches_path,
|
||||
mock_reportbooru_post_searches_path,
|
||||
mock_reportbooru_post_views_path,
|
||||
mock_iqdbs_similar_path,
|
||||
]
|
||||
|
||||
paths.each do |path|
|
||||
get path
|
||||
assert_response :success
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -41,6 +41,13 @@ class PostVersionsControllerTest < ActionDispatch::IntegrationTest
|
||||
assert_response :success
|
||||
assert_equal @post.versions[1].id, response.parsed_body[0]["id"].to_i
|
||||
end
|
||||
|
||||
should "list all versions for search[tag_matches]" do
|
||||
get post_versions_path, as: :json, params: { search: { tag_matches: "tagme" }}
|
||||
assert_response :success
|
||||
assert_equal @post.versions[0].id, response.parsed_body[0]["id"].to_i
|
||||
assert_equal 1, response.parsed_body.length
|
||||
end
|
||||
end
|
||||
|
||||
context "undo action" do
|
||||
|
||||
@@ -1,15 +1,30 @@
|
||||
require 'test_helper'
|
||||
|
||||
class UploadsControllerTest < ActionDispatch::IntegrationTest
|
||||
def assert_uploaded(file_path, user, **upload_params)
|
||||
file = Rack::Test::UploadedFile.new("#{Rails.root}/#{file_path}")
|
||||
def self.should_upload_successfully(source)
|
||||
should "upload successfully from #{source}" do
|
||||
assert_successful_upload(source, user: create(:user, created_at: 1.month.ago))
|
||||
end
|
||||
end
|
||||
|
||||
assert_difference(["Upload.count", "Post.count"]) do
|
||||
post_auth uploads_path, user, params: { upload: { file: file, **upload_params }}
|
||||
assert_redirected_to Upload.last
|
||||
def assert_successful_upload(source_or_file_path, user: @user, **params)
|
||||
if source_or_file_path =~ %r{\Ahttps?://}i
|
||||
source = { source: source_or_file_path }
|
||||
else
|
||||
file = Rack::Test::UploadedFile.new(Rails.root.join(source_or_file_path))
|
||||
source = { file: file }
|
||||
end
|
||||
|
||||
Upload.last
|
||||
assert_difference(["Upload.count"]) do
|
||||
post_auth uploads_path, user, params: { upload: { tag_string: "abc", rating: "e", **source, **params }}
|
||||
end
|
||||
|
||||
upload = Upload.last
|
||||
assert_response :redirect
|
||||
assert_redirected_to upload
|
||||
assert_equal("completed", upload.status)
|
||||
assert_equal(Post.last, upload.post)
|
||||
assert_equal(upload.post.md5, upload.md5)
|
||||
end
|
||||
|
||||
context "The uploads controller" do
|
||||
@@ -18,6 +33,17 @@ class UploadsControllerTest < ActionDispatch::IntegrationTest
|
||||
mock_iqdb_service!
|
||||
end
|
||||
|
||||
context "image proxy action" do
|
||||
should "work" do
|
||||
url = "https://i.pximg.net/img-original/img/2017/11/21/17/06/44/65985331_p0.png"
|
||||
get_auth image_proxy_uploads_path, @user, params: { url: url }
|
||||
|
||||
assert_response :success
|
||||
assert_equal("image/png", response.media_type)
|
||||
assert_equal(15_573, response.body.size)
|
||||
end
|
||||
end
|
||||
|
||||
context "batch action" do
|
||||
context "for twitter galleries" do
|
||||
should "render" do
|
||||
@@ -259,32 +285,65 @@ class UploadsControllerTest < ActionDispatch::IntegrationTest
|
||||
end
|
||||
|
||||
context "uploading a file from your computer" do
|
||||
should "work for a jpeg file" do
|
||||
upload = assert_uploaded("test/files/test.jpg", @user, tag_string: "aaa", rating: "e", source: "aaa")
|
||||
should_upload_successfully("test/files/test.jpg")
|
||||
should_upload_successfully("test/files/test.png")
|
||||
should_upload_successfully("test/files/test-static-32x32.gif")
|
||||
should_upload_successfully("test/files/test-animated-86x52.gif")
|
||||
should_upload_successfully("test/files/test-300x300.mp4")
|
||||
should_upload_successfully("test/files/test-512x512.webm")
|
||||
should_upload_successfully("test/files/compressed.swf")
|
||||
end
|
||||
|
||||
assert_equal("jpg", upload.post.file_ext)
|
||||
assert_equal("aaa", upload.post.source)
|
||||
assert_equal(500, upload.post.image_width)
|
||||
assert_equal(335, upload.post.image_height)
|
||||
end
|
||||
context "uploading a file from a source" do
|
||||
should_upload_successfully("https://www.artstation.com/artwork/04XA4")
|
||||
should_upload_successfully("https://dantewontdie.artstation.com/projects/YZK5q")
|
||||
should_upload_successfully("https://cdna.artstation.com/p/assets/images/images/006/029/978/large/amama-l-z.jpg")
|
||||
|
||||
should "work for a webm file" do
|
||||
upload = assert_uploaded("test/files/test-512x512.webm", @user, tag_string: "aaa", rating: "e", source: "aaa")
|
||||
should_upload_successfully("https://www.deviantart.com/aeror404/art/Holiday-Elincia-424551484")
|
||||
should_upload_successfully("https://noizave.deviantart.com/art/test-no-download-697415967")
|
||||
should_upload_successfully("https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/8b472d70-a0d6-41b5-9a66-c35687090acc/d23jbr4-8a06af02-70cb-46da-8a96-42a6ba73cdb4.jpg/v1/fill/w_786,h_1017,q_70,strp/silverhawks_quicksilver_by_edsfox_d23jbr4-pre.jpg")
|
||||
|
||||
assert_equal("webm", upload.post.file_ext)
|
||||
assert_equal("aaa", upload.post.source)
|
||||
assert_equal(512, upload.post.image_width)
|
||||
assert_equal(512, upload.post.image_height)
|
||||
end
|
||||
should_upload_successfully("https://www.hentai-foundry.com/pictures/user/Afrobull/795025/kuroeda")
|
||||
should_upload_successfully("https://pictures.hentai-foundry.com/a/Afrobull/795025/Afrobull-795025-kuroeda.png")
|
||||
|
||||
should "work for a flash file" do
|
||||
upload = assert_uploaded("test/files/compressed.swf", @user, tag_string: "aaa", rating: "e", source: "aaa")
|
||||
should_upload_successfully("https://yande.re/post/show/482880")
|
||||
should_upload_successfully("https://files.yande.re/image/7ecfdead705d7b956b26b1d37b98d089/yande.re%20482880.jpg")
|
||||
|
||||
assert_equal("swf", upload.post.file_ext)
|
||||
assert_equal("aaa", upload.post.source)
|
||||
assert_equal(607, upload.post.image_width)
|
||||
assert_equal(756, upload.post.image_height)
|
||||
end
|
||||
should_upload_successfully("https://konachan.com/post/show/270916")
|
||||
should_upload_successfully("https://konachan.com/image/ca12cdb79a66d242e95a6f958341bf05/Konachan.com%20-%20270916.png")
|
||||
|
||||
should_upload_successfully("http://lohas.nicoseiga.jp/o/910aecf08e542285862954017f8a33a8c32a8aec/1433298801/4937663")
|
||||
should_upload_successfully("http://seiga.nicovideo.jp/seiga/im4937663")
|
||||
should_upload_successfully("https://seiga.nicovideo.jp/image/source/9146749")
|
||||
should_upload_successfully("https://seiga.nicovideo.jp/watch/mg389884")
|
||||
should_upload_successfully("https://dic.nicovideo.jp/oekaki/52833.png")
|
||||
should_upload_successfully("https://lohas.nicoseiga.jp/o/971eb8af9bbcde5c2e51d5ef3a2f62d6d9ff5552/1589933964/3583893")
|
||||
should_upload_successfully("http://lohas.nicoseiga.jp/priv/3521156?e=1382558156&h=f2e089256abd1d453a455ec8f317a6c703e2cedf")
|
||||
should_upload_successfully("http://lohas.nicoseiga.jp/priv/b80f86c0d8591b217e7513a9e175e94e00f3c7a1/1384936074/3583893")
|
||||
should_upload_successfully("http://lohas.nicoseiga.jp/material/5746c5/4459092")
|
||||
# XXX should_upload_successfully("https://dcdn.cdn.nimg.jp/priv/62a56a7f67d3d3746ae5712db9cac7d465f4a339/1592186183/10466669")
|
||||
# XXX should_upload_successfully("https://dcdn.cdn.nimg.jp/nicoseiga/lohas/o/8ba0a9b2ea34e1ef3b5cc50785bd10cd63ec7e4a/1592187477/10466669")
|
||||
|
||||
should_upload_successfully("http://nijie.info/view.php?id=213043")
|
||||
should_upload_successfully("https://nijie.info/view_popup.php?id=213043")
|
||||
should_upload_successfully("https://pic.nijie.net/03/nijie_picture/728995_20170505014820_0.jpg")
|
||||
|
||||
should_upload_successfully("https://pawoo.net/web/statuses/1202176")
|
||||
should_upload_successfully("https://img.pawoo.net/media_attachments/files/000/128/953/original/4c0a06087b03343f.png")
|
||||
|
||||
should_upload_successfully("https://www.pixiv.net/en/artworks/64476642")
|
||||
should_upload_successfully("https://i.pximg.net/img-original/img/2017/08/18/00/09/21/64476642_p0.jpg")
|
||||
|
||||
should_upload_successfully("https://noizave.tumblr.com/post/162206271767")
|
||||
should_upload_successfully("https://media.tumblr.com/3bbfcbf075ddf969c996641b264086fd/tumblr_os2buiIOt51wsfqepo1_1280.png")
|
||||
|
||||
should_upload_successfully("https://twitter.com/noizave/status/875768175136317440")
|
||||
should_upload_successfully("https://pbs.twimg.com/media/DCdZ_FhUIAAYKFN?format=jpg&name=medium")
|
||||
should_upload_successfully("https://pbs.twimg.com/profile_banners/1225702850002468864/1588597370/1500x500")
|
||||
# XXX should_upload_successfully("https://video.twimg.com/tweet_video/EWHWVrmVcAAp4Vw.mp4")
|
||||
|
||||
should_upload_successfully("https://www.weibo.com/5501756072/J2UNKfbqV")
|
||||
should_upload_successfully("https://wx1.sinaimg.cn/mw690/0060kO5aly1gezsyt5xvhj30ok0sgtc9.jpg")
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@@ -10,8 +10,11 @@ class WikiPagesControllerTest < ActionDispatch::IntegrationTest
|
||||
context "index action" do
|
||||
setup do
|
||||
as(@user) do
|
||||
@wiki_page_abc = create(:wiki_page, :title => "abc")
|
||||
@wiki_page_def = create(:wiki_page, :title => "def")
|
||||
@tagme = create(:wiki_page, title: "tagme")
|
||||
@deleted = create(:wiki_page, title: "deleted", is_deleted: true)
|
||||
@vocaloid = create(:wiki_page, title: "vocaloid")
|
||||
@miku = create(:wiki_page, title: "hatsune_miku", other_names: ["初音ミク"], body: "miku is a [[vocaloid]]")
|
||||
create(:tag, name: "hatsune_miku", category: Tag.categories.character)
|
||||
end
|
||||
end
|
||||
|
||||
@@ -20,22 +23,24 @@ class WikiPagesControllerTest < ActionDispatch::IntegrationTest
|
||||
assert_response :success
|
||||
end
|
||||
|
||||
should "list all wiki_pages (with search)" do
|
||||
get wiki_pages_path, params: {:search => {:title => "abc"}}
|
||||
assert_response :success
|
||||
assert_select "tr td:first-child", text: "abc"
|
||||
end
|
||||
|
||||
should "list wiki_pages without tags with order=post_count" do
|
||||
get wiki_pages_path, params: {:search => {:title => "abc", :order => "post_count"}}
|
||||
assert_response :success
|
||||
assert_select "tr td:first-child", text: "abc"
|
||||
end
|
||||
|
||||
should "redirect the legacy title param to the show page" do
|
||||
get wiki_pages_path(title: "abc")
|
||||
assert_redirected_to wiki_pages_path(search: { title_normalize: "abc" }, redirect: true)
|
||||
get wiki_pages_path(title: "tagme")
|
||||
assert_redirected_to wiki_pages_path(search: { title_normalize: "tagme" }, redirect: true)
|
||||
end
|
||||
|
||||
should respond_to_search(title: "tagme").with { @tagme }
|
||||
should respond_to_search(title: "tagme", order: "post_count").with { @tagme }
|
||||
should respond_to_search(title_normalize: "TAGME ").with { @tagme }
|
||||
|
||||
should respond_to_search(tag: { category: Tag.categories.character }).with { @miku }
|
||||
should respond_to_search(hide_deleted: "true").with { [@miku, @vocaloid, @tagme] }
|
||||
should respond_to_search(linked_to: "vocaloid").with { @miku }
|
||||
should respond_to_search(not_linked_to: "vocaloid").with { [@vocaloid, @deleted, @tagme] }
|
||||
|
||||
should respond_to_search(other_names_match: "初音ミク").with { @miku }
|
||||
should respond_to_search(other_names_match: "初*").with { @miku }
|
||||
should respond_to_search(other_names_present: "true").with { @miku }
|
||||
should respond_to_search(other_names_present: "false").with { [@vocaloid, @deleted, @tagme] }
|
||||
end
|
||||
|
||||
context "search action" do
|
||||
|
||||
@@ -43,15 +43,15 @@ class ActiveSupport::TestCase
|
||||
|
||||
setup do
|
||||
Socket.stubs(:gethostname).returns("www.example.com")
|
||||
WebMock.allow_net_connect!
|
||||
|
||||
storage_manager = StorageManager::Local.new(base_dir: Dir.mktmpdir("uploads-test-storage-"))
|
||||
@temp_dir = Dir.mktmpdir("danbooru-temp-")
|
||||
storage_manager = StorageManager::Local.new(base_dir: @temp_dir)
|
||||
Danbooru.config.stubs(:storage_manager).returns(storage_manager)
|
||||
Danbooru.config.stubs(:backup_storage_manager).returns(StorageManager::Null.new)
|
||||
end
|
||||
|
||||
teardown do
|
||||
FileUtils.rm_rf(Danbooru.config.storage_manager.base_dir)
|
||||
FileUtils.rm_rf(@temp_dir)
|
||||
Cache.clear
|
||||
end
|
||||
|
||||
@@ -61,6 +61,8 @@ class ActiveSupport::TestCase
|
||||
end
|
||||
|
||||
class ActionDispatch::IntegrationTest
|
||||
extend ControllerHelper
|
||||
|
||||
register_encoder :xml, response_parser: ->(body) { Nokogiri.XML(body) }
|
||||
|
||||
def method_authenticated(method_name, url, user, **options)
|
||||
|
||||
47
test/test_helpers/controller_helper.rb
Normal file
47
test/test_helpers/controller_helper.rb
Normal file
@@ -0,0 +1,47 @@
|
||||
module ControllerHelper
|
||||
# A custom Shoulda matcher that tests that a controller's index endpoint
|
||||
# responds to a search correctly. See https://thoughtbot.com/blog/shoulda-matchers.
|
||||
#
|
||||
# Usage:
|
||||
#
|
||||
# # Tests that `/tags.json?search[name]=touhou` returns the `touhou` tag.
|
||||
# subject { TagsController }
|
||||
# setup { @touhou = create(:tag, name: "touhou") }
|
||||
# should respond_to_search(name: "touhou").with { @touhou }
|
||||
#
|
||||
def respond_to_search(search_params)
|
||||
RespondToSearchMatcher.new(search_params)
|
||||
end
|
||||
|
||||
class RespondToSearchMatcher < Struct.new(:params)
|
||||
def description
|
||||
"should respond to a search for #{params}"
|
||||
end
|
||||
|
||||
def matches?(subject, &block)
|
||||
search_params = { search: params }
|
||||
expected_items = @test_case.instance_eval(&@expected)
|
||||
|
||||
@test_case.instance_eval do
|
||||
# calls e.g. "wiki_pages_path" if we're in WikiPagesControllerTest.
|
||||
index_url = send("#{subject.controller_path}_path")
|
||||
get index_url, as: :json, params: search_params
|
||||
|
||||
expected_ids = Array(expected_items).map(&:id)
|
||||
responded_ids = response.parsed_body.map { |item| item["id"] }
|
||||
|
||||
assert_response :success
|
||||
assert_equal(expected_ids, responded_ids)
|
||||
end
|
||||
end
|
||||
|
||||
def with(&block)
|
||||
@expected = block
|
||||
self
|
||||
end
|
||||
|
||||
def in_context(test_case)
|
||||
@test_case = test_case
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,10 +1,8 @@
|
||||
module DownloadTestHelper
|
||||
def assert_downloaded(expected_filesize, source, referer = nil)
|
||||
download = Downloads::File.new(source, referer)
|
||||
tempfile, strategy = download.download!
|
||||
assert_equal(expected_filesize, tempfile.size, "Tested source URL: #{source}")
|
||||
rescue Net::OpenTimeout
|
||||
skip "Remote connection to #{source} failed"
|
||||
strategy = Sources::Strategies.find(source, referer)
|
||||
file = strategy.download_file!
|
||||
assert_equal(expected_filesize, file.size, "Tested source URL: #{source}")
|
||||
end
|
||||
|
||||
def assert_rewritten(expected_source, test_source, test_referer = nil)
|
||||
@@ -16,19 +14,4 @@ module DownloadTestHelper
|
||||
def assert_not_rewritten(source, referer = nil)
|
||||
assert_rewritten(source, source, referer)
|
||||
end
|
||||
|
||||
def assert_http_exists(url, headers: {})
|
||||
res = HTTParty.head(url, Danbooru.config.httparty_options.deep_merge(headers: headers))
|
||||
assert_equal(true, res.success?)
|
||||
end
|
||||
|
||||
def assert_http_status(code, url, headers: {})
|
||||
res = HTTParty.head(url, Danbooru.config.httparty_options.deep_merge(headers: headers))
|
||||
assert_equal(code, res.code)
|
||||
end
|
||||
|
||||
def assert_http_size(size, url, headers: {})
|
||||
res = HTTParty.head(url, Danbooru.config.httparty_options.deep_merge(headers: headers))
|
||||
assert_equal(size, res.content_length)
|
||||
end
|
||||
end
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
module ReportbooruHelper
|
||||
def mock_request(url, method: :get, status: 200, body: nil, http: Danbooru::Http.any_instance)
|
||||
def mock_request(url, method: :get, status: 200, body: nil, http: Danbooru::Http.any_instance, **options)
|
||||
response = HTTP::Response.new(status: status, body: body, version: "1.1")
|
||||
http.stubs(method).with(url).returns(response)
|
||||
http.stubs(method).with(url, **options).returns(response)
|
||||
end
|
||||
|
||||
def mock_post_search_rankings(date = Date.today, rankings)
|
||||
|
||||
@@ -6,15 +6,11 @@ class ArtistTest < ActiveSupport::TestCase
|
||||
|
||||
assert_equal(1, artists.size)
|
||||
assert_equal(expected_name, artists.first.name, "Testing URL: #{source_url}")
|
||||
rescue Net::OpenTimeout, PixivApiClient::Error
|
||||
skip "Remote connection failed for #{source_url}"
|
||||
end
|
||||
|
||||
def assert_artist_not_found(source_url)
|
||||
artists = ArtistFinder.find_artists(source_url).to_a
|
||||
assert_equal(0, artists.size, "Testing URL: #{source_url}")
|
||||
rescue Net::OpenTimeout
|
||||
skip "Remote connection failed for #{source_url}"
|
||||
end
|
||||
|
||||
context "An artist" do
|
||||
@@ -172,15 +168,11 @@ class ArtistTest < ActiveSupport::TestCase
|
||||
a2 = FactoryBot.create(:artist, :name => "subway", :url_string => "http://subway.com/x/test.jpg")
|
||||
a3 = FactoryBot.create(:artist, :name => "minko", :url_string => "https://minko.com/x/test.jpg")
|
||||
|
||||
begin
|
||||
assert_artist_found("rembrandt", "http://rembrandt.com/x/test.jpg")
|
||||
assert_artist_found("rembrandt", "http://rembrandt.com/x/another.jpg")
|
||||
assert_artist_not_found("http://nonexistent.com/test.jpg")
|
||||
assert_artist_found("minko", "https://minko.com/x/test.jpg")
|
||||
assert_artist_found("minko", "http://minko.com/x/test.jpg")
|
||||
rescue Net::OpenTimeout
|
||||
skip "network failure"
|
||||
end
|
||||
assert_artist_found("rembrandt", "http://rembrandt.com/x/test.jpg")
|
||||
assert_artist_found("rembrandt", "http://rembrandt.com/x/another.jpg")
|
||||
assert_artist_not_found("http://nonexistent.com/test.jpg")
|
||||
assert_artist_found("minko", "https://minko.com/x/test.jpg")
|
||||
assert_artist_found("minko", "http://minko.com/x/test.jpg")
|
||||
end
|
||||
|
||||
should "be case-insensitive to domains when finding matches by url" do
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
require 'test_helper'
|
||||
require 'webmock/minitest'
|
||||
|
||||
class CloudflareServiceTest < ActiveSupport::TestCase
|
||||
def setup
|
||||
@@ -8,16 +7,11 @@ class CloudflareServiceTest < ActiveSupport::TestCase
|
||||
|
||||
context "#purge_cache" do
|
||||
should "make calls to cloudflare's api" do
|
||||
stub_request(:any, "api.cloudflare.com")
|
||||
@cloudflare.purge_cache(["http://localhost/file.txt"])
|
||||
url = "http://www.example.com/file.jpg"
|
||||
mock_request("https://api.cloudflare.com/client/v4/zones/123/purge_cache", method: :delete, json: { files: [url] })
|
||||
|
||||
assert_requested(:delete, "https://api.cloudflare.com/client/v4/zones/123/purge_cache", times: 1)
|
||||
end
|
||||
end
|
||||
|
||||
context "#ips" do
|
||||
should "work" do
|
||||
refute_empty(@cloudflare.ips)
|
||||
response = @cloudflare.purge_cache([url])
|
||||
assert_equal(200, response.status)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@@ -4,18 +4,20 @@ class DanbooruHttpTest < ActiveSupport::TestCase
|
||||
context "Danbooru::Http" do
|
||||
context "#get method" do
|
||||
should "work for all basic methods" do
|
||||
%i[get put post delete].each do |method|
|
||||
%i[get head put post delete].each do |method|
|
||||
response = Danbooru::Http.send(method, "https://httpbin.org/status/200")
|
||||
assert_equal(200, response.status)
|
||||
end
|
||||
end
|
||||
|
||||
should "follow redirects" do
|
||||
skip "Skipping test (https://github.com/postmanlabs/httpbin/issues/617)"
|
||||
response = Danbooru::Http.get("https://httpbin.org/absolute-redirect/3")
|
||||
assert_equal(200, response.status)
|
||||
end
|
||||
|
||||
should "fail if redirected too many times" do
|
||||
skip "Skipping test (https://github.com/postmanlabs/httpbin/issues/617)"
|
||||
response = Danbooru::Http.get("https://httpbin.org/absolute-redirect/10")
|
||||
assert_equal(598, response.status)
|
||||
end
|
||||
@@ -26,8 +28,10 @@ class DanbooruHttpTest < ActiveSupport::TestCase
|
||||
end
|
||||
|
||||
should "fail if the request takes too long to download" do
|
||||
response = Danbooru::Http.timeout(1).get("https://httpbin.org/drip?duration=5&numbytes=5")
|
||||
assert_equal(599, response.status)
|
||||
# XXX should return status 599 instead
|
||||
assert_raises(HTTP::TimeoutError) do
|
||||
response = Danbooru::Http.timeout(1).get("https://httpbin.org/drip?duration=10&numbytes=10").flush
|
||||
end
|
||||
end
|
||||
|
||||
should "automatically decompress gzipped responses" do
|
||||
@@ -36,13 +40,131 @@ class DanbooruHttpTest < ActiveSupport::TestCase
|
||||
assert_equal(true, response.parse["gzipped"])
|
||||
end
|
||||
|
||||
should "cache requests" do
|
||||
response1 = Danbooru::Http.cache(1.minute).get("https://httpbin.org/uuid")
|
||||
should "automatically parse html responses" do
|
||||
response = Danbooru::Http.get("https://httpbin.org/html")
|
||||
assert_equal(200, response.status)
|
||||
assert_instance_of(Nokogiri::HTML5::Document, response.parse)
|
||||
assert_equal("Herman Melville - Moby-Dick", response.parse.css("h1").text)
|
||||
end
|
||||
|
||||
should "automatically parse xml responses" do
|
||||
response = Danbooru::Http.get("https://httpbin.org/xml")
|
||||
assert_equal(200, response.status)
|
||||
assert_equal(true, response.parse[:slideshow].present?)
|
||||
end
|
||||
|
||||
should "track cookies between requests" do
|
||||
http = Danbooru::Http.use(:session)
|
||||
|
||||
resp1 = http.get("https://httpbin.org/cookies/set/abc/1")
|
||||
resp2 = http.get("https://httpbin.org/cookies/set/def/2")
|
||||
resp3 = http.get("https://httpbin.org/cookies")
|
||||
assert_equal({ abc: "1", def: "2" }, resp3.parse["cookies"].symbolize_keys)
|
||||
|
||||
resp4 = http.cookies(def: 3, ghi: 4).get("https://httpbin.org/cookies")
|
||||
assert_equal({ abc: "1", def: "3", ghi: "4" }, resp4.parse["cookies"].symbolize_keys)
|
||||
end
|
||||
end
|
||||
|
||||
context "cache feature" do
|
||||
should "cache multiple requests to the same url" do
|
||||
http = Danbooru::Http.cache(1.hour)
|
||||
|
||||
response1 = http.get("https://httpbin.org/uuid")
|
||||
assert_equal(200, response1.status)
|
||||
|
||||
response2 = Danbooru::Http.cache(1.minute).get("https://httpbin.org/uuid")
|
||||
response2 = http.get("https://httpbin.org/uuid")
|
||||
assert_equal(200, response2.status)
|
||||
assert_equal(response2.body, response1.body)
|
||||
assert_equal(response2.to_s, response1.to_s)
|
||||
end
|
||||
|
||||
should "cache cookies correctly" do
|
||||
http = Danbooru::Http.cache(1.hour)
|
||||
|
||||
resp1 = http.get("https://httpbin.org/cookies")
|
||||
resp2 = http.get("https://httpbin.org/cookies/set/abc/1")
|
||||
resp3 = http.get("https://httpbin.org/cookies/set/def/2")
|
||||
resp4 = http.get("https://httpbin.org/cookies")
|
||||
|
||||
assert_equal(200, resp1.status)
|
||||
assert_equal(200, resp2.status)
|
||||
assert_equal(200, resp3.status)
|
||||
assert_equal(200, resp4.status)
|
||||
|
||||
assert_equal({}, resp1.parse["cookies"].symbolize_keys)
|
||||
assert_equal({ abc: "1" }, resp2.parse["cookies"].symbolize_keys)
|
||||
assert_equal({ abc: "1", def: "2" }, resp3.parse["cookies"].symbolize_keys)
|
||||
assert_equal({ abc: "1", def: "2" }, resp4.parse["cookies"].symbolize_keys)
|
||||
end
|
||||
end
|
||||
|
||||
context "retriable feature" do
|
||||
should "retry immediately if no Retry-After header is sent" do
|
||||
response_429 = ::HTTP::Response.new(status: 429, version: "1.1", body: "")
|
||||
response_200 = ::HTTP::Response.new(status: 200, version: "1.1", body: "")
|
||||
HTTP::Client.any_instance.expects(:perform).times(2).returns(response_429, response_200)
|
||||
|
||||
response = Danbooru::Http.use(:retriable).get("https://httpbin.org/status/429")
|
||||
assert_equal(200, response.status)
|
||||
end
|
||||
|
||||
should "retry if the Retry-After header is an integer" do
|
||||
response_503 = ::HTTP::Response.new(status: 503, version: "1.1", headers: { "Retry-After": "1" }, body: "")
|
||||
response_200 = ::HTTP::Response.new(status: 200, version: "1.1", body: "")
|
||||
HTTP::Client.any_instance.expects(:perform).times(2).returns(response_503, response_200)
|
||||
|
||||
response = Danbooru::Http.use(:retriable).get("https://httpbin.org/status/503")
|
||||
assert_equal(200, response.status)
|
||||
end
|
||||
|
||||
should "retry if the Retry-After header is a date" do
|
||||
response_503 = ::HTTP::Response.new(status: 503, version: "1.1", headers: { "Retry-After": 2.seconds.from_now.httpdate }, body: "")
|
||||
response_200 = ::HTTP::Response.new(status: 200, version: "1.1", body: "")
|
||||
HTTP::Client.any_instance.expects(:perform).times(2).returns(response_503, response_200)
|
||||
|
||||
response = Danbooru::Http.use(:retriable).get("https://httpbin.org/status/503")
|
||||
assert_equal(200, response.status)
|
||||
end
|
||||
end
|
||||
|
||||
context "#download method" do
|
||||
should "download files" do
|
||||
response, file = Danbooru::Http.download_media("https://httpbin.org/bytes/1000")
|
||||
|
||||
assert_equal(200, response.status)
|
||||
assert_equal(1000, file.size)
|
||||
end
|
||||
|
||||
should "follow redirects when downloading files" do
|
||||
skip "Skipping test (https://github.com/postmanlabs/httpbin/issues/617)"
|
||||
response, file = Danbooru::Http.download_media("https://httpbin.org/redirect-to?url=https://httpbin.org/bytes/1000")
|
||||
|
||||
assert_equal(200, response.status)
|
||||
assert_equal(1000, file.size)
|
||||
end
|
||||
|
||||
should "fail if the url points to a private IP" do
|
||||
assert_raises(Danbooru::Http::DownloadError) do
|
||||
Danbooru::Http.public_only.download_media("https://127.0.0.1.xip.io")
|
||||
end
|
||||
end
|
||||
|
||||
should "fail if the url redirects to a private IP" do
|
||||
assert_raises(Danbooru::Http::DownloadError) do
|
||||
Danbooru::Http.public_only.download_media("https://httpbin.org/redirect-to?url=https://127.0.0.1.xip.io")
|
||||
end
|
||||
end
|
||||
|
||||
should "fail if a download is too large" do
|
||||
assert_raises(Danbooru::Http::FileTooLargeError) do
|
||||
response, file = Danbooru::Http.max_size(500).download_media("https://httpbin.org/bytes/1000")
|
||||
end
|
||||
end
|
||||
|
||||
should "fail if a streaming download is too large" do
|
||||
assert_raises(Danbooru::Http::FileTooLargeError) do
|
||||
response, file = Danbooru::Http.max_size(500).download_media("https://httpbin.org/stream-bytes/1000")
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@@ -3,53 +3,27 @@ require 'test_helper'
|
||||
module Downloads
|
||||
class ArtStationTest < ActiveSupport::TestCase
|
||||
context "a download for a (small) artstation image" do
|
||||
setup do
|
||||
@asset = "https://cdnb3.artstation.com/p/assets/images/images/003/716/071/small/aoi-ogata-hate-city.jpg?1476754974"
|
||||
@download = Downloads::File.new(@asset)
|
||||
end
|
||||
|
||||
should "download the /4k/ image instead" do
|
||||
file, strategy = @download.download!
|
||||
assert_equal(1_880_910, ::File.size(file.path))
|
||||
assert_downloaded(1_880_910, "https://cdnb3.artstation.com/p/assets/images/images/003/716/071/small/aoi-ogata-hate-city.jpg?1476754974")
|
||||
end
|
||||
end
|
||||
|
||||
context "for an image where an original does not exist" do
|
||||
setup do
|
||||
@asset = "https://cdna.artstation.com/p/assets/images/images/004/730/278/large/mendel-oh-dragonll.jpg"
|
||||
@download = Downloads::File.new(@asset)
|
||||
end
|
||||
|
||||
should "not try to download the original" do
|
||||
file, strategy = @download.download!
|
||||
assert_equal(483_192, ::File.size(file.path))
|
||||
assert_downloaded(483_192, "https://cdna.artstation.com/p/assets/images/images/004/730/278/large/mendel-oh-dragonll.jpg")
|
||||
end
|
||||
end
|
||||
|
||||
context "a download for an ArtStation image hosted on CloudFlare" do
|
||||
setup do
|
||||
@asset = "https://cdnb.artstation.com/p/assets/images/images/003/716/071/large/aoi-ogata-hate-city.jpg?1476754974"
|
||||
end
|
||||
|
||||
should "return the original file, not the polished file" do
|
||||
@asset = "https://cdnb.artstation.com/p/assets/images/images/003/716/071/large/aoi-ogata-hate-city.jpg?1476754974"
|
||||
assert_downloaded(1_880_910, @asset)
|
||||
end
|
||||
|
||||
should "return the original filesize, not the polished filesize" do
|
||||
assert_equal(1_880_910, Downloads::File.new(@asset).size)
|
||||
end
|
||||
end
|
||||
|
||||
context "a download for a https://$artist.artstation.com/projects/$id page" do
|
||||
setup do
|
||||
@source = "https://dantewontdie.artstation.com/projects/YZK5q"
|
||||
@download = Downloads::File.new(@source)
|
||||
end
|
||||
|
||||
should "download the original image instead" do
|
||||
file, strategy = @download.download!
|
||||
|
||||
assert_equal(247_350, ::File.size(file.path))
|
||||
assert_downloaded(247_350, "https://dantewontdie.artstation.com/projects/YZK5q")
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@@ -1,81 +0,0 @@
|
||||
require 'test_helper'
|
||||
|
||||
module Downloads
|
||||
class FileTest < ActiveSupport::TestCase
|
||||
context "A post download" do
|
||||
setup do
|
||||
@source = "http://www.google.com/intl/en_ALL/images/logo.gif"
|
||||
@download = Downloads::File.new(@source)
|
||||
end
|
||||
|
||||
context "for a banned IP" do
|
||||
setup do
|
||||
Resolv.expects(:getaddress).returns("127.0.0.1").at_least_once
|
||||
end
|
||||
|
||||
should "not try to download the file" do
|
||||
assert_raise(Downloads::File::Error) { Downloads::File.new("http://evil.com").download! }
|
||||
end
|
||||
|
||||
should "not try to fetch the size" do
|
||||
assert_raise(Downloads::File::Error) { Downloads::File.new("http://evil.com").size }
|
||||
end
|
||||
|
||||
should "not follow redirects to banned IPs" do
|
||||
url = "http://httpbin.org/redirect-to?url=http://127.0.0.1"
|
||||
stub_request(:get, url).to_return(status: 301, headers: { "Location": "http://127.0.0.1" })
|
||||
|
||||
assert_raise(Downloads::File::Error) { Downloads::File.new(url).download! }
|
||||
end
|
||||
|
||||
should "not follow redirects that resolve to a banned IP" do
|
||||
url = "http://httpbin.org/redirect-to?url=http://127.0.0.1.nip.io"
|
||||
stub_request(:get, url).to_return(status: 301, headers: { "Location": "http://127.0.0.1.xip.io" })
|
||||
|
||||
assert_raise(Downloads::File::Error) { Downloads::File.new(url).download! }
|
||||
end
|
||||
end
|
||||
|
||||
context "that fails" do
|
||||
should "retry three times before giving up" do
|
||||
HTTParty.expects(:get).times(3).raises(Errno::ETIMEDOUT)
|
||||
assert_raises(Errno::ETIMEDOUT) { @download.download! }
|
||||
end
|
||||
|
||||
should "return an uncorrupted file on the second try" do
|
||||
bomb = stub("bomb")
|
||||
bomb.stubs(:code).raises(IOError)
|
||||
resp = stub("resp", success?: true)
|
||||
|
||||
chunk = stub("a")
|
||||
chunk.stubs(:code).returns(200)
|
||||
chunk.stubs(:size).returns(1)
|
||||
chunk.stubs(:to_s).returns("a")
|
||||
|
||||
HTTParty.expects(:get).twice.multiple_yields(chunk, bomb).then.multiple_yields(chunk, chunk).returns(resp)
|
||||
@download.stubs(:is_cloudflare?).returns(false)
|
||||
tempfile, _strategy = @download.download!
|
||||
|
||||
assert_equal("aa", tempfile.read)
|
||||
end
|
||||
end
|
||||
|
||||
should "throw an exception when the file is larger than the maximum" do
|
||||
assert_raise(Downloads::File::Error) do
|
||||
@download.download!(max_size: 1)
|
||||
end
|
||||
end
|
||||
|
||||
should "store the file in the tempfile path" do
|
||||
tempfile, strategy = @download.download!
|
||||
assert_operator(tempfile.size, :>, 0, "should have data")
|
||||
end
|
||||
|
||||
should "correctly save the file when following 302 redirects" do
|
||||
download = Downloads::File.new("https://yande.re/post/show/578014")
|
||||
file, strategy = download.download!(url: download.preview_url)
|
||||
assert_equal(19134, file.size)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -122,32 +122,17 @@ module Downloads
|
||||
assert_downloaded(@file_size, @file_url, @ref)
|
||||
end
|
||||
end
|
||||
|
||||
context "downloading a pixiv fanbox image" do
|
||||
should_eventually "work" do
|
||||
@source = "https://www.pixiv.net/fanbox/creator/12491073/post/82406"
|
||||
@file_url = "https://fanbox.pixiv.net/images/post/82406/D833IKA7FIesJXL8xx39rrG0.jpeg"
|
||||
@file_size = 873_387
|
||||
|
||||
assert_not_rewritten(@file_url, @source)
|
||||
assert_downloaded(@file_size, @file_url, @source)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
context "An ugoira site for pixiv" do
|
||||
setup do
|
||||
@download = Downloads::File.new("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364")
|
||||
@tempfile, strategy = @download.download!
|
||||
@tempfile.close!
|
||||
end
|
||||
|
||||
should "capture the data" do
|
||||
assert_equal(2, @download.data[:ugoira_frame_data].size)
|
||||
if @download.data[:ugoira_frame_data][0]["file"]
|
||||
@strategy = Sources::Strategies.find("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364")
|
||||
|
||||
assert_equal(2, @strategy.data[:ugoira_frame_data].size)
|
||||
if @strategy.data[:ugoira_frame_data][0]["file"]
|
||||
assert_equal([{"file" => "000000.jpg", "delay" => 125}, {"file" => "000001.jpg", "delay" => 125}], @download.data[:ugoira_frame_data])
|
||||
else
|
||||
assert_equal([{"delay_msec" => 125}, {"delay_msec" => 125}], @download.data[:ugoira_frame_data])
|
||||
assert_equal([{"delay_msec" => 125}, {"delay_msec" => 125}], @strategy.data[:ugoira_frame_data])
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@@ -98,6 +98,10 @@ class MediaFileTest < ActiveSupport::TestCase
|
||||
should "determine the correct extension for a flash file" do
|
||||
assert_equal(:swf, MediaFile.open("test/files/compressed.swf").file_ext)
|
||||
end
|
||||
|
||||
should "not fail for empty files" do
|
||||
assert_equal(:bin, MediaFile.open("test/files/test-empty.bin").file_ext)
|
||||
end
|
||||
end
|
||||
|
||||
should "determine the correct md5 for a jpeg file" do
|
||||
|
||||
@@ -277,6 +277,19 @@ class PostQueryBuilderTest < ActiveSupport::TestCase
|
||||
assert_tag_match([child, parent], "-child:garbage")
|
||||
end
|
||||
|
||||
should "return posts when using the status of the parent/child" do
|
||||
parent_of_deleted = create(:post)
|
||||
deleted = create(:post, is_deleted: true, tag_string: "parent:#{parent_of_deleted.id}")
|
||||
child_of_deleted = create(:post, tag_string: "parent:#{deleted.id}")
|
||||
all = [child_of_deleted, deleted, parent_of_deleted]
|
||||
|
||||
assert_tag_match([child_of_deleted], "parent:deleted")
|
||||
assert_tag_match(all - [child_of_deleted], "-parent:deleted")
|
||||
|
||||
assert_tag_match([parent_of_deleted], "child:deleted")
|
||||
assert_tag_match(all - [parent_of_deleted], "-child:deleted")
|
||||
end
|
||||
|
||||
should "return posts for the favgroup:<name> metatag" do
|
||||
post1 = create(:post)
|
||||
post2 = create(:post)
|
||||
@@ -757,8 +770,8 @@ class PostQueryBuilderTest < ActiveSupport::TestCase
|
||||
create(:saved_search, query: "aaa", labels: ["zzz"], user: CurrentUser.user)
|
||||
create(:saved_search, query: "bbb", user: CurrentUser.user)
|
||||
|
||||
Redis.any_instance.stubs(:exists).with("search:aaa").returns(true)
|
||||
Redis.any_instance.stubs(:exists).with("search:bbb").returns(true)
|
||||
Redis.any_instance.stubs(:exists?).with("search:aaa").returns(true)
|
||||
Redis.any_instance.stubs(:exists?).with("search:bbb").returns(true)
|
||||
Redis.any_instance.stubs(:smembers).with("search:aaa").returns([@post1.id])
|
||||
Redis.any_instance.stubs(:smembers).with("search:bbb").returns([@post2.id])
|
||||
|
||||
|
||||
@@ -4,20 +4,20 @@ class ReportbooruServiceTest < ActiveSupport::TestCase
|
||||
def setup
|
||||
@service = ReportbooruService.new(reportbooru_server: "http://localhost:1234")
|
||||
@post = create(:post)
|
||||
@date = "2000-01-01"
|
||||
@date = Date.parse("2000-01-01")
|
||||
end
|
||||
|
||||
context "#popular_posts" do
|
||||
should "return the list of popular posts on success" do
|
||||
body = "[[#{@post.id},100.0]]"
|
||||
@service.http.expects(:get).with("http://localhost:1234/post_views/rank?date=#{@date}").returns(HTTP::Response.new(status: 200, body: body, version: "1.1"))
|
||||
mock_post_view_rankings(@date, [[@post.id, 100]])
|
||||
|
||||
posts = @service.popular_posts(@date)
|
||||
assert_equal([@post], posts)
|
||||
end
|
||||
|
||||
should "return nothing on failure" do
|
||||
@service.http.expects(:get).with("http://localhost:1234/post_views/rank?date=#{@date}").returns(HTTP::Response.new(status: 500, body: "", version: "1.1"))
|
||||
Danbooru::Http.any_instance.expects(:get).with("http://localhost:1234/post_views/rank?date=#{@date}").returns(HTTP::Response.new(status: 500, body: "", version: "1.1"))
|
||||
Danbooru::Http.any_instance.expects(:get).with("http://localhost:1234/post_views/rank?date=#{@date.yesterday}").returns(HTTP::Response.new(status: 500, body: "", version: "1.1"))
|
||||
|
||||
assert_equal([], @service.popular_posts(@date))
|
||||
end
|
||||
|
||||
@@ -43,7 +43,7 @@ module Sources
|
||||
|
||||
should "get the image url" do
|
||||
assert_equal("https://pic.nijie.net/03/nijie_picture/728995_20170505014820_0.jpg", @site.image_url)
|
||||
assert_http_size(132_555, @site.image_url)
|
||||
assert_downloaded(132_555, @site.image_url)
|
||||
end
|
||||
|
||||
should "get the canonical url" do
|
||||
@@ -53,7 +53,7 @@ module Sources
|
||||
should "get the preview url" do
|
||||
assert_equal("https://pic.nijie.net/03/__rs_l170x170/nijie_picture/728995_20170505014820_0.jpg", @site.preview_url)
|
||||
assert_equal([@site.preview_url], @site.preview_urls)
|
||||
assert_http_exists(@site.preview_url)
|
||||
assert_downloaded(132_555, @site.preview_url)
|
||||
end
|
||||
|
||||
should "get the profile" do
|
||||
@@ -187,8 +187,6 @@ module Sources
|
||||
desc = <<-EOS.strip_heredoc.chomp
|
||||
foo [b]bold[/b] [i]italics[/i] [s]strike[/s] red
|
||||
|
||||
|
||||
|
||||
<http://nijie.info/view.php?id=218944>
|
||||
EOS
|
||||
|
||||
@@ -207,8 +205,8 @@ module Sources
|
||||
assert_equal("https://nijie.info/members.php?id=236014", site.profile_url)
|
||||
assert_nothing_raised { site.to_h }
|
||||
|
||||
assert_http_size(3619, site.image_url)
|
||||
assert_http_exists(site.preview_url)
|
||||
assert_downloaded(3619, site.image_url)
|
||||
assert_downloaded(3619, site.preview_url)
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
@@ -15,10 +15,7 @@ module Sources
|
||||
|
||||
def get_source(source)
|
||||
@site = Sources::Strategies.find(source)
|
||||
|
||||
@site
|
||||
rescue Net::OpenTimeout
|
||||
skip "Remote connection to #{source} failed"
|
||||
end
|
||||
|
||||
context "in all cases" do
|
||||
@@ -73,17 +70,6 @@ module Sources
|
||||
end
|
||||
end
|
||||
|
||||
context "A https://www.pixiv.net/fanbox/creator/*/post/* source" do
|
||||
should_eventually "work" do
|
||||
@site = Sources::Strategies.find("http://www.pixiv.net/fanbox/creator/554149/post/82555")
|
||||
|
||||
assert_equal("TYONE(お仕事募集中)", @site.artist_name)
|
||||
assert_equal("https://www.pixiv.net/users/554149", @site.profile_url)
|
||||
assert_equal("https://fanbox.pixiv.net/images/post/82555/Lyyeb6dDLcQZmy09nqLZapuS.jpeg", @site.image_url)
|
||||
assert_nothing_raised { @site.to_h }
|
||||
end
|
||||
end
|
||||
|
||||
context "A https://www.pixiv.net/*/artworks/* source" do
|
||||
should "work" do
|
||||
@site = Sources::Strategies.find("https://www.pixiv.net/en/artworks/64476642")
|
||||
@@ -249,6 +235,10 @@ module Sources
|
||||
assert_includes(@translated_tags, "foo")
|
||||
end
|
||||
|
||||
should "not translate tags for digital media" do
|
||||
assert_equal(false, @tags.include?("Photoshop"))
|
||||
end
|
||||
|
||||
should "normalize 10users入り tags" do
|
||||
assert_includes(@tags, "風景10users入り")
|
||||
assert_includes(@translated_tags, "scenery")
|
||||
@@ -280,7 +270,7 @@ module Sources
|
||||
should "not translate '1000users入り' to '1'" do
|
||||
FactoryBot.create(:tag, name: "1", post_count: 1)
|
||||
source = get_source("https://www.pixiv.net/member_illust.php?mode=medium&illust_id=60665428")
|
||||
tags = %w[1000users入り Fate/GrandOrder アルジュナ(Fate) アルトリア・ペンドラゴン イシュタル(Fate) グランブルーファンタジー マシュ・キリエライト マーリン(Fate) 両儀式 手袋 CLIP\ STUDIO\ PAINT Photoshop]
|
||||
tags = %w[1000users入り Fate/GrandOrder アルジュナ(Fate) アルトリア・ペンドラゴン イシュタル(Fate) グランブルーファンタジー マシュ・キリエライト マーリン(Fate) 両儀式 手袋]
|
||||
|
||||
assert_equal(tags.sort, source.tags.map(&:first).sort)
|
||||
assert_equal(["fate/grand_order"], source.translated_tags.map(&:name))
|
||||
|
||||
@@ -169,7 +169,7 @@ module Sources
|
||||
|
||||
context "The source for a 'http://ve.media.tumblr.com/*' video post with inline images" do
|
||||
setup do
|
||||
@url = "https://ve.media.tumblr.com/tumblr_os31dkexhK1wsfqep.mp4"
|
||||
@url = "https://va.media.tumblr.com/tumblr_os31dkexhK1wsfqep.mp4"
|
||||
@ref = "https://noizave.tumblr.com/post/162222617101"
|
||||
end
|
||||
|
||||
@@ -177,7 +177,7 @@ module Sources
|
||||
should "get the video and inline images" do
|
||||
site = Sources::Strategies.find(@url, @ref)
|
||||
urls = %w[
|
||||
https://ve.media.tumblr.com/tumblr_os31dkexhK1wsfqep.mp4
|
||||
https://va.media.tumblr.com/tumblr_os31dkexhK1wsfqep.mp4
|
||||
https://media.tumblr.com/afed9f5b3c33c39dc8c967e262955de2/tumblr_inline_os31dclyCR1v11u29_1280.png
|
||||
]
|
||||
|
||||
|
||||
@@ -244,6 +244,14 @@ module Sources
|
||||
end
|
||||
end
|
||||
|
||||
context "A profile banner image" do
|
||||
should "work" do
|
||||
@site = Sources::Strategies.find("https://pbs.twimg.com/profile_banners/1225702850002468864/1588597370/1500x500")
|
||||
assert_equal(@site.image_url, @site.url)
|
||||
assert_nothing_raised { @site.to_h }
|
||||
end
|
||||
end
|
||||
|
||||
context "A tweet containing non-normalized Unicode text" do
|
||||
should "be normalized to nfkc" do
|
||||
site = Sources::Strategies.find("https://twitter.com/aprilarcus/status/367557195186970624")
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user