Merge branch 'master' into fix-pixiv-profile-url

.bundle/config

@@ -0,0 +1,3 @@
+---
+BUNDLE_BUILD__NOKOGIRI: "--use-system-libraries"
+BUNDLE_BUILD__NOKOGUMBO: "--without-libxml2"

.editorconfig

@@ -0,0 +1,14 @@
+root = true
+
+[**.{js,rb,css,erb,md,json,yml}]
+indent_style = space
+indent_size = 2
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+[**.html.erb]
+indent_size = unset
+
+[app/javascript/vendor/**]
+indent_size = unset

.eslintrc.yml

@@ -8,9 +8,13 @@ parserOptions:
 globals:
   $: false
   require: false
+parser: babel-eslint
+plugins:
+  - babel
 rules:
   # https://eslint.org/docs/rules/
   array-callback-return: error
+  babel/no-unused-expressions: error
   block-scoped-var: error
   consistent-return: error
   default-case: error
@@ -32,7 +36,7 @@ rules:
   no-sequences: error
   no-shadow: error
   no-shadow-restricted-names: error
-  no-unused-expressions: error
+  #no-unused-expressions: error
   no-unused-vars:
     - error
     - argsIgnorePattern: "^_"

.github/workflows/test.yaml

@@ -64,7 +64,7 @@ jobs:
     - name: Install OS dependencies
       run: |
         apt-get update
-        apt-get -y install --no-install-recommends build-essential ruby ruby-dev ruby-bundler git nodejs yarnpkg webpack ffmpeg mkvtoolnix libvips-dev libxml2-dev postgresql-server-dev-all wget
+        apt-get -y install --no-install-recommends build-essential ruby ruby-dev ruby-bundler git nodejs yarnpkg webpack ffmpeg mkvtoolnix libvips-dev libxml2-dev libxslt-dev zlib1g-dev postgresql-server-dev-all wget
         ln -sf /usr/bin/yarnpkg /usr/bin/yarn
 
     - name: Install Ruby dependencies

.gitignore

@@ -2,7 +2,6 @@
 .yarn-integrity
 .gitconfig
 .git/
-.bundle/
 config/database.yml
 config/danbooru_local_config.rb
 config/deploy/*.rb

Gemfile

@@ -7,7 +7,6 @@ gem "pg"
 gem "delayed_job"
 gem "delayed_job_active_record"
 gem "simple_form"
-gem "mechanize"
 gem "whenever", :require => false
 gem "sanitize"
 gem 'ruby-vips'
@@ -28,7 +27,6 @@ gem 'daemons'
 gem 'oauth2'
 gem 'bootsnap'
 gem 'addressable'
-gem 'httparty'
 gem 'rakismet'
 gem 'recaptcha', require: "recaptcha/rails"
 gem 'activemodel-serializers-xml'
@@ -47,9 +45,7 @@ gem 'http'
 gem 'activerecord-hierarchical_query'
 gem 'pundit'
 gem 'mail'
-
+gem 'nokogiri'
-# locked to 1.10.9 to workaround an incompatibility with nokogumbo 2.0.2.
-gem 'nokogiri', '~> 1.10.9'
 
 group :production, :staging do
   gem 'unicorn', :platforms => :ruby
@@ -65,7 +61,6 @@ end
 group :development do
   gem 'rubocop'
   gem 'rubocop-rails'
-  gem 'sinatra'
   gem 'meta_request'
   gem 'rack-mini-profiler'
   gem 'stackprof'
@@ -86,7 +81,6 @@ group :test do
   gem "mocha", require: "mocha/minitest"
   gem "ffaker"
   gem "simplecov", "~> 0.17.0", require: false
-  gem "webmock", require: "webmock/minitest"
   gem "minitest-ci"
   gem "minitest-reporters", require: "minitest/reporters"
   gem "mock_redis"

Gemfile.lock

@@ -8,63 +8,63 @@ GIT
 GEM
   remote: https://rubygems.org/
   specs:
-    actioncable (6.0.3.1)
+    actioncable (6.0.3.2)
-      actionpack (= 6.0.3.1)
+      actionpack (= 6.0.3.2)
       nio4r (~> 2.0)
       websocket-driver (>= 0.6.1)
-    actionmailbox (6.0.3.1)
+    actionmailbox (6.0.3.2)
-      actionpack (= 6.0.3.1)
+      actionpack (= 6.0.3.2)
-      activejob (= 6.0.3.1)
+      activejob (= 6.0.3.2)
-      activerecord (= 6.0.3.1)
+      activerecord (= 6.0.3.2)
-      activestorage (= 6.0.3.1)
+      activestorage (= 6.0.3.2)
-      activesupport (= 6.0.3.1)
+      activesupport (= 6.0.3.2)
       mail (>= 2.7.1)
-    actionmailer (6.0.3.1)
+    actionmailer (6.0.3.2)
-      actionpack (= 6.0.3.1)
+      actionpack (= 6.0.3.2)
-      actionview (= 6.0.3.1)
+      actionview (= 6.0.3.2)
-      activejob (= 6.0.3.1)
+      activejob (= 6.0.3.2)
       mail (~> 2.5, >= 2.5.4)
       rails-dom-testing (~> 2.0)
-    actionpack (6.0.3.1)
+    actionpack (6.0.3.2)
-      actionview (= 6.0.3.1)
+      actionview (= 6.0.3.2)
-      activesupport (= 6.0.3.1)
+      activesupport (= 6.0.3.2)
       rack (~> 2.0, >= 2.0.8)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.2.0)
-    actiontext (6.0.3.1)
+    actiontext (6.0.3.2)
-      actionpack (= 6.0.3.1)
+      actionpack (= 6.0.3.2)
-      activerecord (= 6.0.3.1)
+      activerecord (= 6.0.3.2)
-      activestorage (= 6.0.3.1)
+      activestorage (= 6.0.3.2)
-      activesupport (= 6.0.3.1)
+      activesupport (= 6.0.3.2)
       nokogiri (>= 1.8.5)
-    actionview (6.0.3.1)
+    actionview (6.0.3.2)
-      activesupport (= 6.0.3.1)
+      activesupport (= 6.0.3.2)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.1, >= 1.2.0)
-    activejob (6.0.3.1)
+    activejob (6.0.3.2)
-      activesupport (= 6.0.3.1)
+      activesupport (= 6.0.3.2)
       globalid (>= 0.3.6)
-    activemodel (6.0.3.1)
+    activemodel (6.0.3.2)
-      activesupport (= 6.0.3.1)
+      activesupport (= 6.0.3.2)
     activemodel-serializers-xml (1.0.2)
       activemodel (> 5.x)
       activesupport (> 5.x)
       builder (~> 3.1)
-    activerecord (6.0.3.1)
+    activerecord (6.0.3.2)
-      activemodel (= 6.0.3.1)
+      activemodel (= 6.0.3.2)
-      activesupport (= 6.0.3.1)
+      activesupport (= 6.0.3.2)
     activerecord-hierarchical_query (1.2.3)
       activerecord (>= 5.0, < 6.1)
       pg (>= 0.21, < 1.3)
-    activestorage (6.0.3.1)
+    activestorage (6.0.3.2)
-      actionpack (= 6.0.3.1)
+      actionpack (= 6.0.3.2)
-      activejob (= 6.0.3.1)
+      activejob (= 6.0.3.2)
-      activerecord (= 6.0.3.1)
+      activerecord (= 6.0.3.2)
       marcel (~> 0.3.1)
-    activesupport (6.0.3.1)
+    activesupport (6.0.3.2)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
@@ -77,7 +77,7 @@ GEM
     ansi (1.5.0)
     ast (2.4.1)
     aws-eventstream (1.1.0)
-    aws-partitions (1.329.0)
+    aws-partitions (1.332.0)
     aws-sdk-core (3.100.0)
       aws-eventstream (~> 1, >= 1.0.2)
       aws-partitions (~> 1, >= 1.239.0)
@@ -86,8 +86,8 @@ GEM
     aws-sdk-sqs (1.27.1)
       aws-sdk-core (~> 3, >= 3.99.0)
       aws-sigv4 (~> 1.1)
-    aws-sigv4 (1.1.4)
+    aws-sigv4 (1.2.0)
-      aws-eventstream (~> 1.0, >= 1.0.2)
+      aws-eventstream (~> 1, >= 1.0.2)
     bcrypt (3.1.13)
     bootsnap (1.4.6)
       msgpack (~> 1.0)
@@ -110,7 +110,7 @@ GEM
       sshkit (~> 1.3)
     capistrano3-unicorn (0.2.1)
       capistrano (~> 3.1, >= 3.1.0)
-    capybara (3.32.2)
+    capybara (3.33.0)
       addressable
       mini_mime (>= 0.1.3)
       nokogiri (~> 1.8)
@@ -122,9 +122,6 @@ GEM
     chronic (0.10.2)
     coderay (1.1.3)
     concurrent-ruby (1.1.6)
-    connection_pool (2.2.3)
-    crack (0.4.3)
-      safe_yaml (~> 1.0.0)
     crass (1.0.6)
     daemons (1.3.1)
     delayed_job (4.1.8)
@@ -141,8 +138,8 @@ GEM
       dotenv (= 2.7.5)
       railties (>= 3.2, < 6.1)
     erubi (1.9.0)
-    factory_bot (5.2.0)
+    factory_bot (6.0.2)
-      activesupport (>= 4.2.0)
+      activesupport (>= 5.0.0)
     faraday (1.0.1)
       multipart-post (>= 1.2, < 3)
     ffaker (2.15.0)
@@ -156,7 +153,6 @@ GEM
       ffi (~> 1.0)
     globalid (0.4.2)
       activesupport (>= 4.2.0)
-    hashdiff (1.0.1)
     http (4.4.1)
       addressable (~> 2.3)
       http-cookie (~> 1.0)
@@ -167,9 +163,6 @@ GEM
     http-form_data (2.3.0)
     http-parser (1.2.1)
       ffi-compiler (>= 1.0, < 2.0)
-    httparty (0.18.1)
-      mime-types (~> 3.0)
-      multi_xml (>= 0.5.2)
     i18n (1.8.3)
       concurrent-ruby (~> 1.0)
     ipaddress_2 (0.13.0)
@@ -184,34 +177,22 @@ GEM
     listen (3.2.1)
       rb-fsevent (~> 0.10, >= 0.10.3)
       rb-inotify (~> 0.9, >= 0.9.10)
-    loofah (2.5.0)
+    loofah (2.6.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
     mail (2.7.1)
       mini_mime (>= 0.1.1)
     marcel (0.3.3)
       mimemagic (~> 0.3.2)
-    mechanize (2.7.6)
-      domain_name (~> 0.5, >= 0.5.1)
-      http-cookie (~> 1.0)
-      mime-types (>= 1.17.2)
-      net-http-digest_auth (~> 1.1, >= 1.1.1)
-      net-http-persistent (>= 2.5.2)
-      nokogiri (~> 1.6)
-      ntlm-http (~> 0.1, >= 0.1.1)
-      webrobots (>= 0.0.9, < 0.2)
     memoist (0.16.2)
     memory_profiler (0.9.14)
     meta_request (0.7.2)
       rack-contrib (>= 1.1, < 3)
       railties (>= 3.0.0, < 7)
     method_source (1.0.0)
-    mime-types (3.3.1)
-      mime-types-data (~> 3.2015)
-    mime-types-data (3.2020.0512)
     mimemagic (0.3.5)
     mini_mime (1.0.2)
-    mini_portile2 (2.4.0)
+    mini_portile2 (2.5.0)
     minitest (5.14.1)
     minitest-ci (3.4.0)
       minitest (>= 5.0.6)
@@ -221,17 +202,12 @@ GEM
       minitest (>= 5.0)
       ruby-progressbar
     mocha (1.11.2)
-    mock_redis (0.23.0)
+    mock_redis (0.24.0)
     msgpack (1.3.3)
     msgpack (1.3.3-x64-mingw32)
     multi_json (1.14.1)
     multi_xml (0.6.0)
     multipart-post (2.1.1)
-    mustermann (1.1.1)
-      ruby2_keywords (~> 0.0.1)
-    net-http-digest_auth (1.4.1)
-    net-http-persistent (4.0.0)
-      connection_pool (~> 2.2)
     net-scp (3.0.0)
       net-ssh (>= 2.6.5, < 7.0.0)
     net-sftp (3.0.0)
@@ -239,22 +215,20 @@ GEM
     net-ssh (6.1.0)
     newrelic_rpm (6.11.0.365)
     nio4r (2.5.2)
-    nokogiri (1.10.9)
+    nokogiri (1.11.0.rc2)
-      mini_portile2 (~> 2.4.0)
+      mini_portile2 (~> 2.5.0)
-    nokogiri (1.10.9-x64-mingw32)
+    nokogiri (1.11.0.rc2-x64-mingw32)
-      mini_portile2 (~> 2.4.0)
     nokogumbo (2.0.2)
       nokogiri (~> 1.8, >= 1.8.4)
-    ntlm-http (0.1.1)
     oauth2 (1.4.4)
       faraday (>= 0.8, < 2.0)
       jwt (>= 1.0, < 3.0)
       multi_json (~> 1.3)
       multi_xml (~> 0.5)
       rack (>= 1.2, < 3)
-    parallel (1.19.1)
+    parallel (1.19.2)
-    parser (2.7.1.3)
+    parser (2.7.1.4)
-      ast (~> 2.4.0)
+      ast (~> 2.4.1)
     pg (1.2.3)
     pg (1.2.3-x64-mingw32)
     pry (0.13.1)
@@ -275,35 +249,33 @@ GEM
       rack (~> 2.0)
     rack-mini-profiler (2.0.2)
       rack (>= 1.2.0)
-    rack-protection (2.0.8.1)
-      rack
     rack-proxy (0.6.5)
       rack
     rack-test (1.1.0)
       rack (>= 1.0, < 3)
-    rails (6.0.3.1)
+    rails (6.0.3.2)
-      actioncable (= 6.0.3.1)
+      actioncable (= 6.0.3.2)
-      actionmailbox (= 6.0.3.1)
+      actionmailbox (= 6.0.3.2)
-      actionmailer (= 6.0.3.1)
+      actionmailer (= 6.0.3.2)
-      actionpack (= 6.0.3.1)
+      actionpack (= 6.0.3.2)
-      actiontext (= 6.0.3.1)
+      actiontext (= 6.0.3.2)
-      actionview (= 6.0.3.1)
+      actionview (= 6.0.3.2)
-      activejob (= 6.0.3.1)
+      activejob (= 6.0.3.2)
-      activemodel (= 6.0.3.1)
+      activemodel (= 6.0.3.2)
-      activerecord (= 6.0.3.1)
+      activerecord (= 6.0.3.2)
-      activestorage (= 6.0.3.1)
+      activestorage (= 6.0.3.2)
-      activesupport (= 6.0.3.1)
+      activesupport (= 6.0.3.2)
       bundler (>= 1.3.0)
-      railties (= 6.0.3.1)
+      railties (= 6.0.3.2)
       sprockets-rails (>= 2.0.0)
     rails-dom-testing (2.0.3)
       activesupport (>= 4.2.0)
       nokogiri (>= 1.6)
     rails-html-sanitizer (1.3.0)
       loofah (~> 2.3)
-    railties (6.0.3.1)
+    railties (6.0.3.2)
-      actionpack (= 6.0.3.1)
+      actionpack (= 6.0.3.2)
-      activesupport (= 6.0.3.1)
+      activesupport (= 6.0.3.2)
       method_source
       rake (>= 0.8.7)
       thor (>= 0.20.3, < 2.0)
@@ -343,9 +315,7 @@ GEM
     ruby-progressbar (1.10.1)
     ruby-vips (2.0.17)
       ffi (~> 1.9)
-    ruby2_keywords (0.0.2)
     rubyzip (2.3.0)
-    safe_yaml (1.0.5)
     sanitize (5.2.1)
       crass (~> 1.0.2)
       nokogiri (>= 1.8.0)
@@ -368,11 +338,6 @@ GEM
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.2)
-    sinatra (2.0.8.1)
-      mustermann (~> 1.0)
-      rack (~> 2.0)
-      rack-protection (= 2.0.8.1)
-      tilt (~> 2.0)
     sprockets (4.0.2)
       concurrent-ruby (~> 1.0)
       rack (> 1, < 3)
@@ -389,7 +354,6 @@ GEM
     stripe (5.22.0)
     thor (1.0.1)
     thread_safe (0.3.6)
-    tilt (2.0.10)
     tzinfo (1.2.7)
       thread_safe (~> 0.1)
     unf (0.1.4)
@@ -403,16 +367,11 @@ GEM
     unicorn-worker-killer (0.4.4)
       get_process_mem (~> 0)
       unicorn (>= 4, < 6)
-    webmock (3.8.3)
-      addressable (>= 2.3.6)
-      crack (>= 0.3.2)
-      hashdiff (>= 0.4.0, < 2.0.0)
     webpacker (5.1.1)
       activesupport (>= 5.2)
       rack-proxy (>= 0.6.1)
       railties (>= 5.2)
       semantic_range (>= 2.3.0)
-    webrobots (0.1.2)
     websocket-driver (0.7.2)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)
@@ -450,12 +409,10 @@ DEPENDENCIES
   ffaker
   flamegraph
   http
-  httparty
   ipaddress_2
   jquery-rails
   listen
   mail
-  mechanize
   memoist
   memory_profiler
   meta_request
@@ -465,7 +422,7 @@ DEPENDENCIES
   mock_redis
   net-sftp
   newrelic_rpm
-  nokogiri (~> 1.10.9)
+  nokogiri
   oauth2
   pg
   pry-byebug
@@ -492,13 +449,11 @@ DEPENDENCIES
   shoulda-matchers
   simple_form
   simplecov (~> 0.17.0)
-  sinatra
   stackprof
   streamio-ffmpeg
   stripe
   unicorn
   unicorn-worker-killer
-  webmock
   webpacker (>= 4.0.x)
   whenever
 

INSTALL.debian

@@ -32,9 +32,6 @@ if [[ -z "$HOSTNAME" ]] ; then
   exit 1
 fi
 
-echo -n "* Enter the VLAN IP address for this server (ex: 172.16.0.1, enter nothing to skip): "
-read VLAN_IP_ADDR
-
 # Install packages
 echo "* Installing packages..."
 
@@ -52,17 +49,6 @@ apt-get -y install $LIBSSL_DEV_PKG build-essential automake libxml2-dev libxslt-
 apt-get -y install libpq-dev postgresql-client
 apt-get -y install liblcms2-dev $LIBJPEG_TURBO_DEV_PKG libexpat1-dev libgif-dev libpng-dev libexif-dev
 
-# vrack specific stuff
-if [ -n "$VLAN_IP_ADDR" ] ; then
-  apt-get -y install vlan
-  modprobe 8021q
-  echo "8021q" >> /etc/modules
-  vconfig add eno2 99
-  ip addr add $VLAN_IP_ADDR/24 dev eno2.99
-  ip link set up eno2.99
-  curl -L -s $GITHUB_INSTALL_SCRIPTS/vrack-cfg.yaml -o /etc/netplan/01-netcfg.yaml
-fi
-
 curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -
 echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list
 curl -sSL https://deb.nodesource.com/setup_10.x | sudo -E bash -

app/controllers/explore/posts_controller.rb

@@ -28,11 +28,11 @@ module Explore
 
     def searches
       @date, @scale, @min_date, @max_date = parse_date(params)
-      @search_service = ReportbooruService.new
+      @searches = ReportbooruService.new.popular_searches(@date)
     end
 
     def missed_searches
-      @search_service = ReportbooruService.new
+      @missed_searches = ReportbooruService.new.missed_search_rankings
     end
 
     private

app/controllers/mock_services_controller.rb

@@ -0,0 +1,48 @@
+class MockServicesController < ApplicationController
+  skip_forgery_protection
+  respond_to :json
+
+  before_action do
+    raise User::PrivilegeError if Rails.env.production?
+  end
+
+  def recommender_recommend
+    @data = posts.map { |post| [post.id, rand(0.0..1.0)] }
+    render json: @data
+  end
+
+  def recommender_similar
+    @data = posts.map { |post| [post.id, rand(0.0..1.0)] }
+    render json: @data
+  end
+
+  def reportbooru_missed_searches
+    @data = tags.map { |tag| "#{tag.name} #{rand(1.0..1000.0)}" }.join("\n")
+    render json: @data
+  end
+
+  def reportbooru_post_searches
+    @data = tags.map { |tag| [tag.name, rand(1..1000)] }
+    render json: @data
+  end
+
+  def reportbooru_post_views
+    @data = posts.map { |post| [post.id, rand(1..1000)] }
+    render json: @data
+  end
+
+  def iqdbs_similar
+    @data = posts.map { |post| { post_id: post.id, score: rand(0..100)} }
+    render json: @data
+  end
+
+  private
+
+  def posts(limit = 10)
+    Post.last(limit)
+  end
+
+  def tags(limit = 10)
+    Tag.order(post_count: :desc).limit(limit)
+  end
+end

app/controllers/uploads_controller.rb

@@ -21,7 +21,7 @@ class UploadsController < ApplicationController
   def image_proxy
     authorize Upload
     resp = ImageProxy.get_image(params[:url])
-    send_data resp.body, :type => resp.content_type, :disposition => "inline"
+    send_data resp.body, type: resp.mime_type, disposition: "inline"
   end
 
   def index

app/javascript/src/javascripts/autocomplete.js.erb

@@ -9,6 +9,7 @@ Autocomplete.ORDER_METATAGS = <%= PostQueryBuilder::ORDER_METATAGS.to_json.html_
 Autocomplete.DISAPPROVAL_REASONS = <%= PostDisapproval::REASONS.to_json.html_safe %>;
 /* eslint-enable */
 
+Autocomplete.MISC_STATUSES = ["deleted", "active", "pending", "flagged", "banned", "modqueue", "unmoderated"];
 Autocomplete.TAG_PREFIXES = "-|~|" + Object.keys(Autocomplete.TAG_CATEGORIES).map(category => category + ":").join("|");
 Autocomplete.METATAGS_REGEX = Autocomplete.METATAGS.concat(Object.keys(Autocomplete.TAG_CATEGORIES)).join("|");
 Autocomplete.TERM_REGEX = new RegExp(`([-~]*)(?:(${Autocomplete.METATAGS_REGEX}):)?(\\S*)$`, "i");
@@ -268,9 +269,7 @@ Autocomplete.render_item = function(list, item) {
 
 Autocomplete.static_metatags = {
   order: Autocomplete.ORDER_METATAGS,
-  status: [
+  status: ["any"].concat(Autocomplete.MISC_STATUSES),
-    "any", "deleted", "active", "pending", "flagged", "banned", "modqueue", "unmoderated"
-  ],
   rating: [
     "safe", "questionable", "explicit"
   ],
@@ -280,12 +279,8 @@ Autocomplete.static_metatags = {
   embedded: [
     "true", "false"
   ],
-  child: [
+  child: ["any", "none"].concat(Autocomplete.MISC_STATUSES),
-    "any", "none"
+  parent: ["any", "none"].concat(Autocomplete.MISC_STATUSES),
-  ],
-  parent: [
-    "any", "none"
-  ],
   filetype: [
     "jpg", "png", "gif", "swf", "zip", "webm", "mp4"
   ],

app/javascript/src/javascripts/posts.js.erb

@@ -300,10 +300,10 @@ Post.initialize_favlist = function() {
   });
 }
 
-Post.view_original = function(e) {
+Post.view_original = function(e = null) {
   if (Utility.test_max_width(660)) {
     // Do the default behavior (navigate to image)
-    return false;
+    return;
   }
 
   var $image = $("#image");
@@ -316,13 +316,13 @@ Post.view_original = function(e) {
   });
   Note.Box.scale_all();
   $("body").attr("data-post-current-image-size", "original");
-  return false;
+  e?.preventDefault();
 }
 
-Post.view_large = function(e) {
+Post.view_large = function(e = null) {
   if (Utility.test_max_width(660)) {
     // Do the default behavior (navigate to image)
-    return false;
+    return;
   }
 
   var $image = $("#image");
@@ -335,7 +335,7 @@ Post.view_large = function(e) {
   });
   Note.Box.scale_all();
   $("body").attr("data-post-current-image-size", "large");
-  return false;
+  e?.preventDefault();
 }
 
 Post.toggle_fit_window = function(e) {

app/logical/cloudflare_service.rb

@@ -9,15 +9,6 @@ class CloudflareService
     api_token.present? && zone.present?
   end
 
-  def ips(expiry: 24.hours)
-    response = Danbooru::Http.cache(expiry).get("https://api.cloudflare.com/client/v4/ips")
-    return [] if response.code != 200
-
-    result = response.parse["result"]
-    ips = result["ipv4_cidrs"] + result["ipv6_cidrs"]
-    ips.map { |ip| IPAddr.new(ip) }
-  end
-
   def purge_cache(urls)
     return unless enabled?
 

app/logical/current_user.rb

@@ -24,15 +24,6 @@ class CurrentUser
     scoped(user, &block)
   end
 
-  def self.as_system(&block)
-    if block_given?
-      scoped(::User.system, "127.0.0.1", &block)
-    else
-      self.user = User.system
-      self.ip_addr = "127.0.0.1"
-    end
-  end
-
   def self.user
     RequestStore[:current_user]
   end

app/logical/danbooru/http.rb

@@ -1,18 +1,43 @@
+require "danbooru/http/html_adapter"
+require "danbooru/http/xml_adapter"
+require "danbooru/http/cache"
+require "danbooru/http/redirector"
+require "danbooru/http/retriable"
+require "danbooru/http/session"
+
 module Danbooru
   class Http
-    DEFAULT_TIMEOUT = 3
+    class DownloadError < StandardError; end
+    class FileTooLargeError < StandardError; end
+
+    DEFAULT_TIMEOUT = 10
     MAX_REDIRECTS = 5
 
-    attr_writer :cache, :http
+    attr_accessor :max_size, :http
 
     class << self
-      delegate :get, :put, :post, :delete, :cache, :follow, :timeout, :auth, :basic_auth, :headers, to: :new
+      delegate :get, :head, :put, :post, :delete, :cache, :follow, :max_size, :timeout, :auth, :basic_auth, :headers, :cookies, :use, :public_only, :download_media, to: :new
+    end
+
+    def initialize
+      @http ||=
+        ::Danbooru::Http::ApplicationClient.new
+        .timeout(DEFAULT_TIMEOUT)
+        .headers("Accept-Encoding" => "gzip")
+        .headers("User-Agent": "#{Danbooru.config.canonical_app_name}/#{Rails.application.config.x.git_hash}")
+        .use(:auto_inflate)
+        .use(redirector: { max_redirects: MAX_REDIRECTS })
+        .use(:session)
     end
 
     def get(url, **options)
       request(:get, url, **options)
     end
 
+    def head(url, **options)
+      request(:head, url, **options)
+    end
+
     def put(url, **options)
       request(:get, url, **options)
     end
@@ -25,14 +50,14 @@ module Danbooru
       request(:delete, url, **options)
     end
 
-    def cache(expiry)
-      dup.tap { |o| o.cache = expiry.to_i }
-    end
-
     def follow(*args)
       dup.tap { |o| o.http = o.http.follow(*args) }
     end
 
+    def max_size(size)
+      dup.tap { |o| o.max_size = size }
+    end
+
     def timeout(*args)
       dup.tap { |o| o.http = o.http.timeout(*args) }
     end
@@ -49,43 +74,72 @@ module Danbooru
       dup.tap { |o| o.http = o.http.headers(*args) }
     end
 
+    def cookies(*args)
+      dup.tap { |o| o.http = o.http.cookies(*args) }
+    end
+
+    def use(*args)
+      dup.tap { |o| o.http = o.http.use(*args) }
+    end
+
+    def cache(expires_in)
+      use(cache: { expires_in: expires_in })
+    end
+
+    # allow requests only to public IPs, not to local or private networks.
+    def public_only
+      dup.tap do |o|
+        o.http = o.http.dup.tap do |http|
+          http.default_options = http.default_options.with_socket_class(ValidatingSocket)
+        end
+      end
+    end
+
+    concerning :DownloadMethods do
+      def download_media(url, no_polish: true, **options)
+        url = Addressable::URI.heuristic_parse(url)
+        response = headers(Referer: url.origin).get(url)
+
+        # prevent Cloudflare Polish from modifying images.
+        if no_polish && response.headers["CF-Polished"].present?
+          url.query_values = url.query_values.to_h.merge(danbooru_no_polish: SecureRandom.uuid)
+          return download_media(url, no_polish: false)
+        end
+
+        file = download_response(response, **options)
+        [response, MediaFile.open(file)]
+      end
+
+      def download_response(response, file: Tempfile.new("danbooru-download-", binmode: true))
+        raise DownloadError, "Downloading #{response.uri} failed with code #{response.status}" if response.status != 200
+        raise FileTooLargeError, response if @max_size && response.content_length.to_i > @max_size
+
+        size = 0
+        response.body.each do |chunk|
+          size += chunk.size
+          raise FileTooLargeError if @max_size && size > @max_size
+          file.write(chunk)
+        end
+
+        file.rewind
+        file
+      end
+    end
+
     protected
 
     def request(method, url, **options)
-      if @cache.present?
-        cached_request(method, url, **options)
-      else
-        raw_request(method, url, **options)
-      end
-    rescue HTTP::Redirector::TooManyRedirectsError
-      ::HTTP::Response.new(status: 598, body: "", version: "1.1")
-    rescue HTTP::TimeoutError
-      # return a synthetic http error on connection timeouts
-      ::HTTP::Response.new(status: 599, body: "", version: "1.1")
-    end
-
-    def cached_request(method, url, **options)
-      key = Cache.hash({ method: method, url: url, headers: http.default_options.headers.to_h, **options }.to_json)
-
-      cached_response = Cache.get(key, @cache) do
-        response = raw_request(method, url, **options)
-        { status: response.status, body: response.to_s, headers: response.headers.to_h, version: "1.1" }
-      end
-
-      ::HTTP::Response.new(**cached_response)
-    end
-
-    def raw_request(method, url, **options)
       http.send(method, url, **options)
+    rescue ValidatingSocket::ProhibitedIpError
+      fake_response(597, "")
+    rescue HTTP::Redirector::TooManyRedirectsError
+      fake_response(598, "")
+    rescue HTTP::TimeoutError
+      fake_response(599, "")
     end
 
-    def http
+    def fake_response(status, body)
-      @http ||= ::HTTP.
+      ::HTTP::Response.new(status: status, version: "1.1", body: ::HTTP::Response::Body.new(body))
-        follow(strict: false, max_hops: MAX_REDIRECTS).
-        timeout(DEFAULT_TIMEOUT).
-        use(:auto_inflate).
-        headers(Danbooru.config.http_headers).
-        headers("Accept-Encoding" => "gzip")
     end
   end
 end

app/logical/danbooru/http/application_client.rb

@@ -0,0 +1,31 @@
+# An extension to HTTP::Client that lets us write Rack-style middlewares that
+# hook into the request/response cycle and override how requests are made. This
+# works by extending http.rb's concept of features (HTTP::Feature) to give them
+# a `perform` method that takes a http request and returns a http response.
+# This can be used to intercept and modify requests and return arbitrary responses.
+
+module Danbooru
+  class Http
+    class ApplicationClient < HTTP::Client
+      # Override `perform` to call the `perform` method on features first.
+      def perform(request, options)
+        features = options.features.values.reverse.select do |feature|
+          feature.respond_to?(:perform)
+        end
+
+        perform = proc { |req| super(req, options) }
+        callback_chain = features.reduce(perform) do |callback_chain, feature|
+          proc { |req| feature.perform(req, &callback_chain) }
+        end
+
+        callback_chain.call(request)
+      end
+
+      # Override `branch` to return an ApplicationClient instead of a
+      # HTTP::Client so that chaining works.
+      def branch(...)
+        ApplicationClient.new(...)
+      end
+    end
+  end
+end

app/logical/danbooru/http/cache.rb

@@ -0,0 +1,30 @@
+module Danbooru
+  class Http
+    class Cache < HTTP::Feature
+      HTTP::Options.register_feature :cache, self
+
+      attr_reader :expires_in
+
+      def initialize(expires_in:)
+        @expires_in = expires_in
+      end
+
+      def perform(request, &block)
+        ::Cache.get(cache_key(request), expires_in) do
+          response = yield request
+
+          # XXX hack to remove connection state from response body so we can serialize it for caching.
+          response.flush
+          response.body.instance_variable_set(:@connection, nil)
+          response.body.instance_variable_set(:@stream, nil)
+
+          response
+        end
+      end
+
+      def cache_key(request)
+        "http:" + ::Cache.hash({ method: request.verb, url: request.uri.to_s, headers: request.headers.sort }.to_json)
+      end
+    end
+  end
+end

app/logical/danbooru/http/html_adapter.rb

@@ -0,0 +1,12 @@
+module Danbooru
+  class Http
+    class HtmlAdapter < HTTP::MimeType::Adapter
+      HTTP::MimeType.register_adapter "text/html", self
+      HTTP::MimeType.register_alias "text/html", :html
+
+      def decode(str)
+        Nokogiri::HTML5(str)
+      end
+    end
+  end
+end

app/logical/danbooru/http/redirector.rb

@@ -0,0 +1,40 @@
+# A HTTP::Feature that automatically follows HTTP redirects.
+# https://developer.mozilla.org/en-US/docs/Web/HTTP/Redirections
+
+module Danbooru
+  class Http
+    class Redirector < HTTP::Feature
+      HTTP::Options.register_feature :redirector, self
+
+      attr_reader :max_redirects
+
+      def initialize(max_redirects: 5)
+        @max_redirects = max_redirects
+      end
+
+      def perform(request, &block)
+        response = yield request
+
+        redirects = max_redirects
+        while response.status.redirect?
+          raise HTTP::Redirector::TooManyRedirectsError if redirects <= 0
+
+          response = yield build_redirect(request, response)
+          redirects -= 1
+        end
+
+        response
+      end
+
+      def build_redirect(request, response)
+        location = response.headers["Location"].to_s
+        uri = HTTP::URI.parse(location)
+
+        verb = request.verb
+        verb = :get if response.status == 303 && !request.verb.in?([:get, :head])
+
+        request.redirect(uri, verb)
+      end
+    end
+  end
+end

app/logical/danbooru/http/retriable.rb

@@ -0,0 +1,54 @@
+# A HTTP::Feature that automatically retries requests that return a 429 error
+# or a Retry-After header. Usage: `Danbooru::Http.use(:retriable).get(url)`.
+#
+# https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/429
+# https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
+
+module Danbooru
+  class Http
+    class Retriable < HTTP::Feature
+      HTTP::Options.register_feature :retriable, self
+
+      attr_reader :max_retries, :max_delay
+
+      def initialize(max_retries: 2, max_delay: 5.seconds)
+        @max_retries = max_retries
+        @max_delay = max_delay
+      end
+
+      def perform(request, &block)
+        response = yield request
+
+        retries = max_retries
+        while retriable?(response) && retries > 0 && retry_delay(response) <= max_delay
+          DanbooruLogger.info "Retrying url=#{request.uri} status=#{response.status} retries=#{retries} delay=#{retry_delay(response)}"
+
+          retries -= 1
+          sleep(retry_delay(response))
+          response = yield request
+        end
+
+        response
+      end
+
+      def retriable?(response)
+        response.status == 429 || response.headers["Retry-After"].present?
+      end
+
+      def retry_delay(response, current_time: Time.zone.now)
+        retry_after = response.headers["Retry-After"]
+
+        if retry_after.blank?
+          0.seconds
+        elsif retry_after =~ /\A\d+\z/
+          retry_after.to_i.seconds
+        else
+          retry_at = Time.zone.parse(retry_after)
+          return 0.seconds if retry_at.blank?
+
+          [retry_at - current_time, 0].max.seconds
+        end
+      end
+    end
+  end
+end

app/logical/danbooru/http/session.rb

@@ -0,0 +1,37 @@
+module Danbooru
+  class Http
+    class Session < HTTP::Feature
+      HTTP::Options.register_feature :session, self
+
+      attr_reader :cookie_jar
+
+      def initialize(cookie_jar: HTTP::CookieJar.new)
+        @cookie_jar = cookie_jar
+      end
+
+      def perform(request)
+        add_cookies(request)
+        response = yield request
+        save_cookies(response)
+        response
+      end
+
+      def add_cookies(request)
+        cookies = cookies_for_request(request)
+        request.headers["Cookie"] = cookies if cookies.present?
+      end
+
+      def cookies_for_request(request)
+        saved_cookies = cookie_jar.each(request.uri).map { |c| [c.name, c.value] }.to_h
+        request_cookies = HTTP::Cookie.cookie_value_to_hash(request.headers["Cookie"].to_s)
+        saved_cookies.merge(request_cookies).map { |name, value| "#{name}=#{value}" }.join("; ")
+      end
+
+      def save_cookies(response)
+        response.cookies.each do |cookie|
+          cookie_jar.add(cookie)
+        end
+      end
+    end
+  end
+end

app/logical/danbooru/http/xml_adapter.rb

@@ -0,0 +1,12 @@
+module Danbooru
+  class Http
+    class XmlAdapter < HTTP::MimeType::Adapter
+      HTTP::MimeType.register_adapter "application/xml", self
+      HTTP::MimeType.register_alias "application/xml", :xml
+
+      def decode(str)
+        Hash.from_xml(str).with_indifferent_access
+      end
+    end
+  end
+end

app/logical/downloads/file.rb

@@ -1,123 +0,0 @@
-require 'resolv'
-
-module Downloads
-  class File
-    include ActiveModel::Validations
-    class Error < StandardError; end
-
-    RETRIABLE_ERRORS = [Errno::ECONNRESET, Errno::ETIMEDOUT, Errno::EIO, Errno::EHOSTUNREACH, Errno::ECONNREFUSED, Timeout::Error, IOError]
-
-    delegate :data, to: :strategy
-    attr_reader :url, :referer
-
-    validate :validate_url
-
-    def initialize(url, referer = nil)
-      @url = Addressable::URI.parse(url) rescue nil
-      @referer = referer
-      validate!
-    end
-
-    def size
-      res = HTTParty.head(uncached_url, **httparty_options, timeout: 3)
-
-      if res.success?
-        res.content_length
-      else
-        raise HTTParty::ResponseError.new(res)
-      end
-    end
-
-    def download!(url: uncached_url, tries: 3, **options)
-      Retriable.retriable(on: RETRIABLE_ERRORS, tries: tries, base_interval: 0) do
-        file = http_get_streaming(url, headers: strategy.headers, **options)
-        return [file, strategy]
-      end
-    end
-
-    def validate_url
-      errors[:base] << "URL must not be blank" if url.blank?
-      errors[:base] << "'#{url}' is not a valid url" if !url.host.present?
-      errors[:base] << "'#{url}' is not a valid url. Did you mean 'http://#{url}'?" if !url.scheme.in?(%w[http https])
-    end
-
-    def http_get_streaming(url, file: Tempfile.new(binmode: true), headers: {}, max_size: Danbooru.config.max_file_size)
-      size = 0
-
-      res = HTTParty.get(url, httparty_options) do |chunk|
-        next if chunk.code == 302
-
-        size += chunk.size
-        raise Error.new("File is too large (max size: #{max_size})") if size > max_size && max_size > 0
-
-        file.write(chunk)
-      end
-
-      if res.success?
-        file.rewind
-        return file
-      else
-        raise Error.new("HTTP error code: #{res.code} #{res.message}")
-      end
-    end
-
-    # Prevent Cloudflare from potentially mangling the image. See issue #3528.
-    def uncached_url
-      return file_url unless is_cloudflare?(file_url)
-
-      url = file_url.dup
-      url.query_values = url.query_values.to_h.merge(danbooru_no_cache: SecureRandom.uuid)
-      url
-    end
-
-    def preview_url
-      @preview_url ||= Addressable::URI.parse(strategy.preview_url)
-    end
-
-    def file_url
-      @file_url ||= Addressable::URI.parse(strategy.image_url)
-    end
-
-    def strategy
-      @strategy ||= Sources::Strategies.find(url.to_s, referer)
-    end
-
-    def httparty_options
-      {
-        timeout: 10,
-        stream_body: true,
-        headers: strategy.headers,
-        connection_adapter: ValidatingConnectionAdapter
-      }.deep_merge(Danbooru.config.httparty_options)
-    end
-
-    def is_cloudflare?(url)
-      ip_addr = IPAddr.new(Resolv.getaddress(url.hostname))
-      CloudflareService.new.ips.any? { |subnet| subnet.include?(ip_addr) }
-    end
-
-    def self.banned_ip?(ip)
-      ip = IPAddress.parse(ip.to_s) unless ip.is_a?(IPAddress)
-
-      if ip.ipv4?
-        ip.loopback? || ip.link_local? || ip.multicast? || ip.private?
-      elsif ip.ipv6?
-        ip.loopback? || ip.link_local? || ip.unique_local? || ip.unspecified?
-      end
-    end
-  end
-
-  # Hook into HTTParty to validate the IP before following redirects.
-  # https://www.rubydoc.info/github/jnunemaker/httparty/HTTParty/ConnectionAdapter
-  class ValidatingConnectionAdapter < HTTParty::ConnectionAdapter
-    def self.call(uri, options)
-      ip_addr = IPAddress.parse(::Resolv.getaddress(uri.hostname))
-
-      if Downloads::File.banned_ip?(ip_addr)
-        raise Downloads::File::Error, "Downloads from #{ip_addr} are not allowed"
-      end
-
-      super(uri, options)
-    end
-  end
-end

app/logical/image_proxy.rb

@@ -1,4 +1,6 @@
 class ImageProxy
+  class Error < StandardError; end
+
   def self.needs_proxy?(url)
     fake_referer_for(url).present?
   end
@@ -8,16 +10,13 @@ class ImageProxy
   end
 
   def self.get_image(url)
-    if url.blank?
+    raise Error, "URL not present" unless url.present?
-      raise "Must specify url"
+    raise Error, "Proxy not allowed for this url (url=#{url})" unless needs_proxy?(url)
-    end
 
-    if !needs_proxy?(url)
+    referer = fake_referer_for(url)
-      raise "Proxy not allowed for this site"
+    response = Danbooru::Http.headers(Referer: referer).get(url)
-    end
+    raise Error, "Couldn't proxy image (code=#{response.status}, url=#{url})" unless response.status.success?
 
-    response = HTTParty.get(url, Danbooru.config.httparty_options.deep_merge(headers: {"Referer" => fake_referer_for(url)}))
-    raise "HTTP error code: #{response.code} #{response.message}" unless response.success?
     response
   end
 end

app/logical/iqdb_proxy.rb

@@ -12,8 +12,9 @@ class IqdbProxy
   end
 
   def download(url, type)
-    download = Downloads::File.new(url)
+    strategy = Sources::Strategies.find(url)
-    file, strategy = download.download!(url: download.send(type))
+    download_url = strategy.send(type)
+    file = strategy.download_file!(download_url)
     file
   end
 
@@ -32,7 +33,7 @@ class IqdbProxy
       file = download(params[:image_url], :url)
       results = query(file: file, limit: limit)
     elsif params[:file_url].present?
-      file = download(params[:file_url], :file_url)
+      file = download(params[:file_url], :image_url)
       results = query(file: file, limit: limit)
     elsif params[:post_id].present?
       url = Post.find(params[:post_id]).preview_file_url
@@ -50,9 +51,12 @@ class IqdbProxy
     file.try(:close)
   end
 
-  def query(params)
+  def query(file: nil, url: nil, limit: 20)
     raise NotImplementedError, "the IQDBs service isn't configured" unless enabled?
-    response = http.post("#{iqdbs_server}/similar", body: params)
+
+    file = HTTP::FormData::File.new(file) if file
+    form = { file: file, url: url, limit: limit }.compact
+    response = http.timeout(30).post("#{iqdbs_server}/similar", form: form)
 
     raise Error, "IQDB error: #{response.status}" if response.status != 200
     raise Error, "IQDB error: #{response.parse["error"]}" if response.parse.is_a?(Hash)

app/logical/media_file.rb

@@ -43,6 +43,8 @@ class MediaFile
     else
       :bin
     end
+  rescue EOFError
+    :bin
   end
 
   def self.videos_enabled?

app/logical/nico_seiga_api_client.rb

@@ -4,8 +4,7 @@ class NicoSeigaApiClient
 
   attr_reader :http
 
-  # XXX temp disable following redirects.
+  def initialize(work_id:, type:, http: Danbooru::Http.new)
-  def initialize(work_id:, type:, http: Danbooru::Http.follow(nil))
     @work_id = work_id
     @work_type = type
     @http = http
@@ -80,28 +79,19 @@ class NicoSeigaApiClient
   end
 
   def get(url)
-    cookie_header = Cache.get("nicoseiga-cookie-header") || regenerate_cookie_header
-
-    resp = http.headers({Cookie: cookie_header}).cache(1.minute).get(url)
-
-    if resp.headers["Location"] =~ %r{seiga\.nicovideo\.jp/login/}i
-      cookie_header = regenerate_cookie_header
-      resp = http.headers({Cookie: cookie_header}).cache(1.minute).get(url)
-    end
-
-    resp
-  end
-
-  def regenerate_cookie_header
     form = {
       mail_tel: Danbooru.config.nico_seiga_login,
       password: Danbooru.config.nico_seiga_password
     }
-    resp = http.post("https://account.nicovideo.jp/api/v1/login", form: form)
-    cookies = resp.cookies.map { |c| c.name + "=" + c.value }
-    cookies << "accept_fetish_warning=2"
 
-    Cache.put("nicoseiga-cookie-header", cookies.join(";"), 1.week)
+    # XXX should fail gracefully instead of raising exception
+    resp = http.cache(1.hour).post("https://account.nicovideo.jp/login/redirector?site=seiga", form: form)
+    raise RuntimeError, "NicoSeiga login failed (status=#{resp.status} url=#{url})" if resp.status != 200
+
+    resp = http.cache(1.minute).get(url)
+    #raise RuntimeError, "NicoSeiga get failed (status=#{resp.status} url=#{url})" if resp.status != 200
+
+    resp
   end
 
   memoize :api_response, :manga_api_response, :user_api_response

app/logical/pixiv_api_client.rb

@@ -1,5 +1,3 @@
-require 'resolv-replace'
-
 class PixivApiClient
   extend Memoist
 
@@ -8,6 +6,21 @@ class PixivApiClient
   CLIENT_SECRET = "HP3RmkgAmEGro0gn1x9ioawQE8WMfvLXDz3ZqxpK"
   CLIENT_HASH_SALT = "28c1fdd170a5204386cb1313c7077b34f83e4aaf4aa829ce78c231e05b0bae2c"
 
+  # Tools to not include in the tags list. We don't tag digital media, so
+  # including these results in bad translated tags suggestions.
+  TOOLS_BLACKLIST = %w[
+    Photoshop Illustrator Fireworks Flash Painter PaintShopPro pixiv\ Sketch
+    CLIP\ STUDIO\ PAINT IllustStudio ComicStudio RETAS\ STUDIO SAI PhotoStudio
+    Pixia NekoPaint PictBear openCanvas ArtRage Expression Inkscape GIMP
+    CGillust COMICWORKS MS_Paint EDGE AzPainter AzPainter2 AzDrawing
+    PicturePublisher SketchBookPro Processing 4thPaint GraphicsGale mdiapp
+    Paintgraphic AfterEffects drawr CLIP\ PAINT\ Lab FireAlpaca Pixelmator
+    AzDrawing2 MediBang\ Paint Krita ibisPaint Procreate Live2D
+    Lightwave3D Shade Poser STRATA AnimationMaster XSI CARRARA CINEMA4D Maya
+    3dsMax Blender ZBrush Metasequoia Sunny3D Bryce Vue Hexagon\ King SketchUp
+    VistaPro Sculptris Comi\ Po! modo DAZ\ Studio 3D-Coat
+  ]
+
   class Error < StandardError; end
   class BadIDError < Error; end
 
@@ -24,7 +37,7 @@ class PixivApiClient
       @artist_commentary_title = json["title"].to_s
       @artist_commentary_desc = json["caption"].to_s
       @tags = json["tags"].reject {|x| x =~ /^http:/}
-      @tags += json["tools"]
+      @tags += json["tools"] - TOOLS_BLACKLIST
 
       if json["metadata"]
         if json["metadata"]["zip_urls"]
@@ -99,66 +112,13 @@ class PixivApiClient
     end
   end
 
-  class FanboxResponse
-    attr_reader :json
-
-    def initialize(json)
-      @json = json
-    end
-
-    def name
-      json["body"]["user"]["name"]
-    end
-
-    def user_id
-      json["body"]["user"]["userId"]
-    end
-
-    def moniker
-      ""
-    end
-
-    def page_count
-      json["body"]["body"]["images"].size
-    end
-
-    def artist_commentary_title
-      json["body"]["title"]
-    end
-
-    def artist_commentary_desc
-      json["body"]["body"]["text"]
-    end
-
-    def tags
-      []
-    end
-
-    def pages
-      if json["body"]["body"]
-        json["body"]["body"]["images"].map {|x| x["originalUrl"]}
-      else
-        []
-      end
-    end
-  end
-
   def work(illust_id)
-    headers = Danbooru.config.http_headers.merge(
+    params = { image_sizes: "large", include_stats: "true" }
-      "Referer" => "http://www.pixiv.net",
-      "Content-Type" => "application/x-www-form-urlencoded",
-      "Authorization" => "Bearer #{access_token}"
-    )
-    params = {
-      "image_sizes" => "large",
-      "include_stats" => "true"
-    }
-
     url = "https://public-api.secure.pixiv.net/v#{API_VERSION}/works/#{illust_id.to_i}.json"
-    response = Danbooru::Http.cache(1.minute).headers(headers).get(url, params: params)
+    response = api_client.cache(1.minute).get(url, params: params)
     json = response.parse
 
-    if response.code == 200
+    if response.status == 200
       WorkResponse.new(json["response"][0])
     elsif json["status"] == "failure" && json.dig("errors", "system", "message") =~ /対象のイラストは見つかりませんでした。/
       raise BadIDError.new("Pixiv ##{illust_id} not found: work was deleted, made private, or ID is invalid.")
@@ -169,32 +129,12 @@ class PixivApiClient
     raise Error.new("Pixiv API call failed (status=#{response.code} body=#{response.body})")
   end
 
-  def fanbox(fanbox_id)
-    url = "https://www.pixiv.net/ajax/fanbox/post?postId=#{fanbox_id.to_i}"
-    resp = agent.get(url)
-    json = JSON.parse(resp.body)
-    if resp.code == "200"
-      FanboxResponse.new(json)
-    elsif json["status"] == "failure"
-      raise Error.new("Pixiv API call failed (status=#{resp.code} body=#{body})")
-    end
-  rescue JSON::ParserError
-    raise Error.new("Pixiv API call failed (status=#{resp.code} body=#{body})")
-  end
-
   def novel(novel_id)
-    headers = Danbooru.config.http_headers.merge(
-      "Referer" => "http://www.pixiv.net",
-      "Content-Type" => "application/x-www-form-urlencoded",
-      "Authorization" => "Bearer #{access_token}"
-    )
-
     url = "https://public-api.secure.pixiv.net/v#{API_VERSION}/novels/#{novel_id.to_i}.json"
-    resp = HTTParty.get(url, Danbooru.config.httparty_options.deep_merge(headers: headers))
+    resp = api_client.cache(1.minute).get(url)
-    body = resp.body.force_encoding("utf-8")
+    json = resp.parse
-    json = JSON.parse(body)
 
-    if resp.success?
+    if resp.status == 200
       NovelResponse.new(json["response"][0])
     elsif json["status"] == "failure" && json.dig("errors", "system", "message") =~ /対象のイラストは見つかりませんでした。/
       raise Error.new("Pixiv API call failed (status=#{resp.code} body=#{body})")
@@ -204,10 +144,8 @@ class PixivApiClient
   end
 
   def access_token
-    Cache.get("pixiv-papi-access-token", 3000) do
+    # truncate timestamp to 1-hour resolution so that it doesn't break caching.
-      access_token = nil
+    client_time = Time.zone.now.utc.change(min: 0).rfc3339
-
-      client_time = Time.now.rfc3339
     client_hash = Digest::MD5.hexdigest(client_time + CLIENT_HASH_SALT)
 
     headers = {
@@ -215,6 +153,7 @@ class PixivApiClient
       "X-Client-Time": client_time,
       "X-Client-Hash": client_hash
     }
+
     params = {
       username: Danbooru.config.pixiv_login,
       password: Danbooru.config.pixiv_password,
@@ -222,24 +161,24 @@ class PixivApiClient
       client_id: CLIENT_ID,
       client_secret: CLIENT_SECRET
     }
-      url = "https://oauth.secure.pixiv.net/auth/token"
 
-      resp = HTTParty.post(url, Danbooru.config.httparty_options.deep_merge(body: params, headers: headers))
+    resp = http.headers(headers).cache(1.hour).post("https://oauth.secure.pixiv.net/auth/token", form: params)
-      body = resp.body.force_encoding("utf-8")
+    return nil unless resp.status == 200
 
-      if resp.success?
+    resp.parse.dig("response", "access_token")
-        json = JSON.parse(body)
-        access_token = json["response"]["access_token"]
-      else
-        raise Error.new("Pixiv API access token call failed (status=#{resp.code} body=#{body})")
   end
 
-      access_token
+  def api_client
-    end
+    http.headers(
+      "Referer": "http://www.pixiv.net",
+      "Content-Type": "application/x-www-form-urlencoded",
+      "Authorization": "Bearer #{access_token}"
+    )
   end
 
-  def agent
+  def http
-    PixivWebAgent.build
+    Danbooru::Http.new
   end
-  memoize :agent
+
+  memoize :access_token, :api_client, :http
 end

app/logical/pixiv_web_agent.rb

@@ -1,74 +0,0 @@
-class PixivWebAgent
-  SESSION_CACHE_KEY = "pixiv-phpsessid"
-  COMIC_SESSION_CACHE_KEY = "pixiv-comicsessid"
-  SESSION_COOKIE_KEY = "PHPSESSID"
-  COMIC_SESSION_COOKIE_KEY = "_pixiv-comic_session"
-
-  def self.phpsessid(agent)
-    agent.cookies.select { |cookie| cookie.name == SESSION_COOKIE_KEY }.first.try(:value)
-  end
-
-  def self.build
-    mech = Mechanize.new
-    mech.keep_alive = false
-
-    phpsessid = Cache.get(SESSION_CACHE_KEY)
-    comicsessid = Cache.get(COMIC_SESSION_CACHE_KEY)
-
-    if phpsessid
-      cookie = Mechanize::Cookie.new(SESSION_COOKIE_KEY, phpsessid)
-      cookie.domain = ".pixiv.net"
-      cookie.path = "/"
-      mech.cookie_jar.add(cookie)
-
-      if comicsessid
-        cookie = Mechanize::Cookie.new(COMIC_SESSION_COOKIE_KEY, comicsessid)
-        cookie.domain = ".pixiv.net"
-        cookie.path = "/"
-        mech.cookie_jar.add(cookie)
-      end
-    else
-      headers = {
-        "Origin" => "https://accounts.pixiv.net",
-        "Referer" => "https://accounts.pixiv.net/login?lang=en^source=pc&view_type=page&ref=wwwtop_accounts_index"
-      }
-
-      params = {
-        pixiv_id: Danbooru.config.pixiv_login,
-        password: Danbooru.config.pixiv_password,
-        captcha: nil,
-        g_captcha_response: nil,
-        source: "pc",
-        post_key: nil
-      }
-
-      mech.get("https://accounts.pixiv.net/login?lang=en&source=pc&view_type=page&ref=wwwtop_accounts_index") do |page|
-        json = page.search("input#init-config").first.attr("value")
-        if json =~ /pixivAccount\.postKey":"([a-f0-9]+)/
-          params[:post_key] = $1
-        end
-      end
-
-      mech.post("https://accounts.pixiv.net/api/login?lang=en", params, headers)
-      if mech.current_page.body =~ /"error":false/
-        cookie = mech.cookies.select {|x| x.name == SESSION_COOKIE_KEY}.first
-        if cookie
-          Cache.put(SESSION_CACHE_KEY, cookie.value, 1.week)
-        end
-      end
-
-      begin
-        mech.get("https://comic.pixiv.net") do
-          cookie = mech.cookies.select {|x| x.name == COMIC_SESSION_COOKIE_KEY}.first
-          if cookie
-            Cache.put(COMIC_SESSION_CACHE_KEY, cookie.value, 1.week)
-          end
-        end
-      rescue Net::HTTPServiceUnavailable
-        # ignore
-      end
-    end
-
-    mech
-  end
-end

app/logical/post_query_builder.rb

@@ -307,6 +307,8 @@ class PostQueryBuilder
       Post.where(parent: nil)
     when "any"
       Post.where.not(parent: nil)
+    when /pending|flagged|modqueue|deleted|banned|active|unmoderated/
+      Post.where.not(parent: nil).where(parent: status_matches(parent))
     when /\A\d+\z/
       Post.where(id: parent).or(Post.where(parent: parent))
     else
@@ -320,6 +322,8 @@ class PostQueryBuilder
       Post.where(has_children: false)
     when "any"
       Post.where(has_children: true)
+    when /pending|flagged|modqueue|deleted|banned|active|unmoderated/
+      Post.where(has_children: true).where(children: status_matches(child))
     else
       Post.none
     end

app/logical/reportbooru_service.rb

@@ -20,29 +20,30 @@ class ReportbooruService
     body.lines.map(&:split).map { [_1, _2.to_i] }
   end
 
-  def post_search_rankings(date = Date.today, expires_in: 1.minutes)
+  def post_search_rankings(date, expires_in: 1.minutes)
-    return [] unless enabled?
+    request("#{reportbooru_server}/post_searches/rank?date=#{date}", expires_in)
-
-    response = http.cache(expires_in).get("#{reportbooru_server}/post_searches/rank?date=#{date}")
-    return [] if response.status != 200
-    JSON.parse(response.to_s.force_encoding("utf-8"))
   end
 
-  def post_view_rankings(date = Date.today, expires_in: 1.minutes)
+  def post_view_rankings(date, expires_in: 1.minutes)
-    return [] unless enabled?
+    request("#{reportbooru_server}/post_views/rank?date=#{date}", expires_in)
-
-    response = http.get("#{reportbooru_server}/post_views/rank?date=#{date}")
-    return [] if response.status != 200
-    JSON.parse(response.to_s.force_encoding("utf-8"))
   end
 
-  def popular_searches(date = Date.today, limit: 100)
+  def popular_searches(date, limit: 100)
     ranking = post_search_rankings(date)
     ranking.take(limit).map(&:first)
   end
 
-  def popular_posts(date = Date.today, limit: 100)
+  def popular_posts(date, limit: 100)
     ranking = post_view_rankings(date)
+    ranking = post_view_rankings(date.yesterday) if ranking.blank?
     ranking.take(limit).map { |x| Post.find(x[0]) }
   end
+
+  def request(url, expires_in)
+    return [] unless enabled?
+
+    response = http.cache(expires_in).get(url)
+    return [] if response.status != 200
+    JSON.parse(response.to_s.force_encoding("utf-8"))
+  end
 end

app/logical/sources/strategies/art_station.rb

@@ -147,7 +147,7 @@ module Sources::Strategies
         urls = urls.reverse
       end
 
-      chosen_url = urls.find { |url| http_exists?(url, headers) }
+      chosen_url = urls.find { |url| http_exists?(url) }
       chosen_url || url
     end
   end

app/logical/sources/strategies/base.rb

@@ -14,6 +14,8 @@
 module Sources
   module Strategies
     class Base
+      class DownloadError < StandardError; end
+
       attr_reader :url, :referer_url, :urls, :parsed_url, :parsed_referer, :parsed_urls
 
       extend Memoist
@@ -35,9 +37,9 @@ module Sources
       #   <tt>referrer_url</tt> so the strategy can discover the HTML
       #   page and other information.
       def initialize(url, referer_url = nil)
-        @url = url
+        @url = url.to_s
-        @referer_url = referer_url
+        @referer_url = referer_url&.to_s
-        @urls = [url, referer_url].select(&:present?)
+        @urls = [@url, @referer_url].select(&:present?)
 
         @parsed_url = Addressable::URI.heuristic_parse(url) rescue nil
         @parsed_referer = Addressable::URI.heuristic_parse(referer_url) rescue nil
@@ -139,15 +141,28 @@ module Sources
       # Subclasses should merge in any required headers needed to access resources
       # on the site.
       def headers
-        Danbooru.config.http_headers
+        {}
       end
 
       # Returns the size of the image resource without actually downloading the file.
       def size
-        Downloads::File.new(image_url).size
+        http.head(image_url).content_length.to_i
       end
       memoize :size
 
+      # Download the file at the given url, or at the main image url by default.
+      def download_file!(download_url = image_url)
+        raise DownloadError, "Download failed: couldn't find download url for #{url}" if download_url.blank?
+        response, file = http.download_media(download_url)
+        raise DownloadError, "Download failed: #{download_url} returned error #{response.status}" if response.status != 200
+        file
+      end
+
+      def http
+        Danbooru::Http.public_only.timeout(30).max_size(Danbooru.config.max_file_size)
+      end
+      memoize :http
+
       # The url to use for artist finding purposes. This will be stored in the
       # artist entry. Normally this will be the profile url.
       def normalize_for_artist_finder
@@ -274,9 +289,8 @@ module Sources
         to_h.to_json
       end
 
-      def http_exists?(url, headers)
+      def http_exists?(url, headers = {})
-        res = HTTParty.head(url, Danbooru.config.httparty_options.deep_merge(headers: headers))
+        http.headers(headers).head(url).status.success?
-        res.success?
       end
 
       # Convert commentary to dtext by stripping html tags. Sites can override

app/logical/sources/strategies/hentai_foundry.rb

@@ -64,11 +64,10 @@ module Sources
       def page
         return nil if page_url.blank?
 
-        doc = Cache.get("hentai-foundry:#{page_url}", 1.minute) do
+        response = Danbooru::Http.new.cache(1.minute).get("#{page_url}?enterAgree=1")
-          HTTParty.get("#{page_url}?enterAgree=1").body
+        return nil unless response.status == 200
-        end
 
-        Nokogiri::HTML(doc)
+        response.parse
       end
 
       def tags

app/logical/sources/strategies/nico_seiga.rb

@@ -73,8 +73,7 @@ module Sources
       end
 
       def image_url
-        return if image_urls.blank?
+        return url if image_urls.blank? || api_client.blank?
-        return url if api_client.blank?
 
         img = case url
         when DIRECT || CDN_DIRECT then "https://seiga.nicovideo.jp/image/source/#{image_id_from_url(url)}"
@@ -83,7 +82,7 @@ module Sources
         end
 
         resp = api_client.get(img)
-        if resp.headers["Location"] =~ %r{https?://.+/(\w+/\d+/\d+)\z}i
+        if resp.uri.to_s =~ %r{https?://.+/(\w+/\d+/\d+)\z}i
           "https://lohas.nicoseiga.jp/priv/#{$1}"
         else
           img
@@ -181,12 +180,12 @@ module Sources
 
       def api_client
         if illust_id.present?
-          NicoSeigaApiClient.new(work_id: illust_id, type: "illust")
+          NicoSeigaApiClient.new(work_id: illust_id, type: "illust", http: http)
         elsif manga_id.present?
-          NicoSeigaApiClient.new(work_id: manga_id, type: "manga")
+          NicoSeigaApiClient.new(work_id: manga_id, type: "manga", http: http)
         elsif image_id.present?
           # We default to illust to attempt getting the api anyway
-          NicoSeigaApiClient.new(work_id: image_id, type: "illust")
+          NicoSeigaApiClient.new(work_id: image_id, type: "illust", http: http)
         end
       end
       memoize :api_client

app/logical/sources/strategies/nijie.rb

@@ -178,54 +178,21 @@ module Sources
       def page
         return nil if page_url.blank?
 
-        doc = agent.get(page_url)
+        http = Danbooru::Http.new
+        form = { email: Danbooru.config.nijie_login, password: Danbooru.config.nijie_password }
 
-        if doc.search("div#header-login-container").any?
+        # XXX `retriable` must come after `cache` so that retries don't return cached error responses.
-          # Session cache is invalid, clear it and log in normally.
+        response = http.cache(1.hour).use(retriable: { max_retries: 20 }).post("https://nijie.info/login_int.php", form: form)
-          Cache.delete("nijie-session")
+        DanbooruLogger.info "Nijie login failed (#{url}, #{response.status})" if response.status != 200
-          doc = agent.get(page_url)
+        return nil unless response.status == 200
+
+        response = http.cookies(R18: 1).cache(1.minute).get(page_url)
+        return nil unless response.status == 200
+
+        response&.parse
       end
 
-        doc
-      rescue Mechanize::ResponseCodeError => e
-        return nil if e.response_code.to_i == 404
-        raise
-      end
       memoize :page
-
-      def agent
-        mech = Mechanize.new
-
-        session = Cache.get("nijie-session")
-        if session
-          cookie = Mechanize::Cookie.new("NIJIEIJIEID", session)
-          cookie.domain = ".nijie.info"
-          cookie.path = "/"
-          mech.cookie_jar.add(cookie)
-        else
-          mech.get("https://nijie.info/login.php") do |page|
-            page.form_with(:action => "/login_int.php") do |form|
-              form['email'] = Danbooru.config.nijie_login
-              form['password'] = Danbooru.config.nijie_password
-            end.click_button
-          end
-          session = mech.cookie_jar.cookies.select {|c| c.name == "NIJIEIJIEID"}.first
-          Cache.put("nijie-session", session.value, 1.day) if session
-        end
-
-        # This cookie needs to be set to allow viewing of adult works while anonymous
-        cookie = Mechanize::Cookie.new("R18", "1")
-        cookie.domain = ".nijie.info"
-        cookie.path = "/"
-        mech.cookie_jar.add(cookie)
-
-        mech
-      rescue Mechanize::ResponseCodeError => e
-        raise unless e.response_code.to_i == 429
-        sleep(5)
-        retry
-      end
-      memoize :agent
     end
   end
 end

app/logical/sources/strategies/null.rb

@@ -47,7 +47,7 @@ module Sources
         when %r{\Ahttps?://c(?:s|han|[1-4])\.sankakucomplex\.com/data(?:/sample)?/(?:[a-f0-9]{2}/){2}(?:sample-|preview)?([a-f0-9]{32})}i
           "https://chan.sankakucomplex.com/en/post/show?md5=#{$1}"
 
-        when %r{\Ahttps?://(?:www|s(?:tatic|[1-4]))\.zerochan\.net/.+(?:\.|\/)(\d+)(?:\.(?:jpe?g?))?\z}i
+        when %r{\Ahttps?://(?:www|s(?:tatic|[1-4]))\.zerochan\.net/.+(?:\.|\/)(\d+)(?:\.(?:jpe?g?|png))?\z}i
           "https://www.zerochan.net/#{$1}#full"
 
         when %r{\Ahttps?://static[1-6]?\.minitokyo\.net/(?:downloads|view)/(?:\d{2}/){2}(\d+)}i

app/logical/sources/strategies/pixiv.rb

@@ -64,9 +64,6 @@ module Sources
       ORIG_IMAGE = %r{#{PXIMG}/img-original/img/#{DATE}/(?<illust_id>\d+)_p(?<page>\d+)\.#{EXT}\z}i
       STACC_PAGE = %r{\A#{WEB}/stacc/#{MONIKER}/?\z}i
       NOVEL_PAGE = %r{(?:\Ahttps?://www\.pixiv\.net/novel/show\.php\?id=(\d+))}
-      FANBOX_ACCOUNT = %r{(?:\Ahttps?://www\.pixiv\.net/fanbox/creator/\d+\z)}
-      FANBOX_IMAGE = %r{(?:\Ahttps?://fanbox\.pixiv\.net/images/post/(\d+))}
-      FANBOX_PAGE = %r{(?:\Ahttps?://www\.pixiv\.net/fanbox/creator/\d+/post/(\d+))}
 
       def self.to_dtext(text)
         if text.nil?
@@ -127,14 +124,6 @@ module Sources
           return "https://www.pixiv.net/novel/show.php?id=#{novel_id}&mode=cover"
         end
 
-        if fanbox_id.present?
-          return "https://www.pixiv.net/fanbox/creator/#{metadata.user_id}/post/#{fanbox_id}"
-        end
-
-        if fanbox_account_id.present?
-          return "https://www.pixiv.net/fanbox/creator/#{fanbox_account_id}"
-        end
-
         if illust_id.present?
           return "https://www.pixiv.net/artworks/#{illust_id}"
         end
@@ -192,17 +181,7 @@ module Sources
       end
 
       def headers
-        if fanbox_id.present?
+        { "Referer" => "https://www.pixiv.net" }
-          # need the session to download fanbox images
-          return {
-            "Referer" => "https://www.pixiv.net/fanbox",
-            "Cookie" => HTTP::Cookie.cookie_value(agent.cookies)
-          }
-        end
-
-        {
-          "Referer" => "https://www.pixiv.net"
-        }
       end
 
       def normalize_for_source
@@ -242,10 +221,6 @@ module Sources
       end
 
       def image_urls_sub
-        if url =~ FANBOX_IMAGE
-          return [url]
-        end
-
         # there's too much normalization bullshit we have to deal with
         # raw urls, so just fetch the canonical url from the api every
         # time.
@@ -265,7 +240,7 @@ module Sources
       # even though it makes sense to reference page_url here, it will only look
       # at (url, referer_url).
       def illust_id
-        return nil if novel_id.present? || fanbox_id.present?
+        return nil if novel_id.present?
 
         parsed_urls.each do |url|
           # http://www.pixiv.net/member_illust.php?mode=medium&illust_id=18557054
@@ -328,46 +303,11 @@ module Sources
       end
       memoize :novel_id
 
-      def fanbox_id
-        [url, referer_url].each do |x|
-          if x =~ FANBOX_PAGE
-            return $1
-          end
-
-          if x =~ FANBOX_IMAGE
-            return $1
-          end
-        end
-
-        nil
-      end
-      memoize :fanbox_id
-
-      def fanbox_account_id
-        [url, referer_url].each do |x|
-          if x =~ FANBOX_ACCOUNT
-            return x
-          end
-        end
-
-        nil
-      end
-      memoize :fanbox_account_id
-
-      def agent
-        PixivWebAgent.build
-      end
-      memoize :agent
-
       def metadata
         if novel_id.present?
           return PixivApiClient.new.novel(novel_id)
         end
 
-        if fanbox_id.present?
-          return PixivApiClient.new.fanbox(fanbox_id)
-        end
-
         PixivApiClient.new.work(illust_id)
       end
       memoize :metadata

app/logical/sources/strategies/tumblr.rb

@@ -23,7 +23,7 @@ module Sources::Strategies
     OLD_IMAGE = %r{\Ahttps?://#{DOMAIN}/(?<dir>#{MD5}/)?#{FILENAME}_(?<size>\w+)\.#{EXT}\z}i
 
     IMAGE = %r{\Ahttps?://#{DOMAIN}/}i
-    VIDEO = %r{\Ahttps?://(?:vtt|ve\.media)\.tumblr\.com/}i
+    VIDEO = %r{\Ahttps?://(?:vtt|ve|va\.media)\.tumblr\.com/}i
     POST = %r{\Ahttps?://(?<blog_name>[^.]+)\.tumblr\.com/(?:post|image)/(?<post_id>\d+)}i
 
     def self.enabled?
@@ -168,7 +168,7 @@ module Sources::Strategies
       end
 
       candidates.find do |candidate|
-        http_exists?(candidate, headers)
+        http_exists?(candidate)
       end
     end
 

app/logical/sources/strategies/twitter.rb

@@ -200,7 +200,7 @@ module Sources::Strategies
     end
 
     def api_response
-      return {} unless self.class.enabled?
+      return {} unless self.class.enabled? && status_id.present?
       api_client.status(status_id)
     end
 

app/logical/tag_relationship_retirement_service.rb

@@ -11,14 +11,6 @@ module TagRelationshipRetirementService
     "This topic deals with tag relationships created two or more years ago that have not been used since. They will be retired. This topic will be updated as an automated system retires expired relationships."
   end
 
-  def dry_run
-    [TagAlias, TagImplication].each do |model|
-      each_candidate(model) do |rel|
-        puts "#{rel.relationship} #{rel.antecedent_name} -> #{rel.consequent_name} retired"
-      end
-    end
-  end
-
   def forum_topic
     topic = ForumTopic.where(title: forum_topic_title).first
     if topic.nil?

app/logical/upload_service/controller_helper.rb

@@ -7,13 +7,10 @@ class UploadService
         # this gets called from UploadsController#new so we need to preprocess async
         UploadPreprocessorDelayedStartJob.perform_later(url, ref, CurrentUser.user)
 
-        begin
+        strategy = Sources::Strategies.find(url, ref)
-          download = Downloads::File.new(url, ref)
+        remote_size = strategy.size
-          remote_size = download.size
-        rescue Exception
-        end
 
-        [upload, remote_size]
+        return [upload, remote_size]
       end
 
       if file

app/logical/upload_service/utils.rb

@@ -71,13 +71,13 @@ class UploadService
       return file if file.present?
       raise "No file or source URL provided" if upload.source_url.blank?
 
-      download = Downloads::File.new(upload.source_url, upload.referer_url)
+      strategy = Sources::Strategies.find(upload.source_url, upload.referer_url)
-      file, strategy = download.download!
+      file = strategy.download_file!
 
-      if download.data[:ugoira_frame_data].present?
+      if strategy.data[:ugoira_frame_data].present?
         upload.context = {
           "ugoira" => {
-            "frame_data" => download.data[:ugoira_frame_data],
+            "frame_data" => strategy.data[:ugoira_frame_data],
             "content_type" => "image/jpeg"
           }
         }

app/logical/validating_socket.rb

@@ -0,0 +1,27 @@
+# A TCPSocket wrapper that disallows connections to local or private IPs. Used for SSRF protection.
+# https://owasp.org/www-community/attacks/Server_Side_Request_Forgery
+
+require "resolv"
+
+class ValidatingSocket < TCPSocket
+  class ProhibitedIpError < StandardError; end
+
+  def initialize(hostname, port)
+    ip = validate_hostname!(hostname)
+    super(ip, port)
+  end
+
+  def validate_hostname!(hostname)
+    ip = IPAddress.parse(::Resolv.getaddress(hostname))
+    raise ProhibitedIpError, "Connection to #{hostname} failed; #{ip} is a prohibited IP" if prohibited_ip?(ip)
+    ip.to_s
+  end
+
+  def prohibited_ip?(ip)
+    if ip.ipv4?
+      ip.loopback? || ip.link_local? || ip.multicast? || ip.private?
+    elsif ip.ipv6?
+      ip.loopback? || ip.link_local? || ip.unique_local? || ip.unspecified?
+    end
+  end
+end

app/models/moderation_report.rb

@@ -34,7 +34,7 @@ class ModerationReport < ApplicationRecord
   def forum_topic
     topic = ForumTopic.find_by_title(forum_topic_title)
     if topic.nil?
-      CurrentUser.as_system do
+      CurrentUser.scoped(User.system) do
         topic = ForumTopic.create!(creator: User.system, title: forum_topic_title, category_id: 0, min_level: User::Levels::MODERATOR)
         forum_post = ForumPost.create!(creator: User.system, body: forum_topic_body, topic: topic)
       end

app/models/post_version.rb

@@ -33,7 +33,7 @@ class PostVersion < ApplicationRecord
     end
 
     def tag_matches(string)
-      tag = string.split(/\S+/)[0]
+      tag = string.match(/\S+/)[0]
       return all if tag.nil?
       tag = "*#{tag}*" unless tag =~ /\*/
       where_ilike(:tags, tag)

app/models/saved_search.rb

@@ -18,8 +18,7 @@ class SavedSearch < ApplicationRecord
         post_ids = Set.new
         queries.each do |query|
           redis_key = "search:#{query}"
-          # XXX change to `exists?` (ref: https://github.com-sds/mock_redis/pull/188
+          if redis.exists?(redis_key)
-          if redis.exists(redis_key)
             sub_ids = redis.smembers(redis_key).map(&:to_i)
             post_ids.merge(sub_ids)
           else
@@ -116,7 +115,7 @@ class SavedSearch < ApplicationRecord
 
       def populate(query, timeout: 10_000)
         redis_key = "search:#{query}"
-        return if redis.exists(redis_key)
+        return if redis.exists?(redis_key)
 
         post_ids = Post.with_timeout(timeout, [], query: query) do
           Post.system_tag_match(query).limit(QUERY_LIMIT).pluck(:id)

app/models/wiki_page.rb

@@ -53,7 +53,11 @@ class WikiPage < ApplicationRecord
     end
 
     def linked_to(title)
-      where(id: DtextLink.wiki_page.wiki_link.where(link_target: title).select(:model_id))
+      where(dtext_links: DtextLink.wiki_page.wiki_link.where(link_target: normalize_title(title)))
+    end
+
+    def not_linked_to(title)
+      where.not(dtext_links: DtextLink.wiki_page.wiki_link.where(link_target: normalize_title(title)))
     end
 
     def default_order
@@ -82,6 +86,10 @@ class WikiPage < ApplicationRecord
         q = q.linked_to(params[:linked_to])
       end
 
+      if params[:not_linked_to].present?
+        q = q.not_linked_to(params[:not_linked_to])
+      end
+
       if params[:hide_deleted].to_s.truthy?
         q = q.where("is_deleted = false")
       end
@@ -146,6 +154,7 @@ class WikiPage < ApplicationRecord
   end
 
   def self.normalize_title(title)
+    return if title.blank?
     title.downcase.delete_prefix("~").gsub(/[[:space:]]+/, "_").gsub(/__/, "_").gsub(/\A_|_\z/, "")
   end
 

app/policies/forum_post_policy.rb

@@ -24,7 +24,7 @@ class ForumPostPolicy < ApplicationPolicy
   end
 
   def votable?
-    unbanned? && show? && record.bulk_update_request.present? && record.bulk_update_request.is_pending?
+    unbanned? && show? && record.bulk_update_request.present? && record.bulk_update_request.is_pending? && record.bulk_update_request.user_id != user.id
   end
 
   def reportable?

app/views/artists/_show.html.erb

@@ -8,12 +8,14 @@
     <% if @artist.is_banned? && !policy(@artist).can_view_banned? %>
       <p>The artist requested removal of this page.</p>
     <% else %>
-      <% if @artist.wiki_page.present? %>
+      <% if @artist.wiki_page.present? && !@artist.wiki_page.is_deleted? %>
+        <div class="artist-wiki">
           <div class="prose">
             <%= format_text(@artist.wiki_page.body, :disable_mentions => true) %>
           </div>
 
           <p><%= link_to "View wiki page", @artist.wiki_page %></p>
+        </div>
       <% end %>
 
       <%= yield %>

app/views/explore/posts/missed_searches.html.erb

@@ -15,7 +15,7 @@
         </tr>
       </thead>
       <tbody>
-        <% @search_service.missed_search_rankings.each do |tags, count| %>
+        <% @missed_searches.each do |tags, count| %>
           <tr class="tag-type-<%= Tag.category_for(tags) %>">
             <td><%= link_to tags, posts_path(:tags => tags) %></td>
             <td>

app/views/explore/posts/searches.html.erb

@@ -13,7 +13,7 @@
         </tr>
       </thead>
       <tbody>
-        <% @search_service.post_search_rankings(@date).each do |tags, count| %>
+        <% @searches.each do |tags, count| %>
           <tr class="tag-type-<%= Tag.category_for(tags) %>">
             <td><%= link_to tags, posts_path(:tags => tags) %></td>
             <td style="text-align: right;"><%= count.to_i %></td>

app/views/wiki_pages/search.html.erb

@@ -4,6 +4,8 @@
       <%= f.input :title_normalize, label: "Title", hint: "Use * for wildcard searches", input_html: { "data-autocomplete": "wiki-page" } %>
       <%= f.input :other_names_match, label: "Other names", hint: "Use * for wildcard searches" %>
       <%= f.input :body_matches, label: "Body" %>
+      <%= f.input :linked_to, hint: "Which wikis link to the specified wiki.", input_html: { "data-autocomplete": "wiki-page" } %>
+      <%= f.input :not_linked_to, hint: "Which wikis do not link to the specified wiki.", input_html: { "data-autocomplete": "wiki-page" } %>
       <%= f.input :other_names_present, as: :select %>
       <%= f.input :hide_deleted, as: :select, include_blank: false %>
       <%= f.input :order, collection: [%w[Name title], %w[Date time], %w[Posts post_count]], include_blank: false %>

app/views/wiki_pages/show.html.erb

@@ -28,7 +28,7 @@
       <%= format_text(@wiki_page.body) %>
     <% end %>
 
-    <% if @wiki_page.artist %>
+    <% if @wiki_page.artist.present? && !@wiki_page.artist.is_deleted? %>
       <p><%= link_to "View artist", @wiki_page.artist %></p>
     <% end %>
 

config/cable.yml

@@ -1,10 +0,0 @@
-development:
-  adapter: async
-
-test:
-  adapter: test
-
-production:
-  adapter: redis
-  url: <%= ENV.fetch("REDIS_URL") { "redis://localhost:6379/1" } %>
-  channel_prefix: danbooru_production

config/danbooru_default_config.rb

@@ -317,13 +317,15 @@ module Danbooru
     # A list of tags that should be removed when a post is replaced. Regexes allowed.
     def post_replacement_tag_removals
       %w[replaceme .*_sample resized upscaled downscaled md5_mismatch
-      jpeg_artifacts corrupted_image source_request non-web_source]
+      jpeg_artifacts corrupted_image missing_image missing_sample missing_thumbnail
+      resolution_mismatch source_larger source_smaller source_request non-web_source]
     end
 
     # Posts with these tags will be highlighted in the modqueue.
     def modqueue_warning_tags
       %w[hard_translated self_upload nude_filter third-party_edit screencap
-      duplicate image_sample md5_mismatch resized upscaled downscaled]
+      duplicate image_sample md5_mismatch resized upscaled downscaled
+      resolution_mismatch source_larger source_smaller]
     end
 
     def stripe_secret_key
@@ -338,22 +340,6 @@ module Danbooru
     def twitter_api_secret
     end
 
-    # The default headers to be sent with outgoing http requests. Some external
-    # services will fail if you don't set a valid User-Agent.
-    def http_headers
-      {
-        "User-Agent" => "#{Danbooru.config.canonical_app_name}/#{Rails.application.config.x.git_hash}"
-      }
-    end
-
-    def httparty_options
-      # proxy example:
-      # {http_proxyaddr: "", http_proxyport: "", http_proxyuser: nil, http_proxypass: nil}
-      {
-        headers: Danbooru.config.http_headers
-      }
-    end
-
     # you should override this
     def email_key
       "zDMSATq0W3hmA5p3rKTgD"
@@ -374,14 +360,20 @@ module Danbooru
       false
     end
 
-    # reportbooru options - see https://github.com/r888888888/reportbooru
+    # The URL for the Reportbooru server (https://github.com/evazion/reportbooru).
+    # Optional. Used for tracking post views, popular searches, and missed searches.
+    # Set to http://localhost/mock/reportbooru to enable a fake reportbooru
+    # server for development purposes.
     def reportbooru_server
     end
 
     def reportbooru_key
     end
 
-    # iqdbs options - see https://github.com/r888888888/iqdbs
+    # The URL for the IQDBs server (https://github.com/evazion/iqdbs).
+    # Optional. Used for dupe detection and reverse image searches.
+    # Set to http://localhost/mock/iqdbs to enable a fake iqdb server for
+    # development purposes.
     def iqdbs_server
     end
 
@@ -459,6 +451,10 @@ module Danbooru
     def cloudflare_zone
     end
 
+    # The URL for the recommender server (https://github.com/evazion/recommender).
+    # Optional. Used to generate post recommendations.
+    # Set to http://localhost/mock/recommender to enable a fake recommender
+    # server for development purposes.
     def recommender_server
     end
 

config/docker/Dockerfile.danbooru

@@ -13,14 +13,20 @@ RUN \
     webpack \
     libvips-dev \
     libxml2-dev \
+    libxslt-dev \
+    zlib1g-dev \
     postgresql-server-dev-all && \
     # webpacker expects the binary to be called `yarn`, but debian/ubuntu installs it as `yarnpkg`.
     ln -sf /usr/bin/yarnpkg /usr/bin/yarn
 
 WORKDIR /build
 
+COPY .bundle .bundle
 COPY Gemfile Gemfile.lock ./
-RUN BUNDLE_DEPLOYMENT=true bundle install --jobs 4
+RUN \
+  bundle config set deployment true --local && \
+  bundle config set path vendor/bundle && \
+  bundle install --jobs 4
 
 COPY package.json yarn.lock ./
 RUN yarn install
@@ -44,6 +50,8 @@ RUN \
     mkvtoolnix \
     libvips \
     libxml2 \
+    libxslt1.1 \
+    zlib1g \
     postgresql-client
 
 USER danbooru

config/initializers/mechanize_patch.rb

@@ -1,64 +0,0 @@
-require 'mechanize'
-
-if Rails.env.test?
-  # something about the root certs on the travis ci image causes Mechanize
-  # to intermittently fail. this is a monkey patch to reset the connection
-  # after every request to avoid dealing wtiht he issue.
-  #
-  # from http://scottwb.com/blog/2013/11/09/defeating-the-infamous-mechanize-too-many-connection-resets-bug/
-  class Mechanize::HTTP::Agent
-    MAX_RESET_RETRIES = 10
-
-    # We need to replace the core Mechanize HTTP method:
-    #
-    #   Mechanize::HTTP::Agent#fetch
-    #
-    # with a wrapper that handles the infamous "too many connection resets"
-    # Mechanize bug that is described here:
-    #
-    #   https://github.com/sparklemotion/mechanize/issues/123
-    #
-    # The wrapper shuts down the persistent HTTP connection when it fails with
-    # this error, and simply tries again. In practice, this only ever needs to
-    # be retried once, but I am going to let it retry a few times
-    # (MAX_RESET_RETRIES), just in case.
-    #
-    def fetch_with_retry(
-      uri,
-      method    = :get,
-      headers   = {},
-      params    = [],
-      referer   = current_page,
-      redirects = 0
-    )
-      action      = "#{method.to_s.upcase} #{uri}"
-      retry_count = 0
-
-      begin
-        fetch_without_retry(uri, method, headers, params, referer, redirects)
-      rescue Net::HTTP::Persistent::Error => e
-        # Pass on any other type of error.
-        raise unless e.message =~ /too many connection resets/
-
-        # Pass on the error if we've tried too many times.
-        if retry_count >= MAX_RESET_RETRIES
-          print "R"
-          # puts "**** WARN: Mechanize retried connection reset #{MAX_RESET_RETRIES} times and never succeeded: #{action}"
-          raise
-        end
-
-        # Otherwise, shutdown the persistent HTTP connection and try again.
-        print "R"
-        # puts "**** WARN: Mechanize retrying connection reset error: #{action}"
-        retry_count += 1
-        self.http.shutdown
-        retry
-      end
-    end
-
-    # Alias so #fetch actually uses our new #fetch_with_retry to wrap the
-    # old one aliased as #fetch_without_retry.
-    alias fetch_without_retry fetch
-    alias fetch fetch_with_retry
-  end
-end

config/routes.rb

@@ -372,6 +372,14 @@ Rails.application.routes.draw do
   get "/static/contact" => "static#contact", :as => "contact"
   get "/static/dtext_help" => "static#dtext_help", :as => "dtext_help"
 
+  get "/mock/recommender/recommend/:user_id" => "mock_services#recommender_recommend", as: "mock_recommender_recommend"
+  get "/mock/recommender/similiar/:post_id" => "mock_services#recommender_similar", as: "mock_recommender_similar"
+  get "/mock/reportbooru/missed_searches" => "mock_services#reportbooru_missed_searches", as: "mock_reportbooru_missed_searches"
+  get "/mock/reportbooru/post_searches/rank" => "mock_services#reportbooru_post_searches", as: "mock_reportbooru_post_searches"
+  get "/mock/reportbooru/post_views/rank" => "mock_services#reportbooru_post_views", as: "mock_reportbooru_post_views"
+  get "/mock/iqdbs/similar" => "mock_services#iqdbs_similar", as: "mock_iqdbs_similar"
+  post "/mock/iqdbs/similar" => "mock_services#iqdbs_similar"
+
   root :to => "posts#index"
 
   get "*other", :to => "static#not_found"

config/secrets.yml

@@ -1,26 +0,0 @@
-# Be sure to restart your server when you modify this file.
-
-# Your secret key is used for verifying the integrity of signed cookies.
-# If you change this key, all old signed cookies will become invalid!
-
-# Make sure the secret is at least 30 characters and all random,
-# no regular words or you'll be exposed to dictionary attacks.
-# You can use `rake secret` to generate a secure secret key.
-
-# Make sure the secrets in this file are kept private
-# if you're sharing your code publicly.
-
-development:
-  secret_key_base: bcc62a512b9c055c292c17742f1e65bd6d88fa37f4d01c8475103809f3ac4c03e3e98605c47d55cd8801333010ea98920a61b722770629926759624bce732539
-
-test:
-  secret_key_base: 60e32a818af77bdfc40bca866e3b4d7b88d7ba767057ffc9e4532279358af8c67d42f2b99c084b700727303ce25b812a592b52723ebc1e3b812fd09a1f969435
-
-# Do not keep production secrets in the repository,
-# instead read values from the environment.
-production:
-  secret_key_base: <%= ENV["SECRET_KEY_BASE"] %>
-
-staging:
-  secret_key_base: <%= ENV["SECRET_KEY_BASE"] %>
-  

config/storage.yml

@@ -1,34 +0,0 @@
-test:
-  service: Disk
-  root: <%= Rails.root.join("tmp/storage") %>
-
-local:
-  service: Disk
-  root: <%= Rails.root.join("storage") %>
-
-# Use rails credentials:edit to set the AWS secrets (as aws:access_key_id|secret_access_key)
-# amazon:
-#   service: S3
-#   access_key_id: <%= Rails.application.credentials.dig(:aws, :access_key_id) %>
-#   secret_access_key: <%= Rails.application.credentials.dig(:aws, :secret_access_key) %>
-#   region: us-east-1
-#   bucket: your_own_bucket
-
-# Remember not to checkin your GCS keyfile to a repository
-# google:
-#   service: GCS
-#   project: your_project
-#   credentials: <%= Rails.root.join("path/to/gcs.keyfile") %>
-#   bucket: your_own_bucket
-
-# Use rails credentials:edit to set the Azure Storage secret (as azure_storage:storage_access_key)
-# microsoft:
-#   service: AzureStorage
-#   storage_account_name: your_account_name
-#   storage_access_key: <%= Rails.application.credentials.dig(:azure_storage, :storage_access_key) %>
-#   container: your_container_name
-
-# mirror:
-#   service: Mirror
-#   primary: local
-#   mirrors: [ amazon, google, microsoft ]

config/unicorn/production.rb

@@ -6,7 +6,7 @@ worker_processes 20
 
 timeout 180
 # listen "127.0.0.1:9000", :tcp_nopush => true
-listen "/tmp/.unicorn.sock", :backlog => 512
+listen "/tmp/.unicorn.sock", backlog: 1024
 
 # Spawn unicorn master worker for user apps (group: apps)
 user 'danbooru', 'danbooru'

package.json

@@ -27,8 +27,10 @@
     "webpack-cli": "^3.3.0"
   },
   "devDependencies": {
-    "eslint": "^6.0.0",
+    "babel-eslint": "^10.1.0",
+    "eslint": "^7.0.0",
     "eslint-loader": "^4.0.0",
+    "eslint-plugin-babel": "^5.3.0",
     "eslint-plugin-ignore-erb": "^0.1.1",
     "stylelint": "^13.0.0",
     "stylelint-config-standard": "^20.0.0",

script/install/app_server.sh

@@ -1,101 +0,0 @@
-#!/bin/bash
-
-# this is a version of the install script designed to be run on
-# app servers (that is, they won't install PostgreSQL server).
-#
-# Run: curl -L -s https://raw.githubusercontent.com/r888888888/danbooru/master/script/install/app_server.sh | sh
-
-export RUBY_VERSION=2.6.3
-export GITHUB_INSTALL_SCRIPTS=https://raw.githubusercontent.com/r888888888/danbooru/master/script/install
-export VIPS_VERSION=8.7.0
-
-if [[ "$(whoami)" != "root" ]] ; then
-  echo "You must run this script as root"
-  exit 1
-fi
-
-echo "* DANBOORU INSTALLATION SCRIPT"
-echo "*"
-echo "* This script will install all the necessary packages to run Danbooru on an"
-echo "* Ubuntu server."
-echo
-
-echo -n "* Enter the VLAN IP address for this server: "
-read VLAN_IP_ADDR
-
-# Install packages
-echo "* Installing packages..."
-
-apt-get update
-apt-get -y install libssl-dev build-essential automake libxml2-dev libxslt-dev ncurses-dev sudo libreadline-dev flex bison ragel redis git curl libcurl4-openssl-dev sendmail-bin sendmail nginx ssh coreutils ffmpeg mkvtoolnix
-apt-get -y install libpq-dev postgresql-client
-apt-get -y install liblcms2-dev libjpeg-turbo8-dev libexpat1-dev libgif-dev libpng-dev libexif-dev
-
-# vrack specific stuff
-apt-get -y install vlan
-modprobe 8021q
-echo "8021q" >> /etc/modules
-vconfig add eno2 99
-ip addr add $VLAN_IP_ADDR/24 dev eno2.99
-ip link set up eno2.99
-curl -L -s $GITHUB_INSTALL_SCRIPTS/vrack-cfg.yaml -o /etc/netplan/01-netcfg.yaml
-
-curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -
-echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list
-curl -sSL https://deb.nodesource.com/setup_10.x | sudo -E bash -
-apt-get update
-apt-get -y install nodejs yarn
-apt-get remove cmdtest
-
-# compile and install libvips (the version in apt is too old)
-cd /tmp
-wget -q https://github.com/libvips/libvips/releases/download/v$VIPS_VERSION/vips-$VIPS_VERSION.tar.gz
-tar xzf vips-$VIPS_VERSION.tar.gz
-cd vips-$VIPS_VERSION
-./configure --prefix=/usr
-make install
-ldconfig
-
-# Create user account
-useradd -m danbooru
-chsh -s /bin/bash danbooru
-usermod -G danbooru,sudo danbooru
-
-# Set up Postgres
-git clone https://github.com/r888888888/test_parser.git /tmp/test_parser
-cd /tmp/test_parser
-make install
-
-# Install rbenv
-echo "* Installing rbenv..."
-cd /tmp
-sudo -u danbooru git clone git://github.com/sstephenson/rbenv.git ~danbooru/.rbenv
-sudo -u danbooru touch ~danbooru/.bash_profile
-echo 'export PATH="$HOME/.rbenv/bin:$PATH"' >> ~danbooru/.bash_profile
-echo 'eval "$(rbenv init -)"' >> ~danbooru/.bash_profile
-sudo -u danbooru mkdir -p ~danbooru/.rbenv/plugins
-sudo -u danbooru git clone git://github.com/sstephenson/ruby-build.git ~danbooru/.rbenv/plugins/ruby-build
-sudo -u danbooru bash -l -c "rbenv install $RUBY_VERSION"
-sudo -u danbooru bash -l -c "rbenv global $RUBY_VERSION"
-
-# Install gems
-echo "* Installing gems..."
-sudo -u danbooru bash -l -c 'gem install --no-ri --no-rdoc bundler'
-
-# Setup danbooru account
-echo "* Enter a new password for the danbooru account"
-passwd danbooru
-
-echo "* Setting up SSH keys for the danbooru account"
-sudo -u danbooru ssh-keygen
-sudo -u danbooru cat ~danbooru/.ssh/id_rsa.pub >> ~danbooru/.ssh/authorized_keys
-
-echo "* TODO:"
-echo "on kagamihara:"
-echo "script/install/distribute_new_pubkey.sh"
-echo
-echo "on this server:"
-echo "rsync -av kagamihara:/etc/nginx/nginx.conf /etc/nginx"
-echo "rsync -av kagamihara:/etc/nginx/conf.d /etc/nginx"
-echo "rsync -av kagamihara:/etc/nginx/sites-enabled /etc/nginx"
-echo "rsync -av kagamihara:/etc/logrotate.d /etc/logrotate.d"

script/install/distribute_new_pubkey.sh

@@ -1,10 +0,0 @@
-#!/bin/sh
-
-HOSTS="kagamihara shima saitou"
-
-echo "Enter new SSH pubkey: "
-read $key
-
-for host in $HOSTS ; do
-  ssh danbooru@$host echo $key >> .ssh/authorized_keys
-done

script/install/nginx.service.d-local.conf

@@ -1,2 +0,0 @@
-[Service]
-LimitNOFILE=10000

script/install/nginx/conf.d/common.conf

@@ -1,92 +0,0 @@
-ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
-ssl_ciphers 'ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS';
-ssl_prefer_server_ciphers on;
-ssl_session_timeout 4h;
-ssl_session_cache shared:SSL:20m;
-ssl_session_tickets off;
-ssl_stapling on;
-ssl_stapling_verify on;
-ssl_dhparam /etc/nginx/ssl/dhparam.pem;
-resolver 8.8.8.8 8.8.4.4;
-
-root /var/www/danbooru/current/public;
-index index.html;
-access_log off;
-error_log /var/www/danbooru/shared/log/server.error.log;
-try_files $uri/index.html $uri.html $uri @app;
-client_max_body_size 35m;
-error_page 503 @maintenance;
-error_page 404 /404.html;
-error_page 500 502 503 504 /500.html;
-
-location /assets {
-  expires max;
-  break;
-}
-
-location /data/preview {
-  expires max;
-  break;
-}
-
-location /posts/mobile {
-  return 404;
-}
-
-location /users {
-  limit_req zone=users burst=5;
-  limit_req_status 429;
-  try_files $uri @app_server;
-}
-
-location /posts {
-  limit_req zone=posts burst=20;
-  limit_req_status 429;
-  try_files $uri @app_server;
-}
-
-location /data {
-  valid_referers none *.donmai.us donmai.us ~\.google\. ~\.bing\. ~\.yahoo\.;
-
-  if ($invalid_referer) {
-    return 403;
-  }
-
-  rewrite ^/data/sample/__.+?__(.+) /data/sample/$1 last;
-  rewrite ^/data/__.+?__(.+) /data/$1 last;
-
-  expires max;
-  break;
-}
-
-location /maintenance.html {
-  expires 10;
-}
-
-if (-f $document_root/maintenance.html) {
-  return 503;
-}
-
-if ($http_user_agent ~ (WinHttp\.WinHttpRequest\.5) ) {
-  return 403;
-}
-
-location @maintenance {
-  rewrite ^(.*)$ /maintenance.html last;
-}
-
-location @app_server {
-  proxy_pass http://app_server;
-  proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-  proxy_set_header X-Real-IP $remote_addr;
-  proxy_redirect off;
-  proxy_set_header Host $http_host;
-  proxy_set_header X-Forwarded-Proto $scheme;
-  proxy_buffer_size 128k;
-  proxy_buffers 4 256k;
-  proxy_busy_buffers_size 256k;
-}
-
-location / {
-  try_files $uri @app_server;
-}

script/install/nginx/nginx.conf

@@ -1,82 +0,0 @@
-user www-data;
-worker_processes auto;
-pid /var/run/nginx.pid;
-
-events {
-  use epoll;
-  worker_connections 10000;
-  multi_accept on;
-  accept_mutex on;
-}
-
-http {
-  limit_req_zone $binary_remote_addr zone=users:10m rate=5r/s;
-  limit_req_zone $binary_remote_addr zone=posts:100m rate=10r/s;
-
-  ##
-  # Basic Settings
-  ##
-
-  sendfile on;
-  tcp_nopush on;
-  tcp_nodelay on;
-  keepalive_timeout 5;
-  types_hash_max_size 2048;
-  # server_tokens off;
-
-  server_names_hash_bucket_size 128;
-  # server_name_in_redirect off;
-
-  include /etc/nginx/mime.types;
-  default_type application/octet-stream;
-
-  ##
-  # Logging Settings
-  ##
-
-  access_log off;
-  error_log /var/log/nginx/error.log;
-
-  ##
-  # Gzip Settings
-  ##
-
-  gzip on;
-  gzip_disable "msie6";
-
-  gzip_http_version 1.1;
-  gzip_vary on;
-  gzip_comp_level 5;
-  gzip_proxied any;
-  gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/rss+xml text/javascript application/atom+xml;
-
-  # curl https://www.cloudflare.com/ips-v4 | sort
-  set_real_ip_from 103.21.244.0/22;
-  set_real_ip_from 103.22.200.0/22;
-  set_real_ip_from 103.31.4.0/22;
-  set_real_ip_from 104.16.0.0/12;
-  set_real_ip_from 108.162.192.0/18;
-  set_real_ip_from 131.0.72.0/22;
-  set_real_ip_from 141.101.64.0/18;
-  set_real_ip_from 162.158.0.0/15;
-  set_real_ip_from 172.64.0.0/13;
-  set_real_ip_from 173.245.48.0/20;
-  set_real_ip_from 188.114.96.0/20;
-  set_real_ip_from 190.93.240.0/20;
-  set_real_ip_from 197.234.240.0/22;
-  set_real_ip_from 198.41.128.0/17;
-  set_real_ip_from 199.27.128.0/21;
-
-  # curl https://www.cloudflare.com/ips-v4 | sort
-  set_real_ip_from 2400:cb00::/32;
-  set_real_ip_from 2606:4700::/32;
-  set_real_ip_from 2803:f800::/32;
-  set_real_ip_from 2405:b500::/32;
-  set_real_ip_from 2405:8100::/32;
-  set_real_ip_from 2a06:98c0::/29;
-  set_real_ip_from 2c0f:f248::/32;
-
-  real_ip_header CF-Connecting-IP;
-
-  include /etc/nginx/sites-enabled/*.conf;
-}

script/install/nginx/sites-enabled/danbooru.conf

@@ -1,68 +0,0 @@
-server {
-  listen 443 ssl http2 default_server;
-  server_name danbooru.donmai.us;
-
-  ssl_certificate /etc/nginx/ssl/danbooru.chain.pem;
-  ssl_certificate_key /etc/nginx/ssl/danbooru.key;
-
-  include /etc/nginx/conf.d/common.conf;
-}
-
-server {
-  listen 443 ssl http2;
-  server_name safebooru.donmai.us;
-
-  ssl_certificate /etc/nginx/ssl/safebooru.chain.pem;
-  ssl_certificate_key /etc/nginx/ssl/safebooru.key;
-	
-  include /etc/nginx/conf.d/common.conf;
-}
-
-server {
-  listen 443 ssl http2;
-  server_name kagamihara.donmai.us;
-
-  ssl_certificate /etc/letsencrypt/live/kagamihara.donmai.us/fullchain.pem; # managed by Certbot
-  ssl_certificate_key /etc/letsencrypt/live/kagamihara.donmai.us/privkey.pem; # managed by Certbot
-
-  include /etc/nginx/conf.d/common.conf;
-}
-
-server {
-  listen 443 ssl http2;
-  server_name saitou.donmai.us;
-
-  ssl_certificate /etc/letsencrypt/live/saitou.donmai.us/fullchain.pem; # managed by Certbot
-  ssl_certificate_key /etc/letsencrypt/live/saitou.donmai.us/privkey.pem; # managed by Certbot
-
-  include /etc/nginx/conf.d/common.conf;
-}
-
-server {
-  listen 443 ssl http2;
-  server_name shima.donmai.us;
-
-  ssl_certificate /etc/letsencrypt/live/shima.donmai.us/fullchain.pem; # managed by Certbot
-  ssl_certificate_key /etc/letsencrypt/live/shima.donmai.us/privkey.pem; # managed by Certbot
-
-  include /etc/nginx/conf.d/common.conf;
-}
-
-# redirect HTTP to HTTPS.
-server {
-  listen 80;
-  server_name safebooru.donmai.us danbooru.donmai.us kagamihara.donmai.us saitou.donmai.us shima.donmai.us;
-  return 301 https://$host$request_uri;
-}
-
-# redirect donmai.us and www.donmai.us to danbooru.donmai.us.
-server {
-  listen 80;
-  listen 443 ssl;
-  server_name donmai.us	www.donmai.us;
-  return 301 https://danbooru.donmai.us$request_uri;
-}
-      
-upstream app_server {
-  server unix:/tmp/.unicorn.sock fail_timeout=0;
-}

script/install/vrack-cfg.yaml

@@ -1,10 +0,0 @@
-network:
-  version: 2
-  renderer: networkd
-  ethernets:
-    eno2: {}
-  vlans:
-    eno2.99:
-      id: 99
-      link: eno2
-      addresses: [172.16.0.1]

script/mock_services/README.md

@@ -1,7 +0,0 @@
-These are mocked services to be used for development purposes.
-
-- danbooru: port 3000
-- recommender: port 3001
-- iqdbs: port 3002
-- reportbooru: port 3003
-

script/mock_services/iqdbs.rb

@@ -1,14 +0,0 @@
-require 'sinatra'
-require 'json'
-require_relative './mock_service_helper'
-
-set :port, 3002
-
-configure do
-  POST_IDS = MockServiceHelper.fetch_post_ids
-end
-
-get '/similar' do
-  content_type :json
-  POST_IDS[0..10].map {|x| {post_id: x}}.to_json
-end

script/mock_services/mock_service_helper.rb

@@ -1,22 +0,0 @@
-require 'socket'
-require 'timeout'
-require 'httparty'
-
-module MockServiceHelper
-  module_function
-
-  DANBOORU_PORT = 3000
-
-  def fetch_post_ids
-    begin
-      s = TCPSocket.new("localhost", DANBOORU_PORT)
-      s.close
-    rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH
-      sleep 1
-      retry
-    end
-
-    json = HTTParty.get("http://localhost:#{DANBOORU_PORT}/posts.json?random=true&limit=10").body
-    return JSON.parse(json).map {|x| x["id"]}
-  end
-end

script/mock_services/recommender.rb

@@ -1,19 +0,0 @@
-require 'sinatra'
-require 'json'
-require_relative './mock_service_helper'
-
-set :port, 3001
-
-configure do
-  POST_IDS = MockServiceHelper.fetch_post_ids
-end
-
-get '/recommend/:user_id' do
-  content_type :json
-  POST_IDS[0..10].map {|x| [x, "1.000"]}.to_json
-end
-
-get '/similar/:post_id' do
-  content_type :json
-  POST_IDS[0..6].map {|x| [x, "1.000"]}.to_json
-end

script/mock_services/reportbooru.rb

@@ -1,22 +0,0 @@
-require 'sinatra'
-require 'json'
-
-set :port, 3003
-
-get '/missed_searches' do
-  content_type :text
-  return "abcdefg 10.0\nblahblahblah 20.0\n"
-end
-
-get '/post_searches/rank' do
-  content_type :json
-  return [["abc", 100], ["def", 200]].to_json
-end
-
-get '/reports/user_similarity' do
-  # todo
-end
-
-post '/post_views' do
-  # todo
-end

test/functional/artists_controller_test.rb

@@ -4,11 +4,8 @@ class ArtistsControllerTest < ActionDispatch::IntegrationTest
   def assert_artist_found(expected_artist, source_url = nil)
     if source_url
       get_auth artists_path(format: "json", search: { url_matches: source_url }), @user
-      if response.body =~ /Net::OpenTimeout/
-        skip "Remote connection to #{source_url} failed"
-        return
-      end
     end
+
     assert_response :success
     json = JSON.parse(response.body)
     assert_equal(1, json.size, "Testing URL: #{source_url}")
@@ -17,10 +14,6 @@ class ArtistsControllerTest < ActionDispatch::IntegrationTest
 
   def assert_artist_not_found(source_url)
     get_auth artists_path(format: "json", search: { url_matches: source_url }), @user
-    if response.body =~ /Net::OpenTimeout/
-      skip "Remote connection to #{source_url} failed"
-      return
-    end
 
     assert_response :success
     json = JSON.parse(response.body)
@@ -54,6 +47,22 @@ class ArtistsControllerTest < ActionDispatch::IntegrationTest
         get artist_path(@artist.id)
         assert_response :success
       end
+
+      should "show active wikis" do
+        as(@user) { create(:wiki_page, title: @artist.name) }
+        get artist_path(@artist.id)
+
+        assert_response :success
+        assert_select ".artist-wiki", count: 1
+      end
+
+      should "not show deleted wikis" do
+        as(@user) { create(:wiki_page, title: @artist.name, is_deleted: true) }
+        get artist_path(@artist.id)
+
+        assert_response :success
+        assert_select ".artist-wiki", count: 0
+      end
     end
 
     context "new action" do

test/functional/forum_post_votes_controller_test.rb

@@ -36,6 +36,13 @@ class ForumPostVotesControllerTest < ActionDispatch::IntegrationTest
           assert_response 403
         end
       end
+
+      should "not allow creators to vote on their own BURs" do
+        assert_difference("ForumPostVote.count", 0) do
+          post_auth forum_post_votes_path(format: :js), @bulk_update_request.user, params: { forum_post_id: @forum_post.id, forum_post_vote: { score: 1 }}
+          assert_response 403
+        end
+      end
     end
 
     context "destroy action" do

test/functional/mock_services_controller_test.rb

@@ -0,0 +1,28 @@
+require 'test_helper'
+
+class MockServicesControllerTest < ActionDispatch::IntegrationTest
+  context "The mock services controller" do
+    setup do
+      create(:post)
+      create(:tag)
+    end
+
+    context "for all actions" do
+      should "work" do
+        paths = [
+          mock_recommender_recommend_path(42),
+          mock_recommender_similar_path(42),
+          mock_reportbooru_missed_searches_path,
+          mock_reportbooru_post_searches_path,
+          mock_reportbooru_post_views_path,
+          mock_iqdbs_similar_path,
+        ]
+
+        paths.each do |path|
+          get path
+          assert_response :success
+        end
+      end
+    end
+  end
+end

test/functional/post_versions_controller_test.rb

@@ -41,6 +41,13 @@ class PostVersionsControllerTest < ActionDispatch::IntegrationTest
         assert_response :success
         assert_equal @post.versions[1].id, response.parsed_body[0]["id"].to_i
       end
+
+      should "list all versions for search[tag_matches]" do
+        get post_versions_path, as: :json, params: { search: { tag_matches: "tagme" }}
+        assert_response :success
+        assert_equal @post.versions[0].id, response.parsed_body[0]["id"].to_i
+        assert_equal 1, response.parsed_body.length
+      end
     end
 
     context "undo action" do

test/functional/uploads_controller_test.rb

@@ -1,15 +1,30 @@
 require 'test_helper'
 
 class UploadsControllerTest < ActionDispatch::IntegrationTest
-  def assert_uploaded(file_path, user, **upload_params)
+  def self.should_upload_successfully(source)
-    file = Rack::Test::UploadedFile.new("#{Rails.root}/#{file_path}")
+    should "upload successfully from #{source}" do
-
+      assert_successful_upload(source, user: create(:user, created_at: 1.month.ago))
-    assert_difference(["Upload.count", "Post.count"]) do
+    end
-      post_auth uploads_path, user, params: { upload: { file: file, **upload_params }}
-      assert_redirected_to Upload.last
   end
 
-    Upload.last
+  def assert_successful_upload(source_or_file_path, user: @user, **params)
+    if source_or_file_path =~ %r{\Ahttps?://}i
+      source = { source: source_or_file_path }
+    else
+      file = Rack::Test::UploadedFile.new(Rails.root.join(source_or_file_path))
+      source = { file: file }
+    end
+
+    assert_difference(["Upload.count"]) do
+      post_auth uploads_path, user, params: { upload: { tag_string: "abc", rating: "e", **source, **params }}
+    end
+
+    upload = Upload.last
+    assert_response :redirect
+    assert_redirected_to upload
+    assert_equal("completed", upload.status)
+    assert_equal(Post.last, upload.post)
+    assert_equal(upload.post.md5, upload.md5)
   end
 
   context "The uploads controller" do
@@ -18,6 +33,17 @@ class UploadsControllerTest < ActionDispatch::IntegrationTest
       mock_iqdb_service!
     end
 
+    context "image proxy action" do
+      should "work" do
+        url = "https://i.pximg.net/img-original/img/2017/11/21/17/06/44/65985331_p0.png"
+        get_auth image_proxy_uploads_path, @user, params: { url: url }
+
+        assert_response :success
+        assert_equal("image/png", response.media_type)
+        assert_equal(15_573, response.body.size)
+      end
+    end
+
     context "batch action" do
       context "for twitter galleries" do
         should "render" do
@@ -259,32 +285,65 @@ class UploadsControllerTest < ActionDispatch::IntegrationTest
       end
 
       context "uploading a file from your computer" do
-        should "work for a jpeg file" do
+        should_upload_successfully("test/files/test.jpg")
-          upload = assert_uploaded("test/files/test.jpg", @user, tag_string: "aaa", rating: "e", source: "aaa")
+        should_upload_successfully("test/files/test.png")
-
+        should_upload_successfully("test/files/test-static-32x32.gif")
-          assert_equal("jpg", upload.post.file_ext)
+        should_upload_successfully("test/files/test-animated-86x52.gif")
-          assert_equal("aaa", upload.post.source)
+        should_upload_successfully("test/files/test-300x300.mp4")
-          assert_equal(500, upload.post.image_width)
+        should_upload_successfully("test/files/test-512x512.webm")
-          assert_equal(335, upload.post.image_height)
+        should_upload_successfully("test/files/compressed.swf")
       end
 
-        should "work for a webm file" do
+      context "uploading a file from a source" do
-          upload = assert_uploaded("test/files/test-512x512.webm", @user, tag_string: "aaa", rating: "e", source: "aaa")
+        should_upload_successfully("https://www.artstation.com/artwork/04XA4")
+        should_upload_successfully("https://dantewontdie.artstation.com/projects/YZK5q")
+        should_upload_successfully("https://cdna.artstation.com/p/assets/images/images/006/029/978/large/amama-l-z.jpg")
 
-          assert_equal("webm", upload.post.file_ext)
+        should_upload_successfully("https://www.deviantart.com/aeror404/art/Holiday-Elincia-424551484")
-          assert_equal("aaa", upload.post.source)
+        should_upload_successfully("https://noizave.deviantart.com/art/test-no-download-697415967")
-          assert_equal(512, upload.post.image_width)
+        should_upload_successfully("https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/8b472d70-a0d6-41b5-9a66-c35687090acc/d23jbr4-8a06af02-70cb-46da-8a96-42a6ba73cdb4.jpg/v1/fill/w_786,h_1017,q_70,strp/silverhawks_quicksilver_by_edsfox_d23jbr4-pre.jpg")
-          assert_equal(512, upload.post.image_height)
-        end
 
-        should "work for a flash file" do
+        should_upload_successfully("https://www.hentai-foundry.com/pictures/user/Afrobull/795025/kuroeda")
-          upload = assert_uploaded("test/files/compressed.swf", @user, tag_string: "aaa", rating: "e", source: "aaa")
+        should_upload_successfully("https://pictures.hentai-foundry.com/a/Afrobull/795025/Afrobull-795025-kuroeda.png")
 
-          assert_equal("swf", upload.post.file_ext)
+        should_upload_successfully("https://yande.re/post/show/482880")
-          assert_equal("aaa", upload.post.source)
+        should_upload_successfully("https://files.yande.re/image/7ecfdead705d7b956b26b1d37b98d089/yande.re%20482880.jpg")
-          assert_equal(607, upload.post.image_width)
+
-          assert_equal(756, upload.post.image_height)
+        should_upload_successfully("https://konachan.com/post/show/270916")
-        end
+        should_upload_successfully("https://konachan.com/image/ca12cdb79a66d242e95a6f958341bf05/Konachan.com%20-%20270916.png")
+
+        should_upload_successfully("http://lohas.nicoseiga.jp/o/910aecf08e542285862954017f8a33a8c32a8aec/1433298801/4937663")
+        should_upload_successfully("http://seiga.nicovideo.jp/seiga/im4937663")
+        should_upload_successfully("https://seiga.nicovideo.jp/image/source/9146749")
+        should_upload_successfully("https://seiga.nicovideo.jp/watch/mg389884")
+        should_upload_successfully("https://dic.nicovideo.jp/oekaki/52833.png")
+        should_upload_successfully("https://lohas.nicoseiga.jp/o/971eb8af9bbcde5c2e51d5ef3a2f62d6d9ff5552/1589933964/3583893")
+        should_upload_successfully("http://lohas.nicoseiga.jp/priv/3521156?e=1382558156&h=f2e089256abd1d453a455ec8f317a6c703e2cedf")
+        should_upload_successfully("http://lohas.nicoseiga.jp/priv/b80f86c0d8591b217e7513a9e175e94e00f3c7a1/1384936074/3583893")
+        should_upload_successfully("http://lohas.nicoseiga.jp/material/5746c5/4459092")
+        # XXX should_upload_successfully("https://dcdn.cdn.nimg.jp/priv/62a56a7f67d3d3746ae5712db9cac7d465f4a339/1592186183/10466669")
+        # XXX should_upload_successfully("https://dcdn.cdn.nimg.jp/nicoseiga/lohas/o/8ba0a9b2ea34e1ef3b5cc50785bd10cd63ec7e4a/1592187477/10466669")
+
+        should_upload_successfully("http://nijie.info/view.php?id=213043")
+        should_upload_successfully("https://nijie.info/view_popup.php?id=213043")
+        should_upload_successfully("https://pic.nijie.net/03/nijie_picture/728995_20170505014820_0.jpg")
+
+        should_upload_successfully("https://pawoo.net/web/statuses/1202176")
+        should_upload_successfully("https://img.pawoo.net/media_attachments/files/000/128/953/original/4c0a06087b03343f.png")
+
+        should_upload_successfully("https://www.pixiv.net/en/artworks/64476642")
+        should_upload_successfully("https://i.pximg.net/img-original/img/2017/08/18/00/09/21/64476642_p0.jpg")
+
+        should_upload_successfully("https://noizave.tumblr.com/post/162206271767")
+        should_upload_successfully("https://media.tumblr.com/3bbfcbf075ddf969c996641b264086fd/tumblr_os2buiIOt51wsfqepo1_1280.png")
+
+        should_upload_successfully("https://twitter.com/noizave/status/875768175136317440")
+        should_upload_successfully("https://pbs.twimg.com/media/DCdZ_FhUIAAYKFN?format=jpg&name=medium")
+        should_upload_successfully("https://pbs.twimg.com/profile_banners/1225702850002468864/1588597370/1500x500")
+        # XXX should_upload_successfully("https://video.twimg.com/tweet_video/EWHWVrmVcAAp4Vw.mp4")
+
+        should_upload_successfully("https://www.weibo.com/5501756072/J2UNKfbqV")
+        should_upload_successfully("https://wx1.sinaimg.cn/mw690/0060kO5aly1gezsyt5xvhj30ok0sgtc9.jpg")
       end
     end
   end

test/functional/wiki_pages_controller_test.rb

@@ -10,8 +10,11 @@ class WikiPagesControllerTest < ActionDispatch::IntegrationTest
     context "index action" do
       setup do
         as(@user) do
-          @wiki_page_abc = create(:wiki_page, :title => "abc")
+          @tagme = create(:wiki_page, title: "tagme")
-          @wiki_page_def = create(:wiki_page, :title => "def")
+          @deleted = create(:wiki_page, title: "deleted", is_deleted: true)
+          @vocaloid = create(:wiki_page, title: "vocaloid")
+          @miku = create(:wiki_page, title: "hatsune_miku", other_names: ["初音ミク"], body: "miku is a [[vocaloid]]")
+          create(:tag, name: "hatsune_miku", category: Tag.categories.character)
         end
       end
 
@@ -20,22 +23,24 @@ class WikiPagesControllerTest < ActionDispatch::IntegrationTest
         assert_response :success
       end
 
-      should "list all wiki_pages (with search)" do
-        get wiki_pages_path, params: {:search => {:title => "abc"}}
-        assert_response :success
-        assert_select "tr td:first-child", text: "abc"
-      end
-
-      should "list wiki_pages without tags with order=post_count" do
-        get wiki_pages_path, params: {:search => {:title => "abc", :order => "post_count"}}
-        assert_response :success
-        assert_select "tr td:first-child", text: "abc"
-      end
-
       should "redirect the legacy title param to the show page" do
-        get wiki_pages_path(title: "abc")
+        get wiki_pages_path(title: "tagme")
-        assert_redirected_to wiki_pages_path(search: { title_normalize: "abc" }, redirect: true)
+        assert_redirected_to wiki_pages_path(search: { title_normalize: "tagme" }, redirect: true)
       end
+
+      should respond_to_search(title: "tagme").with { @tagme }
+      should respond_to_search(title: "tagme", order: "post_count").with { @tagme }
+      should respond_to_search(title_normalize: "TAGME  ").with { @tagme }
+
+      should respond_to_search(tag: { category: Tag.categories.character }).with { @miku }
+      should respond_to_search(hide_deleted: "true").with { [@miku, @vocaloid, @tagme] }
+      should respond_to_search(linked_to: "vocaloid").with { @miku }
+      should respond_to_search(not_linked_to: "vocaloid").with { [@vocaloid, @deleted, @tagme] }
+
+      should respond_to_search(other_names_match: "初音ミク").with { @miku }
+      should respond_to_search(other_names_match: "初*").with { @miku }
+      should respond_to_search(other_names_present: "true").with { @miku }
+      should respond_to_search(other_names_present: "false").with { [@vocaloid, @deleted, @tagme] }
     end
 
     context "search action" do

test/test_helper.rb

@@ -43,15 +43,15 @@ class ActiveSupport::TestCase
 
   setup do
     Socket.stubs(:gethostname).returns("www.example.com")
-    WebMock.allow_net_connect!
 
-    storage_manager = StorageManager::Local.new(base_dir: Dir.mktmpdir("uploads-test-storage-"))
+    @temp_dir = Dir.mktmpdir("danbooru-temp-")
+    storage_manager = StorageManager::Local.new(base_dir: @temp_dir)
     Danbooru.config.stubs(:storage_manager).returns(storage_manager)
     Danbooru.config.stubs(:backup_storage_manager).returns(StorageManager::Null.new)
   end
 
   teardown do
-    FileUtils.rm_rf(Danbooru.config.storage_manager.base_dir)
+    FileUtils.rm_rf(@temp_dir)
     Cache.clear
   end
 
@@ -61,6 +61,8 @@ class ActiveSupport::TestCase
 end
 
 class ActionDispatch::IntegrationTest
+  extend ControllerHelper
+
   register_encoder :xml, response_parser: ->(body) { Nokogiri.XML(body) }
 
   def method_authenticated(method_name, url, user, **options)

test/test_helpers/controller_helper.rb

@@ -0,0 +1,47 @@
+module ControllerHelper
+  # A custom Shoulda matcher that tests that a controller's index endpoint
+  # responds to a search correctly. See https://thoughtbot.com/blog/shoulda-matchers.
+  #
+  # Usage:
+  #
+  #   # Tests that `/tags.json?search[name]=touhou` returns the `touhou` tag.
+  #   subject { TagsController }
+  #   setup { @touhou = create(:tag, name: "touhou") }
+  #   should respond_to_search(name: "touhou").with { @touhou }
+  #
+  def respond_to_search(search_params)
+    RespondToSearchMatcher.new(search_params)
+  end
+
+  class RespondToSearchMatcher < Struct.new(:params)
+    def description
+      "should respond to a search for #{params}"
+    end
+
+    def matches?(subject, &block)
+      search_params = { search: params }
+      expected_items = @test_case.instance_eval(&@expected)
+
+      @test_case.instance_eval do
+        # calls e.g. "wiki_pages_path" if we're in WikiPagesControllerTest.
+        index_url = send("#{subject.controller_path}_path")
+        get index_url, as: :json, params: search_params
+
+        expected_ids = Array(expected_items).map(&:id)
+        responded_ids = response.parsed_body.map { |item| item["id"] }
+
+        assert_response :success
+        assert_equal(expected_ids, responded_ids)
+      end
+    end
+
+    def with(&block)
+      @expected = block
+      self
+    end
+
+    def in_context(test_case)
+      @test_case = test_case
+    end
+  end
+end

test/test_helpers/download_test_helper.rb

@@ -1,10 +1,8 @@
 module DownloadTestHelper
   def assert_downloaded(expected_filesize, source, referer = nil)
-    download = Downloads::File.new(source, referer)
+    strategy = Sources::Strategies.find(source, referer)
-    tempfile, strategy = download.download!
+    file = strategy.download_file!
-    assert_equal(expected_filesize, tempfile.size, "Tested source URL: #{source}")
+    assert_equal(expected_filesize, file.size, "Tested source URL: #{source}")
-  rescue Net::OpenTimeout
-    skip "Remote connection to #{source} failed"
   end
 
   def assert_rewritten(expected_source, test_source, test_referer = nil)
@@ -16,19 +14,4 @@ module DownloadTestHelper
   def assert_not_rewritten(source, referer = nil)
     assert_rewritten(source, source, referer)
   end
-
-  def assert_http_exists(url, headers: {})
-    res = HTTParty.head(url, Danbooru.config.httparty_options.deep_merge(headers: headers))
-    assert_equal(true, res.success?)
-  end
-
-  def assert_http_status(code, url, headers: {})
-    res = HTTParty.head(url, Danbooru.config.httparty_options.deep_merge(headers: headers))
-    assert_equal(code, res.code)
-  end
-
-  def assert_http_size(size, url, headers: {})
-    res = HTTParty.head(url, Danbooru.config.httparty_options.deep_merge(headers: headers))
-    assert_equal(size, res.content_length)
-  end
 end

test/test_helpers/reportbooru_helper.rb

@@ -1,7 +1,7 @@
 module ReportbooruHelper
-  def mock_request(url, method: :get, status: 200, body: nil, http: Danbooru::Http.any_instance)
+  def mock_request(url, method: :get, status: 200, body: nil, http: Danbooru::Http.any_instance, **options)
     response = HTTP::Response.new(status: status, body: body, version: "1.1")
-    http.stubs(method).with(url).returns(response)
+    http.stubs(method).with(url, **options).returns(response)
   end
 
   def mock_post_search_rankings(date = Date.today, rankings)

test/unit/artist_test.rb

@@ -6,15 +6,11 @@ class ArtistTest < ActiveSupport::TestCase
 
     assert_equal(1, artists.size)
     assert_equal(expected_name, artists.first.name, "Testing URL: #{source_url}")
-  rescue Net::OpenTimeout, PixivApiClient::Error
-    skip "Remote connection failed for #{source_url}"
   end
 
   def assert_artist_not_found(source_url)
     artists = ArtistFinder.find_artists(source_url).to_a
     assert_equal(0, artists.size, "Testing URL: #{source_url}")
-  rescue Net::OpenTimeout
-    skip "Remote connection failed for #{source_url}"
   end
 
   context "An artist" do
@@ -172,15 +168,11 @@ class ArtistTest < ActiveSupport::TestCase
       a2 = FactoryBot.create(:artist, :name => "subway", :url_string => "http://subway.com/x/test.jpg")
       a3 = FactoryBot.create(:artist, :name => "minko", :url_string => "https://minko.com/x/test.jpg")
 
-      begin
       assert_artist_found("rembrandt", "http://rembrandt.com/x/test.jpg")
       assert_artist_found("rembrandt", "http://rembrandt.com/x/another.jpg")
       assert_artist_not_found("http://nonexistent.com/test.jpg")
       assert_artist_found("minko", "https://minko.com/x/test.jpg")
       assert_artist_found("minko", "http://minko.com/x/test.jpg")
-      rescue Net::OpenTimeout
-        skip "network failure"
-      end
     end
 
     should "be case-insensitive to domains when finding matches by url" do

test/unit/cloudflare_service_test.rb

@@ -1,5 +1,4 @@
 require 'test_helper'
-require 'webmock/minitest'
 
 class CloudflareServiceTest < ActiveSupport::TestCase
   def setup
@@ -8,16 +7,11 @@ class CloudflareServiceTest < ActiveSupport::TestCase
 
   context "#purge_cache" do
     should "make calls to cloudflare's api" do
-      stub_request(:any, "api.cloudflare.com")
+      url = "http://www.example.com/file.jpg"
-      @cloudflare.purge_cache(["http://localhost/file.txt"])
+      mock_request("https://api.cloudflare.com/client/v4/zones/123/purge_cache", method: :delete, json: { files: [url] })
 
-      assert_requested(:delete, "https://api.cloudflare.com/client/v4/zones/123/purge_cache", times: 1)
+      response = @cloudflare.purge_cache([url])
-    end
+      assert_equal(200, response.status)
-  end
-
-  context "#ips" do
-    should "work" do
-      refute_empty(@cloudflare.ips)
     end
   end
 end

test/unit/danbooru_http_test.rb

@@ -4,18 +4,20 @@ class DanbooruHttpTest < ActiveSupport::TestCase
   context "Danbooru::Http" do
     context "#get method" do
       should "work for all basic methods" do
-        %i[get put post delete].each do |method|
+        %i[get head put post delete].each do |method|
           response = Danbooru::Http.send(method, "https://httpbin.org/status/200")
           assert_equal(200, response.status)
         end
       end
 
       should "follow redirects" do
+        skip "Skipping test (https://github.com/postmanlabs/httpbin/issues/617)"
         response = Danbooru::Http.get("https://httpbin.org/absolute-redirect/3")
         assert_equal(200, response.status)
       end
 
       should "fail if redirected too many times" do
+        skip "Skipping test (https://github.com/postmanlabs/httpbin/issues/617)"
         response = Danbooru::Http.get("https://httpbin.org/absolute-redirect/10")
         assert_equal(598, response.status)
       end
@@ -26,8 +28,10 @@ class DanbooruHttpTest < ActiveSupport::TestCase
       end
 
       should "fail if the request takes too long to download" do
-        response = Danbooru::Http.timeout(1).get("https://httpbin.org/drip?duration=5&numbytes=5")
+        # XXX should return status 599 instead
-        assert_equal(599, response.status)
+        assert_raises(HTTP::TimeoutError) do
+          response = Danbooru::Http.timeout(1).get("https://httpbin.org/drip?duration=10&numbytes=10").flush
+        end
       end
 
       should "automatically decompress gzipped responses" do
@@ -36,13 +40,131 @@ class DanbooruHttpTest < ActiveSupport::TestCase
         assert_equal(true, response.parse["gzipped"])
       end
 
-      should "cache requests" do
+      should "automatically parse html responses" do
-        response1 = Danbooru::Http.cache(1.minute).get("https://httpbin.org/uuid")
+        response = Danbooru::Http.get("https://httpbin.org/html")
+        assert_equal(200, response.status)
+        assert_instance_of(Nokogiri::HTML5::Document, response.parse)
+        assert_equal("Herman Melville - Moby-Dick", response.parse.css("h1").text)
+      end
+
+      should "automatically parse xml responses" do
+        response = Danbooru::Http.get("https://httpbin.org/xml")
+        assert_equal(200, response.status)
+        assert_equal(true, response.parse[:slideshow].present?)
+      end
+
+      should "track cookies between requests" do
+        http = Danbooru::Http.use(:session)
+
+        resp1 = http.get("https://httpbin.org/cookies/set/abc/1")
+        resp2 = http.get("https://httpbin.org/cookies/set/def/2")
+        resp3 = http.get("https://httpbin.org/cookies")
+        assert_equal({ abc: "1", def: "2" }, resp3.parse["cookies"].symbolize_keys)
+
+        resp4 = http.cookies(def: 3, ghi: 4).get("https://httpbin.org/cookies")
+        assert_equal({ abc: "1", def: "3", ghi: "4" }, resp4.parse["cookies"].symbolize_keys)
+      end
+    end
+
+    context "cache feature" do
+      should "cache multiple requests to the same url" do
+        http = Danbooru::Http.cache(1.hour)
+
+        response1 = http.get("https://httpbin.org/uuid")
         assert_equal(200, response1.status)
 
-        response2 = Danbooru::Http.cache(1.minute).get("https://httpbin.org/uuid")
+        response2 = http.get("https://httpbin.org/uuid")
         assert_equal(200, response2.status)
-        assert_equal(response2.body, response1.body)
+        assert_equal(response2.to_s, response1.to_s)
+      end
+
+      should "cache cookies correctly" do
+        http = Danbooru::Http.cache(1.hour)
+
+        resp1 = http.get("https://httpbin.org/cookies")
+        resp2 = http.get("https://httpbin.org/cookies/set/abc/1")
+        resp3 = http.get("https://httpbin.org/cookies/set/def/2")
+        resp4 = http.get("https://httpbin.org/cookies")
+
+        assert_equal(200, resp1.status)
+        assert_equal(200, resp2.status)
+        assert_equal(200, resp3.status)
+        assert_equal(200, resp4.status)
+
+        assert_equal({}, resp1.parse["cookies"].symbolize_keys)
+        assert_equal({ abc: "1" }, resp2.parse["cookies"].symbolize_keys)
+        assert_equal({ abc: "1", def: "2" }, resp3.parse["cookies"].symbolize_keys)
+        assert_equal({ abc: "1", def: "2" }, resp4.parse["cookies"].symbolize_keys)
+      end
+    end
+
+    context "retriable feature" do
+      should "retry immediately if no Retry-After header is sent" do
+        response_429 = ::HTTP::Response.new(status: 429, version: "1.1", body: "")
+        response_200 = ::HTTP::Response.new(status: 200, version: "1.1", body: "")
+        HTTP::Client.any_instance.expects(:perform).times(2).returns(response_429, response_200)
+
+        response = Danbooru::Http.use(:retriable).get("https://httpbin.org/status/429")
+        assert_equal(200, response.status)
+      end
+
+      should "retry if the Retry-After header is an integer" do
+        response_503 = ::HTTP::Response.new(status: 503, version: "1.1", headers: { "Retry-After": "1" }, body: "")
+        response_200 = ::HTTP::Response.new(status: 200, version: "1.1", body: "")
+        HTTP::Client.any_instance.expects(:perform).times(2).returns(response_503, response_200)
+
+        response = Danbooru::Http.use(:retriable).get("https://httpbin.org/status/503")
+        assert_equal(200, response.status)
+      end
+
+      should "retry if the Retry-After header is a date" do
+        response_503 = ::HTTP::Response.new(status: 503, version: "1.1", headers: { "Retry-After": 2.seconds.from_now.httpdate }, body: "")
+        response_200 = ::HTTP::Response.new(status: 200, version: "1.1", body: "")
+        HTTP::Client.any_instance.expects(:perform).times(2).returns(response_503, response_200)
+
+        response = Danbooru::Http.use(:retriable).get("https://httpbin.org/status/503")
+        assert_equal(200, response.status)
+      end
+    end
+
+    context "#download method" do
+      should "download files" do
+        response, file = Danbooru::Http.download_media("https://httpbin.org/bytes/1000")
+
+        assert_equal(200, response.status)
+        assert_equal(1000, file.size)
+      end
+
+      should "follow redirects when downloading files" do
+        skip "Skipping test (https://github.com/postmanlabs/httpbin/issues/617)"
+        response, file = Danbooru::Http.download_media("https://httpbin.org/redirect-to?url=https://httpbin.org/bytes/1000")
+
+        assert_equal(200, response.status)
+        assert_equal(1000, file.size)
+      end
+
+      should "fail if the url points to a private IP" do
+        assert_raises(Danbooru::Http::DownloadError) do
+          Danbooru::Http.public_only.download_media("https://127.0.0.1.xip.io")
+        end
+      end
+
+      should "fail if the url redirects to a private IP" do
+        assert_raises(Danbooru::Http::DownloadError) do
+          Danbooru::Http.public_only.download_media("https://httpbin.org/redirect-to?url=https://127.0.0.1.xip.io")
+        end
+      end
+
+      should "fail if a download is too large" do
+        assert_raises(Danbooru::Http::FileTooLargeError) do
+          response, file = Danbooru::Http.max_size(500).download_media("https://httpbin.org/bytes/1000")
+        end
+      end
+
+      should "fail if a streaming download is too large" do
+        assert_raises(Danbooru::Http::FileTooLargeError) do
+          response, file = Danbooru::Http.max_size(500).download_media("https://httpbin.org/stream-bytes/1000")
+        end
       end
     end
   end

test/unit/downloads/art_station_test.rb

@@ -3,53 +3,27 @@ require 'test_helper'
 module Downloads
   class ArtStationTest < ActiveSupport::TestCase
     context "a download for a (small) artstation image" do
-      setup do
-        @asset = "https://cdnb3.artstation.com/p/assets/images/images/003/716/071/small/aoi-ogata-hate-city.jpg?1476754974"
-        @download = Downloads::File.new(@asset)
-      end
-
       should "download the /4k/ image instead" do
-        file, strategy = @download.download!
+        assert_downloaded(1_880_910, "https://cdnb3.artstation.com/p/assets/images/images/003/716/071/small/aoi-ogata-hate-city.jpg?1476754974")
-        assert_equal(1_880_910, ::File.size(file.path))
       end
     end
 
     context "for an image where an original does not exist" do
-      setup do
-        @asset = "https://cdna.artstation.com/p/assets/images/images/004/730/278/large/mendel-oh-dragonll.jpg"
-        @download = Downloads::File.new(@asset)
-      end
-
       should "not try to download the original" do
-        file, strategy = @download.download!
+        assert_downloaded(483_192, "https://cdna.artstation.com/p/assets/images/images/004/730/278/large/mendel-oh-dragonll.jpg")
-        assert_equal(483_192, ::File.size(file.path))
       end
     end
 
     context "a download for an ArtStation image hosted on CloudFlare" do
-      setup do
-        @asset = "https://cdnb.artstation.com/p/assets/images/images/003/716/071/large/aoi-ogata-hate-city.jpg?1476754974"
-      end
-
       should "return the original file, not the polished file" do
+        @asset = "https://cdnb.artstation.com/p/assets/images/images/003/716/071/large/aoi-ogata-hate-city.jpg?1476754974"
         assert_downloaded(1_880_910, @asset)
       end
-
-      should "return the original filesize, not the polished filesize" do
-        assert_equal(1_880_910, Downloads::File.new(@asset).size)
-      end
     end
 
     context "a download for a https://$artist.artstation.com/projects/$id page" do
-      setup do
-        @source = "https://dantewontdie.artstation.com/projects/YZK5q"
-        @download = Downloads::File.new(@source)
-      end
-
       should "download the original image instead" do
-        file, strategy = @download.download!
+        assert_downloaded(247_350, "https://dantewontdie.artstation.com/projects/YZK5q")
-
-        assert_equal(247_350, ::File.size(file.path))
       end
     end
   end

test/unit/downloads/file_test.rb

@@ -1,81 +0,0 @@
-require 'test_helper'
-
-module Downloads
-  class FileTest < ActiveSupport::TestCase
-    context "A post download" do
-      setup do
-        @source = "http://www.google.com/intl/en_ALL/images/logo.gif"
-        @download = Downloads::File.new(@source)
-      end
-
-      context "for a banned IP" do
-        setup do
-          Resolv.expects(:getaddress).returns("127.0.0.1").at_least_once
-        end
-
-        should "not try to download the file" do
-          assert_raise(Downloads::File::Error) { Downloads::File.new("http://evil.com").download! }
-        end
-
-        should "not try to fetch the size" do
-          assert_raise(Downloads::File::Error) { Downloads::File.new("http://evil.com").size }
-        end
-
-        should "not follow redirects to banned IPs" do
-          url = "http://httpbin.org/redirect-to?url=http://127.0.0.1"
-          stub_request(:get, url).to_return(status: 301, headers: { "Location": "http://127.0.0.1" })
-
-          assert_raise(Downloads::File::Error) { Downloads::File.new(url).download! }
-        end
-
-        should "not follow redirects that resolve to a banned IP" do
-          url = "http://httpbin.org/redirect-to?url=http://127.0.0.1.nip.io"
-          stub_request(:get, url).to_return(status: 301, headers: { "Location": "http://127.0.0.1.xip.io" })
-
-          assert_raise(Downloads::File::Error) { Downloads::File.new(url).download! }
-        end
-      end
-
-      context "that fails" do
-        should "retry three times before giving up" do
-          HTTParty.expects(:get).times(3).raises(Errno::ETIMEDOUT)
-          assert_raises(Errno::ETIMEDOUT) { @download.download! }
-        end
-
-        should "return an uncorrupted file on the second try" do
-          bomb = stub("bomb")
-          bomb.stubs(:code).raises(IOError)
-          resp = stub("resp", success?: true)
-
-          chunk = stub("a")
-          chunk.stubs(:code).returns(200)
-          chunk.stubs(:size).returns(1)
-          chunk.stubs(:to_s).returns("a")
-
-          HTTParty.expects(:get).twice.multiple_yields(chunk, bomb).then.multiple_yields(chunk, chunk).returns(resp)
-          @download.stubs(:is_cloudflare?).returns(false)
-          tempfile, _strategy = @download.download!
-
-          assert_equal("aa", tempfile.read)
-        end
-      end
-
-      should "throw an exception when the file is larger than the maximum" do
-        assert_raise(Downloads::File::Error) do
-          @download.download!(max_size: 1)
-        end
-      end
-
-      should "store the file in the tempfile path" do
-        tempfile, strategy = @download.download!
-        assert_operator(tempfile.size, :>, 0, "should have data")
-      end
-
-      should "correctly save the file when following 302 redirects" do
-        download = Downloads::File.new("https://yande.re/post/show/578014")
-        file, strategy = download.download!(url: download.preview_url)
-        assert_equal(19134, file.size)
-      end
-    end
-  end
-end

test/unit/downloads/pixiv_test.rb

@@ -122,32 +122,17 @@ module Downloads
           assert_downloaded(@file_size, @file_url, @ref)
         end
       end
-
-      context "downloading a pixiv fanbox image" do
-        should_eventually "work" do
-          @source = "https://www.pixiv.net/fanbox/creator/12491073/post/82406"
-          @file_url = "https://fanbox.pixiv.net/images/post/82406/D833IKA7FIesJXL8xx39rrG0.jpeg"
-          @file_size = 873_387
-
-          assert_not_rewritten(@file_url, @source)
-          assert_downloaded(@file_size, @file_url, @source)
-        end
-      end
     end
 
     context "An ugoira site for pixiv" do
-      setup do
-        @download = Downloads::File.new("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364")
-        @tempfile, strategy = @download.download!
-        @tempfile.close!
-      end
-
       should "capture the data" do
-        assert_equal(2, @download.data[:ugoira_frame_data].size)
+        @strategy = Sources::Strategies.find("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364")
-        if @download.data[:ugoira_frame_data][0]["file"]
+
+        assert_equal(2, @strategy.data[:ugoira_frame_data].size)
+        if @strategy.data[:ugoira_frame_data][0]["file"]
           assert_equal([{"file" => "000000.jpg", "delay" => 125}, {"file" => "000001.jpg", "delay" => 125}], @download.data[:ugoira_frame_data])
         else
-          assert_equal([{"delay_msec" => 125}, {"delay_msec" => 125}], @download.data[:ugoira_frame_data])
+          assert_equal([{"delay_msec" => 125}, {"delay_msec" => 125}], @strategy.data[:ugoira_frame_data])
         end
       end
     end

test/unit/media_file_test.rb

@@ -98,6 +98,10 @@ class MediaFileTest < ActiveSupport::TestCase
     should "determine the correct extension for a flash file" do
       assert_equal(:swf, MediaFile.open("test/files/compressed.swf").file_ext)
     end
+
+    should "not fail for empty files" do
+      assert_equal(:bin, MediaFile.open("test/files/test-empty.bin").file_ext)
+    end
   end
 
   should "determine the correct md5 for a jpeg file" do

test/unit/post_query_builder_test.rb

@@ -277,6 +277,19 @@ class PostQueryBuilderTest < ActiveSupport::TestCase
       assert_tag_match([child, parent], "-child:garbage")
     end
 
+    should "return posts when using the status of the parent/child" do
+      parent_of_deleted = create(:post)
+      deleted = create(:post, is_deleted: true, tag_string: "parent:#{parent_of_deleted.id}")
+      child_of_deleted = create(:post, tag_string: "parent:#{deleted.id}")
+      all = [child_of_deleted, deleted, parent_of_deleted]
+
+      assert_tag_match([child_of_deleted], "parent:deleted")
+      assert_tag_match(all - [child_of_deleted], "-parent:deleted")
+
+      assert_tag_match([parent_of_deleted], "child:deleted")
+      assert_tag_match(all - [parent_of_deleted], "-child:deleted")
+    end
+
     should "return posts for the favgroup:<name> metatag" do
       post1 = create(:post)
       post2 = create(:post)
@@ -757,8 +770,8 @@ class PostQueryBuilderTest < ActiveSupport::TestCase
       create(:saved_search, query: "aaa", labels: ["zzz"], user: CurrentUser.user)
       create(:saved_search, query: "bbb", user: CurrentUser.user)
 
-      Redis.any_instance.stubs(:exists).with("search:aaa").returns(true)
+      Redis.any_instance.stubs(:exists?).with("search:aaa").returns(true)
-      Redis.any_instance.stubs(:exists).with("search:bbb").returns(true)
+      Redis.any_instance.stubs(:exists?).with("search:bbb").returns(true)
       Redis.any_instance.stubs(:smembers).with("search:aaa").returns([@post1.id])
       Redis.any_instance.stubs(:smembers).with("search:bbb").returns([@post2.id])
 

test/unit/reportbooru_service_test.rb

@@ -4,20 +4,20 @@ class ReportbooruServiceTest < ActiveSupport::TestCase
   def setup
     @service = ReportbooruService.new(reportbooru_server: "http://localhost:1234")
     @post = create(:post)
-    @date = "2000-01-01"
+    @date = Date.parse("2000-01-01")
   end
 
   context "#popular_posts" do
     should "return the list of popular posts on success" do
-      body = "[[#{@post.id},100.0]]"
+      mock_post_view_rankings(@date, [[@post.id, 100]])
-      @service.http.expects(:get).with("http://localhost:1234/post_views/rank?date=#{@date}").returns(HTTP::Response.new(status: 200, body: body, version: "1.1"))
 
       posts = @service.popular_posts(@date)
       assert_equal([@post], posts)
     end
 
     should "return nothing on failure" do
-      @service.http.expects(:get).with("http://localhost:1234/post_views/rank?date=#{@date}").returns(HTTP::Response.new(status: 500, body: "", version: "1.1"))
+      Danbooru::Http.any_instance.expects(:get).with("http://localhost:1234/post_views/rank?date=#{@date}").returns(HTTP::Response.new(status: 500, body: "", version: "1.1"))
+      Danbooru::Http.any_instance.expects(:get).with("http://localhost:1234/post_views/rank?date=#{@date.yesterday}").returns(HTTP::Response.new(status: 500, body: "", version: "1.1"))
 
       assert_equal([], @service.popular_posts(@date))
     end

test/unit/sources/nijie_test.rb

@@ -43,7 +43,7 @@ module Sources
 
       should "get the image url" do
         assert_equal("https://pic.nijie.net/03/nijie_picture/728995_20170505014820_0.jpg", @site.image_url)
-        assert_http_size(132_555, @site.image_url)
+        assert_downloaded(132_555, @site.image_url)
       end
 
       should "get the canonical url" do
@@ -53,7 +53,7 @@ module Sources
       should "get the preview url" do
         assert_equal("https://pic.nijie.net/03/__rs_l170x170/nijie_picture/728995_20170505014820_0.jpg", @site.preview_url)
         assert_equal([@site.preview_url], @site.preview_urls)
-        assert_http_exists(@site.preview_url)
+        assert_downloaded(132_555, @site.preview_url)
       end
 
       should "get the profile" do
@@ -187,8 +187,6 @@ module Sources
         desc = <<-EOS.strip_heredoc.chomp
           foo [b]bold[/b] [i]italics[/i] [s]strike[/s] red
 
-
-
           <http://nijie.info/view.php?id=218944>
         EOS
 
@@ -207,8 +205,8 @@ module Sources
         assert_equal("https://nijie.info/members.php?id=236014", site.profile_url)
         assert_nothing_raised { site.to_h }
 
-        assert_http_size(3619, site.image_url)
+        assert_downloaded(3619, site.image_url)
-        assert_http_exists(site.preview_url)
+        assert_downloaded(3619, site.preview_url)
       end
     end
 

test/unit/sources/pixiv_test.rb

@@ -15,10 +15,7 @@ module Sources
 
     def get_source(source)
       @site = Sources::Strategies.find(source)
-
       @site
-    rescue Net::OpenTimeout
-      skip "Remote connection to #{source} failed"
     end
 
     context "in all cases" do
@@ -73,17 +70,6 @@ module Sources
         end
       end
 
-      context "A https://www.pixiv.net/fanbox/creator/*/post/* source" do
-        should_eventually "work" do
-          @site = Sources::Strategies.find("http://www.pixiv.net/fanbox/creator/554149/post/82555")
-
-          assert_equal("TYONE(お仕事募集中)", @site.artist_name)
-          assert_equal("https://www.pixiv.net/users/554149", @site.profile_url)
-          assert_equal("https://fanbox.pixiv.net/images/post/82555/Lyyeb6dDLcQZmy09nqLZapuS.jpeg", @site.image_url)
-          assert_nothing_raised { @site.to_h }
-        end
-      end
-
       context "A https://www.pixiv.net/*/artworks/* source" do
         should "work" do
           @site = Sources::Strategies.find("https://www.pixiv.net/en/artworks/64476642")
@@ -249,6 +235,10 @@ module Sources
           assert_includes(@translated_tags, "foo")
         end
 
+        should "not translate tags for digital media" do
+          assert_equal(false, @tags.include?("Photoshop"))
+        end
+
         should "normalize 10users入り tags" do
           assert_includes(@tags, "風景10users入り")
           assert_includes(@translated_tags, "scenery")
@@ -280,7 +270,7 @@ module Sources
         should "not translate '1000users入り' to '1'" do
           FactoryBot.create(:tag, name: "1", post_count: 1)
           source = get_source("https://www.pixiv.net/member_illust.php?mode=medium&illust_id=60665428")
-          tags = %w[1000users入り Fate/GrandOrder アルジュナ(Fate) アルトリア・ペンドラゴン イシュタル(Fate) グランブルーファンタジー マシュ・キリエライト マーリン(Fate) 両儀式 手袋 CLIP\ STUDIO\ PAINT Photoshop]
+          tags = %w[1000users入り Fate/GrandOrder アルジュナ(Fate) アルトリア・ペンドラゴン イシュタル(Fate) グランブルーファンタジー マシュ・キリエライト マーリン(Fate) 両儀式 手袋]
 
           assert_equal(tags.sort, source.tags.map(&:first).sort)
           assert_equal(["fate/grand_order"], source.translated_tags.map(&:name))

test/unit/sources/tumblr_test.rb

@@ -169,7 +169,7 @@ module Sources
 
     context "The source for a 'http://ve.media.tumblr.com/*' video post with inline images" do
       setup do
-        @url = "https://ve.media.tumblr.com/tumblr_os31dkexhK1wsfqep.mp4"
+        @url = "https://va.media.tumblr.com/tumblr_os31dkexhK1wsfqep.mp4"
         @ref = "https://noizave.tumblr.com/post/162222617101"
       end
 
@@ -177,7 +177,7 @@ module Sources
         should "get the video and inline images" do
           site = Sources::Strategies.find(@url, @ref)
           urls = %w[
-            https://ve.media.tumblr.com/tumblr_os31dkexhK1wsfqep.mp4
+            https://va.media.tumblr.com/tumblr_os31dkexhK1wsfqep.mp4
             https://media.tumblr.com/afed9f5b3c33c39dc8c967e262955de2/tumblr_inline_os31dclyCR1v11u29_1280.png
           ]
 

test/unit/sources/twitter_test.rb

@@ -244,6 +244,14 @@ module Sources
       end
     end
 
+    context "A profile banner image" do
+      should "work" do
+        @site = Sources::Strategies.find("https://pbs.twimg.com/profile_banners/1225702850002468864/1588597370/1500x500")
+        assert_equal(@site.image_url, @site.url)
+        assert_nothing_raised { @site.to_h }
+      end
+    end
+
     context "A tweet containing non-normalized Unicode text" do
       should "be normalized to nfkc" do
         site = Sources::Strategies.find("https://twitter.com/aprilarcus/status/367557195186970624")