Merge branch 'master' into mobile-mode-default-image-size

.bundle/config

@@ -0,0 +1,3 @@
+---
+BUNDLE_BUILD__NOKOGIRI: "--use-system-libraries"
+BUNDLE_BUILD__NOKOGUMBO: "--without-libxml2"

.codeclimate.yml

@@ -1,17 +1,25 @@
 version: 2
 checks:
   argument-count:
+    enabled: false
+  complex-logic:
     config:
-      threshold: 4
+      threshold: 8
   file-lines:
     config:
-      threshold: 500
+      threshold: 1000
+  method-complexity:
+    config:
+      threshold: 15
   method-count:
-    config:
+    enabled: false
-      threshold: 40
   method-lines:
+    enabled: false
+  nested-control-flow:
     config:
-      threshold: 100
+      threshold: 4
+  return-statements:
+    enabled: false
 plugins:
   eslint:
     enabled: true

.codecov.yml

@@ -0,0 +1 @@
+comment: false

.editorconfig

@@ -0,0 +1,14 @@
+root = true
+
+[**.{js,rb,css,erb,md,json,yml}]
+indent_style = space
+indent_size = 2
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+[**.html.erb]
+indent_size = unset
+
+[app/javascript/vendor/**]
+indent_size = unset

.eslintrc.yml

@@ -8,9 +8,13 @@ parserOptions:
 globals:
   $: false
   require: false
+parser: babel-eslint
+plugins:
+  - babel
 rules:
   # https://eslint.org/docs/rules/
   array-callback-return: error
+  babel/no-unused-expressions: error
   block-scoped-var: error
   consistent-return: error
   default-case: error
@@ -32,7 +36,7 @@ rules:
   no-sequences: error
   no-shadow: error
   no-shadow-restricted-names: error
-  no-unused-expressions: error
+  #no-unused-expressions: error
   no-unused-vars:
     - error
     - argsIgnorePattern: "^_"

.github/workflows/test.yaml

@@ -1,20 +1,40 @@
-name: Test
+name: Github
 
-on: [push, pull_request]
+# Trigger on pushes to master or pull requests to master, but not both.
+on:
+  push:
+    branches:
+    - master
+  pull_request:
+    branches:
+    - master
 
 jobs:
+  # https://github.com/Marr11317/ConflictAdviser
+  notify-merge-conflicts:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: Marr11317/ConflictAdviser@v1
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          conflict_label: "merge conflict"
+          comment: 'Rebase needed'
+
   test:
     runs-on: ubuntu-latest
     container: ubuntu:20.04
 
+    defaults:
+      run:
+        shell: bash
+
     env:
       DEBIAN_FRONTEND: noninteractive
       PARALLEL_WORKERS: 8 # number of parallel tests to run
+      RUBYOPT: -W0 # silence ruby warnings
+      VIPS_WARNING: 0 # silence libvips warnings
 
-      # Code Climate configuration. https://docs.codeclimate.com/docs/finding-your-test-coverage-token
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
-      CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }}
-      GIT_COMMIT_SHA: ${{ github.sha }}
-      GIT_BRANCH: ${{ github.ref }}
 
       DATABASE_URL: postgresql://danbooru:danbooru@postgres/danbooru
       ARCHIVE_DATABASE_URL: postgresql://danbooru:danbooru@postgres/danbooru
@@ -45,32 +65,25 @@ jobs:
           POSTGRES_PASSWORD: danbooru
 
     steps:
-    - name: Check out code
-      uses: actions/checkout@v2
-#    - name: Save environment
-#      run: env | egrep "DANBOORU|DATABASE_URL" > ~/.env
-#    - name: Install docker-compose
-#      run: sudo apt-get update && sudo apt-get -y install docker-compose
-#    - name: Run tests
-#      run: docker-compose --env-file ~/.env -f config/docker/docker-compose.test.yaml -p danbooru up
     - name: Install OS dependencies
       run: |
         apt-get update
-        apt-get -y install --no-install-recommends build-essential ruby ruby-dev ruby-bundler git nodejs yarnpkg webpack ffmpeg mkvtoolnix libvips-dev libxml2-dev postgresql-server-dev-all wget
+        apt-get -y install --no-install-recommends build-essential ruby ruby-dev ruby-bundler git nodejs yarnpkg webpack ffmpeg mkvtoolnix libvips-dev libxml2-dev libxslt-dev zlib1g-dev postgresql-server-dev-all wget curl git
         ln -sf /usr/bin/yarnpkg /usr/bin/yarn
+
+    - name: Check out code
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+
     - name: Install Ruby dependencies
       run: BUNDLE_DEPLOYMENT=true bundle install --jobs 4
+
     - name: Install Javascript dependencies
       run: yarn install
+
     - name: Prepare database
       run: config/docker/prepare-tests.sh
-    # https://docs.codeclimate.com/docs/configuring-test-coverage
+
-    - name: Prepare test coverage for Code Climate
-      run: |
-        wget https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64
-        chmod +x test-reporter-latest-linux-amd64
-        ./test-reporter-latest-linux-amd64 before-build
     - name: Run tests
       run: bin/rails test
-    - name: Upload test coverage to Code Climate
-      run: ./test-reporter-latest-linux-amd64 after-build

.gitignore

@@ -2,7 +2,6 @@
 .yarn-integrity
 .gitconfig
 .git/
-.bundle/
 config/database.yml
 config/danbooru_local_config.rb
 config/deploy/*.rb

.rubocop.yml

@@ -1,8 +1,9 @@
+# some of these settings are overriden in also test/.rubocop.yml
+
 require:
   - rubocop-rails
 
 AllCops:
-  TargetRubyVersion: 2.7.0
   NewCops: enable
   Exclude:
     - "bin/*"
@@ -41,14 +42,22 @@ Layout/SpaceInsideHashLiteralBraces:
 Metrics/AbcSize:
   Enabled: false
 
+Metrics/BlockLength:
+  Max: 50
+  ExcludedMethods:
+    - concerning
+    - context
+    - should
+
 Metrics/BlockNesting:
+  CountBlocks: false
   Max: 4
 
 Metrics/ClassLength:
   Max: 500
 
 Metrics/CyclomaticComplexity:
-  Max: 10
+  Enabled: false
 
 Metrics/MethodLength:
   Max: 100
@@ -60,7 +69,7 @@ Metrics/ParameterLists:
   Max: 4
 
 Metrics/PerceivedComplexity:
-  Enabled: false
+  Max: 20
 
 Lint/InheritException:
   EnforcedStyle: standard_error
@@ -110,7 +119,9 @@ Style/NumericPredicate:
 Style/PercentLiteralDelimiters:
   PreferredDelimiters:
     "default": "[]"
-    "%r": "!!"
+
+Style/ParallelAssignment:
+  Enabled: false
 
 Style/PerlBackrefs:
   Enabled: false

.simplecov

@@ -1,8 +1,12 @@
 SimpleCov.start "rails" do
   add_group "Libraries", ["app/logical", "lib"]
   add_group "Presenters", "app/presenters"
-  #enable_coverage :branch
+  add_group "Policies", "app/policies"
-  #minimum_coverage line: 85, branch: 75
+  enable_coverage :branch
-  #minimum_coverage_by_file 50
+
-  #coverage_dir "tmp/coverage"
+  # https://github.com/codecov/codecov-ruby#submit-only-in-ci-example
+  if ENV["CODECOV_TOKEN"]
+    require "codecov"
+    SimpleCov.formatter = SimpleCov::Formatter::Codecov
+  end
 end

Gemfile

@@ -7,7 +7,6 @@ gem "pg"
 gem "delayed_job"
 gem "delayed_job_active_record"
 gem "simple_form"
-gem "mechanize"
 gem "whenever", :require => false
 gem "sanitize"
 gem 'ruby-vips'
@@ -28,14 +27,11 @@ gem 'daemons'
 gem 'oauth2'
 gem 'bootsnap'
 gem 'addressable'
-gem 'httparty'
 gem 'rakismet'
 gem 'recaptcha', require: "recaptcha/rails"
 gem 'activemodel-serializers-xml'
-gem 'jquery-rails'
 gem 'webpacker', '>= 4.0.x'
 gem 'rake'
-gem 'retriable'
 gem 'redis'
 gem 'request_store'
 gem 'builder'
@@ -47,9 +43,7 @@ gem 'http'
 gem 'activerecord-hierarchical_query'
 gem 'pundit'
 gem 'mail'
-
+gem 'nokogiri'
-# locked to 1.10.9 to workaround an incompatibility with nokogumbo 2.0.2.
-gem 'nokogiri', '~> 1.10.9'
 
 group :production, :staging do
   gem 'unicorn', :platforms => :ruby
@@ -65,7 +59,6 @@ end
 group :development do
   gem 'rubocop'
   gem 'rubocop-rails'
-  gem 'sinatra'
   gem 'meta_request'
   gem 'rack-mini-profiler'
   gem 'stackprof'
@@ -85,11 +78,11 @@ group :test do
   gem "factory_bot"
   gem "mocha", require: "mocha/minitest"
   gem "ffaker"
-  gem "simplecov", "~> 0.17.0", require: false
+  gem "simplecov", require: false
-  gem "webmock", require: "webmock/minitest"
   gem "minitest-ci"
   gem "minitest-reporters", require: "minitest/reporters"
   gem "mock_redis"
   gem "capybara"
   gem "selenium-webdriver"
+  gem "codecov", require: false
 end

Gemfile.lock

@@ -1,70 +1,70 @@
 GIT
   remote: https://github.com/evazion/dtext_rb.git
-  revision: 507e97e0963822c20351c82620c28cc8e23423d5
+  revision: a95bf1d537cbdba4585adb8e123f03f001f56fd7
   specs:
-    dtext_rb (1.10.5)
+    dtext_rb (1.10.6)
       nokogiri (~> 1.8)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    actioncable (6.0.3.1)
+    actioncable (6.0.3.2)
-      actionpack (= 6.0.3.1)
+      actionpack (= 6.0.3.2)
       nio4r (~> 2.0)
       websocket-driver (>= 0.6.1)
-    actionmailbox (6.0.3.1)
+    actionmailbox (6.0.3.2)
-      actionpack (= 6.0.3.1)
+      actionpack (= 6.0.3.2)
-      activejob (= 6.0.3.1)
+      activejob (= 6.0.3.2)
-      activerecord (= 6.0.3.1)
+      activerecord (= 6.0.3.2)
-      activestorage (= 6.0.3.1)
+      activestorage (= 6.0.3.2)
-      activesupport (= 6.0.3.1)
+      activesupport (= 6.0.3.2)
       mail (>= 2.7.1)
-    actionmailer (6.0.3.1)
+    actionmailer (6.0.3.2)
-      actionpack (= 6.0.3.1)
+      actionpack (= 6.0.3.2)
-      actionview (= 6.0.3.1)
+      actionview (= 6.0.3.2)
-      activejob (= 6.0.3.1)
+      activejob (= 6.0.3.2)
       mail (~> 2.5, >= 2.5.4)
       rails-dom-testing (~> 2.0)
-    actionpack (6.0.3.1)
+    actionpack (6.0.3.2)
-      actionview (= 6.0.3.1)
+      actionview (= 6.0.3.2)
-      activesupport (= 6.0.3.1)
+      activesupport (= 6.0.3.2)
       rack (~> 2.0, >= 2.0.8)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.2.0)
-    actiontext (6.0.3.1)
+    actiontext (6.0.3.2)
-      actionpack (= 6.0.3.1)
+      actionpack (= 6.0.3.2)
-      activerecord (= 6.0.3.1)
+      activerecord (= 6.0.3.2)
-      activestorage (= 6.0.3.1)
+      activestorage (= 6.0.3.2)
-      activesupport (= 6.0.3.1)
+      activesupport (= 6.0.3.2)
       nokogiri (>= 1.8.5)
-    actionview (6.0.3.1)
+    actionview (6.0.3.2)
-      activesupport (= 6.0.3.1)
+      activesupport (= 6.0.3.2)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.1, >= 1.2.0)
-    activejob (6.0.3.1)
+    activejob (6.0.3.2)
-      activesupport (= 6.0.3.1)
+      activesupport (= 6.0.3.2)
       globalid (>= 0.3.6)
-    activemodel (6.0.3.1)
+    activemodel (6.0.3.2)
-      activesupport (= 6.0.3.1)
+      activesupport (= 6.0.3.2)
     activemodel-serializers-xml (1.0.2)
       activemodel (> 5.x)
       activesupport (> 5.x)
       builder (~> 3.1)
-    activerecord (6.0.3.1)
+    activerecord (6.0.3.2)
-      activemodel (= 6.0.3.1)
+      activemodel (= 6.0.3.2)
-      activesupport (= 6.0.3.1)
+      activesupport (= 6.0.3.2)
     activerecord-hierarchical_query (1.2.3)
       activerecord (>= 5.0, < 6.1)
       pg (>= 0.21, < 1.3)
-    activestorage (6.0.3.1)
+    activestorage (6.0.3.2)
-      actionpack (= 6.0.3.1)
+      actionpack (= 6.0.3.2)
-      activejob (= 6.0.3.1)
+      activejob (= 6.0.3.2)
-      activerecord (= 6.0.3.1)
+      activerecord (= 6.0.3.2)
       marcel (~> 0.3.1)
-    activesupport (6.0.3.1)
+    activesupport (6.0.3.2)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
@@ -75,42 +75,42 @@ GEM
     airbrussh (1.4.0)
       sshkit (>= 1.6.1, != 1.7.0)
     ansi (1.5.0)
-    ast (2.4.0)
+    ast (2.4.1)
     aws-eventstream (1.1.0)
-    aws-partitions (1.326.0)
+    aws-partitions (1.341.0)
-    aws-sdk-core (3.98.0)
+    aws-sdk-core (3.103.0)
       aws-eventstream (~> 1, >= 1.0.2)
       aws-partitions (~> 1, >= 1.239.0)
       aws-sigv4 (~> 1.1)
       jmespath (~> 1.0)
-    aws-sdk-sqs (1.26.0)
+    aws-sdk-sqs (1.30.0)
-      aws-sdk-core (~> 3, >= 3.71.0)
+      aws-sdk-core (~> 3, >= 3.99.0)
       aws-sigv4 (~> 1.1)
-    aws-sigv4 (1.1.4)
+    aws-sigv4 (1.2.1)
-      aws-eventstream (~> 1.0, >= 1.0.2)
+      aws-eventstream (~> 1, >= 1.0.2)
     bcrypt (3.1.13)
     bootsnap (1.4.6)
       msgpack (~> 1.0)
     builder (3.2.4)
     byebug (11.1.3)
-    capistrano (3.14.0)
+    capistrano (3.14.1)
       airbrussh (>= 1.0.0)
       i18n
       rake (>= 10.0.0)
       sshkit (>= 1.9.0)
-    capistrano-bundler (1.6.0)
+    capistrano-bundler (2.0.0)
       capistrano (~> 3.1)
     capistrano-deploytags (1.0.7)
       capistrano (>= 3.7.0)
-    capistrano-rails (1.5.0)
+    capistrano-rails (1.6.1)
       capistrano (~> 3.1)
-      capistrano-bundler (~> 1.1)
+      capistrano-bundler (>= 1.1, < 3)
-    capistrano-rbenv (2.1.6)
+    capistrano-rbenv (2.2.0)
       capistrano (~> 3.1)
       sshkit (~> 1.3)
     capistrano3-unicorn (0.2.1)
       capistrano (~> 3.1, >= 3.1.0)
-    capybara (3.32.2)
+    capybara (3.33.0)
       addressable
       mini_mime (>= 0.1.3)
       nokogiri (~> 1.8)
@@ -120,11 +120,13 @@ GEM
       xpath (~> 3.2)
     childprocess (3.0.0)
     chronic (0.10.2)
+    codecov (0.2.0)
+      colorize
+      json
+      simplecov
     coderay (1.1.3)
+    colorize (0.8.1)
     concurrent-ruby (1.1.6)
-    connection_pool (2.2.3)
-    crack (0.4.3)
-      safe_yaml (~> 1.0.0)
     crass (1.0.6)
     daemons (1.3.1)
     delayed_job (4.1.8)
@@ -132,22 +134,21 @@ GEM
     delayed_job_active_record (4.1.4)
       activerecord (>= 3.0, < 6.1)
       delayed_job (>= 3.0, < 5)
-    diff-lcs (1.3)
+    diff-lcs (1.4.4)
     docile (1.3.2)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
-    dotenv (2.7.5)
+    dotenv (2.7.6)
-    dotenv-rails (2.7.5)
+    dotenv-rails (2.7.6)
-      dotenv (= 2.7.5)
+      dotenv (= 2.7.6)
-      railties (>= 3.2, < 6.1)
+      railties (>= 3.2)
     erubi (1.9.0)
-    factory_bot (5.2.0)
+    factory_bot (6.1.0)
-      activesupport (>= 4.2.0)
+      activesupport (>= 5.0.0)
     faraday (1.0.1)
       multipart-post (>= 1.2, < 3)
     ffaker (2.15.0)
-    ffi (1.13.0)
+    ffi (1.13.1)
-    ffi (1.13.0-x64-mingw32)
     ffi-compiler (1.0.1)
       ffi (>= 1.0.0)
       rake
@@ -156,7 +157,6 @@ GEM
       ffi (~> 1.0)
     globalid (0.4.2)
       activesupport (>= 4.2.0)
-    hashdiff (1.0.1)
     http (4.4.1)
       addressable (~> 2.3)
       http-cookie (~> 1.0)
@@ -167,48 +167,29 @@ GEM
     http-form_data (2.3.0)
     http-parser (1.2.1)
       ffi-compiler (>= 1.0, < 2.0)
-    httparty (0.18.0)
-      mime-types (~> 3.0)
-      multi_xml (>= 0.5.2)
     i18n (1.8.3)
       concurrent-ruby (~> 1.0)
     ipaddress_2 (0.13.0)
     jmespath (1.4.0)
-    jquery-rails (4.3.5)
+    json (2.3.1)
-      rails-dom-testing (>= 1, < 3)
-      railties (>= 4.2.0)
-      thor (>= 0.14, < 2.0)
-    json (2.3.0)
     jwt (2.2.1)
     kgio (2.11.3)
     listen (3.2.1)
       rb-fsevent (~> 0.10, >= 0.10.3)
       rb-inotify (~> 0.9, >= 0.9.10)
-    loofah (2.5.0)
+    loofah (2.6.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
     mail (2.7.1)
       mini_mime (>= 0.1.1)
     marcel (0.3.3)
       mimemagic (~> 0.3.2)
-    mechanize (2.7.6)
-      domain_name (~> 0.5, >= 0.5.1)
-      http-cookie (~> 1.0)
-      mime-types (>= 1.17.2)
-      net-http-digest_auth (~> 1.1, >= 1.1.1)
-      net-http-persistent (>= 2.5.2)
-      nokogiri (~> 1.6)
-      ntlm-http (~> 0.1, >= 0.1.1)
-      webrobots (>= 0.0.9, < 0.2)
     memoist (0.16.2)
     memory_profiler (0.9.14)
     meta_request (0.7.2)
       rack-contrib (>= 1.1, < 3)
       railties (>= 3.0.0, < 7)
     method_source (1.0.0)
-    mime-types (3.3.1)
-      mime-types-data (~> 3.2015)
-    mime-types-data (3.2020.0512)
     mimemagic (0.3.5)
     mini_mime (1.0.2)
     mini_portile2 (2.4.0)
@@ -221,42 +202,32 @@ GEM
       minitest (>= 5.0)
       ruby-progressbar
     mocha (1.11.2)
-    mock_redis (0.23.0)
+    mock_redis (0.25.0)
     msgpack (1.3.3)
-    msgpack (1.3.3-x64-mingw32)
+    multi_json (1.15.0)
-    multi_json (1.14.1)
     multi_xml (0.6.0)
     multipart-post (2.1.1)
-    mustermann (1.1.1)
-      ruby2_keywords (~> 0.0.1)
-    net-http-digest_auth (1.4.1)
-    net-http-persistent (4.0.0)
-      connection_pool (~> 2.2)
     net-scp (3.0.0)
       net-ssh (>= 2.6.5, < 7.0.0)
     net-sftp (3.0.0)
       net-ssh (>= 5.0.0, < 7.0.0)
-    net-ssh (6.0.2)
+    net-ssh (6.1.0)
     newrelic_rpm (6.11.0.365)
     nio4r (2.5.2)
-    nokogiri (1.10.9)
+    nokogiri (1.10.10)
-      mini_portile2 (~> 2.4.0)
-    nokogiri (1.10.9-x64-mingw32)
       mini_portile2 (~> 2.4.0)
     nokogumbo (2.0.2)
       nokogiri (~> 1.8, >= 1.8.4)
-    ntlm-http (0.1.1)
     oauth2 (1.4.4)
       faraday (>= 0.8, < 2.0)
       jwt (>= 1.0, < 3.0)
       multi_json (~> 1.3)
       multi_xml (~> 0.5)
       rack (>= 1.2, < 3)
-    parallel (1.19.1)
+    parallel (1.19.2)
-    parser (2.7.1.3)
+    parser (2.7.1.4)
-      ast (~> 2.4.0)
+      ast (~> 2.4.1)
     pg (1.2.3)
-    pg (1.2.3-x64-mingw32)
     pry (0.13.1)
       coderay (~> 1.1)
       method_source (~> 1.0)
@@ -270,40 +241,38 @@ GEM
       nio4r (~> 2.0)
     pundit (2.1.0)
       activesupport (>= 3.0.0)
-    rack (2.2.2)
+    rack (2.2.3)
     rack-contrib (2.2.0)
       rack (~> 2.0)
     rack-mini-profiler (2.0.2)
       rack (>= 1.2.0)
-    rack-protection (2.0.8.1)
-      rack
     rack-proxy (0.6.5)
       rack
     rack-test (1.1.0)
       rack (>= 1.0, < 3)
-    rails (6.0.3.1)
+    rails (6.0.3.2)
-      actioncable (= 6.0.3.1)
+      actioncable (= 6.0.3.2)
-      actionmailbox (= 6.0.3.1)
+      actionmailbox (= 6.0.3.2)
-      actionmailer (= 6.0.3.1)
+      actionmailer (= 6.0.3.2)
-      actionpack (= 6.0.3.1)
+      actionpack (= 6.0.3.2)
-      actiontext (= 6.0.3.1)
+      actiontext (= 6.0.3.2)
-      actionview (= 6.0.3.1)
+      actionview (= 6.0.3.2)
-      activejob (= 6.0.3.1)
+      activejob (= 6.0.3.2)
-      activemodel (= 6.0.3.1)
+      activemodel (= 6.0.3.2)
-      activerecord (= 6.0.3.1)
+      activerecord (= 6.0.3.2)
-      activestorage (= 6.0.3.1)
+      activestorage (= 6.0.3.2)
-      activesupport (= 6.0.3.1)
+      activesupport (= 6.0.3.2)
       bundler (>= 1.3.0)
-      railties (= 6.0.3.1)
+      railties (= 6.0.3.2)
       sprockets-rails (>= 2.0.0)
     rails-dom-testing (2.0.3)
       activesupport (>= 4.2.0)
       nokogiri (>= 1.6)
     rails-html-sanitizer (1.3.0)
       loofah (~> 2.3)
-    railties (6.0.3.1)
+    railties (6.0.3.2)
-      actionpack (= 6.0.3.1)
+      actionpack (= 6.0.3.2)
-      activesupport (= 6.0.3.1)
+      activesupport (= 6.0.3.2)
       method_source
       rake (>= 0.8.7)
       thor (>= 0.20.3, < 2.0)
@@ -316,25 +285,24 @@ GEM
       ffi (~> 1.0)
     recaptcha (5.5.0)
       json
-    redis (4.1.4)
+    redis (4.2.1)
     regexp_parser (1.7.1)
     request_store (1.5.0)
       rack (>= 1.4)
     responders (3.0.1)
       actionpack (>= 5.0)
       railties (>= 5.0)
-    retriable (3.1.2)
     rexml (3.2.4)
-    rubocop (0.85.1)
+    rubocop (0.88.0)
       parallel (~> 1.10)
-      parser (>= 2.7.0.1)
+      parser (>= 2.7.1.1)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.7)
       rexml
-      rubocop-ast (>= 0.0.3)
+      rubocop-ast (>= 0.1.0, < 1.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 2.0)
-    rubocop-ast (0.0.3)
+    rubocop-ast (0.1.0)
       parser (>= 2.7.0.1)
     rubocop-rails (2.6.0)
       activesupport (>= 4.2.0)
@@ -343,10 +311,8 @@ GEM
     ruby-progressbar (1.10.1)
     ruby-vips (2.0.17)
       ffi (~> 1.9)
-    ruby2_keywords (0.0.2)
     rubyzip (2.3.0)
-    safe_yaml (1.0.5)
+    sanitize (5.2.1)
-    sanitize (5.2.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.8.0)
       nokogumbo (~> 2.0)
@@ -357,23 +323,17 @@ GEM
       childprocess (>= 0.5, < 4.0)
       rubyzip (>= 1.2.2)
     semantic_range (2.3.0)
-    shoulda-context (1.2.2)
+    shoulda-context (2.0.0)
     shoulda-matchers (4.3.0)
       activesupport (>= 4.2.0)
     simple_form (5.0.2)
       actionpack (>= 5.0)
       activemodel (>= 5.0)
-    simplecov (0.17.1)
+    simplecov (0.18.5)
       docile (~> 1.1)
-      json (>= 1.8, < 3)
+      simplecov-html (~> 0.11)
-      simplecov-html (~> 0.10.0)
+    simplecov-html (0.12.2)
-    simplecov-html (0.10.2)
+    sprockets (4.0.2)
-    sinatra (2.0.8.1)
-      mustermann (~> 1.0)
-      rack (~> 2.0)
-      rack-protection (= 2.0.8.1)
-      tilt (~> 2.0)
-    sprockets (4.0.0)
       concurrent-ruby (~> 1.0)
       rack (> 1, < 3)
     sprockets-rails (3.2.1)
@@ -389,13 +349,11 @@ GEM
     stripe (5.22.0)
     thor (1.0.1)
     thread_safe (0.3.6)
-    tilt (2.0.10)
     tzinfo (1.2.7)
       thread_safe (~> 0.1)
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.7.7)
-    unf_ext (0.0.7.7-x64-mingw32)
     unicode-display_width (1.7.0)
     unicorn (5.5.5)
       kgio (~> 2.6)
@@ -403,28 +361,22 @@ GEM
     unicorn-worker-killer (0.4.4)
       get_process_mem (~> 0)
       unicorn (>= 4, < 6)
-    webmock (3.8.3)
-      addressable (>= 2.3.6)
-      crack (>= 0.3.2)
-      hashdiff (>= 0.4.0, < 2.0.0)
     webpacker (5.1.1)
       activesupport (>= 5.2)
       rack-proxy (>= 0.6.1)
       railties (>= 5.2)
       semantic_range (>= 2.3.0)
-    webrobots (0.1.2)
+    websocket-driver (0.7.3)
-    websocket-driver (0.7.2)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)
     whenever (1.0.0)
       chronic (>= 0.6.3)
     xpath (3.2.0)
       nokogiri (~> 1.8)
-    zeitwerk (2.3.0)
+    zeitwerk (2.3.1)
 
 PLATFORMS
   ruby
-  x64-mingw32
 
 DEPENDENCIES
   activemodel-serializers-xml
@@ -440,6 +392,7 @@ DEPENDENCIES
   capistrano-rbenv
   capistrano3-unicorn
   capybara
+  codecov
   daemons
   delayed_job
   delayed_job_active_record
@@ -450,12 +403,9 @@ DEPENDENCIES
   ffaker
   flamegraph
   http
-  httparty
   ipaddress_2
-  jquery-rails
   listen
   mail
-  mechanize
   memoist
   memory_profiler
   meta_request
@@ -465,7 +415,7 @@ DEPENDENCIES
   mock_redis
   net-sftp
   newrelic_rpm
-  nokogiri (~> 1.10.9)
+  nokogiri
   oauth2
   pg
   pry-byebug
@@ -480,7 +430,6 @@ DEPENDENCIES
   redis
   request_store
   responders
-  retriable
   rubocop
   rubocop-rails
   ruby-vips
@@ -491,14 +440,12 @@ DEPENDENCIES
   shoulda-context
   shoulda-matchers
   simple_form
-  simplecov (~> 0.17.0)
+  simplecov
-  sinatra
   stackprof
   streamio-ffmpeg
   stripe
   unicorn
   unicorn-worker-killer
-  webmock
   webpacker (>= 4.0.x)
   whenever
 

INSTALL.debian

@@ -32,9 +32,6 @@ if [[ -z "$HOSTNAME" ]] ; then
   exit 1
 fi
 
-echo -n "* Enter the VLAN IP address for this server (ex: 172.16.0.1, enter nothing to skip): "
-read VLAN_IP_ADDR
-
 # Install packages
 echo "* Installing packages..."
 
@@ -52,17 +49,6 @@ apt-get -y install $LIBSSL_DEV_PKG build-essential automake libxml2-dev libxslt-
 apt-get -y install libpq-dev postgresql-client
 apt-get -y install liblcms2-dev $LIBJPEG_TURBO_DEV_PKG libexpat1-dev libgif-dev libpng-dev libexif-dev
 
-# vrack specific stuff
-if [ -n "$VLAN_IP_ADDR" ] ; then
-  apt-get -y install vlan
-  modprobe 8021q
-  echo "8021q" >> /etc/modules
-  vconfig add eno2 99
-  ip addr add $VLAN_IP_ADDR/24 dev eno2.99
-  ip link set up eno2.99
-  curl -L -s $GITHUB_INSTALL_SCRIPTS/vrack-cfg.yaml -o /etc/netplan/01-netcfg.yaml
-fi
-
 curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -
 echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list
 curl -sSL https://deb.nodesource.com/setup_10.x | sudo -E bash -

README.md

@@ -1,4 +1,4 @@
-[![Test Coverage](https://api.codeclimate.com/v1/badges/a51f46fb460a35104d82/test_coverage)](https://codeclimate.com/github/danbooru/danbooru/test_coverage) [![Discord](https://img.shields.io/discord/310432830138089472?label=Discord)](https://discord.gg/eSVKkUF)
+[![codecov](https://codecov.io/gh/danbooru/danbooru/branch/master/graph/badge.svg)](https://codecov.io/gh/danbooru/danbooru) [![Discord](https://img.shields.io/discord/310432830138089472?label=Discord)](https://discord.gg/eSVKkUF)
 
 ## Installation
 

app/controllers/application_controller.rb

@@ -88,6 +88,8 @@ class ApplicationController < ActionController::Base
 
   def rescue_exception(exception)
     case exception
+    when ActionView::Template::Error
+      rescue_exception(exception.cause)
     when ActiveRecord::QueryCanceled
       render_error_page(500, exception, template: "static/search_timeout", message: "The database timed out running your query.")
     when ActionController::BadRequest

app/controllers/autocomplete_controller.rb

@@ -0,0 +1,16 @@
+class AutocompleteController < ApplicationController
+  respond_to :xml, :json
+
+  def index
+    @tags = Tag.names_matches_with_aliases(params[:query], params.fetch(:limit, 10).to_i)
+
+    if request.variant.opensearch?
+      expires_in 1.hour
+      results = [params[:query], @tags.map(&:pretty_name)]
+      respond_with(results)
+    else
+      # XXX
+      respond_with(@tags.map(&:attributes))
+    end
+  end
+end

app/controllers/comments_controller.rb

@@ -97,6 +97,7 @@ class CommentsController < ApplicationController
 
     if request.format.atom?
       @comments = @comments.includes(:creator, :post)
+      @comments = @comments.select { |comment| comment.post.visible? }
     elsif request.format.html?
       @comments = @comments.includes(:creator, :updater, post: :uploader)
       @comments = @comments.includes(:votes) if CurrentUser.is_member?

app/controllers/explore/posts_controller.rb

@@ -22,23 +22,25 @@ module Explore
 
     def viewed
       @date, @scale, @min_date, @max_date = parse_date(params)
-      @posts = PostViewCountService.new.popular_posts(@date)
+      @posts = ReportbooruService.new.popular_posts(@date)
       respond_with(@posts)
     end
 
     def searches
       @date, @scale, @min_date, @max_date = parse_date(params)
-      @search_service = PopularSearchService.new(@date)
+      @searches = ReportbooruService.new.post_search_rankings(@date)
+      respond_with(@searches)
     end
 
     def missed_searches
-      @search_service = MissedSearchService.new
+      @missed_searches = ReportbooruService.new.missed_search_rankings
+      respond_with(@missed_searches)
     end
 
     private
 
     def parse_date(params)
-      date = params[:date].present? ? Date.parse(params[:date]) : Time.zone.today
+      date = params[:date].present? ? Date.parse(params[:date]) : Date.today
       scale = params[:scale].in?(["day", "week", "month"]) ? params[:scale] : "day"
       min_date = date.send("beginning_of_#{scale}")
       max_date = date.send("next_#{scale}").send("beginning_of_#{scale}")

app/controllers/iqdb_queries_controller.rb

@@ -5,7 +5,7 @@ class IqdbQueriesController < ApplicationController
     # XXX allow bare search params for backwards compatibility.
     search_params.merge!(params.slice(:url, :image_url, :file_url, :post_id, :limit, :similarity, :high_similarity).permit!)
 
-    @high_similarity_matches, @low_similarity_matches, @matches = IqdbProxy.search(search_params)
+    @high_similarity_matches, @low_similarity_matches, @matches = IqdbProxy.new.search(search_params)
     respond_with(@matches, template: "iqdb_queries/show")
   end
 

app/controllers/mock_services_controller.rb

@@ -0,0 +1,48 @@
+class MockServicesController < ApplicationController
+  skip_forgery_protection
+  respond_to :json
+
+  before_action do
+    raise User::PrivilegeError if Rails.env.production?
+  end
+
+  def recommender_recommend
+    @data = posts.map { |post| [post.id, rand(0.0..1.0)] }
+    render json: @data
+  end
+
+  def recommender_similar
+    @data = posts.map { |post| [post.id, rand(0.0..1.0)] }
+    render json: @data
+  end
+
+  def reportbooru_missed_searches
+    @data = tags.map { |tag| "#{tag.name} #{rand(1.0..1000.0)}" }.join("\n")
+    render json: @data
+  end
+
+  def reportbooru_post_searches
+    @data = tags.map { |tag| [tag.name, rand(1..1000)] }
+    render json: @data
+  end
+
+  def reportbooru_post_views
+    @data = posts.map { |post| [post.id, rand(1..1000)] }
+    render json: @data
+  end
+
+  def iqdbs_similar
+    @data = posts.map { |post| { post_id: post.id, score: rand(0..100)} }
+    render json: @data
+  end
+
+  private
+
+  def posts(limit = 10)
+    Post.last(limit)
+  end
+
+  def tags(limit = 10)
+    Tag.order(post_count: :desc).limit(limit)
+  end
+end

app/controllers/static_controller.rb

@@ -1,4 +1,7 @@
 class StaticController < ApplicationController
+  def privacy_policy
+  end
+
   def terms_of_service
   end
 
@@ -13,13 +16,40 @@ class StaticController < ApplicationController
     redirect_to wiki_page_path("help:dtext") unless request.format.js?
   end
 
+  def opensearch
+  end
+
   def site_map
   end
 
-  def sitemap
+  def sitemap_index
-    @popular_search_service = PopularSearchService.new(Date.yesterday)
+    @sitemap = params[:sitemap]
-    @posts = Post.where("created_at > ?", 1.week.ago).order(score: :desc).limit(200)
+    @limit = params.fetch(:limit, 10000).to_i
-    @posts = @posts.select(&:visible?)
+
-    render layout: false
+    case @sitemap
+    when "artists"
+      @relation = Artist.undeleted
+      @search = { is_deleted: "false" }
+    when "forum_topics"
+      @relation = ForumTopic.undeleted
+      @search = { is_deleted: "false" }
+    when "pools"
+      @relation = Pool.undeleted
+      @search = { is_deleted: "false" }
+    when "posts"
+      @relation = Post.order(id: :asc)
+      @serach = {}
+    when "tags"
+      @relation = Tag.nonempty
+      @search = {}
+    when "users"
+      @relation = User.all
+      @search = {}
+    when "wiki_pages"
+      @relation = WikiPage.undeleted
+      @search = { is_deleted: "false" }
+    else
+      raise NotImplementedError
+    end
   end
 end

app/controllers/uploads_controller.rb

@@ -21,7 +21,7 @@ class UploadsController < ApplicationController
   def image_proxy
     authorize Upload
     resp = ImageProxy.get_image(params[:url])
-    send_data resp.body, :type => resp.content_type, :disposition => "inline"
+    send_data resp.body, type: resp.mime_type, disposition: "inline"
   end
 
   def index

app/controllers/user_name_change_requests_controller.rb

@@ -19,7 +19,7 @@ class UserNameChangeRequestsController < ApplicationController
   end
 
   def index
-    @change_requests = authorize UserNameChangeRequest.visible(CurrentUser.user).order("id desc").paginate(params[:page], :limit => params[:limit])
+    @change_requests = authorize UserNameChangeRequest.visible(CurrentUser.user).paginated_search(params)
     respond_with(@change_requests)
   end
 end

app/controllers/users_controller.rb

@@ -27,7 +27,7 @@ class UsersController < ApplicationController
   def index
     if params[:name].present?
       @user = User.find_by_name!(params[:name])
-      redirect_to user_path(@user)
+      redirect_to user_path(@user, variant: params[:variant])
       return
     end
 
@@ -42,7 +42,9 @@ class UsersController < ApplicationController
 
   def show
     @user = authorize User.find(params[:id])
-    respond_with(@user, methods: @user.full_attributes)
+    respond_with(@user, methods: @user.full_attributes) do |format|
+      format.html.tooltip { render layout: false }
+    end
   end
 
   def profile

app/helpers/application_helper.rb

@@ -121,6 +121,10 @@ module ApplicationHelper
     raw content_tag(:time, duration, datetime: datetime, title: title)
   end
 
+  def humanized_number(number)
+    number_to_human number, units: { thousand: "k", million: "m" }, format: "%n%u"
+  end
+
   def time_ago_in_words_tagged(time, compact: false)
     if time.nil?
       tag.em(tag.time("unknown"))
@@ -162,8 +166,10 @@ module ApplicationHelper
     end
   end
 
-  def link_to_ip(ip)
+  def link_to_ip(ip, shorten: false, **options)
-    link_to ip, ip_addresses_path(search: { ip_addr: ip, group_by: "user" })
+    ip_addr = IPAddr.new(ip.to_s)
+    ip_addr.prefix = 64 if ip_addr.ipv6? && shorten
+    link_to ip_addr.to_s, ip_addresses_path(search: { ip_addr: ip, group_by: "user" }), **options
   end
 
   def link_to_search(search)
@@ -186,12 +192,13 @@ module ApplicationHelper
   def link_to_user(user)
     return "anonymous" if user.blank?
 
-    user_class = "user-#{user.level_string.downcase}"
+    user_class = "user user-#{user.level_string.downcase}"
     user_class += " user-post-approver" if user.can_approve_posts?
     user_class += " user-post-uploader" if user.can_upload_free?
     user_class += " user-banned" if user.is_banned?
-    user_class += " with-style" if CurrentUser.user.style_usernames?
+
-    link_to(user.pretty_name, user_path(user), :class => user_class)
+    data = { "user-id": user.id, "user-name": user.name, "user-level": user.level }
+    link_to(user.pretty_name, user_path(user), class: user_class, data: data)
   end
 
   def mod_link_to_user(user, positive_or_negative)
@@ -217,21 +224,8 @@ module ApplicationHelper
     tag.div(text, class: "prose", **options)
   end
 
-  def dtext_field(object, name, options = {})
+  def dtext_preview_button(preview_field)
-    options[:name] ||= name.capitalize
+    tag.input value: "Preview", type: "button", class: "dtext-preview-button", "data-preview-field": preview_field
-    options[:input_id] ||= "#{object}_#{name}"
-    options[:input_name] ||= "#{object}[#{name}]"
-    options[:value] ||= instance_variable_get("@#{object}").try(name)
-    options[:preview_id] ||= "dtext-preview"
-    options[:classes] ||= ""
-    options[:hint] ||= ""
-    options[:type] ||= "text"
-
-    render "dtext/form", options
-  end
-
-  def dtext_preview_button(object, name, input_id: "#{object}_#{name}", preview_id: "dtext-preview")
-    tag.input value: "Preview", type: "button", class: "dtext-preview-button", "data-input-id": input_id, "data-preview-id": preview_id
   end
 
   def quick_search_form_for(attribute, url, name, autocomplete: nil, redirect: false, &block)
@@ -266,7 +260,7 @@ module ApplicationHelper
   def body_attributes(user, params, current_item = nil)
     current_user_data_attributes = data_attributes_for(user, "current-user", current_user_attributes)
 
-    if current_item.present? && current_item.respond_to?(:html_data_attributes) && current_item.respond_to?(:model_name)
+    if !current_item.nil? && current_item.respond_to?(:html_data_attributes) && current_item.respond_to?(:model_name)
       model_name = current_item.model_name.singular.dasherize
       model_attributes = current_item.html_data_attributes
       current_item_data_attributes = data_attributes_for(current_item, model_name, model_attributes)
@@ -353,6 +347,19 @@ module ApplicationHelper
     end
   end
 
+  def canonical_url(url = nil)
+    if url.present?
+      content_for(:canonical_url) { url }
+    elsif content_for(:canonical_url).present?
+      content_for(:canonical_url)
+    else
+      request_params = request.params.sort.to_h.with_indifferent_access
+      request_params.delete(:page) if request_params[:page].to_i == 1
+      request_params.delete(:limit)
+      url_for(**request_params, host: Danbooru.config.hostname, only_path: false)
+    end
+  end
+
   def atom_feed_tag(title, url = {})
     content_for(:html_header, auto_discovery_link_tag(:atom, url, title: title))
   end

app/helpers/pagination_helper.rb

@@ -20,8 +20,8 @@ module PaginationHelper
     params[:page] =~ /[ab]/ || records.current_page >= Danbooru.config.max_numbered_pages
   end
 
-  def numbered_paginator(records, switch_to_sequential = true)
+  def numbered_paginator(records)
-    if use_sequential_paginator?(records) && switch_to_sequential
+    if use_sequential_paginator?(records)
       return sequential_paginator(records)
     end
 

app/helpers/seo_helper.rb

@@ -0,0 +1,65 @@
+# https://yoast.com/structured-data-schema-ultimate-guide/
+# https://technicalseo.com/tools/schema-markup-generator/
+# https://developers.google.com/search/docs/data-types/sitelinks-searchbox
+# https://developers.google.com/search/docs/data-types/logo
+# https://search.google.com/structured-data/testing-tool/u/0/
+# https://search.google.com/test/rich-results
+# https://schema.org/Organization
+# https://schema.org/WebSite
+
+module SeoHelper
+  def site_description
+    "#{Danbooru.config.canonical_app_name} is the original anime image booru. Search millions of anime pictures categorized by thousands of tags."
+  end
+
+  # https://developers.google.com/search/docs/data-types/video#video-object
+  def json_ld_video_data(post)
+    json_ld_tag({
+      "@context": "https://schema.org",
+      "@type": "VideoObject",
+      "name": page_title,
+      "description": meta_description,
+      "uploadDate": post.created_at.iso8601,
+      "thumbnailUrl": post.preview_file_url,
+      "contentUrl": post.file_url,
+    })
+  end
+
+  def json_ld_website_data
+    urls = [
+      Danbooru.config.twitter_url,
+      Danbooru.config.discord_server_url,
+      Danbooru.config.source_code_url,
+      "https://en.wikipedia.org/wiki/Danbooru"
+    ].compact
+
+    json_ld_tag({
+      "@context": "https://schema.org",
+      "@graph": [
+        {
+          "@type": "Organization",
+          url: root_url(host: Danbooru.config.hostname),
+          name: Danbooru.config.app_name,
+          logo: "#{root_url(host: Danbooru.config.hostname)}images/danbooru-logo-500x500.png",
+          sameAs: urls
+        },
+        {
+          "@type": "WebSite",
+          "@id": root_url(anchor: "website", host: Danbooru.config.hostname),
+          "url": root_url(host: Danbooru.config.hostname),
+          "name": Danbooru.config.app_name,
+          "description": site_description,
+          "potentialAction": [{
+            "@type": "SearchAction",
+            "target": "#{posts_url(host: Danbooru.config.hostname)}?tags={search_term_string}",
+            "query-input": "required name=search_term_string"
+          }]
+        }
+      ]
+    })
+  end
+
+  def json_ld_tag(data)
+    tag.script(data.to_json.html_safe, type: "application/ld+json")
+  end
+end

app/javascript/packs/application.js

@@ -6,12 +6,12 @@ function importAll(r) {
 
 require('@rails/ujs').start();
 require('hammerjs');
-require('stupid-table-plugin');
 require('jquery-hotkeys');
 
 // should start looking for nodejs replacements
 importAll(require.context('../vendor', true, /\.js$/));
 
+require('jquery');
 require("jquery-ui/ui/effects/effect-shake");
 require("jquery-ui/ui/widgets/autocomplete");
 require("jquery-ui/ui/widgets/button");
@@ -33,6 +33,7 @@ require("@fortawesome/fontawesome-free/css/regular.css");
 importAll(require.context('../src/javascripts', true, /\.js(\.erb)?$/));
 importAll(require.context('../src/styles', true, /\.s?css(?:\.erb)?$/));
 
+export { default as jQuery } from "jquery";
 export { default as Autocomplete } from '../src/javascripts/autocomplete.js.erb';
 export { default as Blacklist } from '../src/javascripts/blacklists.js';
 export { default as Comment } from '../src/javascripts/comments.js';
@@ -47,5 +48,6 @@ export { default as PostVersion } from '../src/javascripts/post_version.js';
 export { default as RelatedTag } from '../src/javascripts/related_tag.js';
 export { default as Shortcuts } from '../src/javascripts/shortcuts.js';
 export { default as Upload } from '../src/javascripts/uploads.js.erb';
+export { default as UserTooltip } from '../src/javascripts/user_tooltips.js';
 export { default as Utility } from '../src/javascripts/utility.js';
 export { default as Ugoira } from '../src/javascripts/ugoira.js';

app/javascript/src/javascripts/autocomplete.js.erb

@@ -9,6 +9,7 @@ Autocomplete.ORDER_METATAGS = <%= PostQueryBuilder::ORDER_METATAGS.to_json.html_
 Autocomplete.DISAPPROVAL_REASONS = <%= PostDisapproval::REASONS.to_json.html_safe %>;
 /* eslint-enable */
 
+Autocomplete.MISC_STATUSES = ["deleted", "active", "pending", "flagged", "banned", "modqueue", "unmoderated"];
 Autocomplete.TAG_PREFIXES = "-|~|" + Object.keys(Autocomplete.TAG_CATEGORIES).map(category => category + ":").join("|");
 Autocomplete.METATAGS_REGEX = Autocomplete.METATAGS.concat(Object.keys(Autocomplete.TAG_CATEGORIES)).join("|");
 Autocomplete.TERM_REGEX = new RegExp(`([-~]*)(?:(${Autocomplete.METATAGS_REGEX}):)?(\\S*)$`, "i");
@@ -37,7 +38,7 @@ Autocomplete.initialize_all = function() {
     });
 
     this.initialize_tag_autocomplete();
-    this.initialize_mention_autocomplete($(".autocomplete-mentions textarea"));
+    this.initialize_mention_autocomplete($("form div.input.dtext textarea"));
     this.initialize_fields($('[data-autocomplete="tag"]'), Autocomplete.tag_source);
     this.initialize_fields($('[data-autocomplete="artist"]'), Autocomplete.artist_source);
     this.initialize_fields($('[data-autocomplete="pool"]'), Autocomplete.pool_source);
@@ -248,9 +249,6 @@ Autocomplete.render_item = function(list, item) {
   } else if (item.type === "user") {
     var level_class = "user-" + item.level.toLowerCase();
     $link.addClass(level_class);
-    if (CurrentUser.data("style-usernames")) {
-      $link.addClass("with-style");
-    }
   } else if (item.type === "pool") {
     $link.addClass("pool-category-" + item.category);
   }
@@ -268,9 +266,7 @@ Autocomplete.render_item = function(list, item) {
 
 Autocomplete.static_metatags = {
   order: Autocomplete.ORDER_METATAGS,
-  status: [
+  status: ["any"].concat(Autocomplete.MISC_STATUSES),
-    "any", "deleted", "active", "pending", "flagged", "banned", "modqueue", "unmoderated"
-  ],
   rating: [
     "safe", "questionable", "explicit"
   ],
@@ -280,12 +276,8 @@ Autocomplete.static_metatags = {
   embedded: [
     "true", "false"
   ],
-  child: [
+  child: ["any", "none"].concat(Autocomplete.MISC_STATUSES),
-    "any", "none"
+  parent: ["any", "none"].concat(Autocomplete.MISC_STATUSES),
-  ],
-  parent: [
-    "any", "none"
-  ],
   filetype: [
     "jpg", "png", "gif", "swf", "zip", "webm", "mp4"
   ],

app/javascript/src/javascripts/dtext.js

@@ -39,8 +39,11 @@ Dtext.call_edit = function(e, $button, $input, $preview) {
 
 Dtext.click_button = function(e) {
   var $button = $(e.target);
-  var $input = $("#" + $button.data("input-id"));
+  var $form = $button.parents("form");
-  var $preview = $("#" + $button.data("preview-id"));
+  var fieldName = $button.data("preview-field");
+  var $inputContainer = $form.find(`div.input.${fieldName} .dtext-previewable`);
+  var $input = $inputContainer.find("> input, > textarea");
+  var $preview = $inputContainer.find("div.dtext-preview");
 
   if ($button.val().match(/preview/i)) {
     Dtext.call_preview(e, $button, $input, $preview);

app/javascript/src/javascripts/forum_posts.js

@@ -7,17 +7,15 @@ ForumPost.initialize_all = function() {
 }
 
 ForumPost.initialize_edit_links = function() {
-  $(".edit_forum_post_link").on("click.danbooru", function(e) {
+  $(document).on("click.danbooru", ".edit_forum_post_link", function(e) {
-    var link_id = $(this).attr("id");
+    let $form = $(this).parents("article.forum-post").find("form.edit_forum_post");
-    var forum_post_id = link_id.match(/^edit_forum_post_link_(\d+)$/)[1];
+    $form.fadeToggle("fast");
-    $("#edit_forum_post_" + forum_post_id).fadeToggle("fast");
     e.preventDefault();
   });
 
-  $(".edit_forum_topic_link").on("click.danbooru", function(e) {
+  $(document).on("click.danbooru", ".edit_forum_topic_link", function(e) {
-    var link_id = $(this).attr("id");
+    let $form = $(this).parents("article.forum-post").find("form.edit_forum_topic");
-    var forum_topic_id = link_id.match(/^edit_forum_topic_link_(\d+)$/)[1];
+    $form.fadeToggle("fast");
-    $("#edit_forum_topic_" + forum_topic_id).fadeToggle("fast");
     e.preventDefault();
   });
 

app/javascript/src/javascripts/pools.js

@@ -20,11 +20,6 @@ Pool.initialize_add_to_pool_link = function() {
     e.preventDefault();
     $("#add-to-pool-dialog").dialog("open");
   });
-
-  $("#recent-pools li").on("click.danbooru", function(e) {
-    e.preventDefault();
-    $("#pool_name").val($(this).attr("data-value"));
-  });
 }
 
 Pool.initialize_simple_edit = function() {

app/javascript/src/javascripts/post_mode_menu.js

@@ -70,7 +70,7 @@ PostModeMenu.initialize_edit_form = function() {
 
   $(document).on("click.danbooru", "#quick-edit-form input[type=submit]", async function(e) {
     e.preventDefault();
-    let post_id = $("#quick-edit-form").data("post-id");
+    let post_id = $("#quick-edit-form").attr("data-post-id");
     await Post.update(post_id, "quick-edit", { post: { tag_string: $("#post_tag_string").val() }});
   });
 }

app/javascript/src/javascripts/post_tooltips.js

@@ -1,28 +1,76 @@
-import CurrentUser from './current_user'
+import CurrentUser from './current_user';
-import Utility from './utility'
+import Utility from './utility';
-
+import { delegate, hideAll } from 'tippy.js';
-require('qtip2');
+import 'tippy.js/dist/tippy.css';
-require('qtip2/dist/jquery.qtip.css');
 
 let PostTooltip = {};
 
-PostTooltip.render_tooltip = async function (event, qtip) {
+PostTooltip.POST_SELECTOR = "*:not(.ui-sortable-handle) > .post-preview img, .dtext-post-id-link";
+PostTooltip.SHOW_DELAY = 500;
+PostTooltip.HIDE_DELAY = 125;
+PostTooltip.DURATION = 250;
+
+PostTooltip.initialize = function () {
+  if (PostTooltip.disabled()) {
+    return;
+  }
+
+  delegate("body", {
+    allowHTML: true,
+    appendTo: document.body,
+    delay: [PostTooltip.SHOW_DELAY, PostTooltip.HIDE_DELAY],
+    duration: PostTooltip.DURATION,
+    interactive: true,
+    maxWidth: "none",
+    target: PostTooltip.POST_SELECTOR,
+    theme: "common-tooltip post-tooltip",
+    touch: false,
+
+    onCreate: PostTooltip.on_create,
+    onShow: PostTooltip.on_show,
+    onHide: PostTooltip.on_hide,
+  });
+
+  $(document).on("click.danbooru.postTooltip", ".post-tooltip-disable", PostTooltip.on_disable_tooltips);
+};
+
+PostTooltip.on_create = function (instance) {
+  let title = instance.reference.getAttribute("title");
+
+  if (title) {
+    instance.reference.setAttribute("data-title", title);
+    instance.reference.setAttribute("title", "");
+  }
+};
+
+PostTooltip.on_show = async function (instance) {
   let post_id = null;
   let preview = false;
+  let $target = $(instance.reference);
+  let $tooltip = $(instance.popper);
 
-  if ($(this).is(".dtext-post-id-link")) {
+  hideAll({ exclude: instance });
+
+  // skip if tooltip has already been rendered.
+  if ($tooltip.has(".post-tooltip-body").length) {
+    return;
+  }
+
+  if ($target.is(".dtext-post-id-link")) {
     preview = true;
-    post_id = /\/posts\/(\d+)/.exec($(this).attr("href"))[1];
+    post_id = /\/posts\/(\d+)/.exec($target.attr("href"))[1];
   } else {
-    post_id = $(this).parents("[data-id]").data("id");
+    post_id = $target.parents("[data-id]").data("id");
   }
 
   try {
-    qtip.cache.request = $.get(`/posts/${post_id}`, { variant: "tooltip", preview: preview });
+    $tooltip.addClass("tooltip-loading");
-    let html = await qtip.cache.request;
 
-    qtip.set("content.text", html);
+    instance._request = $.get(`/posts/${post_id}`, { variant: "tooltip", preview: preview });
-    qtip.elements.tooltip.removeClass("post-tooltip-loading");
+    let html = await instance._request;
+    instance.setContent(html);
+
+    $tooltip.removeClass("tooltip-loading");
   } catch (error) {
     if (error.status !== 0 && error.statusText !== "abort") {
       Utility.error(`Error displaying tooltip for post #${post_id} (error: ${error.status} ${error.statusText})`);
@@ -30,98 +78,19 @@ PostTooltip.render_tooltip = async function (event, qtip) {
   }
 };
 
-// Hide the tooltip the first time it is shown, while we wait on the ajax call to complete.
+PostTooltip.on_hide = function (instance) {
-PostTooltip.on_show = function (event, qtip) {
+  if (instance._request?.state() === "pending") {
-  if (!qtip.cache.hasBeenShown) {
+    instance._request.abort();
-    qtip.elements.tooltip.addClass("post-tooltip-loading");
-    qtip.cache.hasBeenShown = true;
   }
-};
+}
-
-PostTooltip.POST_SELECTOR = "*:not(.ui-sortable-handle) > .post-preview img, .dtext-post-id-link";
-
-// http://qtip2.com/options
-PostTooltip.QTIP_OPTIONS = {
-  style: {
-    classes: "qtip-light post-tooltip",
-    tip: false
-  },
-  content: PostTooltip.render_tooltip,
-  overwrite: false,
-  position: {
-    viewport: true,
-    my: "bottom left",
-    at: "top left",
-    effect: false,
-    adjust: {
-      y: -2,
-      method: "shift",
-    },
-  },
-  show: {
-    solo: true,
-    delay: 750,
-    effect: false,
-    ready: true,
-    event: "mouseenter",
-  },
-  hide: {
-    delay: 250,
-    fixed: true,
-    effect: false,
-    event: "unfocus click mouseleave",
-  },
-  events: {
-    show: PostTooltip.on_show,
-  },
-};
-
-PostTooltip.initialize = function () {
-  $(document).on("mouseenter.danbooru.postTooltip", PostTooltip.POST_SELECTOR, function (event) {
-    if (PostTooltip.disabled()) {
-      $(this).qtip("disable");
-    } else {
-      $(this).qtip(PostTooltip.QTIP_OPTIONS, event);
-    }
-  });
-
-  // Cancel pending ajax requests when we mouse out of the thumbnail.
-  $(document).on("mouseleave.danbooru.postTooltip", PostTooltip.POST_SELECTOR, function (event) {
-    let qtip = $(event.target).qtip("api");
-
-    if (qtip && qtip.cache && qtip.cache.request && qtip.cache.request.state() === "pending") {
-      qtip.cache.request.abort();
-    }
-  });
-
-  $(document).on("click.danbooru.postTooltip", ".post-tooltip-disable", PostTooltip.on_disable_tooltips);
-
-  // Hide tooltips when pressing keys or clicking thumbnails.
-  $(document).on("keydown.danbooru.postTooltip", PostTooltip.hide);
-  $(document).on("click.danbooru.postTooltip", PostTooltip.POST_SELECTOR, PostTooltip.hide);
-
-  // Disable tooltips on touch devices. https://developer.mozilla.org/en-US/docs/Web/API/Touch_events/Supporting_both_TouchEvent_and_MouseEvent
-  PostTooltip.isTouching = false;
-  $(document).on("touchstart.danbooru.postTooltip", function (event) { PostTooltip.isTouching = true; });
-  $(document).on("touchend.danbooru.postTooltip",   function (event) { PostTooltip.isTouching = false; });
-};
-
-PostTooltip.hide = function (event) {
-  // Hide on any key except control (user may be control-clicking link inside tooltip).
-  if (event.type === "keydown" && event.ctrlKey === true) {
-    return;
-  }
-
-  $(".post-tooltip:visible").qtip("hide");
-};
 
 PostTooltip.disabled = function (event) {
-  return PostTooltip.isTouching || CurrentUser.data("disable-post-tooltips");
+  return CurrentUser.data("disable-post-tooltips");
 };
 
 PostTooltip.on_disable_tooltips = async function (event) {
   event.preventDefault();
-  $(event.target).parents(".qtip").qtip("hide");
+  hideAll();
 
   if (CurrentUser.data("is-anonymous")) {
     Utility.notice('You must <a href="/session/new">login</a> to disable tooltips');

app/javascript/src/javascripts/posts.js.erb

@@ -300,10 +300,10 @@ Post.initialize_favlist = function() {
   });
 }
 
-Post.view_original = function(e) {
+Post.view_original = function(e = null) {
   if (Utility.test_max_width(660)) {
     // Do the default behavior (navigate to image)
-    return false;
+    return;
   }
 
   var $image = $("#image");
@@ -316,13 +316,13 @@ Post.view_original = function(e) {
   });
   Note.Box.scale_all();
   $("body").attr("data-post-current-image-size", "original");
-  return false;
+  e?.preventDefault();
 }
 
-Post.view_large = function(e) {
+Post.view_large = function(e = null) {
   if (Utility.test_max_width(660)) {
     // Do the default behavior (navigate to image)
-    return false;
+    return;
   }
 
   var $image = $("#image");
@@ -335,7 +335,7 @@ Post.view_large = function(e) {
   });
   Note.Box.scale_all();
   $("body").attr("data-post-current-image-size", "large");
-  return false;
+  e?.preventDefault();
 }
 
 Post.toggle_fit_window = function(e) {

app/javascript/src/javascripts/saved_searches.js

@@ -1,11 +0,0 @@
-let SavedSearch = {};
-
-SavedSearch.initialize_all = function() {
-  if ($("#c-saved-searches").length) {
-    $("#c-saved-searches table").stupidtable();
-  }
-}
-
-$(SavedSearch.initialize_all);
-
-export default SavedSearch

app/javascript/src/javascripts/user_tooltips.js

@@ -0,0 +1,85 @@
+import Utility from './utility';
+import { delegate } from 'tippy.js';
+import 'tippy.js/dist/tippy.css';
+
+let UserTooltip = {};
+
+UserTooltip.SELECTOR = "*:not(.user-tooltip-name) > a.user, a.dtext-user-id-link, a.dtext-user-mention-link";
+UserTooltip.SHOW_DELAY = 500;
+UserTooltip.HIDE_DELAY = 125;
+UserTooltip.DURATION = 250;
+UserTooltip.MAX_WIDTH = 600;
+
+UserTooltip.initialize = function () {
+  delegate("body", {
+    allowHTML: true,
+    appendTo: document.body,
+    delay: [UserTooltip.SHOW_DELAY, UserTooltip.HIDE_DELAY],
+    duration: UserTooltip.DURATION,
+    interactive: true,
+    maxWidth: UserTooltip.MAX_WIDTH,
+    target: UserTooltip.SELECTOR,
+    theme: "common-tooltip user-tooltip",
+    touch: false,
+
+    onShow: UserTooltip.on_show,
+    onHide: UserTooltip.on_hide,
+  });
+
+  delegate("body", {
+    allowHTML: true,
+    interactive: true,
+    theme: "common-tooltip",
+    target: ".user-tooltip-menu-button",
+    placement: "bottom",
+    touch: false,
+    trigger: "click",
+    content: (element) => {
+      return $(element).parents(".user-tooltip").find(".user-tooltip-menu").get(0);
+    }
+  });
+};
+
+UserTooltip.on_show = async function (instance) {
+  let $target = $(instance.reference);
+  let $tooltip = $(instance.popper);
+
+  // skip if tooltip has already been rendered.
+  if ($tooltip.has(".user-tooltip-body").length) {
+    return;
+  }
+
+  try {
+    $tooltip.addClass("tooltip-loading");
+
+    if ($target.is("a.dtext-user-id-link")) {
+      let user_id = /\/users\/(\d+)/.exec($target.attr("href"))[1];
+      instance._request = $.get(`/users/${user_id}`, { variant: "tooltip" });
+    } else if ($target.is("a.user")) {
+      let user_id = $target.attr("data-user-id");
+      instance._request = $.get(`/users/${user_id}`, { variant: "tooltip" });
+    } else if ($target.is("a.dtext-user-mention-link")) {
+      let user_name = $target.attr("data-user-name");
+      instance._request = $.get(`/users`, { name: user_name, variant: "tooltip" });
+    }
+
+    let html = await instance._request;
+    instance.setContent(html);
+
+    $tooltip.removeClass("tooltip-loading");
+  } catch (error) {
+    if (error.status !== 0 && error.statusText !== "abort") {
+      Utility.error(`Error displaying tooltip (error: ${error.status} ${error.statusText})`);
+    }
+  }
+};
+
+UserTooltip.on_hide = function (instance) {
+  if (instance._request?.state() === "pending") {
+    instance._request.abort();
+  }
+}
+
+$(document).ready(UserTooltip.initialize);
+
+export default UserTooltip

app/javascript/src/styles/base/020_base.scss

@@ -129,6 +129,14 @@ table tfoot {
   font-size: 0.9em;
 }
 
+a.link-plain {
+  color: unset;
+
+  &:hover {
+    text-decoration: underline;
+  }
+}
+
 .fixed-width-container {
   max-width: 70em;
 }

app/javascript/src/styles/base/040_colors.css

@@ -2,6 +2,7 @@
   --body-background-color: white;
 
   --text-color: hsl(0, 0%, 15%);
+  --inverse-text-color: white;
   --muted-text-color: hsl(0, 0%, 55%);
   --header-color: hsl(0, 0%, 15%);
 
@@ -72,7 +73,8 @@
   --comment-sticky-background-color: var(--subnav-menu-background-color);
 
   --post-tooltip-background-color: var(--body-background-color);
-  --post-tooltip-border-color: #767676;
+  --post-tooltip-border-color: hsla(210, 100%, 3%, 0.15);
+  --post-tooltip-box-shadow: 0 4px 14px -2px hsla(210, 100%, 3%, 0.10);
   --post-tooltip-header-background-color: var(--subnav-menu-background-color);
   --post-tooltip-info-color: #555;
   --post-tooltip-scrollbar-background: #999999;
@@ -81,6 +83,9 @@
   --post-tooltip-scrollbar-track-background: #EEEEEE;
   --post-tooltip-scrollbar-track-border: 0 none white;
 
+  --user-tooltip-positive-feedback-color: orange;
+  --user-tooltip-negative-feedback-color: red;
+
   --preview-current-post-background: rgba(0, 0, 0, 0.1);
 
   --autocomplete-selected-background-color: var(--subnav-menu-background-color);
@@ -200,6 +205,7 @@
   --user-platinum-color: gray;
   --user-gold-color: #00F;
   --user-member-color: var(--link-color);
+  --user-banned-color: black;
 
   --news-updates-background: #EEE;
   --news-updates-border: 2px solid #666;
@@ -260,6 +266,7 @@ body[data-current-user-theme="dark"] {
 
   /* main text colors */
   --text-color: var(--grey-5);
+  --inverse-text-color: white;
   --muted-text-color: var(--grey-4);
   --header-color: var(--grey-6);
 
@@ -282,6 +289,7 @@ body[data-current-user-theme="dark"] {
   --collection-pool-color: var(--general-tag-color);
   --collection-pool-hover-color: var(--general-tag-hover-color);
 
+  --user-banned-color: var(--grey-1);
   --user-member-color: var(--blue-1);
   --user-gold-color: var(--yellow-1);
   --user-platinum-color: var(--grey-4);
@@ -394,6 +402,9 @@ body[data-current-user-theme="dark"] {
   --post-tooltip-scrollbar-track-background: var(--grey-1);
   --post-tooltip-scrollbar-track-border: none;
 
+  --user-tooltip-positive-feedback-color: var(--yellow-1);
+  --user-tooltip-negative-feedback-color: var(--red-1);
+
   --preview-pending-color: var(--blue-1);
   --preview-flagged-color: var(--red-1);
   --preview-deleted-color: var(--grey-5);

app/javascript/src/styles/common/messages.scss

@@ -17,7 +17,8 @@ div.list-of-messages {
 
       a.message-timestamp {
         font-style: italic;
-        color: var(--text-color);
+        font-size: 0.90em;
+        color: var(--muted-text-color);
         &:hover { text-decoration: underline; }
       }
     }

app/javascript/src/styles/common/simple_form.scss

@@ -39,7 +39,7 @@ form.simple_form {
       padding-left: 1em;
     }
 
-    &.text {
+    &.text, &.dtext {
       .hint {
         padding-left: 0;
         display: block;

app/javascript/src/styles/common/user_styles.scss

@@ -1,23 +1,25 @@
-a.user-admin.with-style {
+body[data-current-user-style-usernames="true"] {
-  color: var(--user-admin-color);
+  a.user-admin {
-}
+    color: var(--user-admin-color);
+  }
 
-a.user-moderator.with-style {
+  a.user-moderator {
-  color: var(--user-moderator-color);
+    color: var(--user-moderator-color);
-}
+  }
 
-a.user-builder.with-style {
+  a.user-builder {
-  color: var(--user-builder-color);
+    color: var(--user-builder-color);
-}
+  }
 
-a.user-platinum.with-style {
+  a.user-platinum {
-  color: var(--user-platinum-color);
+    color: var(--user-platinum-color);
-}
+  }
 
-a.user-gold.with-style {
+  a.user-gold {
-  color: var(--user-gold-color);
+    color: var(--user-gold-color);
-}
+  }
 
-a.user-member.with-style {
+  a.user-member {
-  color: var(--user-member-color);
+    color: var(--user-member-color);
+  }
 }

app/javascript/src/styles/specific/common_tooltips.scss

@@ -0,0 +1,51 @@
+div[data-tippy-root].tooltip-loading {
+  visibility: hidden !important;
+}
+
+.tippy-box[data-theme~="common-tooltip"] {
+  box-sizing: border-box;
+
+  border: 1px solid var(--post-tooltip-border-color);
+  border-radius: 4px;
+
+  color: var(--text-color);
+  background-color: var(--post-tooltip-background-color);
+  background-clip: padding-box;
+  box-shadow: var(--post-tooltip-box-shadow);
+
+  /* bordered arrow styling; see https://github.com/atomiks/tippyjs/blob/master/src/scss/themes/light-border.scss */
+  &[data-placement^=bottom] {
+    > .tippy-arrow:before {
+      border-bottom-color: var(--post-tooltip-background-color);
+      bottom: 16px;
+    }
+
+    > .tippy-arrow:after {
+      border-bottom-color: var(--post-tooltip-border-color);
+      border-width: 0 7px 7px;
+      top: -8px;
+      left: 1px;
+    }
+  }
+
+  &[data-placement^=top] {
+    > .tippy-arrow:before {
+      border-top-color: var(--post-tooltip-background-color);
+    }
+
+    > .tippy-arrow:after {
+      border-top-color: var(--post-tooltip-border-color);
+      border-width: 7px 7px 0;
+      top: 17px;
+      left: 1px;
+    }
+  }
+
+  > .tippy-arrow:after {
+    border-color: transparent;
+    border-style: solid;
+    content: "";
+    position: absolute;
+    z-index: -1;
+  }
+}

app/javascript/src/styles/specific/post_tooltips.scss

@@ -1,6 +1,5 @@
 $tooltip-line-height: 16px;
-$tooltip-body-height: $tooltip-line-height * 6; // 6 lines high.
+$tooltip-body-height: $tooltip-line-height * 4; // 4 lines high.
-$tooltip-width: 164px * 3 - 10; // 3 thumbnails wide.
 
 @mixin thin-scrollbar {
   &::-webkit-scrollbar {
@@ -46,16 +45,13 @@ $tooltip-width: 164px * 3 - 10; // 3 thumbnails wide.
   }
 }
 
-.post-tooltip {
+.tippy-box[data-theme~="post-tooltip"] {
-  max-width: $tooltip-width;
+  min-width: 20em;
-  min-width: $tooltip-width;
+  max-width: 40em !important;
-  box-sizing: border-box;
   font-size: 11px;
   line-height: $tooltip-line-height;
-  border-color: var(--post-tooltip-border-color);
-  background-color: var(--post-tooltip-background-color);
 
-  .qtip-content {
+  .tippy-content {
     padding: 0;
 
     > * {
@@ -85,38 +81,32 @@ $tooltip-width: 164px * 3 - 10; // 3 thumbnails wide.
       .post-tooltip-body-right { flex: 1; }
     }
 
-    .post-tooltip-header {
+    div.post-tooltip-header {
       background-color: var(--post-tooltip-header-background-color);
       display: flex;
       white-space: nowrap;
-      overflow: hidden;
+      align-items: center;
 
-      .post-tooltip-header-left {
+      .post-tooltip-info {
-        flex: 1;
+        margin-right: 0.5em;
+        color: var(--post-tooltip-info-color);
+        font-size: 10px;
+        flex: 0;
       }
 
-      .post-tooltip-header-right {
+      a.user {
+        margin-right: 0.5em;
+        overflow: hidden;
+        white-space: nowrap;
+        text-overflow: ellipsis;
+        min-width: 0;
+        max-width: 11em;
+      }
+
+      .post-tooltip-source {
         flex: 1;
         text-align: right;
       }
-
-      .fa-xs {
-        vertical-align: baseline;
-      }
-
-      .post-tooltip-disable {
-        margin-left: 0.5em;
-      }
     }
-
-    .post-tooltip-info {
-      margin-left: 0.5em;
-      color: var(--post-tooltip-info-color);
-      font-size: 10px;
-    }
-  }
-
-  &.post-tooltip-loading {
-    visibility: hidden;
   }
 }

app/javascript/src/styles/specific/privacy_policy.scss

@@ -0,0 +1,5 @@
+#c-static #a-privacy-policy {
+  .summary {
+    font-style: italic;
+  }
+}

app/javascript/src/styles/specific/user_tooltips.scss

@@ -0,0 +1,103 @@
+.tippy-box[data-theme~="user-tooltip"] {
+  line-height: 1.25em;
+  min-width: 350px;
+
+  .user-tooltip-header {
+    margin-bottom: 1em;
+    display: grid;
+    grid:
+      "avatar header-top menu"
+      "avatar header-bottom menu" /
+      32px 1fr 32px;
+    column-gap: 0.25em;
+
+    .user-tooltip-avatar {
+      font-size: 32px;
+      grid-area: avatar;
+      align-self: center;
+    }
+
+    .user-tooltip-header-top {
+      grid-area: header-top;
+
+      .user-tooltip-badge {
+        color: var(--inverse-text-color);
+        font-size: 0.70em;
+        padding: 2px 4px;
+        margin-right: 0.25em;
+        border-radius: 3px;
+
+        &.user-tooltip-badge-admin { background-color: var(--user-admin-color); }
+        &.user-tooltip-badge-moderator { background-color: var(--user-moderator-color); }
+        &.user-tooltip-badge-approver { background-color: var(--user-builder-color); }
+        &.user-tooltip-badge-contributor { background-color: var(--user-builder-color); }
+        &.user-tooltip-badge-builder { background-color: var(--user-builder-color); }
+        &.user-tooltip-badge-platinum { background-color: var(--user-platinum-color); }
+        &.user-tooltip-badge-gold { background-color: var(--user-gold-color); }
+        &.user-tooltip-badge-member { background-color: var(--user-member-color); }
+        &.user-tooltip-badge-banned { background-color: var(--user-banned-color); }
+
+        &.user-tooltip-badge-positive-feedback {
+          color: var(--user-tooltip-positive-feedback-color);
+          border: 1px solid;
+        }
+
+        &.user-tooltip-badge-negative-feedback {
+          color: var(--user-tooltip-negative-feedback-color);
+          border: 1px solid;
+        }
+      }
+    }
+
+    .user-tooltip-header-bottom {
+      grid-area: header-bottom;
+      color: var(--muted-text-color);
+      font-size: 0.75em;
+    }
+
+    a.user-tooltip-menu-button {
+      color: var(--muted-text-color);
+      grid-area: menu;
+      align-self: center;
+      text-align: center;
+      width: 28px;
+      height: 28px;
+      line-height: 28px;
+      border-radius: 50%;
+
+      &:hover {
+        background-color: var(--subnav-menu-background-color);
+      }
+    }
+
+    > ul.user-tooltip-menu {
+      display: none;
+    }
+
+    ul.user-tooltip-menu {
+      .icon {
+        width: 1.5em;
+      }
+    }
+  }
+
+  .user-tooltip-stats {
+    display: grid;
+    grid: auto / repeat(3, 1fr);
+    column-gap: 1em;
+    row-gap: 0.5em;
+
+    .user-tooltip-stat-item {
+      text-align: center;
+
+      .user-tooltip-stat-value {
+        font-weight: bold;
+      }
+
+      .user-tooltip-stat-name {
+        font-size: 0.90em;
+        color: var(--muted-text-color);
+      }
+    }
+  }
+}

app/logical/apng_inspector.rb

@@ -57,7 +57,7 @@ class APNGInspector
         # if we did, file is probably maliciously formed
         # fail gracefully without marking the file as corrupt
         chunks += 1
-        if chunks > 100000
+        if chunks > 100_000
           iend_reached = true
           break
         end
@@ -66,7 +66,8 @@ class APNGInspector
         file.seek(current_pos + chunk_len + 4, IO::SEEK_SET)
       end
     end
-    return iend_reached
+
+    iend_reached
   end
 
   def inspect!
@@ -105,6 +106,7 @@ class APNGInspector
     if framedata.nil? || framedata.length != 4
       return -1
     end
-    return framedata.unpack1("N".freeze)
+
+    framedata.unpack1("N".freeze)
   end
 end

app/logical/artist_finder.rb

@@ -6,7 +6,7 @@ module ArtistFinder
   SITE_BLACKLIST = [
     "artstation.com/artist", # http://www.artstation.com/artist/serafleur/
     "www.artstation.com", # http://www.artstation.com/serafleur/
-    %r!cdn[ab]?\.artstation\.com/p/assets/images/images!i, # https://cdna.artstation.com/p/assets/images/images/001/658/068/large/yang-waterkuma-b402.jpg?1450269769
+    %r{cdn[ab]?\.artstation\.com/p/assets/images/images}i, # https://cdna.artstation.com/p/assets/images/images/001/658/068/large/yang-waterkuma-b402.jpg?1450269769
     "ask.fm", # http://ask.fm/mikuroko_396
     "bcyimg.com",
     "bcyimg.com/drawer", # https://img9.bcyimg.com/drawer/32360/post/178vu/46229ec06e8111e79558c1b725ebc9e6.jpg
@@ -52,7 +52,7 @@ module ArtistFinder
     "hentai-foundry.com",
     "hentai-foundry.com/pictures/user", # http://www.hentai-foundry.com/pictures/user/aaaninja/
     "hentai-foundry.com/user", # http://www.hentai-foundry.com/user/aaaninja/profile
-    %r!pictures\.hentai-foundry\.com(?:/\w)?!i, # http://pictures.hentai-foundry.com/a/aaaninja/
+    %r{pictures\.hentai-foundry\.com(?:/\w)?}i, # http://pictures.hentai-foundry.com/a/aaaninja/
     "i.imgur.com", # http://i.imgur.com/Ic9q3.jpg
     "instagram.com", # http://www.instagram.com/serafleur.art/
     "iwara.tv",
@@ -62,13 +62,15 @@ module ArtistFinder
     "monappy.jp",
     "monappy.jp/u", # https://monappy.jp/u/abara_bone
     "mstdn.jp", # https://mstdn.jp/@oneb
+    "www.newgrounds.com", # https://jessxjess.newgrounds.com/
+    "newgrounds.com/art/view/", # https://www.newgrounds.com/art/view/jessxjess/avatar-korra
     "nicoseiga.jp",
     "nicoseiga.jp/priv", # http://lohas.nicoseiga.jp/priv/2017365fb6cfbdf47ad26c7b6039feb218c5e2d4/1498430264/6820259
     "nicovideo.jp",
     "nicovideo.jp/user", # http://www.nicovideo.jp/user/317609
     "nicovideo.jp/user/illust", # http://seiga.nicovideo.jp/user/illust/29075429
     "nijie.info", # http://nijie.info/members.php?id=15235
-    %r!nijie\.info/nijie_picture!i, # http://pic03.nijie.info/nijie_picture/32243_20150609224803_0.png
+    %r{nijie\.info/nijie_picture}i, # http://pic03.nijie.info/nijie_picture/32243_20150609224803_0.png
     "patreon.com", # http://patreon.com/serafleur
     "pawoo.net", # https://pawoo.net/@148nasuka
     "pawoo.net/web/accounts", # https://pawoo.net/web/accounts/228341
@@ -120,7 +122,7 @@ module ArtistFinder
 
   SITE_BLACKLIST_REGEXP = Regexp.union(SITE_BLACKLIST.map do |domain|
     domain = Regexp.escape(domain) if domain.is_a?(String)
-    %r!\Ahttps?://(?:[a-zA-Z0-9_-]+\.)*#{domain}/\z!i
+    %r{\Ahttps?://(?:[a-zA-Z0-9_-]+\.)*#{domain}/\z}i
   end)
 
   def find_artists(url)
@@ -128,7 +130,7 @@ module ArtistFinder
     artists = []
 
     while artists.empty? && url.size > 10
-      u = url.sub(/\/+$/, "") + "/"
+      u = url.sub(%r{/+$}, "") + "/"
       u = u.to_escaped_for_sql_like.gsub(/\*/, '%') + '%'
       artists += Artist.joins(:urls).where(["artists.is_deleted = FALSE AND artist_urls.normalized_url LIKE ? ESCAPE E'\\\\'", u]).limit(10).order("artists.name").all
       url = File.dirname(url) + "/"

app/logical/bulk_update_request_processor.rb

@@ -148,8 +148,6 @@ class BulkUpdateRequestProcessor
     end.join("\n")
   end
 
-  private
-
   def self.is_tag_move_allowed?(antecedent_name, consequent_name)
     antecedent_tag = Tag.find_by_name(Tag.normalize_name(antecedent_name))
     consequent_tag = Tag.find_by_name(Tag.normalize_name(consequent_name))

app/logical/cloudflare_service.rb

@@ -9,15 +9,6 @@ class CloudflareService
     api_token.present? && zone.present?
   end
 
-  def ips(expiry: 24.hours)
-    response = Danbooru::Http.cache(expiry).get("https://api.cloudflare.com/client/v4/ips")
-    return [] if response.code != 200
-
-    result = response.parse["result"]
-    ips = result["ipv4_cidrs"] + result["ipv6_cidrs"]
-    ips.map { |ip| IPAddr.new(ip) }
-  end
-
   def purge_cache(urls)
     return unless enabled?
 

app/logical/concerns/has_bit_flags.rb

@@ -16,7 +16,7 @@ module HasBitFlags
       end
 
       define_method("#{attribute}=") do |val|
-        if val.to_s =~ /t|1|y/
+        if val.to_s =~ /[t1y]/
           send("#{field}=", send(field) | bit_flag)
         else
           send("#{field}=", send(field) & ~bit_flag)

app/logical/concerns/mentionable.rb

@@ -11,8 +11,6 @@ module Mentionable
     # - user_field
     def mentionable(options = {})
       @mentionable_options = options
-
-      message_field = mentionable_option(:message_field)
       after_save :queue_mention_messages
     end
 

app/logical/concerns/searchable.rb

@@ -89,7 +89,7 @@ module Searchable
   def where_array_count(attr, value)
     qualified_column = "cardinality(#{qualified_column_for(attr)})"
     range = PostQueryBuilder.new(nil).parse_range(value, :integer)
-    where_operator("cardinality(#{qualified_column_for(attr)})", *range)
+    where_operator(qualified_column, *range)
   end
 
   def search_boolean_attribute(attribute, params)
@@ -170,7 +170,7 @@ module Searchable
     end
   end
 
-  def search_text_attribute(attr, params, **options)
+  def search_text_attribute(attr, params)
     if params[attr].present?
       where(attr => params[attr])
     elsif params[:"#{attr}_eq"].present?
@@ -279,7 +279,8 @@ module Searchable
         return find_ordered(parse_ids[1])
       end
     end
-    return default_order
+
+    default_order
   end
 
   def default_order

app/logical/current_user.rb

@@ -24,15 +24,6 @@ class CurrentUser
     scoped(user, &block)
   end
 
-  def self.as_system(&block)
-    if block_given?
-      scoped(::User.system, "127.0.0.1", &block)
-    else
-      self.user = User.system
-      self.ip_addr = "127.0.0.1"
-    end
-  end
-
   def self.user
     RequestStore[:current_user]
   end

app/logical/d_text.rb

@@ -11,7 +11,7 @@ class DText
     html = DTextRagel.parse(text, **options)
     html = postprocess(html, *data)
     html
-  rescue DTextRagel::Error => e
+  rescue DTextRagel::Error
     ""
   end
 
@@ -135,7 +135,7 @@ class DText
     fragment = Nokogiri::HTML.fragment(html)
 
     titles = fragment.css("a.dtext-wiki-link").map do |node|
-      title = node["href"][%r!\A/wiki_pages/(.*)\z!i, 1]
+      title = node["href"][%r{\A/wiki_pages/(.*)\z}i, 1]
       title = CGI.unescape(title)
       title = WikiPage.normalize_title(title)
       title
@@ -163,7 +163,7 @@ class DText
     string = string.dup
 
     string.gsub!(/\s*\[#{tag}\](?!\])\s*/mi, "\n\n[#{tag}]\n\n")
-    string.gsub!(/\s*\[\/#{tag}\]\s*/mi, "\n\n[/#{tag}]\n\n")
+    string.gsub!(%r{\s*\[/#{tag}\]\s*}mi, "\n\n[/#{tag}]\n\n")
     string.gsub!(/(?:\r?\n){3,}/, "\n\n")
     string.strip!
 
@@ -203,7 +203,7 @@ class DText
       end
     end
 
-    text = text.gsub(/\A[[:space:]]+|[[:space:]]+\z/, "")
+    text.gsub(/\A[[:space:]]+|[[:space:]]+\z/, "")
   end
 
   def self.from_html(text, inline: false, &block)

app/logical/danbooru/http.rb

@@ -1,17 +1,49 @@
+require "danbooru/http/html_adapter"
+require "danbooru/http/xml_adapter"
+require "danbooru/http/cache"
+require "danbooru/http/redirector"
+require "danbooru/http/retriable"
+require "danbooru/http/session"
+require "danbooru/http/spoof_referrer"
+require "danbooru/http/unpolish_cloudflare"
+
 module Danbooru
   class Http
-    DEFAULT_TIMEOUT = 3
+    class DownloadError < StandardError; end
+    class FileTooLargeError < StandardError; end
 
-    attr_writer :cache, :http
+    DEFAULT_TIMEOUT = 10
+    MAX_REDIRECTS = 5
+
+    attr_accessor :max_size, :http
 
     class << self
-      delegate :get, :post, :delete, :cache, :auth, :basic_auth, :headers, to: :new
+      delegate :get, :head, :put, :post, :delete, :cache, :follow, :max_size, :timeout, :auth, :basic_auth, :headers, :cookies, :use, :public_only, :download_media, to: :new
+    end
+
+    def initialize
+      @http ||=
+        ::Danbooru::Http::ApplicationClient.new
+        .timeout(DEFAULT_TIMEOUT)
+        .headers("Accept-Encoding" => "gzip")
+        .headers("User-Agent": "#{Danbooru.config.canonical_app_name}/#{Rails.application.config.x.git_hash}")
+        .use(:auto_inflate)
+        .use(redirector: { max_redirects: MAX_REDIRECTS })
+        .use(:session)
     end
 
     def get(url, **options)
       request(:get, url, **options)
     end
 
+    def head(url, **options)
+      request(:head, url, **options)
+    end
+
+    def put(url, **options)
+      request(:get, url, **options)
+    end
+
     def post(url, **options)
       request(:post, url, **options)
     end
@@ -20,8 +52,16 @@ module Danbooru
       request(:delete, url, **options)
     end
 
-    def cache(expiry)
+    def follow(*args)
-      dup.tap { |o| o.cache = expiry.to_i }
+      dup.tap { |o| o.http = o.http.follow(*args) }
+    end
+
+    def max_size(size)
+      dup.tap { |o| o.max_size = size }
+    end
+
+    def timeout(*args)
+      dup.tap { |o| o.http = o.http.timeout(*args) }
     end
 
     def auth(*args)
@@ -36,36 +76,66 @@ module Danbooru
       dup.tap { |o| o.http = o.http.headers(*args) }
     end
 
+    def cookies(*args)
+      dup.tap { |o| o.http = o.http.cookies(*args) }
+    end
+
+    def use(*args)
+      dup.tap { |o| o.http = o.http.use(*args) }
+    end
+
+    def cache(expires_in)
+      use(cache: { expires_in: expires_in })
+    end
+
+    # allow requests only to public IPs, not to local or private networks.
+    def public_only
+      dup.tap do |o|
+        o.http = o.http.dup.tap do |http|
+          http.default_options = http.default_options.with_socket_class(ValidatingSocket)
+        end
+      end
+    end
+
+    concerning :DownloadMethods do
+      def download_media(url, file: Tempfile.new("danbooru-download-", binmode: true))
+        response = get(url)
+
+        raise DownloadError, "Downloading #{response.uri} failed with code #{response.status}" if response.status != 200
+        raise FileTooLargeError, response if @max_size && response.content_length.to_i > @max_size
+
+        size = 0
+        response.body.each do |chunk|
+          size += chunk.size
+          raise FileTooLargeError if @max_size && size > @max_size
+          file.write(chunk)
+        end
+
+        file.rewind
+        [response, MediaFile.open(file)]
+      end
+    end
+
     protected
 
     def request(method, url, **options)
-      if @cache.present?
-        cached_request(method, url, **options)
-      else
-        raw_request(method, url, **options)
-      end
-    rescue HTTP::TimeoutError
-      # return a synthetic http error on connection timeouts
-      ::HTTP::Response.new(status: 522, body: "", version: "1.1")
-    end
-
-    def cached_request(method, url, **options)
-      key = Cache.hash({ method: method, url: url, headers: http.default_options.headers.to_h, **options }.to_json)
-
-      cached_response = Cache.get(key, @cache) do
-        response = raw_request(method, url, **options)
-        { status: response.status, body: response.to_s, headers: response.headers.to_h, version: "1.1" }
-      end
-
-      ::HTTP::Response.new(**cached_response)
-    end
-
-    def raw_request(method, url, **options)
       http.send(method, url, **options)
+    rescue OpenSSL::SSL::SSLError
+      fake_response(590, "")
+    rescue ValidatingSocket::ProhibitedIpError
+      fake_response(591, "")
+    rescue HTTP::Redirector::TooManyRedirectsError
+      fake_response(596, "")
+    rescue HTTP::TimeoutError
+      fake_response(597, "")
+    rescue HTTP::ConnectionError
+      fake_response(598, "")
+    rescue HTTP::Error
+      fake_response(599, "")
     end
 
-    def http
+    def fake_response(status, body)
-      @http ||= ::HTTP.timeout(DEFAULT_TIMEOUT).use(:auto_inflate).headers(Danbooru.config.http_headers).headers("Accept-Encoding" => "gzip")
+      ::HTTP::Response.new(status: status, version: "1.1", body: ::HTTP::Response::Body.new(body))
     end
   end
 end

app/logical/danbooru/http/application_client.rb

@@ -0,0 +1,31 @@
+# An extension to HTTP::Client that lets us write Rack-style middlewares that
+# hook into the request/response cycle and override how requests are made. This
+# works by extending http.rb's concept of features (HTTP::Feature) to give them
+# a `perform` method that takes a http request and returns a http response.
+# This can be used to intercept and modify requests and return arbitrary responses.
+
+module Danbooru
+  class Http
+    class ApplicationClient < HTTP::Client
+      # Override `perform` to call the `perform` method on features first.
+      def perform(request, options)
+        features = options.features.values.reverse.select do |feature|
+          feature.respond_to?(:perform)
+        end
+
+        perform = proc { |req| super(req, options) }
+        callback_chain = features.reduce(perform) do |callback_chain, feature|
+          proc { |req| feature.perform(req, &callback_chain) }
+        end
+
+        callback_chain.call(request)
+      end
+
+      # Override `branch` to return an ApplicationClient instead of a
+      # HTTP::Client so that chaining works.
+      def branch(...)
+        ApplicationClient.new(...)
+      end
+    end
+  end
+end

app/logical/danbooru/http/cache.rb

@@ -0,0 +1,30 @@
+module Danbooru
+  class Http
+    class Cache < HTTP::Feature
+      HTTP::Options.register_feature :cache, self
+
+      attr_reader :expires_in
+
+      def initialize(expires_in:)
+        @expires_in = expires_in
+      end
+
+      def perform(request, &block)
+        ::Cache.get(cache_key(request), expires_in) do
+          response = yield request
+
+          # XXX hack to remove connection state from response body so we can serialize it for caching.
+          response.flush
+          response.body.instance_variable_set(:@connection, nil)
+          response.body.instance_variable_set(:@stream, nil)
+
+          response
+        end
+      end
+
+      def cache_key(request)
+        "http:" + ::Cache.hash({ method: request.verb, url: request.uri.to_s, headers: request.headers.sort }.to_json)
+      end
+    end
+  end
+end

app/logical/danbooru/http/html_adapter.rb

@@ -0,0 +1,12 @@
+module Danbooru
+  class Http
+    class HtmlAdapter < HTTP::MimeType::Adapter
+      HTTP::MimeType.register_adapter "text/html", self
+      HTTP::MimeType.register_alias "text/html", :html
+
+      def decode(str)
+        Nokogiri::HTML5(str)
+      end
+    end
+  end
+end

app/logical/danbooru/http/redirector.rb

@@ -0,0 +1,40 @@
+# A HTTP::Feature that automatically follows HTTP redirects.
+# https://developer.mozilla.org/en-US/docs/Web/HTTP/Redirections
+
+module Danbooru
+  class Http
+    class Redirector < HTTP::Feature
+      HTTP::Options.register_feature :redirector, self
+
+      attr_reader :max_redirects
+
+      def initialize(max_redirects: 5)
+        @max_redirects = max_redirects
+      end
+
+      def perform(request, &block)
+        response = yield request
+
+        redirects = max_redirects
+        while response.status.redirect?
+          raise HTTP::Redirector::TooManyRedirectsError if redirects <= 0
+
+          response = yield build_redirect(request, response)
+          redirects -= 1
+        end
+
+        response
+      end
+
+      def build_redirect(request, response)
+        location = response.headers["Location"].to_s
+        uri = HTTP::URI.parse(location)
+
+        verb = request.verb
+        verb = :get if response.status == 303 && !request.verb.in?([:get, :head])
+
+        request.redirect(uri, verb)
+      end
+    end
+  end
+end

app/logical/danbooru/http/retriable.rb

@@ -0,0 +1,54 @@
+# A HTTP::Feature that automatically retries requests that return a 429 error
+# or a Retry-After header. Usage: `Danbooru::Http.use(:retriable).get(url)`.
+#
+# https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/429
+# https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
+
+module Danbooru
+  class Http
+    class Retriable < HTTP::Feature
+      HTTP::Options.register_feature :retriable, self
+
+      attr_reader :max_retries, :max_delay
+
+      def initialize(max_retries: 2, max_delay: 5.seconds)
+        @max_retries = max_retries
+        @max_delay = max_delay
+      end
+
+      def perform(request, &block)
+        response = yield request
+
+        retries = max_retries
+        while retriable?(response) && retries > 0 && retry_delay(response) <= max_delay
+          DanbooruLogger.info "Retrying url=#{request.uri} status=#{response.status} retries=#{retries} delay=#{retry_delay(response)}"
+
+          retries -= 1
+          sleep(retry_delay(response))
+          response = yield request
+        end
+
+        response
+      end
+
+      def retriable?(response)
+        response.status == 429 || response.headers["Retry-After"].present?
+      end
+
+      def retry_delay(response, current_time: Time.zone.now)
+        retry_after = response.headers["Retry-After"]
+
+        if retry_after.blank?
+          0.seconds
+        elsif retry_after =~ /\A\d+\z/
+          retry_after.to_i.seconds
+        else
+          retry_at = Time.zone.parse(retry_after)
+          return 0.seconds if retry_at.blank?
+
+          [retry_at - current_time, 0].max.seconds
+        end
+      end
+    end
+  end
+end

app/logical/danbooru/http/session.rb

@@ -0,0 +1,37 @@
+module Danbooru
+  class Http
+    class Session < HTTP::Feature
+      HTTP::Options.register_feature :session, self
+
+      attr_reader :cookie_jar
+
+      def initialize(cookie_jar: HTTP::CookieJar.new)
+        @cookie_jar = cookie_jar
+      end
+
+      def perform(request)
+        add_cookies(request)
+        response = yield request
+        save_cookies(response)
+        response
+      end
+
+      def add_cookies(request)
+        cookies = cookies_for_request(request)
+        request.headers["Cookie"] = cookies if cookies.present?
+      end
+
+      def cookies_for_request(request)
+        saved_cookies = cookie_jar.each(request.uri).map { |c| [c.name, c.value] }.to_h
+        request_cookies = HTTP::Cookie.cookie_value_to_hash(request.headers["Cookie"].to_s)
+        saved_cookies.merge(request_cookies).map { |name, value| "#{name}=#{value}" }.join("; ")
+      end
+
+      def save_cookies(response)
+        response.cookies.each do |cookie|
+          cookie_jar.add(cookie)
+        end
+      end
+    end
+  end
+end

app/logical/danbooru/http/spoof_referrer.rb

@@ -0,0 +1,13 @@
+module Danbooru
+  class Http
+    class SpoofReferrer < HTTP::Feature
+      HTTP::Options.register_feature :spoof_referrer, self
+
+      def perform(request, &block)
+        request.headers["Referer"] = request.uri.origin unless request.headers["Referer"].present?
+        response = yield request
+        response
+      end
+    end
+  end
+end

app/logical/danbooru/http/unpolish_cloudflare.rb

@@ -0,0 +1,20 @@
+# Bypass Cloudflare Polish (https://support.cloudflare.com/hc/en-us/articles/360000607372-Using-Cloudflare-Polish-to-compress-images)
+
+module Danbooru
+  class Http
+    class UnpolishCloudflare < HTTP::Feature
+      HTTP::Options.register_feature :unpolish_cloudflare, self
+
+      def perform(request, &block)
+        response = yield request
+
+        if response.headers["CF-Polished"].present?
+          request.uri.query_values = request.uri.query_values.to_h.merge(danbooru_no_polish: SecureRandom.uuid)
+          response = yield request
+        end
+
+        response
+      end
+    end
+  end
+end

app/logical/danbooru/http/xml_adapter.rb

@@ -0,0 +1,12 @@
+module Danbooru
+  class Http
+    class XmlAdapter < HTTP::MimeType::Adapter
+      HTTP::MimeType.register_adapter "application/xml", self
+      HTTP::MimeType.register_alias "application/xml", :xml
+
+      def decode(str)
+        Hash.from_xml(str).with_indifferent_access
+      end
+    end
+  end
+end

app/logical/danbooru_maintenance.rb

@@ -2,14 +2,13 @@ module DanbooruMaintenance
   module_function
 
   def hourly
+    safely { Upload.prune! }
   end
 
   def daily
     safely { PostPruner.new.prune! }
-    safely { Upload.prune! }
     safely { Delayed::Job.where('created_at < ?', 45.days.ago).delete_all }
     safely { PostDisapproval.prune! }
-    safely { PostDisapproval.dmail_messages! }
     safely { regenerate_post_counts! }
     safely { TokenBucket.prune! }
     safely { BulkUpdateRequestPruner.warn_old }

app/logical/downloads/file.rb

@@ -1,123 +0,0 @@
-require 'resolv'
-
-module Downloads
-  class File
-    include ActiveModel::Validations
-    class Error < StandardError; end
-
-    RETRIABLE_ERRORS = [Errno::ECONNRESET, Errno::ETIMEDOUT, Errno::EIO, Errno::EHOSTUNREACH, Errno::ECONNREFUSED, Timeout::Error, IOError]
-
-    delegate :data, to: :strategy
-    attr_reader :url, :referer
-
-    validate :validate_url
-
-    def initialize(url, referer = nil)
-      @url = Addressable::URI.parse(url) rescue nil
-      @referer = referer
-      validate!
-    end
-
-    def size
-      res = HTTParty.head(uncached_url, **httparty_options, timeout: 3)
-
-      if res.success?
-        res.content_length
-      else
-        raise HTTParty::ResponseError.new(res)
-      end
-    end
-
-    def download!(url: uncached_url, tries: 3, **options)
-      Retriable.retriable(on: RETRIABLE_ERRORS, tries: tries, base_interval: 0) do
-        file = http_get_streaming(url, headers: strategy.headers, **options)
-        return [file, strategy]
-      end
-    end
-
-    def validate_url
-      errors[:base] << "URL must not be blank" if url.blank?
-      errors[:base] << "'#{url}' is not a valid url" if !url.host.present?
-      errors[:base] << "'#{url}' is not a valid url. Did you mean 'http://#{url}'?" if !url.scheme.in?(%w[http https])
-    end
-
-    def http_get_streaming(url, file: Tempfile.new(binmode: true), headers: {}, max_size: Danbooru.config.max_file_size)
-      size = 0
-
-      res = HTTParty.get(url, httparty_options) do |chunk|
-        next if chunk.code == 302
-
-        size += chunk.size
-        raise Error.new("File is too large (max size: #{max_size})") if size > max_size && max_size > 0
-
-        file.write(chunk)
-      end
-
-      if res.success?
-        file.rewind
-        return file
-      else
-        raise Error.new("HTTP error code: #{res.code} #{res.message}")
-      end
-    end
-
-    # Prevent Cloudflare from potentially mangling the image. See issue #3528.
-    def uncached_url
-      return file_url unless is_cloudflare?(file_url)
-
-      url = file_url.dup
-      url.query_values = url.query_values.to_h.merge(danbooru_no_cache: SecureRandom.uuid)
-      url
-    end
-
-    def preview_url
-      @preview_url ||= Addressable::URI.parse(strategy.preview_url)
-    end
-
-    def file_url
-      @file_url ||= Addressable::URI.parse(strategy.image_url)
-    end
-
-    def strategy
-      @strategy ||= Sources::Strategies.find(url.to_s, referer)
-    end
-
-    def httparty_options
-      {
-        timeout: 10,
-        stream_body: true,
-        headers: strategy.headers,
-        connection_adapter: ValidatingConnectionAdapter
-      }.deep_merge(Danbooru.config.httparty_options)
-    end
-
-    def is_cloudflare?(url)
-      ip_addr = IPAddr.new(Resolv.getaddress(url.hostname))
-      CloudflareService.new.ips.any? { |subnet| subnet.include?(ip_addr) }
-    end
-
-    def self.banned_ip?(ip)
-      ip = IPAddress.parse(ip.to_s) unless ip.is_a?(IPAddress)
-
-      if ip.ipv4?
-        ip.loopback? || ip.link_local? || ip.multicast? || ip.private?
-      elsif ip.ipv6?
-        ip.loopback? || ip.link_local? || ip.unique_local? || ip.unspecified?
-      end
-    end
-  end
-
-  # Hook into HTTParty to validate the IP before following redirects.
-  # https://www.rubydoc.info/github/jnunemaker/httparty/HTTParty/ConnectionAdapter
-  class ValidatingConnectionAdapter < HTTParty::ConnectionAdapter
-    def self.call(uri, options)
-      ip_addr = IPAddress.parse(::Resolv.getaddress(uri.hostname))
-
-      if Downloads::File.banned_ip?(ip_addr)
-        raise Downloads::File::Error, "Downloads from #{ip_addr} are not allowed"
-      end
-
-      super(uri, options)
-    end
-  end
-end

app/logical/dtext_input.rb

@@ -0,0 +1,30 @@
+# A custom SimpleForm input for DText fields.
+#
+# Usage:
+#
+#   <%= f.input :body, as: :dtext %>
+#   <%= f.input :reason, as: :dtext, inline: true %>
+#
+# https://github.com/heartcombo/simple_form/wiki/Custom-inputs-examples
+# https://github.com/heartcombo/simple_form/blob/master/lib/simple_form/inputs/string_input.rb
+# https://github.com/heartcombo/simple_form/blob/master/lib/simple_form/inputs/text_input.rb
+
+class DtextInput < SimpleForm::Inputs::Base
+  enable :placeholder, :maxlength, :minlength
+
+  def input(wrapper_options)
+    t = template
+    merged_input_options = merge_wrapper_options(input_html_options, wrapper_options)
+
+    t.tag.div(class: "dtext-previewable") do
+      if options[:inline]
+        t.concat @builder.text_field(attribute_name, merged_input_options)
+      else
+        t.concat @builder.text_area(attribute_name, { rows: 20, cols: 30 }.merge(merged_input_options))
+      end
+
+      t.concat t.tag.div(id: "dtext-preview", class: "dtext-preview prose")
+      t.concat t.tag.span(t.link_to("Formatting help", t.dtext_help_path, remote: true, method: :get), class: "hint dtext-hint")
+    end
+  end
+end

app/logical/image_proxy.rb

@@ -1,4 +1,6 @@
 class ImageProxy
+  class Error < StandardError; end
+
   def self.needs_proxy?(url)
     fake_referer_for(url).present?
   end
@@ -8,19 +10,13 @@ class ImageProxy
   end
 
   def self.get_image(url)
-    if url.blank?
+    raise Error, "URL not present" unless url.present?
-      raise "Must specify url"
+    raise Error, "Proxy not allowed for this url (url=#{url})" unless needs_proxy?(url)
-    end
 
-    if !needs_proxy?(url)
+    referer = fake_referer_for(url)
-      raise "Proxy not allowed for this site"
+    response = Danbooru::Http.timeout(30).headers(Referer: referer).get(url)
-    end
+    raise Error, "Couldn't proxy image (code=#{response.status}, url=#{url})" unless response.status.success?
 
-    response = HTTParty.get(url, Danbooru.config.httparty_options.deep_merge(headers: {"Referer" => fake_referer_for(url)}))
+    response
-    if response.success?
-      return response
-    else
-      raise "HTTP error code: #{response.code} #{response.message}"
-    end
   end
 end

app/logical/ip_lookup.rb

@@ -16,7 +16,7 @@ class IpLookup
   end
 
   def info
-    return {} unless api_key.present?
+    return {} if api_key.blank?
     response = Danbooru::Http.cache(cache_duration).get("https://api.ipregistry.co/#{ip_addr}?key=#{api_key}")
     return {} if response.status != 200
     json = response.parse.deep_symbolize_keys.with_indifferent_access

app/logical/iqdb_proxy.rb

@@ -1,19 +1,24 @@
 class IqdbProxy
   class Error < StandardError; end
+  attr_reader :http, :iqdbs_server
 
-  def self.enabled?
+  def initialize(http: Danbooru::Http.new, iqdbs_server: Danbooru.config.iqdbs_server)
-    Danbooru.config.iqdbs_server.present?
+    @iqdbs_server = iqdbs_server
+    @http = http
   end
 
-  def self.download(url, type)
+  def enabled?
-    download = Downloads::File.new(url)
+    iqdbs_server.present?
-    file, strategy = download.download!(url: download.send(type))
+  end
+
+  def download(url, type)
+    strategy = Sources::Strategies.find(url)
+    download_url = strategy.send(type)
+    file = strategy.download_file!(download_url)
     file
   end
 
-  def self.search(params)
+  def search(params)
-    raise NotImplementedError, "the IQDBs service isn't configured" unless enabled?
-
     limit = params[:limit]&.to_i&.clamp(1, 1000) || 20
     similarity = params[:similarity]&.to_f&.clamp(0.0, 100.0) || 0.0
     high_similarity = params[:high_similarity]&.to_f&.clamp(0.0, 100.0) || 65.0
@@ -28,7 +33,7 @@ class IqdbProxy
       file = download(params[:image_url], :url)
       results = query(file: file, limit: limit)
     elsif params[:file_url].present?
-      file = download(params[:file_url], :file_url)
+      file = download(params[:file_url], :image_url)
       results = query(file: file, limit: limit)
     elsif params[:post_id].present?
       url = Post.find(params[:post_id]).preview_file_url
@@ -46,15 +51,21 @@ class IqdbProxy
     file.try(:close)
   end
 
-  def self.query(params)
+  def query(file: nil, url: nil, limit: 20)
-    response = HTTParty.post("#{Danbooru.config.iqdbs_server}/similar", body: params, **Danbooru.config.httparty_options)
+    raise NotImplementedError, "the IQDBs service isn't configured" unless enabled?
-    raise Error, "IQDB error: #{response.code} #{response.message}" unless response.success?
+
-    raise Error, "IQDB error: #{response.parsed_response["error"]}" if response.parsed_response.is_a?(Hash)
+    file = HTTP::FormData::File.new(file) if file
-    raise Error, "IQDB error: #{response.parsed_response.first}" if response.parsed_response.try(:first).is_a?(String)
+    form = { file: file, url: url, limit: limit }.compact
-    response.parsed_response
+    response = http.timeout(30).post("#{iqdbs_server}/similar", form: form)
+
+    raise Error, "IQDB error: #{response.status}" if response.status != 200
+    raise Error, "IQDB error: #{response.parse["error"]}" if response.parse.is_a?(Hash)
+    raise Error, "IQDB error: #{response.parse.first}" if response.parse.try(:first).is_a?(String)
+
+    response.parse
   end
 
-  def self.decorate_posts(json)
+  def decorate_posts(json)
     post_ids = json.map { |match| match["post_id"] }
     posts = Post.where(id: post_ids).group_by(&:id).transform_values(&:first)
 

app/logical/media_file.rb

@@ -43,6 +43,8 @@ class MediaFile
     else
       :bin
     end
+  rescue EOFError
+    :bin
   end
 
   def self.videos_enabled?

app/logical/media_file/flash.rb

@@ -42,7 +42,7 @@ class MediaFile::Flash < MediaFile
     signature = contents[0..2]
 
     # SWF version
-    version = contents[3].unpack('C').join.to_i
+    _version = contents[3].unpack('C').join.to_i
 
     # Determine the length of the uncompressed stream
     length = contents[4..7].unpack('V').join.to_i
@@ -50,7 +50,7 @@ class MediaFile::Flash < MediaFile
     # If we do, in fact, have compression
     if signature == 'CWS'
       # Decompress the body of the SWF
-      body = Zlib::Inflate.inflate( contents[8..length] )
+      body = Zlib::Inflate.inflate(contents[8..length])
 
       # And reconstruct the stream contents to the first 8 bytes (header)
       # Plus our decompressed body
@@ -58,10 +58,10 @@ class MediaFile::Flash < MediaFile
     end
 
     # Determine the nbits of our dimensions rectangle
-    nbits = contents.unpack('C'*contents.length)[8] >> 3
+    nbits = contents.unpack('C' * contents.length)[8] >> 3
 
     # Determine how many bits long this entire RECT structure is
-    rectbits = 5 + nbits * 4    # 5 bits for nbits, as well as nbits * number of fields (4)
+    rectbits = 5 + nbits * 4 # 5 bits for nbits, as well as nbits * number of fields (4)
 
     # Determine how many bytes rectbits composes (ceil(rectbits/8))
     rectbytes = (rectbits.to_f / 8).ceil
@@ -70,11 +70,11 @@ class MediaFile::Flash < MediaFile
     rect = contents[8..(8 + rectbytes)].unpack("#{'B8' * rectbytes}").join
 
     # Read in nbits incremenets starting from 5
-    dimensions = Array.new
+    dimensions = []
     4.times do |n|
       s = 5 + (n * nbits)     # Calculate our start index
       e = s + (nbits - 1)     # Calculate our end index
-      dimensions[n] = rect[s..e].to_i(2)    # Read that range (binary) and convert it to an integer
+      dimensions[n] = rect[s..e].to_i(2) # Read that range (binary) and convert it to an integer
     end
 
     # The values we have here are in "twips"

app/logical/missed_search_service.rb

@@ -1,29 +0,0 @@
-# queries reportbooru to find missed post searches
-class MissedSearchService
-  def self.enabled?
-    Danbooru.config.reportbooru_server.present?
-  end
-
-  def initialize
-    if !MissedSearchService.enabled?
-      raise NotImplementedError.new("the Reportbooru service isn't configured. Missed searches are not available.")
-    end
-  end
-
-  def each_search(&block)
-    fetch_data.scan(/(.+?) (\d+)\.0\n/).each(&block)
-  end
-
-  def fetch_data
-    Cache.get("ms", 1.minute) do
-      url = URI.parse("#{Danbooru.config.reportbooru_server}/missed_searches")
-      response = HTTParty.get(url, Danbooru.config.httparty_options.reverse_merge(timeout: 6))
-      if response.success?
-        response = response.body
-      else
-        response = ""
-      end
-      response.force_encoding("utf-8")
-    end
-  end
-end

app/logical/nico_seiga_api_client.rb

@@ -1,83 +1,102 @@
 class NicoSeigaApiClient
   extend Memoist
-  BASE_URL = "http://seiga.nicovideo.jp/api"
+  XML_API = "https://seiga.nicovideo.jp/api"
-  attr_reader :illust_id
 
-  def self.agent
+  attr_reader :http
-    mech = Mechanize.new
-    mech.redirect_ok = false
-    mech.keep_alive = false
 
-    session = Cache.get("nico-seiga-session")
+  def initialize(work_id:, type:, http: Danbooru::Http.new)
-    if session
+    @work_id = work_id
-      cookie = Mechanize::Cookie.new("user_session", session)
+    @work_type = type
-      cookie.domain = ".nicovideo.jp"
+    @http = http
-      cookie.path = "/"
+  end
-      mech.cookie_jar.add(cookie)
+
-    else
+  def image_ids
-      mech.get("https://account.nicovideo.jp/login") do |page|
+    if @work_type == "illust"
-        page.form_with(:id => "login_form") do |form|
+      [api_response["id"]]
-          form["mail_tel"] = Danbooru.config.nico_seiga_login
+    elsif @work_type == "manga"
-          form["password"] = Danbooru.config.nico_seiga_password
+      manga_api_response.map do |x|
-        end.click_button
+        case x["meta"]["source_url"]
-      end
+        when %r{/thumb/(\d+)\w}i then Regexp.last_match(1)
-      session = mech.cookie_jar.cookies.select {|c| c.name == "user_session"}.first
+        when %r{nicoseiga\.cdn\.nimg\.jp/drm/image/\w+/(\d+)\w}i then Regexp.last_match(1)
-      if session
+        end
-        Cache.put("nico-seiga-session", session.value, 1.week)
-      else
-        raise "Session not found"
       end
     end
-
-    # This cookie needs to be set to allow viewing of adult works
-    cookie = Mechanize::Cookie.new("skip_fetish_warning", "1")
-    cookie.domain = "seiga.nicovideo.jp"
-    cookie.path = "/"
-    mech.cookie_jar.add(cookie)
-
-    mech.redirect_ok = true
-    mech
-  end
-
-  def initialize(illust_id:, user_id: nil)
-    @illust_id = illust_id
-    @user_id = user_id
-  end
-
-  def image_id
-    illust_xml["response"]["image"]["id"].to_i
-  end
-
-  def user_id
-    @user_id || illust_xml["response"]["image"]["user_id"].to_i
   end
 
   def title
-    illust_xml["response"]["image"]["title"]
+    api_response["title"]
   end
 
-  def desc
+  def description
-    illust_xml["response"]["image"]["description"]
+    api_response["description"]
   end
 
-  def moniker
+  def tags
-    artist_xml["response"]["user"]["nickname"]
+    api_response.dig("tag_list", "tag").to_a.map { |t| t["name"] }.compact
   end
 
-  def illust_xml
+  def user_id
-    get("#{BASE_URL}/illust/info?id=#{illust_id}")
+    api_response["user_id"]
   end
 
-  def artist_xml
+  def user_name
-    get("#{BASE_URL}/user/info?id=#{user_id}")
+    if @work_type == "illust"
+      api_response["nickname"]
+    elsif @work_type == "manga"
+      user_api_response(user_id)["nickname"]
+    end
+  end
+
+  def api_response
+    if @work_type == "illust"
+      resp = get("https://sp.seiga.nicovideo.jp/ajax/seiga/im#{@work_id}")
+      return {} if resp.blank? || resp.code.to_i == 404
+      api_response = JSON.parse(resp)["target_image"]
+
+    elsif @work_type == "manga"
+      resp = http.cache(1.minute).get("#{XML_API}/theme/info?id=#{@work_id}")
+      return {} if resp.blank? || resp.code.to_i == 404
+      api_response = Hash.from_xml(resp.to_s)["response"]["theme"]
+    end
+
+    api_response || {}
+  rescue JSON::ParserError
+    {}
+  end
+
+  def manga_api_response
+    resp = get("https://ssl.seiga.nicovideo.jp/api/v1/app/manga/episodes/#{@work_id}/frames")
+    return {} if resp.blank? || resp.code.to_i == 404
+    JSON.parse(resp)["data"]["result"]
+  rescue JSON::ParserError
+    {}
+  end
+
+  def user_api_response(user_id)
+    resp = http.cache(1.minute).get("#{XML_API}/user/info?id=#{user_id}")
+    return {} if resp.blank? || resp.code.to_i == 404
+    Hash.from_xml(resp.to_s)["response"]["user"]
+  end
+
+  def login
+    form = {
+      mail_tel: Danbooru.config.nico_seiga_login,
+      password: Danbooru.config.nico_seiga_password
+    }
+
+    # XXX should fail gracefully instead of raising exception
+    resp = http.cache(1.hour).post("https://account.nicovideo.jp/login/redirector?site=seiga", form: form)
+    raise RuntimeError, "NicoSeiga login failed (status=#{resp.status} url=#{url})" if resp.status != 200
+
+    http
   end
 
   def get(url)
-    response = Danbooru::Http.cache(1.minute).get(url)
+    resp = login.cache(1.minute).get(url)
-    raise "nico seiga api call failed (code=#{response.code}, body=#{response.body})" if response.code != 200
+    #raise RuntimeError, "NicoSeiga get failed (status=#{resp.status} url=#{url})" if resp.status != 200
 
-    Hash.from_xml(response.to_s)
+    resp
   end
 
-  memoize :artist_xml, :illust_xml
+  memoize :api_response, :manga_api_response, :user_api_response
 end

app/logical/nico_seiga_manga_api_client.rb

@@ -1,60 +0,0 @@
-class NicoSeigaMangaApiClient
-  extend Memoist
-  BASE_URL = "https://seiga.nicovideo.jp/api"
-  attr_reader :theme_id
-
-  def initialize(theme_id)
-    @theme_id = theme_id
-  end
-
-  def user_id
-    theme_info_xml["response"]["theme"]["user_id"].to_i
-  end
-
-  def title
-    theme_info_xml["response"]["theme"]["title"]
-  end
-
-  def desc
-    theme_info_xml["response"]["theme"]["description"]
-  end
-
-  def moniker
-    artist_xml["response"]["user"]["nickname"]
-  end
-
-  def image_ids
-    images = theme_data_xml["response"]["image_list"]["image"]
-    images = [images] unless images.is_a?(Array)
-    images.map {|x| x["id"]}
-  end
-
-  def tags
-    theme_info_xml["response"]["theme"]["tag_list"]["tag"].map {|x| x["name"]}
-  end
-
-  def theme_data_xml
-    uri = "#{BASE_URL}/theme/data?theme_id=#{theme_id}"
-    body = NicoSeigaApiClient.agent.get(uri).body
-    Hash.from_xml(body)
-  end
-
-  def theme_info_xml
-    uri = "#{BASE_URL}/theme/info?id=#{theme_id}"
-    body = NicoSeigaApiClient.agent.get(uri).body
-    Hash.from_xml(body)
-  end
-
-  def artist_xml
-    get("#{BASE_URL}/user/info?id=#{user_id}")
-  end
-
-  def get(url)
-    response = Danbooru::Http.cache(1.minute).get(url)
-    raise "nico seiga api call failed (code=#{response.code}, body=#{response.body})" if response.code != 200
-
-    Hash.from_xml(response.to_s)
-  end
-
-  memoize :theme_data_xml, :theme_info_xml, :artist_xml
-end

app/logical/pagination_extension.rb

@@ -3,9 +3,9 @@ module PaginationExtension
 
   attr_accessor :current_page, :records_per_page, :paginator_count, :paginator_mode
 
-  def paginate(page, limit: nil, count: nil, search_count: nil)
+  def paginate(page, limit: nil, max_limit: 1000, count: nil, search_count: nil)
     @records_per_page = limit || Danbooru.config.posts_per_page
-    @records_per_page = @records_per_page.to_i.clamp(1, 1000)
+    @records_per_page = @records_per_page.to_i.clamp(1, max_limit)
 
     if count.present?
       @paginator_count = count
@@ -61,27 +61,35 @@ module PaginationExtension
   end
 
   def prev_page
-    return nil if is_first_page?
+    if is_first_page?
-
+      nil
-    if paginator_mode == :numbered
+    elsif paginator_mode == :numbered
       current_page - 1
-    elsif paginator_mode == :sequential_before
+    elsif paginator_mode == :sequential_before && records.present?
       "a#{records.first.id}"
-    elsif paginator_mode == :sequential_after
+    elsif paginator_mode == :sequential_after && records.present?
       "b#{records.last.id}"
+    else
+      nil
     end
+  rescue ActiveRecord::QueryCanceled
+    nil
   end
 
   def next_page
-    return nil if is_last_page?
+    if is_last_page?
-
+      nil
-    if paginator_mode == :numbered
+    elsif paginator_mode == :numbered
       current_page + 1
-    elsif paginator_mode == :sequential_before
+    elsif paginator_mode == :sequential_before && records.present?
       "b#{records.last.id}"
-    elsif paginator_mode == :sequential_after
+    elsif paginator_mode == :sequential_after && records.present?
       "a#{records.first.id}"
+    else
+      nil
     end
+  rescue ActiveRecord::QueryCanceled
+    nil
   end
 
   # XXX Hack: in sequential pagination we fetch one more record than we
@@ -106,10 +114,7 @@ module PaginationExtension
   def total_count
     @paginator_count ||= unscoped.from(except(:offset, :limit, :order).reorder(nil)).count
   rescue ActiveRecord::StatementInvalid => e
-    if e.to_s =~ /statement timeout/
+    raise unless e.to_s =~ /statement timeout/
-      @paginator_count ||= 1_000_000
+    @paginator_count ||= 1_000_000
-    else
-      raise
-    end
   end
 end

app/logical/pawoo_api_client.rb

@@ -128,15 +128,15 @@ class PawooApiClient
       rescue
         data = {}
       end
-      return Account.new(data)
+      Account.new(data)
     end
   end
 
   private
 
   def fetch_access_token
-    raise MissingConfigurationError.new("missing pawoo client id") if Danbooru.config.pawoo_client_id.nil?
+    raise MissingConfigurationError, "missing pawoo client id" if Danbooru.config.pawoo_client_id.nil?
-    raise MissingConfigurationError.new("missing pawoo client secret") if Danbooru.config.pawoo_client_secret.nil?
+    raise MissingConfigurationError, "missing pawoo client secret" if Danbooru.config.pawoo_client_secret.nil?
 
     Cache.get("pawoo-token") do
       result = client.client_credentials.get_token

app/logical/pixiv_api_client.rb

@@ -1,5 +1,3 @@
-require 'resolv-replace'
-
 class PixivApiClient
   extend Memoist
 
@@ -114,66 +112,13 @@ class PixivApiClient
     end
   end
 
-  class FanboxResponse
-    attr_reader :json
-
-    def initialize(json)
-      @json = json
-    end
-
-    def name
-      json["body"]["user"]["name"]
-    end
-
-    def user_id
-      json["body"]["user"]["userId"]
-    end
-
-    def moniker
-      ""
-    end
-
-    def page_count
-      json["body"]["body"]["images"].size
-    end
-
-    def artist_commentary_title
-      json["body"]["title"]
-    end
-
-    def artist_commentary_desc
-      json["body"]["body"]["text"]
-    end
-
-    def tags
-      []
-    end
-
-    def pages
-      if json["body"]["body"]
-        json["body"]["body"]["images"].map {|x| x["originalUrl"]}
-      else
-        []
-      end
-    end
-  end
-
   def work(illust_id)
-    headers = Danbooru.config.http_headers.merge(
+    params = { image_sizes: "large", include_stats: "true" }
-      "Referer" => "http://www.pixiv.net",
-      "Content-Type" => "application/x-www-form-urlencoded",
-      "Authorization" => "Bearer #{access_token}"
-    )
-    params = {
-      "image_sizes" => "large",
-      "include_stats" => "true"
-    }
-
     url = "https://public-api.secure.pixiv.net/v#{API_VERSION}/works/#{illust_id.to_i}.json"
-    response = Danbooru::Http.cache(1.minute).headers(headers).get(url, params: params)
+    response = api_client.cache(1.minute).get(url, params: params)
     json = response.parse
 
-    if response.code == 200
+    if response.status == 200
       WorkResponse.new(json["response"][0])
     elsif json["status"] == "failure" && json.dig("errors", "system", "message") =~ /対象のイラストは見つかりませんでした。/
       raise BadIDError.new("Pixiv ##{illust_id} not found: work was deleted, made private, or ID is invalid.")
@@ -184,32 +129,12 @@ class PixivApiClient
     raise Error.new("Pixiv API call failed (status=#{response.code} body=#{response.body})")
   end
 
-  def fanbox(fanbox_id)
-    url = "https://www.pixiv.net/ajax/fanbox/post?postId=#{fanbox_id.to_i}"
-    resp = agent.get(url)
-    json = JSON.parse(resp.body)
-    if resp.code == "200"
-      FanboxResponse.new(json)
-    elsif json["status"] == "failure"
-      raise Error.new("Pixiv API call failed (status=#{resp.code} body=#{body})")
-    end
-  rescue JSON::ParserError
-    raise Error.new("Pixiv API call failed (status=#{resp.code} body=#{body})")
-  end
-
   def novel(novel_id)
-    headers = Danbooru.config.http_headers.merge(
-      "Referer" => "http://www.pixiv.net",
-      "Content-Type" => "application/x-www-form-urlencoded",
-      "Authorization" => "Bearer #{access_token}"
-    )
-
     url = "https://public-api.secure.pixiv.net/v#{API_VERSION}/novels/#{novel_id.to_i}.json"
-    resp = HTTParty.get(url, Danbooru.config.httparty_options.deep_merge(headers: headers))
+    resp = api_client.cache(1.minute).get(url)
-    body = resp.body.force_encoding("utf-8")
+    json = resp.parse
-    json = JSON.parse(body)
 
-    if resp.success?
+    if resp.status == 200
       NovelResponse.new(json["response"][0])
     elsif json["status"] == "failure" && json.dig("errors", "system", "message") =~ /対象のイラストは見つかりませんでした。/
       raise Error.new("Pixiv API call failed (status=#{resp.code} body=#{body})")
@@ -219,42 +144,41 @@ class PixivApiClient
   end
 
   def access_token
-    Cache.get("pixiv-papi-access-token", 3000) do
+    # truncate timestamp to 1-hour resolution so that it doesn't break caching.
-      access_token = nil
+    client_time = Time.zone.now.utc.change(min: 0).rfc3339
+    client_hash = Digest::MD5.hexdigest(client_time + CLIENT_HASH_SALT)
 
-      client_time = Time.now.rfc3339
+    headers = {
-      client_hash = Digest::MD5.hexdigest(client_time + CLIENT_HASH_SALT)
+      "Referer": "http://www.pixiv.net",
+      "X-Client-Time": client_time,
+      "X-Client-Hash": client_hash
+    }
 
-      headers = {
+    params = {
-        "Referer": "http://www.pixiv.net",
+      username: Danbooru.config.pixiv_login,
-        "X-Client-Time": client_time,
+      password: Danbooru.config.pixiv_password,
-        "X-Client-Hash": client_hash
+      grant_type: "password",
-      }
+      client_id: CLIENT_ID,
-      params = {
+      client_secret: CLIENT_SECRET
-        username: Danbooru.config.pixiv_login,
+    }
-        password: Danbooru.config.pixiv_password,
-        grant_type: "password",
-        client_id: CLIENT_ID,
-        client_secret: CLIENT_SECRET
-      }
-      url = "https://oauth.secure.pixiv.net/auth/token"
 
-      resp = HTTParty.post(url, Danbooru.config.httparty_options.deep_merge(body: params, headers: headers))
+    resp = http.headers(headers).cache(1.hour).post("https://oauth.secure.pixiv.net/auth/token", form: params)
-      body = resp.body.force_encoding("utf-8")
+    return nil unless resp.status == 200
 
-      if resp.success?
+    resp.parse.dig("response", "access_token")
-        json = JSON.parse(body)
-        access_token = json["response"]["access_token"]
-      else
-        raise Error.new("Pixiv API access token call failed (status=#{resp.code} body=#{body})")
-      end
-
-      access_token
-    end
   end
 
-  def agent
+  def api_client
-    PixivWebAgent.build
+    http.headers(
+      "Referer": "http://www.pixiv.net",
+      "Content-Type": "application/x-www-form-urlencoded",
+      "Authorization": "Bearer #{access_token}"
+    )
   end
-  memoize :agent
+
+  def http
+    Danbooru::Http.new
+  end
+
+  memoize :access_token, :api_client, :http
 end

app/logical/pixiv_web_agent.rb

@@ -1,74 +0,0 @@
-class PixivWebAgent
-  SESSION_CACHE_KEY = "pixiv-phpsessid"
-  COMIC_SESSION_CACHE_KEY = "pixiv-comicsessid"
-  SESSION_COOKIE_KEY = "PHPSESSID"
-  COMIC_SESSION_COOKIE_KEY = "_pixiv-comic_session"
-
-  def self.phpsessid(agent)
-    agent.cookies.select { |cookie| cookie.name == SESSION_COOKIE_KEY }.first.try(:value)
-  end
-
-  def self.build
-    mech = Mechanize.new
-    mech.keep_alive = false
-
-    phpsessid = Cache.get(SESSION_CACHE_KEY)
-    comicsessid = Cache.get(COMIC_SESSION_CACHE_KEY)
-
-    if phpsessid
-      cookie = Mechanize::Cookie.new(SESSION_COOKIE_KEY, phpsessid)
-      cookie.domain = ".pixiv.net"
-      cookie.path = "/"
-      mech.cookie_jar.add(cookie)
-
-      if comicsessid
-        cookie = Mechanize::Cookie.new(COMIC_SESSION_COOKIE_KEY, comicsessid)
-        cookie.domain = ".pixiv.net"
-        cookie.path = "/"
-        mech.cookie_jar.add(cookie)
-      end
-    else
-      headers = {
-        "Origin" => "https://accounts.pixiv.net",
-        "Referer" => "https://accounts.pixiv.net/login?lang=en^source=pc&view_type=page&ref=wwwtop_accounts_index"
-      }
-
-      params = {
-        pixiv_id: Danbooru.config.pixiv_login,
-        password: Danbooru.config.pixiv_password,
-        captcha: nil,
-        g_captcha_response: nil,
-        source: "pc",
-        post_key: nil
-      }
-
-      mech.get("https://accounts.pixiv.net/login?lang=en&source=pc&view_type=page&ref=wwwtop_accounts_index") do |page|
-        json = page.search("input#init-config").first.attr("value")
-        if json =~ /pixivAccount\.postKey":"([a-f0-9]+)/
-          params[:post_key] = $1
-        end
-      end
-
-      mech.post("https://accounts.pixiv.net/api/login?lang=en", params, headers)
-      if mech.current_page.body =~ /"error":false/
-        cookie = mech.cookies.select {|x| x.name == SESSION_COOKIE_KEY}.first
-        if cookie
-          Cache.put(SESSION_CACHE_KEY, cookie.value, 1.week)
-        end
-      end
-
-      begin
-        mech.get("https://comic.pixiv.net") do |page|
-          cookie = mech.cookies.select {|x| x.name == COMIC_SESSION_COOKIE_KEY}.first
-          if cookie
-            Cache.put(COMIC_SESSION_CACHE_KEY, cookie.value, 1.week)
-          end
-        end
-      rescue Net::HTTPServiceUnavailable
-        # ignore
-      end
-    end
-
-    mech
-  end
-end

app/logical/popular_search_service.rb

@@ -1,61 +0,0 @@
-# queries reportbooru to find popular post searches
-class PopularSearchService
-  attr_reader :date
-
-  def self.enabled?
-    Danbooru.config.reportbooru_server.present?
-  end
-
-  def initialize(date)
-    if !PopularSearchService.enabled?
-      raise NotImplementedError.new("the Reportbooru service isn't configured. Popular searches are not available.")
-    end
-
-    @date = date
-  end
-
-  def each_search(limit = 100, &block)
-    JSON.parse(fetch_data.to_s).slice(0, limit).each(&block)
-  end
-
-  def tags
-    JSON.parse(fetch_data.to_s).map {|x| x[0]}
-  end
-
-  def fetch_data
-    return [] unless self.class.enabled?
-
-    dates = date.strftime("%Y-%m-%d")
-
-    data = Cache.get("ps-day-#{dates}", 1.minute) do
-      url = "#{Danbooru.config.reportbooru_server}/post_searches/rank?date=#{dates}"
-      response = HTTParty.get(url, Danbooru.config.httparty_options.reverse_merge(timeout: 3))
-      if response.success?
-        response = response.body
-      else
-        response = "[]"
-      end
-      response
-    end.to_s.force_encoding("utf-8")
-
-    if data.blank? || data == "[]"
-      dates = date.yesterday.strftime("%Y-%m-%d")
-
-      data = Cache.get("ps-day-#{dates}", 1.minute) do
-        url = "#{Danbooru.config.reportbooru_server}/post_searches/rank?date=#{dates}"
-        response = HTTParty.get(url, Danbooru.config.httparty_options.reverse_merge(timeout: 3))
-        if response.success?
-          response = response.body
-        else
-          response = "[]"
-        end
-        response
-      end.to_s.force_encoding("utf-8")
-    end
-
-    data
-  rescue StandardError => e
-    DanbooruLogger.log(e)
-    return []
-  end
-end

app/logical/post_query_builder.rb

@@ -307,6 +307,8 @@ class PostQueryBuilder
       Post.where(parent: nil)
     when "any"
       Post.where.not(parent: nil)
+    when /pending|flagged|modqueue|deleted|banned|active|unmoderated/
+      Post.where.not(parent: nil).where(parent: status_matches(parent))
     when /\A\d+\z/
       Post.where(id: parent).or(Post.where(parent: parent))
     else
@@ -320,6 +322,8 @@ class PostQueryBuilder
       Post.where(has_children: false)
     when "any"
       Post.where(has_children: true)
+    when /pending|flagged|modqueue|deleted|banned|active|unmoderated/
+      Post.where(has_children: true).where(children: status_matches(child))
     else
       Post.none
     end

app/logical/post_sets/post.rb

@@ -24,9 +24,8 @@ module PostSets
     end
 
     def wiki_page
-      return nil unless tag.present? && tag.wiki_page.present?
+      return nil unless normalized_query.has_single_tag?
-      return nil unless !tag.wiki_page.is_deleted?
+      @wiki_page ||= WikiPage.undeleted.find_by(title: normalized_query.tags.first.name)
-      tag.wiki_page
     end
 
     def tag
@@ -77,7 +76,11 @@ module PostSets
     end
 
     def per_page
-      (@per_page || query.find_metatag(:limit) || CurrentUser.user.per_page).to_i.clamp(0, MAX_PER_PAGE)
+      (@per_page || query.find_metatag(:limit) || CurrentUser.user.per_page).to_i.clamp(0, max_per_page)
+    end
+
+    def max_per_page
+      (format == "sitemap") ? 10_000 : MAX_PER_PAGE
     end
 
     def is_random?
@@ -94,7 +97,7 @@ module PostSets
     end
 
     def get_random_posts
-      per_page.times.inject([]) do |all, x|
+      per_page.times.inject([]) do |all, _|
         all << ::Post.user_tag_match(tag_string).random
       end.compact.uniq
     end
@@ -104,15 +107,15 @@ module PostSets
         @post_count = get_post_count
 
         if is_random?
-          temp = get_random_posts
+          get_random_posts
         else
-          temp = normalized_query.build.paginate(page, count: post_count, search_count: !post_count.nil?, limit: per_page)
+          normalized_query.build.paginate(page, count: post_count, search_count: !post_count.nil?, limit: per_page, max_limit: max_per_page).load
         end
       end
     end
 
     def hide_from_crawler?
-      return true if current_page > 1
+      return true if current_page > 50
       return false if query.is_empty_search? || query.is_simple_tag? || query.is_metatag?(:order, :rank)
       true
     end
@@ -160,20 +163,16 @@ module PostSets
         elsif query.is_metatag?(:search)
           saved_search_tags
         elsif query.is_empty_search? || query.is_metatag?(:order, :rank)
-          popular_tags
+          popular_tags.presence || frequent_tags
         elsif query.is_single_term?
-          similar_tags
+          similar_tags.presence || frequent_tags
         else
           frequent_tags
         end
       end
 
       def popular_tags
-        if PopularSearchService.enabled?
+        ReportbooruService.new.popular_searches(Date.today, limit: MAX_SIDEBAR_TAGS)
-          PopularSearchService.new(Date.today).tags
-        else
-          frequent_tags
-        end
       end
 
       def similar_tags

app/logical/post_view_count_service.rb

@@ -1,38 +0,0 @@
-class PostViewCountService
-  def self.enabled?
-    Danbooru.config.reportbooru_server.present?
-  end
-
-  def initialize
-    if !PostViewCountService.enabled?
-      raise NotImplementedError.new("the Reportbooru service isn't configured. Post views are not available.")
-    end
-  end
-
-  def fetch_count(post_id)
-    url = URI.parse("#{Danbooru.config.reportbooru_server}/post_views/#{post_id}")
-    response = HTTParty.get(url, Danbooru.config.httparty_options.reverse_merge(timeout: 6))
-    if response.success?
-      return JSON.parse(response.body)
-    else
-      return nil
-    end
-  end
-
-  def fetch_rank(date = Date.today)
-    url = URI.parse("#{Danbooru.config.reportbooru_server}/post_views/rank?date=#{date}")
-    response = HTTParty.get(url, Danbooru.config.httparty_options.reverse_merge(timeout: 6))
-    if response.success?
-      return JSON.parse(response.body)
-    else
-      return nil
-    end
-  rescue JSON::ParserError
-    nil
-  end
-
-  def popular_posts(date = Date.today)
-    ranking = fetch_rank(date) || []
-    ranking.slice(0, 50).map {|x| Post.find(x[0])}
-  end
-end

app/logical/reportbooru_service.rb

@@ -0,0 +1,50 @@
+class ReportbooruService
+  attr_reader :http, :reportbooru_server
+
+  def initialize(http: Danbooru::Http.new, reportbooru_server: Danbooru.config.reportbooru_server)
+    @reportbooru_server = reportbooru_server
+    @http = http
+  end
+
+  def enabled?
+    reportbooru_server.present?
+  end
+
+  def missed_search_rankings(expires_in: 1.minutes)
+    return [] unless enabled?
+
+    response = http.cache(expires_in).get("#{reportbooru_server}/missed_searches")
+    return [] if response.status != 200
+
+    body = response.to_s.force_encoding("utf-8")
+    body.lines.map(&:split).map { [_1, _2.to_i] }
+  end
+
+  def post_search_rankings(date, expires_in: 1.minutes)
+    request("#{reportbooru_server}/post_searches/rank?date=#{date}", expires_in)
+  end
+
+  def post_view_rankings(date, expires_in: 1.minutes)
+    request("#{reportbooru_server}/post_views/rank?date=#{date}", expires_in)
+  end
+
+  def popular_searches(date, limit: 100)
+    ranking = post_search_rankings(date)
+    ranking = post_search_rankings(date.yesterday) if ranking.blank?
+    ranking.take(limit).map(&:first)
+  end
+
+  def popular_posts(date, limit: 100)
+    ranking = post_view_rankings(date)
+    ranking = post_view_rankings(date.yesterday) if ranking.blank?
+    ranking.take(limit).map { |x| Post.find(x[0]) }
+  end
+
+  def request(url, expires_in)
+    return [] unless enabled?
+
+    response = http.cache(expires_in).get(url)
+    return [] if response.status != 200
+    JSON.parse(response.to_s.force_encoding("utf-8"))
+  end
+end

app/logical/sources/strategies.rb

@@ -1,7 +1,7 @@
 module Sources
   module Strategies
     def self.all
-      return [
+      [
         Strategies::Pixiv,
         Strategies::NicoSeiga,
         Strategies::Twitter,
@@ -13,7 +13,8 @@ module Sources
         Strategies::Pawoo,
         Strategies::Moebooru,
         Strategies::HentaiFoundry,
-        Strategies::Weibo
+        Strategies::Weibo,
+        Strategies::Newgrounds
       ]
     end
 

app/logical/sources/strategies/art_station.rb

@@ -22,15 +22,15 @@
 
 module Sources::Strategies
   class ArtStation < Base
-    PROJECT1 = %r!\Ahttps?://www\.artstation\.com/artwork/(?<project_id>[a-z0-9-]+)/?\z!i
+    PROJECT1 = %r{\Ahttps?://www\.artstation\.com/artwork/(?<project_id>[a-z0-9-]+)/?\z}i
-    PROJECT2 = %r!\Ahttps?://(?<artist_name>[\w-]+)\.artstation\.com/projects/(?<project_id>[a-z0-9-]+)(?:/|\?[\w=-]+)?\z!i
+    PROJECT2 = %r{\Ahttps?://(?<artist_name>[\w-]+)\.artstation\.com/projects/(?<project_id>[a-z0-9-]+)(?:/|\?[\w=-]+)?\z}i
     PROJECT = Regexp.union(PROJECT1, PROJECT2)
     ARTIST1 = %r{\Ahttps?://(?<artist_name>[\w-]+)(?<!www)\.artstation\.com/?\z}i
     ARTIST2 = %r{\Ahttps?://www\.artstation\.com/artist/(?<artist_name>[\w-]+)/?\z}i
     ARTIST3 = %r{\Ahttps?://www\.artstation\.com/(?<artist_name>[\w-]+)/?\z}i
     ARTIST = Regexp.union(ARTIST1, ARTIST2, ARTIST3)
 
-    ASSET = %r!\Ahttps?://cdn\w*\.artstation\.com/p/assets/(?<type>images|covers)/images/(?<id>\d+/\d+/\d+)/(?<size>[^/]+)/(?<filename>.+)\z!i
+    ASSET = %r{\Ahttps?://cdn\w*\.artstation\.com/p/assets/(?<type>images|covers)/images/(?<id>\d+/\d+/\d+)/(?<size>[^/]+)/(?<filename>.+)\z}i
 
     attr_reader :json
 
@@ -144,10 +144,10 @@ module Sources::Strategies
 
       urls = image_url_sizes($~[:type], $~[:id], $~[:filename])
       if size == :smallest
-        urls = urls.reverse()
+        urls = urls.reverse
       end
 
-      chosen_url = urls.find { |url| http_exists?(url, headers) }
+      chosen_url = urls.find { |url| http_exists?(url) }
       chosen_url || url
     end
   end

app/logical/sources/strategies/base.rb

@@ -14,6 +14,8 @@
 module Sources
   module Strategies
     class Base
+      class DownloadError < StandardError; end
+
       attr_reader :url, :referer_url, :urls, :parsed_url, :parsed_referer, :parsed_urls
 
       extend Memoist
@@ -35,9 +37,9 @@ module Sources
       #   <tt>referrer_url</tt> so the strategy can discover the HTML
       #   page and other information.
       def initialize(url, referer_url = nil)
-        @url = url
+        @url = url.to_s
-        @referer_url = referer_url
+        @referer_url = referer_url&.to_s
-        @urls = [url, referer_url].select(&:present?)
+        @urls = [@url, @referer_url].select(&:present?)
 
         @parsed_url = Addressable::URI.heuristic_parse(url) rescue nil
         @parsed_referer = Addressable::URI.heuristic_parse(referer_url) rescue nil
@@ -58,8 +60,8 @@ module Sources
       end
 
       def site_name
-        Addressable::URI.heuristic_parse(url).host
+        Addressable::URI.heuristic_parse(url)&.host
-      rescue Addressable::URI::InvalidURIError => e
+      rescue Addressable::URI::InvalidURIError
         nil
       end
 
@@ -90,9 +92,7 @@ module Sources
       # eventually be assigned as the source for the post, but it does not
       # represent what the downloader will fetch.
       def page_url
-        Rails.logger.warn "Valid page url for (#{url}, #{referer_url}) not found"
+        nil
-
-        return nil
       end
 
       # This will be the url stored in posts. Typically this is the page
@@ -141,14 +141,37 @@ module Sources
       # Subclasses should merge in any required headers needed to access resources
       # on the site.
       def headers
-        return Danbooru.config.http_headers
+        {}
       end
 
       # Returns the size of the image resource without actually downloading the file.
-      def size
+      def remote_size
-        Downloads::File.new(image_url).size
+        response = http_downloader.head(image_url)
+        return nil unless response.status == 200 && response.content_length.present?
+
+        response.content_length.to_i
       end
-      memoize :size
+      memoize :remote_size
+
+      # Download the file at the given url, or at the main image url by default.
+      def download_file!(download_url = image_url)
+        raise DownloadError, "Download failed: couldn't find download url for #{url}" if download_url.blank?
+        response, file = http_downloader.download_media(download_url)
+        raise DownloadError, "Download failed: #{download_url} returned error #{response.status}" if response.status != 200
+        file
+      end
+
+      # A http client for API requests.
+      def http
+        Danbooru::Http.new.public_only
+      end
+      memoize :http
+
+      # A http client for downloading files.
+      def http_downloader
+        http.timeout(30).max_size(Danbooru.config.max_file_size).use(:spoof_referrer).use(:unpolish_cloudflare)
+      end
+      memoize :http_downloader
 
       # The url to use for artist finding purposes. This will be stored in the
       # artist entry. Normally this will be the profile url.
@@ -189,7 +212,7 @@ module Sources
       end
 
       def normalized_tags
-        tags.map { |tag, url| normalize_tag(tag) }.sort.uniq
+        tags.map { |tag, _url| normalize_tag(tag) }.sort.uniq
       end
 
       def normalize_tag(tag)
@@ -243,7 +266,7 @@ module Sources
       end
 
       def to_h
-        return {
+        {
           :artist => {
             :name => artist_name,
             :tag_name => tag_name,
@@ -276,9 +299,8 @@ module Sources
         to_h.to_json
       end
 
-      def http_exists?(url, headers)
+      def http_exists?(url)
-        res = HTTParty.head(url, Danbooru.config.httparty_options.deep_merge(headers: headers))
+        http_downloader.head(url).status.success?
-        res.success?
       end
 
       # Convert commentary to dtext by stripping html tags. Sites can override

app/logical/sources/strategies/deviant_art.rb

@@ -47,18 +47,18 @@
 module Sources
   module Strategies
     class DeviantArt < Base
-      ASSET_SUBDOMAINS = %r{(?:fc|th|pre|img|orig|origin-orig)\d*}i
+      ASSET_SUBDOMAINS = /(?:fc|th|pre|img|orig|origin-orig)\d*/i
       RESERVED_SUBDOMAINS = %r{\Ahttps?://(?:#{ASSET_SUBDOMAINS}|www)\.}i
       MAIN_DOMAIN = %r{\Ahttps?://(?:www\.)?deviantart.com}i
 
-      TITLE = %r{(?<title>[a-z0-9_-]+?)}i
+      TITLE = /(?<title>[a-z0-9_-]+?)/i
-      ARTIST = %r{(?<artist>[a-z0-9_-]+?)}i
+      ARTIST = /(?<artist>[a-z0-9_-]+?)/i
-      DEVIATION_ID = %r{(?<deviation_id>[0-9]+)}i
+      DEVIATION_ID = /(?<deviation_id>[0-9]+)/i
 
-      DA_FILENAME_1 = %r{[a-f0-9]{32}-d(?<base36_deviation_id>[a-z0-9]+)\.}i
+      DA_FILENAME_1 = /[a-f0-9]{32}-d(?<base36_deviation_id>[a-z0-9]+)\./i
-      DA_FILENAME_2 = %r{#{TITLE}(?:_by_#{ARTIST}(?:-d(?<base36_deviation_id>[a-z0-9]+))?)?\.}i
+      DA_FILENAME_2 = /#{TITLE}(?:_by_#{ARTIST}(?:-d(?<base36_deviation_id>[a-z0-9]+))?)?\./i
       DA_FILENAME = Regexp.union(DA_FILENAME_1, DA_FILENAME_2)
-      WIX_FILENAME  = %r{d(?<base36_deviation_id>[a-z0-9]+)[0-9a-f-]+\.\w+(?:/\w+/\w+/[\w,]+/(?<title>[\w-]+)_by_(?<artist>[\w-]+)_d\w+-\w+\.\w+)?.+}i
+      WIX_FILENAME = %r{d(?<base36_deviation_id>[a-z0-9]+)[0-9a-f-]+\.\w+(?:/\w+/\w+/[\w,]+/(?<title>[\w-]+)_by_(?<artist>[\w-]+)_d\w+-\w+\.\w+)?.+}i
 
       NOT_NORMALIZABLE_ASSET = %r{\Ahttps?://#{ASSET_SUBDOMAINS}\.deviantart\.net/.+/[0-9a-f]{32}(?:-[^d]\w+)?\.}i
 
@@ -75,7 +75,7 @@ module Sources
       PATH_PROFILE = %r{#{MAIN_DOMAIN}/#{ARTIST}/?\z}i
       SUBDOMAIN_PROFILE = %r{\Ahttps?://#{ARTIST}\.deviantart\.com/?\z}i
 
-      FAVME = %r{\Ahttps?://(www\.)?fav\.me/d(?<base36_deviation_id>[a-z0-9]+)\z}i
+      FAVME = %r{\Ahttps?://(?:www\.)?fav\.me/d(?<base36_deviation_id>[a-z0-9]+)\z}i
 
       def domains
         ["deviantart.net", "deviantart.com", "fav.me"]
@@ -110,12 +110,12 @@ module Sources
           api_deviation[:videos].max_by { |x| x[:filesize] }[:src]
         else
           src = api_deviation.dig(:content, :src)
-          if deviation_id && deviation_id.to_i <= 790677560 && src =~ /^https:\/\/images-wixmp-/ && src !~ /\.gif\?/
+          if deviation_id && deviation_id.to_i <= 790_677_560 && src =~ %r{\Ahttps://images-wixmp-} && src !~ /\.gif\?/
-            src = src.sub(%r!(/f/[a-f0-9-]+/[a-f0-9-]+)!, '/intermediary\1')
+            src = src.sub(%r{(/f/[a-f0-9-]+/[a-f0-9-]+)}, '/intermediary\1')
-            src = src.sub(%r!/v1/(fit|fill)/.*\z!i, "")
+            src = src.sub(%r{/v1/(fit|fill)/.*\z}i, "")
           end
-          src = src.sub(%r!\Ahttps?://orig\d+\.deviantart\.net!i, "http://origin-orig.deviantart.net")
+          src = src.sub(%r{\Ahttps?://orig\d+\.deviantart\.net}i, "http://origin-orig.deviantart.net")
-          src = src.gsub(%r!q_\d+,strp!, "q_100")
+          src = src.gsub(/q_\d+,strp/, "q_100")
           src
         end
       end
@@ -191,7 +191,7 @@ module Sources
 
             # <a href="https://sa-dui.deviantart.com/journal/About-Commissions-223178193" data-sigil="thumb" class="thumb lit" ...>
             if element["class"].split.include?("lit")
-              deviation_id = element["href"][%r!-(\d+)\z!, 1].to_i
+              deviation_id = element["href"][/-(\d+)\z/, 1].to_i
               element.content = "deviantart ##{deviation_id}"
             else
               element.content = ""
@@ -199,7 +199,7 @@ module Sources
           end
 
           if element.name == "a" && element["href"].present?
-            element["href"] = element["href"].gsub(%r!\Ahttps?://www\.deviantart\.com/users/outgoing\?!i, "")
+            element["href"] = element["href"].gsub(%r{\Ahttps?://www\.deviantart\.com/users/outgoing\?}i, "")
 
             # href may be missing the `http://` bit (ex: `inprnt.com`, `//inprnt.com`). Add it if missing.
             uri = Addressable::URI.heuristic_parse(element["href"]) rescue nil
@@ -283,7 +283,7 @@ module Sources
         return nil if meta.nil?
 
         appurl = meta["content"]
-        uuid = appurl[%r!\ADeviantArt://deviation/(.*)\z!, 1]
+        uuid = appurl[%r{\ADeviantArt://deviation/(.*)\z}, 1]
         uuid
       end
       memoize :uuid

app/logical/sources/strategies/hentai_foundry.rb

@@ -23,11 +23,11 @@
 module Sources
   module Strategies
     class HentaiFoundry < Base
-      BASE_URL =    %r!\Ahttps?://(?:www\.)?hentai-foundry\.com!i
+      BASE_URL =    %r{\Ahttps?://(?:www\.)?hentai-foundry\.com}i
-      PAGE_URL =    %r!#{BASE_URL}/pictures/user/(?<artist_name>[\w-]+)/(?<illust_id>\d+)(?:/[\w.-]*)?(\?[\w=]*)?\z!i
+      PAGE_URL =    %r{#{BASE_URL}/pictures/user/(?<artist_name>[\w-]+)/(?<illust_id>\d+)(?:/[\w.-]*)?(\?[\w=]*)?\z}i
-      OLD_PAGE =    %r!#{BASE_URL}/pic-(?<illust_id>\d+)(?:\.html)?\z!i
+      OLD_PAGE =    %r{#{BASE_URL}/pic-(?<illust_id>\d+)(?:\.html)?\z}i
-      PROFILE_URL = %r!#{BASE_URL}/(?:pictures/)?user/(?<artist_name>[\w-]+)(?:/[a-z]*)?\z!i
+      PROFILE_URL = %r{#{BASE_URL}/(?:pictures/)?user/(?<artist_name>[\w-]+)(?:/[a-z]*)?\z}i
-      IMAGE_URL =   %r!\Ahttps?://pictures\.hentai-foundry\.com/+\w/(?<artist_name>[\w-]+)/(?<illust_id>\d+)(?:(?:/[\w.-]+)?\.\w+)?\z!i
+      IMAGE_URL =   %r{\Ahttps?://pictures\.hentai-foundry\.com/+\w/(?<artist_name>[\w-]+)/(?<illust_id>\d+)(?:(?:/[\w.-]+)?\.\w+)?\z}i
 
       def domains
         ["hentai-foundry.com"]
@@ -64,11 +64,10 @@ module Sources
       def page
         return nil if page_url.blank?
 
-        doc = Cache.get("hentai-foundry:#{page_url}", 1.minute) do
+        response = Danbooru::Http.new.cache(1.minute).get("#{page_url}?enterAgree=1")
-          HTTParty.get("#{page_url}?enterAgree=1").body
+        return nil unless response.status == 200
-        end
 
-        Nokogiri::HTML(doc)
+        response.parse
       end
 
       def tags

app/logical/sources/strategies/moebooru.rb

@@ -32,10 +32,10 @@
 module Sources
   module Strategies
     class Moebooru < Base
-      BASE_URL = %r!\Ahttps?://(?:[^.]+\.)?(?<domain>yande\.re|konachan\.com)!i
+      BASE_URL = %r{\Ahttps?://(?:[^.]+\.)?(?<domain>yande\.re|konachan\.com)}i
-      POST_URL = %r!#{BASE_URL}/post/show/(?<id>\d+)!i
+      POST_URL = %r{#{BASE_URL}/post/show/(?<id>\d+)}i
-      URL_SLUG = %r!/(?:yande\.re%20|Konachan\.com%20-%20)?(?<id>\d+)?.*!i
+      URL_SLUG = %r{/(?:yande\.re%20|Konachan\.com%20-%20)?(?<id>\d+)?.*}i
-      IMAGE_URL = %r!#{BASE_URL}/(?<type>image|jpeg|sample)/(?<md5>\h{32})#{URL_SLUG}?\.(?<ext>jpg|jpeg|png|gif)\z!i
+      IMAGE_URL = %r{#{BASE_URL}/(?<type>image|jpeg|sample)/(?<md5>\h{32})#{URL_SLUG}?\.(?<ext>jpg|jpeg|png|gif)\z}i
 
       delegate :artist_name, :profile_url, :tag_name, :artist_commentary_title, :artist_commentary_desc, :dtext_artist_commentary_title, :dtext_artist_commentary_desc, to: :sub_strategy, allow_nil: true
 
@@ -63,7 +63,7 @@ module Sources
       end
 
       def preview_urls
-        return image_urls unless post_md5.present?
+        return image_urls if post_md5.blank?
         ["https://#{file_host}/data/preview/#{post_md5[0..1]}/#{post_md5[2..3]}/#{post_md5}.jpg"]
       end
 
@@ -155,7 +155,7 @@ module Sources
 
           # the api_response wasn't available because it's a deleted post.
           elsif post_md5.present?
-            %w[jpg png gif].find { |ext| http_exists?("https://#{site_name}/image/#{post_md5}.#{ext}", headers) }
+            %w[jpg png gif].find { |ext| http_exists?("https://#{site_name}/image/#{post_md5}.#{ext}") }
 
           else
             nil

app/logical/sources/strategies/newgrounds.rb

@@ -0,0 +1,111 @@
+# Image Urls
+# * https://art.ngfiles.com/images/1254000/1254722_natthelich_pandora.jpg
+# * https://art.ngfiles.com/images/1033000/1033622_natthelich_fire-emblem-marth-plus-progress-pic.png?f1569487181
+# * https://art.ngfiles.com/comments/57000/iu_57615_7115981.jpg
+#
+# Page URLs
+# * https://www.newgrounds.com/art/view/puddbytes/costanza-at-bat
+# * https://www.newgrounds.com/art/view/natthelich/fire-emblem-marth-plus-progress-pic (multiple)
+#
+# Profile URLs
+# * https://natthelich.newgrounds.com/
+
+module Sources
+  module Strategies
+    class Newgrounds < Base
+      IMAGE_URL   = %r{\Ahttps?://art\.ngfiles\.com/images/\d+/\d+_(?<user_name>[0-9a-z-]+)_(?<illust_title>[0-9a-z-]+)\.\w+}i
+      COMMENT_URL = %r{\Ahttps?://art\.ngfiles\.com/comments/\d+/\w+\.\w+}i
+
+      PAGE_URL    = %r{\Ahttps?://(?:www\.)?newgrounds\.com/art/view/(?<user_name>[0-9a-z-]+)/(?<illust_title>[0-9a-z-]+)(?:\?.*)?}i
+
+      PROFILE_URL = %r{\Ahttps?://(?<artist_name>(?!www)[0-9a-z-]+)\.newgrounds\.com(?:/.*)?}i
+
+      def domains
+        ["newgrounds.com", "ngfiles.com"]
+      end
+
+      def site_name
+        "NewGrounds"
+      end
+
+      def image_urls
+        if url =~ COMMENT_URL || url =~ IMAGE_URL
+          [url]
+        else
+          urls = []
+
+          urls += page&.css(".image img").to_a.map { |img| img["src"] }
+          urls += page&.css("#author_comments img[data-user-image='1']").to_a.map { |img| img["data-smartload-src"] || img["src"] }
+
+          urls.compact
+        end
+      end
+
+      def page_url
+        return nil if illust_title.blank? || user_name.blank?
+
+        "https://www.newgrounds.com/art/view/#{user_name}/#{illust_title}"
+      end
+
+      def page
+        return nil if page_url.blank?
+
+        response = Danbooru::Http.cache(1.minute).get(page_url)
+        return nil if response.status == 404
+
+        response.parse
+      end
+      memoize :page
+
+      def tags
+        page&.css("#sidestats .tags a").to_a.map do |tag|
+          [tag.text, "https://www.newgrounds.com/search/conduct/art?match=tags&tags=" + tag.text]
+        end
+      end
+
+      def normalize_tag(tag)
+        tag = tag.tr("-", "_")
+        super(tag)
+      end
+
+      def artist_name
+        name = page&.css(".item-user .item-details h4 a")&.text&.strip || user_name
+        name&.downcase
+      end
+
+      def other_names
+        [artist_name, user_name].compact.uniq
+      end
+
+      def profile_url
+        # user names are not mutable, artist names are.
+        # However we need the latest name for normalization
+        "https://#{artist_name}.newgrounds.com"
+      end
+
+      def artist_commentary_title
+        page&.css(".pod-head > [itemprop='name']")&.text
+      end
+
+      def artist_commentary_desc
+        page&.css("#author_comments")&.to_html
+      end
+
+      def dtext_artist_commentary_desc
+        DText.from_html(artist_commentary_desc)
+      end
+
+      def normalize_for_source
+        page_url
+      end
+
+      def user_name
+        urls.map { |u| url[PROFILE_URL, :artist_name] || u[IMAGE_URL, :user_name] || u[PAGE_URL, :user_name] }.compact.first
+      end
+
+      def illust_title
+        urls.map { |u| u[IMAGE_URL, :illust_title] || u[PAGE_URL, :illust_title] }.compact.first
+      end
+    end
+  end
+end

app/logical/sources/strategies/nico_seiga.rb

@@ -1,25 +1,54 @@
-# Image Direct URL
+# Direct URL
 # * https://lohas.nicoseiga.jp/o/971eb8af9bbcde5c2e51d5ef3a2f62d6d9ff5552/1589933964/3583893
 # * http://lohas.nicoseiga.jp/priv/3521156?e=1382558156&h=f2e089256abd1d453a455ec8f317a6c703e2cedf
 # * http://lohas.nicoseiga.jp/priv/b80f86c0d8591b217e7513a9e175e94e00f3c7a1/1384936074/3583893
+# * https://dcdn.cdn.nimg.jp/priv/62a56a7f67d3d3746ae5712db9cac7d465f4a339/1592186183/10466669
+# * https://dcdn.cdn.nimg.jp/nicoseiga/lohas/o/8ba0a9b2ea34e1ef3b5cc50785bd10cd63ec7e4a/1592187477/10466669
+#
+# * http://lohas.nicoseiga.jp/material/5746c5/4459092
+#
+# (Manga direct url)
+# * https://lohas.nicoseiga.jp/priv/f5b8966fd53bf7e06cccff9fbb2c4eef62877538/1590752727/8947170
+#
+# Samples
+# * http://lohas.nicoseiga.jp/thumb/2163478i?
+# * https://lohas.nicoseiga.jp/thumb/8947170p
+#
+## The direct urls and samples above can belong to both illust and manga.
+## There's two ways to tell them apart:
+## * visit the /source/ equivalent: illusts redirect to the /o/ intermediary page, manga redirect to /priv/ directly
+## * try an api call: illusts will succeed, manga will fail
+#
+# Source Link
 # * http://seiga.nicovideo.jp/image/source?id=3312222
 #
-# Image Page URL
+# Illust Page URL
 # * https://seiga.nicovideo.jp/seiga/im3521156
+# * https://seiga.nicovideo.jp/seiga/im520647 (anonymous artist)
 #
 # Manga Page URL
 # * http://seiga.nicovideo.jp/watch/mg316708
+#
+# Video Page URL (not supported)
+# * https://www.nicovideo.jp/watch/sm36465441
+#
+# Oekaki
+# * https://dic.nicovideo.jp/oekaki/52833.png
 
 module Sources
   module Strategies
     class NicoSeiga < Base
-      URL = %r!\Ahttps?://(?:\w+\.)?nico(?:seiga|video)\.jp!
+      DIRECT       = %r{\Ahttps?://lohas\.nicoseiga\.jp/(?:priv|o)/(?:\w+/\d+/)?(?<image_id>\d+)(?:\?.+)?}i
-      DIRECT1 = %r!\Ahttps?://lohas\.nicoseiga\.jp/priv/[0-9a-f]+!
+      CDN_DIRECT   = %r{\Ahttps?://dcdn\.cdn\.nimg\.jp/.+/\w+/\d+/(?<image_id>\d+)}i
-      DIRECT2 = %r!\Ahttps?://lohas\.nicoseiga\.jp/o/[0-9a-f]+/\d+/\d+!
+      SOURCE       = %r{\Ahttps?://seiga\.nicovideo\.jp/image/source(?:/|\?id=)(?<image_id>\d+)}i
-      DIRECT3 = %r!\Ahttps?://seiga\.nicovideo\.jp/images/source/\d+!
+
-      PAGE = %r!\Ahttps?://seiga\.nicovideo\.jp/seiga/im(\d+)!i
+      ILLUST_THUMB = %r{\Ahttps?://lohas\.nicoseiga\.jp/thumb/(?<illust_id>\d+)i}i
-      PROFILE = %r!\Ahttps?://seiga\.nicovideo\.jp/user/illust/(\d+)!i
+      MANGA_THUMB  = %r{\Ahttps?://lohas\.nicoseiga\.jp/thumb/(?<image_id>\d+)p}i
-      MANGA_PAGE = %r!\Ahttps?://seiga\.nicovideo\.jp/watch/mg(\d+)!i
+
+      ILLUST_PAGE  = %r{\Ahttps?://(?:sp\.)?seiga\.nicovideo\.jp/seiga/im(?<illust_id>\d+)}i
+      MANGA_PAGE   = %r{\Ahttps?://(?:sp\.)?seiga\.nicovideo\.jp/watch/mg(?<manga_id>\d+)}i
+
+      PROFILE_PAGE = %r{\Ahttps?://seiga\.nicovideo\.jp/user/illust/(?<artist_id>\d+)}i
 
       def domains
         ["nicoseiga.jp", "nicovideo.jp"]
@@ -30,160 +59,136 @@ module Sources
       end
 
       def image_urls
-        if url =~ DIRECT1
+        urls = []
-          return [url]
+        return urls if api_client&.api_response.blank?
+
+        if image_id.present?
+          urls << "https://seiga.nicovideo.jp/image/source/#{image_id}"
+        elsif illust_id.present?
+          urls << "https://seiga.nicovideo.jp/image/source/#{illust_id}"
+        elsif manga_id.present? && api_client.image_ids.present?
+          urls += api_client.image_ids.map { |id| "https://seiga.nicovideo.jp/image/source/#{id}" }
+        end
+        urls
+      end
+
+      def image_url
+        return url if image_urls.blank? || api_client.blank?
+
+        img = case url
+        when DIRECT || CDN_DIRECT then "https://seiga.nicovideo.jp/image/source/#{image_id_from_url(url)}"
+        when SOURCE then url
+        else image_urls.first
         end
 
-        if theme_id
+        resp = api_client.login.head(img)
-          return api_client.image_ids.map do |image_id|
+        if resp.uri.to_s =~ %r{https?://.+/(\w+/\d+/\d+)\z}i
-            "https://seiga.nicovideo.jp/image/source/#{image_id}"
+          "https://lohas.nicoseiga.jp/priv/#{$1}"
-          end
+        else
+          img
         end
+      end
 
-        link = page.search("a#illust_link")
+      def preview_urls
-
+        if illust_id.present?
-        if link.any?
+          ["https://lohas.nicoseiga.jp/thumb/#{illust_id}i"]
-          image_url = "http://seiga.nicovideo.jp" + link[0]["href"]
+        else
-          page = agent.get(image_url) # need to follow this redirect while logged in or it won't work
+          image_urls
-
-          if page.is_a?(Mechanize::Image)
-            return [page.uri.to_s]
-          end
-
-          images = page.search("div.illust_view_big").select {|x| x["data-src"] =~ /\/priv\//}
-
-          if images.any?
-            return ["http://lohas.nicoseiga.jp" + images[0]["data-src"]]
-          end
         end
-
-        raise "image url not found for (#{url}, #{referer_url})"
       end
 
       def page_url
-        [url, referer_url].each do |x|
+        if illust_id.present?
-          if x =~ %r!\Ahttps?://lohas\.nicoseiga\.jp/o/[a-f0-9]+/\d+/(\d+)!
+          "https://seiga.nicovideo.jp/seiga/im#{illust_id}"
-            return "http://seiga.nicovideo.jp/seiga/im#{$1}"
+        elsif manga_id.present?
-          end
+          "https://seiga.nicovideo.jp/watch/mg#{manga_id}"
-
+        elsif image_id.present?
-          if x =~ %r{\Ahttps?://lohas\.nicoseiga\.jp/priv/(\d+)\?e=\d+&h=[a-f0-9]+}i
+          "https://seiga.nicovideo.jp/image/source/#{image_id}"
-            return "http://seiga.nicovideo.jp/seiga/im#{$1}"
-          end
-
-          if x =~ %r{\Ahttps?://lohas\.nicoseiga\.jp/priv/[a-f0-9]+/\d+/(\d+)}i
-            return "http://seiga.nicovideo.jp/seiga/im#{$1}"
-          end
-
-          if x =~ %r{\Ahttps?://lohas\.nicoseiga\.jp/priv/(\d+)}i
-            return "http://seiga.nicovideo.jp/seiga/im#{$1}"
-          end
-
-          if x =~ %r{\Ahttps?://lohas\.nicoseiga\.jp//?thumb/(\d+)i?}i
-            return "http://seiga.nicovideo.jp/seiga/im#{$1}"
-          end
-
-          if x =~ %r{/seiga/im\d+}
-            return x
-          end
-
-          if x =~ %r{/watch/mg\d+}
-            return x
-          end
-
-          if x =~ %r{/image/source\?id=(\d+)}
-            return "http://seiga.nicovideo.jp/seiga/im#{$1}"
-          end
         end
-
-        return super
-      end
-
-      def canonical_url
-        image_url
       end
 
       def profile_url
-        if url =~ PROFILE
+        user_id = api_client&.user_id
-          return url
+        return if user_id.blank? # artists can be anonymous
-        end
 
         "http://seiga.nicovideo.jp/user/illust/#{api_client.user_id}"
       end
 
       def artist_name
-        api_client.moniker
+        return if api_client.blank?
+        api_client.user_name
       end
 
       def artist_commentary_title
+        return if api_client.blank?
         api_client.title
       end
 
       def artist_commentary_desc
-        api_client.desc
+        return if api_client.blank?
+        api_client.description
+      end
+
+      def dtext_artist_commentary_desc
+        DText.from_html(artist_commentary_desc).gsub(/[^\w]im(\d+)/, ' seiga #\1 ')
       end
 
       def normalize_for_source
-        if illust_id.present?
+        # There's no way to tell apart illust from manga from the direct image url alone. What's worse,
-          "https://seiga.nicovideo.jp/seiga/im#{illust_id}"
+        # nicoseiga itself doesn't know how to normalize back to manga, so if it's not an illust type then
-        elsif theme_id.present?
+        # it's impossible to get the original manga page back from the image url alone.
-          "http://seiga.nicovideo.jp/watch/mg#{theme_id}"
+        # /source/ links on the other hand correctly redirect, hence we use them to normalize saved direct sources.
+        if url =~ DIRECT
+          "https://seiga.nicovideo.jp/image/source/#{image_id}"
+        else
+          page_url
         end
       end
 
       def tag_name
+        return if api_client&.user_id.blank?
         "nicoseiga#{api_client.user_id}"
       end
 
       def tags
-        string = page.at("meta[name=keywords]").try(:[], "content") || ""
+        return [] if api_client.blank?
-        string.split(/,/).map do |name|
+
-          [name, "https://seiga.nicovideo.jp/tag/#{CGI.escape(name)}"]
+        base_url = "https://seiga.nicovideo.jp/"
+        base_url += "manga/" if manga_id.present?
+        base_url += "tag/"
+
+        api_client.tags.map do |name|
+          [name, base_url + CGI.escape(name)]
         end
       end
-      memoize :tags
+
+      def image_id
+        image_id_from_url(url)
+      end
+
+      def image_id_from_url(url)
+        url[DIRECT, :image_id] || url[SOURCE, :image_id] || url[MANGA_THUMB, :image_id] || url[CDN_DIRECT, :image_id]
+      end
+
+      def illust_id
+        urls.map { |u| u[ILLUST_PAGE, :illust_id] || u[ILLUST_THUMB, :illust_id] }.compact.first
+      end
+
+      def manga_id
+        urls.compact.map { |u| u[MANGA_PAGE, :manga_id] }.compact.first
+      end
 
       def api_client
-        if illust_id
+        if illust_id.present?
-          NicoSeigaApiClient.new(illust_id: illust_id)
+          NicoSeigaApiClient.new(work_id: illust_id, type: "illust", http: http)
-        elsif theme_id
+        elsif manga_id.present?
-          NicoSeigaMangaApiClient.new(theme_id)
+          NicoSeigaApiClient.new(work_id: manga_id, type: "manga", http: http)
+        elsif image_id.present?
+          # We default to illust to attempt getting the api anyway
+          NicoSeigaApiClient.new(work_id: image_id, type: "illust", http: http)
         end
       end
       memoize :api_client
-
-      def illust_id
-        if page_url =~ PAGE
-          return $1.to_i
-        end
-
-        return nil
-      end
-
-      def theme_id
-        if page_url =~ MANGA_PAGE
-          return $1.to_i
-        end
-
-        return nil
-      end
-
-      def page
-        doc = agent.get(page_url)
-
-        if doc.search("a#link_btn_login").any?
-          # Session cache is invalid, clear it and log in normally.
-          Cache.delete("nico-seiga-session")
-          doc = agent.get(page_url)
-        end
-
-        doc
-      end
-      memoize :page
-
-      def agent
-        NicoSeigaApiClient.agent
-      end
-      memoize :agent
     end
   end
 end

app/logical/sources/strategies/nijie.rb

@@ -44,25 +44,25 @@
 module Sources
   module Strategies
     class Nijie < Base
-      BASE_URL = %r!\Ahttps?://(?:[^.]+\.)?nijie\.info!i
+      BASE_URL = %r{\Ahttps?://(?:[^.]+\.)?nijie\.info}i
-      PAGE_URL = %r!#{BASE_URL}/view(?:_popup)?\.php\?id=(?<illust_id>\d+)!i
+      PAGE_URL = %r{#{BASE_URL}/view(?:_popup)?\.php\?id=(?<illust_id>\d+)}i
-      PROFILE_URL = %r!#{BASE_URL}/members(?:_illust)?\.php\?id=(?<artist_id>\d+)\z!i
+      PROFILE_URL = %r{#{BASE_URL}/members(?:_illust)?\.php\?id=(?<artist_id>\d+)\z}i
 
       # https://pic03.nijie.info/nijie_picture/28310_20131101215959.jpg
       # https://pic03.nijie.info/nijie_picture/236014_20170620101426_0.png
       # http://pic.nijie.net/03/nijie_picture/829001_20190620004513_0.mp4
       # https://pic05.nijie.info/nijie_picture/diff/main/559053_20180604023346_1.png
-      FILENAME1 = %r!(?<artist_id>\d+)_(?<timestamp>\d{14})(?:_\d+)?!i
+      FILENAME1 = /(?<artist_id>\d+)_(?<timestamp>\d{14})(?:_\d+)?/i
 
       # https://pic01.nijie.info/nijie_picture/diff/main/218856_0_236014_20170620101329.png
-      FILENAME2 = %r!(?<illust_id>\d+)_\d+_(?<artist_id>\d+)_(?<timestamp>\d{14})!i
+      FILENAME2 = /(?<illust_id>\d+)_\d+_(?<artist_id>\d+)_(?<timestamp>\d{14})/i
 
       # https://pic04.nijie.info/nijie_picture/diff/main/287736_161475_20181112032855_1.png
-      FILENAME3 = %r!(?<illust_id>\d+)_(?<artist_id>\d+)_(?<timestamp>\d{14})_\d+!i
+      FILENAME3 = /(?<illust_id>\d+)_(?<artist_id>\d+)_(?<timestamp>\d{14})_\d+/i
 
-      IMAGE_BASE_URL = %r!\Ahttps?://(?:pic\d+\.nijie\.info|pic\.nijie\.net)!i
+      IMAGE_BASE_URL = %r{\Ahttps?://(?:pic\d+\.nijie\.info|pic\.nijie\.net)}i
-      DIR = %r!(?:\d+/)?(?:__rs_\w+/)?nijie_picture(?:/diff/main)?!
+      DIR = %r{(?:\d+/)?(?:__rs_\w+/)?nijie_picture(?:/diff/main)?}
-      IMAGE_URL = %r!#{IMAGE_BASE_URL}/#{DIR}/#{Regexp.union(FILENAME1, FILENAME2, FILENAME3)}\.\w+\z!i
+      IMAGE_URL = %r{#{IMAGE_BASE_URL}/#{DIR}/#{Regexp.union(FILENAME1, FILENAME2, FILENAME3)}\.\w+\z}i
 
       def domains
         ["nijie.info", "nijie.net"]
@@ -146,7 +146,7 @@ module Sources
       end
 
       def to_full_image_url(x)
-        x.gsub(%r!__rs_\w+/!i, "").gsub(/\Ahttp:/, "https:")
+        x.gsub(%r{__rs_\w+/}i, "").gsub(/\Ahttp:/, "https:")
       end
 
       def to_preview_url(url)
@@ -178,57 +178,21 @@ module Sources
       def page
         return nil if page_url.blank?
 
-        doc = agent.get(page_url)
+        http = Danbooru::Http.new
+        form = { email: Danbooru.config.nijie_login, password: Danbooru.config.nijie_password }
 
-        if doc.search("div#header-login-container").any?
+        # XXX `retriable` must come after `cache` so that retries don't return cached error responses.
-          # Session cache is invalid, clear it and log in normally.
+        response = http.cache(1.hour).use(retriable: { max_retries: 20 }).post("https://nijie.info/login_int.php", form: form)
-          Cache.delete("nijie-session")
+        DanbooruLogger.info "Nijie login failed (#{url}, #{response.status})" if response.status != 200
-          doc = agent.get(page_url)
+        return nil unless response.status == 200
-        end
 
-        return doc
+        response = http.cookies(R18: 1).cache(1.minute).get(page_url)
-      rescue Mechanize::ResponseCodeError => e
+        return nil unless response.status == 200
-        return nil if e.response_code.to_i == 404
+
-        raise
+        response&.parse
       end
+
       memoize :page
-
-      def agent
-        mech = Mechanize.new
-
-        session = Cache.get("nijie-session")
-        if session
-          cookie = Mechanize::Cookie.new("NIJIEIJIEID", session)
-          cookie.domain = ".nijie.info"
-          cookie.path = "/"
-          mech.cookie_jar.add(cookie)
-        else
-          mech.get("https://nijie.info/login.php") do |page|
-            page.form_with(:action => "/login_int.php") do |form|
-              form['email'] = Danbooru.config.nijie_login
-              form['password'] = Danbooru.config.nijie_password
-            end.click_button
-          end
-          session = mech.cookie_jar.cookies.select {|c| c.name == "NIJIEIJIEID"}.first
-          Cache.put("nijie-session", session.value, 1.day) if session
-        end
-
-        # This cookie needs to be set to allow viewing of adult works while anonymous
-        cookie = Mechanize::Cookie.new("R18", "1")
-        cookie.domain = ".nijie.info"
-        cookie.path = "/"
-        mech.cookie_jar.add(cookie)
-
-        mech
-      rescue Mechanize::ResponseCodeError => x
-        if x.response_code.to_i == 429
-          sleep(5)
-          retry
-        else
-          raise
-        end
-      end
-      memoize :agent
     end
   end
 end

app/logical/sources/strategies/null.rb

@@ -28,7 +28,7 @@ module Sources
         when %r{\Ahttp://p\.twpl\.jp/show/(?:large|orig)/([a-z0-9]+)}i
           "http://p.twipple.jp/#{$1}"
 
-        when %r{\Ahttps?://blog(?:(?:-imgs-)?\d*(?:-origin)?)?\.fc2\.com/(?:(?:[^/]/){3}|(?:[^/]/))([^/]+)/(?:file/)?([^\.]+\.[^\?]+)}i
+        when %r{\Ahttps?://blog(?:(?:-imgs-)?\d*(?:-origin)?)?\.fc2\.com/(?:(?:[^/]/){3}|(?:[^/]/))([^/]+)/(?:file/)?([^.]+\.[^?]+)}i
           username = $1
           filename = $2
           "http://#{username}.blog.fc2.com/img/#{filename}/"
@@ -47,7 +47,7 @@ module Sources
         when %r{\Ahttps?://c(?:s|han|[1-4])\.sankakucomplex\.com/data(?:/sample)?/(?:[a-f0-9]{2}/){2}(?:sample-|preview)?([a-f0-9]{32})}i
           "https://chan.sankakucomplex.com/en/post/show?md5=#{$1}"
 
-        when %r{\Ahttps?://(?:www|s(?:tatic|[1-4]))\.zerochan\.net/.+(?:\.|\/)(\d+)(?:\.(?:jpe?g?))?\z}i
+        when %r{\Ahttps?://(?:www|s(?:tatic|[1-4]))\.zerochan\.net/.+(?:\.|\/)(\d+)(?:\.(?:jpe?g?|png))?\z}i
           "https://www.zerochan.net/#{$1}#full"
 
         when %r{\Ahttps?://static[1-6]?\.minitokyo\.net/(?:downloads|view)/(?:\d{2}/){2}(\d+)}i
@@ -105,7 +105,7 @@ module Sources
         # http://img.toranoana.jp/popup_img18/04/0010/22/87/040010228714-1p.jpg
         # http://img.toranoana.jp/popup_blimg/04/0030/08/30/040030083068-1p.jpg
         # https://ecdnimg.toranoana.jp/ec/img/04/0030/65/34/040030653417-6p.jpg
-        when %r{\Ahttps?://(\w+\.)?toranoana\.jp/(?:popup_(?:bl)?img\d*|ec/img)/\d{2}/\d{4}/\d{2}/\d{2}/(?<work_id>\d+)}i
+        when %r{\Ahttps?://(?:\w+\.)?toranoana\.jp/(?:popup_(?:bl)?img\d*|ec/img)/\d{2}/\d{4}/\d{2}/\d{2}/(?<work_id>\d+)}i
           "https://ec.toranoana.jp/tora_r/ec/item/#{$~[:work_id]}/"
 
         # https://a.hitomi.la/galleries/907838/1.png

app/logical/sources/strategies/pawoo.rb

@@ -16,13 +16,13 @@
 
 module Sources::Strategies
   class Pawoo < Base
-    HOST = %r!\Ahttps?://(www\.)?pawoo\.net!i
+    HOST = %r{\Ahttps?://(www\.)?pawoo\.net}i
-    IMAGE = %r!\Ahttps?://img\.pawoo\.net/media_attachments/files/(\d+/\d+/\d+)!
+    IMAGE = %r{\Ahttps?://img\.pawoo\.net/media_attachments/files/(\d+/\d+/\d+)}
-    NAMED_PROFILE = %r!#{HOST}/@(?<artist_name>\w+)!i
+    NAMED_PROFILE = %r{#{HOST}/@(?<artist_name>\w+)}i
-    ID_PROFILE = %r!#{HOST}/web/accounts/(?<artist_id>\d+)!
+    ID_PROFILE = %r{#{HOST}/web/accounts/(?<artist_id>\d+)}
 
-    STATUS1 = %r!\A#{HOST}/web/statuses/(?<status_id>\d+)!
+    STATUS1 = %r{\A#{HOST}/web/statuses/(?<status_id>\d+)}
-    STATUS2 = %r!\A#{NAMED_PROFILE}/(?<status_id>\d+)!
+    STATUS2 = %r{\A#{NAMED_PROFILE}/(?<status_id>\d+)}
 
     def domains
       ["pawoo.net"]
@@ -37,15 +37,13 @@ module Sources::Strategies
     end
 
     def image_urls
-      if url =~ %r!#{IMAGE}/small/([a-z0-9]+\.\w+)\z!i
+      if url =~ %r{#{IMAGE}/small/([a-z0-9]+\.\w+)\z}i
-        return ["https://img.pawoo.net/media_attachments/files/#{$1}/original/#{$2}"]
+        ["https://img.pawoo.net/media_attachments/files/#{$1}/original/#{$2}"]
+      elsif url =~ %r{#{IMAGE}/original/([a-z0-9]+\.\w+)\z}i
+        [url]
+      else
+        api_response.image_urls
       end
-
-      if url =~ %r!#{IMAGE}/original/([a-z0-9]+\.\w+)\z!i
-        return [url]
-      end
-
-      return api_response.image_urls
     end
 
     def page_url
@@ -55,16 +53,17 @@ module Sources::Strategies
         end
       end
 
-      return super
+      super
     end
 
     def profile_url
       if url =~ PawooApiClient::PROFILE2
-        return "https://pawoo.net/@#{$1}"
+        "https://pawoo.net/@#{$1}"
+      elsif api_response.profile_url.blank?
+        url
+      else
+        api_response.profile_url
       end
-
-      return url if api_response.profile_url.blank?
-      api_response.profile_url
     end
 
     def artist_name
@@ -87,10 +86,6 @@ module Sources::Strategies
       urls.map { |url| url[STATUS1, :status_id] || url[STATUS2, :status_id] }.compact.first
     end
 
-    def artist_commentary_title
-      nil
-    end
-
     def artist_commentary_desc
       api_response.commentary
     end
@@ -99,18 +94,10 @@ module Sources::Strategies
       api_response.tags
     end
 
-    def normalizable_for_artist_finder?
-      true
-    end
-
-    def normalize_for_artist_finder
-      profile_url
-    end
-
     def normalize_for_source
       artist_name = artist_name_from_url
       status_id = status_id_from_url
-      return unless status_id.present?
+      return if status_id.blank?
 
       if artist_name.present?
         "https://pawoo.net/@#{artist_name}/#{status_id}"
@@ -131,7 +118,7 @@ module Sources::Strategies
 
     def api_response
       [url, referer_url].each do |x|
-        if client = PawooApiClient.new.get(x)
+        if (client = PawooApiClient.new.get(x))
           return client
         end
       end

app/logical/sources/strategies/pixiv.rb

@@ -50,37 +50,34 @@
 module Sources
   module Strategies
     class Pixiv < Base
-      MONIKER = %r!(?:[a-zA-Z0-9_-]+)!
+      MONIKER = /(?:[a-zA-Z0-9_-]+)/
-      PROFILE = %r!\Ahttps?://www\.pixiv\.net/member\.php\?id=[0-9]+\z!
+      PROFILE = %r{\Ahttps?://www\.pixiv\.net/member\.php\?id=[0-9]+\z}
-      DATE =    %r!(?<date>\d{4}/\d{2}/\d{2}/\d{2}/\d{2}/\d{2})!i
+      DATE =    %r{(?<date>\d{4}/\d{2}/\d{2}/\d{2}/\d{2}/\d{2})}i
-      EXT =     %r!(?:jpg|jpeg|png|gif)!i
+      EXT =     /(?:jpg|jpeg|png|gif)/i
 
-      WEB =     %r!(?:\A(?:https?://)?www\.pixiv\.net)!
+      WEB =     %r{(?:\A(?:https?://)?www\.pixiv\.net)}
-      I12 =     %r!(?:\A(?:https?://)?i[0-9]+\.pixiv\.net)!
+      I12 =     %r{(?:\A(?:https?://)?i[0-9]+\.pixiv\.net)}
-      IMG =     %r!(?:\A(?:https?://)?img[0-9]*\.pixiv\.net)!
+      IMG =     %r{(?:\A(?:https?://)?img[0-9]*\.pixiv\.net)}
-      PXIMG =   %r!(?:\A(?:https?://)?[^.]+\.pximg\.net)!
+      PXIMG =   %r{(?:\A(?:https?://)?[^.]+\.pximg\.net)}
-      TOUCH =   %r!(?:\A(?:https?://)?touch\.pixiv\.net)!
+      TOUCH =   %r{(?:\A(?:https?://)?touch\.pixiv\.net)}
-      UGOIRA =  %r!#{PXIMG}/img-zip-ugoira/img/#{DATE}/(?<illust_id>\d+)_ugoira1920x1080\.zip\z!i
+      UGOIRA =  %r{#{PXIMG}/img-zip-ugoira/img/#{DATE}/(?<illust_id>\d+)_ugoira1920x1080\.zip\z}i
-      ORIG_IMAGE = %r!#{PXIMG}/img-original/img/#{DATE}/(?<illust_id>\d+)_p(?<page>\d+)\.#{EXT}\z!i
+      ORIG_IMAGE = %r{#{PXIMG}/img-original/img/#{DATE}/(?<illust_id>\d+)_p(?<page>\d+)\.#{EXT}\z}i
-      STACC_PAGE = %r!\A#{WEB}/stacc/#{MONIKER}/?\z!i
+      STACC_PAGE = %r{\A#{WEB}/stacc/#{MONIKER}/?\z}i
-      NOVEL_PAGE = %r!(?:\Ahttps?://www\.pixiv\.net/novel/show\.php\?id=(\d+))!
+      NOVEL_PAGE = %r{(?:\Ahttps?://www\.pixiv\.net/novel/show\.php\?id=(\d+))}
-      FANBOX_ACCOUNT = %r!(?:\Ahttps?://www\.pixiv\.net/fanbox/creator/\d+\z)!
-      FANBOX_IMAGE = %r!(?:\Ahttps?://fanbox\.pixiv\.net/images/post/(\d+))!
-      FANBOX_PAGE = %r!(?:\Ahttps?://www\.pixiv\.net/fanbox/creator/\d+/post/(\d+))!
 
       def self.to_dtext(text)
         if text.nil?
           return nil
         end
 
-        text = text.gsub(%r!https?://www\.pixiv\.net/member_illust\.php\?mode=medium&illust_id=([0-9]+)!i) do |match|
+        text = text.gsub(%r{https?://www\.pixiv\.net/member_illust\.php\?mode=medium&illust_id=([0-9]+)}i) do |_match|
           pixiv_id = $1
           %(pixiv ##{pixiv_id} "»":[/posts?tags=pixiv:#{pixiv_id}])
         end
 
-        text = text.gsub(%r!https?://www\.pixiv\.net/member\.php\?id=([0-9]+)!i) do |match|
+        text = text.gsub(%r{https?://www\.pixiv\.net/member\.php\?id=([0-9]+)}i) do |_match|
           member_id = $1
-          profile_url = "https://www.pixiv.net/member.php?id=#{member_id}"
+          profile_url = "https://www.pixiv.net/users/#{member_id}"
           search_params = {"search[url_matches]" => profile_url}.to_param
 
           %("user/#{member_id}":[#{profile_url}] "»":[/artists?#{search_params}])
@@ -127,25 +124,17 @@ module Sources
           return "https://www.pixiv.net/novel/show.php?id=#{novel_id}&mode=cover"
         end
 
-        if fanbox_id.present?
-          return "https://www.pixiv.net/fanbox/creator/#{metadata.user_id}/post/#{fanbox_id}"
-        end
-
-        if fanbox_account_id.present?
-          return "https://www.pixiv.net/fanbox/creator/#{fanbox_account_id}"
-        end
-
         if illust_id.present?
           return "https://www.pixiv.net/artworks/#{illust_id}"
         end
 
-        return url
+        url
       rescue PixivApiClient::BadIDError
         nil
       end
 
       def canonical_url
-        return image_url
+        image_url
       end
 
       def profile_url
@@ -155,7 +144,7 @@ module Sources
           end
         end
 
-        "https://www.pixiv.net/member.php?id=#{metadata.user_id}"
+        "https://www.pixiv.net/users/#{metadata.user_id}"
       rescue PixivApiClient::BadIDError
         nil
       end
@@ -192,17 +181,7 @@ module Sources
       end
 
       def headers
-        if fanbox_id.present?
+        { "Referer" => "https://www.pixiv.net" }
-          # need the session to download fanbox images
-          return {
-            "Referer" => "https://www.pixiv.net/fanbox",
-            "Cookie" => HTTP::Cookie.cookie_value(agent.cookies)
-          }
-        end
-
-        return {
-          "Referer" => "https://www.pixiv.net"
-        }
       end
 
       def normalize_for_source
@@ -231,7 +210,7 @@ module Sources
         translated_tags = super(tag)
 
         if translated_tags.empty? && tag.include?("/")
-          translated_tags = tag.split("/").flat_map { |tag| super(tag) }
+          translated_tags = tag.split("/").flat_map { |translated_tag| super(translated_tag) }
         end
 
         translated_tags
@@ -242,10 +221,6 @@ module Sources
       end
 
       def image_urls_sub
-        if url =~ FANBOX_IMAGE
-          return [url]
-        end
-
         # there's too much normalization bullshit we have to deal with
         # raw urls, so just fetch the canonical url from the api every
         # time.
@@ -257,7 +232,7 @@ module Sources
           return [ugoira_zip_url]
         end
 
-        return metadata.pages
+        metadata.pages
       end
 
       # in order to prevent recursive loops, this method should not make any
@@ -265,7 +240,7 @@ module Sources
       # even though it makes sense to reference page_url here, it will only look
       # at (url, referer_url).
       def illust_id
-        return nil if novel_id.present? || fanbox_id.present?
+        return nil if novel_id.present?
 
         parsed_urls.each do |url|
           # http://www.pixiv.net/member_illust.php?mode=medium&illust_id=18557054
@@ -276,11 +251,11 @@ module Sources
             return url.query_values["illust_id"].to_i
 
           # http://www.pixiv.net/en/artworks/46324488
-          elsif url.host == "www.pixiv.net" && url.path =~ %r!\A/(?:en/)?artworks/(?<illust_id>\d+)!i
+          elsif url.host == "www.pixiv.net" && url.path =~ %r{\A/(?:en/)?artworks/(?<illust_id>\d+)}i
             return $~[:illust_id].to_i
 
           # http://www.pixiv.net/i/18557054
-          elsif url.host == "www.pixiv.net" && url.path =~ %r!\A/i/(?<illust_id>\d+)\z!i
+          elsif url.host == "www.pixiv.net" && url.path =~ %r{\A/i/(?<illust_id>\d+)\z}i
             return $~[:illust_id].to_i
 
           # http://img18.pixiv.net/img/evazion/14901720.png
@@ -289,8 +264,8 @@ module Sources
           # http://i2.pixiv.net/img18/img/evazion/14901720_s.png
           # http://i1.pixiv.net/img07/img/pasirism/18557054_p1.png
           # http://i1.pixiv.net/img07/img/pasirism/18557054_big_p1.png
-          elsif url.host =~ %r!\A(?:i\d+|img\d+)\.pixiv\.net\z!i &&
+          elsif url.host =~ /\A(?:i\d+|img\d+)\.pixiv\.net\z/i &&
-                url.path =~ %r!\A(?:/img\d+)?/img/#{MONIKER}/(?<illust_id>\d+)(?:_\w+)?\.(?:jpg|jpeg|png|gif|zip)!i
+              url.path =~ %r{\A(?:/img\d+)?/img/#{MONIKER}/(?<illust_id>\d+)(?:_\w+)?\.(?:jpg|jpeg|png|gif|zip)}i
             return $~[:illust_id].to_i
 
           # http://i1.pixiv.net/img-inf/img/2011/05/01/23/28/04/18557054_64x64.jpg
@@ -307,13 +282,13 @@ module Sources
           #
           # https://i.pximg.net/novel-cover-original/img/2019/01/14/01/15/05/10617324_d84daae89092d96bbe66efafec136e42.jpg
           # https://img-sketch.pixiv.net/uploads/medium/file/4463372/8906921629213362989.jpg
-          elsif url.host =~ %r!\A(?:[^.]+\.pximg\.net|i\d+\.pixiv\.net|tc-pximg01\.techorus-cdn\.com)\z!i &&
+          elsif url.host =~ /\A(?:[^.]+\.pximg\.net|i\d+\.pixiv\.net|tc-pximg01\.techorus-cdn\.com)\z/i &&
-                url.path =~ %r!\A(/c/\w+)?/img-[a-z-]+/img/#{DATE}/(?<illust_id>\d+)(?:_\w+)?\.(?:jpg|jpeg|png|gif|zip)!i
+              url.path =~ %r{\A(/c/\w+)?/img-[a-z-]+/img/#{DATE}/(?<illust_id>\d+)(?:_\w+)?\.(?:jpg|jpeg|png|gif|zip)}i
             return $~[:illust_id].to_i
           end
         end
 
-        return nil
+        nil
       end
       memoize :illust_id
 
@@ -324,89 +299,48 @@ module Sources
           end
         end
 
-        return nil
+        nil
       end
       memoize :novel_id
 
-      def fanbox_id
-        [url, referer_url].each do |x|
-          if x =~ FANBOX_PAGE
-            return $1
-          end
-
-          if x =~ FANBOX_IMAGE
-            return $1
-          end
-        end
-
-        return nil
-      end
-      memoize :fanbox_id
-
-      def fanbox_account_id
-        [url, referer_url].each do |x|
-          if x =~ FANBOX_ACCOUNT
-            return x
-          end
-        end
-
-        return nil
-      end
-      memoize :fanbox_account_id
-
-      def agent
-        PixivWebAgent.build
-      end
-      memoize :agent
-
       def metadata
         if novel_id.present?
           return PixivApiClient.new.novel(novel_id)
         end
 
-        if fanbox_id.present?
+        PixivApiClient.new.work(illust_id)
-          return PixivApiClient.new.fanbox(fanbox_id)
-        end
-
-        return PixivApiClient.new.work(illust_id)
       end
       memoize :metadata
 
       def moniker
         # we can sometimes get the moniker from the url
-        if url =~ %r!#{IMG}/img/(#{MONIKER})!i
+        if url =~ %r{#{IMG}/img/(#{MONIKER})}i
-          return $1
+          $1
+        elsif url =~ %r{#{I12}/img[0-9]+/img/(#{MONIKER})}i
+          $1
+        elsif url =~ %r{#{WEB}/stacc/(#{MONIKER})/?$}i
+          $1
+        else
+          metadata.moniker
         end
-
-        if url =~ %r!#{I12}/img[0-9]+/img/(#{MONIKER})!i
-          return $1
-        end
-
-        if url =~ %r!#{WEB}/stacc/(#{MONIKER})/?$!i
-          return $1
-        end
-
-        return metadata.moniker
       rescue PixivApiClient::BadIDError
         nil
       end
       memoize :moniker
 
       def data
-        return {
+        { ugoira_frame_data: ugoira_frame_data }
-          ugoira_frame_data: ugoira_frame_data
-        }
       end
 
       def ugoira_zip_url
         if metadata.pages.is_a?(Hash) && metadata.pages["ugoira600x600"]
-          return metadata.pages["ugoira600x600"].sub("_ugoira600x600.zip", "_ugoira1920x1080.zip")
+          metadata.pages["ugoira600x600"].sub("_ugoira600x600.zip", "_ugoira1920x1080.zip")
         end
       end
       memoize :ugoira_zip_url
 
       def ugoira_frame_data
-        return metadata.json.dig("metadata", "frames")
+        metadata.json.dig("metadata", "frames")
       rescue PixivApiClient::BadIDError
         nil
       end
@@ -415,16 +349,14 @@ module Sources
       def ugoira_content_type
         case metadata.json["image_urls"].to_s
         when /\.jpg/
-          return "image/jpeg"
+          "image/jpeg"
-
         when /\.png/
-          return "image/png"
+          "image/png"
-
         when /\.gif/
-          return "image/gif"
+          "image/gif"
+        else
+          raise Sources::Error, "content type not found for (#{url}, #{referer_url})"
         end
-
-        raise Sources::Error.new("content type not found for (#{url}, #{referer_url})")
       end
       memoize :ugoira_content_type
 
@@ -434,7 +366,7 @@ module Sources
         # http://i2.pixiv.net/img04/img/syounen_no_uta/46170939_p0.jpg
         # http://i1.pixiv.net/c/600x600/img-master/img/2014/09/24/23/25/08/46168376_p0_master1200.jpg
         # http://i1.pixiv.net/img-original/img/2014/09/25/23/09/29/46183440_p0.jpg
-        if url =~ %r!/\d+_p(\d+)(?:_\w+)?\.#{EXT}!i
+        if url =~ %r{/\d+_p(\d+)(?:_\w+)?\.#{EXT}}i
           return $1.to_i
         end
 
@@ -445,7 +377,7 @@ module Sources
           end
         end
 
-        return nil
+        nil
       end
       memoize :manga_page
     end

app/logical/sources/strategies/tumblr.rb

@@ -12,19 +12,19 @@ module Sources::Strategies
   class Tumblr < Base
     SIZES = %w[1280 640 540 500h 500 400 250 100]
 
-    BASE_URL = %r!\Ahttps?://(?:[^/]+\.)*tumblr\.com!i
+    BASE_URL = %r{\Ahttps?://(?:[^/]+\.)*tumblr\.com}i
-    DOMAIN = %r{(data|(\d+\.)?media)\.tumblr\.com}
+    DOMAIN = /(data|(?:\d+\.)?media)\.tumblr\.com/i
-    MD5 = %r{(?<md5>[0-9a-f]{32})}i
+    MD5 = /(?<md5>[0-9a-f]{32})/i
-    FILENAME = %r{(?<filename>(tumblr_(inline_)?)?[a-z0-9]+(_r[0-9]+)?)}i
+    FILENAME = /(?<filename>(?:tumblr_(?:inline_)?)?[a-z0-9]+(?:_r[0-9]+)?)/i
-    EXT = %r{(?<ext>\w+)}
+    EXT = /(?<ext>\w+)/
 
     # old: https://66.media.tumblr.com/2c6f55531618b4335c67e29157f5c1fc/tumblr_pz4a44xdVj1ssucdno1_1280.png
     # new: https://66.media.tumblr.com/168dabd09d5ad69eb5fedcf94c45c31a/3dbfaec9b9e0c2e3-72/s640x960/bf33a1324f3f36d2dc64f011bfeab4867da62bc8.png
-    OLD_IMAGE = %r!\Ahttps?://#{DOMAIN}/(?<dir>#{MD5}/)?#{FILENAME}_(?<size>\w+)\.#{EXT}\z!i
+    OLD_IMAGE = %r{\Ahttps?://#{DOMAIN}/(?<dir>#{MD5}/)?#{FILENAME}_(?<size>\w+)\.#{EXT}\z}i
 
-    IMAGE = %r!\Ahttps?://#{DOMAIN}/!i
+    IMAGE = %r{\Ahttps?://#{DOMAIN}/}i
-    VIDEO = %r!\Ahttps?://(?:vtt|ve\.media)\.tumblr\.com/!i
+    VIDEO = %r{\Ahttps?://(?:vtt|ve|va\.media)\.tumblr\.com/}i
-    POST = %r!\Ahttps?://(?<blog_name>[^.]+)\.tumblr\.com/(?:post|image)/(?<post_id>\d+)!i
+    POST = %r{\Ahttps?://(?<blog_name>[^.]+)\.tumblr\.com/(?:post|image)/(?<post_id>\d+)}i
 
     def self.enabled?
       Danbooru.config.tumblr_consumer_key.present?
@@ -68,7 +68,7 @@ module Sources::Strategies
 
     def preview_urls
       image_urls.map do |x|
-        x.sub(%r!_1280\.(jpg|png|gif|jpeg)\z!, '_250.\1')
+        x.sub(/_1280\.(jpg|png|gif|jpeg)\z/, '_250.\1')
       end
     end
 
@@ -168,7 +168,7 @@ module Sources::Strategies
       end
 
       candidates.find do |candidate|
-        http_exists?(candidate, headers)
+        http_exists?(candidate)
       end
     end
 

app/logical/sources/strategies/twitter.rb

@@ -1,20 +1,20 @@
 module Sources::Strategies
   class Twitter < Base
-    PAGE = %r!\Ahttps?://(?:mobile\.)?twitter\.com!i
+    PAGE = %r{\Ahttps?://(?:mobile\.)?twitter\.com}i
-    PROFILE = %r!\Ahttps?://(?:mobile\.)?twitter.com/(?<username>[a-z0-9_]+)!i
+    PROFILE = %r{\Ahttps?://(?:mobile\.)?twitter.com/(?<username>[a-z0-9_]+)}i
 
     # https://pbs.twimg.com/media/EBGbJe_U8AA4Ekb.jpg
     # https://pbs.twimg.com/media/EBGbJe_U8AA4Ekb?format=jpg&name=900x900
     # https://pbs.twimg.com/tweet_video_thumb/ETkN_L3X0AMy1aT.jpg
     # https://pbs.twimg.com/ext_tw_video_thumb/1243725361986375680/pu/img/JDA7g7lcw7wK-PIv.jpg
     # https://pbs.twimg.com/amplify_video_thumb/1215590775364259840/img/lolCkEEioFZTb5dl.jpg
-    BASE_IMAGE_URL = %r!\Ahttps?://pbs\.twimg\.com/(?<media_type>media|tweet_video_thumb|ext_tw_video_thumb|amplify_video_thumb)!i
+    BASE_IMAGE_URL = %r{\Ahttps?://pbs\.twimg\.com/(?<media_type>media|tweet_video_thumb|ext_tw_video_thumb|amplify_video_thumb)}i
-    FILENAME1 = %r!(?<file_name>[a-zA-Z0-9_-]+)\.(?<file_ext>\w+)!i
+    FILENAME1 = /(?<file_name>[a-zA-Z0-9_-]+)\.(?<file_ext>\w+)/i
-    FILENAME2 = %r!(?<file_name>[a-zA-Z0-9_-]+)\?.*format=(?<file_ext>\w+)!i
+    FILENAME2 = /(?<file_name>[a-zA-Z0-9_-]+)\?.*format=(?<file_ext>\w+)/i
-    FILEPATH1 = %r!(?<file_path>\d+/[\w_-]+/img)!i
+    FILEPATH1 = %r{(?<file_path>\d+/[\w_-]+/img)}i
-    FILEPATH2 = %r!(?<file_path>\d+/img)!i
+    FILEPATH2 = %r{(?<file_path>\d+/img)}i
-    IMAGE_URL1 = %r!#{BASE_IMAGE_URL}/#{Regexp.union(FILENAME1, FILENAME2)}!i
+    IMAGE_URL1 = %r{#{BASE_IMAGE_URL}/#{Regexp.union(FILENAME1, FILENAME2)}}i
-    IMAGE_URL2 = %r!#{BASE_IMAGE_URL}/#{Regexp.union(FILEPATH1, FILEPATH2)}/#{FILENAME1}!i
+    IMAGE_URL2 = %r{#{BASE_IMAGE_URL}/#{Regexp.union(FILEPATH1, FILEPATH2)}/#{FILENAME1}}i
 
     # Twitter provides a list but it's inaccurate; some names ('intent') aren't
     # included and other names in the list aren't actually reserved.
@@ -47,7 +47,7 @@ module Sources::Strategies
         return $1
       end
 
-      return nil
+      nil
     end
 
     def self.artist_name_from_url(url)
@@ -78,7 +78,7 @@ module Sources::Strategies
           elsif media[:type].in?(["video", "animated_gif"])
             variants = media.dig(:video_info, :variants)
             videos = variants.select { |variant| variant[:content_type] == "video/mp4" }
-            video = videos.max_by { |video| video[:bitrate].to_i }
+            video = videos.max_by { |v| v[:bitrate].to_i }
             video[:url]
           end
         end
@@ -137,10 +137,6 @@ module Sources::Strategies
       api_response[:full_text].to_s
     end
 
-    def normalizable_for_artist_finder?
-      url =~ PAGE
-    end
-
     def normalize_for_artist_finder
       profile_url.try(:downcase).presence || url
     end
@@ -193,9 +189,9 @@ module Sources::Strategies
 
       desc = artist_commentary_desc.unicode_normalize(:nfkc)
       desc = CGI.unescapeHTML(desc)
-      desc = desc.gsub(%r!https?://t\.co/[a-zA-Z0-9]+!i, url_replacements)
+      desc = desc.gsub(%r{https?://t\.co/[a-zA-Z0-9]+}i, url_replacements)
-      desc = desc.gsub(%r!#([^[:space:]]+)!, '"#\\1":[https://twitter.com/hashtag/\\1]')
+      desc = desc.gsub(/#([^[:space:]]+)/, '"#\\1":[https://twitter.com/hashtag/\\1]')
-      desc = desc.gsub(%r!@([a-zA-Z0-9_]+)!, '"@\\1":[https://twitter.com/\\1]')
+      desc = desc.gsub(/@([a-zA-Z0-9_]+)/, '"@\\1":[https://twitter.com/\\1]')
       desc.strip
     end
 
@@ -204,7 +200,7 @@ module Sources::Strategies
     end
 
     def api_response
-      return {} if !self.class.enabled?
+      return {} unless self.class.enabled? && status_id.present?
       api_client.status(status_id)
     end
 

app/logical/sources/strategies/weibo.rb

@@ -38,7 +38,7 @@ module Sources
 
       PAGE_URL_1    = %r{\Ahttps?://(?:www\.)?weibo\.com/(?<artist_short_id>\d+)/(?<illust_base62_id>\w+)(?:\?.*)?\z}i
       PAGE_URL_2    = %r{#{PROFILE_URL_2}/(?:wbphotos/large/mid|talbum/detail/photo_id)/(?<illust_long_id>\d+)(?:/pid/(?<image_id>\w{32}))?}i
-      PAGE_URL_3    = %r{\Ahttps?://m\.weibo\.cn/(detail/(?<illust_long_id>\d+)|status/(?<illust_base62_id>\w+))}i
+      PAGE_URL_3    = %r{\Ahttps?://m\.weibo\.cn/(?:detail/(?<illust_long_id>\d+)|status/(?<illust_base62_id>\w+))}i
       PAGE_URL_4    = %r{\Ahttps?://tw\.weibo\.com/(?:(?<artist_short_id>\d+)|\w+)/(?<illust_long_id>\d+)}i
 
       IMAGE_URL     = %r{\Ahttps?://\w{3}\.sinaimg\.cn/\w+/(?<image_id>\w{32})\.}i
@@ -203,12 +203,12 @@ module Sources
       end
 
       def api_response
-        return nil if mobile_url.blank?
+        return {} if mobile_url.blank?
 
         resp = Danbooru::Http.cache(1.minute).get(mobile_url)
         json_string = resp.to_s[/var \$render_data = \[(.*)\]\[0\]/m, 1]
 
-        return nil if json_string.blank?
+        return {} if json_string.blank?
 
         JSON.parse(json_string)["status"]
       end

app/logical/spam_detector.rb

@@ -7,10 +7,11 @@ class SpamDetector
   # if a person receives more than 10 automatic spam reports within a 1 hour
   # window, automatically ban them forever.
   AUTOBAN_THRESHOLD = 10
-  AUTOBAN_WINDOW = 1.hours
+  AUTOBAN_WINDOW = 1.hour
-  AUTOBAN_DURATION = 999999
+  AUTOBAN_DURATION = 999_999
 
   attr_accessor :record, :user, :user_ip, :content, :comment_type
+
   rakismet_attrs author: proc { user.name },
                  author_email: proc { user.email_address&.address },
                  blog_lang: "en",
@@ -84,8 +85,8 @@ class SpamDetector
     end
 
     is_spam
-  rescue StandardError => exception
+  rescue StandardError => e
-    DanbooruLogger.log(exception)
+    DanbooruLogger.log(e)
     false
   end
 end