The artist requested removal of this page.
<% else %> - <% if @artist.wiki_page.present? %> -<%= link_to "View wiki page", @artist.wiki_page %>
+<%= link_to "View wiki page", @artist.wiki_page %>
+<%= link_to "View artist", @wiki_page.artist %>
<% end %> diff --git a/config/cable.yml b/config/cable.yml deleted file mode 100644 index feea78f65..000000000 --- a/config/cable.yml +++ /dev/null @@ -1,10 +0,0 @@ -development: - adapter: async - -test: - adapter: test - -production: - adapter: redis - url: <%= ENV.fetch("REDIS_URL") { "redis://localhost:6379/1" } %> - channel_prefix: danbooru_production diff --git a/config/danbooru_default_config.rb b/config/danbooru_default_config.rb index c3f22f050..f9570949c 100644 --- a/config/danbooru_default_config.rb +++ b/config/danbooru_default_config.rb @@ -317,13 +317,15 @@ module Danbooru # A list of tags that should be removed when a post is replaced. Regexes allowed. def post_replacement_tag_removals %w[replaceme .*_sample resized upscaled downscaled md5_mismatch - jpeg_artifacts corrupted_image source_request non-web_source] + jpeg_artifacts corrupted_image missing_image missing_sample missing_thumbnail + resolution_mismatch source_larger source_smaller source_request non-web_source] end # Posts with these tags will be highlighted in the modqueue. def modqueue_warning_tags %w[hard_translated self_upload nude_filter third-party_edit screencap - duplicate image_sample md5_mismatch resized upscaled downscaled] + duplicate image_sample md5_mismatch resized upscaled downscaled + resolution_mismatch source_larger source_smaller] end def stripe_secret_key @@ -338,22 +340,6 @@ module Danbooru def twitter_api_secret end - # The default headers to be sent with outgoing http requests. Some external - # services will fail if you don't set a valid User-Agent. - def http_headers - { - "User-Agent" => "#{Danbooru.config.canonical_app_name}/#{Rails.application.config.x.git_hash}" - } - end - - def httparty_options - # proxy example: - # {http_proxyaddr: "", http_proxyport: "", http_proxyuser: nil, http_proxypass: nil} - { - headers: Danbooru.config.http_headers - } - end - # you should override this def email_key "zDMSATq0W3hmA5p3rKTgD" @@ -374,14 +360,20 @@ module Danbooru false end - # reportbooru options - see https://github.com/r888888888/reportbooru + # The URL for the Reportbooru server (https://github.com/evazion/reportbooru). + # Optional. Used for tracking post views, popular searches, and missed searches. + # Set to http://localhost/mock/reportbooru to enable a fake reportbooru + # server for development purposes. def reportbooru_server end def reportbooru_key end - # iqdbs options - see https://github.com/r888888888/iqdbs + # The URL for the IQDBs server (https://github.com/evazion/iqdbs). + # Optional. Used for dupe detection and reverse image searches. + # Set to http://localhost/mock/iqdbs to enable a fake iqdb server for + # development purposes. def iqdbs_server end @@ -459,6 +451,10 @@ module Danbooru def cloudflare_zone end + # The URL for the recommender server (https://github.com/evazion/recommender). + # Optional. Used to generate post recommendations. + # Set to http://localhost/mock/recommender to enable a fake recommender + # server for development purposes. def recommender_server end diff --git a/config/docker/Dockerfile.danbooru b/config/docker/Dockerfile.danbooru index df2af68bd..013644a89 100644 --- a/config/docker/Dockerfile.danbooru +++ b/config/docker/Dockerfile.danbooru @@ -13,14 +13,20 @@ RUN \ webpack \ libvips-dev \ libxml2-dev \ + libxslt-dev \ + zlib1g-dev \ postgresql-server-dev-all && \ # webpacker expects the binary to be called `yarn`, but debian/ubuntu installs it as `yarnpkg`. ln -sf /usr/bin/yarnpkg /usr/bin/yarn WORKDIR /build +COPY .bundle .bundle COPY Gemfile Gemfile.lock ./ -RUN BUNDLE_DEPLOYMENT=true bundle install --jobs 4 +RUN \ + bundle config set deployment true --local && \ + bundle config set path vendor/bundle && \ + bundle install --jobs 4 COPY package.json yarn.lock ./ RUN yarn install @@ -44,6 +50,8 @@ RUN \ mkvtoolnix \ libvips \ libxml2 \ + libxslt1.1 \ + zlib1g \ postgresql-client USER danbooru diff --git a/config/initializers/mechanize_patch.rb b/config/initializers/mechanize_patch.rb deleted file mode 100644 index 875c28b9d..000000000 --- a/config/initializers/mechanize_patch.rb +++ /dev/null @@ -1,64 +0,0 @@ -require 'mechanize' - -if Rails.env.test? - # something about the root certs on the travis ci image causes Mechanize - # to intermittently fail. this is a monkey patch to reset the connection - # after every request to avoid dealing wtiht he issue. - # - # from http://scottwb.com/blog/2013/11/09/defeating-the-infamous-mechanize-too-many-connection-resets-bug/ - class Mechanize::HTTP::Agent - MAX_RESET_RETRIES = 10 - - # We need to replace the core Mechanize HTTP method: - # - # Mechanize::HTTP::Agent#fetch - # - # with a wrapper that handles the infamous "too many connection resets" - # Mechanize bug that is described here: - # - # https://github.com/sparklemotion/mechanize/issues/123 - # - # The wrapper shuts down the persistent HTTP connection when it fails with - # this error, and simply tries again. In practice, this only ever needs to - # be retried once, but I am going to let it retry a few times - # (MAX_RESET_RETRIES), just in case. - # - def fetch_with_retry( - uri, - method = :get, - headers = {}, - params = [], - referer = current_page, - redirects = 0 - ) - action = "#{method.to_s.upcase} #{uri}" - retry_count = 0 - - begin - fetch_without_retry(uri, method, headers, params, referer, redirects) - rescue Net::HTTP::Persistent::Error => e - # Pass on any other type of error. - raise unless e.message =~ /too many connection resets/ - - # Pass on the error if we've tried too many times. - if retry_count >= MAX_RESET_RETRIES - print "R" - # puts "**** WARN: Mechanize retried connection reset #{MAX_RESET_RETRIES} times and never succeeded: #{action}" - raise - end - - # Otherwise, shutdown the persistent HTTP connection and try again. - print "R" - # puts "**** WARN: Mechanize retrying connection reset error: #{action}" - retry_count += 1 - self.http.shutdown - retry - end - end - - # Alias so #fetch actually uses our new #fetch_with_retry to wrap the - # old one aliased as #fetch_without_retry. - alias fetch_without_retry fetch - alias fetch fetch_with_retry - end -end diff --git a/config/routes.rb b/config/routes.rb index 95392b1e1..9b8ff2e6e 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -372,6 +372,14 @@ Rails.application.routes.draw do get "/static/contact" => "static#contact", :as => "contact" get "/static/dtext_help" => "static#dtext_help", :as => "dtext_help" + get "/mock/recommender/recommend/:user_id" => "mock_services#recommender_recommend", as: "mock_recommender_recommend" + get "/mock/recommender/similiar/:post_id" => "mock_services#recommender_similar", as: "mock_recommender_similar" + get "/mock/reportbooru/missed_searches" => "mock_services#reportbooru_missed_searches", as: "mock_reportbooru_missed_searches" + get "/mock/reportbooru/post_searches/rank" => "mock_services#reportbooru_post_searches", as: "mock_reportbooru_post_searches" + get "/mock/reportbooru/post_views/rank" => "mock_services#reportbooru_post_views", as: "mock_reportbooru_post_views" + get "/mock/iqdbs/similar" => "mock_services#iqdbs_similar", as: "mock_iqdbs_similar" + post "/mock/iqdbs/similar" => "mock_services#iqdbs_similar" + root :to => "posts#index" get "*other", :to => "static#not_found" diff --git a/config/secrets.yml b/config/secrets.yml deleted file mode 100644 index a3d89cfec..000000000 --- a/config/secrets.yml +++ /dev/null @@ -1,26 +0,0 @@ -# Be sure to restart your server when you modify this file. - -# Your secret key is used for verifying the integrity of signed cookies. -# If you change this key, all old signed cookies will become invalid! - -# Make sure the secret is at least 30 characters and all random, -# no regular words or you'll be exposed to dictionary attacks. -# You can use `rake secret` to generate a secure secret key. - -# Make sure the secrets in this file are kept private -# if you're sharing your code publicly. - -development: - secret_key_base: bcc62a512b9c055c292c17742f1e65bd6d88fa37f4d01c8475103809f3ac4c03e3e98605c47d55cd8801333010ea98920a61b722770629926759624bce732539 - -test: - secret_key_base: 60e32a818af77bdfc40bca866e3b4d7b88d7ba767057ffc9e4532279358af8c67d42f2b99c084b700727303ce25b812a592b52723ebc1e3b812fd09a1f969435 - -# Do not keep production secrets in the repository, -# instead read values from the environment. -production: - secret_key_base: <%= ENV["SECRET_KEY_BASE"] %> - -staging: - secret_key_base: <%= ENV["SECRET_KEY_BASE"] %> - \ No newline at end of file diff --git a/config/storage.yml b/config/storage.yml deleted file mode 100644 index d32f76e8f..000000000 --- a/config/storage.yml +++ /dev/null @@ -1,34 +0,0 @@ -test: - service: Disk - root: <%= Rails.root.join("tmp/storage") %> - -local: - service: Disk - root: <%= Rails.root.join("storage") %> - -# Use rails credentials:edit to set the AWS secrets (as aws:access_key_id|secret_access_key) -# amazon: -# service: S3 -# access_key_id: <%= Rails.application.credentials.dig(:aws, :access_key_id) %> -# secret_access_key: <%= Rails.application.credentials.dig(:aws, :secret_access_key) %> -# region: us-east-1 -# bucket: your_own_bucket - -# Remember not to checkin your GCS keyfile to a repository -# google: -# service: GCS -# project: your_project -# credentials: <%= Rails.root.join("path/to/gcs.keyfile") %> -# bucket: your_own_bucket - -# Use rails credentials:edit to set the Azure Storage secret (as azure_storage:storage_access_key) -# microsoft: -# service: AzureStorage -# storage_account_name: your_account_name -# storage_access_key: <%= Rails.application.credentials.dig(:azure_storage, :storage_access_key) %> -# container: your_container_name - -# mirror: -# service: Mirror -# primary: local -# mirrors: [ amazon, google, microsoft ] diff --git a/config/unicorn/production.rb b/config/unicorn/production.rb index e53d8a24f..7ac586706 100644 --- a/config/unicorn/production.rb +++ b/config/unicorn/production.rb @@ -6,7 +6,7 @@ worker_processes 20 timeout 180 # listen "127.0.0.1:9000", :tcp_nopush => true -listen "/tmp/.unicorn.sock", :backlog => 512 +listen "/tmp/.unicorn.sock", backlog: 1024 # Spawn unicorn master worker for user apps (group: apps) user 'danbooru', 'danbooru' diff --git a/package.json b/package.json index 28fa41a81..e715e9198 100644 --- a/package.json +++ b/package.json @@ -27,8 +27,10 @@ "webpack-cli": "^3.3.0" }, "devDependencies": { - "eslint": "^6.0.0", + "babel-eslint": "^10.1.0", + "eslint": "^7.0.0", "eslint-loader": "^4.0.0", + "eslint-plugin-babel": "^5.3.0", "eslint-plugin-ignore-erb": "^0.1.1", "stylelint": "^13.0.0", "stylelint-config-standard": "^20.0.0", diff --git a/script/install/app_server.sh b/script/install/app_server.sh deleted file mode 100644 index a5a14d8ac..000000000 --- a/script/install/app_server.sh +++ /dev/null @@ -1,101 +0,0 @@ -#!/bin/bash - -# this is a version of the install script designed to be run on -# app servers (that is, they won't install PostgreSQL server). -# -# Run: curl -L -s https://raw.githubusercontent.com/r888888888/danbooru/master/script/install/app_server.sh | sh - -export RUBY_VERSION=2.6.3 -export GITHUB_INSTALL_SCRIPTS=https://raw.githubusercontent.com/r888888888/danbooru/master/script/install -export VIPS_VERSION=8.7.0 - -if [[ "$(whoami)" != "root" ]] ; then - echo "You must run this script as root" - exit 1 -fi - -echo "* DANBOORU INSTALLATION SCRIPT" -echo "*" -echo "* This script will install all the necessary packages to run Danbooru on an" -echo "* Ubuntu server." -echo - -echo -n "* Enter the VLAN IP address for this server: " -read VLAN_IP_ADDR - -# Install packages -echo "* Installing packages..." - -apt-get update -apt-get -y install libssl-dev build-essential automake libxml2-dev libxslt-dev ncurses-dev sudo libreadline-dev flex bison ragel redis git curl libcurl4-openssl-dev sendmail-bin sendmail nginx ssh coreutils ffmpeg mkvtoolnix -apt-get -y install libpq-dev postgresql-client -apt-get -y install liblcms2-dev libjpeg-turbo8-dev libexpat1-dev libgif-dev libpng-dev libexif-dev - -# vrack specific stuff -apt-get -y install vlan -modprobe 8021q -echo "8021q" >> /etc/modules -vconfig add eno2 99 -ip addr add $VLAN_IP_ADDR/24 dev eno2.99 -ip link set up eno2.99 -curl -L -s $GITHUB_INSTALL_SCRIPTS/vrack-cfg.yaml -o /etc/netplan/01-netcfg.yaml - -curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - -echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list -curl -sSL https://deb.nodesource.com/setup_10.x | sudo -E bash - -apt-get update -apt-get -y install nodejs yarn -apt-get remove cmdtest - -# compile and install libvips (the version in apt is too old) -cd /tmp -wget -q https://github.com/libvips/libvips/releases/download/v$VIPS_VERSION/vips-$VIPS_VERSION.tar.gz -tar xzf vips-$VIPS_VERSION.tar.gz -cd vips-$VIPS_VERSION -./configure --prefix=/usr -make install -ldconfig - -# Create user account -useradd -m danbooru -chsh -s /bin/bash danbooru -usermod -G danbooru,sudo danbooru - -# Set up Postgres -git clone https://github.com/r888888888/test_parser.git /tmp/test_parser -cd /tmp/test_parser -make install - -# Install rbenv -echo "* Installing rbenv..." -cd /tmp -sudo -u danbooru git clone git://github.com/sstephenson/rbenv.git ~danbooru/.rbenv -sudo -u danbooru touch ~danbooru/.bash_profile -echo 'export PATH="$HOME/.rbenv/bin:$PATH"' >> ~danbooru/.bash_profile -echo 'eval "$(rbenv init -)"' >> ~danbooru/.bash_profile -sudo -u danbooru mkdir -p ~danbooru/.rbenv/plugins -sudo -u danbooru git clone git://github.com/sstephenson/ruby-build.git ~danbooru/.rbenv/plugins/ruby-build -sudo -u danbooru bash -l -c "rbenv install $RUBY_VERSION" -sudo -u danbooru bash -l -c "rbenv global $RUBY_VERSION" - -# Install gems -echo "* Installing gems..." -sudo -u danbooru bash -l -c 'gem install --no-ri --no-rdoc bundler' - -# Setup danbooru account -echo "* Enter a new password for the danbooru account" -passwd danbooru - -echo "* Setting up SSH keys for the danbooru account" -sudo -u danbooru ssh-keygen -sudo -u danbooru cat ~danbooru/.ssh/id_rsa.pub >> ~danbooru/.ssh/authorized_keys - -echo "* TODO:" -echo "on kagamihara:" -echo "script/install/distribute_new_pubkey.sh" -echo -echo "on this server:" -echo "rsync -av kagamihara:/etc/nginx/nginx.conf /etc/nginx" -echo "rsync -av kagamihara:/etc/nginx/conf.d /etc/nginx" -echo "rsync -av kagamihara:/etc/nginx/sites-enabled /etc/nginx" -echo "rsync -av kagamihara:/etc/logrotate.d /etc/logrotate.d" diff --git a/script/install/distribute_new_pubkey.sh b/script/install/distribute_new_pubkey.sh deleted file mode 100644 index 52feaa245..000000000 --- a/script/install/distribute_new_pubkey.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh - -HOSTS="kagamihara shima saitou" - -echo "Enter new SSH pubkey: " -read $key - -for host in $HOSTS ; do - ssh danbooru@$host echo $key >> .ssh/authorized_keys -done diff --git a/script/install/nginx.service.d-local.conf b/script/install/nginx.service.d-local.conf deleted file mode 100644 index 01c1b16bc..000000000 --- a/script/install/nginx.service.d-local.conf +++ /dev/null @@ -1,2 +0,0 @@ -[Service] -LimitNOFILE=10000 diff --git a/script/install/nginx/conf.d/common.conf b/script/install/nginx/conf.d/common.conf deleted file mode 100644 index 677a14da7..000000000 --- a/script/install/nginx/conf.d/common.conf +++ /dev/null @@ -1,92 +0,0 @@ -ssl_protocols TLSv1 TLSv1.1 TLSv1.2; -ssl_ciphers 'ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS'; -ssl_prefer_server_ciphers on; -ssl_session_timeout 4h; -ssl_session_cache shared:SSL:20m; -ssl_session_tickets off; -ssl_stapling on; -ssl_stapling_verify on; -ssl_dhparam /etc/nginx/ssl/dhparam.pem; -resolver 8.8.8.8 8.8.4.4; - -root /var/www/danbooru/current/public; -index index.html; -access_log off; -error_log /var/www/danbooru/shared/log/server.error.log; -try_files $uri/index.html $uri.html $uri @app; -client_max_body_size 35m; -error_page 503 @maintenance; -error_page 404 /404.html; -error_page 500 502 503 504 /500.html; - -location /assets { - expires max; - break; -} - -location /data/preview { - expires max; - break; -} - -location /posts/mobile { - return 404; -} - -location /users { - limit_req zone=users burst=5; - limit_req_status 429; - try_files $uri @app_server; -} - -location /posts { - limit_req zone=posts burst=20; - limit_req_status 429; - try_files $uri @app_server; -} - -location /data { - valid_referers none *.donmai.us donmai.us ~\.google\. ~\.bing\. ~\.yahoo\.; - - if ($invalid_referer) { - return 403; - } - - rewrite ^/data/sample/__.+?__(.+) /data/sample/$1 last; - rewrite ^/data/__.+?__(.+) /data/$1 last; - - expires max; - break; -} - -location /maintenance.html { - expires 10; -} - -if (-f $document_root/maintenance.html) { - return 503; -} - -if ($http_user_agent ~ (WinHttp\.WinHttpRequest\.5) ) { - return 403; -} - -location @maintenance { - rewrite ^(.*)$ /maintenance.html last; -} - -location @app_server { - proxy_pass http://app_server; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Real-IP $remote_addr; - proxy_redirect off; - proxy_set_header Host $http_host; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_buffer_size 128k; - proxy_buffers 4 256k; - proxy_busy_buffers_size 256k; -} - -location / { - try_files $uri @app_server; -} diff --git a/script/install/nginx/nginx.conf b/script/install/nginx/nginx.conf deleted file mode 100644 index f8d9b7e99..000000000 --- a/script/install/nginx/nginx.conf +++ /dev/null @@ -1,82 +0,0 @@ -user www-data; -worker_processes auto; -pid /var/run/nginx.pid; - -events { - use epoll; - worker_connections 10000; - multi_accept on; - accept_mutex on; -} - -http { - limit_req_zone $binary_remote_addr zone=users:10m rate=5r/s; - limit_req_zone $binary_remote_addr zone=posts:100m rate=10r/s; - - ## - # Basic Settings - ## - - sendfile on; - tcp_nopush on; - tcp_nodelay on; - keepalive_timeout 5; - types_hash_max_size 2048; - # server_tokens off; - - server_names_hash_bucket_size 128; - # server_name_in_redirect off; - - include /etc/nginx/mime.types; - default_type application/octet-stream; - - ## - # Logging Settings - ## - - access_log off; - error_log /var/log/nginx/error.log; - - ## - # Gzip Settings - ## - - gzip on; - gzip_disable "msie6"; - - gzip_http_version 1.1; - gzip_vary on; - gzip_comp_level 5; - gzip_proxied any; - gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/rss+xml text/javascript application/atom+xml; - - # curl https://www.cloudflare.com/ips-v4 | sort - set_real_ip_from 103.21.244.0/22; - set_real_ip_from 103.22.200.0/22; - set_real_ip_from 103.31.4.0/22; - set_real_ip_from 104.16.0.0/12; - set_real_ip_from 108.162.192.0/18; - set_real_ip_from 131.0.72.0/22; - set_real_ip_from 141.101.64.0/18; - set_real_ip_from 162.158.0.0/15; - set_real_ip_from 172.64.0.0/13; - set_real_ip_from 173.245.48.0/20; - set_real_ip_from 188.114.96.0/20; - set_real_ip_from 190.93.240.0/20; - set_real_ip_from 197.234.240.0/22; - set_real_ip_from 198.41.128.0/17; - set_real_ip_from 199.27.128.0/21; - - # curl https://www.cloudflare.com/ips-v4 | sort - set_real_ip_from 2400:cb00::/32; - set_real_ip_from 2606:4700::/32; - set_real_ip_from 2803:f800::/32; - set_real_ip_from 2405:b500::/32; - set_real_ip_from 2405:8100::/32; - set_real_ip_from 2a06:98c0::/29; - set_real_ip_from 2c0f:f248::/32; - - real_ip_header CF-Connecting-IP; - - include /etc/nginx/sites-enabled/*.conf; -} diff --git a/script/install/nginx/sites-enabled/danbooru.conf b/script/install/nginx/sites-enabled/danbooru.conf deleted file mode 100644 index 30d19cda3..000000000 --- a/script/install/nginx/sites-enabled/danbooru.conf +++ /dev/null @@ -1,68 +0,0 @@ -server { - listen 443 ssl http2 default_server; - server_name danbooru.donmai.us; - - ssl_certificate /etc/nginx/ssl/danbooru.chain.pem; - ssl_certificate_key /etc/nginx/ssl/danbooru.key; - - include /etc/nginx/conf.d/common.conf; -} - -server { - listen 443 ssl http2; - server_name safebooru.donmai.us; - - ssl_certificate /etc/nginx/ssl/safebooru.chain.pem; - ssl_certificate_key /etc/nginx/ssl/safebooru.key; - - include /etc/nginx/conf.d/common.conf; -} - -server { - listen 443 ssl http2; - server_name kagamihara.donmai.us; - - ssl_certificate /etc/letsencrypt/live/kagamihara.donmai.us/fullchain.pem; # managed by Certbot - ssl_certificate_key /etc/letsencrypt/live/kagamihara.donmai.us/privkey.pem; # managed by Certbot - - include /etc/nginx/conf.d/common.conf; -} - -server { - listen 443 ssl http2; - server_name saitou.donmai.us; - - ssl_certificate /etc/letsencrypt/live/saitou.donmai.us/fullchain.pem; # managed by Certbot - ssl_certificate_key /etc/letsencrypt/live/saitou.donmai.us/privkey.pem; # managed by Certbot - - include /etc/nginx/conf.d/common.conf; -} - -server { - listen 443 ssl http2; - server_name shima.donmai.us; - - ssl_certificate /etc/letsencrypt/live/shima.donmai.us/fullchain.pem; # managed by Certbot - ssl_certificate_key /etc/letsencrypt/live/shima.donmai.us/privkey.pem; # managed by Certbot - - include /etc/nginx/conf.d/common.conf; -} - -# redirect HTTP to HTTPS. -server { - listen 80; - server_name safebooru.donmai.us danbooru.donmai.us kagamihara.donmai.us saitou.donmai.us shima.donmai.us; - return 301 https://$host$request_uri; -} - -# redirect donmai.us and www.donmai.us to danbooru.donmai.us. -server { - listen 80; - listen 443 ssl; - server_name donmai.us www.donmai.us; - return 301 https://danbooru.donmai.us$request_uri; -} - -upstream app_server { - server unix:/tmp/.unicorn.sock fail_timeout=0; -} diff --git a/script/install/vrack-cfg.yaml b/script/install/vrack-cfg.yaml deleted file mode 100644 index cbd7434a3..000000000 --- a/script/install/vrack-cfg.yaml +++ /dev/null @@ -1,10 +0,0 @@ -network: - version: 2 - renderer: networkd - ethernets: - eno2: {} - vlans: - eno2.99: - id: 99 - link: eno2 - addresses: [172.16.0.1] diff --git a/script/mock_services/README.md b/script/mock_services/README.md deleted file mode 100644 index 6f8f84e8a..000000000 --- a/script/mock_services/README.md +++ /dev/null @@ -1,7 +0,0 @@ -These are mocked services to be used for development purposes. - -- danbooru: port 3000 -- recommender: port 3001 -- iqdbs: port 3002 -- reportbooru: port 3003 - diff --git a/script/mock_services/iqdbs.rb b/script/mock_services/iqdbs.rb deleted file mode 100644 index a3cca08c3..000000000 --- a/script/mock_services/iqdbs.rb +++ /dev/null @@ -1,14 +0,0 @@ -require 'sinatra' -require 'json' -require_relative './mock_service_helper' - -set :port, 3002 - -configure do - POST_IDS = MockServiceHelper.fetch_post_ids -end - -get '/similar' do - content_type :json - POST_IDS[0..10].map {|x| {post_id: x}}.to_json -end diff --git a/script/mock_services/mock_service_helper.rb b/script/mock_services/mock_service_helper.rb deleted file mode 100644 index 2b259bc4c..000000000 --- a/script/mock_services/mock_service_helper.rb +++ /dev/null @@ -1,22 +0,0 @@ -require 'socket' -require 'timeout' -require 'httparty' - -module MockServiceHelper - module_function - - DANBOORU_PORT = 3000 - - def fetch_post_ids - begin - s = TCPSocket.new("localhost", DANBOORU_PORT) - s.close - rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH - sleep 1 - retry - end - - json = HTTParty.get("http://localhost:#{DANBOORU_PORT}/posts.json?random=true&limit=10").body - return JSON.parse(json).map {|x| x["id"]} - end -end diff --git a/script/mock_services/recommender.rb b/script/mock_services/recommender.rb deleted file mode 100644 index cb5f2b317..000000000 --- a/script/mock_services/recommender.rb +++ /dev/null @@ -1,19 +0,0 @@ -require 'sinatra' -require 'json' -require_relative './mock_service_helper' - -set :port, 3001 - -configure do - POST_IDS = MockServiceHelper.fetch_post_ids -end - -get '/recommend/:user_id' do - content_type :json - POST_IDS[0..10].map {|x| [x, "1.000"]}.to_json -end - -get '/similar/:post_id' do - content_type :json - POST_IDS[0..6].map {|x| [x, "1.000"]}.to_json -end diff --git a/script/mock_services/reportbooru.rb b/script/mock_services/reportbooru.rb deleted file mode 100644 index 3aa40edb3..000000000 --- a/script/mock_services/reportbooru.rb +++ /dev/null @@ -1,22 +0,0 @@ -require 'sinatra' -require 'json' - -set :port, 3003 - -get '/missed_searches' do - content_type :text - return "abcdefg 10.0\nblahblahblah 20.0\n" -end - -get '/post_searches/rank' do - content_type :json - return [["abc", 100], ["def", 200]].to_json -end - -get '/reports/user_similarity' do - # todo -end - -post '/post_views' do - # todo -end diff --git a/test/files/test-empty.bin b/test/files/test-empty.bin new file mode 100644 index 000000000..e69de29bb diff --git a/test/functional/artists_controller_test.rb b/test/functional/artists_controller_test.rb index 2b24f3cc7..2ae68d78e 100644 --- a/test/functional/artists_controller_test.rb +++ b/test/functional/artists_controller_test.rb @@ -4,11 +4,8 @@ class ArtistsControllerTest < ActionDispatch::IntegrationTest def assert_artist_found(expected_artist, source_url = nil) if source_url get_auth artists_path(format: "json", search: { url_matches: source_url }), @user - if response.body =~ /Net::OpenTimeout/ - skip "Remote connection to #{source_url} failed" - return - end end + assert_response :success json = JSON.parse(response.body) assert_equal(1, json.size, "Testing URL: #{source_url}") @@ -17,10 +14,6 @@ class ArtistsControllerTest < ActionDispatch::IntegrationTest def assert_artist_not_found(source_url) get_auth artists_path(format: "json", search: { url_matches: source_url }), @user - if response.body =~ /Net::OpenTimeout/ - skip "Remote connection to #{source_url} failed" - return - end assert_response :success json = JSON.parse(response.body) @@ -54,6 +47,22 @@ class ArtistsControllerTest < ActionDispatch::IntegrationTest get artist_path(@artist.id) assert_response :success end + + should "show active wikis" do + as(@user) { create(:wiki_page, title: @artist.name) } + get artist_path(@artist.id) + + assert_response :success + assert_select ".artist-wiki", count: 1 + end + + should "not show deleted wikis" do + as(@user) { create(:wiki_page, title: @artist.name, is_deleted: true) } + get artist_path(@artist.id) + + assert_response :success + assert_select ".artist-wiki", count: 0 + end end context "new action" do diff --git a/test/functional/forum_post_votes_controller_test.rb b/test/functional/forum_post_votes_controller_test.rb index 7143c038d..5d693d26b 100644 --- a/test/functional/forum_post_votes_controller_test.rb +++ b/test/functional/forum_post_votes_controller_test.rb @@ -36,6 +36,13 @@ class ForumPostVotesControllerTest < ActionDispatch::IntegrationTest assert_response 403 end end + + should "not allow creators to vote on their own BURs" do + assert_difference("ForumPostVote.count", 0) do + post_auth forum_post_votes_path(format: :js), @bulk_update_request.user, params: { forum_post_id: @forum_post.id, forum_post_vote: { score: 1 }} + assert_response 403 + end + end end context "destroy action" do diff --git a/test/functional/mock_services_controller_test.rb b/test/functional/mock_services_controller_test.rb new file mode 100644 index 000000000..5f33cc691 --- /dev/null +++ b/test/functional/mock_services_controller_test.rb @@ -0,0 +1,28 @@ +require 'test_helper' + +class MockServicesControllerTest < ActionDispatch::IntegrationTest + context "The mock services controller" do + setup do + create(:post) + create(:tag) + end + + context "for all actions" do + should "work" do + paths = [ + mock_recommender_recommend_path(42), + mock_recommender_similar_path(42), + mock_reportbooru_missed_searches_path, + mock_reportbooru_post_searches_path, + mock_reportbooru_post_views_path, + mock_iqdbs_similar_path, + ] + + paths.each do |path| + get path + assert_response :success + end + end + end + end +end diff --git a/test/functional/post_versions_controller_test.rb b/test/functional/post_versions_controller_test.rb index a57914cff..521d99adb 100644 --- a/test/functional/post_versions_controller_test.rb +++ b/test/functional/post_versions_controller_test.rb @@ -41,6 +41,13 @@ class PostVersionsControllerTest < ActionDispatch::IntegrationTest assert_response :success assert_equal @post.versions[1].id, response.parsed_body[0]["id"].to_i end + + should "list all versions for search[tag_matches]" do + get post_versions_path, as: :json, params: { search: { tag_matches: "tagme" }} + assert_response :success + assert_equal @post.versions[0].id, response.parsed_body[0]["id"].to_i + assert_equal 1, response.parsed_body.length + end end context "undo action" do diff --git a/test/functional/uploads_controller_test.rb b/test/functional/uploads_controller_test.rb index 9df37a1b4..10fab5471 100644 --- a/test/functional/uploads_controller_test.rb +++ b/test/functional/uploads_controller_test.rb @@ -1,15 +1,30 @@ require 'test_helper' class UploadsControllerTest < ActionDispatch::IntegrationTest - def assert_uploaded(file_path, user, **upload_params) - file = Rack::Test::UploadedFile.new("#{Rails.root}/#{file_path}") + def self.should_upload_successfully(source) + should "upload successfully from #{source}" do + assert_successful_upload(source, user: create(:user, created_at: 1.month.ago)) + end + end - assert_difference(["Upload.count", "Post.count"]) do - post_auth uploads_path, user, params: { upload: { file: file, **upload_params }} - assert_redirected_to Upload.last + def assert_successful_upload(source_or_file_path, user: @user, **params) + if source_or_file_path =~ %r{\Ahttps?://}i + source = { source: source_or_file_path } + else + file = Rack::Test::UploadedFile.new(Rails.root.join(source_or_file_path)) + source = { file: file } end - Upload.last + assert_difference(["Upload.count"]) do + post_auth uploads_path, user, params: { upload: { tag_string: "abc", rating: "e", **source, **params }} + end + + upload = Upload.last + assert_response :redirect + assert_redirected_to upload + assert_equal("completed", upload.status) + assert_equal(Post.last, upload.post) + assert_equal(upload.post.md5, upload.md5) end context "The uploads controller" do @@ -18,6 +33,17 @@ class UploadsControllerTest < ActionDispatch::IntegrationTest mock_iqdb_service! end + context "image proxy action" do + should "work" do + url = "https://i.pximg.net/img-original/img/2017/11/21/17/06/44/65985331_p0.png" + get_auth image_proxy_uploads_path, @user, params: { url: url } + + assert_response :success + assert_equal("image/png", response.media_type) + assert_equal(15_573, response.body.size) + end + end + context "batch action" do context "for twitter galleries" do should "render" do @@ -259,32 +285,65 @@ class UploadsControllerTest < ActionDispatch::IntegrationTest end context "uploading a file from your computer" do - should "work for a jpeg file" do - upload = assert_uploaded("test/files/test.jpg", @user, tag_string: "aaa", rating: "e", source: "aaa") + should_upload_successfully("test/files/test.jpg") + should_upload_successfully("test/files/test.png") + should_upload_successfully("test/files/test-static-32x32.gif") + should_upload_successfully("test/files/test-animated-86x52.gif") + should_upload_successfully("test/files/test-300x300.mp4") + should_upload_successfully("test/files/test-512x512.webm") + should_upload_successfully("test/files/compressed.swf") + end - assert_equal("jpg", upload.post.file_ext) - assert_equal("aaa", upload.post.source) - assert_equal(500, upload.post.image_width) - assert_equal(335, upload.post.image_height) - end + context "uploading a file from a source" do + should_upload_successfully("https://www.artstation.com/artwork/04XA4") + should_upload_successfully("https://dantewontdie.artstation.com/projects/YZK5q") + should_upload_successfully("https://cdna.artstation.com/p/assets/images/images/006/029/978/large/amama-l-z.jpg") - should "work for a webm file" do - upload = assert_uploaded("test/files/test-512x512.webm", @user, tag_string: "aaa", rating: "e", source: "aaa") + should_upload_successfully("https://www.deviantart.com/aeror404/art/Holiday-Elincia-424551484") + should_upload_successfully("https://noizave.deviantart.com/art/test-no-download-697415967") + should_upload_successfully("https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/8b472d70-a0d6-41b5-9a66-c35687090acc/d23jbr4-8a06af02-70cb-46da-8a96-42a6ba73cdb4.jpg/v1/fill/w_786,h_1017,q_70,strp/silverhawks_quicksilver_by_edsfox_d23jbr4-pre.jpg") - assert_equal("webm", upload.post.file_ext) - assert_equal("aaa", upload.post.source) - assert_equal(512, upload.post.image_width) - assert_equal(512, upload.post.image_height) - end + should_upload_successfully("https://www.hentai-foundry.com/pictures/user/Afrobull/795025/kuroeda") + should_upload_successfully("https://pictures.hentai-foundry.com/a/Afrobull/795025/Afrobull-795025-kuroeda.png") - should "work for a flash file" do - upload = assert_uploaded("test/files/compressed.swf", @user, tag_string: "aaa", rating: "e", source: "aaa") + should_upload_successfully("https://yande.re/post/show/482880") + should_upload_successfully("https://files.yande.re/image/7ecfdead705d7b956b26b1d37b98d089/yande.re%20482880.jpg") - assert_equal("swf", upload.post.file_ext) - assert_equal("aaa", upload.post.source) - assert_equal(607, upload.post.image_width) - assert_equal(756, upload.post.image_height) - end + should_upload_successfully("https://konachan.com/post/show/270916") + should_upload_successfully("https://konachan.com/image/ca12cdb79a66d242e95a6f958341bf05/Konachan.com%20-%20270916.png") + + should_upload_successfully("http://lohas.nicoseiga.jp/o/910aecf08e542285862954017f8a33a8c32a8aec/1433298801/4937663") + should_upload_successfully("http://seiga.nicovideo.jp/seiga/im4937663") + should_upload_successfully("https://seiga.nicovideo.jp/image/source/9146749") + should_upload_successfully("https://seiga.nicovideo.jp/watch/mg389884") + should_upload_successfully("https://dic.nicovideo.jp/oekaki/52833.png") + should_upload_successfully("https://lohas.nicoseiga.jp/o/971eb8af9bbcde5c2e51d5ef3a2f62d6d9ff5552/1589933964/3583893") + should_upload_successfully("http://lohas.nicoseiga.jp/priv/3521156?e=1382558156&h=f2e089256abd1d453a455ec8f317a6c703e2cedf") + should_upload_successfully("http://lohas.nicoseiga.jp/priv/b80f86c0d8591b217e7513a9e175e94e00f3c7a1/1384936074/3583893") + should_upload_successfully("http://lohas.nicoseiga.jp/material/5746c5/4459092") + # XXX should_upload_successfully("https://dcdn.cdn.nimg.jp/priv/62a56a7f67d3d3746ae5712db9cac7d465f4a339/1592186183/10466669") + # XXX should_upload_successfully("https://dcdn.cdn.nimg.jp/nicoseiga/lohas/o/8ba0a9b2ea34e1ef3b5cc50785bd10cd63ec7e4a/1592187477/10466669") + + should_upload_successfully("http://nijie.info/view.php?id=213043") + should_upload_successfully("https://nijie.info/view_popup.php?id=213043") + should_upload_successfully("https://pic.nijie.net/03/nijie_picture/728995_20170505014820_0.jpg") + + should_upload_successfully("https://pawoo.net/web/statuses/1202176") + should_upload_successfully("https://img.pawoo.net/media_attachments/files/000/128/953/original/4c0a06087b03343f.png") + + should_upload_successfully("https://www.pixiv.net/en/artworks/64476642") + should_upload_successfully("https://i.pximg.net/img-original/img/2017/08/18/00/09/21/64476642_p0.jpg") + + should_upload_successfully("https://noizave.tumblr.com/post/162206271767") + should_upload_successfully("https://media.tumblr.com/3bbfcbf075ddf969c996641b264086fd/tumblr_os2buiIOt51wsfqepo1_1280.png") + + should_upload_successfully("https://twitter.com/noizave/status/875768175136317440") + should_upload_successfully("https://pbs.twimg.com/media/DCdZ_FhUIAAYKFN?format=jpg&name=medium") + should_upload_successfully("https://pbs.twimg.com/profile_banners/1225702850002468864/1588597370/1500x500") + # XXX should_upload_successfully("https://video.twimg.com/tweet_video/EWHWVrmVcAAp4Vw.mp4") + + should_upload_successfully("https://www.weibo.com/5501756072/J2UNKfbqV") + should_upload_successfully("https://wx1.sinaimg.cn/mw690/0060kO5aly1gezsyt5xvhj30ok0sgtc9.jpg") end end end diff --git a/test/functional/wiki_pages_controller_test.rb b/test/functional/wiki_pages_controller_test.rb index 119483201..2ea1619c6 100644 --- a/test/functional/wiki_pages_controller_test.rb +++ b/test/functional/wiki_pages_controller_test.rb @@ -10,8 +10,11 @@ class WikiPagesControllerTest < ActionDispatch::IntegrationTest context "index action" do setup do as(@user) do - @wiki_page_abc = create(:wiki_page, :title => "abc") - @wiki_page_def = create(:wiki_page, :title => "def") + @tagme = create(:wiki_page, title: "tagme") + @deleted = create(:wiki_page, title: "deleted", is_deleted: true) + @vocaloid = create(:wiki_page, title: "vocaloid") + @miku = create(:wiki_page, title: "hatsune_miku", other_names: ["初音ミク"], body: "miku is a [[vocaloid]]") + create(:tag, name: "hatsune_miku", category: Tag.categories.character) end end @@ -20,22 +23,24 @@ class WikiPagesControllerTest < ActionDispatch::IntegrationTest assert_response :success end - should "list all wiki_pages (with search)" do - get wiki_pages_path, params: {:search => {:title => "abc"}} - assert_response :success - assert_select "tr td:first-child", text: "abc" - end - - should "list wiki_pages without tags with order=post_count" do - get wiki_pages_path, params: {:search => {:title => "abc", :order => "post_count"}} - assert_response :success - assert_select "tr td:first-child", text: "abc" - end - should "redirect the legacy title param to the show page" do - get wiki_pages_path(title: "abc") - assert_redirected_to wiki_pages_path(search: { title_normalize: "abc" }, redirect: true) + get wiki_pages_path(title: "tagme") + assert_redirected_to wiki_pages_path(search: { title_normalize: "tagme" }, redirect: true) end + + should respond_to_search(title: "tagme").with { @tagme } + should respond_to_search(title: "tagme", order: "post_count").with { @tagme } + should respond_to_search(title_normalize: "TAGME ").with { @tagme } + + should respond_to_search(tag: { category: Tag.categories.character }).with { @miku } + should respond_to_search(hide_deleted: "true").with { [@miku, @vocaloid, @tagme] } + should respond_to_search(linked_to: "vocaloid").with { @miku } + should respond_to_search(not_linked_to: "vocaloid").with { [@vocaloid, @deleted, @tagme] } + + should respond_to_search(other_names_match: "初音ミク").with { @miku } + should respond_to_search(other_names_match: "初*").with { @miku } + should respond_to_search(other_names_present: "true").with { @miku } + should respond_to_search(other_names_present: "false").with { [@vocaloid, @deleted, @tagme] } end context "search action" do diff --git a/test/test_helper.rb b/test/test_helper.rb index 700bf1e6d..892046f7e 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -43,15 +43,15 @@ class ActiveSupport::TestCase setup do Socket.stubs(:gethostname).returns("www.example.com") - WebMock.allow_net_connect! - storage_manager = StorageManager::Local.new(base_dir: Dir.mktmpdir("uploads-test-storage-")) + @temp_dir = Dir.mktmpdir("danbooru-temp-") + storage_manager = StorageManager::Local.new(base_dir: @temp_dir) Danbooru.config.stubs(:storage_manager).returns(storage_manager) Danbooru.config.stubs(:backup_storage_manager).returns(StorageManager::Null.new) end teardown do - FileUtils.rm_rf(Danbooru.config.storage_manager.base_dir) + FileUtils.rm_rf(@temp_dir) Cache.clear end @@ -61,6 +61,8 @@ class ActiveSupport::TestCase end class ActionDispatch::IntegrationTest + extend ControllerHelper + register_encoder :xml, response_parser: ->(body) { Nokogiri.XML(body) } def method_authenticated(method_name, url, user, **options) diff --git a/test/test_helpers/controller_helper.rb b/test/test_helpers/controller_helper.rb new file mode 100644 index 000000000..716bfaa97 --- /dev/null +++ b/test/test_helpers/controller_helper.rb @@ -0,0 +1,47 @@ +module ControllerHelper + # A custom Shoulda matcher that tests that a controller's index endpoint + # responds to a search correctly. See https://thoughtbot.com/blog/shoulda-matchers. + # + # Usage: + # + # # Tests that `/tags.json?search[name]=touhou` returns the `touhou` tag. + # subject { TagsController } + # setup { @touhou = create(:tag, name: "touhou") } + # should respond_to_search(name: "touhou").with { @touhou } + # + def respond_to_search(search_params) + RespondToSearchMatcher.new(search_params) + end + + class RespondToSearchMatcher < Struct.new(:params) + def description + "should respond to a search for #{params}" + end + + def matches?(subject, &block) + search_params = { search: params } + expected_items = @test_case.instance_eval(&@expected) + + @test_case.instance_eval do + # calls e.g. "wiki_pages_path" if we're in WikiPagesControllerTest. + index_url = send("#{subject.controller_path}_path") + get index_url, as: :json, params: search_params + + expected_ids = Array(expected_items).map(&:id) + responded_ids = response.parsed_body.map { |item| item["id"] } + + assert_response :success + assert_equal(expected_ids, responded_ids) + end + end + + def with(&block) + @expected = block + self + end + + def in_context(test_case) + @test_case = test_case + end + end +end diff --git a/test/test_helpers/download_test_helper.rb b/test/test_helpers/download_test_helper.rb index 72805cf9b..68e14abc6 100644 --- a/test/test_helpers/download_test_helper.rb +++ b/test/test_helpers/download_test_helper.rb @@ -1,10 +1,8 @@ module DownloadTestHelper def assert_downloaded(expected_filesize, source, referer = nil) - download = Downloads::File.new(source, referer) - tempfile, strategy = download.download! - assert_equal(expected_filesize, tempfile.size, "Tested source URL: #{source}") - rescue Net::OpenTimeout - skip "Remote connection to #{source} failed" + strategy = Sources::Strategies.find(source, referer) + file = strategy.download_file! + assert_equal(expected_filesize, file.size, "Tested source URL: #{source}") end def assert_rewritten(expected_source, test_source, test_referer = nil) @@ -16,19 +14,4 @@ module DownloadTestHelper def assert_not_rewritten(source, referer = nil) assert_rewritten(source, source, referer) end - - def assert_http_exists(url, headers: {}) - res = HTTParty.head(url, Danbooru.config.httparty_options.deep_merge(headers: headers)) - assert_equal(true, res.success?) - end - - def assert_http_status(code, url, headers: {}) - res = HTTParty.head(url, Danbooru.config.httparty_options.deep_merge(headers: headers)) - assert_equal(code, res.code) - end - - def assert_http_size(size, url, headers: {}) - res = HTTParty.head(url, Danbooru.config.httparty_options.deep_merge(headers: headers)) - assert_equal(size, res.content_length) - end end diff --git a/test/test_helpers/reportbooru_helper.rb b/test/test_helpers/reportbooru_helper.rb index 6a99dc5d4..3ec926202 100644 --- a/test/test_helpers/reportbooru_helper.rb +++ b/test/test_helpers/reportbooru_helper.rb @@ -1,7 +1,7 @@ module ReportbooruHelper - def mock_request(url, method: :get, status: 200, body: nil, http: Danbooru::Http.any_instance) + def mock_request(url, method: :get, status: 200, body: nil, http: Danbooru::Http.any_instance, **options) response = HTTP::Response.new(status: status, body: body, version: "1.1") - http.stubs(method).with(url).returns(response) + http.stubs(method).with(url, **options).returns(response) end def mock_post_search_rankings(date = Date.today, rankings) diff --git a/test/unit/artist_test.rb b/test/unit/artist_test.rb index b9dc3a90c..44beba428 100644 --- a/test/unit/artist_test.rb +++ b/test/unit/artist_test.rb @@ -6,15 +6,11 @@ class ArtistTest < ActiveSupport::TestCase assert_equal(1, artists.size) assert_equal(expected_name, artists.first.name, "Testing URL: #{source_url}") - rescue Net::OpenTimeout, PixivApiClient::Error - skip "Remote connection failed for #{source_url}" end def assert_artist_not_found(source_url) artists = ArtistFinder.find_artists(source_url).to_a assert_equal(0, artists.size, "Testing URL: #{source_url}") - rescue Net::OpenTimeout - skip "Remote connection failed for #{source_url}" end context "An artist" do @@ -172,15 +168,11 @@ class ArtistTest < ActiveSupport::TestCase a2 = FactoryBot.create(:artist, :name => "subway", :url_string => "http://subway.com/x/test.jpg") a3 = FactoryBot.create(:artist, :name => "minko", :url_string => "https://minko.com/x/test.jpg") - begin - assert_artist_found("rembrandt", "http://rembrandt.com/x/test.jpg") - assert_artist_found("rembrandt", "http://rembrandt.com/x/another.jpg") - assert_artist_not_found("http://nonexistent.com/test.jpg") - assert_artist_found("minko", "https://minko.com/x/test.jpg") - assert_artist_found("minko", "http://minko.com/x/test.jpg") - rescue Net::OpenTimeout - skip "network failure" - end + assert_artist_found("rembrandt", "http://rembrandt.com/x/test.jpg") + assert_artist_found("rembrandt", "http://rembrandt.com/x/another.jpg") + assert_artist_not_found("http://nonexistent.com/test.jpg") + assert_artist_found("minko", "https://minko.com/x/test.jpg") + assert_artist_found("minko", "http://minko.com/x/test.jpg") end should "be case-insensitive to domains when finding matches by url" do diff --git a/test/unit/cloudflare_service_test.rb b/test/unit/cloudflare_service_test.rb index faced9207..cb9bc86be 100644 --- a/test/unit/cloudflare_service_test.rb +++ b/test/unit/cloudflare_service_test.rb @@ -1,5 +1,4 @@ require 'test_helper' -require 'webmock/minitest' class CloudflareServiceTest < ActiveSupport::TestCase def setup @@ -8,16 +7,11 @@ class CloudflareServiceTest < ActiveSupport::TestCase context "#purge_cache" do should "make calls to cloudflare's api" do - stub_request(:any, "api.cloudflare.com") - @cloudflare.purge_cache(["http://localhost/file.txt"]) + url = "http://www.example.com/file.jpg" + mock_request("https://api.cloudflare.com/client/v4/zones/123/purge_cache", method: :delete, json: { files: [url] }) - assert_requested(:delete, "https://api.cloudflare.com/client/v4/zones/123/purge_cache", times: 1) - end - end - - context "#ips" do - should "work" do - refute_empty(@cloudflare.ips) + response = @cloudflare.purge_cache([url]) + assert_equal(200, response.status) end end end diff --git a/test/unit/danbooru_http_test.rb b/test/unit/danbooru_http_test.rb index 42e585bef..79ba9e0d5 100644 --- a/test/unit/danbooru_http_test.rb +++ b/test/unit/danbooru_http_test.rb @@ -4,18 +4,20 @@ class DanbooruHttpTest < ActiveSupport::TestCase context "Danbooru::Http" do context "#get method" do should "work for all basic methods" do - %i[get put post delete].each do |method| + %i[get head put post delete].each do |method| response = Danbooru::Http.send(method, "https://httpbin.org/status/200") assert_equal(200, response.status) end end should "follow redirects" do + skip "Skipping test (https://github.com/postmanlabs/httpbin/issues/617)" response = Danbooru::Http.get("https://httpbin.org/absolute-redirect/3") assert_equal(200, response.status) end should "fail if redirected too many times" do + skip "Skipping test (https://github.com/postmanlabs/httpbin/issues/617)" response = Danbooru::Http.get("https://httpbin.org/absolute-redirect/10") assert_equal(598, response.status) end @@ -26,8 +28,10 @@ class DanbooruHttpTest < ActiveSupport::TestCase end should "fail if the request takes too long to download" do - response = Danbooru::Http.timeout(1).get("https://httpbin.org/drip?duration=5&numbytes=5") - assert_equal(599, response.status) + # XXX should return status 599 instead + assert_raises(HTTP::TimeoutError) do + response = Danbooru::Http.timeout(1).get("https://httpbin.org/drip?duration=10&numbytes=10").flush + end end should "automatically decompress gzipped responses" do @@ -36,13 +40,131 @@ class DanbooruHttpTest < ActiveSupport::TestCase assert_equal(true, response.parse["gzipped"]) end - should "cache requests" do - response1 = Danbooru::Http.cache(1.minute).get("https://httpbin.org/uuid") + should "automatically parse html responses" do + response = Danbooru::Http.get("https://httpbin.org/html") + assert_equal(200, response.status) + assert_instance_of(Nokogiri::HTML5::Document, response.parse) + assert_equal("Herman Melville - Moby-Dick", response.parse.css("h1").text) + end + + should "automatically parse xml responses" do + response = Danbooru::Http.get("https://httpbin.org/xml") + assert_equal(200, response.status) + assert_equal(true, response.parse[:slideshow].present?) + end + + should "track cookies between requests" do + http = Danbooru::Http.use(:session) + + resp1 = http.get("https://httpbin.org/cookies/set/abc/1") + resp2 = http.get("https://httpbin.org/cookies/set/def/2") + resp3 = http.get("https://httpbin.org/cookies") + assert_equal({ abc: "1", def: "2" }, resp3.parse["cookies"].symbolize_keys) + + resp4 = http.cookies(def: 3, ghi: 4).get("https://httpbin.org/cookies") + assert_equal({ abc: "1", def: "3", ghi: "4" }, resp4.parse["cookies"].symbolize_keys) + end + end + + context "cache feature" do + should "cache multiple requests to the same url" do + http = Danbooru::Http.cache(1.hour) + + response1 = http.get("https://httpbin.org/uuid") assert_equal(200, response1.status) - response2 = Danbooru::Http.cache(1.minute).get("https://httpbin.org/uuid") + response2 = http.get("https://httpbin.org/uuid") assert_equal(200, response2.status) - assert_equal(response2.body, response1.body) + assert_equal(response2.to_s, response1.to_s) + end + + should "cache cookies correctly" do + http = Danbooru::Http.cache(1.hour) + + resp1 = http.get("https://httpbin.org/cookies") + resp2 = http.get("https://httpbin.org/cookies/set/abc/1") + resp3 = http.get("https://httpbin.org/cookies/set/def/2") + resp4 = http.get("https://httpbin.org/cookies") + + assert_equal(200, resp1.status) + assert_equal(200, resp2.status) + assert_equal(200, resp3.status) + assert_equal(200, resp4.status) + + assert_equal({}, resp1.parse["cookies"].symbolize_keys) + assert_equal({ abc: "1" }, resp2.parse["cookies"].symbolize_keys) + assert_equal({ abc: "1", def: "2" }, resp3.parse["cookies"].symbolize_keys) + assert_equal({ abc: "1", def: "2" }, resp4.parse["cookies"].symbolize_keys) + end + end + + context "retriable feature" do + should "retry immediately if no Retry-After header is sent" do + response_429 = ::HTTP::Response.new(status: 429, version: "1.1", body: "") + response_200 = ::HTTP::Response.new(status: 200, version: "1.1", body: "") + HTTP::Client.any_instance.expects(:perform).times(2).returns(response_429, response_200) + + response = Danbooru::Http.use(:retriable).get("https://httpbin.org/status/429") + assert_equal(200, response.status) + end + + should "retry if the Retry-After header is an integer" do + response_503 = ::HTTP::Response.new(status: 503, version: "1.1", headers: { "Retry-After": "1" }, body: "") + response_200 = ::HTTP::Response.new(status: 200, version: "1.1", body: "") + HTTP::Client.any_instance.expects(:perform).times(2).returns(response_503, response_200) + + response = Danbooru::Http.use(:retriable).get("https://httpbin.org/status/503") + assert_equal(200, response.status) + end + + should "retry if the Retry-After header is a date" do + response_503 = ::HTTP::Response.new(status: 503, version: "1.1", headers: { "Retry-After": 2.seconds.from_now.httpdate }, body: "") + response_200 = ::HTTP::Response.new(status: 200, version: "1.1", body: "") + HTTP::Client.any_instance.expects(:perform).times(2).returns(response_503, response_200) + + response = Danbooru::Http.use(:retriable).get("https://httpbin.org/status/503") + assert_equal(200, response.status) + end + end + + context "#download method" do + should "download files" do + response, file = Danbooru::Http.download_media("https://httpbin.org/bytes/1000") + + assert_equal(200, response.status) + assert_equal(1000, file.size) + end + + should "follow redirects when downloading files" do + skip "Skipping test (https://github.com/postmanlabs/httpbin/issues/617)" + response, file = Danbooru::Http.download_media("https://httpbin.org/redirect-to?url=https://httpbin.org/bytes/1000") + + assert_equal(200, response.status) + assert_equal(1000, file.size) + end + + should "fail if the url points to a private IP" do + assert_raises(Danbooru::Http::DownloadError) do + Danbooru::Http.public_only.download_media("https://127.0.0.1.xip.io") + end + end + + should "fail if the url redirects to a private IP" do + assert_raises(Danbooru::Http::DownloadError) do + Danbooru::Http.public_only.download_media("https://httpbin.org/redirect-to?url=https://127.0.0.1.xip.io") + end + end + + should "fail if a download is too large" do + assert_raises(Danbooru::Http::FileTooLargeError) do + response, file = Danbooru::Http.max_size(500).download_media("https://httpbin.org/bytes/1000") + end + end + + should "fail if a streaming download is too large" do + assert_raises(Danbooru::Http::FileTooLargeError) do + response, file = Danbooru::Http.max_size(500).download_media("https://httpbin.org/stream-bytes/1000") + end end end end diff --git a/test/unit/downloads/art_station_test.rb b/test/unit/downloads/art_station_test.rb index 0ebacd585..83b1710ca 100644 --- a/test/unit/downloads/art_station_test.rb +++ b/test/unit/downloads/art_station_test.rb @@ -3,53 +3,27 @@ require 'test_helper' module Downloads class ArtStationTest < ActiveSupport::TestCase context "a download for a (small) artstation image" do - setup do - @asset = "https://cdnb3.artstation.com/p/assets/images/images/003/716/071/small/aoi-ogata-hate-city.jpg?1476754974" - @download = Downloads::File.new(@asset) - end - should "download the /4k/ image instead" do - file, strategy = @download.download! - assert_equal(1_880_910, ::File.size(file.path)) + assert_downloaded(1_880_910, "https://cdnb3.artstation.com/p/assets/images/images/003/716/071/small/aoi-ogata-hate-city.jpg?1476754974") end end context "for an image where an original does not exist" do - setup do - @asset = "https://cdna.artstation.com/p/assets/images/images/004/730/278/large/mendel-oh-dragonll.jpg" - @download = Downloads::File.new(@asset) - end - should "not try to download the original" do - file, strategy = @download.download! - assert_equal(483_192, ::File.size(file.path)) + assert_downloaded(483_192, "https://cdna.artstation.com/p/assets/images/images/004/730/278/large/mendel-oh-dragonll.jpg") end end context "a download for an ArtStation image hosted on CloudFlare" do - setup do - @asset = "https://cdnb.artstation.com/p/assets/images/images/003/716/071/large/aoi-ogata-hate-city.jpg?1476754974" - end - should "return the original file, not the polished file" do + @asset = "https://cdnb.artstation.com/p/assets/images/images/003/716/071/large/aoi-ogata-hate-city.jpg?1476754974" assert_downloaded(1_880_910, @asset) end - - should "return the original filesize, not the polished filesize" do - assert_equal(1_880_910, Downloads::File.new(@asset).size) - end end context "a download for a https://$artist.artstation.com/projects/$id page" do - setup do - @source = "https://dantewontdie.artstation.com/projects/YZK5q" - @download = Downloads::File.new(@source) - end - should "download the original image instead" do - file, strategy = @download.download! - - assert_equal(247_350, ::File.size(file.path)) + assert_downloaded(247_350, "https://dantewontdie.artstation.com/projects/YZK5q") end end end diff --git a/test/unit/downloads/file_test.rb b/test/unit/downloads/file_test.rb deleted file mode 100644 index 7e37787dd..000000000 --- a/test/unit/downloads/file_test.rb +++ /dev/null @@ -1,81 +0,0 @@ -require 'test_helper' - -module Downloads - class FileTest < ActiveSupport::TestCase - context "A post download" do - setup do - @source = "http://www.google.com/intl/en_ALL/images/logo.gif" - @download = Downloads::File.new(@source) - end - - context "for a banned IP" do - setup do - Resolv.expects(:getaddress).returns("127.0.0.1").at_least_once - end - - should "not try to download the file" do - assert_raise(Downloads::File::Error) { Downloads::File.new("http://evil.com").download! } - end - - should "not try to fetch the size" do - assert_raise(Downloads::File::Error) { Downloads::File.new("http://evil.com").size } - end - - should "not follow redirects to banned IPs" do - url = "http://httpbin.org/redirect-to?url=http://127.0.0.1" - stub_request(:get, url).to_return(status: 301, headers: { "Location": "http://127.0.0.1" }) - - assert_raise(Downloads::File::Error) { Downloads::File.new(url).download! } - end - - should "not follow redirects that resolve to a banned IP" do - url = "http://httpbin.org/redirect-to?url=http://127.0.0.1.nip.io" - stub_request(:get, url).to_return(status: 301, headers: { "Location": "http://127.0.0.1.xip.io" }) - - assert_raise(Downloads::File::Error) { Downloads::File.new(url).download! } - end - end - - context "that fails" do - should "retry three times before giving up" do - HTTParty.expects(:get).times(3).raises(Errno::ETIMEDOUT) - assert_raises(Errno::ETIMEDOUT) { @download.download! } - end - - should "return an uncorrupted file on the second try" do - bomb = stub("bomb") - bomb.stubs(:code).raises(IOError) - resp = stub("resp", success?: true) - - chunk = stub("a") - chunk.stubs(:code).returns(200) - chunk.stubs(:size).returns(1) - chunk.stubs(:to_s).returns("a") - - HTTParty.expects(:get).twice.multiple_yields(chunk, bomb).then.multiple_yields(chunk, chunk).returns(resp) - @download.stubs(:is_cloudflare?).returns(false) - tempfile, _strategy = @download.download! - - assert_equal("aa", tempfile.read) - end - end - - should "throw an exception when the file is larger than the maximum" do - assert_raise(Downloads::File::Error) do - @download.download!(max_size: 1) - end - end - - should "store the file in the tempfile path" do - tempfile, strategy = @download.download! - assert_operator(tempfile.size, :>, 0, "should have data") - end - - should "correctly save the file when following 302 redirects" do - download = Downloads::File.new("https://yande.re/post/show/578014") - file, strategy = download.download!(url: download.preview_url) - assert_equal(19134, file.size) - end - end - end -end diff --git a/test/unit/downloads/pixiv_test.rb b/test/unit/downloads/pixiv_test.rb index a22f8a700..02d923c02 100644 --- a/test/unit/downloads/pixiv_test.rb +++ b/test/unit/downloads/pixiv_test.rb @@ -122,32 +122,17 @@ module Downloads assert_downloaded(@file_size, @file_url, @ref) end end - - context "downloading a pixiv fanbox image" do - should_eventually "work" do - @source = "https://www.pixiv.net/fanbox/creator/12491073/post/82406" - @file_url = "https://fanbox.pixiv.net/images/post/82406/D833IKA7FIesJXL8xx39rrG0.jpeg" - @file_size = 873_387 - - assert_not_rewritten(@file_url, @source) - assert_downloaded(@file_size, @file_url, @source) - end - end end context "An ugoira site for pixiv" do - setup do - @download = Downloads::File.new("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364") - @tempfile, strategy = @download.download! - @tempfile.close! - end - should "capture the data" do - assert_equal(2, @download.data[:ugoira_frame_data].size) - if @download.data[:ugoira_frame_data][0]["file"] + @strategy = Sources::Strategies.find("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364") + + assert_equal(2, @strategy.data[:ugoira_frame_data].size) + if @strategy.data[:ugoira_frame_data][0]["file"] assert_equal([{"file" => "000000.jpg", "delay" => 125}, {"file" => "000001.jpg", "delay" => 125}], @download.data[:ugoira_frame_data]) else - assert_equal([{"delay_msec" => 125}, {"delay_msec" => 125}], @download.data[:ugoira_frame_data]) + assert_equal([{"delay_msec" => 125}, {"delay_msec" => 125}], @strategy.data[:ugoira_frame_data]) end end end diff --git a/test/unit/media_file_test.rb b/test/unit/media_file_test.rb index a7516198d..4c77cf5b3 100644 --- a/test/unit/media_file_test.rb +++ b/test/unit/media_file_test.rb @@ -98,6 +98,10 @@ class MediaFileTest < ActiveSupport::TestCase should "determine the correct extension for a flash file" do assert_equal(:swf, MediaFile.open("test/files/compressed.swf").file_ext) end + + should "not fail for empty files" do + assert_equal(:bin, MediaFile.open("test/files/test-empty.bin").file_ext) + end end should "determine the correct md5 for a jpeg file" do diff --git a/test/unit/post_query_builder_test.rb b/test/unit/post_query_builder_test.rb index d74b08e8e..54b8ca27f 100644 --- a/test/unit/post_query_builder_test.rb +++ b/test/unit/post_query_builder_test.rb @@ -277,6 +277,19 @@ class PostQueryBuilderTest < ActiveSupport::TestCase assert_tag_match([child, parent], "-child:garbage") end + should "return posts when using the status of the parent/child" do + parent_of_deleted = create(:post) + deleted = create(:post, is_deleted: true, tag_string: "parent:#{parent_of_deleted.id}") + child_of_deleted = create(:post, tag_string: "parent:#{deleted.id}") + all = [child_of_deleted, deleted, parent_of_deleted] + + assert_tag_match([child_of_deleted], "parent:deleted") + assert_tag_match(all - [child_of_deleted], "-parent:deleted") + + assert_tag_match([parent_of_deleted], "child:deleted") + assert_tag_match(all - [parent_of_deleted], "-child:deleted") + end + should "return posts for the favgroup: