Merge branch 'master' into fix-pixiv-profile-url

test/functional/artists_controller_test.rb

@@ -4,11 +4,8 @@ class ArtistsControllerTest < ActionDispatch::IntegrationTest
   def assert_artist_found(expected_artist, source_url = nil)
     if source_url
       get_auth artists_path(format: "json", search: { url_matches: source_url }), @user
-      if response.body =~ /Net::OpenTimeout/
-        skip "Remote connection to #{source_url} failed"
-        return
-      end
     end
+
     assert_response :success
     json = JSON.parse(response.body)
     assert_equal(1, json.size, "Testing URL: #{source_url}")
@@ -17,10 +14,6 @@ class ArtistsControllerTest < ActionDispatch::IntegrationTest
 
   def assert_artist_not_found(source_url)
     get_auth artists_path(format: "json", search: { url_matches: source_url }), @user
-    if response.body =~ /Net::OpenTimeout/
-      skip "Remote connection to #{source_url} failed"
-      return
-    end
 
     assert_response :success
     json = JSON.parse(response.body)
@@ -54,6 +47,22 @@ class ArtistsControllerTest < ActionDispatch::IntegrationTest
         get artist_path(@artist.id)
         assert_response :success
       end
+
+      should "show active wikis" do
+        as(@user) { create(:wiki_page, title: @artist.name) }
+        get artist_path(@artist.id)
+
+        assert_response :success
+        assert_select ".artist-wiki", count: 1
+      end
+
+      should "not show deleted wikis" do
+        as(@user) { create(:wiki_page, title: @artist.name, is_deleted: true) }
+        get artist_path(@artist.id)
+
+        assert_response :success
+        assert_select ".artist-wiki", count: 0
+      end
     end
 
     context "new action" do

test/functional/forum_post_votes_controller_test.rb

@@ -36,6 +36,13 @@ class ForumPostVotesControllerTest < ActionDispatch::IntegrationTest
           assert_response 403
         end
       end
+
+      should "not allow creators to vote on their own BURs" do
+        assert_difference("ForumPostVote.count", 0) do
+          post_auth forum_post_votes_path(format: :js), @bulk_update_request.user, params: { forum_post_id: @forum_post.id, forum_post_vote: { score: 1 }}
+          assert_response 403
+        end
+      end
     end
 
     context "destroy action" do

test/functional/mock_services_controller_test.rb

@@ -0,0 +1,28 @@
+require 'test_helper'
+
+class MockServicesControllerTest < ActionDispatch::IntegrationTest
+  context "The mock services controller" do
+    setup do
+      create(:post)
+      create(:tag)
+    end
+
+    context "for all actions" do
+      should "work" do
+        paths = [
+          mock_recommender_recommend_path(42),
+          mock_recommender_similar_path(42),
+          mock_reportbooru_missed_searches_path,
+          mock_reportbooru_post_searches_path,
+          mock_reportbooru_post_views_path,
+          mock_iqdbs_similar_path,
+        ]
+
+        paths.each do |path|
+          get path
+          assert_response :success
+        end
+      end
+    end
+  end
+end

test/functional/post_versions_controller_test.rb

@@ -41,6 +41,13 @@ class PostVersionsControllerTest < ActionDispatch::IntegrationTest
         assert_response :success
         assert_equal @post.versions[1].id, response.parsed_body[0]["id"].to_i
       end
+
+      should "list all versions for search[tag_matches]" do
+        get post_versions_path, as: :json, params: { search: { tag_matches: "tagme" }}
+        assert_response :success
+        assert_equal @post.versions[0].id, response.parsed_body[0]["id"].to_i
+        assert_equal 1, response.parsed_body.length
+      end
     end
 
     context "undo action" do

test/functional/uploads_controller_test.rb

@@ -1,15 +1,30 @@
 require 'test_helper'
 
 class UploadsControllerTest < ActionDispatch::IntegrationTest
-  def assert_uploaded(file_path, user, **upload_params)
-    file = Rack::Test::UploadedFile.new("#{Rails.root}/#{file_path}")
+  def self.should_upload_successfully(source)
+    should "upload successfully from #{source}" do
+      assert_successful_upload(source, user: create(:user, created_at: 1.month.ago))
+    end
+  end
 
-    assert_difference(["Upload.count", "Post.count"]) do
-      post_auth uploads_path, user, params: { upload: { file: file, **upload_params }}
-      assert_redirected_to Upload.last
+  def assert_successful_upload(source_or_file_path, user: @user, **params)
+    if source_or_file_path =~ %r{\Ahttps?://}i
+      source = { source: source_or_file_path }
+    else
+      file = Rack::Test::UploadedFile.new(Rails.root.join(source_or_file_path))
+      source = { file: file }
     end
 
-    Upload.last
+    assert_difference(["Upload.count"]) do
+      post_auth uploads_path, user, params: { upload: { tag_string: "abc", rating: "e", **source, **params }}
+    end
+
+    upload = Upload.last
+    assert_response :redirect
+    assert_redirected_to upload
+    assert_equal("completed", upload.status)
+    assert_equal(Post.last, upload.post)
+    assert_equal(upload.post.md5, upload.md5)
   end
 
   context "The uploads controller" do
@@ -18,6 +33,17 @@ class UploadsControllerTest < ActionDispatch::IntegrationTest
       mock_iqdb_service!
     end
 
+    context "image proxy action" do
+      should "work" do
+        url = "https://i.pximg.net/img-original/img/2017/11/21/17/06/44/65985331_p0.png"
+        get_auth image_proxy_uploads_path, @user, params: { url: url }
+
+        assert_response :success
+        assert_equal("image/png", response.media_type)
+        assert_equal(15_573, response.body.size)
+      end
+    end
+
     context "batch action" do
       context "for twitter galleries" do
         should "render" do
@@ -259,32 +285,65 @@ class UploadsControllerTest < ActionDispatch::IntegrationTest
       end
 
       context "uploading a file from your computer" do
-        should "work for a jpeg file" do
-          upload = assert_uploaded("test/files/test.jpg", @user, tag_string: "aaa", rating: "e", source: "aaa")
+        should_upload_successfully("test/files/test.jpg")
+        should_upload_successfully("test/files/test.png")
+        should_upload_successfully("test/files/test-static-32x32.gif")
+        should_upload_successfully("test/files/test-animated-86x52.gif")
+        should_upload_successfully("test/files/test-300x300.mp4")
+        should_upload_successfully("test/files/test-512x512.webm")
+        should_upload_successfully("test/files/compressed.swf")
+      end
 
-          assert_equal("jpg", upload.post.file_ext)
-          assert_equal("aaa", upload.post.source)
-          assert_equal(500, upload.post.image_width)
-          assert_equal(335, upload.post.image_height)
-        end
+      context "uploading a file from a source" do
+        should_upload_successfully("https://www.artstation.com/artwork/04XA4")
+        should_upload_successfully("https://dantewontdie.artstation.com/projects/YZK5q")
+        should_upload_successfully("https://cdna.artstation.com/p/assets/images/images/006/029/978/large/amama-l-z.jpg")
 
-        should "work for a webm file" do
-          upload = assert_uploaded("test/files/test-512x512.webm", @user, tag_string: "aaa", rating: "e", source: "aaa")
+        should_upload_successfully("https://www.deviantart.com/aeror404/art/Holiday-Elincia-424551484")
+        should_upload_successfully("https://noizave.deviantart.com/art/test-no-download-697415967")
+        should_upload_successfully("https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/8b472d70-a0d6-41b5-9a66-c35687090acc/d23jbr4-8a06af02-70cb-46da-8a96-42a6ba73cdb4.jpg/v1/fill/w_786,h_1017,q_70,strp/silverhawks_quicksilver_by_edsfox_d23jbr4-pre.jpg")
 
-          assert_equal("webm", upload.post.file_ext)
-          assert_equal("aaa", upload.post.source)
-          assert_equal(512, upload.post.image_width)
-          assert_equal(512, upload.post.image_height)
-        end
+        should_upload_successfully("https://www.hentai-foundry.com/pictures/user/Afrobull/795025/kuroeda")
+        should_upload_successfully("https://pictures.hentai-foundry.com/a/Afrobull/795025/Afrobull-795025-kuroeda.png")
 
-        should "work for a flash file" do
-          upload = assert_uploaded("test/files/compressed.swf", @user, tag_string: "aaa", rating: "e", source: "aaa")
+        should_upload_successfully("https://yande.re/post/show/482880")
+        should_upload_successfully("https://files.yande.re/image/7ecfdead705d7b956b26b1d37b98d089/yande.re%20482880.jpg")
 
-          assert_equal("swf", upload.post.file_ext)
-          assert_equal("aaa", upload.post.source)
-          assert_equal(607, upload.post.image_width)
-          assert_equal(756, upload.post.image_height)
-        end
+        should_upload_successfully("https://konachan.com/post/show/270916")
+        should_upload_successfully("https://konachan.com/image/ca12cdb79a66d242e95a6f958341bf05/Konachan.com%20-%20270916.png")
+
+        should_upload_successfully("http://lohas.nicoseiga.jp/o/910aecf08e542285862954017f8a33a8c32a8aec/1433298801/4937663")
+        should_upload_successfully("http://seiga.nicovideo.jp/seiga/im4937663")
+        should_upload_successfully("https://seiga.nicovideo.jp/image/source/9146749")
+        should_upload_successfully("https://seiga.nicovideo.jp/watch/mg389884")
+        should_upload_successfully("https://dic.nicovideo.jp/oekaki/52833.png")
+        should_upload_successfully("https://lohas.nicoseiga.jp/o/971eb8af9bbcde5c2e51d5ef3a2f62d6d9ff5552/1589933964/3583893")
+        should_upload_successfully("http://lohas.nicoseiga.jp/priv/3521156?e=1382558156&h=f2e089256abd1d453a455ec8f317a6c703e2cedf")
+        should_upload_successfully("http://lohas.nicoseiga.jp/priv/b80f86c0d8591b217e7513a9e175e94e00f3c7a1/1384936074/3583893")
+        should_upload_successfully("http://lohas.nicoseiga.jp/material/5746c5/4459092")
+        # XXX should_upload_successfully("https://dcdn.cdn.nimg.jp/priv/62a56a7f67d3d3746ae5712db9cac7d465f4a339/1592186183/10466669")
+        # XXX should_upload_successfully("https://dcdn.cdn.nimg.jp/nicoseiga/lohas/o/8ba0a9b2ea34e1ef3b5cc50785bd10cd63ec7e4a/1592187477/10466669")
+
+        should_upload_successfully("http://nijie.info/view.php?id=213043")
+        should_upload_successfully("https://nijie.info/view_popup.php?id=213043")
+        should_upload_successfully("https://pic.nijie.net/03/nijie_picture/728995_20170505014820_0.jpg")
+
+        should_upload_successfully("https://pawoo.net/web/statuses/1202176")
+        should_upload_successfully("https://img.pawoo.net/media_attachments/files/000/128/953/original/4c0a06087b03343f.png")
+
+        should_upload_successfully("https://www.pixiv.net/en/artworks/64476642")
+        should_upload_successfully("https://i.pximg.net/img-original/img/2017/08/18/00/09/21/64476642_p0.jpg")
+
+        should_upload_successfully("https://noizave.tumblr.com/post/162206271767")
+        should_upload_successfully("https://media.tumblr.com/3bbfcbf075ddf969c996641b264086fd/tumblr_os2buiIOt51wsfqepo1_1280.png")
+
+        should_upload_successfully("https://twitter.com/noizave/status/875768175136317440")
+        should_upload_successfully("https://pbs.twimg.com/media/DCdZ_FhUIAAYKFN?format=jpg&name=medium")
+        should_upload_successfully("https://pbs.twimg.com/profile_banners/1225702850002468864/1588597370/1500x500")
+        # XXX should_upload_successfully("https://video.twimg.com/tweet_video/EWHWVrmVcAAp4Vw.mp4")
+
+        should_upload_successfully("https://www.weibo.com/5501756072/J2UNKfbqV")
+        should_upload_successfully("https://wx1.sinaimg.cn/mw690/0060kO5aly1gezsyt5xvhj30ok0sgtc9.jpg")
       end
     end
   end

test/functional/wiki_pages_controller_test.rb

@@ -10,8 +10,11 @@ class WikiPagesControllerTest < ActionDispatch::IntegrationTest
     context "index action" do
       setup do
         as(@user) do
-          @wiki_page_abc = create(:wiki_page, :title => "abc")
-          @wiki_page_def = create(:wiki_page, :title => "def")
+          @tagme = create(:wiki_page, title: "tagme")
+          @deleted = create(:wiki_page, title: "deleted", is_deleted: true)
+          @vocaloid = create(:wiki_page, title: "vocaloid")
+          @miku = create(:wiki_page, title: "hatsune_miku", other_names: ["初音ミク"], body: "miku is a [[vocaloid]]")
+          create(:tag, name: "hatsune_miku", category: Tag.categories.character)
         end
       end
 
@@ -20,22 +23,24 @@ class WikiPagesControllerTest < ActionDispatch::IntegrationTest
         assert_response :success
       end
 
-      should "list all wiki_pages (with search)" do
-        get wiki_pages_path, params: {:search => {:title => "abc"}}
-        assert_response :success
-        assert_select "tr td:first-child", text: "abc"
-      end
-
-      should "list wiki_pages without tags with order=post_count" do
-        get wiki_pages_path, params: {:search => {:title => "abc", :order => "post_count"}}
-        assert_response :success
-        assert_select "tr td:first-child", text: "abc"
-      end
-
       should "redirect the legacy title param to the show page" do
-        get wiki_pages_path(title: "abc")
-        assert_redirected_to wiki_pages_path(search: { title_normalize: "abc" }, redirect: true)
+        get wiki_pages_path(title: "tagme")
+        assert_redirected_to wiki_pages_path(search: { title_normalize: "tagme" }, redirect: true)
       end
+
+      should respond_to_search(title: "tagme").with { @tagme }
+      should respond_to_search(title: "tagme", order: "post_count").with { @tagme }
+      should respond_to_search(title_normalize: "TAGME  ").with { @tagme }
+
+      should respond_to_search(tag: { category: Tag.categories.character }).with { @miku }
+      should respond_to_search(hide_deleted: "true").with { [@miku, @vocaloid, @tagme] }
+      should respond_to_search(linked_to: "vocaloid").with { @miku }
+      should respond_to_search(not_linked_to: "vocaloid").with { [@vocaloid, @deleted, @tagme] }
+
+      should respond_to_search(other_names_match: "初音ミク").with { @miku }
+      should respond_to_search(other_names_match: "初*").with { @miku }
+      should respond_to_search(other_names_present: "true").with { @miku }
+      should respond_to_search(other_names_present: "false").with { [@vocaloid, @deleted, @tagme] }
     end
 
     context "search action" do

test/test_helper.rb

@@ -43,15 +43,15 @@ class ActiveSupport::TestCase
 
   setup do
     Socket.stubs(:gethostname).returns("www.example.com")
-    WebMock.allow_net_connect!
 
-    storage_manager = StorageManager::Local.new(base_dir: Dir.mktmpdir("uploads-test-storage-"))
+    @temp_dir = Dir.mktmpdir("danbooru-temp-")
+    storage_manager = StorageManager::Local.new(base_dir: @temp_dir)
     Danbooru.config.stubs(:storage_manager).returns(storage_manager)
     Danbooru.config.stubs(:backup_storage_manager).returns(StorageManager::Null.new)
   end
 
   teardown do
-    FileUtils.rm_rf(Danbooru.config.storage_manager.base_dir)
+    FileUtils.rm_rf(@temp_dir)
     Cache.clear
   end
 
@@ -61,6 +61,8 @@ class ActiveSupport::TestCase
 end
 
 class ActionDispatch::IntegrationTest
+  extend ControllerHelper
+
   register_encoder :xml, response_parser: ->(body) { Nokogiri.XML(body) }
 
   def method_authenticated(method_name, url, user, **options)

test/test_helpers/controller_helper.rb

@@ -0,0 +1,47 @@
+module ControllerHelper
+  # A custom Shoulda matcher that tests that a controller's index endpoint
+  # responds to a search correctly. See https://thoughtbot.com/blog/shoulda-matchers.
+  #
+  # Usage:
+  #
+  #   # Tests that `/tags.json?search[name]=touhou` returns the `touhou` tag.
+  #   subject { TagsController }
+  #   setup { @touhou = create(:tag, name: "touhou") }
+  #   should respond_to_search(name: "touhou").with { @touhou }
+  #
+  def respond_to_search(search_params)
+    RespondToSearchMatcher.new(search_params)
+  end
+
+  class RespondToSearchMatcher < Struct.new(:params)
+    def description
+      "should respond to a search for #{params}"
+    end
+
+    def matches?(subject, &block)
+      search_params = { search: params }
+      expected_items = @test_case.instance_eval(&@expected)
+
+      @test_case.instance_eval do
+        # calls e.g. "wiki_pages_path" if we're in WikiPagesControllerTest.
+        index_url = send("#{subject.controller_path}_path")
+        get index_url, as: :json, params: search_params
+
+        expected_ids = Array(expected_items).map(&:id)
+        responded_ids = response.parsed_body.map { |item| item["id"] }
+
+        assert_response :success
+        assert_equal(expected_ids, responded_ids)
+      end
+    end
+
+    def with(&block)
+      @expected = block
+      self
+    end
+
+    def in_context(test_case)
+      @test_case = test_case
+    end
+  end
+end

test/test_helpers/download_test_helper.rb

@@ -1,10 +1,8 @@
 module DownloadTestHelper
   def assert_downloaded(expected_filesize, source, referer = nil)
-    download = Downloads::File.new(source, referer)
-    tempfile, strategy = download.download!
-    assert_equal(expected_filesize, tempfile.size, "Tested source URL: #{source}")
-  rescue Net::OpenTimeout
-    skip "Remote connection to #{source} failed"
+    strategy = Sources::Strategies.find(source, referer)
+    file = strategy.download_file!
+    assert_equal(expected_filesize, file.size, "Tested source URL: #{source}")
   end
 
   def assert_rewritten(expected_source, test_source, test_referer = nil)
@@ -16,19 +14,4 @@ module DownloadTestHelper
   def assert_not_rewritten(source, referer = nil)
     assert_rewritten(source, source, referer)
   end
-
-  def assert_http_exists(url, headers: {})
-    res = HTTParty.head(url, Danbooru.config.httparty_options.deep_merge(headers: headers))
-    assert_equal(true, res.success?)
-  end
-
-  def assert_http_status(code, url, headers: {})
-    res = HTTParty.head(url, Danbooru.config.httparty_options.deep_merge(headers: headers))
-    assert_equal(code, res.code)
-  end
-
-  def assert_http_size(size, url, headers: {})
-    res = HTTParty.head(url, Danbooru.config.httparty_options.deep_merge(headers: headers))
-    assert_equal(size, res.content_length)
-  end
 end

test/test_helpers/reportbooru_helper.rb

@@ -1,7 +1,7 @@
 module ReportbooruHelper
-  def mock_request(url, method: :get, status: 200, body: nil, http: Danbooru::Http.any_instance)
+  def mock_request(url, method: :get, status: 200, body: nil, http: Danbooru::Http.any_instance, **options)
     response = HTTP::Response.new(status: status, body: body, version: "1.1")
-    http.stubs(method).with(url).returns(response)
+    http.stubs(method).with(url, **options).returns(response)
   end
 
   def mock_post_search_rankings(date = Date.today, rankings)

test/unit/artist_test.rb

@@ -6,15 +6,11 @@ class ArtistTest < ActiveSupport::TestCase
 
     assert_equal(1, artists.size)
     assert_equal(expected_name, artists.first.name, "Testing URL: #{source_url}")
-  rescue Net::OpenTimeout, PixivApiClient::Error
-    skip "Remote connection failed for #{source_url}"
   end
 
   def assert_artist_not_found(source_url)
     artists = ArtistFinder.find_artists(source_url).to_a
     assert_equal(0, artists.size, "Testing URL: #{source_url}")
-  rescue Net::OpenTimeout
-    skip "Remote connection failed for #{source_url}"
   end
 
   context "An artist" do
@@ -172,15 +168,11 @@ class ArtistTest < ActiveSupport::TestCase
       a2 = FactoryBot.create(:artist, :name => "subway", :url_string => "http://subway.com/x/test.jpg")
       a3 = FactoryBot.create(:artist, :name => "minko", :url_string => "https://minko.com/x/test.jpg")
 
-      begin
-        assert_artist_found("rembrandt", "http://rembrandt.com/x/test.jpg")
-        assert_artist_found("rembrandt", "http://rembrandt.com/x/another.jpg")
-        assert_artist_not_found("http://nonexistent.com/test.jpg")
-        assert_artist_found("minko", "https://minko.com/x/test.jpg")
-        assert_artist_found("minko", "http://minko.com/x/test.jpg")
-      rescue Net::OpenTimeout
-        skip "network failure"
-      end
+      assert_artist_found("rembrandt", "http://rembrandt.com/x/test.jpg")
+      assert_artist_found("rembrandt", "http://rembrandt.com/x/another.jpg")
+      assert_artist_not_found("http://nonexistent.com/test.jpg")
+      assert_artist_found("minko", "https://minko.com/x/test.jpg")
+      assert_artist_found("minko", "http://minko.com/x/test.jpg")
     end
 
     should "be case-insensitive to domains when finding matches by url" do

test/unit/cloudflare_service_test.rb

@@ -1,5 +1,4 @@
 require 'test_helper'
-require 'webmock/minitest'
 
 class CloudflareServiceTest < ActiveSupport::TestCase
   def setup
@@ -8,16 +7,11 @@ class CloudflareServiceTest < ActiveSupport::TestCase
 
   context "#purge_cache" do
     should "make calls to cloudflare's api" do
-      stub_request(:any, "api.cloudflare.com")
-      @cloudflare.purge_cache(["http://localhost/file.txt"])
+      url = "http://www.example.com/file.jpg"
+      mock_request("https://api.cloudflare.com/client/v4/zones/123/purge_cache", method: :delete, json: { files: [url] })
 
-      assert_requested(:delete, "https://api.cloudflare.com/client/v4/zones/123/purge_cache", times: 1)
-    end
-  end
-
-  context "#ips" do
-    should "work" do
-      refute_empty(@cloudflare.ips)
+      response = @cloudflare.purge_cache([url])
+      assert_equal(200, response.status)
     end
   end
 end

test/unit/danbooru_http_test.rb

@@ -4,18 +4,20 @@ class DanbooruHttpTest < ActiveSupport::TestCase
   context "Danbooru::Http" do
     context "#get method" do
       should "work for all basic methods" do
-        %i[get put post delete].each do |method|
+        %i[get head put post delete].each do |method|
           response = Danbooru::Http.send(method, "https://httpbin.org/status/200")
           assert_equal(200, response.status)
         end
       end
 
       should "follow redirects" do
+        skip "Skipping test (https://github.com/postmanlabs/httpbin/issues/617)"
         response = Danbooru::Http.get("https://httpbin.org/absolute-redirect/3")
         assert_equal(200, response.status)
       end
 
       should "fail if redirected too many times" do
+        skip "Skipping test (https://github.com/postmanlabs/httpbin/issues/617)"
         response = Danbooru::Http.get("https://httpbin.org/absolute-redirect/10")
         assert_equal(598, response.status)
       end
@@ -26,8 +28,10 @@ class DanbooruHttpTest < ActiveSupport::TestCase
       end
 
       should "fail if the request takes too long to download" do
-        response = Danbooru::Http.timeout(1).get("https://httpbin.org/drip?duration=5&numbytes=5")
-        assert_equal(599, response.status)
+        # XXX should return status 599 instead
+        assert_raises(HTTP::TimeoutError) do
+          response = Danbooru::Http.timeout(1).get("https://httpbin.org/drip?duration=10&numbytes=10").flush
+        end
       end
 
       should "automatically decompress gzipped responses" do
@@ -36,13 +40,131 @@ class DanbooruHttpTest < ActiveSupport::TestCase
         assert_equal(true, response.parse["gzipped"])
       end
 
-      should "cache requests" do
-        response1 = Danbooru::Http.cache(1.minute).get("https://httpbin.org/uuid")
+      should "automatically parse html responses" do
+        response = Danbooru::Http.get("https://httpbin.org/html")
+        assert_equal(200, response.status)
+        assert_instance_of(Nokogiri::HTML5::Document, response.parse)
+        assert_equal("Herman Melville - Moby-Dick", response.parse.css("h1").text)
+      end
+
+      should "automatically parse xml responses" do
+        response = Danbooru::Http.get("https://httpbin.org/xml")
+        assert_equal(200, response.status)
+        assert_equal(true, response.parse[:slideshow].present?)
+      end
+
+      should "track cookies between requests" do
+        http = Danbooru::Http.use(:session)
+
+        resp1 = http.get("https://httpbin.org/cookies/set/abc/1")
+        resp2 = http.get("https://httpbin.org/cookies/set/def/2")
+        resp3 = http.get("https://httpbin.org/cookies")
+        assert_equal({ abc: "1", def: "2" }, resp3.parse["cookies"].symbolize_keys)
+
+        resp4 = http.cookies(def: 3, ghi: 4).get("https://httpbin.org/cookies")
+        assert_equal({ abc: "1", def: "3", ghi: "4" }, resp4.parse["cookies"].symbolize_keys)
+      end
+    end
+
+    context "cache feature" do
+      should "cache multiple requests to the same url" do
+        http = Danbooru::Http.cache(1.hour)
+
+        response1 = http.get("https://httpbin.org/uuid")
         assert_equal(200, response1.status)
 
-        response2 = Danbooru::Http.cache(1.minute).get("https://httpbin.org/uuid")
+        response2 = http.get("https://httpbin.org/uuid")
         assert_equal(200, response2.status)
-        assert_equal(response2.body, response1.body)
+        assert_equal(response2.to_s, response1.to_s)
+      end
+
+      should "cache cookies correctly" do
+        http = Danbooru::Http.cache(1.hour)
+
+        resp1 = http.get("https://httpbin.org/cookies")
+        resp2 = http.get("https://httpbin.org/cookies/set/abc/1")
+        resp3 = http.get("https://httpbin.org/cookies/set/def/2")
+        resp4 = http.get("https://httpbin.org/cookies")
+
+        assert_equal(200, resp1.status)
+        assert_equal(200, resp2.status)
+        assert_equal(200, resp3.status)
+        assert_equal(200, resp4.status)
+
+        assert_equal({}, resp1.parse["cookies"].symbolize_keys)
+        assert_equal({ abc: "1" }, resp2.parse["cookies"].symbolize_keys)
+        assert_equal({ abc: "1", def: "2" }, resp3.parse["cookies"].symbolize_keys)
+        assert_equal({ abc: "1", def: "2" }, resp4.parse["cookies"].symbolize_keys)
+      end
+    end
+
+    context "retriable feature" do
+      should "retry immediately if no Retry-After header is sent" do
+        response_429 = ::HTTP::Response.new(status: 429, version: "1.1", body: "")
+        response_200 = ::HTTP::Response.new(status: 200, version: "1.1", body: "")
+        HTTP::Client.any_instance.expects(:perform).times(2).returns(response_429, response_200)
+
+        response = Danbooru::Http.use(:retriable).get("https://httpbin.org/status/429")
+        assert_equal(200, response.status)
+      end
+
+      should "retry if the Retry-After header is an integer" do
+        response_503 = ::HTTP::Response.new(status: 503, version: "1.1", headers: { "Retry-After": "1" }, body: "")
+        response_200 = ::HTTP::Response.new(status: 200, version: "1.1", body: "")
+        HTTP::Client.any_instance.expects(:perform).times(2).returns(response_503, response_200)
+
+        response = Danbooru::Http.use(:retriable).get("https://httpbin.org/status/503")
+        assert_equal(200, response.status)
+      end
+
+      should "retry if the Retry-After header is a date" do
+        response_503 = ::HTTP::Response.new(status: 503, version: "1.1", headers: { "Retry-After": 2.seconds.from_now.httpdate }, body: "")
+        response_200 = ::HTTP::Response.new(status: 200, version: "1.1", body: "")
+        HTTP::Client.any_instance.expects(:perform).times(2).returns(response_503, response_200)
+
+        response = Danbooru::Http.use(:retriable).get("https://httpbin.org/status/503")
+        assert_equal(200, response.status)
+      end
+    end
+
+    context "#download method" do
+      should "download files" do
+        response, file = Danbooru::Http.download_media("https://httpbin.org/bytes/1000")
+
+        assert_equal(200, response.status)
+        assert_equal(1000, file.size)
+      end
+
+      should "follow redirects when downloading files" do
+        skip "Skipping test (https://github.com/postmanlabs/httpbin/issues/617)"
+        response, file = Danbooru::Http.download_media("https://httpbin.org/redirect-to?url=https://httpbin.org/bytes/1000")
+
+        assert_equal(200, response.status)
+        assert_equal(1000, file.size)
+      end
+
+      should "fail if the url points to a private IP" do
+        assert_raises(Danbooru::Http::DownloadError) do
+          Danbooru::Http.public_only.download_media("https://127.0.0.1.xip.io")
+        end
+      end
+
+      should "fail if the url redirects to a private IP" do
+        assert_raises(Danbooru::Http::DownloadError) do
+          Danbooru::Http.public_only.download_media("https://httpbin.org/redirect-to?url=https://127.0.0.1.xip.io")
+        end
+      end
+
+      should "fail if a download is too large" do
+        assert_raises(Danbooru::Http::FileTooLargeError) do
+          response, file = Danbooru::Http.max_size(500).download_media("https://httpbin.org/bytes/1000")
+        end
+      end
+
+      should "fail if a streaming download is too large" do
+        assert_raises(Danbooru::Http::FileTooLargeError) do
+          response, file = Danbooru::Http.max_size(500).download_media("https://httpbin.org/stream-bytes/1000")
+        end
       end
     end
   end

test/unit/downloads/art_station_test.rb

@@ -3,53 +3,27 @@ require 'test_helper'
 module Downloads
   class ArtStationTest < ActiveSupport::TestCase
     context "a download for a (small) artstation image" do
-      setup do
-        @asset = "https://cdnb3.artstation.com/p/assets/images/images/003/716/071/small/aoi-ogata-hate-city.jpg?1476754974"
-        @download = Downloads::File.new(@asset)
-      end
-
       should "download the /4k/ image instead" do
-        file, strategy = @download.download!
-        assert_equal(1_880_910, ::File.size(file.path))
+        assert_downloaded(1_880_910, "https://cdnb3.artstation.com/p/assets/images/images/003/716/071/small/aoi-ogata-hate-city.jpg?1476754974")
       end
     end
 
     context "for an image where an original does not exist" do
-      setup do
-        @asset = "https://cdna.artstation.com/p/assets/images/images/004/730/278/large/mendel-oh-dragonll.jpg"
-        @download = Downloads::File.new(@asset)
-      end
-
       should "not try to download the original" do
-        file, strategy = @download.download!
-        assert_equal(483_192, ::File.size(file.path))
+        assert_downloaded(483_192, "https://cdna.artstation.com/p/assets/images/images/004/730/278/large/mendel-oh-dragonll.jpg")
       end
     end
 
     context "a download for an ArtStation image hosted on CloudFlare" do
-      setup do
-        @asset = "https://cdnb.artstation.com/p/assets/images/images/003/716/071/large/aoi-ogata-hate-city.jpg?1476754974"
-      end
-
       should "return the original file, not the polished file" do
+        @asset = "https://cdnb.artstation.com/p/assets/images/images/003/716/071/large/aoi-ogata-hate-city.jpg?1476754974"
         assert_downloaded(1_880_910, @asset)
       end
-
-      should "return the original filesize, not the polished filesize" do
-        assert_equal(1_880_910, Downloads::File.new(@asset).size)
-      end
     end
 
     context "a download for a https://$artist.artstation.com/projects/$id page" do
-      setup do
-        @source = "https://dantewontdie.artstation.com/projects/YZK5q"
-        @download = Downloads::File.new(@source)
-      end
-
       should "download the original image instead" do
-        file, strategy = @download.download!
-
-        assert_equal(247_350, ::File.size(file.path))
+        assert_downloaded(247_350, "https://dantewontdie.artstation.com/projects/YZK5q")
       end
     end
   end

test/unit/downloads/file_test.rb

@@ -1,81 +0,0 @@
-require 'test_helper'
-
-module Downloads
-  class FileTest < ActiveSupport::TestCase
-    context "A post download" do
-      setup do
-        @source = "http://www.google.com/intl/en_ALL/images/logo.gif"
-        @download = Downloads::File.new(@source)
-      end
-
-      context "for a banned IP" do
-        setup do
-          Resolv.expects(:getaddress).returns("127.0.0.1").at_least_once
-        end
-
-        should "not try to download the file" do
-          assert_raise(Downloads::File::Error) { Downloads::File.new("http://evil.com").download! }
-        end
-
-        should "not try to fetch the size" do
-          assert_raise(Downloads::File::Error) { Downloads::File.new("http://evil.com").size }
-        end
-
-        should "not follow redirects to banned IPs" do
-          url = "http://httpbin.org/redirect-to?url=http://127.0.0.1"
-          stub_request(:get, url).to_return(status: 301, headers: { "Location": "http://127.0.0.1" })
-
-          assert_raise(Downloads::File::Error) { Downloads::File.new(url).download! }
-        end
-
-        should "not follow redirects that resolve to a banned IP" do
-          url = "http://httpbin.org/redirect-to?url=http://127.0.0.1.nip.io"
-          stub_request(:get, url).to_return(status: 301, headers: { "Location": "http://127.0.0.1.xip.io" })
-
-          assert_raise(Downloads::File::Error) { Downloads::File.new(url).download! }
-        end
-      end
-
-      context "that fails" do
-        should "retry three times before giving up" do
-          HTTParty.expects(:get).times(3).raises(Errno::ETIMEDOUT)
-          assert_raises(Errno::ETIMEDOUT) { @download.download! }
-        end
-
-        should "return an uncorrupted file on the second try" do
-          bomb = stub("bomb")
-          bomb.stubs(:code).raises(IOError)
-          resp = stub("resp", success?: true)
-
-          chunk = stub("a")
-          chunk.stubs(:code).returns(200)
-          chunk.stubs(:size).returns(1)
-          chunk.stubs(:to_s).returns("a")
-
-          HTTParty.expects(:get).twice.multiple_yields(chunk, bomb).then.multiple_yields(chunk, chunk).returns(resp)
-          @download.stubs(:is_cloudflare?).returns(false)
-          tempfile, _strategy = @download.download!
-
-          assert_equal("aa", tempfile.read)
-        end
-      end
-
-      should "throw an exception when the file is larger than the maximum" do
-        assert_raise(Downloads::File::Error) do
-          @download.download!(max_size: 1)
-        end
-      end
-
-      should "store the file in the tempfile path" do
-        tempfile, strategy = @download.download!
-        assert_operator(tempfile.size, :>, 0, "should have data")
-      end
-
-      should "correctly save the file when following 302 redirects" do
-        download = Downloads::File.new("https://yande.re/post/show/578014")
-        file, strategy = download.download!(url: download.preview_url)
-        assert_equal(19134, file.size)
-      end
-    end
-  end
-end

test/unit/downloads/pixiv_test.rb

@@ -122,32 +122,17 @@ module Downloads
           assert_downloaded(@file_size, @file_url, @ref)
         end
       end
-
-      context "downloading a pixiv fanbox image" do
-        should_eventually "work" do
-          @source = "https://www.pixiv.net/fanbox/creator/12491073/post/82406"
-          @file_url = "https://fanbox.pixiv.net/images/post/82406/D833IKA7FIesJXL8xx39rrG0.jpeg"
-          @file_size = 873_387
-
-          assert_not_rewritten(@file_url, @source)
-          assert_downloaded(@file_size, @file_url, @source)
-        end
-      end
     end
 
     context "An ugoira site for pixiv" do
-      setup do
-        @download = Downloads::File.new("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364")
-        @tempfile, strategy = @download.download!
-        @tempfile.close!
-      end
-
       should "capture the data" do
-        assert_equal(2, @download.data[:ugoira_frame_data].size)
-        if @download.data[:ugoira_frame_data][0]["file"]
+        @strategy = Sources::Strategies.find("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364")
+
+        assert_equal(2, @strategy.data[:ugoira_frame_data].size)
+        if @strategy.data[:ugoira_frame_data][0]["file"]
           assert_equal([{"file" => "000000.jpg", "delay" => 125}, {"file" => "000001.jpg", "delay" => 125}], @download.data[:ugoira_frame_data])
         else
-          assert_equal([{"delay_msec" => 125}, {"delay_msec" => 125}], @download.data[:ugoira_frame_data])
+          assert_equal([{"delay_msec" => 125}, {"delay_msec" => 125}], @strategy.data[:ugoira_frame_data])
         end
       end
     end

test/unit/media_file_test.rb

@@ -98,6 +98,10 @@ class MediaFileTest < ActiveSupport::TestCase
     should "determine the correct extension for a flash file" do
       assert_equal(:swf, MediaFile.open("test/files/compressed.swf").file_ext)
     end
+
+    should "not fail for empty files" do
+      assert_equal(:bin, MediaFile.open("test/files/test-empty.bin").file_ext)
+    end
   end
 
   should "determine the correct md5 for a jpeg file" do

test/unit/post_query_builder_test.rb

@@ -277,6 +277,19 @@ class PostQueryBuilderTest < ActiveSupport::TestCase
       assert_tag_match([child, parent], "-child:garbage")
     end
 
+    should "return posts when using the status of the parent/child" do
+      parent_of_deleted = create(:post)
+      deleted = create(:post, is_deleted: true, tag_string: "parent:#{parent_of_deleted.id}")
+      child_of_deleted = create(:post, tag_string: "parent:#{deleted.id}")
+      all = [child_of_deleted, deleted, parent_of_deleted]
+
+      assert_tag_match([child_of_deleted], "parent:deleted")
+      assert_tag_match(all - [child_of_deleted], "-parent:deleted")
+
+      assert_tag_match([parent_of_deleted], "child:deleted")
+      assert_tag_match(all - [parent_of_deleted], "-child:deleted")
+    end
+
     should "return posts for the favgroup:<name> metatag" do
       post1 = create(:post)
       post2 = create(:post)
@@ -757,8 +770,8 @@ class PostQueryBuilderTest < ActiveSupport::TestCase
       create(:saved_search, query: "aaa", labels: ["zzz"], user: CurrentUser.user)
       create(:saved_search, query: "bbb", user: CurrentUser.user)
 
-      Redis.any_instance.stubs(:exists).with("search:aaa").returns(true)
-      Redis.any_instance.stubs(:exists).with("search:bbb").returns(true)
+      Redis.any_instance.stubs(:exists?).with("search:aaa").returns(true)
+      Redis.any_instance.stubs(:exists?).with("search:bbb").returns(true)
       Redis.any_instance.stubs(:smembers).with("search:aaa").returns([@post1.id])
       Redis.any_instance.stubs(:smembers).with("search:bbb").returns([@post2.id])
 

test/unit/reportbooru_service_test.rb

@@ -4,20 +4,20 @@ class ReportbooruServiceTest < ActiveSupport::TestCase
   def setup
     @service = ReportbooruService.new(reportbooru_server: "http://localhost:1234")
     @post = create(:post)
-    @date = "2000-01-01"
+    @date = Date.parse("2000-01-01")
   end
 
   context "#popular_posts" do
     should "return the list of popular posts on success" do
-      body = "[[#{@post.id},100.0]]"
-      @service.http.expects(:get).with("http://localhost:1234/post_views/rank?date=#{@date}").returns(HTTP::Response.new(status: 200, body: body, version: "1.1"))
+      mock_post_view_rankings(@date, [[@post.id, 100]])
 
       posts = @service.popular_posts(@date)
       assert_equal([@post], posts)
     end
 
     should "return nothing on failure" do
-      @service.http.expects(:get).with("http://localhost:1234/post_views/rank?date=#{@date}").returns(HTTP::Response.new(status: 500, body: "", version: "1.1"))
+      Danbooru::Http.any_instance.expects(:get).with("http://localhost:1234/post_views/rank?date=#{@date}").returns(HTTP::Response.new(status: 500, body: "", version: "1.1"))
+      Danbooru::Http.any_instance.expects(:get).with("http://localhost:1234/post_views/rank?date=#{@date.yesterday}").returns(HTTP::Response.new(status: 500, body: "", version: "1.1"))
 
       assert_equal([], @service.popular_posts(@date))
     end

test/unit/sources/nijie_test.rb

@@ -43,7 +43,7 @@ module Sources
 
       should "get the image url" do
         assert_equal("https://pic.nijie.net/03/nijie_picture/728995_20170505014820_0.jpg", @site.image_url)
-        assert_http_size(132_555, @site.image_url)
+        assert_downloaded(132_555, @site.image_url)
       end
 
       should "get the canonical url" do
@@ -53,7 +53,7 @@ module Sources
       should "get the preview url" do
         assert_equal("https://pic.nijie.net/03/__rs_l170x170/nijie_picture/728995_20170505014820_0.jpg", @site.preview_url)
         assert_equal([@site.preview_url], @site.preview_urls)
-        assert_http_exists(@site.preview_url)
+        assert_downloaded(132_555, @site.preview_url)
       end
 
       should "get the profile" do
@@ -187,8 +187,6 @@ module Sources
         desc = <<-EOS.strip_heredoc.chomp
           foo [b]bold[/b] [i]italics[/i] [s]strike[/s] red
 
-
-
           <http://nijie.info/view.php?id=218944>
         EOS
 
@@ -207,8 +205,8 @@ module Sources
         assert_equal("https://nijie.info/members.php?id=236014", site.profile_url)
         assert_nothing_raised { site.to_h }
 
-        assert_http_size(3619, site.image_url)
-        assert_http_exists(site.preview_url)
+        assert_downloaded(3619, site.image_url)
+        assert_downloaded(3619, site.preview_url)
       end
     end
 

test/unit/sources/pixiv_test.rb

@@ -15,10 +15,7 @@ module Sources
 
     def get_source(source)
       @site = Sources::Strategies.find(source)
-
       @site
-    rescue Net::OpenTimeout
-      skip "Remote connection to #{source} failed"
     end
 
     context "in all cases" do
@@ -73,17 +70,6 @@ module Sources
         end
       end
 
-      context "A https://www.pixiv.net/fanbox/creator/*/post/* source" do
-        should_eventually "work" do
-          @site = Sources::Strategies.find("http://www.pixiv.net/fanbox/creator/554149/post/82555")
-
-          assert_equal("TYONE(お仕事募集中)", @site.artist_name)
-          assert_equal("https://www.pixiv.net/users/554149", @site.profile_url)
-          assert_equal("https://fanbox.pixiv.net/images/post/82555/Lyyeb6dDLcQZmy09nqLZapuS.jpeg", @site.image_url)
-          assert_nothing_raised { @site.to_h }
-        end
-      end
-
       context "A https://www.pixiv.net/*/artworks/* source" do
         should "work" do
           @site = Sources::Strategies.find("https://www.pixiv.net/en/artworks/64476642")
@@ -249,6 +235,10 @@ module Sources
           assert_includes(@translated_tags, "foo")
         end
 
+        should "not translate tags for digital media" do
+          assert_equal(false, @tags.include?("Photoshop"))
+        end
+
         should "normalize 10users入り tags" do
           assert_includes(@tags, "風景10users入り")
           assert_includes(@translated_tags, "scenery")
@@ -280,7 +270,7 @@ module Sources
         should "not translate '1000users入り' to '1'" do
           FactoryBot.create(:tag, name: "1", post_count: 1)
           source = get_source("https://www.pixiv.net/member_illust.php?mode=medium&illust_id=60665428")
-          tags = %w[1000users入り Fate/GrandOrder アルジュナ(Fate) アルトリア・ペンドラゴン イシュタル(Fate) グランブルーファンタジー マシュ・キリエライト マーリン(Fate) 両儀式 手袋 CLIP\ STUDIO\ PAINT Photoshop]
+          tags = %w[1000users入り Fate/GrandOrder アルジュナ(Fate) アルトリア・ペンドラゴン イシュタル(Fate) グランブルーファンタジー マシュ・キリエライト マーリン(Fate) 両儀式 手袋]
 
           assert_equal(tags.sort, source.tags.map(&:first).sort)
           assert_equal(["fate/grand_order"], source.translated_tags.map(&:name))

test/unit/sources/tumblr_test.rb

@@ -169,7 +169,7 @@ module Sources
 
     context "The source for a 'http://ve.media.tumblr.com/*' video post with inline images" do
       setup do
-        @url = "https://ve.media.tumblr.com/tumblr_os31dkexhK1wsfqep.mp4"
+        @url = "https://va.media.tumblr.com/tumblr_os31dkexhK1wsfqep.mp4"
         @ref = "https://noizave.tumblr.com/post/162222617101"
       end
 
@@ -177,7 +177,7 @@ module Sources
         should "get the video and inline images" do
           site = Sources::Strategies.find(@url, @ref)
           urls = %w[
-            https://ve.media.tumblr.com/tumblr_os31dkexhK1wsfqep.mp4
+            https://va.media.tumblr.com/tumblr_os31dkexhK1wsfqep.mp4
             https://media.tumblr.com/afed9f5b3c33c39dc8c967e262955de2/tumblr_inline_os31dclyCR1v11u29_1280.png
           ]
 

test/unit/sources/twitter_test.rb

@@ -244,6 +244,14 @@ module Sources
       end
     end
 
+    context "A profile banner image" do
+      should "work" do
+        @site = Sources::Strategies.find("https://pbs.twimg.com/profile_banners/1225702850002468864/1588597370/1500x500")
+        assert_equal(@site.image_url, @site.url)
+        assert_nothing_raised { @site.to_h }
+      end
+    end
+
     context "A tweet containing non-normalized Unicode text" do
       should "be normalized to nfkc" do
         site = Sources::Strategies.find("https://twitter.com/aprilarcus/status/367557195186970624")

test/unit/storage_manager_test.rb

@@ -1,8 +1,6 @@
 require 'test_helper'
 
 class StorageManagerTest < ActiveSupport::TestCase
-  BASE_DIR = "#{Rails.root}/tmp/test-storage"
-
   setup do
     CurrentUser.ip_addr = "127.0.0.1"
   end
@@ -45,25 +43,21 @@ class StorageManagerTest < ActiveSupport::TestCase
 
   context "StorageManager::Local" do
     setup do
-      @storage_manager = StorageManager::Local.new(base_dir: BASE_DIR, base_url: "/data")
-    end
-
-    teardown do
-      FileUtils.rm_rf(BASE_DIR)
+      @storage_manager = StorageManager::Local.new(base_dir: @temp_dir, base_url: "/data")
     end
 
     context "#store method" do
       should "store the file" do
-        @storage_manager.store(StringIO.new("data"), "#{BASE_DIR}/test.txt")
+        @storage_manager.store(StringIO.new("data"), "#{@temp_dir}/test.txt")
 
-        assert("data", File.read("#{BASE_DIR}/test.txt"))
+        assert("data", File.read("#{@temp_dir}/test.txt"))
       end
 
       should "overwrite the file if it already exists" do
-        @storage_manager.store(StringIO.new("foo"), "#{BASE_DIR}/test.txt")
-        @storage_manager.store(StringIO.new("bar"), "#{BASE_DIR}/test.txt")
+        @storage_manager.store(StringIO.new("foo"), "#{@temp_dir}/test.txt")
+        @storage_manager.store(StringIO.new("bar"), "#{@temp_dir}/test.txt")
 
-        assert("bar", File.read("#{BASE_DIR}/test.txt"))
+        assert("bar", File.read("#{@temp_dir}/test.txt"))
       end
     end
 
@@ -72,7 +66,7 @@ class StorageManagerTest < ActiveSupport::TestCase
         @storage_manager.store(StringIO.new("data"), "test.txt")
         @storage_manager.delete("test.txt")
 
-        assert_not(File.exist?("#{BASE_DIR}/test.txt"))
+        assert_not(File.exist?("#{@temp_dir}/test.txt"))
       end
 
       should "not fail if the file doesn't exist" do
@@ -88,9 +82,9 @@ class StorageManagerTest < ActiveSupport::TestCase
         @storage_manager.store_file(StringIO.new("data"), @post, :large)
         @storage_manager.store_file(StringIO.new("data"), @post, :original)
 
-        @file_path = "#{BASE_DIR}/preview/#{@post.md5}.jpg"
-        @large_file_path = "#{BASE_DIR}/sample/sample-#{@post.md5}.jpg"
-        @preview_file_path = "#{BASE_DIR}/#{@post.md5}.#{@post.file_ext}"
+        @file_path = "#{@temp_dir}/preview/#{@post.md5}.jpg"
+        @large_file_path = "#{@temp_dir}/sample/sample-#{@post.md5}.jpg"
+        @preview_file_path = "#{@temp_dir}/#{@post.md5}.#{@post.file_ext}"
       end
 
       should "store the files at the correct path" do
@@ -134,12 +128,12 @@ class StorageManagerTest < ActiveSupport::TestCase
     context "when the original_subdir option is used" do
       should "store original files at the correct path" do
         @post = FactoryBot.create(:post, file_ext: "png")
-        @storage_manager = StorageManager::Local.new(base_dir: BASE_DIR, base_url: "/data", original_subdir: "original/")
+        @storage_manager = StorageManager::Local.new(base_dir: @temp_dir, base_url: "/data", original_subdir: "original/")
 
-        assert_equal("#{BASE_DIR}/original/#{@post.md5}.png", @storage_manager.file_path(@post, @post.file_ext, :original))
+        assert_equal("#{@temp_dir}/original/#{@post.md5}.png", @storage_manager.file_path(@post, @post.file_ext, :original))
 
         @storage_manager.store_file(StringIO.new("data"), @post, :original)
-        assert_equal(true, File.exist?("#{BASE_DIR}/original/#{@post.md5}.png"))
+        assert_equal(true, File.exist?("#{@temp_dir}/original/#{@post.md5}.png"))
       end
     end
   end
@@ -151,24 +145,20 @@ class StorageManagerTest < ActiveSupport::TestCase
 
       @storage_manager = StorageManager::Hybrid.new do |id, md5, file_ext, type|
         if id.odd?
-          StorageManager::Local.new(base_dir: "#{BASE_DIR}/i1", base_url: "/i1")
+          StorageManager::Local.new(base_dir: "#{@temp_dir}/i1", base_url: "/i1")
         else
-          StorageManager::Local.new(base_dir: "#{BASE_DIR}/i2", base_url: "/i2")
+          StorageManager::Local.new(base_dir: "#{@temp_dir}/i2", base_url: "/i2")
         end
       end
     end
 
-    teardown do
-      FileUtils.rm_rf(BASE_DIR)
-    end
-
     context "#store_file method" do
       should "store odd-numbered posts under /i1 and even-numbered posts under /i2" do
         @storage_manager.store_file(StringIO.new("post1"), @post1, :original)
         @storage_manager.store_file(StringIO.new("post2"), @post2, :original)
 
-        assert(File.exist?("#{BASE_DIR}/i1/#{@post1.md5}.png"))
-        assert(File.exist?("#{BASE_DIR}/i2/#{@post2.md5}.png"))
+        assert(File.exist?("#{@temp_dir}/i1/#{@post1.md5}.png"))
+        assert(File.exist?("#{@temp_dir}/i2/#{@post2.md5}.png"))
       end
     end
 

test/unit/upload_service_test.rb

@@ -131,12 +131,9 @@ class UploadServiceTest < ActiveSupport::TestCase
         end
 
         should "download the file" do
-          begin
-            @service = UploadService::Preprocessor.new(source: @source, referer_url: @ref)
-            @upload = @service.start!
-          rescue Net::OpenTimeout
-            skip "network failure"
-          end
+          @service = UploadService::Preprocessor.new(source: @source, referer_url: @ref)
+          @upload = @service.start!
+
           assert_equal("preprocessed", @upload.status)
           assert_equal(294591, @upload.file_size)
           assert_equal("jpg", @upload.file_ext)
@@ -155,11 +152,8 @@ class UploadServiceTest < ActiveSupport::TestCase
           skip unless MediaFile::Ugoira.videos_enabled?
 
           @service = UploadService::Preprocessor.new(source: @source)
-          begin
-            @upload = @service.start!
-          rescue Net::OpenTimeout
-            skip "network problems"
-          end
+          @upload = @service.start!
+
           assert_equal("preprocessed", @upload.status)
           assert_equal(2804, @upload.file_size)
           assert_equal("zip", @upload.file_ext)
@@ -176,11 +170,8 @@ class UploadServiceTest < ActiveSupport::TestCase
 
         should "download the file" do
           @service = UploadService::Preprocessor.new(source: @source)
-          begin
-            @upload = @service.start!
-          rescue Net::OpenTimeout
-            skip "network problems"
-          end
+          @upload = @service.start!
+
           assert_equal("preprocessed", @upload.status)
           assert_equal(181309, @upload.file_size)
           assert_equal("jpg", @upload.file_ext)
@@ -212,7 +203,7 @@ class UploadServiceTest < ActiveSupport::TestCase
       context "on timeout errors" do
         setup do
           @source = "https://cdn.donmai.us/original/93/f4/93f4dd66ef1eb11a89e56d31f9adc8d0.jpg"
-          HTTParty.stubs(:get).raises(Net::ReadTimeout)
+          Danbooru::Http.any_instance.stubs(:get).raises(HTTP::TimeoutError)
         end
 
         should "leave the upload in an error state" do
@@ -512,36 +503,28 @@ class UploadServiceTest < ActiveSupport::TestCase
 
       context "a post with a pixiv html source" do
         should "replace with the full size image" do
-          begin
-            as(@user) do
-              @post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350")
-            end
-
-            assert_equal(80, @post.image_width)
-            assert_equal(82, @post.image_height)
-            assert_equal(16275, @post.file_size)
-            assert_equal("png", @post.file_ext)
-            assert_equal("4ceadc314938bc27f3574053a3e1459a", @post.md5)
-            assert_equal("4ceadc314938bc27f3574053a3e1459a", Digest::MD5.file(@post.file).hexdigest)
-            assert_equal("https://i.pximg.net/img-original/img/2017/04/04/08/54/15/62247350_p0.png", @post.replacements.last.replacement_url)
-            assert_equal("https://i.pximg.net/img-original/img/2017/04/04/08/54/15/62247350_p0.png", @post.source)
-          rescue Net::OpenTimeout
-            skip "Remote connection to Pixiv failed"
+          as(@user) do
+            @post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350")
           end
+
+          assert_equal(80, @post.image_width)
+          assert_equal(82, @post.image_height)
+          assert_equal(16275, @post.file_size)
+          assert_equal("png", @post.file_ext)
+          assert_equal("4ceadc314938bc27f3574053a3e1459a", @post.md5)
+          assert_equal("4ceadc314938bc27f3574053a3e1459a", Digest::MD5.file(@post.file).hexdigest)
+          assert_equal("https://i.pximg.net/img-original/img/2017/04/04/08/54/15/62247350_p0.png", @post.replacements.last.replacement_url)
+          assert_equal("https://i.pximg.net/img-original/img/2017/04/04/08/54/15/62247350_p0.png", @post.source)
         end
 
         should "delete the old files after thirty days" do
-          begin
-            @post.unstub(:queue_delete_files)
-            FileUtils.expects(:rm_f).times(3)
+          @post.unstub(:queue_delete_files)
+          FileUtils.expects(:rm_f).times(3)
 
-            as(@user) { @post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350") }
+          as(@user) { @post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350") }
 
-            travel_to((PostReplacement::DELETION_GRACE_PERIOD + 1).days.from_now) do
-              perform_enqueued_jobs
-            end
-          rescue Net::OpenTimeout
-            skip "Remote connection to Pixiv failed"
+          travel_to((PostReplacement::DELETION_GRACE_PERIOD + 1).days.from_now) do
+            perform_enqueued_jobs
           end
         end
       end
@@ -568,34 +551,30 @@ class UploadServiceTest < ActiveSupport::TestCase
 
       context "a post that is replaced to another file then replaced back to the original file" do
         should "not delete the original files" do
-          begin
-            skip unless MediaFile::Ugoira.videos_enabled?
-            @post.unstub(:queue_delete_files)
+          skip unless MediaFile::Ugoira.videos_enabled?
+          @post.unstub(:queue_delete_files)
 
-            # this is called thrice to delete the file for 62247364
-            FileUtils.expects(:rm_f).times(3)
+          # this is called thrice to delete the file for 62247364
+          FileUtils.expects(:rm_f).times(3)
 
-            as(@user) do
-              @post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350")
-              @post.reload
-              @post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364")
-              @post.reload
-              Upload.destroy_all
-              @post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350")
-            end
-
-            assert_nothing_raised { @post.file(:original) }
-            assert_nothing_raised { @post.file(:preview) }
-
-            assert_enqueued_jobs 3, only: DeletePostFilesJob
-            travel PostReplacement::DELETION_GRACE_PERIOD + 1.day
-            assert_raise(Post::DeletionError) { perform_enqueued_jobs }
-
-            assert_nothing_raised { @post.file(:original) }
-            assert_nothing_raised { @post.file(:preview) }
-          rescue Net::OpenTimeout
-            skip "Remote connection to Pixiv failed"
+          as(@user) do
+            @post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350")
+            @post.reload
+            @post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364")
+            @post.reload
+            Upload.destroy_all
+            @post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350")
           end
+
+          assert_nothing_raised { @post.file(:original) }
+          assert_nothing_raised { @post.file(:preview) }
+
+          assert_enqueued_jobs 3, only: DeletePostFilesJob
+          travel PostReplacement::DELETION_GRACE_PERIOD + 1.day
+          assert_raise(Post::DeletionError) { perform_enqueued_jobs }
+
+          assert_nothing_raised { @post.file(:original) }
+          assert_nothing_raised { @post.file(:preview) }
         end
       end
 
@@ -609,39 +588,35 @@ class UploadServiceTest < ActiveSupport::TestCase
 
         should "not delete the still active files" do
           # swap the images between @post1 and @post2.
-          begin
-            as(@user) do
-              skip unless MediaFile::Ugoira.videos_enabled?
+          as(@user) do
+            skip unless MediaFile::Ugoira.videos_enabled?
 
-              @post1.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350")
-              @post2.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364")
-              assert_equal("4ceadc314938bc27f3574053a3e1459a", @post1.md5)
-              assert_equal("cad1da177ef309bf40a117c17b8eecf5", @post2.md5)
+            @post1.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350")
+            @post2.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364")
+            assert_equal("4ceadc314938bc27f3574053a3e1459a", @post1.md5)
+            assert_equal("cad1da177ef309bf40a117c17b8eecf5", @post2.md5)
 
-              @post2.reload
-              @post2.replace!(replacement_url: "https://cdn.donmai.us/original/d3/4e/d34e4cf0a437a5d65f8e82b7bcd02606.jpg")
-              assert_equal("d34e4cf0a437a5d65f8e82b7bcd02606", @post2.md5)
-              Upload.destroy_all
-              @post1.reload
-              @post2.reload
+            @post2.reload
+            @post2.replace!(replacement_url: "https://cdn.donmai.us/original/d3/4e/d34e4cf0a437a5d65f8e82b7bcd02606.jpg")
+            assert_equal("d34e4cf0a437a5d65f8e82b7bcd02606", @post2.md5)
+            Upload.destroy_all
+            @post1.reload
+            @post2.reload
 
-              @post1.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364")
-              @post2.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350")
-              assert_equal("cad1da177ef309bf40a117c17b8eecf5", @post1.md5)
-              assert_equal("4ceadc314938bc27f3574053a3e1459a", @post2.md5)
-            end
-
-            travel_to (PostReplacement::DELETION_GRACE_PERIOD + 1).days.from_now do
-              assert_raise(Post::DeletionError) do
-                perform_enqueued_jobs
-              end
-            end
-
-            assert_nothing_raised { @post1.file(:original) }
-            assert_nothing_raised { @post2.file(:original) }
-          rescue Net::OpenTimeout
-            skip "Remote connection to Pixiv failed"
+            @post1.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364")
+            @post2.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350")
+            assert_equal("cad1da177ef309bf40a117c17b8eecf5", @post1.md5)
+            assert_equal("4ceadc314938bc27f3574053a3e1459a", @post2.md5)
           end
+
+          travel_to (PostReplacement::DELETION_GRACE_PERIOD + 1).days.from_now do
+            assert_raise(Post::DeletionError) do
+              perform_enqueued_jobs
+            end
+          end
+
+          assert_nothing_raised { @post1.file(:original) }
+          assert_nothing_raised { @post2.file(:original) }
         end
       end
 
@@ -885,12 +860,8 @@ class UploadServiceTest < ActiveSupport::TestCase
       end
 
       should "record the canonical source" do
-        begin
-          post = subject.new({}).create_post_from_upload(@upload)
-          assert_equal(@source, post.source)
-        rescue Net::OpenTimeout
-          skip "network failure"
-        end
+        post = subject.new({}).create_post_from_upload(@upload)
+        assert_equal(@source, post.source)
       end
     end