tests: fix broken tests.

app/logical/source/extractor/moebooru.rb

@@ -66,6 +66,10 @@ module Source
           elsif api_response[:file_url].present?
             api_response[:file_url][/\.(jpg|jpeg|png|gif)\z/i, 1]
 
+          # on yande.re, the file extension doesn't matter, any random string will work. This means we can't guess the true file extension.
+          elsif domain == "yande.re"
+            "png"
+
           # the api_response wasn't available because it's a deleted post.
           elsif post_md5.present?
             %w[jpg png gif].find { |ext| http_exists?("https://#{domain}/image/#{post_md5}.#{ext}") }

app/models/media_asset.rb

@@ -246,6 +246,8 @@ class MediaAsset < ApplicationRecord
       # This can't be called inside a transaction because the transaction will
       # fail if there's a RecordNotUnique error when the asset already exists.
       def upload!(media_file, &block)
+        media_file = MediaFile.open(media_file) unless media_file.is_a?(MediaFile)
+
         raise Error, "File is corrupt" if media_file.is_corrupt?
 
         media_asset = create!(file: media_file, status: :processing)

test/functional/artists_controller_test.rb

@@ -70,8 +70,7 @@ class ArtistsControllerTest < ActionDispatch::IntegrationTest
         @artist = create(:artist, is_banned: true)
         get artist_path(@artist.id)
 
-        assert_response :success
+        assert_response 451
-        assert_select "meta[name=robots][content=noindex]"
       end
     end
 

test/functional/moderator/post/posts_controller_test.rb

@@ -11,7 +11,7 @@ module Moderator
           end
 
           as(@user) do
-            @post = create(:post)
+            @post = create(:post_with_file)
           end
         end
 

test/test_helpers/source_test_helper.rb

@@ -51,7 +51,7 @@ module SourceTestHelper
       should "correctly match a strategy to an artist with the same profile url" do
         assert_not_nil(Danbooru::URL.parse(strategy.profile_url))
         assert_equal(profile_url, strategy.profile_url)
-        artist = FactoryBot.create(:artist, name: strategy.artist_name, url_string: profile_url)
+        artist = FactoryBot.create(:artist, name: strategy.tag_name, url_string: profile_url)
         assert_equal([artist], strategy.artists)
       end
     else

test/unit/downloads/art_station_test.rb

@@ -1,32 +0,0 @@
-require 'test_helper'
-
-module Downloads
-  class ArtStationTest < ActiveSupport::TestCase
-    context "a download for a (small) artstation image" do
-      should "download the /4k/ image instead" do
-        assert_downloaded(1_816_438, "https://cdnb3.artstation.com/p/assets/images/images/003/716/071/small/aoi-ogata-hate-city.jpg?1476754974")
-      end
-    end
-
-    context "for an image where an original does not exist" do
-      should "not try to download the original" do
-        skip if ENV["CI"].present?
-
-        assert_downloaded(452_795, "https://cdna.artstation.com/p/assets/images/images/004/730/278/large/mendel-oh-dragonll.jpg")
-      end
-    end
-
-    context "a download for an ArtStation image hosted on CloudFlare" do
-      should "return the original file, not the polished file" do
-        @asset = "https://cdnb.artstation.com/p/assets/images/images/003/716/071/large/aoi-ogata-hate-city.jpg?1476754974"
-        assert_downloaded(1_816_438, @asset)
-      end
-    end
-
-    context "a download for a https://$artist.artstation.com/projects/$id page" do
-      should "download the original image instead" do
-        assert_downloaded(210_709, "https://dantewontdie.artstation.com/projects/YZK5q")
-      end
-    end
-  end
-end

test/unit/downloads/tumblr_test.rb

@@ -61,30 +61,6 @@ module Downloads
       end
     end
 
-    context "a download for a media.tumblr.com/$id_$size image with a larger size" do
-      should "download the best available version" do
-        skip "Tumblr keys are not set" unless Danbooru.config.tumblr_consumer_key
-        @ref = "https://noizave.tumblr.com/post/162206271767"
-        @source = "http://media.tumblr.com/0DNBGJovY5j3smfeQs8nB53z_400.jpg"
-        @rewrite = "https://media.tumblr.com/0DNBGJovY5j3smfeQs8nB53z_1280.jpg"
-        assert_rewritten(@rewrite, @source, @ref)
-        assert_downloaded(125850, @source)
-        # assert_downloaded(153_885, @source)
-      end
-    end
-
-    context "a download for a media.tumblr.com/tumblr_$id_$size.jpg image" do
-      should "download the best available version" do
-        skip "Tumblr keys are not set" unless Danbooru.config.tumblr_consumer_key
-        @ref = "https://noizave.tumblr.com/post/162206271767"
-        @source = "http://media.tumblr.com/tumblr_m24kbxqKAX1rszquso1_250.jpg"
-        @rewrite = "https://media.tumblr.com/tumblr_m24kbxqKAX1rszquso1_1280.jpg"
-        assert_rewritten(@rewrite, @source, @ref)
-        assert_downloaded(105963, @source, @ref)
-        # assert_downloaded(296_399, @source)
-      end
-    end
-
     context "a download for a *.tumblr.com/post/* html page" do
       should "download the best available version" do
         skip "Tumblr keys are not set" unless Danbooru.config.tumblr_consumer_key

test/unit/favorite_group_test.rb

@@ -1,6 +1,6 @@
 require 'test_helper'
 
-class FavoriteTest < ActiveSupport::TestCase
+class FavoriteGroupTest < ActiveSupport::TestCase
   def setup
     @fav_group = create(:favorite_group)
   end
@@ -22,7 +22,7 @@ class FavoriteTest < ActiveSupport::TestCase
 
   context "expunging a post" do
     should "remove it from all favorite groups" do
-      @post = FactoryBot.create(:post)
+      @post = create(:post_with_file, filename: "test.jpg")
 
       @fav_group.add(@post)
       assert_equal([@post.id], @fav_group.post_ids)

test/unit/ip_geolocation_test.rb

@@ -52,7 +52,7 @@ class IpGeolocationTest < ActiveSupport::TestCase
 
       should "work for a mobile IP" do
         @ip = IpGeolocation.create_or_update!("37.173.153.166")
-        assert_equal("Free Mobile SAS", @ip.carrier)
+        assert_match(/Free Mobile/, @ip.carrier)
       end
 
       should "work for a proxy IP" do

test/unit/post_query_builder_test.rb

@@ -1164,9 +1164,9 @@ class PostQueryBuilderTest < ActiveSupport::TestCase
     end
 
     should "return posts for an exif:<value> metatag" do
-      jpg = create(:post, media_asset: create(:media_asset, file: "test/files/test.jpg"))
+      jpg = create(:post_with_file, filename: "test.jpg")
-      gif = create(:post, media_asset: create(:media_asset, file: "test/files/test.gif"))
+      gif = create(:post_with_file, filename: "test.gif")
-      png = create(:post, media_asset: create(:media_asset, file: "test/files/test.png"))
+      png = create(:post_with_file, filename: "test.png")
 
       assert_tag_match([jpg], "exif:File:ColorComponents")
       assert_tag_match([jpg], "exif:File:ColorComponents=3")
@@ -1536,7 +1536,7 @@ class PostQueryBuilderTest < ActiveSupport::TestCase
         end
 
         should "not fail for a two tag search by a member" do
-          post1 = create(:post, tag_string: "aaa bbb rating:s")
+          post1 = create(:post, tag_string: "aaa bbb rating:g")
           post2 = create(:post, tag_string: "aaa bbb rating:e")
 
           assert_fast_count(1, "aaa bbb", { safe_mode: true })

test/unit/post_test.rb

@@ -1112,7 +1112,7 @@ class PostTest < ActiveSupport::TestCase
 
       context "an animated gif missing the animated_gif tag" do
         should "automatically add the animated_gif tag" do
-          @media_asset = MediaAsset.create!(file: "test/files/test-animated-86x52.gif")
+          @media_asset = MediaAsset.upload!("test/files/test-animated-86x52.gif")
           @post.update!(md5: @media_asset.md5)
           @post.reload.update!(tag_string: "tagme")
           assert_equal("animated animated_gif tagme", @post.tag_string)
@@ -1121,7 +1121,7 @@ class PostTest < ActiveSupport::TestCase
 
       context "an animated png missing the animated_png tag" do
         should "automatically add the animated_png tag" do
-          @media_asset = MediaAsset.create!(file: "test/files/test-animated-256x256.png")
+          @media_asset = MediaAsset.upload!("test/files/test-animated-256x256.png")
           @post.update!(md5: @media_asset.md5)
           @post.reload.update!(tag_string: "tagme")
           assert_equal("animated animated_png tagme", @post.tag_string)
@@ -1130,7 +1130,7 @@ class PostTest < ActiveSupport::TestCase
 
       context "a greyscale image missing the greyscale tag" do
         should "automatically add the greyscale tag" do
-          @media_asset = MediaAsset.create!(file: "test/files/test-grey-no-profile.jpg")
+          @media_asset = MediaAsset.upload!("test/files/test-grey-no-profile.jpg")
           @post.update!(md5: @media_asset.md5)
           @post.reload.update!(tag_string: "tagme")
           assert_equal("greyscale tagme", @post.tag_string)
@@ -1139,7 +1139,7 @@ class PostTest < ActiveSupport::TestCase
 
       context "an exif-rotated image missing the exif_rotation tag" do
         should "automatically add the exif_rotation tag" do
-          @media_asset = MediaAsset.create!(file: "test/files/test-rotation-90cw.jpg")
+          @media_asset = MediaAsset.upload!("test/files/test-rotation-90cw.jpg")
           @post.update!(md5: @media_asset.md5)
           @post.reload.update!(tag_string: "tagme")
           assert_equal("exif_rotation tagme", @post.tag_string)
@@ -1148,7 +1148,7 @@ class PostTest < ActiveSupport::TestCase
 
       context "a PNG with the exif orientation flag" do
         should "not add the exif_rotation tag" do
-          @media_asset = MediaAsset.create!(file: "test/files/test-rotation-90cw.png")
+          @media_asset = MediaAsset.upload!("test/files/test-rotation-90cw.png")
           @post.update!(md5: @media_asset.md5)
           @post.reload.update!(tag_string: "tagme")
           assert_equal("tagme", @post.tag_string)
@@ -1157,12 +1157,12 @@ class PostTest < ActiveSupport::TestCase
 
       context "a non-repeating GIF missing the non-repeating_animation tag" do
         should "automatically add the non-repeating_animation tag" do
-          @media_asset = MediaAsset.create!(file: "test/files/test-animated-86x52-loop-1.gif")
+          @media_asset = MediaAsset.upload!("test/files/test-animated-86x52-loop-1.gif")
           @post.update!(md5: @media_asset.md5)
           @post.reload.update!(tag_string: "tagme")
           assert_equal("animated animated_gif non-repeating_animation tagme", @post.tag_string)
 
-          @media_asset = MediaAsset.create!(file: "test/files/test-animated-86x52-loop-2.gif")
+          @media_asset = MediaAsset.upload!("test/files/test-animated-86x52-loop-2.gif")
           @post.update!(md5: @media_asset.md5)
           @post.reload.update!(tag_string: "tagme")
           assert_equal("animated animated_gif non-repeating_animation tagme", @post.tag_string)

test/unit/sources/art_station_test.rb

@@ -33,37 +33,18 @@ module Sources
       end
     end
 
-    context "The source site for an art station projects page" do
+    context "An ArtStation /projects/ URL" do
-      setup do
+      strategy_should_work(
-        @site = Source::Extractor.find("https://dantewontdie.artstation.com/projects/YZK5q")
+        "https://dantewontdie.artstation.com/projects/YZK5q",
-      end
+        image_urls: ["https://cdn.artstation.com/p/assets/images/images/006/066/534/4k/yinan-cui-reika.jpg?1495781565"],
-
+        page_url: "https://dantewontdie.artstation.com/projects/YZK5q",
-      should "get the image url" do
+        profile_url: "https://www.artstation.com/dantewontdie",
-        url = "https://cdn.artstation.com/p/assets/images/images/006/066/534/4k/yinan-cui-reika.jpg?1495781565"
+        artist_name: "dantewontdie",
-        assert_equal([url], @site.image_urls)
+        tags: %w[gantz Reika],
-      end
+        artist_commentary_title: "Reika ",
-
+        dtext_artist_commentary_desc: "From Gantz.",
-      should "get the page url" do
+        download_size: 210_899,
-        assert_equal("https://dantewontdie.artstation.com/projects/YZK5q", @site.page_url)
+      )
-      end
-
-      should "get the profile" do
-        assert_equal("https://www.artstation.com/dantewontdie", @site.profile_url)
-      end
-
-      should "get the artist name" do
-        assert_equal("dantewontdie", @site.artist_name)
-      end
-
-      should "get the tags" do
-        assert_equal(%w[gantz Reika], @site.tags.map(&:first))
-        assert_equal(%w[gantz reika], @site.normalized_tags)
-      end
-
-      should "get the artist commentary" do
-        assert_equal("Reika ", @site.artist_commentary_title)
-        assert_equal("From Gantz.", @site.dtext_artist_commentary_desc)
-      end
     end
 
     context "The source site for a www.artstation.com/artwork/$slug page" do
@@ -182,6 +163,30 @@ module Sources
       end
     end
 
+    context "A /small/ ArtStation image URL" do
+      strategy_should_work(
+        "https://cdnb3.artstation.com/p/assets/images/images/003/716/071/small/aoi-ogata-hate-city.jpg?1476754974",
+        image_urls: ["https://cdn.artstation.com/p/assets/images/images/003/716/071/4k/aoi-ogata-hate-city.jpg?1476754974"],
+        download_size: 1_816_628,
+      )
+    end
+
+    context "A /large/ ArtStation image URL (1)" do
+      strategy_should_work(
+        "https://cdnb.artstation.com/p/assets/images/images/003/716/071/large/aoi-ogata-hate-city.jpg?1476754974",
+        image_urls: ["https://cdn.artstation.com/p/assets/images/images/003/716/071/4k/aoi-ogata-hate-city.jpg?1476754974"],
+        download_size: 1_816_628,
+      )
+    end
+
+    context "A /large/ ArtStation image URL (2)" do
+      strategy_should_work(
+        "https://cdna.artstation.com/p/assets/images/images/004/730/278/large/mendel-oh-dragonll.jpg",
+        image_urls: ["https://cdn.artstation.com/p/assets/images/images/004/730/278/4k/mendel-oh-dragonll.jpg"],
+        download_size: 452_985,
+      )
+    end
+
     should "work for artists with underscores in their name" do
       site = Source::Extractor.find("https://hosi_na.artstation.com/projects/3oEk3B")
       assert_equal("hosi_na", site.artist_name)

test/unit/sources/deviant_art_test.rb

@@ -211,9 +211,16 @@ module Sources
 
     context "A source with malformed links in the artist commentary" do
       should "fix the links" do
-        @site = Source::Extractor.find("https://teemutaiga.deviantart.com/art/Kisu-620666655")
+        @site = Source::Extractor.find("https://www.deviantart.com/dishwasher1910/art/Solar-Sisters-792488305")
 
-        assert_match(%r{"Print available at Inprnt":\[http://www.inprnt.com/gallery/teemutaiga/kisu\]}, @site.dtext_artist_commentary_desc)
+        assert_equal(<<~EOS.chomp, @site.dtext_artist_commentary_desc)
+          Solar sisters 
+
+          HD images , Psd file and alternative version available on my Patreon :
+          "www.patreon.com/Dishwasher1910":[https://www.patreon.com/Dishwasher1910]
+          You can buy the print here :
+          "www.inprnt.com/gallery/dishwas…":[https://www.inprnt.com/gallery/dishwasher1910/solar-sisters/]
+        EOS
       end
     end
 

test/unit/sources/nijie_test.rb

@@ -38,43 +38,20 @@ module Sources
       end
     end
 
-    context "The source site for a nijie page" do
+    context "A Nijie page" do
-      setup do
+      strategy_should_work(
-        CurrentUser.user = FactoryBot.create(:user)
+        "https://nijie.info/view.php?id=213043",
-        CurrentUser.ip_addr = "127.0.0.1"
+        image_urls: ["https://pic.nijie.net/07/nijie/17/95/728995/illust/0_0_403fdd541191110c_c25585.jpg"],
-
+        download_size: 132_555,
-        @site = Source::Extractor.find("https://nijie.info/view.php?id=213043")
+        artist_name: "莚",
-      end
+        profile_url: "https://nijie.info/members.php?id=728995",
-
+        artist_commentary_title: "ジャージの下は",
-      should "get the image url" do
+        dtext_artist_commentary_desc: "「リトルウィッチアカデミア」から無自覚サキュバスぶりを発揮するアーシュラ先生です",
-        assert_equal(["https://pic.nijie.net/07/nijie/17/95/728995/illust/0_0_403fdd541191110c_c25585.jpg"], @site.image_urls)
+        tags: %w[眼鏡 リトルウィッチアカデミア アーシュラ先生],
-        assert_downloaded(132_555, @site.image_urls.sole)
+      )
-      end
+    end
-
-      should "get the page url" do
-        assert_equal("https://nijie.info/view.php?id=213043", @site.page_url)
-      end
-
-      should "get the profile" do
-        assert_equal("https://nijie.info/members.php?id=728995", @site.profile_url)
-      end
-
-      should "get the artist name" do
-        assert_equal("莚", @site.artist_name)
-      end
-
-      should "get the tags" do
-        tags = [
-          ["眼鏡", "https://nijie.info/search.php?word=%E7%9C%BC%E9%8F%A1"],
-          ["谷間", "https://nijie.info/search.php?word=%E8%B0%B7%E9%96%93"],
-          ["リトルウィッチアカデミア", "https://nijie.info/search.php?word=%E3%83%AA%E3%83%88%E3%83%AB%E3%82%A6%E3%82%A3%E3%83%83%E3%83%81%E3%82%A2%E3%82%AB%E3%83%87%E3%83%9F%E3%82%A2"],
-          ["アーシュラ先生", "https://nijie.info/search.php?word=%E3%82%A2%E3%83%BC%E3%82%B7%E3%83%A5%E3%83%A9%E5%85%88%E7%94%9F"],
-          ["上着全開", "https://nijie.info/search.php?word=%E4%B8%8A%E7%9D%80%E5%85%A8%E9%96%8B"]
-        ]
-
-        assert_equal(tags, @site.tags)
-      end
 
+    context "A Nijie post" do
       should "normalize （）characters in tags" do
         FactoryBot.create(:tag, :name => "kaga")
         FactoryBot.create(:wiki_page, :title => "kaga", :other_names => "加賀(艦これ)")
@@ -84,14 +61,6 @@ module Sources
         assert_includes(@site.tags.map(&:first), "加賀（艦これ）")
         assert_includes(@site.translated_tags.map(&:name), "kaga")
       end
-
-      should "get the commentary" do
-        title = "ジャージの下は"
-        desc = "「リトルウィッチアカデミア」から無自覚サキュバスぶりを発揮するアーシュラ先生です"
-
-        assert_equal(title, @site.dtext_artist_commentary_title)
-        assert_equal(desc, @site.dtext_artist_commentary_desc)
-      end
     end
 
     context "For long commentaries that may be truncated" do

test/unit/sources/tinami_test.rb

@@ -56,7 +56,7 @@ module Sources
           assert_equal(<<~EOS.chomp, source.artist_commentary_desc)
             リゼロのレムのプライズをクリアドレス仕様にリペイント。透け透けキラキラな感じに改装してみたものです。
 
-            >https://youtu.be/nkjZkEALg94
+            >https://youtu.be/nkjZkEALg94 
 
             製作日記的な動画です( ´∀｀ )
 

test/unit/sources/tumblr_test.rb

@@ -227,6 +227,24 @@ module Sources
       end
     end
 
+    context "A *.media.tumblr.com/$hash/tumblr_$id_$size.png URL with a referer" do
+      strategy_should_work(
+        "https://64.media.tumblr.com/3bbfcbf075ddf969c996641b264086fd/tumblr_os2buiIOt51wsfqepo1_400.png",
+        referer: "https://noizave.tumblr.com/post/162206271767",
+        image_urls: ["https://media.tumblr.com/3bbfcbf075ddf969c996641b264086fd/tumblr_os2buiIOt51wsfqepo1_1280.png"],
+        download_size: 3655,
+      )
+    end
+
+    context "A *.media.tumblr.com/tumblr_$id_$size.jpg URL with a referer" do
+      strategy_should_work(
+        "http://media.tumblr.com/tumblr_m24kbxqKAX1rszquso1_250.jpg",
+        referer: "https://noizave.tumblr.com/post/162206271767",
+        image_urls: ["https://media.tumblr.com/tumblr_m24kbxqKAX1rszquso1_1280.jpg"],
+        download_size: 105_963,
+      )
+    end
+
     context "generating page urls" do
       should "work" do
         source1 = "https://octrain1020.tumblr.com/post/190713122589"