diff --git a/app/logical/source/extractor/moebooru.rb b/app/logical/source/extractor/moebooru.rb index eac927752..30171da47 100644 --- a/app/logical/source/extractor/moebooru.rb +++ b/app/logical/source/extractor/moebooru.rb @@ -66,6 +66,10 @@ module Source elsif api_response[:file_url].present? api_response[:file_url][/\.(jpg|jpeg|png|gif)\z/i, 1] + # on yande.re, the file extension doesn't matter, any random string will work. This means we can't guess the true file extension. + elsif domain == "yande.re" + "png" + # the api_response wasn't available because it's a deleted post. elsif post_md5.present? %w[jpg png gif].find { |ext| http_exists?("https://#{domain}/image/#{post_md5}.#{ext}") } diff --git a/app/models/media_asset.rb b/app/models/media_asset.rb index f8b113b03..14cad07d3 100644 --- a/app/models/media_asset.rb +++ b/app/models/media_asset.rb @@ -246,6 +246,8 @@ class MediaAsset < ApplicationRecord # This can't be called inside a transaction because the transaction will # fail if there's a RecordNotUnique error when the asset already exists. def upload!(media_file, &block) + media_file = MediaFile.open(media_file) unless media_file.is_a?(MediaFile) + raise Error, "File is corrupt" if media_file.is_corrupt? media_asset = create!(file: media_file, status: :processing) diff --git a/test/functional/artists_controller_test.rb b/test/functional/artists_controller_test.rb index 8452d5e8e..79dc088b9 100644 --- a/test/functional/artists_controller_test.rb +++ b/test/functional/artists_controller_test.rb @@ -70,8 +70,7 @@ class ArtistsControllerTest < ActionDispatch::IntegrationTest @artist = create(:artist, is_banned: true) get artist_path(@artist.id) - assert_response :success - assert_select "meta[name=robots][content=noindex]" + assert_response 451 end end diff --git a/test/functional/moderator/post/posts_controller_test.rb b/test/functional/moderator/post/posts_controller_test.rb index 6aff77f78..1a79e0e2f 100644 --- a/test/functional/moderator/post/posts_controller_test.rb +++ b/test/functional/moderator/post/posts_controller_test.rb @@ -11,7 +11,7 @@ module Moderator end as(@user) do - @post = create(:post) + @post = create(:post_with_file) end end diff --git a/test/test_helpers/source_test_helper.rb b/test/test_helpers/source_test_helper.rb index f1ed8521d..b8d3e379a 100644 --- a/test/test_helpers/source_test_helper.rb +++ b/test/test_helpers/source_test_helper.rb @@ -51,7 +51,7 @@ module SourceTestHelper should "correctly match a strategy to an artist with the same profile url" do assert_not_nil(Danbooru::URL.parse(strategy.profile_url)) assert_equal(profile_url, strategy.profile_url) - artist = FactoryBot.create(:artist, name: strategy.artist_name, url_string: profile_url) + artist = FactoryBot.create(:artist, name: strategy.tag_name, url_string: profile_url) assert_equal([artist], strategy.artists) end else diff --git a/test/unit/downloads/art_station_test.rb b/test/unit/downloads/art_station_test.rb deleted file mode 100644 index 33fe75a6b..000000000 --- a/test/unit/downloads/art_station_test.rb +++ /dev/null @@ -1,32 +0,0 @@ -require 'test_helper' - -module Downloads - class ArtStationTest < ActiveSupport::TestCase - context "a download for a (small) artstation image" do - should "download the /4k/ image instead" do - assert_downloaded(1_816_438, "https://cdnb3.artstation.com/p/assets/images/images/003/716/071/small/aoi-ogata-hate-city.jpg?1476754974") - end - end - - context "for an image where an original does not exist" do - should "not try to download the original" do - skip if ENV["CI"].present? - - assert_downloaded(452_795, "https://cdna.artstation.com/p/assets/images/images/004/730/278/large/mendel-oh-dragonll.jpg") - end - end - - context "a download for an ArtStation image hosted on CloudFlare" do - should "return the original file, not the polished file" do - @asset = "https://cdnb.artstation.com/p/assets/images/images/003/716/071/large/aoi-ogata-hate-city.jpg?1476754974" - assert_downloaded(1_816_438, @asset) - end - end - - context "a download for a https://$artist.artstation.com/projects/$id page" do - should "download the original image instead" do - assert_downloaded(210_709, "https://dantewontdie.artstation.com/projects/YZK5q") - end - end - end -end diff --git a/test/unit/downloads/tumblr_test.rb b/test/unit/downloads/tumblr_test.rb index 35f6f23dd..59112351b 100644 --- a/test/unit/downloads/tumblr_test.rb +++ b/test/unit/downloads/tumblr_test.rb @@ -61,30 +61,6 @@ module Downloads end end - context "a download for a media.tumblr.com/$id_$size image with a larger size" do - should "download the best available version" do - skip "Tumblr keys are not set" unless Danbooru.config.tumblr_consumer_key - @ref = "https://noizave.tumblr.com/post/162206271767" - @source = "http://media.tumblr.com/0DNBGJovY5j3smfeQs8nB53z_400.jpg" - @rewrite = "https://media.tumblr.com/0DNBGJovY5j3smfeQs8nB53z_1280.jpg" - assert_rewritten(@rewrite, @source, @ref) - assert_downloaded(125850, @source) - # assert_downloaded(153_885, @source) - end - end - - context "a download for a media.tumblr.com/tumblr_$id_$size.jpg image" do - should "download the best available version" do - skip "Tumblr keys are not set" unless Danbooru.config.tumblr_consumer_key - @ref = "https://noizave.tumblr.com/post/162206271767" - @source = "http://media.tumblr.com/tumblr_m24kbxqKAX1rszquso1_250.jpg" - @rewrite = "https://media.tumblr.com/tumblr_m24kbxqKAX1rszquso1_1280.jpg" - assert_rewritten(@rewrite, @source, @ref) - assert_downloaded(105963, @source, @ref) - # assert_downloaded(296_399, @source) - end - end - context "a download for a *.tumblr.com/post/* html page" do should "download the best available version" do skip "Tumblr keys are not set" unless Danbooru.config.tumblr_consumer_key diff --git a/test/unit/favorite_group_test.rb b/test/unit/favorite_group_test.rb index 09743eb65..ac496ba6f 100644 --- a/test/unit/favorite_group_test.rb +++ b/test/unit/favorite_group_test.rb @@ -1,6 +1,6 @@ require 'test_helper' -class FavoriteTest < ActiveSupport::TestCase +class FavoriteGroupTest < ActiveSupport::TestCase def setup @fav_group = create(:favorite_group) end @@ -22,7 +22,7 @@ class FavoriteTest < ActiveSupport::TestCase context "expunging a post" do should "remove it from all favorite groups" do - @post = FactoryBot.create(:post) + @post = create(:post_with_file, filename: "test.jpg") @fav_group.add(@post) assert_equal([@post.id], @fav_group.post_ids) diff --git a/test/unit/ip_geolocation_test.rb b/test/unit/ip_geolocation_test.rb index 0060a8df4..4d5c74d39 100644 --- a/test/unit/ip_geolocation_test.rb +++ b/test/unit/ip_geolocation_test.rb @@ -52,7 +52,7 @@ class IpGeolocationTest < ActiveSupport::TestCase should "work for a mobile IP" do @ip = IpGeolocation.create_or_update!("37.173.153.166") - assert_equal("Free Mobile SAS", @ip.carrier) + assert_match(/Free Mobile/, @ip.carrier) end should "work for a proxy IP" do diff --git a/test/unit/post_query_builder_test.rb b/test/unit/post_query_builder_test.rb index fa1b79c0a..66fe4603a 100644 --- a/test/unit/post_query_builder_test.rb +++ b/test/unit/post_query_builder_test.rb @@ -1164,9 +1164,9 @@ class PostQueryBuilderTest < ActiveSupport::TestCase end should "return posts for an exif: metatag" do - jpg = create(:post, media_asset: create(:media_asset, file: "test/files/test.jpg")) - gif = create(:post, media_asset: create(:media_asset, file: "test/files/test.gif")) - png = create(:post, media_asset: create(:media_asset, file: "test/files/test.png")) + jpg = create(:post_with_file, filename: "test.jpg") + gif = create(:post_with_file, filename: "test.gif") + png = create(:post_with_file, filename: "test.png") assert_tag_match([jpg], "exif:File:ColorComponents") assert_tag_match([jpg], "exif:File:ColorComponents=3") @@ -1536,7 +1536,7 @@ class PostQueryBuilderTest < ActiveSupport::TestCase end should "not fail for a two tag search by a member" do - post1 = create(:post, tag_string: "aaa bbb rating:s") + post1 = create(:post, tag_string: "aaa bbb rating:g") post2 = create(:post, tag_string: "aaa bbb rating:e") assert_fast_count(1, "aaa bbb", { safe_mode: true }) diff --git a/test/unit/post_test.rb b/test/unit/post_test.rb index 8f2ccf322..ce2745427 100644 --- a/test/unit/post_test.rb +++ b/test/unit/post_test.rb @@ -1112,7 +1112,7 @@ class PostTest < ActiveSupport::TestCase context "an animated gif missing the animated_gif tag" do should "automatically add the animated_gif tag" do - @media_asset = MediaAsset.create!(file: "test/files/test-animated-86x52.gif") + @media_asset = MediaAsset.upload!("test/files/test-animated-86x52.gif") @post.update!(md5: @media_asset.md5) @post.reload.update!(tag_string: "tagme") assert_equal("animated animated_gif tagme", @post.tag_string) @@ -1121,7 +1121,7 @@ class PostTest < ActiveSupport::TestCase context "an animated png missing the animated_png tag" do should "automatically add the animated_png tag" do - @media_asset = MediaAsset.create!(file: "test/files/test-animated-256x256.png") + @media_asset = MediaAsset.upload!("test/files/test-animated-256x256.png") @post.update!(md5: @media_asset.md5) @post.reload.update!(tag_string: "tagme") assert_equal("animated animated_png tagme", @post.tag_string) @@ -1130,7 +1130,7 @@ class PostTest < ActiveSupport::TestCase context "a greyscale image missing the greyscale tag" do should "automatically add the greyscale tag" do - @media_asset = MediaAsset.create!(file: "test/files/test-grey-no-profile.jpg") + @media_asset = MediaAsset.upload!("test/files/test-grey-no-profile.jpg") @post.update!(md5: @media_asset.md5) @post.reload.update!(tag_string: "tagme") assert_equal("greyscale tagme", @post.tag_string) @@ -1139,7 +1139,7 @@ class PostTest < ActiveSupport::TestCase context "an exif-rotated image missing the exif_rotation tag" do should "automatically add the exif_rotation tag" do - @media_asset = MediaAsset.create!(file: "test/files/test-rotation-90cw.jpg") + @media_asset = MediaAsset.upload!("test/files/test-rotation-90cw.jpg") @post.update!(md5: @media_asset.md5) @post.reload.update!(tag_string: "tagme") assert_equal("exif_rotation tagme", @post.tag_string) @@ -1148,7 +1148,7 @@ class PostTest < ActiveSupport::TestCase context "a PNG with the exif orientation flag" do should "not add the exif_rotation tag" do - @media_asset = MediaAsset.create!(file: "test/files/test-rotation-90cw.png") + @media_asset = MediaAsset.upload!("test/files/test-rotation-90cw.png") @post.update!(md5: @media_asset.md5) @post.reload.update!(tag_string: "tagme") assert_equal("tagme", @post.tag_string) @@ -1157,12 +1157,12 @@ class PostTest < ActiveSupport::TestCase context "a non-repeating GIF missing the non-repeating_animation tag" do should "automatically add the non-repeating_animation tag" do - @media_asset = MediaAsset.create!(file: "test/files/test-animated-86x52-loop-1.gif") + @media_asset = MediaAsset.upload!("test/files/test-animated-86x52-loop-1.gif") @post.update!(md5: @media_asset.md5) @post.reload.update!(tag_string: "tagme") assert_equal("animated animated_gif non-repeating_animation tagme", @post.tag_string) - @media_asset = MediaAsset.create!(file: "test/files/test-animated-86x52-loop-2.gif") + @media_asset = MediaAsset.upload!("test/files/test-animated-86x52-loop-2.gif") @post.update!(md5: @media_asset.md5) @post.reload.update!(tag_string: "tagme") assert_equal("animated animated_gif non-repeating_animation tagme", @post.tag_string) diff --git a/test/unit/sources/art_station_test.rb b/test/unit/sources/art_station_test.rb index 2adfa1ce9..57bbd0a87 100644 --- a/test/unit/sources/art_station_test.rb +++ b/test/unit/sources/art_station_test.rb @@ -33,37 +33,18 @@ module Sources end end - context "The source site for an art station projects page" do - setup do - @site = Source::Extractor.find("https://dantewontdie.artstation.com/projects/YZK5q") - end - - should "get the image url" do - url = "https://cdn.artstation.com/p/assets/images/images/006/066/534/4k/yinan-cui-reika.jpg?1495781565" - assert_equal([url], @site.image_urls) - end - - should "get the page url" do - assert_equal("https://dantewontdie.artstation.com/projects/YZK5q", @site.page_url) - end - - should "get the profile" do - assert_equal("https://www.artstation.com/dantewontdie", @site.profile_url) - end - - should "get the artist name" do - assert_equal("dantewontdie", @site.artist_name) - end - - should "get the tags" do - assert_equal(%w[gantz Reika], @site.tags.map(&:first)) - assert_equal(%w[gantz reika], @site.normalized_tags) - end - - should "get the artist commentary" do - assert_equal("Reika ", @site.artist_commentary_title) - assert_equal("From Gantz.", @site.dtext_artist_commentary_desc) - end + context "An ArtStation /projects/ URL" do + strategy_should_work( + "https://dantewontdie.artstation.com/projects/YZK5q", + image_urls: ["https://cdn.artstation.com/p/assets/images/images/006/066/534/4k/yinan-cui-reika.jpg?1495781565"], + page_url: "https://dantewontdie.artstation.com/projects/YZK5q", + profile_url: "https://www.artstation.com/dantewontdie", + artist_name: "dantewontdie", + tags: %w[gantz Reika], + artist_commentary_title: "Reika ", + dtext_artist_commentary_desc: "From Gantz.", + download_size: 210_899, + ) end context "The source site for a www.artstation.com/artwork/$slug page" do @@ -182,6 +163,30 @@ module Sources end end + context "A /small/ ArtStation image URL" do + strategy_should_work( + "https://cdnb3.artstation.com/p/assets/images/images/003/716/071/small/aoi-ogata-hate-city.jpg?1476754974", + image_urls: ["https://cdn.artstation.com/p/assets/images/images/003/716/071/4k/aoi-ogata-hate-city.jpg?1476754974"], + download_size: 1_816_628, + ) + end + + context "A /large/ ArtStation image URL (1)" do + strategy_should_work( + "https://cdnb.artstation.com/p/assets/images/images/003/716/071/large/aoi-ogata-hate-city.jpg?1476754974", + image_urls: ["https://cdn.artstation.com/p/assets/images/images/003/716/071/4k/aoi-ogata-hate-city.jpg?1476754974"], + download_size: 1_816_628, + ) + end + + context "A /large/ ArtStation image URL (2)" do + strategy_should_work( + "https://cdna.artstation.com/p/assets/images/images/004/730/278/large/mendel-oh-dragonll.jpg", + image_urls: ["https://cdn.artstation.com/p/assets/images/images/004/730/278/4k/mendel-oh-dragonll.jpg"], + download_size: 452_985, + ) + end + should "work for artists with underscores in their name" do site = Source::Extractor.find("https://hosi_na.artstation.com/projects/3oEk3B") assert_equal("hosi_na", site.artist_name) diff --git a/test/unit/sources/deviant_art_test.rb b/test/unit/sources/deviant_art_test.rb index ab6f7a71f..adbd46fc0 100644 --- a/test/unit/sources/deviant_art_test.rb +++ b/test/unit/sources/deviant_art_test.rb @@ -211,9 +211,16 @@ module Sources context "A source with malformed links in the artist commentary" do should "fix the links" do - @site = Source::Extractor.find("https://teemutaiga.deviantart.com/art/Kisu-620666655") + @site = Source::Extractor.find("https://www.deviantart.com/dishwasher1910/art/Solar-Sisters-792488305") - assert_match(%r{"Print available at Inprnt":\[http://www.inprnt.com/gallery/teemutaiga/kisu\]}, @site.dtext_artist_commentary_desc) + assert_equal(<<~EOS.chomp, @site.dtext_artist_commentary_desc) + Solar sisters + + HD images , Psd file and alternative version available on my Patreon : + "www.patreon.com/Dishwasher1910":[https://www.patreon.com/Dishwasher1910] + You can buy the print here : + "www.inprnt.com/gallery/dishwas…":[https://www.inprnt.com/gallery/dishwasher1910/solar-sisters/] + EOS end end diff --git a/test/unit/sources/nijie_test.rb b/test/unit/sources/nijie_test.rb index ad6814499..f09eb4835 100644 --- a/test/unit/sources/nijie_test.rb +++ b/test/unit/sources/nijie_test.rb @@ -38,43 +38,20 @@ module Sources end end - context "The source site for a nijie page" do - setup do - CurrentUser.user = FactoryBot.create(:user) - CurrentUser.ip_addr = "127.0.0.1" - - @site = Source::Extractor.find("https://nijie.info/view.php?id=213043") - end - - should "get the image url" do - assert_equal(["https://pic.nijie.net/07/nijie/17/95/728995/illust/0_0_403fdd541191110c_c25585.jpg"], @site.image_urls) - assert_downloaded(132_555, @site.image_urls.sole) - end - - should "get the page url" do - assert_equal("https://nijie.info/view.php?id=213043", @site.page_url) - end - - should "get the profile" do - assert_equal("https://nijie.info/members.php?id=728995", @site.profile_url) - end - - should "get the artist name" do - assert_equal("莚", @site.artist_name) - end - - should "get the tags" do - tags = [ - ["眼鏡", "https://nijie.info/search.php?word=%E7%9C%BC%E9%8F%A1"], - ["谷間", "https://nijie.info/search.php?word=%E8%B0%B7%E9%96%93"], - ["リトルウィッチアカデミア", "https://nijie.info/search.php?word=%E3%83%AA%E3%83%88%E3%83%AB%E3%82%A6%E3%82%A3%E3%83%83%E3%83%81%E3%82%A2%E3%82%AB%E3%83%87%E3%83%9F%E3%82%A2"], - ["アーシュラ先生", "https://nijie.info/search.php?word=%E3%82%A2%E3%83%BC%E3%82%B7%E3%83%A5%E3%83%A9%E5%85%88%E7%94%9F"], - ["上着全開", "https://nijie.info/search.php?word=%E4%B8%8A%E7%9D%80%E5%85%A8%E9%96%8B"] - ] - - assert_equal(tags, @site.tags) - end + context "A Nijie page" do + strategy_should_work( + "https://nijie.info/view.php?id=213043", + image_urls: ["https://pic.nijie.net/07/nijie/17/95/728995/illust/0_0_403fdd541191110c_c25585.jpg"], + download_size: 132_555, + artist_name: "莚", + profile_url: "https://nijie.info/members.php?id=728995", + artist_commentary_title: "ジャージの下は", + dtext_artist_commentary_desc: "「リトルウィッチアカデミア」から無自覚サキュバスぶりを発揮するアーシュラ先生です", + tags: %w[眼鏡 リトルウィッチアカデミア アーシュラ先生], + ) + end + context "A Nijie post" do should "normalize ()characters in tags" do FactoryBot.create(:tag, :name => "kaga") FactoryBot.create(:wiki_page, :title => "kaga", :other_names => "加賀(艦これ)") @@ -84,14 +61,6 @@ module Sources assert_includes(@site.tags.map(&:first), "加賀(艦これ)") assert_includes(@site.translated_tags.map(&:name), "kaga") end - - should "get the commentary" do - title = "ジャージの下は" - desc = "「リトルウィッチアカデミア」から無自覚サキュバスぶりを発揮するアーシュラ先生です" - - assert_equal(title, @site.dtext_artist_commentary_title) - assert_equal(desc, @site.dtext_artist_commentary_desc) - end end context "For long commentaries that may be truncated" do diff --git a/test/unit/sources/tinami_test.rb b/test/unit/sources/tinami_test.rb index 5a531967d..79056f709 100644 --- a/test/unit/sources/tinami_test.rb +++ b/test/unit/sources/tinami_test.rb @@ -56,7 +56,7 @@ module Sources assert_equal(<<~EOS.chomp, source.artist_commentary_desc) リゼロのレムのプライズをクリアドレス仕様にリペイント。透け透けキラキラな感じに改装してみたものです。 - >https://youtu.be/nkjZkEALg94 + >https://youtu.be/nkjZkEALg94 製作日記的な動画です( ´∀` ) diff --git a/test/unit/sources/tumblr_test.rb b/test/unit/sources/tumblr_test.rb index f847628b0..35db0c9d5 100644 --- a/test/unit/sources/tumblr_test.rb +++ b/test/unit/sources/tumblr_test.rb @@ -227,6 +227,24 @@ module Sources end end + context "A *.media.tumblr.com/$hash/tumblr_$id_$size.png URL with a referer" do + strategy_should_work( + "https://64.media.tumblr.com/3bbfcbf075ddf969c996641b264086fd/tumblr_os2buiIOt51wsfqepo1_400.png", + referer: "https://noizave.tumblr.com/post/162206271767", + image_urls: ["https://media.tumblr.com/3bbfcbf075ddf969c996641b264086fd/tumblr_os2buiIOt51wsfqepo1_1280.png"], + download_size: 3655, + ) + end + + context "A *.media.tumblr.com/tumblr_$id_$size.jpg URL with a referer" do + strategy_should_work( + "http://media.tumblr.com/tumblr_m24kbxqKAX1rszquso1_250.jpg", + referer: "https://noizave.tumblr.com/post/162206271767", + image_urls: ["https://media.tumblr.com/tumblr_m24kbxqKAX1rszquso1_1280.jpg"], + download_size: 105_963, + ) + end + context "generating page urls" do should "work" do source1 = "https://octrain1020.tumblr.com/post/190713122589"