jaculabilis/danbooru
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'master' into fix-pixiv-profile-url

Browse Source
This commit is contained in:
evazion
2020-06-24 00:06:55 -05:00
committed by GitHub
parent 158a4aa916 8ef2b7772d
commit 185693b99b
103 changed files with 1639 additions and 2247 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
test/unit/artist_test.rb
View File

@@ -6,15 +6,11 @@ class ArtistTest < ActiveSupport::TestCase
assert_equal(1, artists.size)
assert_equal(expected_name, artists.first.name, "Testing URL: #{source_url}")
rescue Net::OpenTimeout, PixivApiClient::Error
skip "Remote connection failed for #{source_url}"
end
def assert_artist_not_found(source_url)
artists = ArtistFinder.find_artists(source_url).to_a
assert_equal(0, artists.size, "Testing URL: #{source_url}")
rescue Net::OpenTimeout
skip "Remote connection failed for #{source_url}"
end
context "An artist" do
@@ -172,15 +168,11 @@ class ArtistTest < ActiveSupport::TestCase
a2 = FactoryBot.create(:artist, :name => "subway", :url_string => "http://subway.com/x/test.jpg")
a3 = FactoryBot.create(:artist, :name => "minko", :url_string => "https://minko.com/x/test.jpg")
begin
assert_artist_found("rembrandt", "http://rembrandt.com/x/test.jpg")
assert_artist_found("rembrandt", "http://rembrandt.com/x/another.jpg")
assert_artist_not_found("http://nonexistent.com/test.jpg")
assert_artist_found("minko", "https://minko.com/x/test.jpg")
assert_artist_found("minko", "http://minko.com/x/test.jpg")
rescue Net::OpenTimeout
skip "network failure"
end
assert_artist_found("rembrandt", "http://rembrandt.com/x/test.jpg")
assert_artist_found("rembrandt", "http://rembrandt.com/x/another.jpg")
assert_artist_not_found("http://nonexistent.com/test.jpg")
assert_artist_found("minko", "https://minko.com/x/test.jpg")
assert_artist_found("minko", "http://minko.com/x/test.jpg")
end
should "be case-insensitive to domains when finding matches by url" do

14
test/unit/cloudflare_service_test.rb
View File

@@ -1,5 +1,4 @@
require 'test_helper'
require 'webmock/minitest'
class CloudflareServiceTest < ActiveSupport::TestCase
def setup
@@ -8,16 +7,11 @@ class CloudflareServiceTest < ActiveSupport::TestCase
context "#purge_cache" do
should "make calls to cloudflare's api" do
stub_request(:any, "api.cloudflare.com")
@cloudflare.purge_cache(["http://localhost/file.txt"])
url = "http://www.example.com/file.jpg"
mock_request("https://api.cloudflare.com/client/v4/zones/123/purge_cache", method: :delete, json: { files: [url] })
assert_requested(:delete, "https://api.cloudflare.com/client/v4/zones/123/purge_cache", times: 1)
end
end
context "#ips" do
should "work" do
refute_empty(@cloudflare.ips)
response = @cloudflare.purge_cache([url])
assert_equal(200, response.status)
end
end
end

136
test/unit/danbooru_http_test.rb
View File

@@ -4,18 +4,20 @@ class DanbooruHttpTest < ActiveSupport::TestCase
context "Danbooru::Http" do
context "#get method" do
should "work for all basic methods" do
%i[get put post delete].each do |method|
%i[get head put post delete].each do |method|
response = Danbooru::Http.send(method, "https://httpbin.org/status/200")
assert_equal(200, response.status)
end
end
should "follow redirects" do
skip "Skipping test (https://github.com/postmanlabs/httpbin/issues/617)"
response = Danbooru::Http.get("https://httpbin.org/absolute-redirect/3")
assert_equal(200, response.status)
end
should "fail if redirected too many times" do
skip "Skipping test (https://github.com/postmanlabs/httpbin/issues/617)"
response = Danbooru::Http.get("https://httpbin.org/absolute-redirect/10")
assert_equal(598, response.status)
end
@@ -26,8 +28,10 @@ class DanbooruHttpTest < ActiveSupport::TestCase
end
should "fail if the request takes too long to download" do
response = Danbooru::Http.timeout(1).get("https://httpbin.org/drip?duration=5&numbytes=5")
assert_equal(599, response.status)
# XXX should return status 599 instead
assert_raises(HTTP::TimeoutError) do
response = Danbooru::Http.timeout(1).get("https://httpbin.org/drip?duration=10&numbytes=10").flush
end
end
should "automatically decompress gzipped responses" do
@@ -36,13 +40,131 @@ class DanbooruHttpTest < ActiveSupport::TestCase
assert_equal(true, response.parse["gzipped"])
end
should "cache requests" do
response1 = Danbooru::Http.cache(1.minute).get("https://httpbin.org/uuid")
should "automatically parse html responses" do
response = Danbooru::Http.get("https://httpbin.org/html")
assert_equal(200, response.status)
assert_instance_of(Nokogiri::HTML5::Document, response.parse)
assert_equal("Herman Melville - Moby-Dick", response.parse.css("h1").text)
end
should "automatically parse xml responses" do
response = Danbooru::Http.get("https://httpbin.org/xml")
assert_equal(200, response.status)
assert_equal(true, response.parse[:slideshow].present?)
end
should "track cookies between requests" do
http = Danbooru::Http.use(:session)
resp1 = http.get("https://httpbin.org/cookies/set/abc/1")
resp2 = http.get("https://httpbin.org/cookies/set/def/2")
resp3 = http.get("https://httpbin.org/cookies")
assert_equal({ abc: "1", def: "2" }, resp3.parse["cookies"].symbolize_keys)
resp4 = http.cookies(def: 3, ghi: 4).get("https://httpbin.org/cookies")
assert_equal({ abc: "1", def: "3", ghi: "4" }, resp4.parse["cookies"].symbolize_keys)
end
end
context "cache feature" do
should "cache multiple requests to the same url" do
http = Danbooru::Http.cache(1.hour)
response1 = http.get("https://httpbin.org/uuid")
assert_equal(200, response1.status)
response2 = Danbooru::Http.cache(1.minute).get("https://httpbin.org/uuid")
response2 = http.get("https://httpbin.org/uuid")
assert_equal(200, response2.status)
assert_equal(response2.body, response1.body)
assert_equal(response2.to_s, response1.to_s)
end
should "cache cookies correctly" do
http = Danbooru::Http.cache(1.hour)
resp1 = http.get("https://httpbin.org/cookies")
resp2 = http.get("https://httpbin.org/cookies/set/abc/1")
resp3 = http.get("https://httpbin.org/cookies/set/def/2")
resp4 = http.get("https://httpbin.org/cookies")
assert_equal(200, resp1.status)
assert_equal(200, resp2.status)
assert_equal(200, resp3.status)
assert_equal(200, resp4.status)
assert_equal({}, resp1.parse["cookies"].symbolize_keys)
assert_equal({ abc: "1" }, resp2.parse["cookies"].symbolize_keys)
assert_equal({ abc: "1", def: "2" }, resp3.parse["cookies"].symbolize_keys)
assert_equal({ abc: "1", def: "2" }, resp4.parse["cookies"].symbolize_keys)
end
end
context "retriable feature" do
should "retry immediately if no Retry-After header is sent" do
response_429 = ::HTTP::Response.new(status: 429, version: "1.1", body: "")
response_200 = ::HTTP::Response.new(status: 200, version: "1.1", body: "")
HTTP::Client.any_instance.expects(:perform).times(2).returns(response_429, response_200)
response = Danbooru::Http.use(:retriable).get("https://httpbin.org/status/429")
assert_equal(200, response.status)
end
should "retry if the Retry-After header is an integer" do
response_503 = ::HTTP::Response.new(status: 503, version: "1.1", headers: { "Retry-After": "1" }, body: "")
response_200 = ::HTTP::Response.new(status: 200, version: "1.1", body: "")
HTTP::Client.any_instance.expects(:perform).times(2).returns(response_503, response_200)
response = Danbooru::Http.use(:retriable).get("https://httpbin.org/status/503")
assert_equal(200, response.status)
end
should "retry if the Retry-After header is a date" do
response_503 = ::HTTP::Response.new(status: 503, version: "1.1", headers: { "Retry-After": 2.seconds.from_now.httpdate }, body: "")
response_200 = ::HTTP::Response.new(status: 200, version: "1.1", body: "")
HTTP::Client.any_instance.expects(:perform).times(2).returns(response_503, response_200)
response = Danbooru::Http.use(:retriable).get("https://httpbin.org/status/503")
assert_equal(200, response.status)
end
end
context "#download method" do
should "download files" do
response, file = Danbooru::Http.download_media("https://httpbin.org/bytes/1000")
assert_equal(200, response.status)
assert_equal(1000, file.size)
end
should "follow redirects when downloading files" do
skip "Skipping test (https://github.com/postmanlabs/httpbin/issues/617)"
response, file = Danbooru::Http.download_media("https://httpbin.org/redirect-to?url=https://httpbin.org/bytes/1000")
assert_equal(200, response.status)
assert_equal(1000, file.size)
end
should "fail if the url points to a private IP" do
assert_raises(Danbooru::Http::DownloadError) do
Danbooru::Http.public_only.download_media("https://127.0.0.1.xip.io")
end
end
should "fail if the url redirects to a private IP" do
assert_raises(Danbooru::Http::DownloadError) do
Danbooru::Http.public_only.download_media("https://httpbin.org/redirect-to?url=https://127.0.0.1.xip.io")
end
end
should "fail if a download is too large" do
assert_raises(Danbooru::Http::FileTooLargeError) do
response, file = Danbooru::Http.max_size(500).download_media("https://httpbin.org/bytes/1000")
end
end
should "fail if a streaming download is too large" do
assert_raises(Danbooru::Http::FileTooLargeError) do
response, file = Danbooru::Http.max_size(500).download_media("https://httpbin.org/stream-bytes/1000")
end
end
end
end

34
test/unit/downloads/art_station_test.rb
View File

@@ -3,53 +3,27 @@ require 'test_helper'
module Downloads
class ArtStationTest < ActiveSupport::TestCase
context "a download for a (small) artstation image" do
setup do
@asset = "https://cdnb3.artstation.com/p/assets/images/images/003/716/071/small/aoi-ogata-hate-city.jpg?1476754974"
@download = Downloads::File.new(@asset)
end
should "download the /4k/ image instead" do
file, strategy = @download.download!
assert_equal(1_880_910, ::File.size(file.path))
assert_downloaded(1_880_910, "https://cdnb3.artstation.com/p/assets/images/images/003/716/071/small/aoi-ogata-hate-city.jpg?1476754974")
end
end
context "for an image where an original does not exist" do
setup do
@asset = "https://cdna.artstation.com/p/assets/images/images/004/730/278/large/mendel-oh-dragonll.jpg"
@download = Downloads::File.new(@asset)
end
should "not try to download the original" do
file, strategy = @download.download!
assert_equal(483_192, ::File.size(file.path))
assert_downloaded(483_192, "https://cdna.artstation.com/p/assets/images/images/004/730/278/large/mendel-oh-dragonll.jpg")
end
end
context "a download for an ArtStation image hosted on CloudFlare" do
setup do
@asset = "https://cdnb.artstation.com/p/assets/images/images/003/716/071/large/aoi-ogata-hate-city.jpg?1476754974"
end
should "return the original file, not the polished file" do
@asset = "https://cdnb.artstation.com/p/assets/images/images/003/716/071/large/aoi-ogata-hate-city.jpg?1476754974"
assert_downloaded(1_880_910, @asset)
end
should "return the original filesize, not the polished filesize" do
assert_equal(1_880_910, Downloads::File.new(@asset).size)
end
end
context "a download for a https://$artist.artstation.com/projects/$id page" do
setup do
@source = "https://dantewontdie.artstation.com/projects/YZK5q"
@download = Downloads::File.new(@source)
end
should "download the original image instead" do
file, strategy = @download.download!
assert_equal(247_350, ::File.size(file.path))
assert_downloaded(247_350, "https://dantewontdie.artstation.com/projects/YZK5q")
end
end
end

81
test/unit/downloads/file_test.rb
View File

@@ -1,81 +0,0 @@
require 'test_helper'
module Downloads
class FileTest < ActiveSupport::TestCase
context "A post download" do
setup do
@source = "http://www.google.com/intl/en_ALL/images/logo.gif"
@download = Downloads::File.new(@source)
end
context "for a banned IP" do
setup do
Resolv.expects(:getaddress).returns("127.0.0.1").at_least_once
end
should "not try to download the file" do
assert_raise(Downloads::File::Error) { Downloads::File.new("http://evil.com").download! }
end
should "not try to fetch the size" do
assert_raise(Downloads::File::Error) { Downloads::File.new("http://evil.com").size }
end
should "not follow redirects to banned IPs" do
url = "http://httpbin.org/redirect-to?url=http://127.0.0.1"
stub_request(:get, url).to_return(status: 301, headers: { "Location": "http://127.0.0.1" })
assert_raise(Downloads::File::Error) { Downloads::File.new(url).download! }
end
should "not follow redirects that resolve to a banned IP" do
url = "http://httpbin.org/redirect-to?url=http://127.0.0.1.nip.io"
stub_request(:get, url).to_return(status: 301, headers: { "Location": "http://127.0.0.1.xip.io" })
assert_raise(Downloads::File::Error) { Downloads::File.new(url).download! }
end
end
context "that fails" do
should "retry three times before giving up" do
HTTParty.expects(:get).times(3).raises(Errno::ETIMEDOUT)
assert_raises(Errno::ETIMEDOUT) { @download.download! }
end
should "return an uncorrupted file on the second try" do
bomb = stub("bomb")
bomb.stubs(:code).raises(IOError)
resp = stub("resp", success?: true)
chunk = stub("a")
chunk.stubs(:code).returns(200)
chunk.stubs(:size).returns(1)
chunk.stubs(:to_s).returns("a")
HTTParty.expects(:get).twice.multiple_yields(chunk, bomb).then.multiple_yields(chunk, chunk).returns(resp)
@download.stubs(:is_cloudflare?).returns(false)
tempfile, _strategy = @download.download!
assert_equal("aa", tempfile.read)
end
end
should "throw an exception when the file is larger than the maximum" do
assert_raise(Downloads::File::Error) do
@download.download!(max_size: 1)
end
end
should "store the file in the tempfile path" do
tempfile, strategy = @download.download!
assert_operator(tempfile.size, :>, 0, "should have data")
end
should "correctly save the file when following 302 redirects" do
download = Downloads::File.new("https://yande.re/post/show/578014")
file, strategy = download.download!(url: download.preview_url)
assert_equal(19134, file.size)
end
end
end
end

25
test/unit/downloads/pixiv_test.rb
View File

@@ -122,32 +122,17 @@ module Downloads
assert_downloaded(@file_size, @file_url, @ref)
end
end
context "downloading a pixiv fanbox image" do
should_eventually "work" do
@source = "https://www.pixiv.net/fanbox/creator/12491073/post/82406"
@file_url = "https://fanbox.pixiv.net/images/post/82406/D833IKA7FIesJXL8xx39rrG0.jpeg"
@file_size = 873_387
assert_not_rewritten(@file_url, @source)
assert_downloaded(@file_size, @file_url, @source)
end
end
end
context "An ugoira site for pixiv" do
setup do
@download = Downloads::File.new("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364")
@tempfile, strategy = @download.download!
@tempfile.close!
end
should "capture the data" do
assert_equal(2, @download.data[:ugoira_frame_data].size)
if @download.data[:ugoira_frame_data][0]["file"]
@strategy = Sources::Strategies.find("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364")
assert_equal(2, @strategy.data[:ugoira_frame_data].size)
if @strategy.data[:ugoira_frame_data][0]["file"]
assert_equal([{"file" => "000000.jpg", "delay" => 125}, {"file" => "000001.jpg", "delay" => 125}], @download.data[:ugoira_frame_data])
else
assert_equal([{"delay_msec" => 125}, {"delay_msec" => 125}], @download.data[:ugoira_frame_data])
assert_equal([{"delay_msec" => 125}, {"delay_msec" => 125}], @strategy.data[:ugoira_frame_data])
end
end
end

4
test/unit/media_file_test.rb
View File

@@ -98,6 +98,10 @@ class MediaFileTest < ActiveSupport::TestCase
should "determine the correct extension for a flash file" do
assert_equal(:swf, MediaFile.open("test/files/compressed.swf").file_ext)
end
should "not fail for empty files" do
assert_equal(:bin, MediaFile.open("test/files/test-empty.bin").file_ext)
end
end
should "determine the correct md5 for a jpeg file" do

17
test/unit/post_query_builder_test.rb
View File

@@ -277,6 +277,19 @@ class PostQueryBuilderTest < ActiveSupport::TestCase
assert_tag_match([child, parent], "-child:garbage")
end
should "return posts when using the status of the parent/child" do
parent_of_deleted = create(:post)
deleted = create(:post, is_deleted: true, tag_string: "parent:#{parent_of_deleted.id}")
child_of_deleted = create(:post, tag_string: "parent:#{deleted.id}")
all = [child_of_deleted, deleted, parent_of_deleted]
assert_tag_match([child_of_deleted], "parent:deleted")
assert_tag_match(all - [child_of_deleted], "-parent:deleted")
assert_tag_match([parent_of_deleted], "child:deleted")
assert_tag_match(all - [parent_of_deleted], "-child:deleted")
end
should "return posts for the favgroup:<name> metatag" do
post1 = create(:post)
post2 = create(:post)
@@ -757,8 +770,8 @@ class PostQueryBuilderTest < ActiveSupport::TestCase
create(:saved_search, query: "aaa", labels: ["zzz"], user: CurrentUser.user)
create(:saved_search, query: "bbb", user: CurrentUser.user)
Redis.any_instance.stubs(:exists).with("search:aaa").returns(true)
Redis.any_instance.stubs(:exists).with("search:bbb").returns(true)
Redis.any_instance.stubs(:exists?).with("search:aaa").returns(true)
Redis.any_instance.stubs(:exists?).with("search:bbb").returns(true)
Redis.any_instance.stubs(:smembers).with("search:aaa").returns([@post1.id])
Redis.any_instance.stubs(:smembers).with("search:bbb").returns([@post2.id])

8
test/unit/reportbooru_service_test.rb
View File

@@ -4,20 +4,20 @@ class ReportbooruServiceTest < ActiveSupport::TestCase
def setup
@service = ReportbooruService.new(reportbooru_server: "http://localhost:1234")
@post = create(:post)
@date = "2000-01-01"
@date = Date.parse("2000-01-01")
end
context "#popular_posts" do
should "return the list of popular posts on success" do
body = "[[#{@post.id},100.0]]"
@service.http.expects(:get).with("http://localhost:1234/post_views/rank?date=#{@date}").returns(HTTP::Response.new(status: 200, body: body, version: "1.1"))
mock_post_view_rankings(@date, [[@post.id, 100]])
posts = @service.popular_posts(@date)
assert_equal([@post], posts)
end
should "return nothing on failure" do
@service.http.expects(:get).with("http://localhost:1234/post_views/rank?date=#{@date}").returns(HTTP::Response.new(status: 500, body: "", version: "1.1"))
Danbooru::Http.any_instance.expects(:get).with("http://localhost:1234/post_views/rank?date=#{@date}").returns(HTTP::Response.new(status: 500, body: "", version: "1.1"))
Danbooru::Http.any_instance.expects(:get).with("http://localhost:1234/post_views/rank?date=#{@date.yesterday}").returns(HTTP::Response.new(status: 500, body: "", version: "1.1"))
assert_equal([], @service.popular_posts(@date))
end

10
test/unit/sources/nijie_test.rb
View File

@@ -43,7 +43,7 @@ module Sources
should "get the image url" do
assert_equal("https://pic.nijie.net/03/nijie_picture/728995_20170505014820_0.jpg", @site.image_url)
assert_http_size(132_555, @site.image_url)
assert_downloaded(132_555, @site.image_url)
end
should "get the canonical url" do
@@ -53,7 +53,7 @@ module Sources
should "get the preview url" do
assert_equal("https://pic.nijie.net/03/__rs_l170x170/nijie_picture/728995_20170505014820_0.jpg", @site.preview_url)
assert_equal([@site.preview_url], @site.preview_urls)
assert_http_exists(@site.preview_url)
assert_downloaded(132_555, @site.preview_url)
end
should "get the profile" do
@@ -187,8 +187,6 @@ module Sources
desc = <<-EOS.strip_heredoc.chomp
foo [b]bold[/b] [i]italics[/i] [s]strike[/s] red
<http://nijie.info/view.php?id=218944>
EOS
@@ -207,8 +205,8 @@ module Sources
assert_equal("https://nijie.info/members.php?id=236014", site.profile_url)
assert_nothing_raised { site.to_h }
assert_http_size(3619, site.image_url)
assert_http_exists(site.preview_url)
assert_downloaded(3619, site.image_url)
assert_downloaded(3619, site.preview_url)
end
end

20
test/unit/sources/pixiv_test.rb
View File

@@ -15,10 +15,7 @@ module Sources
def get_source(source)
@site = Sources::Strategies.find(source)
@site
rescue Net::OpenTimeout
skip "Remote connection to #{source} failed"
end
context "in all cases" do
@@ -73,17 +70,6 @@ module Sources
end
end
context "A https://www.pixiv.net/fanbox/creator/*/post/* source" do
should_eventually "work" do
@site = Sources::Strategies.find("http://www.pixiv.net/fanbox/creator/554149/post/82555")
assert_equal("TYONE(お仕事募集中)", @site.artist_name)
assert_equal("https://www.pixiv.net/users/554149", @site.profile_url)
assert_equal("https://fanbox.pixiv.net/images/post/82555/Lyyeb6dDLcQZmy09nqLZapuS.jpeg", @site.image_url)
assert_nothing_raised { @site.to_h }
end
end
context "A https://www.pixiv.net/*/artworks/* source" do
should "work" do
@site = Sources::Strategies.find("https://www.pixiv.net/en/artworks/64476642")
@@ -249,6 +235,10 @@ module Sources
assert_includes(@translated_tags, "foo")
end
should "not translate tags for digital media" do
assert_equal(false, @tags.include?("Photoshop"))
end
should "normalize 10users入り tags" do
assert_includes(@tags, "風景10users入り")
assert_includes(@translated_tags, "scenery")
@@ -280,7 +270,7 @@ module Sources
should "not translate '1000users入り' to '1'" do
FactoryBot.create(:tag, name: "1", post_count: 1)
source = get_source("https://www.pixiv.net/member_illust.php?mode=medium&illust_id=60665428")
tags = %w[1000users入り Fate/GrandOrder アルジュナ(Fate) アルトリア・ペンドラゴン イシュタル(Fate) グランブルーファンタジー マシュ・キリエライト マーリン(Fate) 両儀式 手袋 CLIP\ STUDIO\ PAINT Photoshop]
tags = %w[1000users入り Fate/GrandOrder アルジュナ(Fate) アルトリア・ペンドラゴン イシュタル(Fate) グランブルーファンタジー マシュ・キリエライト マーリン(Fate) 両儀式 手袋]
assert_equal(tags.sort, source.tags.map(&:first).sort)
assert_equal(["fate/grand_order"], source.translated_tags.map(&:name))

4
test/unit/sources/tumblr_test.rb
View File

@@ -169,7 +169,7 @@ module Sources
context "The source for a 'http://ve.media.tumblr.com/*' video post with inline images" do
setup do
@url = "https://ve.media.tumblr.com/tumblr_os31dkexhK1wsfqep.mp4"
@url = "https://va.media.tumblr.com/tumblr_os31dkexhK1wsfqep.mp4"
@ref = "https://noizave.tumblr.com/post/162222617101"
end
@@ -177,7 +177,7 @@ module Sources
should "get the video and inline images" do
site = Sources::Strategies.find(@url, @ref)
urls = %w[
https://ve.media.tumblr.com/tumblr_os31dkexhK1wsfqep.mp4
https://va.media.tumblr.com/tumblr_os31dkexhK1wsfqep.mp4
https://media.tumblr.com/afed9f5b3c33c39dc8c967e262955de2/tumblr_inline_os31dclyCR1v11u29_1280.png
]

8
test/unit/sources/twitter_test.rb
View File

@@ -244,6 +244,14 @@ module Sources
end
end
context "A profile banner image" do
should "work" do
@site = Sources::Strategies.find("https://pbs.twimg.com/profile_banners/1225702850002468864/1588597370/1500x500")
assert_equal(@site.image_url, @site.url)
assert_nothing_raised { @site.to_h }
end
end
context "A tweet containing non-normalized Unicode text" do
should "be normalized to nfkc" do
site = Sources::Strategies.find("https://twitter.com/aprilarcus/status/367557195186970624")

44
test/unit/storage_manager_test.rb
View File

@@ -1,8 +1,6 @@
require 'test_helper'
class StorageManagerTest < ActiveSupport::TestCase
BASE_DIR = "#{Rails.root}/tmp/test-storage"
setup do
CurrentUser.ip_addr = "127.0.0.1"
end
@@ -45,25 +43,21 @@ class StorageManagerTest < ActiveSupport::TestCase
context "StorageManager::Local" do
setup do
@storage_manager = StorageManager::Local.new(base_dir: BASE_DIR, base_url: "/data")
end
teardown do
FileUtils.rm_rf(BASE_DIR)
@storage_manager = StorageManager::Local.new(base_dir: @temp_dir, base_url: "/data")
end
context "#store method" do
should "store the file" do
@storage_manager.store(StringIO.new("data"), "#{BASE_DIR}/test.txt")
@storage_manager.store(StringIO.new("data"), "#{@temp_dir}/test.txt")
assert("data", File.read("#{BASE_DIR}/test.txt"))
assert("data", File.read("#{@temp_dir}/test.txt"))
end
should "overwrite the file if it already exists" do
@storage_manager.store(StringIO.new("foo"), "#{BASE_DIR}/test.txt")
@storage_manager.store(StringIO.new("bar"), "#{BASE_DIR}/test.txt")
@storage_manager.store(StringIO.new("foo"), "#{@temp_dir}/test.txt")
@storage_manager.store(StringIO.new("bar"), "#{@temp_dir}/test.txt")
assert("bar", File.read("#{BASE_DIR}/test.txt"))
assert("bar", File.read("#{@temp_dir}/test.txt"))
end
end
@@ -72,7 +66,7 @@ class StorageManagerTest < ActiveSupport::TestCase
@storage_manager.store(StringIO.new("data"), "test.txt")
@storage_manager.delete("test.txt")
assert_not(File.exist?("#{BASE_DIR}/test.txt"))
assert_not(File.exist?("#{@temp_dir}/test.txt"))
end
should "not fail if the file doesn't exist" do
@@ -88,9 +82,9 @@ class StorageManagerTest < ActiveSupport::TestCase
@storage_manager.store_file(StringIO.new("data"), @post, :large)
@storage_manager.store_file(StringIO.new("data"), @post, :original)
@file_path = "#{BASE_DIR}/preview/#{@post.md5}.jpg"
@large_file_path = "#{BASE_DIR}/sample/sample-#{@post.md5}.jpg"
@preview_file_path = "#{BASE_DIR}/#{@post.md5}.#{@post.file_ext}"
@file_path = "#{@temp_dir}/preview/#{@post.md5}.jpg"
@large_file_path = "#{@temp_dir}/sample/sample-#{@post.md5}.jpg"
@preview_file_path = "#{@temp_dir}/#{@post.md5}.#{@post.file_ext}"
end
should "store the files at the correct path" do
@@ -134,12 +128,12 @@ class StorageManagerTest < ActiveSupport::TestCase
context "when the original_subdir option is used" do
should "store original files at the correct path" do
@post = FactoryBot.create(:post, file_ext: "png")
@storage_manager = StorageManager::Local.new(base_dir: BASE_DIR, base_url: "/data", original_subdir: "original/")
@storage_manager = StorageManager::Local.new(base_dir: @temp_dir, base_url: "/data", original_subdir: "original/")
assert_equal("#{BASE_DIR}/original/#{@post.md5}.png", @storage_manager.file_path(@post, @post.file_ext, :original))
assert_equal("#{@temp_dir}/original/#{@post.md5}.png", @storage_manager.file_path(@post, @post.file_ext, :original))
@storage_manager.store_file(StringIO.new("data"), @post, :original)
assert_equal(true, File.exist?("#{BASE_DIR}/original/#{@post.md5}.png"))
assert_equal(true, File.exist?("#{@temp_dir}/original/#{@post.md5}.png"))
end
end
end
@@ -151,24 +145,20 @@ class StorageManagerTest < ActiveSupport::TestCase
@storage_manager = StorageManager::Hybrid.new do |id, md5, file_ext, type|
if id.odd?
StorageManager::Local.new(base_dir: "#{BASE_DIR}/i1", base_url: "/i1")
StorageManager::Local.new(base_dir: "#{@temp_dir}/i1", base_url: "/i1")
else
StorageManager::Local.new(base_dir: "#{BASE_DIR}/i2", base_url: "/i2")
StorageManager::Local.new(base_dir: "#{@temp_dir}/i2", base_url: "/i2")
end
end
end
teardown do
FileUtils.rm_rf(BASE_DIR)
end
context "#store_file method" do
should "store odd-numbered posts under /i1 and even-numbered posts under /i2" do
@storage_manager.store_file(StringIO.new("post1"), @post1, :original)
@storage_manager.store_file(StringIO.new("post2"), @post2, :original)
assert(File.exist?("#{BASE_DIR}/i1/#{@post1.md5}.png"))
assert(File.exist?("#{BASE_DIR}/i2/#{@post2.md5}.png"))
assert(File.exist?("#{@temp_dir}/i1/#{@post1.md5}.png"))
assert(File.exist?("#{@temp_dir}/i2/#{@post2.md5}.png"))
end
end

173
test/unit/upload_service_test.rb
View File

@@ -131,12 +131,9 @@ class UploadServiceTest < ActiveSupport::TestCase
end
should "download the file" do
begin
@service = UploadService::Preprocessor.new(source: @source, referer_url: @ref)
@upload = @service.start!
rescue Net::OpenTimeout
skip "network failure"
end
@service = UploadService::Preprocessor.new(source: @source, referer_url: @ref)
@upload = @service.start!
assert_equal("preprocessed", @upload.status)
assert_equal(294591, @upload.file_size)
assert_equal("jpg", @upload.file_ext)
@@ -155,11 +152,8 @@ class UploadServiceTest < ActiveSupport::TestCase
skip unless MediaFile::Ugoira.videos_enabled?
@service = UploadService::Preprocessor.new(source: @source)
begin
@upload = @service.start!
rescue Net::OpenTimeout
skip "network problems"
end
@upload = @service.start!
assert_equal("preprocessed", @upload.status)
assert_equal(2804, @upload.file_size)
assert_equal("zip", @upload.file_ext)
@@ -176,11 +170,8 @@ class UploadServiceTest < ActiveSupport::TestCase
should "download the file" do
@service = UploadService::Preprocessor.new(source: @source)
begin
@upload = @service.start!
rescue Net::OpenTimeout
skip "network problems"
end
@upload = @service.start!
assert_equal("preprocessed", @upload.status)
assert_equal(181309, @upload.file_size)
assert_equal("jpg", @upload.file_ext)
@@ -212,7 +203,7 @@ class UploadServiceTest < ActiveSupport::TestCase
context "on timeout errors" do
setup do
@source = "https://cdn.donmai.us/original/93/f4/93f4dd66ef1eb11a89e56d31f9adc8d0.jpg"
HTTParty.stubs(:get).raises(Net::ReadTimeout)
Danbooru::Http.any_instance.stubs(:get).raises(HTTP::TimeoutError)
end
should "leave the upload in an error state" do
@@ -512,36 +503,28 @@ class UploadServiceTest < ActiveSupport::TestCase
context "a post with a pixiv html source" do
should "replace with the full size image" do
begin
as(@user) do
@post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350")
end
assert_equal(80, @post.image_width)
assert_equal(82, @post.image_height)
assert_equal(16275, @post.file_size)
assert_equal("png", @post.file_ext)
assert_equal("4ceadc314938bc27f3574053a3e1459a", @post.md5)
assert_equal("4ceadc314938bc27f3574053a3e1459a", Digest::MD5.file(@post.file).hexdigest)
assert_equal("https://i.pximg.net/img-original/img/2017/04/04/08/54/15/62247350_p0.png", @post.replacements.last.replacement_url)
assert_equal("https://i.pximg.net/img-original/img/2017/04/04/08/54/15/62247350_p0.png", @post.source)
rescue Net::OpenTimeout
skip "Remote connection to Pixiv failed"
as(@user) do
@post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350")
end
assert_equal(80, @post.image_width)
assert_equal(82, @post.image_height)
assert_equal(16275, @post.file_size)
assert_equal("png", @post.file_ext)
assert_equal("4ceadc314938bc27f3574053a3e1459a", @post.md5)
assert_equal("4ceadc314938bc27f3574053a3e1459a", Digest::MD5.file(@post.file).hexdigest)
assert_equal("https://i.pximg.net/img-original/img/2017/04/04/08/54/15/62247350_p0.png", @post.replacements.last.replacement_url)
assert_equal("https://i.pximg.net/img-original/img/2017/04/04/08/54/15/62247350_p0.png", @post.source)
end
should "delete the old files after thirty days" do
begin
@post.unstub(:queue_delete_files)
FileUtils.expects(:rm_f).times(3)
@post.unstub(:queue_delete_files)
FileUtils.expects(:rm_f).times(3)
as(@user) { @post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350") }
as(@user) { @post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350") }
travel_to((PostReplacement::DELETION_GRACE_PERIOD + 1).days.from_now) do
perform_enqueued_jobs
end
rescue Net::OpenTimeout
skip "Remote connection to Pixiv failed"
travel_to((PostReplacement::DELETION_GRACE_PERIOD + 1).days.from_now) do
perform_enqueued_jobs
end
end
end
@@ -568,34 +551,30 @@ class UploadServiceTest < ActiveSupport::TestCase
context "a post that is replaced to another file then replaced back to the original file" do
should "not delete the original files" do
begin
skip unless MediaFile::Ugoira.videos_enabled?
@post.unstub(:queue_delete_files)
skip unless MediaFile::Ugoira.videos_enabled?
@post.unstub(:queue_delete_files)
# this is called thrice to delete the file for 62247364
FileUtils.expects(:rm_f).times(3)
# this is called thrice to delete the file for 62247364
FileUtils.expects(:rm_f).times(3)
as(@user) do
@post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350")
@post.reload
@post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364")
@post.reload
Upload.destroy_all
@post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350")
end
assert_nothing_raised { @post.file(:original) }
assert_nothing_raised { @post.file(:preview) }
assert_enqueued_jobs 3, only: DeletePostFilesJob
travel PostReplacement::DELETION_GRACE_PERIOD + 1.day
assert_raise(Post::DeletionError) { perform_enqueued_jobs }
assert_nothing_raised { @post.file(:original) }
assert_nothing_raised { @post.file(:preview) }
rescue Net::OpenTimeout
skip "Remote connection to Pixiv failed"
as(@user) do
@post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350")
@post.reload
@post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364")
@post.reload
Upload.destroy_all
@post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350")
end
assert_nothing_raised { @post.file(:original) }
assert_nothing_raised { @post.file(:preview) }
assert_enqueued_jobs 3, only: DeletePostFilesJob
travel PostReplacement::DELETION_GRACE_PERIOD + 1.day
assert_raise(Post::DeletionError) { perform_enqueued_jobs }
assert_nothing_raised { @post.file(:original) }
assert_nothing_raised { @post.file(:preview) }
end
end
@@ -609,39 +588,35 @@ class UploadServiceTest < ActiveSupport::TestCase
should "not delete the still active files" do
# swap the images between @post1 and @post2.
begin
as(@user) do
skip unless MediaFile::Ugoira.videos_enabled?
as(@user) do
skip unless MediaFile::Ugoira.videos_enabled?
@post1.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350")
@post2.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364")
assert_equal("4ceadc314938bc27f3574053a3e1459a", @post1.md5)
assert_equal("cad1da177ef309bf40a117c17b8eecf5", @post2.md5)
@post1.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350")
@post2.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364")
assert_equal("4ceadc314938bc27f3574053a3e1459a", @post1.md5)
assert_equal("cad1da177ef309bf40a117c17b8eecf5", @post2.md5)
@post2.reload
@post2.replace!(replacement_url: "https://cdn.donmai.us/original/d3/4e/d34e4cf0a437a5d65f8e82b7bcd02606.jpg")
assert_equal("d34e4cf0a437a5d65f8e82b7bcd02606", @post2.md5)
Upload.destroy_all
@post1.reload
@post2.reload
@post2.reload
@post2.replace!(replacement_url: "https://cdn.donmai.us/original/d3/4e/d34e4cf0a437a5d65f8e82b7bcd02606.jpg")
assert_equal("d34e4cf0a437a5d65f8e82b7bcd02606", @post2.md5)
Upload.destroy_all
@post1.reload
@post2.reload
@post1.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364")
@post2.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350")
assert_equal("cad1da177ef309bf40a117c17b8eecf5", @post1.md5)
assert_equal("4ceadc314938bc27f3574053a3e1459a", @post2.md5)
end
travel_to (PostReplacement::DELETION_GRACE_PERIOD + 1).days.from_now do
assert_raise(Post::DeletionError) do
perform_enqueued_jobs
end
end
assert_nothing_raised { @post1.file(:original) }
assert_nothing_raised { @post2.file(:original) }
rescue Net::OpenTimeout
skip "Remote connection to Pixiv failed"
@post1.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364")
@post2.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350")
assert_equal("cad1da177ef309bf40a117c17b8eecf5", @post1.md5)
assert_equal("4ceadc314938bc27f3574053a3e1459a", @post2.md5)
end
travel_to (PostReplacement::DELETION_GRACE_PERIOD + 1).days.from_now do
assert_raise(Post::DeletionError) do
perform_enqueued_jobs
end
end
assert_nothing_raised { @post1.file(:original) }
assert_nothing_raised { @post2.file(:original) }
end
end
@@ -885,12 +860,8 @@ class UploadServiceTest < ActiveSupport::TestCase
end
should "record the canonical source" do
begin
post = subject.new({}).create_post_from_upload(@upload)
assert_equal(@source, post.source)
rescue Net::OpenTimeout
skip "network failure"
end
post = subject.new({}).create_post_from_upload(@upload)
assert_equal(@source, post.source)
end
end