jaculabilis/danbooru
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'circleci-fixes'

Browse Source
This commit is contained in:
Albert Yi
2018-05-15 10:25:06 -07:00
parent 9310b39fda 8b16934aaa
commit 32ad389581
11 changed files with 168 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
.circleci/config.yml
View File

@@ -43,15 +43,13 @@ jobs:
docker-compose -f config/docker/compose.yml exec archives dockerize -wait tcp://$PGHOST:$PGPORT bash -l -c 'cd /app ; bundle exec rake db:create ; bundle exec rake db:migrate'
docker-compose -f config/docker/compose.yml exec web dockerize -wait tcp://$PGHOST:$PGPORT bash -l -c 'cd /app ; bin/rake db:create ; bin/rake db:migrate'
- run:
name: Test splitting
name: Run tests
command: |
ping -c 5 public-api.secure.pixiv.net
ping -c 5 oauth.secure.pixiv.net
ping -c 5 www.pixiv.net
circleci tests glob test/**/*_test.rb | circleci tests split | xargs -I{} docker-compose -f config/docker/compose.yml exec -T web bash -l -c 'cd /app ; bin/rails test {}'
circleci tests glob test/**/*_test.rb | circleci tests split | xargs -I{} docker-compose -f config/docker/compose.yml exec -T web bash -l -c 'cd /app ; bin/rails test --no-ci-clean {}'
docker cp docker_web_1:/app/test/reports /tmp
- store_test_results:
path: /tmp/reports
- run:
name: Teardown
command: docker-compose -f config/docker/compose.yml down
# - run:
# name: Teardown
# command: docker-compose -f config/docker/compose.yml down

9
app/logical/pixiv_api_client.rb
View File

@@ -154,10 +154,6 @@ class PixivApiClient
else
raise Error.new("Pixiv API call failed (status=#{resp.code} body=#{body})")
end
rescue Net::OpenTimeout
print "R" if Rails.env.test?
sleep(5)
retry
rescue JSON::ParserError
raise Error.new("Pixiv API call failed (status=#{resp.code} body=#{body})")
end
@@ -189,10 +185,5 @@ class PixivApiClient
access_token
end
rescue Net::OpenTimeout
print "R" if Rails.env.test?
sleep(5)
retry
end
end

3
app/models/artist.rb
View File

@@ -484,6 +484,9 @@ class Artist < ApplicationRecord
else
nil
end
rescue Net::OpenTimeout
raise if Rails.env.test?
nil
rescue Exception
nil
end

4
app/models/artist_url.rb
View File

@@ -21,7 +21,9 @@ class ArtistUrl < ApplicationRecord
begin
url = Sources::Site.new(url).normalize_for_artist_finder!
rescue PixivApiClient::Error, Sources::Site::NoStrategyError
rescue PixivApiClient::Error
raise if Rails.env.test?
rescue Sources::Site::NoStrategyError
end
url = url.gsub(/\/+\Z/, "")
url + "/"

2
app/models/bulk_update_request.rb
View File

@@ -118,8 +118,6 @@ class BulkUpdateRequest < ApplicationRecord
update(forum_post_id: forum_post.id)
else
forum_topic = ForumTopic.create(title: title, category_id: 1, original_post_attributes: {body: reason_with_link})
puts forum_topic.errors.full_messages
puts forum_topic.original_post.errors.full_messages
update(forum_topic_id: forum_topic.id, forum_post_id: forum_topic.posts.first.id)
end
end

12
test/functional/artists_controller_test.rb
View File

@@ -4,6 +4,11 @@ class ArtistsControllerTest < ActionDispatch::IntegrationTest
def assert_artist_found(expected_artist, source_url = nil)
if source_url
get_auth finder_artists_path(format: "json", url: source_url), @user
puts response.body
if response.body =~ /Net::OpenTimeout/
skip "Remote connection to #{source_url} failed"
return
end
end
assert_response :success
json = JSON.parse(response.body)
@@ -12,7 +17,14 @@ class ArtistsControllerTest < ActionDispatch::IntegrationTest
end
def assert_artist_not_found(source_url)
tries = 0
get_auth finder_artists_path(format: "json", url: source_url), @user
if response.body =~ /Net::OpenTimeout/
skip "Remote connection to #{source_url} failed"
return
end
assert_response :success
json = JSON.parse(response.body)
assert_equal(0, json.size, "Testing URL: #{source_url}")

8
test/functional/sources_controller_test.rb
View File

@@ -4,8 +4,12 @@ class SourcesControllerTest < ActionDispatch::IntegrationTest
context "The sources controller" do
context "show action" do
should "work for a pixiv URL" do
get source_path, params: { url: "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=14901720", format: "json" }
assert_response :success
begin
get source_path, params: { url: "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=14901720", format: "json" }
assert_response :success
rescue Net::OpenTimeout
skip "Remote connection failed to Pixiv failed"
end
end
should "work for a direct twitter URL with referer" do

13
test/test_helpers/download_test_helper.rb
View File

@@ -3,7 +3,18 @@ require 'ptools'
module DownloadTestHelper
def assert_downloaded(expected_filesize, source)
download = Downloads::File.new(source)
tempfile = download.download!
@retry_count = 0
begin
tempfile = download.download!
rescue Net::OpenTimeout
@retry_count += 1
if @retry_count == 3
skip "Remote connection to #{source} failed"
else
sleep(@retry_count ** 2)
retry
end
end
assert_equal(expected_filesize, tempfile.size, "Tested source URL: #{source}")
end

42
test/unit/artist_test.rb
View File

@@ -2,15 +2,39 @@ require 'test_helper'
class ArtistTest < ActiveSupport::TestCase
def assert_artist_found(expected_name, source_url)
artists = Artist.url_matches(source_url).to_a
tries = 0
assert_equal(1, artists.size)
assert_equal(expected_name, artists.first.name, "Testing URL: #{source_url}")
begin
artists = Artist.url_matches(source_url).to_a
assert_equal(1, artists.size)
assert_equal(expected_name, artists.first.name, "Testing URL: #{source_url}")
rescue Net::OpenTimeout, PixivApiClient::Error
tries += 1
if tries == 3
skip "Remote connection failed for #{source_url}"
else
sleep(tries ** 2)
retry
end
end
end
def assert_artist_not_found(source_url)
artists = Artist.url_matches(source_url).to_a
assert_equal(0, artists.size, "Testing URL: #{source_url}")
tries = 0
begin
artists = Artist.url_matches(source_url).to_a
assert_equal(0, artists.size, "Testing URL: #{source_url}")
rescue Net::OpenTimeout
tries += 1
if tries == 3
skip "Remote connection failed for #{source_url}"
else
sleep(tries ** 2)
retry
end
end
end
context "An artist" do
@@ -157,8 +181,8 @@ class ArtistTest < ActiveSupport::TestCase
end
should "ignore pixiv.net/ and pixiv.net/img/ url matches" do
a1 = FactoryBot.create(:artist, :name => "yomosaka", :url_string => "http://i2.pixiv.net/img100/img/yomosaka/27618292.jpg")
a2 = FactoryBot.create(:artist, :name => "niwatazumi_bf", :url_string => "http://i2.pixiv.net/img16/img/niwatazumi_bf/35488864_big_p6.jpg")
a1 = FactoryBot.create(:artist, :name => "yomosaka", :url_string => "http://i2.pixiv.net/img18/img/evazion/14901720.png")
a2 = FactoryBot.create(:artist, :name => "niwatazumi_bf", :url_string => "http://i2.pixiv.net/img18/img/evazion/14901720_big_p0.png")
assert_equal([], Artist.find_all_by_url("http://i2.pixiv.net/img28/img/kyang692/35563903.jpg"))
end
@@ -261,7 +285,9 @@ class ArtistTest < ActiveSupport::TestCase
end
should "find nothing for bad IDs" do
assert_artist_not_found("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=32049358")
assert_raises(PixivApiClient::BadIDError) do
assert_artist_not_found("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=32049358")
end
end
end

132
test/unit/post_replacement_test.rb
View File

@@ -127,61 +127,77 @@ class PostReplacementTest < ActiveSupport::TestCase
should "rescale the notes" do
assert_equal([80, 82, 80, 82], [@note.x, @note.y, @note.width, @note.height])
assert_difference("@replacer.note_versions.count") do
# replacement image is 80x82, so we're downscaling by 50% (160x164 -> 80x82).
@post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350")
@note.reload
end
begin
assert_difference("@replacer.note_versions.count") do
# replacement image is 80x82, so we're downscaling by 50% (160x164 -> 80x82).
@post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350")
@note.reload
end
assert_equal([40, 41, 40, 41], [@note.x, @note.y, @note.width, @note.height])
assert_equal([40, 41, 40, 41], [@note.x, @note.y, @note.width, @note.height])
rescue Net::OpenTimeout
skip "Remote connection to Pixiv failed"
end
end
end
context "a post with a pixiv html source" do
should "replace with the full size image" do
@post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350")
begin
@post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350")
assert_equal(80, @post.image_width)
assert_equal(82, @post.image_height)
assert_equal(16275, @post.file_size)
assert_equal("png", @post.file_ext)
assert_equal("4ceadc314938bc27f3574053a3e1459a", @post.md5)
assert_equal("4ceadc314938bc27f3574053a3e1459a", Digest::MD5.file(@post.file).hexdigest)
assert_equal("https://i.pximg.net/img-original/img/2017/04/04/08/54/15/62247350_p0.png", @post.source)
assert_equal("https://i.pximg.net/img-original/img/2017/04/04/08/54/15/62247350_p0.png", @post.replacements.last.replacement_url)
assert_equal(80, @post.image_width)
assert_equal(82, @post.image_height)
assert_equal(16275, @post.file_size)
assert_equal("png", @post.file_ext)
assert_equal("4ceadc314938bc27f3574053a3e1459a", @post.md5)
assert_equal("4ceadc314938bc27f3574053a3e1459a", Digest::MD5.file(@post.file).hexdigest)
assert_equal("https://i.pximg.net/img-original/img/2017/04/04/08/54/15/62247350_p0.png", @post.source)
assert_equal("https://i.pximg.net/img-original/img/2017/04/04/08/54/15/62247350_p0.png", @post.replacements.last.replacement_url)
rescue Net::OpenTimeout
skip "Remote connection to Pixiv failed"
end
end
should "delete the old files after thirty days" do
old_file_path, old_preview_file_path = @post.file(:original).path, @post.file(:preview).path
@post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350")
begin
old_file_path, old_preview_file_path = @post.file(:original).path, @post.file(:preview).path
@post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350")
assert(File.exists?(old_file_path))
assert(File.exists?(old_preview_file_path))
assert(File.exists?(old_file_path))
assert(File.exists?(old_preview_file_path))
Timecop.travel(Time.now + PostReplacement::DELETION_GRACE_PERIOD + 1.day) do
Delayed::Worker.new.work_off
Timecop.travel(Time.now + PostReplacement::DELETION_GRACE_PERIOD + 1.day) do
Delayed::Worker.new.work_off
end
assert_not(File.exists?(old_file_path))
assert_not(File.exists?(old_preview_file_path))
rescue Net::OpenTimeout
skip "Remote connection to Pixiv failed"
end
assert_not(File.exists?(old_file_path))
assert_not(File.exists?(old_preview_file_path))
end
end
context "a post that is replaced by a ugoira" do
should "save the frame data" do
skip "ffmpeg not installed" unless check_ffmpeg
@post.replace!(replacement_url: "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364")
@post.reload
begin
@post.replace!(replacement_url: "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364")
@post.reload
assert_equal(80, @post.image_width)
assert_equal(82, @post.image_height)
assert_equal(2804, @post.file_size)
assert_equal("zip", @post.file_ext)
assert_equal("cad1da177ef309bf40a117c17b8eecf5", @post.md5)
assert_equal("cad1da177ef309bf40a117c17b8eecf5", Digest::MD5.file(@post.file).hexdigest)
assert_equal(80, @post.image_width)
assert_equal(82, @post.image_height)
assert_equal(2804, @post.file_size)
assert_equal("zip", @post.file_ext)
assert_equal("cad1da177ef309bf40a117c17b8eecf5", @post.md5)
assert_equal("cad1da177ef309bf40a117c17b8eecf5", Digest::MD5.file(@post.file).hexdigest)
assert_equal("https://i.pximg.net/img-zip-ugoira/img/2017/04/04/08/57/38/62247364_ugoira1920x1080.zip", @post.source)
assert_equal([{"file"=>"000000.jpg", "delay"=>125}, {"file"=>"000001.jpg", "delay"=>125}], @post.pixiv_ugoira_frame_data.data)
assert_equal("https://i.pximg.net/img-zip-ugoira/img/2017/04/04/08/57/38/62247364_ugoira1920x1080.zip", @post.source)
assert_equal([{"file"=>"000000.jpg", "delay"=>125}, {"file"=>"000001.jpg", "delay"=>125}], @post.pixiv_ugoira_frame_data.data)
rescue Net::OpenTimeout
skip "Remote connection to Pixiv failed"
end
end
end
@@ -189,19 +205,23 @@ class PostReplacementTest < ActiveSupport::TestCase
should "not delete the original files" do
skip "ffmpeg is not installed" unless check_ffmpeg
@post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350")
@post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364")
@post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350")
begin
@post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350")
@post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364")
@post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350")
assert_nothing_raised { @post.file(:original) }
assert_nothing_raised { @post.file(:preview) }
assert_nothing_raised { @post.file(:original) }
assert_nothing_raised { @post.file(:preview) }
Timecop.travel(Time.now + PostReplacement::DELETION_GRACE_PERIOD + 1.day) do
Delayed::Worker.new.work_off
Timecop.travel(Time.now + PostReplacement::DELETION_GRACE_PERIOD + 1.day) do
Delayed::Worker.new.work_off
end
assert_nothing_raised { @post.file(:original) }
assert_nothing_raised { @post.file(:preview) }
rescue Net::OpenTimeout
skip "Remote connection to Pixiv failed"
end
assert_nothing_raised { @post.file(:original) }
assert_nothing_raised { @post.file(:preview) }
end
end
@@ -213,18 +233,22 @@ class PostReplacementTest < ActiveSupport::TestCase
@post2 = FactoryBot.create(:post)
# swap the images between @post1 and @post2.
@post1.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350")
@post2.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364")
@post2.replace!(replacement_url: "https://www.google.com/intl/en_ALL/images/logo.gif")
@post1.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364")
@post2.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350")
begin
@post1.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350")
@post2.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364")
@post2.replace!(replacement_url: "https://www.google.com/intl/en_ALL/images/logo.gif")
@post1.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364")
@post2.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350")
Timecop.travel(Time.now + PostReplacement::DELETION_GRACE_PERIOD + 1.day) do
Delayed::Worker.new.work_off
Timecop.travel(Time.now + PostReplacement::DELETION_GRACE_PERIOD + 1.day) do
Delayed::Worker.new.work_off
end
assert_nothing_raised { @post1.file(:original) }
assert_nothing_raised { @post2.file(:original) }
rescue Net::OpenTimeout
skip "Remote connection to Pixiv failed"
end
assert_nothing_raised { @post1.file(:original) }
assert_nothing_raised { @post2.file(:original) }
end
end

15
test/unit/sources/pixiv_test.rb
View File

@@ -4,13 +4,26 @@ module Sources
class PixivTest < ActiveSupport::TestCase
def get_source(source)
@site = Sources::Site.new(source)
@site.get
begin
@site.get
rescue Net::OpenTimeout
@retry_count += 1
if @retry_count == 3
skip "Could not connect to Pixiv"
else
sleep(@retry_count ** 2)
retry
end
end
@site
end
def setup
super
load_pixiv_tokens!
@retry_count = 0
end
def teardown