jaculabilis/danbooru
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #3577 from evazion/feat-storage-manager

Browse Source 
Introduce storage manager concept
This commit is contained in:
Albert Yi
2018-03-27 09:28:17 -07:00
committed by GitHub
parent 8a012d4c91 619a2055fe
commit ce7e31325c
38 changed files with 744 additions and 796 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
app/controllers/iqdb_queries_controller.rb
View File

@@ -49,7 +49,7 @@ protected
def create_by_post
@post = Post.find(params[:post_id])
@download = Iqdb::Download.new(@post.complete_preview_file_url)
@download = Iqdb::Download.new(@post.preview_file_url)
@download.find_similar
@results = @download.matches
end

2
app/controllers/uploads_controller.rb
View File

@@ -6,7 +6,7 @@ class UploadsController < ApplicationController
@upload = Upload.new
@upload_notice_wiki = WikiPage.titled(Danbooru.config.upload_notice_wiki_page).first
if params[:url]
download = Downloads::File.new(params[:url], ".")
download = Downloads::File.new(params[:url])
@normalized_url, _, _ = download.before_download(params[:url], {})
@post = find_post_by_url(@normalized_url)

4
app/logical/apng_inspector.rb
View File

@@ -86,7 +86,7 @@ class APNGInspector
end
@corrupted = !read_success || actl_corrupted
return !@corrupted
self
end
def corrupted?
@@ -109,4 +109,4 @@ class APNGInspector
return framedata.unpack("N".freeze)[0]
end
end
end

9
app/logical/backup_service.rb
View File

@@ -1,9 +0,0 @@
class BackupService
def backup(file_path, options = {})
raise NotImplementedError.new("#{self.class}.backup not implemented")
end
def delete(file_path, options = {})
raise NotImplementedError.new("#{self.class}.delete not implemented")
end
end

9
app/logical/danbooru_image_resizer.rb
View File

@@ -1,6 +1,6 @@
module DanbooruImageResizer
def resize(read_path, write_path, width, height, resize_quality = 90)
image = Magick::Image.read(read_path).first
def resize(file, width, height, resize_quality = 90)
image = Magick::Image.read(file.path).first
geometry = "#{width}x>"
if width == Danbooru.config.small_image_width
@@ -17,14 +17,15 @@ module DanbooruImageResizer
image = flatten(image, width, height)
image.strip!
image.write(write_path) do
output_file = Tempfile.new(binmode: true)
image.write("jpeg:" + output_file.path) do
self.quality = resize_quality
# setting PlaneInterlace enables progressive encoding for JPEGs
self.interlace = Magick::PlaneInterlace
end
image.destroy!
FileUtils.chmod(0664, write_path)
output_file
end
def flatten(image, width, height)

17
app/logical/downloads/file.rb
View File

@@ -3,9 +3,9 @@ module Downloads
class Error < Exception ; end
attr_reader :data, :options
attr_accessor :source, :original_source, :downloaded_source, :file_path
attr_accessor :source, :original_source, :downloaded_source
def initialize(source, file_path, options = {})
def initialize(source, options = {})
# source can potentially get rewritten in the course
# of downloading a file, so check it again
@source = source
@@ -14,9 +14,6 @@ module Downloads
# the URL actually downloaded after rewriting the original source.
@downloaded_source = nil
# where to save the download
@file_path = file_path
# we sometimes need to capture data from the source page
@data = {}
@@ -35,12 +32,13 @@ module Downloads
def download!
url, headers, @data = before_download(@source, @data)
::File.open(@file_path, "wb") do |out|
http_get_streaming(uncached_url(url, headers), out, headers)
end
output_file = Tempfile.new(binmode: true)
http_get_streaming(uncached_url(url, headers), output_file, headers)
@downloaded_source = url
@source = after_download(url)
output_file
end
def before_download(url, datums)
@@ -91,7 +89,8 @@ module Downloads
end
if res.success?
return
file.rewind
return file
else
raise Error.new("HTTP error code: #{res.code} #{res.message}")
end

9
app/logical/null_backup_service.rb
View File

@@ -1,9 +0,0 @@
class NullBackupService
def backup(file_path, options = {})
# do nothing
end
def delete(file_path, options = {})
# do nothing
end
end

29
app/logical/pixiv_ugoira_converter.rb
View File

@@ -1,13 +1,9 @@
class PixivUgoiraConverter
def self.convert(source_path, output_path, preview_path, frame_data)
folder = Zip::File.new(source_path)
write_webm(folder, output_path, frame_data)
write_preview(folder, preview_path)
RemoteFileManager.new(output_path).distribute
RemoteFileManager.new(preview_path).distribute
end
def self.generate_webm(ugoira_file, frame_data)
folder = Zip::File.new(ugoira_file.path)
output_file = Tempfile.new(binmode: true)
write_path = output_file.path
def self.write_webm(folder, write_path, frame_data)
Dir.mktmpdir do |tmpdir|
FileUtils.mkdir_p("#{tmpdir}/images")
folder.each_with_index do |file, i|
@@ -64,14 +60,17 @@ class PixivUgoiraConverter
return
end
end
output_file
end
def self.write_preview(folder, path)
Dir.mktmpdir do |tmpdir|
file = folder.first
temp_path = File.join(tmpdir, file.name)
file.extract(temp_path)
DanbooruImageResizer.resize(temp_path, path, Danbooru.config.small_image_width, Danbooru.config.small_image_width, 85)
end
def self.generate_preview(ugoira_file)
file = Tempfile.new(binmode: true)
zipfile = Zip::File.new(ugoira_file.path)
zipfile.entries.first.extract(file.path) { true } # 'true' means overwrite the existing tempfile.
DanbooruImageResizer.resize(file, Danbooru.config.small_image_width, Danbooru.config.small_image_width, 85)
ensure
file.close!
end
end

22
app/logical/pixiv_ugoira_service.rb
View File

@@ -1,32 +1,10 @@
class PixivUgoiraService
attr_reader :width, :height, :frame_data, :content_type
def self.regen(post)
service = new()
service.load(
:is_ugoira => true,
:ugoira_frame_data => post.pixiv_ugoira_frame_data.data
)
service.generate_resizes(post.file_path, post.large_file_path, post.preview_file_path, false)
end
def save_frame_data(post)
PixivUgoiraFrameData.create(:data => @frame_data, :content_type => @content_type, :post_id => post.id)
end
def generate_resizes(source_path, output_path, preview_path, delay = true)
# Run this a bit in the future to give the upload process time to move the file
if delay
PixivUgoiraConverter.delay(:queue => Socket.gethostname, :run_at => 10.seconds.from_now, :priority => -1).convert(source_path, output_path, preview_path, @frame_data)
else
PixivUgoiraConverter.convert(source_path, output_path, preview_path, @frame_data)
end
# since the resizes will be delayed, just touch the output file so the
# file distribution wont break
FileUtils.touch([output_path, preview_path])
end
def calculate_dimensions(source_path)
folder = Zip::File.new(source_path)
tempfile = Tempfile.new("ugoira-dimensions")

52
app/logical/s3_backup_service.rb
View File

@@ -1,52 +0,0 @@
class S3BackupService < BackupService
attr_reader :client, :bucket
def initialize(client: nil, bucket: Danbooru.config.aws_s3_bucket_name)
@credentials = Aws::Credentials.new(Danbooru.config.aws_access_key_id, Danbooru.config.aws_secret_access_key)
@client = client || Aws::S3::Client.new(credentials: @credentials, region: "us-east-1", logger: Logger.new(STDOUT))
@bucket = bucket
end
def backup(file_path, type: nil, **options)
keys = s3_keys(file_path, type)
keys.each do |key|
upload_to_s3(key, file_path)
end
end
def delete(file_path, type: nil)
keys = s3_keys(file_path, type)
keys.each do |key|
delete_from_s3(key)
end
end
protected
def s3_keys(file_path, type)
name = File.basename(file_path)
case type
when :original
[name]
when :preview
["preview/#{name}"]
when :large
["sample/#{name}"]
else
raise ArgumentError.new("Unknown type: #{type}")
end
end
def delete_from_s3(key)
client.delete_object(bucket: bucket, key: key)
rescue Aws::S3::Errors::NoSuchKey
# ignore
end
def upload_to_s3(key, file_path)
File.open(file_path, "rb") do |body|
base64_md5 = Digest::MD5.base64digest(File.read(file_path))
client.put_object(acl: "public-read", bucket: bucket, key: key, body: body, content_md5: base64_md5)
end
end
end

106
app/logical/storage_manager.rb Normal file
View File

@@ -0,0 +1,106 @@
class StorageManager
class Error < StandardError; end
DEFAULT_BASE_URL = Rails.application.routes.url_helpers.root_url + "data"
DEFAULT_BASE_DIR = "#{Rails.root}/public/data"
attr_reader :base_url, :base_dir, :hierarchical, :tagged_filenames, :large_image_prefix
def initialize(base_url: DEFAULT_BASE_URL, base_dir: DEFAULT_BASE_DIR, hierarchical: false, tagged_filenames: Danbooru.config.enable_seo_post_urls, large_image_prefix: Danbooru.config.large_image_prefix)
@base_url = base_url.chomp("/")
@base_dir = base_dir
@hierarchical = hierarchical
@tagged_filenames = tagged_filenames
@large_image_prefix = large_image_prefix
end
# Store the given file at the given path. If a file already exists at that
# location it should be overwritten atomically. Either the file is fully
# written, or an error is raised and the original file is left unchanged. The
# file should never be in a partially written state.
def store(io, path)
raise NotImplementedError, "store not implemented"
end
# Delete the file at the given path. If the file doesn't exist, no error
# should be raised.
def delete(path)
raise NotImplementedError, "delete not implemented"
end
# Return a readonly copy of the file located at the given path.
def open(path)
raise NotImplementedError, "open not implemented"
end
def store_file(io, post, type)
store(io, file_path(post.md5, post.file_ext, type))
end
def delete_file(post_id, md5, file_ext, type)
delete(file_path(md5, file_ext, type))
end
def open_file(post, type)
open(file_path(post.md5, post.file_ext, type))
end
def file_url(post, type)
subdir = subdir_for(post.md5)
file = file_name(post.md5, post.file_ext, type)
if type == :preview && !post.has_preview?
"#{base_url}/images/download-preview.png"
elsif type == :preview
"#{base_url}/preview/#{subdir}#{file}"
elsif type == :large && post.has_large?
"#{base_url}/sample/#{subdir}#{seo_tags(post)}#{file}"
else
"#{base_url}/#{subdir}#{seo_tags(post)}#{file}"
end
end
protected
def file_path(md5, file_ext, type)
subdir = subdir_for(md5)
file = file_name(md5, file_ext, type)
case type
when :preview
"#{base_dir}/preview/#{subdir}#{file}"
when :large
"#{base_dir}/sample/#{subdir}#{file}"
when :original
"#{base_dir}/#{subdir}#{file}"
end
end
def file_name(md5, file_ext, type)
large_file_ext = (file_ext == "zip") ? "webm" : "jpg"
case type
when :preview
"#{md5}.jpg"
when :large
"#{large_image_prefix}#{md5}.#{large_file_ext}"
when :original
"#{md5}.#{file_ext}"
end
end
def subdir_for(md5)
if hierarchical
"#{md5[0..1]}/#{md5[2..3]}/"
else
""
end
end
def seo_tags(post, user = CurrentUser.user)
return "" if !tagged_filenames || user.disable_tagged_filenames?
tags = post.humanized_essential_tag_string.gsub(/[^a-z0-9]+/, "_").gsub(/(?:^_+)|(?:_+$)/, "").gsub(/_{2,}/, "_")
"__#{tags}__"
end
end

23
app/logical/storage_manager/hybrid.rb Normal file
View File

@@ -0,0 +1,23 @@
class StorageManager::Hybrid < StorageManager
attr_reader :submanager
def initialize(&block)
@submanager = block
end
def store_file(io, post, type)
submanager[post.id, post.md5, post.file_ext, type].store_file(io, post, type)
end
def delete_file(post_id, md5, file_ext, type)
submanager[post_id, md5, file_ext, type].delete_file(post_id, md5, file_ext, type)
end
def open_file(io, post, type)
submanager[post.id, post.md5, post.file_ext, type].open_file(post, type)
end
def file_url(post, type)
submanager[post.id, post.md5, post.file_ext, type].file_url(post, type)
end
end

25
app/logical/storage_manager/local.rb Normal file
View File

@@ -0,0 +1,25 @@
class StorageManager::Local < StorageManager
DEFAULT_PERMISSIONS = 0644
def store(io, dest_path)
temp_path = dest_path + "-" + SecureRandom.uuid + ".tmp"
FileUtils.mkdir_p(File.dirname(temp_path))
bytes_copied = IO.copy_stream(io, temp_path)
raise Error, "store failed: #{bytes_copied}/#{io.size} bytes copied" if bytes_copied != io.size
FileUtils.chmod(DEFAULT_PERMISSIONS, temp_path)
File.rename(temp_path, dest_path)
rescue StandardError => e
FileUtils.rm_f(temp_path)
raise Error, e
end
def delete(path)
FileUtils.rm_f(path)
end
def open(path)
File.open(path, "r", binmode: true)
end
end

13
app/logical/storage_manager/null.rb Normal file
View File

@@ -0,0 +1,13 @@
class StorageManager::Null < StorageManager
def store(io, path)
# no-op
end
def delete(path)
# no-op
end
def open(path)
# no-op
end
end

43
app/logical/storage_manager/s3.rb Normal file
View File

@@ -0,0 +1,43 @@
class StorageManager::S3 < StorageManager
# https://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Client.html#initialize-instance_method
DEFAULT_S3_OPTIONS = {
region: Danbooru.config.aws_region,
credentials: Danbooru.config.aws_credentials,
logger: Rails.logger,
}
# https://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Client.html#put_object-instance_method
DEFAULT_PUT_OPTIONS = {
acl: "public-read",
storage_class: "STANDARD", # STANDARD, STANDARD_IA, REDUCED_REDUNDANCY
cache_control: "public, max-age=#{1.year.to_i}",
#content_type: "image/jpeg" # XXX should set content type
}
attr_reader :bucket, :client, :s3_options
def initialize(bucket, client: nil, s3_options: {}, **options)
@bucket = bucket
@s3_options = DEFAULT_S3_OPTIONS.merge(s3_options)
@client = client || Aws::S3::Client.new(**@s3_options)
super(**options)
end
def store(io, path)
data = io.read
base64_md5 = Digest::MD5.base64digest(data)
client.put_object(bucket: bucket, key: path, body: data, content_md5: base64_md5, **DEFAULT_PUT_OPTIONS)
end
def delete(path)
client.delete_object(bucket: bucket, key: path)
rescue Aws::S3::Errors::NoSuchKey
# ignore
end
def open(path)
file = Tempfile.new(binmode: true)
client.get_object(bucket: bucket: key: path, response_target: file)
file
end
end

76
app/logical/storage_manager/sftp.rb Normal file
View File

@@ -0,0 +1,76 @@
class StorageManager::SFTP < StorageManager
DEFAULT_PERMISSIONS = 0644
# http://net-ssh.github.io/net-ssh/Net/SSH.html#method-c-start
DEFAULT_SSH_OPTIONS = {
timeout: 10,
logger: Rails.logger,
verbose: :fatal,
non_interactive: true,
}
attr_reader :hosts, :ssh_options
def initialize(*hosts, ssh_options: {}, **options)
@hosts = hosts
@ssh_options = DEFAULT_SSH_OPTIONS.merge(ssh_options)
super(**options)
end
def store(file, dest_path)
temp_upload_path = dest_path + "-" + SecureRandom.uuid + ".tmp"
dest_backup_path = dest_path + "-" + SecureRandom.uuid + ".bak"
each_host do |host, sftp|
begin
sftp.upload!(file.path, temp_upload_path)
sftp.setstat!(temp_upload_path, permissions: DEFAULT_PERMISSIONS)
# `rename!` can't overwrite existing files, so if a file already exists
# at dest_path we move it out of the way first.
force { sftp.rename!(dest_path, dest_backup_path) }
force { sftp.rename!(temp_upload_path, dest_path) }
rescue StandardError => e
# if anything fails, try to move the original file back in place (if it was moved).
force { sftp.rename!(dest_backup_path, dest_path) }
raise Error, e
ensure
force { sftp.remove!(temp_upload_path) }
force { sftp.remove!(dest_backup_path) }
end
end
end
def delete(dest_path)
each_host do |host, sftp|
force { sftp.remove!(dest_path) }
end
end
def open(dest_path)
file = Tempfile.new(binmode: true)
Net::SFTP.start(hosts.first, nil, ssh_options) do |sftp|
sftp.download!(dest_path, file.path)
end
file
end
protected
# Ignore "no such file" exceptions for the given operation.
def force
yield
rescue Net::SFTP::StatusException => e
raise Error, e unless e.description == "no such file"
end
def each_host
hosts.each do |host|
Net::SFTP.start(host, nil, ssh_options) do |sftp|
yield host, sftp
end
end
end
end

202
app/models/post.rb
View File

@@ -29,7 +29,6 @@ class Post < ApplicationRecord
before_save :set_tag_counts
before_save :set_pool_category_pseudo_tags
before_create :autoban
after_save :queue_backup, if: :md5_changed?
after_save :create_version
after_save :update_parent_on_save
after_save :apply_post_metatags
@@ -94,144 +93,76 @@ class Post < ApplicationRecord
extend ActiveSupport::Concern
module ClassMethods
def delete_files(post_id, file_path, large_file_path, preview_file_path, force: false)
unless force
# XXX should pass in the md5 instead of parsing it.
preview_file_path =~ %r!/data/preview/(?:test\.)?([a-z0-9]{32})\.jpg\z!
md5 = $1
if Post.where(md5: md5).exists?
raise DeletionError.new("Files still in use; skipping deletion.")
end
def delete_files(post_id, md5, file_ext, force: false)
if Post.where(md5: md5).exists? && !force
raise DeletionError.new("Files still in use; skipping deletion.")
end
backup_service = Danbooru.config.backup_service
backup_service.delete(file_path, type: :original)
backup_service.delete(large_file_path, type: :large)
backup_service.delete(preview_file_path, type: :preview)
Danbooru.config.storage_manager.delete_file(post_id, md5, file_ext, :original)
Danbooru.config.storage_manager.delete_file(post_id, md5, file_ext, :large)
Danbooru.config.storage_manager.delete_file(post_id, md5, file_ext, :preview)
# the large file and the preview don't necessarily exist. if so errors will be ignored.
FileUtils.rm_f(file_path)
FileUtils.rm_f(large_file_path)
FileUtils.rm_f(preview_file_path)
RemoteFileManager.new(file_path).delete
RemoteFileManager.new(large_file_path).delete
RemoteFileManager.new(preview_file_path).delete
Danbooru.config.backup_storage_manager.delete_file(post_id, md5, file_ext, :original)
Danbooru.config.backup_storage_manager.delete_file(post_id, md5, file_ext, :large)
Danbooru.config.backup_storage_manager.delete_file(post_id, md5, file_ext, :preview)
if Danbooru.config.cloudflare_key
md5, ext = File.basename(file_path).split(".")
CloudflareService.new.delete(md5, ext)
CloudflareService.new.delete(md5, file_ext)
end
end
end
def queue_delete_files(grace_period)
Post.delay(queue: "default", run_at: Time.now + grace_period).delete_files(id, md5, file_ext)
end
def delete_files
Post.delete_files(id, file_path, large_file_path, preview_file_path, force: true)
Post.delete_files(id, md5, file_ext, force: true)
end
def distribute_files
if Danbooru.config.build_file_url(self) =~ /^http/
# this post is archived
RemoteFileManager.new(file_path).distribute_to_archive(Danbooru.config.build_file_url(self))
RemoteFileManager.new(preview_file_path).distribute if has_preview?
RemoteFileManager.new(large_file_path).distribute_to_archive(Danbooru.config.build_large_file_url(self)) if has_large?
else
RemoteFileManager.new(file_path).distribute
RemoteFileManager.new(preview_file_path).distribute if has_preview?
RemoteFileManager.new(large_file_path).distribute if has_large?
end
def distribute_files(file, sample_file, preview_file)
storage_manager.store_file(file, self, :original)
storage_manager.store_file(sample_file, self, :large) if sample_file.present?
storage_manager.store_file(preview_file, self, :preview) if preview_file.present?
backup_storage_manager.store_file(file, self, :original)
backup_storage_manager.store_file(sample_file, self, :large) if sample_file.present?
backup_storage_manager.store_file(preview_file, self, :preview) if preview_file.present?
end
def file_path_prefix
Rails.env == "test" ? "test." : ""
def backup_storage_manager
Danbooru.config.backup_storage_manager
end
def file_path
"#{Rails.root}/public/data/#{file_path_prefix}#{md5}.#{file_ext}"
def storage_manager
Danbooru.config.storage_manager
end
def large_file_path
if has_large?
"#{Rails.root}/public/data/sample/#{file_path_prefix}#{Danbooru.config.large_image_prefix}#{md5}.#{large_file_ext}"
else
file_path
end
end
def large_file_ext
if is_ugoira?
"webm"
else
"jpg"
end
end
def preview_file_path
"#{Rails.root}/public/data/preview/#{file_path_prefix}#{md5}.jpg"
end
def file_name
"#{file_path_prefix}#{md5}.#{file_ext}"
def file(type = :original)
storage_manager.open_file(self, type)
end
def file_url
Danbooru.config.build_file_url(self)
end
# this is for the 640x320 version
def cropped_file_url
storage_manager.file_url(self, :original)
end
def large_file_url
if has_large?
Danbooru.config.build_large_file_url(self)
else
file_url
end
end
def seo_tag_string
if Danbooru.config.enable_seo_post_urls && !CurrentUser.user.disable_tagged_filenames?
"__#{seo_tags}__"
else
nil
end
end
def seo_tags
@seo_tags ||= humanized_essential_tag_string.gsub(/[^a-z0-9]+/, "_").gsub(/(?:^_+)|(?:_+$)/, "").gsub(/_{2,}/, "_")
storage_manager.file_url(self, :large)
end
def preview_file_url
if !has_preview?
return "/images/download-preview.png"
end
"/data/preview/#{file_path_prefix}#{md5}.jpg"
end
def complete_preview_file_url
"http://#{Danbooru.config.hostname}#{preview_file_url}"
storage_manager.file_url(self, :preview)
end
def open_graph_image_url
if is_image?
if has_large?
if Danbooru.config.build_large_file_url(self) =~ /http/
large_file_url
else
"http://#{Danbooru.config.hostname}#{large_file_url}"
end
large_file_url
else
if Danbooru.config.build_file_url(self) =~ /http/
file_url
else
"http://#{Danbooru.config.hostname}#{file_url}"
end
file_url
end
else
complete_preview_file_url
preview_file_url
end
end
@@ -243,36 +174,10 @@ class Post < ApplicationRecord
end
end
def file_path_for(user)
if user.default_image_size == "large" && image_width > Danbooru.config.large_image_width
large_file_path
else
file_path
end
end
def is_image?
file_ext =~ /jpg|jpeg|gif|png/i
end
def is_animated_gif?
if file_ext =~ /gif/i && File.exists?(file_path)
return Magick::Image.ping(file_path).length > 1
else
return false
end
end
def is_animated_png?
if file_ext =~ /png/i && File.exists?(file_path)
apng = APNGInspector.new(file_path)
apng.inspect!
return apng.animated?
else
return false
end
end
def is_flash?
file_ext =~ /swf/i
end
@@ -294,9 +199,7 @@ class Post < ApplicationRecord
end
def has_preview?
# for video/ugoira we don't want to try and render a preview that
# might doesn't exist yet
is_image? || ((is_video? || is_ugoira?) && File.exists?(preview_file_path))
is_image? || is_video? || is_ugoira?
end
def has_dimensions?
@@ -304,24 +207,7 @@ class Post < ApplicationRecord
end
def has_ugoira_webm?
created_at < 1.minute.ago || (File.exists?(preview_file_path) && File.size(preview_file_path) > 0)
end
end
module BackupMethods
extend ActiveSupport::Concern
def queue_backup
Post.delay(queue: "default", priority: -1).backup_file(file_path, id: id, type: :original)
Post.delay(queue: "default", priority: -1).backup_file(large_file_path, id: id, type: :large) if has_large?
Post.delay(queue: "default", priority: -1).backup_file(preview_file_path, id: id, type: :preview) if has_preview?
end
module ClassMethods
def backup_file(file_path, options = {})
backup_service = Danbooru.config.backup_service
backup_service.backup(file_path, options)
end
true
end
end
@@ -765,7 +651,6 @@ class Post < ApplicationRecord
return tags if !Danbooru.config.enable_dimension_autotagging
tags -= %w(incredibly_absurdres absurdres highres lowres huge_filesize flash webm mp4)
tags -= %w(animated_gif animated_png) if new_record?
if has_dimensions?
if image_width >= 10_000 || image_height >= 10_000
@@ -794,14 +679,6 @@ class Post < ApplicationRecord
tags << "huge_filesize"
end
if is_animated_gif?
tags << "animated_gif"
end
if is_animated_png?
tags << "animated_png"
end
if is_flash?
tags << "flash"
end
@@ -1747,8 +1624,8 @@ class Post < ApplicationRecord
end
def update_iqdb_async
if File.exists?(preview_file_path) && Post.iqdb_enabled?
Post.iqdb_sqs_service.send_message("update\n#{id}\n#{complete_preview_file_url}")
if Post.iqdb_enabled?
Post.iqdb_sqs_service.send_message("update\n#{id}\n#{preview_file_url}")
end
end
@@ -1854,7 +1731,6 @@ class Post < ApplicationRecord
end
include FileMethods
include BackupMethods
include ImageMethods
include ApprovalMethods
include PresenterMethods

8
app/models/post_replacement.rb
View File

@@ -24,6 +24,8 @@ class PostReplacement < ApplicationRecord
end
def process!
upload = nil
transaction do
upload = Upload.create!(
file: replacement_file,
@@ -47,7 +49,7 @@ class PostReplacement < ApplicationRecord
# md5/file_ext to delete the old files. if saving the post fails,
# this is rolled back so the job won't run.
if md5_changed
Post.delay(queue: "default", run_at: Time.now + DELETION_GRACE_PERIOD).delete_files(post.id, post.file_path, post.large_file_path, post.preview_file_path)
post.queue_delete_files(DELETION_GRACE_PERIOD)
end
self.file_ext = upload.file_ext
@@ -69,8 +71,6 @@ class PostReplacement < ApplicationRecord
if md5_changed
post.comments.create!({creator: User.system, body: comment_replacement_message, do_not_bump_post: true}, without_protection: true)
else
post.queue_backup
end
save!
@@ -79,7 +79,7 @@ class PostReplacement < ApplicationRecord
# point of no return: these things can't be rolled back, so we do them
# only after the transaction successfully commits.
post.distribute_files
upload.distribute_files(post)
post.update_iqdb_async
end

298
app/models/upload.rb
View File

@@ -10,14 +10,11 @@ class Upload < ApplicationRecord
belongs_to :uploader, :class_name => "User"
belongs_to :post
before_validation :initialize_uploader, :on => :create
before_validation :initialize_status, :on => :create
before_create :convert_cgi_file
after_destroy :delete_temp_file
validate :uploader_is_not_limited, :on => :create
validate :file_or_source_is_present, :on => :create
validate :rating_given
attr_accessible :file, :image_width, :image_height, :file_ext, :md5,
:file_size, :as_pending, :source, :file_path, :content_type, :rating,
:file_size, :as_pending, :source, :rating,
:tag_string, :status, :backtrace, :post_id, :md5_confirmation,
:parent_id, :server, :artist_commentary_title,
:artist_commentary_desc, :include_artist_commentary,
@@ -53,12 +50,6 @@ class Upload < ApplicationRecord
end
end
def validate_file_exists
unless file_path && File.exists?(file_path)
raise "file does not exist"
end
end
def validate_file_content_type
unless is_valid_content_type?
raise "invalid content type (only JPEG, PNG, GIF, SWF, MP4, and WebM files are allowed)"
@@ -75,12 +66,6 @@ class Upload < ApplicationRecord
end
end
def validate_md5_confirmation_after_move
if !md5_confirmation.blank? && md5_confirmation != Digest::MD5.file(md5_file_path).hexdigest
raise "md5 mismatch"
end
end
def rating_given
if rating.present?
return true
@@ -93,10 +78,14 @@ class Upload < ApplicationRecord
end
end
def tag_audio
if is_video? && video.audio_channels.present?
self.tag_string = "#{tag_string} video_with_sound"
end
def automatic_tags
return "" unless Danbooru.config.enable_dimension_autotagging
tags = []
tags << "video_with_sound" if is_video_with_audio?
tags << "animated_gif" if is_animated_gif?
tags << "animated_png" if is_animated_png?
tags.join(" ")
end
def validate_video_duration
@@ -110,35 +99,36 @@ class Upload < ApplicationRecord
module ConversionMethods
def process_upload
CurrentUser.scoped(uploader, uploader_ip_addr) do
begin
update_attribute(:status, "processing")
self.source = strip_source
self.source = source.to_s.strip
if is_downloadable?
self.downloaded_source, self.source = download_from_source(temp_file_path)
self.downloaded_source, self.source, self.file = download_from_source(source, referer_url)
else
self.file = self.file.tempfile
end
validate_file_exists
self.content_type = file_header_to_content_type(file_path)
self.file_ext = content_type_to_file_ext(content_type)
self.file_ext = file_header_to_file_ext(file)
self.file_size = file.size
self.md5 = Digest::MD5.file(file.path).hexdigest
validate_file_content_type
calculate_hash(file_path)
validate_md5_uniqueness
validate_md5_confirmation
tag_audio
validate_video_duration
calculate_file_size(file_path)
if has_dimensions?
calculate_dimensions(file_path)
end
generate_resizes(file_path)
move_file
validate_md5_confirmation_after_move
self.tag_string = "#{tag_string} #{automatic_tags}"
self.image_width, self.image_height = calculate_dimensions
save
end
end
def create_post_from_upload
post = convert_to_post
post.distribute_files
distribute_files(post)
if post.save
create_artist_commentary(post) if include_artist_commentary?
ugoira_service.save_frame_data(post) if is_ugoira?
@@ -151,6 +141,14 @@ class Upload < ApplicationRecord
post
end
def distribute_files(post)
preview_file, sample_file = generate_resizes
post.distribute_files(file, sample_file, preview_file)
ensure
preview_file.try(:close!)
sample_file.try(:close!)
end
def process!(force = false)
@tries ||= 0
return if !force && status =~ /processing|completed|error/
@@ -170,13 +168,9 @@ class Upload < ApplicationRecord
rescue Exception => x
update_attributes(:status => "error: #{x.class} - #{x.message}", :backtrace => x.backtrace.join("\n"))
nil
ensure
delete_temp_file
end
def async_conversion?
is_ugoira?
ensure
file.try(:close!)
end
def ugoira_service
@@ -211,23 +205,6 @@ class Upload < ApplicationRecord
end
module FileMethods
def delete_temp_file(path = nil)
FileUtils.rm_f(path || temp_file_path)
end
def move_file
FileUtils.mv(file_path, md5_file_path)
end
def calculate_file_size(source_path)
self.file_size = File.size(source_path)
end
# Calculates the MD5 based on whatever is in temp_file_path
def calculate_hash(source_path)
self.md5 = Digest::MD5.file(source_path).hexdigest
end
def is_image?
%w(jpg gif png).include?(file_ext)
end
@@ -240,75 +217,68 @@ class Upload < ApplicationRecord
%w(webm mp4).include?(file_ext)
end
def is_video_with_audio?
is_video? && video.audio_channels.present?
end
def is_ugoira?
%w(zip).include?(file_ext)
end
def is_animated_gif?
file_ext == "gif" && Magick::Image.ping(file.path).length > 1
end
def is_animated_png?
file_ext == "png" && APNGInspector.new(file.path).inspect!.animated?
end
end
module ResizerMethods
def generate_resizes(source_path)
generate_resize_for(Danbooru.config.small_image_width, Danbooru.config.small_image_width, source_path, 85)
def generate_resizes
if is_video?
preview_file = generate_video_preview_for(video, width, height)
elsif is_ugoira?
preview_file = PixivUgoiraConverter.generate_preview(file)
sample_file = PixivUgoiraConverter.generate_webm(file, ugoira_service.frame_data)
elsif is_image?
preview_file = DanbooruImageResizer.resize(file, Danbooru.config.small_image_width, Danbooru.config.small_image_width, 85)
if is_image? && image_width > Danbooru.config.large_image_width
generate_resize_for(Danbooru.config.large_image_width, nil, source_path)
if image_width > Danbooru.config.large_image_width
sample_file = DanbooruImageResizer.resize(file, Danbooru.config.large_image_width, nil, 90)
end
end
[preview_file, sample_file]
end
def generate_video_preview_for(width, height, output_path)
dimension_ratio = image_width.to_f / image_height
def generate_video_preview_for(video, width, height)
dimension_ratio = video.width.to_f / video.height
if dimension_ratio > 1
height = (width / dimension_ratio).to_i
else
width = (height * dimension_ratio).to_i
end
video.screenshot(output_path, {:seek_time => 0, :resolution => "#{width}x#{height}"})
FileUtils.chmod(0664, output_path)
end
def generate_resize_for(width, height, source_path, quality = 90)
unless File.exists?(source_path)
raise Error.new("file not found")
end
output_path = resized_file_path_for(width)
if is_image?
DanbooruImageResizer.resize(source_path, output_path, width, height, quality)
elsif is_ugoira?
if Delayed::Worker.delay_jobs
# by the time this runs we'll have moved source_path to md5_file_path
ugoira_service.generate_resizes(md5_file_path, resized_file_path_for(Danbooru.config.large_image_width), resized_file_path_for(Danbooru.config.small_image_width))
else
ugoira_service.generate_resizes(source_path, resized_file_path_for(Danbooru.config.large_image_width), resized_file_path_for(Danbooru.config.small_image_width), false)
end
elsif is_video?
generate_video_preview_for(width, height, output_path)
end
output_file = Tempfile.new(binmode: true)
video.screenshot(output_file.path, {:seek_time => 0, :resolution => "#{width}x#{height}"})
output_file
end
end
module DimensionMethods
# Figures out the dimensions of the image.
def calculate_dimensions(file_path)
def calculate_dimensions
if is_video?
self.image_width = video.width
self.image_height = video.height
[video.width, video.height]
elsif is_ugoira?
ugoira_service.calculate_dimensions(file_path)
self.image_width = ugoira_service.width
self.image_height = ugoira_service.height
ugoira_service.calculate_dimensions(file.path)
[ugoira_service.width, ugoira_service.height]
else
File.open(file_path, "rb") do |file|
image_size = ImageSpec.new(file)
self.image_width = image_size.width
self.image_height = image_size.height
end
image_size = ImageSpec.new(file.path)
[image_size.width, image_size.height]
end
end
# Does this file have image dimensions?
def has_dimensions?
%w(jpg gif png swf webm mp4 zip).include?(file_ext)
end
end
module ContentTypeMethods
@@ -316,136 +286,44 @@ class Upload < ApplicationRecord
file_ext =~ /jpg|gif|png|swf|webm|mp4|zip/
end
def content_type_to_file_ext(content_type)
case content_type
when "image/jpeg"
def file_header_to_file_ext(file)
case File.read(file.path, 16)
when /^\xff\xd8/n
"jpg"
when "image/gif"
when /^GIF87a/, /^GIF89a/
"gif"
when "image/png"
when /^\x89PNG\r\n\x1a\n/n
"png"
when "application/x-shockwave-flash"
when /^CWS/, /^FWS/, /^ZWS/
"swf"
when "video/webm"
when /^\x1a\x45\xdf\xa3/n
"webm"
when "video/mp4"
when /^....ftyp(?:isom|3gp5|mp42|MSNV|avc1)/
"mp4"
when "application/zip"
when /^PK\x03\x04/
"zip"
else
"bin"
end
end
def file_header_to_content_type(source_path)
case File.read(source_path, 16)
when /^\xff\xd8/n
"image/jpeg"
when /^GIF87a/, /^GIF89a/
"image/gif"
when /^\x89PNG\r\n\x1a\n/n
"image/png"
when /^CWS/, /^FWS/, /^ZWS/
"application/x-shockwave-flash"
when /^\x1a\x45\xdf\xa3/n
"video/webm"
when /^....ftyp(?:isom|3gp5|mp42|MSNV|avc1)/
"video/mp4"
when /^PK\x03\x04/
"application/zip"
else
"application/octet-stream"
end
end
end
module FilePathMethods
def md5_file_path
prefix = Rails.env == "test" ? "test." : ""
"#{Rails.root}/public/data/#{prefix}#{md5}.#{file_ext}"
end
def resized_file_path_for(width)
prefix = Rails.env == "test" ? "test." : ""
case width
when Danbooru.config.small_image_width
"#{Rails.root}/public/data/preview/#{prefix}#{md5}.jpg"
when Danbooru.config.large_image_width
"#{Rails.root}/public/data/sample/#{prefix}#{Danbooru.config.large_image_prefix}#{md5}.#{large_file_ext}"
end
end
def large_file_ext
if is_ugoira?
"webm"
else
"jpg"
end
end
def temp_file_path
@temp_file_path ||= File.join(Rails.root, "tmp", "upload_#{Time.now.to_f}.#{Process.pid}")
end
end
module DownloaderMethods
def strip_source
source.to_s.strip
end
# Determines whether the source is downloadable
def is_downloadable?
source =~ /^https?:\/\// && file_path.blank?
source =~ /^https?:\/\// && file.blank?
end
# Downloads the file to destination_path
def download_from_source(destination_path)
self.file_path = destination_path
download = Downloads::File.new(source, destination_path, :referer_url => referer_url)
download.download!
def download_from_source(source, referer_url)
download = Downloads::File.new(source, referer_url: referer_url)
file = download.download!
ugoira_service.load(download.data)
[download.downloaded_source, download.source]
end
end
module CgiFileMethods
def convert_cgi_file
return if file.blank? || file.size == 0
self.file_path = temp_file_path
if file.respond_to?(:tempfile) && file.tempfile
FileUtils.cp(file.tempfile.path, file_path)
else
File.open(file_path, 'wb') do |out|
out.write(file.read)
end
end
FileUtils.chmod(0664, file_path)
[download.downloaded_source, download.source, file]
end
end
module StatusMethods
def initialize_status
self.status = "pending"
end
def is_pending?
status == "pending"
end
@@ -480,7 +358,7 @@ class Upload < ApplicationRecord
module VideoMethods
def video
@video ||= FFMPEG::Movie.new(file_path)
@video ||= FFMPEG::Movie.new(file.path)
end
end
@@ -534,8 +412,6 @@ class Upload < ApplicationRecord
include DimensionMethods
include ContentTypeMethods
include DownloaderMethods
include FilePathMethods
include CgiFileMethods
include StatusMethods
include UploaderMethods
include VideoMethods

4
app/presenters/post_presenter.rb
View File

@@ -135,6 +135,10 @@ class PostPresenter < Presenter
@post.humanized_essential_tag_string
end
def filename_for_download
"#{humanized_essential_tag_string} - #{@post.md5}.#{@post.file_ext}"
end
def categorized_tag_groups
string = []

2
app/views/comments/index.atom.builder
View File

@@ -11,7 +11,7 @@ atom_feed(root_url: comments_url(host: Danbooru.config.hostname)) do |feed|
feed.entry(comment, published: comment.created_at, updated: comment.updated_at) do |entry|
entry.title("@#{comment.creator_name} on post ##{comment.post_id} (#{comment.post.humanized_essential_tag_string})")
entry.content(<<-EOS.strip_heredoc, type: "html")
<img src="#{comment.post.complete_preview_file_url}"/>
<img src="#{comment.post.preview_file_url}"/>
#{format_text(comment.body)}
EOS

2
app/views/posts/partials/show/_options.html.erb
View File

@@ -4,7 +4,7 @@
<% if CurrentUser.is_member? %>
<li><%= link_to "Favorite", favorites_path(:post_id => post.id), :remote => true, :method => :post, :id => "add-to-favorites", :title => "Shortcut is F" %></li>
<li><%= link_to "Unfavorite", favorite_path(post), :remote => true, :method => :delete, :id => "remove-from-favorites" %></li>
<li><%= link_to_if post.visible?, "Download", post.file_url, :download => post.presenter.humanized_essential_tag_string + " - " + post.file_name %></li>
<li><%= link_to_if post.visible?, "Download", post.file_url, download: post.presenter.filename_for_download %></li>
<li id="add-to-pool-list"><%= link_to "Add to pool", "#", :id => "pool" %></li>
<% if post.is_note_locked? %>
<li id="add-notes-list"><span id="note-locked-notice">Note locked</span></li>

76
config/danbooru_default_config.rb
View File

@@ -99,20 +99,6 @@ module Danbooru
true
end
# What method to use to backup images.
#
# NullBackupService: Don't backup images at all.
#
# S3BackupService: Backup to Amazon S3. Must configure aws_access_key_id,
# aws_secret_access_key, and aws_s3_bucket_name. Bucket must exist and be writable.
def backup_service
if Rails.env.production?
S3BackupService.new
else
NullBackupService.new
end
end
# What method to use to store images.
# local_flat: Store every image in one directory.
# local_hierarchy: Store every image in a hierarchical directory, based on the post's MD5 hash. On some file systems this may be faster.
@@ -222,12 +208,58 @@ module Danbooru
"danbooru"
end
def build_file_url(post)
"/data/#{post.file_path_prefix}/#{post.md5}.#{post.file_ext}"
# The method to use for storing image files.
def storage_manager
# Store files on the local filesystem.
# base_dir - where to store files (default: under public/data)
# base_url - where to serve files from (default: http://#{hostname}/data)
# hierarchical: false - store files in a single directory
# hierarchical: true - store files in a hierarchical directory structure, based on the MD5 hash
StorageManager::Local.new(base_dir: "#{Rails.root}/public/data", hierarchical: false)
# Store files on one or more remote host(s). Configure SSH settings in
# ~/.ssh_config or in the ssh_options param (ref: http://net-ssh.github.io/net-ssh/Net/SSH.html#method-c-start)
# StorageManager::SFTP.new("i1.example.com", "i2.example.com", base_dir: "/mnt/backup", hierarchical: false, ssh_options: {})
# Store files in an S3 bucket. The bucket must already exist and be
# writable by you. Configure your S3 settings in aws_region and
# aws_credentials below, or in the s3_options param (ref:
# https://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Client.html#initialize-instance_method)
# StorageManager::S3.new("my_s3_bucket", base_url: "https://my_s3_bucket.s3.amazonaws.com/", s3_options: {})
# Select the storage method based on the post's id and type (preview, large, or original).
# StorageManager::Hybrid.new do |id, md5, file_ext, type|
# ssh_options = { user: "danbooru" }
#
# if type.in?([:large, :original]) && id.in?(0..850_000)
# StorageManager::SFTP.new("raikou1.donmai.us", base_url: "https://raikou1.donmai.us", base_dir: "/path/to/files", hierarchical: true, ssh_options: ssh_options)
# elsif type.in?([:large, :original]) && id.in?(850_001..2_000_000)
# StorageManager::SFTP.new("raikou2.donmai.us", base_url: "https://raikou2.donmai.us", base_dir: "/path/to/files", hierarchical: true, ssh_options: ssh_options)
# elsif type.in?([:large, :original]) && id.in?(2_000_001..3_000_000)
# StorageManager::SFTP.new(*all_server_hosts, base_url: "https://hijiribe.donmai.us/data", ssh_options: ssh_options)
# else
# StorageManager::SFTP.new(*all_server_hosts, ssh_options: ssh_options)
# end
# end
end
def build_large_file_url(post)
"/data/sample/#{post.file_path_prefix}#{Danbooru.config.large_image_prefix}#{post.md5}.#{post.large_file_ext}"
# The method to use for backing up image files.
def backup_storage_manager
# Don't perform any backups.
StorageManager::Null.new
# Backup files to /mnt/backup on the local filesystem.
# StorageManager::Local.new(base_dir: "/mnt/backup", hierarchical: false)
# Backup files to /mnt/backup on a remote system. Configure SSH settings
# in ~/.ssh_config or in the ssh_options param (ref: http://net-ssh.github.io/net-ssh/Net/SSH.html#method-c-start)
# StorageManager::SFTP.new("www.example.com", base_dir: "/mnt/backup", ssh_options: {})
# Backup files to an S3 bucket. The bucket must already exist and be
# writable by you. Configure your S3 settings in aws_region and
# aws_credentials below, or in the s3_options param (ref:
# https://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Client.html#initialize-instance_method)
# StorageManager::S3.new("my_s3_bucket_name", s3_options: {})
end
#TAG CONFIGURATION
@@ -611,6 +643,14 @@ module Danbooru
end
# AWS config options
def aws_region
"us-east-1"
end
def aws_credentials
Aws::Credentials.new(Danbooru.config.aws_access_key_id, Danbooru.config.aws_secret_access_key)
end
def aws_access_key_id
end

47
test/factories/upload.rb
View File

@@ -1,5 +1,3 @@
require 'fileutils'
FactoryGirl.define do
factory(:upload) do
rating "s"
@@ -15,51 +13,10 @@ FactoryGirl.define do
end
factory(:jpg_upload) do
content_type "image/jpeg"
file do
f = Tempfile.new
f.write(File.read("#{Rails.root}/test/files/test.jpg"))
f.seek(0)
f
end
end
factory(:exif_jpg_upload) do
content_type "image/jpeg"
file_path do
FileUtils.cp("#{Rails.root}/test/files/test-exif-small.jpg", "#{Rails.root}/tmp")
"#{Rails.root}/tmp/test-exif-small.jpg"
end
end
factory(:blank_jpg_upload) do
content_type "image/jpeg"
file_path do
FileUtils.cp("#{Rails.root}/test/files/test-blank.jpg", "#{Rails.root}/tmp")
"#{Rails.root}/tmp/test-blank.jpg"
end
end
factory(:large_jpg_upload) do
file_ext "jpg"
content_type "image/jpeg"
file_path do
FileUtils.cp("#{Rails.root}/test/files/test-large.jpg", "#{Rails.root}/tmp")
"#{Rails.root}/tmp/test-large.jpg"
end
end
factory(:png_upload) do
file_path do
FileUtils.cp("#{Rails.root}/test/files/test.png", "#{Rails.root}/tmp")
"#{Rails.root}/tmp/test.png"
end
end
factory(:gif_upload) do
file_path do
FileUtils.cp("#{Rails.root}/test/files/test.gif", "#{Rails.root}/tmp")
"#{Rails.root}/tmp/test.gif"
IO.copy_stream("#{Rails.root}/test/files/test.jpg", f.path)
ActionDispatch::Http::UploadedFile.new(tempfile: f, filename: "test.jpg")
end
end
end

2
test/functional/post_replacements_controller_test.rb
View File

@@ -39,7 +39,7 @@ class PostReplacementsControllerTest < ActionController::TestCase
assert_response :success
assert_equal("https://www.google.com/intl/en_ALL/images/logo.gif", @post.source)
assert_equal("e80d1c59a673f560785784fb1ac10959", @post.md5)
assert_equal("e80d1c59a673f560785784fb1ac10959", Digest::MD5.file(@post.file_path).hexdigest)
assert_equal("e80d1c59a673f560785784fb1ac10959", Digest::MD5.file(@post.file(:original)).hexdigest)
end
end

5
test/test_helper.rb
View File

@@ -38,9 +38,14 @@ class ActiveSupport::TestCase
mock_missed_search_service!
WebMock.allow_net_connect!
Danbooru.config.stubs(:enable_sock_puppet_validation?).returns(false)
storage_manager = StorageManager::Local.new(base_dir: "#{Rails.root}/public/data/test")
Danbooru.config.stubs(:storage_manager).returns(storage_manager)
Danbooru.config.stubs(:backup_storage_manager).returns(StorageManager::Null.new)
end
teardown do
FileUtils.rm_rf(Danbooru.config.storage_manager.base_dir)
Cache.clear
end
end

12
test/test_helpers/download_helper.rb
View File

@@ -1,18 +1,14 @@
module DownloadTestHelper
def assert_downloaded(expected_filesize, source)
tempfile = Tempfile.new("danbooru-test")
download = Downloads::File.new(source, tempfile.path)
assert_nothing_raised(Downloads::File::Error) do
download.download!
tempfile = Downloads::File.new(source).download!
assert_equal(expected_filesize, tempfile.size, "Tested source URL: #{source}")
tempfile.close!
end
assert_equal(expected_filesize, tempfile.size, "Tested source URL: #{source}")
end
def assert_rewritten(expected_source, test_source)
tempfile = Tempfile.new("danbooru-test")
download = Downloads::File.new(test_source, tempfile.path)
download = Downloads::File.new(test_source)
rewritten_source, _, _ = download.before_download(test_source, {})
assert_match(expected_source, rewritten_source, "Tested source URL: #{test_source}")

2
test/test_helpers/iqdb_test_helper.rb
View File

@@ -23,7 +23,7 @@ module IqdbTestHelper
end
def mock_iqdb_matches!(post_or_source, matches)
source = post_or_source.is_a?(Post) ? post_or_source.complete_preview_file_url : post_or_source
source = post_or_source.is_a?(Post) ? post_or_source.preview_file_url : post_or_source
url = "http://localhost:3004/similar?key=hunter2&url=#{CGI.escape source}&ref"
body = matches.map { |post| { post_id: post.id } }.to_json

25
test/test_helpers/upload_test_helper.rb
View File

@@ -1,23 +1,10 @@
module UploadTestHelper
def upload_file(path, content_type, filename)
tempfile = Tempfile.new(filename)
FileUtils.copy_file(path, tempfile.path)
def upload_file(path)
file = Tempfile.new(binmode: true)
IO.copy_stream("#{Rails.root}/#{path}", file.path)
uploaded_file = ActionDispatch::Http::UploadedFile.new(tempfile: file, filename: File.basename(path))
(class << tempfile; self; end).class_eval do
alias local_path path
define_method(:tempfile) {self}
define_method(:original_filename) {filename}
define_method(:content_type) {content_type}
end
tempfile
end
def upload_jpeg(path)
upload_file(path, "image/jpeg", File.basename(path))
end
def upload_zip(path)
upload_file(path, "application/zip", File.basename(path))
yield uploaded_file if block_given?
uploaded_file
end
end

10
test/unit/downloads/art_station_test.rb
View File

@@ -5,9 +5,7 @@ module Downloads
context "a download for a (small) artstation image" do
setup do
@source = "https://cdnb3.artstation.com/p/assets/images/images/003/716/071/large/aoi-ogata-hate-city.jpg?1476754974"
@tempfile = Tempfile.new("danbooru-test")
@download = Downloads::File.new(@source, @tempfile.path)
@download.download!
@download = Downloads::File.new(@source)
end
should "download the large image instead" do
@@ -18,8 +16,7 @@ module Downloads
context "for an image where an original does not exist" do
setup do
@source = "https://cdna.artstation.com/p/assets/images/images/004/730/278/large/mendel-oh-dragonll.jpg"
@tempfile = Tempfile.new("danbooru-test")
@download = Downloads::File.new(@source, @tempfile.path)
@download = Downloads::File.new(@source)
@download.download!
end
@@ -38,8 +35,7 @@ module Downloads
context "a download for a https://$artist.artstation.com/projects/$id page" do
setup do
@source = "https://dantewontdie.artstation.com/projects/YZK5q"
@tempfile = Tempfile.new("danbooru-test")
@download = Downloads::File.new(@source, @tempfile.path)
@download = Downloads::File.new(@source)
@download.download!
end

5
test/unit/downloads/deviant_art_test.rb
View File

@@ -5,9 +5,8 @@ module Downloads
context "a download for a deviant art html page" do
setup do
@source = "http://starbitt.deviantart.com/art/09271X-636962118"
@tempfile = Tempfile.new("danbooru-test")
@download = Downloads::File.new(@source, @tempfile.path)
@download.download!
@download = Downloads::File.new(@source)
@tempfile = @download.download!
end
should "set the html page as the source" do

18
test/unit/downloads/file_test.rb
View File

@@ -5,12 +5,7 @@ module Downloads
context "A twitter video download" do
setup do
@source = "https://twitter.com/CincinnatiZoo/status/859073537713328129"
@tempfile = Tempfile.new("danbooru-test")
@download = Downloads::File.new(@source, @tempfile.path)
end
teardown do
@tempfile.close
@download = Downloads::File.new(@source)
end
should "preserve the twitter source" do
@@ -22,12 +17,8 @@ module Downloads
context "A post download" do
setup do
@source = "http://www.google.com/intl/en_ALL/images/logo.gif"
@download = Downloads::File.new(@source)
@tempfile = Tempfile.new("danbooru-test")
@download = Downloads::File.new(@source, @tempfile.path)
end
teardown do
@tempfile.close
end
context "that fails" do
@@ -49,10 +40,9 @@ module Downloads
end
should "store the file in the tempfile path" do
@download.download!
tempfile = @download.download!
assert_equal(@source, @download.source)
assert(::File.exists?(@tempfile.path), "temp file should exist")
assert(::File.size(@tempfile.path) > 0, "should have data")
assert_operator(tempfile.size, :>, 0, "should have data")
end
end
end

10
test/unit/downloads/pixiv_test.rb
View File

@@ -4,13 +4,9 @@ module Downloads
class PixivTest < ActiveSupport::TestCase
context "An ugoira site for pixiv" do
setup do
@tempfile = Tempfile.new("danbooru-test")
@download = Downloads::File.new("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364", @tempfile.path)
@download.download!
end
teardown do
@tempfile.unlink
@download = Downloads::File.new("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364")
@tempfile = @download.download!
@tempfile.close!
end
should "capture the data" do

19
test/unit/pixiv_ugoira_converter_test.rb
View File

@@ -3,9 +3,7 @@ require "test_helper"
class PixivUgoiraConverterTest < ActiveSupport::TestCase
context "An ugoira converter" do
setup do
@zipped_body = "#{Rails.root}/test/fixtures/ugoira.zip"
@write_file = Tempfile.new("converted")
@preview_write_file = Tempfile.new("preview")
@zipfile = upload_file("test/fixtures/ugoira.zip").tempfile
@frame_data = [
{"file" => "000000.jpg", "delay" => 200},
{"file" => "000001.jpg", "delay" => 200},
@@ -15,16 +13,11 @@ class PixivUgoiraConverterTest < ActiveSupport::TestCase
]
end
teardown do
@write_file.unlink
@preview_write_file.unlink
end
should "output to webm" do
@converter = PixivUgoiraConverter
@converter.convert(@zipped_body, @write_file.path, @preview_write_file.path, @frame_data)
assert_operator(File.size(@write_file.path), :>, 1_000)
assert_operator(File.size(@preview_write_file.path), :>, 0)
sample_file = PixivUgoiraConverter.generate_webm(@zipfile, @frame_data)
preview_file = PixivUgoiraConverter.generate_preview(@zipfile)
assert_operator(sample_file.size, :>, 1_000)
assert_operator(preview_file.size, :>, 0)
end
end
end
end

43
test/unit/post_replacement_test.rb
View File

@@ -1,15 +1,6 @@
require 'test_helper'
class PostReplacementTest < ActiveSupport::TestCase
def upload_file(path, filename, &block)
Tempfile.open do |file|
file.write(File.read(path))
file.seek(0)
uploaded_file = ActionDispatch::Http::UploadedFile.new(tempfile: file, filename: filename)
yield uploaded_file
end
end
def setup
super
@@ -68,7 +59,7 @@ class PostReplacementTest < ActiveSupport::TestCase
assert_equal(5969, @post.file_size)
assert_equal("png", @post.file_ext)
assert_equal("8f9327db2597fa57d2f42b4a6c5a9855", @post.md5)
assert_equal("8f9327db2597fa57d2f42b4a6c5a9855", Digest::MD5.file(@post.file_path).hexdigest)
assert_equal("8f9327db2597fa57d2f42b4a6c5a9855", Digest::MD5.file(@post.file).hexdigest)
assert_equal("https://www.google.com/images/branding/googlelogo/1x/googlelogo_color_272x92dp.png", @post.source)
end
end
@@ -102,7 +93,7 @@ class PostReplacementTest < ActiveSupport::TestCase
assert_equal(8558, @post.file_size)
assert_equal("gif", @post.file_ext)
assert_equal("e80d1c59a673f560785784fb1ac10959", @post.md5)
assert_equal("e80d1c59a673f560785784fb1ac10959", Digest::MD5.file(@post.file_path).hexdigest)
assert_equal("e80d1c59a673f560785784fb1ac10959", Digest::MD5.file(@post.file).hexdigest)
assert_equal("https://www.google.com/intl/en_ALL/images/logo.gif", @post.source)
end
@@ -155,18 +146,17 @@ class PostReplacementTest < ActiveSupport::TestCase
assert_equal(16275, @post.file_size)
assert_equal("png", @post.file_ext)
assert_equal("4ceadc314938bc27f3574053a3e1459a", @post.md5)
assert_equal("4ceadc314938bc27f3574053a3e1459a", Digest::MD5.file(@post.file_path).hexdigest)
assert_equal("4ceadc314938bc27f3574053a3e1459a", Digest::MD5.file(@post.file).hexdigest)
assert_equal("https://i.pximg.net/img-original/img/2017/04/04/08/54/15/62247350_p0.png", @post.source)
assert_equal("https://i.pximg.net/img-original/img/2017/04/04/08/54/15/62247350_p0.png", @post.replacements.last.replacement_url)
end
should "delete the old files after three days" do
old_file_path, old_preview_file_path, old_large_file_path = @post.file_path, @post.preview_file_path, @post.large_file_path
should "delete the old files after thirty days" do
old_file_path, old_preview_file_path = @post.file(:original).path, @post.file(:preview).path
@post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350")
assert(File.exists?(old_file_path))
assert(File.exists?(old_preview_file_path))
assert(File.exists?(old_large_file_path))
Timecop.travel(Time.now + PostReplacement::DELETION_GRACE_PERIOD + 1.day) do
Delayed::Worker.new.work_off
@@ -174,7 +164,6 @@ class PostReplacementTest < ActiveSupport::TestCase
assert_not(File.exists?(old_file_path))
assert_not(File.exists?(old_preview_file_path))
assert_not(File.exists?(old_large_file_path))
end
end
@@ -188,7 +177,7 @@ class PostReplacementTest < ActiveSupport::TestCase
assert_equal(2804, @post.file_size)
assert_equal("zip", @post.file_ext)
assert_equal("cad1da177ef309bf40a117c17b8eecf5", @post.md5)
assert_equal("cad1da177ef309bf40a117c17b8eecf5", Digest::MD5.file(@post.file_path).hexdigest)
assert_equal("cad1da177ef309bf40a117c17b8eecf5", Digest::MD5.file(@post.file).hexdigest)
assert_equal("https://i.pximg.net/img-zip-ugoira/img/2017/04/04/08/57/38/62247364_ugoira1920x1080.zip", @post.source)
assert_equal([{"file"=>"000000.jpg", "delay"=>125}, {"file"=>"000001.jpg", "delay"=>125}], @post.pixiv_ugoira_frame_data.data)
@@ -201,17 +190,15 @@ class PostReplacementTest < ActiveSupport::TestCase
@post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364")
@post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350")
assert(File.exists?(@post.file_path))
assert(File.exists?(@post.preview_file_path))
assert(File.exists?(@post.large_file_path))
assert_nothing_raised { @post.file(:original) }
assert_nothing_raised { @post.file(:preview) }
Timecop.travel(Time.now + PostReplacement::DELETION_GRACE_PERIOD + 1.day) do
Delayed::Worker.new.work_off
end
assert(File.exists?(@post.file_path))
assert(File.exists?(@post.preview_file_path))
assert(File.exists?(@post.large_file_path))
assert_nothing_raised { @post.file(:original) }
assert_nothing_raised { @post.file(:preview) }
end
end
@@ -231,14 +218,14 @@ class PostReplacementTest < ActiveSupport::TestCase
Delayed::Worker.new.work_off
end
assert(File.exists?(@post1.file_path))
assert(File.exists?(@post2.file_path))
assert_nothing_raised { @post1.file(:original) }
assert_nothing_raised { @post2.file(:original) }
end
end
context "a post with an uploaded file" do
should "work" do
upload_file("#{Rails.root}/test/files/test.png", "test.png") do |file|
upload_file("test/files/test.png") do |file|
@post.replace!(replacement_file: file, replacement_url: "")
assert_equal(@post.md5, Digest::MD5.file(file.tempfile).hexdigest)
assert_equal("file://test.png", @post.replacements.last.replacement_url)
@@ -268,7 +255,7 @@ class PostReplacementTest < ActiveSupport::TestCase
context "a post with the same file" do
should "not raise a duplicate error" do
upload_file("#{Rails.root}/test/files/test.jpg", "test.jpg") do |file|
upload_file("test/files/test.jpg") do |file|
assert_nothing_raised do
@post.replace!(replacement_file: file, replacement_url: "")
end
@@ -276,7 +263,7 @@ class PostReplacementTest < ActiveSupport::TestCase
end
should "not queue a deletion or log a comment" do
upload_file("#{Rails.root}/test/files/test.jpg", "test.jpg") do |file|
upload_file("test/files/test.jpg") do |file|
assert_no_difference(["@post.comments.count"]) do
@post.replace!(replacement_file: file, replacement_url: "")
end

10
test/unit/post_test.rb
View File

@@ -35,17 +35,15 @@ class PostTest < ActiveSupport::TestCase
end
should "delete the files" do
assert_equal(true, File.exists?(@post.preview_file_path))
assert_equal(true, File.exists?(@post.large_file_path))
assert_equal(true, File.exists?(@post.file_path))
assert_nothing_raised { @post.file(:preview) }
assert_nothing_raised { @post.file(:original) }
TestAfterCommit.with_commits(true) do
@post.expunge!
end
assert_equal(false, File.exists?(@post.preview_file_path))
assert_equal(false, File.exists?(@post.large_file_path))
assert_equal(false, File.exists?(@post.file_path))
assert_raise(StandardError) { @post.file(:preview) }
assert_raise(StandardError) { @post.file(:original) }
end
should "remove all favorites" do

124
test/unit/storage_manager_test.rb Normal file
View File

@@ -0,0 +1,124 @@
require 'test_helper'
class StorageManagerTest < ActiveSupport::TestCase
BASE_DIR = "#{Rails.root}/tmp/test-storage"
setup do
CurrentUser.ip_addr = "127.0.0.1"
end
context "StorageManager::Local" do
setup do
@storage_manager = StorageManager::Local.new(base_dir: BASE_DIR, base_url: "/data")
end
teardown do
FileUtils.rm_rf(BASE_DIR)
end
context "#store method" do
should "store the file" do
@storage_manager.store(StringIO.new("data"), "#{BASE_DIR}/test.txt")
assert("data", File.read("#{BASE_DIR}/test.txt"))
end
should "overwrite the file if it already exists" do
@storage_manager.store(StringIO.new("foo"), "#{BASE_DIR}/test.txt")
@storage_manager.store(StringIO.new("bar"), "#{BASE_DIR}/test.txt")
assert("bar", File.read("#{BASE_DIR}/test.txt"))
end
end
context "#delete method" do
should "delete the file" do
@storage_manager.store(StringIO.new("data"), "test.txt")
@storage_manager.delete("test.txt")
assert_not(File.exist?("#{BASE_DIR}/test.txt"))
end
should "not fail if the file doesn't exist" do
assert_nothing_raised { @storage_manager.delete("dne.txt") }
end
end
context "#store_file and #delete_file methods" do
setup do
@post = FactoryGirl.create(:post, file_ext: "png")
@storage_manager.store_file(StringIO.new("data"), @post, :preview)
@storage_manager.store_file(StringIO.new("data"), @post, :large)
@storage_manager.store_file(StringIO.new("data"), @post, :original)
@file_path = "#{BASE_DIR}/preview/#{@post.md5}.jpg"
@large_file_path = "#{BASE_DIR}/sample/sample-#{@post.md5}.jpg"
@preview_file_path = "#{BASE_DIR}/#{@post.md5}.#{@post.file_ext}"
end
should "store the files at the correct path" do
assert(File.exist?(@file_path))
assert(File.exist?(@large_file_path))
assert(File.exist?(@preview_file_path))
end
should "delete the files" do
@storage_manager.delete_file(@post.id, @post.md5, @post.file_ext, :preview)
@storage_manager.delete_file(@post.id, @post.md5, @post.file_ext, :large)
@storage_manager.delete_file(@post.id, @post.md5, @post.file_ext, :original)
assert_not(File.exist?(@file_path))
assert_not(File.exist?(@large_file_path))
assert_not(File.exist?(@preview_file_path))
end
end
context "#file_url method" do
should "return the correct urls" do
@post = FactoryGirl.create(:post, file_ext: "png")
@storage_manager.stubs(:tagged_filenames).returns(false)
assert_equal("/data/#{@post.md5}.png", @storage_manager.file_url(@post, :original))
assert_equal("/data/sample/sample-#{@post.md5}.jpg", @storage_manager.file_url(@post, :large))
assert_equal("/data/preview/#{@post.md5}.jpg", @storage_manager.file_url(@post, :preview))
end
end
end
context "StorageManager::Hybrid" do
setup do
@post1 = FactoryGirl.build(:post, id: 1, file_ext: "png")
@post2 = FactoryGirl.build(:post, id: 2, file_ext: "png")
@storage_manager = StorageManager::Hybrid.new do |id, md5, file_ext, type|
if id.odd?
StorageManager::Local.new(base_dir: "#{BASE_DIR}/i1", base_url: "/i1")
else
StorageManager::Local.new(base_dir: "#{BASE_DIR}/i2", base_url: "/i2")
end
end
end
teardown do
FileUtils.rm_rf(BASE_DIR)
end
context "#store_file method" do
should "store odd-numbered posts under /i1 and even-numbered posts under /i2" do
@storage_manager.store_file(StringIO.new("post1"), @post1, :original)
@storage_manager.store_file(StringIO.new("post2"), @post2, :original)
assert(File.exist?("#{BASE_DIR}/i1/#{@post1.md5}.png"))
assert(File.exist?("#{BASE_DIR}/i2/#{@post2.md5}.png"))
end
end
context "#file_url method" do
should "generate /i1 urls for odd posts and /i2 urls for even posts" do
assert_equal("/i1/#{@post1.md5}.png", @storage_manager.file_url(@post1, :original))
assert_equal("/i2/#{@post2.md5}.png", @storage_manager.file_url(@post2, :original))
end
end
end
end

175
test/unit/upload_test.rb
View File

@@ -17,21 +17,15 @@ class UploadTest < ActiveSupport::TestCase
teardown do
CurrentUser.user = nil
CurrentUser.ip_addr = nil
@upload.delete_temp_file if @upload
end
context "An upload" do
teardown do
FileUtils.rm_f(Dir.glob("#{Rails.root}/tmp/test.*"))
end
context "from a user that is limited" do
setup do
CurrentUser.user = FactoryGirl.create(:user, :created_at => 1.year.ago)
User.any_instance.stubs(:upload_limit).returns(0)
end
should "fail creation" do
@upload = FactoryGirl.build(:jpg_upload, :tag_string => "")
@upload.save
@@ -41,73 +35,51 @@ class UploadTest < ActiveSupport::TestCase
context "image size calculator" do
should "discover the dimensions for a compressed SWF" do
@upload = FactoryGirl.create(:upload, :file_path => "#{Rails.root}/test/files/compressed.swf")
@upload.calculate_dimensions(@upload.file_path)
assert_equal(607, @upload.image_width)
assert_equal(756, @upload.image_height)
@upload = FactoryGirl.create(:upload, file: upload_file("test/files/compressed.swf"))
assert_equal([607, 756], @upload.calculate_dimensions)
end
should "discover the dimensions for a JPG with JFIF data" do
@upload = FactoryGirl.create(:jpg_upload)
assert_nothing_raised {@upload.calculate_dimensions(@upload.file_path)}
assert_equal(500, @upload.image_width)
assert_equal(335, @upload.image_height)
assert_equal([500, 335], @upload.calculate_dimensions)
end
should "discover the dimensions for a JPG with EXIF data" do
@upload = FactoryGirl.create(:exif_jpg_upload)
assert_nothing_raised {@upload.calculate_dimensions(@upload.file_path)}
assert_equal(529, @upload.image_width)
assert_equal(600, @upload.image_height)
@upload = FactoryGirl.create(:upload, file: upload_file("test/files/test-exif-small.jpg"))
assert_equal([529, 600], @upload.calculate_dimensions)
end
should "discover the dimensions for a JPG with no header data" do
@upload = FactoryGirl.create(:blank_jpg_upload)
assert_nothing_raised {@upload.calculate_dimensions(@upload.file_path)}
assert_equal(668, @upload.image_width)
assert_equal(996, @upload.image_height)
@upload = FactoryGirl.create(:upload, file: upload_file("test/files/test-blank.jpg"))
assert_equal([668, 996], @upload.calculate_dimensions)
end
should "discover the dimensions for a PNG" do
@upload = FactoryGirl.create(:png_upload)
assert_nothing_raised {@upload.calculate_dimensions(@upload.file_path)}
assert_equal(768, @upload.image_width)
assert_equal(1024, @upload.image_height)
@upload = FactoryGirl.create(:upload, file: upload_file("test/files/test.png"))
assert_equal([768, 1024], @upload.calculate_dimensions)
end
should "discover the dimensions for a GIF" do
@upload = FactoryGirl.create(:gif_upload)
assert_nothing_raised {@upload.calculate_dimensions(@upload.file_path)}
assert_equal(400, @upload.image_width)
assert_equal(400, @upload.image_height)
@upload = FactoryGirl.create(:upload, file: upload_file("test/files/test.gif"))
assert_equal([400, 400], @upload.calculate_dimensions)
end
end
context "content type calculator" do
should "know how to parse jpeg, png, gif, and swf file headers" do
@upload = FactoryGirl.create(:jpg_upload)
assert_equal("image/jpeg", @upload.file_header_to_content_type("#{Rails.root}/test/files/test.jpg"))
assert_equal("image/gif", @upload.file_header_to_content_type("#{Rails.root}/test/files/test.gif"))
assert_equal("image/png", @upload.file_header_to_content_type("#{Rails.root}/test/files/test.png"))
assert_equal("application/x-shockwave-flash", @upload.file_header_to_content_type("#{Rails.root}/test/files/compressed.swf"))
assert_equal("application/octet-stream", @upload.file_header_to_content_type("#{Rails.root}/README.md"))
end
should "know how to parse jpeg, png, gif, and swf content types" do
@upload = FactoryGirl.create(:jpg_upload)
assert_equal("jpg", @upload.content_type_to_file_ext("image/jpeg"))
assert_equal("gif", @upload.content_type_to_file_ext("image/gif"))
assert_equal("png", @upload.content_type_to_file_ext("image/png"))
assert_equal("swf", @upload.content_type_to_file_ext("application/x-shockwave-flash"))
assert_equal("bin", @upload.content_type_to_file_ext(""))
@upload = FactoryGirl.build(:jpg_upload)
assert_equal("jpg", @upload.file_header_to_file_ext(File.open("#{Rails.root}/test/files/test.jpg")))
assert_equal("gif", @upload.file_header_to_file_ext(File.open("#{Rails.root}/test/files/test.gif")))
assert_equal("png", @upload.file_header_to_file_ext(File.open("#{Rails.root}/test/files/test.png")))
assert_equal("swf", @upload.file_header_to_file_ext(File.open("#{Rails.root}/test/files/compressed.swf")))
assert_equal("bin", @upload.file_header_to_file_ext(File.open("#{Rails.root}/README.md")))
end
end
context "downloader" do
context "for a zip that is not an ugoira" do
should "not validate" do
FileUtils.cp("#{Rails.root}/test/files/invalid_ugoira.zip", "#{Rails.root}/tmp")
@upload = Upload.create(:file => upload_zip("#{Rails.root}/tmp/invalid_ugoira.zip"), :rating => "q", :tag_string => "xxx")
@upload = FactoryGirl.create(:upload, file: upload_file("test/files/invalid_ugoira.zip"))
@upload.process!
assert_equal("error: RuntimeError - missing frame data for ugoira", @upload.status)
end
@@ -116,30 +88,15 @@ class UploadTest < ActiveSupport::TestCase
context "that is a pixiv ugoira" do
setup do
@url = "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=46378654"
@upload = FactoryGirl.create(:source_upload, :source => @url, :tag_string => "ugoira")
@output_file = Tempfile.new("download")
@upload = FactoryGirl.create(:upload, :source => @url, :tag_string => "ugoira")
end
teardown do
@output_file.unlink
end
should "process successfully" do
@upload.download_from_source(@output_file.path)
assert_operator(File.size(@output_file.path), :>, 1_000)
assert_equal("application/zip", @upload.file_header_to_content_type(@output_file.path))
assert_equal("zip", @upload.content_type_to_file_ext(@upload.file_header_to_content_type(@output_file.path)))
_, _, output_file = @upload.download_from_source(@url, "")
assert_operator(output_file.size, :>, 1_000)
assert_equal("zip", @upload.file_header_to_file_ext(output_file))
end
end
should "initialize the final path after downloading a file" do
@upload = FactoryGirl.create(:source_upload)
path = "#{Rails.root}/tmp/test.download.jpg"
assert_nothing_raised {@upload.download_from_source(path)}
assert(File.exists?(path))
assert_equal(8558, File.size(path))
assert_equal(path, @upload.file_path)
end
end
context "determining if a file is downloadable" do
@@ -161,46 +118,32 @@ class UploadTest < ActiveSupport::TestCase
context "file processor" do
should "parse and process a cgi file representation" do
FileUtils.cp("#{Rails.root}/test/files/test.jpg", "#{Rails.root}/tmp")
@upload = Upload.new(:file => upload_jpeg("#{Rails.root}/tmp/test.jpg"))
assert_nothing_raised {@upload.convert_cgi_file}
assert(File.exists?(@upload.file_path))
assert_equal(28086, File.size(@upload.file_path))
@upload = FactoryGirl.create(:upload, file: upload_file("test/files/test.jpg"))
assert_nothing_raised {@upload.process_upload}
assert_equal(28086, @upload.file_size)
end
should "process a transparent png" do
FileUtils.cp("#{Rails.root}/test/files/alpha.png", "#{Rails.root}/tmp")
@upload = Upload.new(:file => upload_file("#{Rails.root}/tmp/alpha.png", "image/png", "alpha.png"))
assert_nothing_raised {@upload.convert_cgi_file}
assert(File.exists?(@upload.file_path))
assert_equal(1136, File.size(@upload.file_path))
@upload = FactoryGirl.create(:upload, file: upload_file("test/files/alpha.png"))
assert_nothing_raised {@upload.process_upload}
assert_equal(1136, @upload.file_size)
end
end
context "hash calculator" do
should "caculate the hash" do
@upload = FactoryGirl.create(:jpg_upload)
@upload.calculate_hash(@upload.file_path)
@upload.process_upload
assert_equal("ecef68c44edb8a0d6a3070b5f8e8ee76", @upload.md5)
end
end
context "resizer" do
teardown do
FileUtils.rm_f(Dir.glob("#{Rails.root}/public/data/preview/test.*.jpg"))
FileUtils.rm_f(Dir.glob("#{Rails.root}/public/data/sample/test.*.jpg"))
FileUtils.rm_f(Dir.glob("#{Rails.root}/public/data/test.*.jpg"))
end
should "generate several resized versions of the image" do
@upload = FactoryGirl.create(:large_jpg_upload)
@upload.calculate_hash(@upload.file_path)
@upload.calculate_dimensions(@upload.file_path)
assert_nothing_raised {@upload.generate_resizes(@upload.file_path)}
assert(File.exists?(@upload.resized_file_path_for(Danbooru.config.small_image_width)))
assert(File.size(@upload.resized_file_path_for(Danbooru.config.small_image_width)) > 0)
assert(File.exists?(@upload.resized_file_path_for(Danbooru.config.large_image_width)))
assert(File.size(@upload.resized_file_path_for(Danbooru.config.large_image_width)) > 0)
@upload = FactoryGirl.create(:upload, file_ext: "jpg", image_width: 1356, image_height: 911, file: upload_file("test/files/test-large.jpg"))
preview_file, sample_file = @upload.generate_resizes
assert_operator(preview_file.size, :>, 1_000)
assert_operator(sample_file.size, :>, 1_000)
end
end
@@ -215,13 +158,10 @@ class UploadTest < ActiveSupport::TestCase
context "with an artist commentary" do
setup do
@upload = FactoryGirl.create(:source_upload,
:rating => "s",
:uploader_ip_addr => "127.0.0.1",
:tag_string => "hoge foo"
)
@upload.include_artist_commentary = "1"
@upload.artist_commentary_title = ""
@upload.artist_commentary_desc = "blah"
include_artist_commentary: "1",
artist_commentary_title: "",
artist_commentary_desc: "blah",
)
end
should "create an artist commentary when processed" do
@@ -255,15 +195,17 @@ class UploadTest < ActiveSupport::TestCase
assert_equal(post.id, @upload.post_id)
assert_equal("completed", @upload.status)
end
context "automatic tagging" do
should "tag animated png files" do
@upload = FactoryGirl.build(:upload, file_ext: "png", file: upload_file("test/files/apng/normal_apng.png"))
assert_equal("animated_png", @upload.automatic_tags)
end
end
end
should "process completely for a pixiv ugoira" do
@upload = FactoryGirl.create(:source_upload,
:source => "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=46378654",
:rating => "s",
:uploader_ip_addr => "127.0.0.1",
:tag_string => "hoge foo"
)
@upload = FactoryGirl.create(:source_upload, source: "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=46378654")
assert_difference(["PixivUgoiraFrameData.count", "Post.count"]) do
@upload.process!
assert_equal([], @upload.errors.full_messages)
@@ -274,18 +216,18 @@ class UploadTest < ActiveSupport::TestCase
assert_equal(60, post.image_width)
assert_equal(60, post.image_height)
assert_equal("https://i.pximg.net/img-zip-ugoira/img/2014/10/05/23/42/23/46378654_ugoira1920x1080.zip", post.source)
assert_operator(File.size(post.large_file_path), :>, 0)
assert_operator(File.size(post.preview_file_path), :>, 0)
assert_nothing_raised { post.file(:original) }
assert_nothing_raised { post.file(:large) }
assert_nothing_raised { post.file(:preview) }
end
should "process completely for an uploaded image" do
@upload = FactoryGirl.create(:jpg_upload,
:rating => "s",
:uploader_ip_addr => "127.0.0.1",
:tag_string => "hoge foo"
:tag_string => "hoge foo",
:file => upload_file("test/files/test.jpg"),
)
@upload.file = upload_jpeg("#{Rails.root}/test/files/test.jpg")
@upload.convert_cgi_file
assert_difference("Post.count") do
assert_nothing_raised {@upload.process!}
@@ -297,8 +239,8 @@ class UploadTest < ActiveSupport::TestCase
assert_equal("127.0.0.1", post.uploader_ip_addr.to_s)
assert_equal(@upload.md5, post.md5)
assert_equal("jpg", post.file_ext)
assert(File.exists?(post.file_path))
assert_equal(28086, File.size(post.file_path))
assert_nothing_raised { post.file(:original) }
assert_equal(28086, post.file(:original).size)
assert_equal(post.id, @upload.post_id)
assert_equal("completed", @upload.status)
end
@@ -310,16 +252,5 @@ class UploadTest < ActiveSupport::TestCase
assert_nothing_raised {@upload.process!}
end
end
should "delete the temporary file upon completion" do
@upload = FactoryGirl.create(:source_upload,
:rating => "s",
:uploader_ip_addr => "127.0.0.1",
:tag_string => "hoge foo"
)
@upload.process!
assert(!File.exists?(@upload.temp_file_path))
end
end
end