posts: use storage manager to backup files.

* Perform backups synchronously inside `distribute_files` instead of
  asynchronously in `queue_backup`. Asynchronous backups assumed that
  files are stored on the local filesystem, which isn't true in general.

* Remove obsolete backup service classes.
This commit is contained in:
evazion
2018-03-18 16:33:26 -05:00
parent 6d0d1a3ce9
commit f0bf1bc66e
7 changed files with 27 additions and 105 deletions

View File

@@ -1,9 +0,0 @@
class BackupService
def backup(file_path, options = {})
raise NotImplementedError.new("#{self.class}.backup not implemented")
end
def delete(file_path, options = {})
raise NotImplementedError.new("#{self.class}.delete not implemented")
end
end

View File

@@ -1,9 +0,0 @@
class NullBackupService
def backup(file_path, options = {})
# do nothing
end
def delete(file_path, options = {})
# do nothing
end
end

View File

@@ -1,52 +0,0 @@
class S3BackupService < BackupService
attr_reader :client, :bucket
def initialize(client: nil, bucket: Danbooru.config.aws_s3_bucket_name)
@credentials = Aws::Credentials.new(Danbooru.config.aws_access_key_id, Danbooru.config.aws_secret_access_key)
@client = client || Aws::S3::Client.new(credentials: @credentials, region: "us-east-1", logger: Logger.new(STDOUT))
@bucket = bucket
end
def backup(file_path, type: nil, **options)
keys = s3_keys(file_path, type)
keys.each do |key|
upload_to_s3(key, file_path)
end
end
def delete(file_path, type: nil)
keys = s3_keys(file_path, type)
keys.each do |key|
delete_from_s3(key)
end
end
protected
def s3_keys(file_path, type)
name = File.basename(file_path)
case type
when :original
[name]
when :preview
["preview/#{name}"]
when :large
["sample/#{name}"]
else
raise ArgumentError.new("Unknown type: #{type}")
end
end
def delete_from_s3(key)
client.delete_object(bucket: bucket, key: key)
rescue Aws::S3::Errors::NoSuchKey
# ignore
end
def upload_to_s3(key, file_path)
File.open(file_path, "rb") do |body|
base64_md5 = Digest::MD5.base64digest(File.read(file_path))
client.put_object(acl: "public-read", bucket: bucket, key: key, body: body, content_md5: base64_md5)
end
end
end

View File

@@ -29,7 +29,6 @@ class Post < ApplicationRecord
before_save :set_tag_counts
before_save :set_pool_category_pseudo_tags
before_create :autoban
after_save :queue_backup, if: :md5_changed?
after_save :create_version
after_save :update_parent_on_save
after_save :apply_post_metatags
@@ -134,6 +133,10 @@ class Post < ApplicationRecord
storage_manager.store_file(file, self, :original)
storage_manager.store_file(sample_file, self, :large) if sample_file.present?
storage_manager.store_file(preview_file, self, :preview) if preview_file.present?
backup_storage_manager.store_file(file, self, :original)
backup_storage_manager.store_file(sample_file, self, :large) if sample_file.present?
backup_storage_manager.store_file(preview_file, self, :preview) if preview_file.present?
end
def file_path_prefix
@@ -168,6 +171,10 @@ class Post < ApplicationRecord
"#{file_path_prefix}#{md5}.#{file_ext}"
end
def backup_storage_manager
Danbooru.config.backup_storage_manager
end
def storage_manager
Danbooru.config.storage_manager
end
@@ -263,23 +270,6 @@ class Post < ApplicationRecord
end
end
module BackupMethods
extend ActiveSupport::Concern
def queue_backup
Post.delay(queue: "default", priority: -1).backup_file(file_path, id: id, type: :original)
Post.delay(queue: "default", priority: -1).backup_file(large_file_path, id: id, type: :large) if has_large?
Post.delay(queue: "default", priority: -1).backup_file(preview_file_path, id: id, type: :preview) if has_preview?
end
module ClassMethods
def backup_file(file_path, options = {})
backup_service = Danbooru.config.backup_service
backup_service.backup(file_path, options)
end
end
end
module ImageMethods
def twitter_card_supported?
image_width.to_i >= 280 && image_height.to_i >= 150
@@ -1800,7 +1790,6 @@ class Post < ApplicationRecord
end
include FileMethods
include BackupMethods
include ImageMethods
include ApprovalMethods
include PresenterMethods

View File

@@ -71,8 +71,6 @@ class PostReplacement < ApplicationRecord
if md5_changed
post.comments.create!({creator: User.system, body: comment_replacement_message, do_not_bump_post: true}, without_protection: true)
else
post.queue_backup
end
save!