From 782bdd46276078045b4ccadd1226254268a501b6 Mon Sep 17 00:00:00 2001 From: r888888888 Date: Fri, 10 Jan 2014 16:35:40 -0800 Subject: [PATCH 1/5] fix amazonbackup script, trap on epipe exception for db backup --- app/models/amazon_backup.rb | 4 ++-- config/danbooru_default_config.rb | 4 ++++ script/donmai/backup_db_to_s3 | 16 ++++++++++++++-- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/app/models/amazon_backup.rb b/app/models/amazon_backup.rb index 19919a25d..b70814410 100644 --- a/app/models/amazon_backup.rb +++ b/app/models/amazon_backup.rb @@ -24,7 +24,7 @@ class AmazonBackup < ActiveRecord::Base AWS::S3::S3Object.store(File.basename(post.file_path), open(post.file_path, "rb"), Danbooru.config.amazon_s3_bucket_name, "Content-MD5" => base64_md5) end - if post.image? && File.exists?(post.preview_file_path) + if post.is_image? && File.exists?(post.preview_file_path) AWS::S3::S3Object.store("preview/#{post.md5}.jpg", open(post.preview_file_path, "rb"), Danbooru.config.amazon_s3_bucket_name) end @@ -32,7 +32,7 @@ class AmazonBackup < ActiveRecord::Base AWS::S3::S3Object.store("large/#{post.md5}.jpg", open(post.large_file_path, "rb"), Danbooru.config.amazon_s3_bucket_name) end - AmazonBackup.update_id(last_id) + AmazonBackup.update_id(post.id) end rescue Exception => x # probably some network error, retry next time diff --git a/config/danbooru_default_config.rb b/config/danbooru_default_config.rb index e51b83179..07e6a4e08 100644 --- a/config/danbooru_default_config.rb +++ b/config/danbooru_default_config.rb @@ -298,6 +298,10 @@ module Danbooru nil end + def amazon_s3_bucket_name + "danbooru" + end + def enable_dimension_autotagging true end diff --git a/script/donmai/backup_db_to_s3 b/script/donmai/backup_db_to_s3 index a9f2cb60c..f3de65c29 100755 --- a/script/donmai/backup_db_to_s3 +++ b/script/donmai/backup_db_to_s3 @@ -20,7 +20,7 @@ end access_key = open("#{home}/.s3/access_key").read.strip secret_access_key = open("#{home}/.s3/secret_access_key").read.strip -AWS::S3::Base.establish_connection!(:access_key_id => access_key, :secret_access_key => secret_access_key) +AWS::S3::Base.establish_connection!(:access_key_id => access_key, :secret_access_key => secret_access_key, :server => "s3.amazonaws.com") current_backups = AWS::S3::Bucket.find("danbooru-backup").objects.map {|x| x.key}.select {|x| x =~ /^db-/}.sort.reverse @@ -34,5 +34,17 @@ end backup = Dir["/var/www/danbooru2/shared/backup/db-*.dump"].sort.last data = File.open(backup, "rb") filename = data.mtime.strftime("db-%Y-%m-%d-%H-%M") -AWS::S3::S3Object.store(filename, data, "danbooru-backup", :access => :private) +tries = 0 + +begin + AWS::S3::S3Object.store(filename, data, "danbooru-backup", :access => :private) +rescue Errno::EPIPE + tries += 1 + if tries > 3 + raise + else + retry + end +end + puts "Uploaded #{backup} as #{filename}" From e6481d77e1f793ca9a690de7f30d0e7be2f87793 Mon Sep 17 00:00:00 2001 From: r888888888 Date: Fri, 10 Jan 2014 16:48:38 -0800 Subject: [PATCH 2/5] update version --- config/danbooru_default_config.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/danbooru_default_config.rb b/config/danbooru_default_config.rb index 07e6a4e08..44277d123 100644 --- a/config/danbooru_default_config.rb +++ b/config/danbooru_default_config.rb @@ -4,7 +4,7 @@ module Danbooru class Configuration # The version of this Danbooru. def version - "2.39.0" + "2.40.0" end # The name of this Danbooru. From ca26bf0f3bf1b6c6469bcdfc93e88e2309af0b9d Mon Sep 17 00:00:00 2001 From: r888888888 Date: Fri, 10 Jan 2014 16:50:56 -0800 Subject: [PATCH 3/5] hard code s3 server --- app/models/amazon_backup.rb | 3 ++- script/donmai/backup_db_to_s3 | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/app/models/amazon_backup.rb b/app/models/amazon_backup.rb index b70814410..fdcd34083 100644 --- a/app/models/amazon_backup.rb +++ b/app/models/amazon_backup.rb @@ -16,7 +16,8 @@ class AmazonBackup < ActiveRecord::Base Post.where("id > ?", last_id).limit(200).order("id").each do |post| AWS::S3::Base.establish_connection!( :access_key_id => Danbooru.config.amazon_s3_access_key_id, - :secret_access_key => Danbooru.config.amazon_s3_secret_access_key + :secret_access_key => Danbooru.config.amazon_s3_secret_access_key, + :server => "s3-us-east-1.amazonaws.com" ) if File.exists?(post.file_path) diff --git a/script/donmai/backup_db_to_s3 b/script/donmai/backup_db_to_s3 index f3de65c29..f333fca76 100755 --- a/script/donmai/backup_db_to_s3 +++ b/script/donmai/backup_db_to_s3 @@ -20,7 +20,7 @@ end access_key = open("#{home}/.s3/access_key").read.strip secret_access_key = open("#{home}/.s3/secret_access_key").read.strip -AWS::S3::Base.establish_connection!(:access_key_id => access_key, :secret_access_key => secret_access_key, :server => "s3.amazonaws.com") +AWS::S3::Base.establish_connection!(:access_key_id => access_key, :secret_access_key => secret_access_key, :server => "s3-us-east-1.amazonaws.com") current_backups = AWS::S3::Bucket.find("danbooru-backup").objects.map {|x| x.key}.select {|x| x =~ /^db-/}.sort.reverse From efff3a389ff19de8b0611f136df3ce2d1a73d9e0 Mon Sep 17 00:00:00 2001 From: r888888888 Date: Fri, 10 Jan 2014 16:58:57 -0800 Subject: [PATCH 4/5] hard code s3 server --- script/donmai/backup_db_to_s3 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/donmai/backup_db_to_s3 b/script/donmai/backup_db_to_s3 index f333fca76..f3de65c29 100755 --- a/script/donmai/backup_db_to_s3 +++ b/script/donmai/backup_db_to_s3 @@ -20,7 +20,7 @@ end access_key = open("#{home}/.s3/access_key").read.strip secret_access_key = open("#{home}/.s3/secret_access_key").read.strip -AWS::S3::Base.establish_connection!(:access_key_id => access_key, :secret_access_key => secret_access_key, :server => "s3-us-east-1.amazonaws.com") +AWS::S3::Base.establish_connection!(:access_key_id => access_key, :secret_access_key => secret_access_key, :server => "s3.amazonaws.com") current_backups = AWS::S3::Bucket.find("danbooru-backup").objects.map {|x| x.key}.select {|x| x =~ /^db-/}.sort.reverse From d594d525d691b025a9d265a8c072f668cfd1576d Mon Sep 17 00:00:00 2001 From: r888888888 Date: Fri, 10 Jan 2014 17:20:57 -0800 Subject: [PATCH 5/5] fix amazonbackup --- app/models/amazon_backup.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/amazon_backup.rb b/app/models/amazon_backup.rb index fdcd34083..f6bb7447e 100644 --- a/app/models/amazon_backup.rb +++ b/app/models/amazon_backup.rb @@ -17,7 +17,7 @@ class AmazonBackup < ActiveRecord::Base AWS::S3::Base.establish_connection!( :access_key_id => Danbooru.config.amazon_s3_access_key_id, :secret_access_key => Danbooru.config.amazon_s3_secret_access_key, - :server => "s3-us-east-1.amazonaws.com" + :server => "s3.amazonaws.com" ) if File.exists?(post.file_path)