From c8067a46917729668bfaec36d1f0f03bccf15178 Mon Sep 17 00:00:00 2001 From: albert Date: Mon, 15 Aug 2011 19:41:50 -0400 Subject: [PATCH] fixing jobs --- app/assets/stylesheets/application.css.scss | 2 +- app/models/amazon_backup.rb | 44 +++++++++++++++++++ app/models/jobs/backup_to_s3.rb | 27 ------------ app/models/jobs/calculate_post_count.rb | 7 --- app/models/jobs/calculate_related_tags.rb | 12 ----- app/models/jobs/calculate_uploaded_tags.rb | 13 ------ app/models/jobs/create_tag_alias.rb | 12 ----- app/models/jobs/create_tag_implication.rb | 12 ----- app/models/jobs/fix_pixiv_uploads.rb | 13 ------ app/models/jobs/mass_tag_edit.rb | 7 --- app/models/jobs/process_tag_subscriptions.rb | 10 ----- app/models/jobs/process_uploads.rb | 9 ---- app/presenters/post_set_presenters/popular.rb | 2 +- config/schedule.rb | 4 ++ .../20110815233456_create_amazon_backups.rb | 9 ++++ test/fixtures/amazon_backups.yml | 7 +++ test/unit/amazon_backup_test.rb | 7 +++ 17 files changed, 73 insertions(+), 124 deletions(-) create mode 100644 app/models/amazon_backup.rb delete mode 100644 app/models/jobs/backup_to_s3.rb delete mode 100644 app/models/jobs/calculate_post_count.rb delete mode 100644 app/models/jobs/calculate_related_tags.rb delete mode 100644 app/models/jobs/calculate_uploaded_tags.rb delete mode 100644 app/models/jobs/create_tag_alias.rb delete mode 100644 app/models/jobs/create_tag_implication.rb delete mode 100644 app/models/jobs/fix_pixiv_uploads.rb delete mode 100644 app/models/jobs/mass_tag_edit.rb delete mode 100644 app/models/jobs/process_tag_subscriptions.rb delete mode 100644 app/models/jobs/process_uploads.rb create mode 100644 db/migrate/20110815233456_create_amazon_backups.rb create mode 100644 test/fixtures/amazon_backups.yml create mode 100644 test/unit/amazon_backup_test.rb diff --git a/app/assets/stylesheets/application.css.scss b/app/assets/stylesheets/application.css.scss index 9a805d5ab..9310643d5 100644 --- a/app/assets/stylesheets/application.css.scss +++ b/app/assets/stylesheets/application.css.scss @@ -1079,7 +1079,7 @@ footer#page-footer { margin: 1em; text-align: center; padding-top: 1em; - border-top: 2px solid #CCC; + border-top: 1px solid #EEE; } /*** news ticker ***/ diff --git a/app/models/amazon_backup.rb b/app/models/amazon_backup.rb new file mode 100644 index 000000000..e29c2d373 --- /dev/null +++ b/app/models/amazon_backup.rb @@ -0,0 +1,44 @@ +require 'base64' +require 'digest/md5' + +class AmazonBackup < ActiveRecord::Base + def self.last_id + first.last_id + end + + def self.update_id(new_id) + first.update_column(:last_id, new_id) + end + + def self.execute + last_id = AmazonBackup.last_id + + Post.where("id > ?", last_id).limit(200).order("id").each do |post| + AWS::S3::Base.establish_connection!( + :access_key_id => Danbooru.config.amazon_s3_access_key_id, + :secret_access_key => Danbooru.config.amazon_s3_secret_access_key + ) + + if File.exists?(post.file_path) + base64_md5 = Base64.encode64(Digest::MD5.digest(File.read(post.file_path))) + AWS::S3::S3Object.store(post.file_name, open(post.file_path, "rb"), Danbooru.config.amazon_s3_bucket_name, "Content-MD5" => base64_md5) + end + + if post.image? && File.exists?(post.preview_file_path) + AWS::S3::S3Object.store("preview/#{post.md5}.jpg", open(post.preview_file_path, "rb"), Danbooru.config.amazon_s3_bucket_name) + end + + if File.exists?(post.medium_file_path) + AWS::S3::S3Object.store("medium/#{post.md5}.jpg", open(post.medium_file_path, "rb"), Danbooru.config.amazon_s3_bucket_name) + end + + if File.exists?(post.large_file_path) + AWS::S3::S3Object.store("large/#{post.md5}.jpg", open(post.large_file_path, "rb"), Danbooru.config.amazon_s3_bucket_name) + end + + AmazonBackup.update_id(last_id) + end + rescue Exception => x + # probably some network error, retry next time + end +end diff --git a/app/models/jobs/backup_to_s3.rb b/app/models/jobs/backup_to_s3.rb deleted file mode 100644 index 3675ac649..000000000 --- a/app/models/jobs/backup_to_s3.rb +++ /dev/null @@ -1,27 +0,0 @@ -module Jobs - class BackupToS3 < Struct.new(:last_id) - def perform - Post.find(:all, :conditions => ["id > ?", last_id], :limit => 200, :order => "id").each do |post| - AWS::S3::Base.establish_connection!(:access_key_id => CONFIG["amazon_s3_access_key_id"], :secret_access_key => CONFIG["amazon_s3_secret_access_key"]) - if File.exists?(post.file_path) - base64_md5 = Base64.encode64(Digest::MD5.digest(File.read(post.file_path))) - AWS::S3::S3Object.store(post.file_name, open(post.file_path, "rb"), CONFIG["amazon_s3_bucket_name"], "Content-MD5" => base64_md5) - end - - if post.image? && File.exists?(post.preview_path) - AWS::S3::S3Object.store("preview/#{post.md5}.jpg", open(post.preview_path, "rb"), CONFIG["amazon_s3_bucket_name"]) - end - - if File.exists?(post.sample_path) - AWS::S3::S3Object.store("sample/" + CONFIG["sample_filename_prefix"] + "#{post.md5}.jpg", open(post.sample_path, "rb"), CONFIG["amazon_s3_bucket_name"]) - end - - self.last_id = post.id - end - - Delayed::Job.enqueue(BackupToS3.new(last_id)) - rescue Exception => x - # probably some network error, retry next time - end - end -end \ No newline at end of file diff --git a/app/models/jobs/calculate_post_count.rb b/app/models/jobs/calculate_post_count.rb deleted file mode 100644 index 24653b04a..000000000 --- a/app/models/jobs/calculate_post_count.rb +++ /dev/null @@ -1,7 +0,0 @@ -module Jobs - class CalculatePostCount < Struct.new(:tag_name) - def perform - Tag.recalculate_post_count(tag_name) - end - end -end diff --git a/app/models/jobs/calculate_related_tags.rb b/app/models/jobs/calculate_related_tags.rb deleted file mode 100644 index 161faabb4..000000000 --- a/app/models/jobs/calculate_related_tags.rb +++ /dev/null @@ -1,12 +0,0 @@ -module Jobs - class CalculateRelatedTags < Struct.new(:tag_id) - def perform - tag = Tag.find_by_id(tag_id) - - if tag - tag.update_related - tag.save - end - end - end -end diff --git a/app/models/jobs/calculate_uploaded_tags.rb b/app/models/jobs/calculate_uploaded_tags.rb deleted file mode 100644 index a3f750df9..000000000 --- a/app/models/jobs/calculate_uploaded_tags.rb +++ /dev/null @@ -1,13 +0,0 @@ -module Jobs - class CalculateUploadedTags < Struct.new(:user_id) - def perform - tags = [] - user = User.find(user_id) - CONFIG["tag_types"].values.uniq.each do |tag_type| - tags += user.calculate_uploaded_tags(tag_type) - end - user.update_column(:uploaded_tags, tags.join("\n")) - end - end -end - diff --git a/app/models/jobs/create_tag_alias.rb b/app/models/jobs/create_tag_alias.rb deleted file mode 100644 index 1e1d4dfbc..000000000 --- a/app/models/jobs/create_tag_alias.rb +++ /dev/null @@ -1,12 +0,0 @@ -module Jobs - class CreateTagAlias < Struct.new(:antecedent_name, :consequent_name, :creator_id, :creator_ip_addr) - def execute - TagAlias.create( - :antecedent_name => antecedent_name, - :consequent_name => consequent_name, - :creator_id => creator_id, - :creator_ip_addr => creator_ip_addr - ) - end - end -end diff --git a/app/models/jobs/create_tag_implication.rb b/app/models/jobs/create_tag_implication.rb deleted file mode 100644 index 860582c9d..000000000 --- a/app/models/jobs/create_tag_implication.rb +++ /dev/null @@ -1,12 +0,0 @@ -module Jobs - class CreateTagImplication < Struct.new(:antecedent_name, :consequent_name, :creator_id, :creator_ip_addr) - def perform - TagImplication.create( - :antecedent_name => antecedent_name, - :consequent_name => consequent_name, - :creator_id => creator_id, - :creator_ip_addr => creator_ip_addr - ) - end - end -end diff --git a/app/models/jobs/fix_pixiv_uploads.rb b/app/models/jobs/fix_pixiv_uploads.rb deleted file mode 100644 index 3e9dd09dc..000000000 --- a/app/models/jobs/fix_pixiv_uploads.rb +++ /dev/null @@ -1,13 +0,0 @@ -module Jobs - class FixPixivUploads < Struct.new(:last_post_id) - def perform - post_id = nil - - Post.find_each(:conditions => ["GREATEST(width, height) IN (150, 600) AND source LIKE ? AND id > ?", "%pixiv%", last_post_id]) do |post| - post_id = post.id - end - - update_attributes(:data => {:last_post_id => post_id}) - end - end -end diff --git a/app/models/jobs/mass_tag_edit.rb b/app/models/jobs/mass_tag_edit.rb deleted file mode 100644 index c10102a7d..000000000 --- a/app/models/jobs/mass_tag_edit.rb +++ /dev/null @@ -1,7 +0,0 @@ -module Jobs - class MassTagEdit < Struct.new(:start_tags, :result_tags, :updater_id, :updater_ip_addr) - def perform - Tag.mass_edit(start_tags, result_tags, updater_id, updater_ip_addr) - end - end -end diff --git a/app/models/jobs/process_tag_subscriptions.rb b/app/models/jobs/process_tag_subscriptions.rb deleted file mode 100644 index e6e21f265..000000000 --- a/app/models/jobs/process_tag_subscriptions.rb +++ /dev/null @@ -1,10 +0,0 @@ -module Jobs - class ProcessTagSubscriptions < Struct.new(:last_run) - def perform - if last_run.nil? || last_run < 20.minutes.ago - TagSubscription.process_all - Delayed::Job.enqueue(ProcessTagSubscriptions.new(Time.now)) - end - end - end -end diff --git a/app/models/jobs/process_uploads.rb b/app/models/jobs/process_uploads.rb deleted file mode 100644 index c7ba63ae3..000000000 --- a/app/models/jobs/process_uploads.rb +++ /dev/null @@ -1,9 +0,0 @@ -module Jobs - class ProcessUploads - def perform - Upload.find_each(:conditions => ["status = ?", "pending"]) do |upload| - upload.process! - end - end - end -end diff --git a/app/presenters/post_set_presenters/popular.rb b/app/presenters/post_set_presenters/popular.rb index 18f6c6973..754b3fb3f 100644 --- a/app/presenters/post_set_presenters/popular.rb +++ b/app/presenters/post_set_presenters/popular.rb @@ -37,7 +37,7 @@ module PostSetPresenters elsif max_date - min_date == 6 min_date.strftime("Week of %B %d, %Y") else - date.strftime("Month of %B %Y") + date.strftime("%B %Y") end end end diff --git a/config/schedule.rb b/config/schedule.rb index 53fdc0a39..054359354 100644 --- a/config/schedule.rb +++ b/config/schedule.rb @@ -3,3 +3,7 @@ set :output, "/var/log/whenever.log" every 1.hour do TagSubscription.process_all end + +every 1.hour do + AmazonBackup.execute +end diff --git a/db/migrate/20110815233456_create_amazon_backups.rb b/db/migrate/20110815233456_create_amazon_backups.rb new file mode 100644 index 000000000..25c41e8d6 --- /dev/null +++ b/db/migrate/20110815233456_create_amazon_backups.rb @@ -0,0 +1,9 @@ +class CreateAmazonBackups < ActiveRecord::Migration + def change + create_table :amazon_backups do |t| + t.integer :last_id + + t.timestamps + end + end +end diff --git a/test/fixtures/amazon_backups.yml b/test/fixtures/amazon_backups.yml new file mode 100644 index 000000000..833d34ed4 --- /dev/null +++ b/test/fixtures/amazon_backups.yml @@ -0,0 +1,7 @@ +# Read about fixtures at http://api.rubyonrails.org/classes/Fixtures.html + +one: + last_id: 1 + +two: + last_id: 1 diff --git a/test/unit/amazon_backup_test.rb b/test/unit/amazon_backup_test.rb new file mode 100644 index 000000000..cd0312b68 --- /dev/null +++ b/test/unit/amazon_backup_test.rb @@ -0,0 +1,7 @@ +require 'test_helper' + +class AmazonBackupTest < ActiveSupport::TestCase + # test "the truth" do + # assert true + # end +end