From d5fa4b46e306eb27d6c08f70b207b24cbadc61ea Mon Sep 17 00:00:00 2001 From: r888888888 Date: Mon, 28 Dec 2015 12:25:38 -0800 Subject: [PATCH] * refactor aws config options * fix aws calls in savedsearch * remove unused scripts --- app/controllers/saved_searches_controller.rb | 2 +- app/logical/sqs_service.rb | 33 ++++++++ app/models/amazon_backup.rb | 6 +- app/models/saved_search.rb | 87 ++++++-------------- config/danbooru_default_config.rb | 50 +++++++++-- config/initializers/z_action_mailer.rb | 8 +- script/donmai/backup_db | 3 - script/donmai/backup_db_to_s3 | 55 ------------- script/donmai/prune_backup_dbs | 3 - 9 files changed, 108 insertions(+), 139 deletions(-) create mode 100644 app/logical/sqs_service.rb delete mode 100755 script/donmai/backup_db delete mode 100755 script/donmai/backup_db_to_s3 delete mode 100755 script/donmai/prune_backup_dbs diff --git a/app/controllers/saved_searches_controller.rb b/app/controllers/saved_searches_controller.rb index d3cb5f6f1..2c0bd8cff 100644 --- a/app/controllers/saved_searches_controller.rb +++ b/app/controllers/saved_searches_controller.rb @@ -4,7 +4,7 @@ class SavedSearchesController < ApplicationController def index if Danbooru.config.listbooru_server - SavedSearch.delay(:queue => "default").refresh_listbooru(CurrentUser.id) + SavedSearch.refresh_listbooru(CurrentUser.id) end @saved_searches = saved_searches.order("tag_query") diff --git a/app/logical/sqs_service.rb b/app/logical/sqs_service.rb new file mode 100644 index 000000000..fcff916ca --- /dev/null +++ b/app/logical/sqs_service.rb @@ -0,0 +1,33 @@ +class SqsService + attr_reader :url + + def initialize(url) + @url = url + end + + def send_message(string, options = {}) + return unless Danbooru.config.aws_sqs_enabled? + + sqs.send_message( + options.merge( + message_body: string, + queue_url: url + ) + ) + end + +private + + def sqs + @sqs ||= begin + credentials = Aws::Credentials.new( + Danbooru.config.aws_access_key_id, + Danbooru.config.aws_secret_access_key + ) + Aws::SQS::Client.new( + credentials: credentials, + region: Danbooru.config.aws_sqs_region + ) + end + end +end diff --git a/app/models/amazon_backup.rb b/app/models/amazon_backup.rb index b7f86968b..175a99f89 100644 --- a/app/models/amazon_backup.rb +++ b/app/models/amazon_backup.rb @@ -13,14 +13,16 @@ class AmazonBackup < ActiveRecord::Base end def self.execute + return false unless Danbooru.config.aws_s3_enabled? + last_id = AmazonBackup.last_id - credentials = Aws::Credentials.new(Danbooru.config.amazon_s3_access_key_id, Danbooru.config.amazon_s3_secret_access_key) + credentials = Aws::Credentials.new(Danbooru.config.aws_access_key_id, Danbooru.config.aws_secret_access_key) Aws.config.update({ region: "us-east-1", credentials: credentials }) client = Aws::S3::Client.new - bucket = Danbooru.config.amazon_s3_bucket_name + bucket = Danbooru.config.aws_s3_bucket_name Post.where("id > ?", last_id).limit(1000).order("id").each do |post| if File.exists?(post.file_path) diff --git a/app/models/saved_search.rb b/app/models/saved_search.rb index 8556a3f7e..a90c1e01f 100644 --- a/app/models/saved_search.rb +++ b/app/models/saved_search.rb @@ -3,87 +3,48 @@ class SavedSearch < ActiveRecord::Base extend ActiveSupport::Concern module ClassMethods - def posts_search_available? - Danbooru.config.listbooru_server.present? && CurrentUser.is_gold? - end - def refresh_listbooru(user_id) - return unless Danbooru.config.listbooru_auth_key - user = User.find(user_id) - return unless user.is_gold? - - params = { - :user_id => user_id, - :key => Danbooru.config.listbooru_auth_key - } - uri = URI.parse("#{Danbooru.config.listbooru_server}/users") - uri.query = URI.encode_www_form(params) - Net::HTTP.get_response(uri) + return false unless Danbooru.config.listbooru_enabled? + + sqs = SqsService.new(Danbooru.config.aws_sqs_queue_url) + sqs.send_message("refresh\n#{user_id}") end def reset_listbooru(user_id) - return unless Danbooru.config.listbooru_auth_key - - uri = URI.parse("#{Danbooru.config.listbooru_server}/searches") - Net::HTTP.start(uri.host, uri.port) do |http| - req = Net::HTTP::Delete.new("/searches") - req.set_form_data("user_id" => user_id, "all" => "true", "key" => Danbooru.config.listbooru_auth_key) - http.request(req) - end + return false unless Danbooru.config.listbooru_enabled? + sqs = SqsService.new(Danbooru.config.aws_sqs_queue_url) user = User.find(user_id) + + sqs.send_message("delete\n#{user_id}\nall\n") + user.saved_searches.each do |saved_search| - update_listbooru_on_create(user_id, saved_search.category, saved_search.tag_query) + sqs.send_message("create\n#{user_id}\n#{saved_search.category}\n#{saved_search.tag_query}", :delay_seconds => 30) end - end - def update_listbooru_on_create(user_id, name, query) - return unless Danbooru.config.listbooru_auth_key - uri = URI.parse("#{Danbooru.config.listbooru_server}/searches") - Net::HTTP.post_form(uri, {"user_id" => user_id, "name" => name.try(:downcase), "query" => query, "key" => Danbooru.config.listbooru_auth_key}) - end - - def update_listbooru_on_destroy(user_id, name, query) - return unless Danbooru.config.listbooru_auth_key - uri = URI.parse("#{Danbooru.config.listbooru_server}/searches") - Net::HTTP.start(uri.host, uri.port) do |http| - req = Net::HTTP::Delete.new("/searches") - req.set_form_data("user_id" => user_id, "name" => name.try(:downcase), "query" => query, "key" => Danbooru.config.listbooru_auth_key) - http.request(req) - end - end - - def update_listbooru_on_update(user_id, old_name, old_query, new_name, new_query) - return unless Danbooru.config.listbooru_auth_key - uri = URI.parse("#{Danbooru.config.listbooru_server}/searches") - Net::HTTP.start(uri.host, uri.port) do |http| - req = Net::HTTP::Put.new("/searches") - req.set_form_data( - "user_id" => user_id, - "old_name" => old_name.try(:downcase), - "old_query" => old_query, - "new_name" => new_name.try(:downcase), - "new_query" => new_query, - "key" => Danbooru.config.listbooru_auth_key - ) - http.request(req) - end + true end end def update_listbooru_on_create - return unless Danbooru.config.listbooru_auth_key - SavedSearch.delay(:queue => "default").update_listbooru_on_create(user_id, category, tag_query) + return false unless Danbooru.config.listbooru_enabled? + + sqs = SqsService.new(Danbooru.config.aws_sqs_queue_url) + sqs.send_message("create\n#{user_id}\n#{category}\n#{tag_query}") end def update_listbooru_on_destroy - return unless Danbooru.config.listbooru_auth_key - SavedSearch.delay(:queue => "default").update_listbooru_on_destroy(user_id, category, tag_query) + return false unless Danbooru.config.listbooru_enabled? + + sqs = SqsService.new(Danbooru.config.aws_sqs_queue_url) + sqs.send_message("delete\n#{user_id}\n#{category}\n#{tag_query}") end def update_listbooru_on_update - return unless Danbooru.config.listbooru_auth_key - SavedSearch.delay(:queue => "default").update_listbooru_on_update(user_id, category_was, tag_query_was, category, tag_query) + return false unless Danbooru.config.listbooru_enabled? + + sqs = SqsService.new(Danbooru.config.aws_sqs_queue_url) + sqs.send_message("update\n#{user_id}\n#{category_was}\n#{tag_query_was}\n#{category}\n#{tag_query}") end end @@ -110,6 +71,8 @@ class SavedSearch < ActiveRecord::Base end def self.post_ids(user_id, name = nil) + return [] unless Danbooru.config.listbooru_enabled? + params = { "key" => Danbooru.config.listbooru_auth_key, "user_id" => user_id, diff --git a/config/danbooru_default_config.rb b/config/danbooru_default_config.rb index 236742c31..cef84d9b9 100644 --- a/config/danbooru_default_config.rb +++ b/config/danbooru_default_config.rb @@ -295,15 +295,6 @@ module Danbooru nil end - def amazon_ses - # {:smtp_server_name => "smtp server", :user_name => "user name", :ses_smtp_user_name => "smtp user name", :ses_smtp_password => "smtp password"} - nil - end - - def amazon_s3_bucket_name - "danbooru" - end - def enable_dimension_autotagging true end @@ -396,10 +387,51 @@ module Danbooru def addthis_key end + # listbooru options + def listbooru_enabled? + false + end + def listbooru_server end def listbooru_auth_key end + + # AWS config options + def aws_access_key_id + nil + end + + def aws_secret_access_key + nil + end + + def aws_ses_enabled? + false + end + + def aws_ses_options + # {:smtp_server_name => "smtp server", :user_name => "user name", :ses_smtp_user_name => "smtp user name", :ses_smtp_password => "smtp password"} + nil + end + + def aws_s3_enabled? + false + end + + def aws_s3_bucket_name + "danbooru" + end + + def aws_sqs_enabled? + false + end + + def aws_sqs_queue_url + end + + def aws_sqs_region + end end end diff --git a/config/initializers/z_action_mailer.rb b/config/initializers/z_action_mailer.rb index 031154e16..2af5627b5 100644 --- a/config/initializers/z_action_mailer.rb +++ b/config/initializers/z_action_mailer.rb @@ -1,8 +1,8 @@ -if Danbooru.config.amazon_ses && Rails.env == "production" +if Danbooru.config.aws_ses_enabled? && Rails.env == "production" Rails.application.config.action_mailer.smtp_settings = { - :address => Danbooru.config.amazon_ses[:smtp_server_name], - :user_name => Danbooru.config.amazon_ses[:ses_smtp_user_name], - :password => Danbooru.config.amazon_ses[:ses_smtp_password], + :address => Danbooru.config.aws_ses_options[:smtp_server_name], + :user_name => Danbooru.config.aws_ses_options[:ses_smtp_user_name], + :password => Danbooru.config.aws_ses_options[:ses_smtp_password], :authentication => :login, :enable_starttls_auto => true } diff --git a/script/donmai/backup_db b/script/donmai/backup_db deleted file mode 100755 index e132328ce..000000000 --- a/script/donmai/backup_db +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -PGOPTIONS="-c statement_timeout=0" pg_dump -Fc -hdbserver -f /var/www/danbooru2/shared/backup/db-`date +"%Y-%m-%d-%H-%M"`.dump danbooru2 diff --git a/script/donmai/backup_db_to_s3 b/script/donmai/backup_db_to_s3 deleted file mode 100755 index 17ff93af1..000000000 --- a/script/donmai/backup_db_to_s3 +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env ruby - -require 'rubygems' -require 'aws-sdk/s3' - -MAX_BACKUPS = 30 - -home = File.expand_path("~") - -if !File.exists?("#{home}/.s3/access_key") - STDERR.puts "Access key not found" - exit 1 -end - -if !File.exists?("#{home}/.s3/secret_access_key") - STDERR.puts "Secret access key not found" - exit 1 -end - -access_key = open("#{home}/.s3/access_key").read.strip -secret_access_key = open("#{home}/.s3/secret_access_key").read.strip -credentials = Aws::Credentials.new(Danbooru.config.amazon_s3_access_key_id, Danbooru.config.amazon_s3_secret_access_key) -Aws.config.update({ - region: "us-east-1", - credentials: credentials -}) -client = Aws::S3::Client.new -bucket = "danbooru-backup" - -current_backups = client.list_objects(buckets: buckets).contents.map {|x| x.key}.select {|x| x =~ /^db-/}.sort.reverse - -if current_backups.size > MAX_BACKUPS - current_backups[MAX_BACKUPS..-1].each do |old_backup| - client.delete_object(bucket: bucket, key: old_backup) - puts "Deleted old backup #{old_backup}" - end -end - -backup = Dir["/var/www/danbooru2/shared/backup/db-*.dump"].sort.last -data = File.open(backup, "rb") -filename = data.mtime.strftime("db-%Y-%m-%d-%H-%M") -tries = 0 - -begin - client.put_object(bucket: bucket, key: filename, body: data, :acl => "private") -rescue Errno::EPIPE - tries += 1 - if tries > 3 - raise - else - retry - end -end - -puts "Uploaded #{backup} as #{filename}" diff --git a/script/donmai/prune_backup_dbs b/script/donmai/prune_backup_dbs deleted file mode 100755 index 996f89361..000000000 --- a/script/donmai/prune_backup_dbs +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -find /var/www/danbooru2/shared/backup -name 'db-*.dump' -mtime +30 -exec /bin/rm -f {} \;