* refactor aws config options

* fix aws calls in savedsearch
* remove unused scripts

app/controllers/saved_searches_controller.rb

@@ -4,7 +4,7 @@ class SavedSearchesController < ApplicationController
   
   def index
     if Danbooru.config.listbooru_server
-      SavedSearch.delay(:queue => "default").refresh_listbooru(CurrentUser.id)
+      SavedSearch.refresh_listbooru(CurrentUser.id)
     end
 
     @saved_searches = saved_searches.order("tag_query")

app/logical/sqs_service.rb

@@ -0,0 +1,33 @@
+class SqsService
+  attr_reader :url
+
+  def initialize(url)
+    @url = url
+  end
+
+  def send_message(string, options = {})
+    return unless Danbooru.config.aws_sqs_enabled?
+
+    sqs.send_message(
+      options.merge(
+        message_body: string,
+        queue_url: url
+      )
+    )
+  end
+
+private
+
+  def sqs
+    @sqs ||= begin
+      credentials = Aws::Credentials.new(
+        Danbooru.config.aws_access_key_id,
+        Danbooru.config.aws_secret_access_key
+      )
+      Aws::SQS::Client.new(
+        credentials: credentials,
+        region: Danbooru.config.aws_sqs_region
+      )
+    end
+  end
+end

app/models/amazon_backup.rb

@@ -13,14 +13,16 @@ class AmazonBackup < ActiveRecord::Base
   end
 
   def self.execute
+    return false unless Danbooru.config.aws_s3_enabled?
+    
     last_id = AmazonBackup.last_id
-    credentials = Aws::Credentials.new(Danbooru.config.amazon_s3_access_key_id, Danbooru.config.amazon_s3_secret_access_key)
+    credentials = Aws::Credentials.new(Danbooru.config.aws_access_key_id, Danbooru.config.aws_secret_access_key)
     Aws.config.update({
       region: "us-east-1",
       credentials: credentials
     })
     client = Aws::S3::Client.new
-    bucket = Danbooru.config.amazon_s3_bucket_name
+    bucket = Danbooru.config.aws_s3_bucket_name
 
     Post.where("id > ?", last_id).limit(1000).order("id").each do |post|
       if File.exists?(post.file_path)

app/models/saved_search.rb

@@ -3,87 +3,48 @@ class SavedSearch < ActiveRecord::Base
     extend ActiveSupport::Concern
 
     module ClassMethods
-      def posts_search_available?
-        Danbooru.config.listbooru_server.present? && CurrentUser.is_gold?
-      end
-
       def refresh_listbooru(user_id)
-        return unless Danbooru.config.listbooru_auth_key
+        return false unless Danbooru.config.listbooru_enabled?
-        user = User.find(user_id)
-        return unless user.is_gold?
 
-        params = {
+        sqs = SqsService.new(Danbooru.config.aws_sqs_queue_url)
-          :user_id => user_id,
+        sqs.send_message("refresh\n#{user_id}")
-          :key => Danbooru.config.listbooru_auth_key
-        }
-        uri = URI.parse("#{Danbooru.config.listbooru_server}/users")
-        uri.query = URI.encode_www_form(params)
-        Net::HTTP.get_response(uri)
       end
 
       def reset_listbooru(user_id)
-        return unless Danbooru.config.listbooru_auth_key
+        return false unless Danbooru.config.listbooru_enabled?
-
-        uri = URI.parse("#{Danbooru.config.listbooru_server}/searches")
-        Net::HTTP.start(uri.host, uri.port) do |http|
-          req = Net::HTTP::Delete.new("/searches")
-          req.set_form_data("user_id" => user_id, "all" => "true", "key" => Danbooru.config.listbooru_auth_key)
-          http.request(req)
-        end
 
+        sqs = SqsService.new(Danbooru.config.aws_sqs_queue_url)
         user = User.find(user_id)
+
+        sqs.send_message("delete\n#{user_id}\nall\n")
+
         user.saved_searches.each do |saved_search|
-          update_listbooru_on_create(user_id, saved_search.category, saved_search.tag_query)
+          sqs.send_message("create\n#{user_id}\n#{saved_search.category}\n#{saved_search.tag_query}", :delay_seconds => 30)
         end
-      end
 
-      def update_listbooru_on_create(user_id, name, query)
+        true
-        return unless Danbooru.config.listbooru_auth_key
-        uri = URI.parse("#{Danbooru.config.listbooru_server}/searches")
-        Net::HTTP.post_form(uri, {"user_id" => user_id, "name" => name.try(:downcase), "query" => query, "key" => Danbooru.config.listbooru_auth_key})
-      end
-
-      def update_listbooru_on_destroy(user_id, name, query)
-        return unless Danbooru.config.listbooru_auth_key
-        uri = URI.parse("#{Danbooru.config.listbooru_server}/searches")
-        Net::HTTP.start(uri.host, uri.port) do |http|
-          req = Net::HTTP::Delete.new("/searches")
-          req.set_form_data("user_id" => user_id, "name" => name.try(:downcase), "query" => query, "key" => Danbooru.config.listbooru_auth_key)
-          http.request(req)
-        end
-      end
-
-      def update_listbooru_on_update(user_id, old_name, old_query, new_name, new_query)
-        return unless Danbooru.config.listbooru_auth_key
-        uri = URI.parse("#{Danbooru.config.listbooru_server}/searches")
-        Net::HTTP.start(uri.host, uri.port) do |http|
-          req = Net::HTTP::Put.new("/searches")
-          req.set_form_data(
-            "user_id" => user_id, 
-            "old_name" => old_name.try(:downcase),
-            "old_query" => old_query,
-            "new_name" => new_name.try(:downcase), 
-            "new_query" => new_query, 
-            "key" => Danbooru.config.listbooru_auth_key
-          )
-          http.request(req)
-        end
       end
     end
 
     def update_listbooru_on_create
-      return unless Danbooru.config.listbooru_auth_key
+      return false unless Danbooru.config.listbooru_enabled?
-      SavedSearch.delay(:queue => "default").update_listbooru_on_create(user_id, category, tag_query)
+
+      sqs = SqsService.new(Danbooru.config.aws_sqs_queue_url)
+      sqs.send_message("create\n#{user_id}\n#{category}\n#{tag_query}")
     end
 
     def update_listbooru_on_destroy
-      return unless Danbooru.config.listbooru_auth_key
+      return false unless Danbooru.config.listbooru_enabled?
-      SavedSearch.delay(:queue => "default").update_listbooru_on_destroy(user_id, category, tag_query)
+
+      sqs = SqsService.new(Danbooru.config.aws_sqs_queue_url)
+      sqs.send_message("delete\n#{user_id}\n#{category}\n#{tag_query}")
     end
 
     def update_listbooru_on_update
-      return unless Danbooru.config.listbooru_auth_key
+      return false unless Danbooru.config.listbooru_enabled?
-      SavedSearch.delay(:queue => "default").update_listbooru_on_update(user_id, category_was, tag_query_was, category, tag_query)
+
+      sqs = SqsService.new(Danbooru.config.aws_sqs_queue_url)
+      sqs.send_message("update\n#{user_id}\n#{category_was}\n#{tag_query_was}\n#{category}\n#{tag_query}")
     end
   end
 
@@ -110,6 +71,8 @@ class SavedSearch < ActiveRecord::Base
   end
 
   def self.post_ids(user_id, name = nil)
+    return [] unless Danbooru.config.listbooru_enabled?
+
     params = {
       "key" => Danbooru.config.listbooru_auth_key,
       "user_id" => user_id,

config/danbooru_default_config.rb

@@ -295,15 +295,6 @@ module Danbooru
       nil
     end
 
-    def amazon_ses
-      # {:smtp_server_name => "smtp server", :user_name => "user name", :ses_smtp_user_name => "smtp user name", :ses_smtp_password => "smtp password"}
-      nil
-    end
-
-    def amazon_s3_bucket_name
-      "danbooru"
-    end
-
     def enable_dimension_autotagging
       true
     end
@@ -396,10 +387,51 @@ module Danbooru
     def addthis_key
     end
 
+    # listbooru options
+    def listbooru_enabled?
+      false
+    end
+
     def listbooru_server
     end
 
     def listbooru_auth_key
     end
+
+    # AWS config options
+    def aws_access_key_id
+      nil
+    end
+
+    def aws_secret_access_key
+      nil
+    end
+
+    def aws_ses_enabled?
+      false
+    end
+
+    def aws_ses_options
+      # {:smtp_server_name => "smtp server", :user_name => "user name", :ses_smtp_user_name => "smtp user name", :ses_smtp_password => "smtp password"}
+      nil
+    end
+
+    def aws_s3_enabled?
+      false
+    end
+
+    def aws_s3_bucket_name
+      "danbooru"
+    end
+
+    def aws_sqs_enabled?
+      false
+    end
+
+    def aws_sqs_queue_url
+    end
+
+    def aws_sqs_region
+    end
   end
 end

config/initializers/z_action_mailer.rb

@@ -1,8 +1,8 @@
-if Danbooru.config.amazon_ses && Rails.env == "production"
+if Danbooru.config.aws_ses_enabled? && Rails.env == "production"
   Rails.application.config.action_mailer.smtp_settings = {
-    :address => Danbooru.config.amazon_ses[:smtp_server_name],
+    :address => Danbooru.config.aws_ses_options[:smtp_server_name],
-    :user_name => Danbooru.config.amazon_ses[:ses_smtp_user_name],
+    :user_name => Danbooru.config.aws_ses_options[:ses_smtp_user_name],
-    :password => Danbooru.config.amazon_ses[:ses_smtp_password],
+    :password => Danbooru.config.aws_ses_options[:ses_smtp_password],
     :authentication => :login,
     :enable_starttls_auto => true
   }

script/donmai/backup_db

@@ -1,3 +0,0 @@
-#!/bin/bash
-
-PGOPTIONS="-c statement_timeout=0" pg_dump -Fc -hdbserver -f /var/www/danbooru2/shared/backup/db-`date +"%Y-%m-%d-%H-%M"`.dump danbooru2

script/donmai/backup_db_to_s3

@@ -1,55 +0,0 @@
-#!/usr/bin/env ruby
-
-require 'rubygems'
-require 'aws-sdk/s3'
-
-MAX_BACKUPS = 30
-
-home = File.expand_path("~")
-
-if !File.exists?("#{home}/.s3/access_key")
-  STDERR.puts "Access key not found"
-  exit 1
-end
-
-if !File.exists?("#{home}/.s3/secret_access_key")
-  STDERR.puts "Secret access key not found"
-  exit 1
-end
-
-access_key = open("#{home}/.s3/access_key").read.strip
-secret_access_key = open("#{home}/.s3/secret_access_key").read.strip
-credentials = Aws::Credentials.new(Danbooru.config.amazon_s3_access_key_id, Danbooru.config.amazon_s3_secret_access_key)
-Aws.config.update({
-  region: "us-east-1",
-  credentials: credentials
-})
-client = Aws::S3::Client.new
-bucket = "danbooru-backup"
-
-current_backups = client.list_objects(buckets: buckets).contents.map {|x| x.key}.select {|x| x =~ /^db-/}.sort.reverse
-
-if current_backups.size > MAX_BACKUPS
-  current_backups[MAX_BACKUPS..-1].each do |old_backup|
-    client.delete_object(bucket: bucket, key: old_backup)
-    puts "Deleted old backup #{old_backup}"
-  end
-end
-
-backup = Dir["/var/www/danbooru2/shared/backup/db-*.dump"].sort.last
-data = File.open(backup, "rb")
-filename = data.mtime.strftime("db-%Y-%m-%d-%H-%M")
-tries = 0
-
-begin
-  client.put_object(bucket: bucket, key: filename, body: data, :acl => "private")
-rescue Errno::EPIPE
-  tries += 1
-  if tries > 3
-    raise
-  else
-    retry
-  end
-end
-
-puts "Uploaded #{backup} as #{filename}"

script/donmai/prune_backup_dbs

@@ -1,3 +0,0 @@
-#!/bin/bash
-
-find /var/www/danbooru2/shared/backup -name 'db-*.dump' -mtime +30 -exec /bin/rm -f {} \;