jaculabilis/danbooru
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

* refactor aws config options

Browse Source 
* fix aws calls in savedsearch
* remove unused scripts
This commit is contained in:
r888888888
2015-12-28 12:25:38 -08:00
parent 811a60bb9d
commit d5fa4b46e3
9 changed files with 108 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
app/controllers/saved_searches_controller.rb
View File

@@ -4,7 +4,7 @@ class SavedSearchesController < ApplicationController
def index
if Danbooru.config.listbooru_server
SavedSearch.delay(:queue => "default").refresh_listbooru(CurrentUser.id)
SavedSearch.refresh_listbooru(CurrentUser.id)
end
@saved_searches = saved_searches.order("tag_query")

33
app/logical/sqs_service.rb Normal file
View File

@@ -0,0 +1,33 @@
class SqsService
attr_reader :url
def initialize(url)
@url = url
end
def send_message(string, options = {})
return unless Danbooru.config.aws_sqs_enabled?
sqs.send_message(
options.merge(
message_body: string,
queue_url: url
)
)
end
private
def sqs
@sqs ||= begin
credentials = Aws::Credentials.new(
Danbooru.config.aws_access_key_id,
Danbooru.config.aws_secret_access_key
)
Aws::SQS::Client.new(
credentials: credentials,
region: Danbooru.config.aws_sqs_region
)
end
end
end

6
app/models/amazon_backup.rb
View File

@@ -13,14 +13,16 @@ class AmazonBackup < ActiveRecord::Base
end
def self.execute
return false unless Danbooru.config.aws_s3_enabled?
last_id = AmazonBackup.last_id
credentials = Aws::Credentials.new(Danbooru.config.amazon_s3_access_key_id, Danbooru.config.amazon_s3_secret_access_key)
credentials = Aws::Credentials.new(Danbooru.config.aws_access_key_id, Danbooru.config.aws_secret_access_key)
Aws.config.update({
region: "us-east-1",
credentials: credentials
})
client = Aws::S3::Client.new
bucket = Danbooru.config.amazon_s3_bucket_name
bucket = Danbooru.config.aws_s3_bucket_name
Post.where("id > ?", last_id).limit(1000).order("id").each do |post|
if File.exists?(post.file_path)

87
app/models/saved_search.rb
View File

@@ -3,87 +3,48 @@ class SavedSearch < ActiveRecord::Base
extend ActiveSupport::Concern
module ClassMethods
def posts_search_available?
Danbooru.config.listbooru_server.present? && CurrentUser.is_gold?
end
def refresh_listbooru(user_id)
return unless Danbooru.config.listbooru_auth_key
user = User.find(user_id)
return unless user.is_gold?
params = {
:user_id => user_id,
:key => Danbooru.config.listbooru_auth_key
}
uri = URI.parse("#{Danbooru.config.listbooru_server}/users")
uri.query = URI.encode_www_form(params)
Net::HTTP.get_response(uri)
return false unless Danbooru.config.listbooru_enabled?
sqs = SqsService.new(Danbooru.config.aws_sqs_queue_url)
sqs.send_message("refresh\n#{user_id}")
end
def reset_listbooru(user_id)
return unless Danbooru.config.listbooru_auth_key
uri = URI.parse("#{Danbooru.config.listbooru_server}/searches")
Net::HTTP.start(uri.host, uri.port) do |http|
req = Net::HTTP::Delete.new("/searches")
req.set_form_data("user_id" => user_id, "all" => "true", "key" => Danbooru.config.listbooru_auth_key)
http.request(req)
end
return false unless Danbooru.config.listbooru_enabled?
sqs = SqsService.new(Danbooru.config.aws_sqs_queue_url)
user = User.find(user_id)
sqs.send_message("delete\n#{user_id}\nall\n")
user.saved_searches.each do |saved_search|
update_listbooru_on_create(user_id, saved_search.category, saved_search.tag_query)
sqs.send_message("create\n#{user_id}\n#{saved_search.category}\n#{saved_search.tag_query}", :delay_seconds => 30)
end
end
def update_listbooru_on_create(user_id, name, query)
return unless Danbooru.config.listbooru_auth_key
uri = URI.parse("#{Danbooru.config.listbooru_server}/searches")
Net::HTTP.post_form(uri, {"user_id" => user_id, "name" => name.try(:downcase), "query" => query, "key" => Danbooru.config.listbooru_auth_key})
end
def update_listbooru_on_destroy(user_id, name, query)
return unless Danbooru.config.listbooru_auth_key
uri = URI.parse("#{Danbooru.config.listbooru_server}/searches")
Net::HTTP.start(uri.host, uri.port) do |http|
req = Net::HTTP::Delete.new("/searches")
req.set_form_data("user_id" => user_id, "name" => name.try(:downcase), "query" => query, "key" => Danbooru.config.listbooru_auth_key)
http.request(req)
end
end
def update_listbooru_on_update(user_id, old_name, old_query, new_name, new_query)
return unless Danbooru.config.listbooru_auth_key
uri = URI.parse("#{Danbooru.config.listbooru_server}/searches")
Net::HTTP.start(uri.host, uri.port) do |http|
req = Net::HTTP::Put.new("/searches")
req.set_form_data(
"user_id" => user_id,
"old_name" => old_name.try(:downcase),
"old_query" => old_query,
"new_name" => new_name.try(:downcase),
"new_query" => new_query,
"key" => Danbooru.config.listbooru_auth_key
)
http.request(req)
end
true
end
end
def update_listbooru_on_create
return unless Danbooru.config.listbooru_auth_key
SavedSearch.delay(:queue => "default").update_listbooru_on_create(user_id, category, tag_query)
return false unless Danbooru.config.listbooru_enabled?
sqs = SqsService.new(Danbooru.config.aws_sqs_queue_url)
sqs.send_message("create\n#{user_id}\n#{category}\n#{tag_query}")
end
def update_listbooru_on_destroy
return unless Danbooru.config.listbooru_auth_key
SavedSearch.delay(:queue => "default").update_listbooru_on_destroy(user_id, category, tag_query)
return false unless Danbooru.config.listbooru_enabled?
sqs = SqsService.new(Danbooru.config.aws_sqs_queue_url)
sqs.send_message("delete\n#{user_id}\n#{category}\n#{tag_query}")
end
def update_listbooru_on_update
return unless Danbooru.config.listbooru_auth_key
SavedSearch.delay(:queue => "default").update_listbooru_on_update(user_id, category_was, tag_query_was, category, tag_query)
return false unless Danbooru.config.listbooru_enabled?
sqs = SqsService.new(Danbooru.config.aws_sqs_queue_url)
sqs.send_message("update\n#{user_id}\n#{category_was}\n#{tag_query_was}\n#{category}\n#{tag_query}")
end
end
@@ -110,6 +71,8 @@ class SavedSearch < ActiveRecord::Base
end
def self.post_ids(user_id, name = nil)
return [] unless Danbooru.config.listbooru_enabled?
params = {
"key" => Danbooru.config.listbooru_auth_key,
"user_id" => user_id,

50
config/danbooru_default_config.rb
View File

@@ -295,15 +295,6 @@ module Danbooru
nil
end
def amazon_ses
# {:smtp_server_name => "smtp server", :user_name => "user name", :ses_smtp_user_name => "smtp user name", :ses_smtp_password => "smtp password"}
nil
end
def amazon_s3_bucket_name
"danbooru"
end
def enable_dimension_autotagging
true
end
@@ -396,10 +387,51 @@ module Danbooru
def addthis_key
end
# listbooru options
def listbooru_enabled?
false
end
def listbooru_server
end
def listbooru_auth_key
end
# AWS config options
def aws_access_key_id
nil
end
def aws_secret_access_key
nil
end
def aws_ses_enabled?
false
end
def aws_ses_options
# {:smtp_server_name => "smtp server", :user_name => "user name", :ses_smtp_user_name => "smtp user name", :ses_smtp_password => "smtp password"}
nil
end
def aws_s3_enabled?
false
end
def aws_s3_bucket_name
"danbooru"
end
def aws_sqs_enabled?
false
end
def aws_sqs_queue_url
end
def aws_sqs_region
end
end
end

8
config/initializers/z_action_mailer.rb
View File

@@ -1,8 +1,8 @@
if Danbooru.config.amazon_ses && Rails.env == "production"
if Danbooru.config.aws_ses_enabled? && Rails.env == "production"
Rails.application.config.action_mailer.smtp_settings = {
:address => Danbooru.config.amazon_ses[:smtp_server_name],
:user_name => Danbooru.config.amazon_ses[:ses_smtp_user_name],
:password => Danbooru.config.amazon_ses[:ses_smtp_password],
:address => Danbooru.config.aws_ses_options[:smtp_server_name],
:user_name => Danbooru.config.aws_ses_options[:ses_smtp_user_name],
:password => Danbooru.config.aws_ses_options[:ses_smtp_password],
:authentication => :login,
:enable_starttls_auto => true
}

3
script/donmai/backup_db
View File

@@ -1,3 +0,0 @@
#!/bin/bash
PGOPTIONS="-c statement_timeout=0" pg_dump -Fc -hdbserver -f /var/www/danbooru2/shared/backup/db-`date +"%Y-%m-%d-%H-%M"`.dump danbooru2

55
script/donmai/backup_db_to_s3
View File

@@ -1,55 +0,0 @@
#!/usr/bin/env ruby
require 'rubygems'
require 'aws-sdk/s3'
MAX_BACKUPS = 30
home = File.expand_path("~")
if !File.exists?("#{home}/.s3/access_key")
STDERR.puts "Access key not found"
exit 1
end
if !File.exists?("#{home}/.s3/secret_access_key")
STDERR.puts "Secret access key not found"
exit 1
end
access_key = open("#{home}/.s3/access_key").read.strip
secret_access_key = open("#{home}/.s3/secret_access_key").read.strip
credentials = Aws::Credentials.new(Danbooru.config.amazon_s3_access_key_id, Danbooru.config.amazon_s3_secret_access_key)
Aws.config.update({
region: "us-east-1",
credentials: credentials
})
client = Aws::S3::Client.new
bucket = "danbooru-backup"
current_backups = client.list_objects(buckets: buckets).contents.map {|x| x.key}.select {|x| x =~ /^db-/}.sort.reverse
if current_backups.size > MAX_BACKUPS
current_backups[MAX_BACKUPS..-1].each do |old_backup|
client.delete_object(bucket: bucket, key: old_backup)
puts "Deleted old backup #{old_backup}"
end
end
backup = Dir["/var/www/danbooru2/shared/backup/db-*.dump"].sort.last
data = File.open(backup, "rb")
filename = data.mtime.strftime("db-%Y-%m-%d-%H-%M")
tries = 0
begin
client.put_object(bucket: bucket, key: filename, body: data, :acl => "private")
rescue Errno::EPIPE
tries += 1
if tries > 3
raise
else
retry
end
end
puts "Uploaded #{backup} as #{filename}"

3
script/donmai/prune_backup_dbs
View File

@@ -1,3 +0,0 @@
#!/bin/bash
find /var/www/danbooru2/shared/backup -name 'db-*.dump' -mtime +30 -exec /bin/rm -f {} \;