From 316690f39370b1b5b9f6d3a5ffc65fbd326f113d Mon Sep 17 00:00:00 2001 From: evazion Date: Fri, 13 Dec 2019 01:49:42 -0600 Subject: [PATCH] storage manager: remove S3 backend. Remove in favor of the rclone S3 backend. --- Gemfile | 1 - Gemfile.lock | 8 --- app/logical/storage_manager/s3.rb | 47 -------------- config/danbooru_default_config.rb | 12 ---- lib/tasks/images.rake | 100 ------------------------------ 5 files changed, 168 deletions(-) delete mode 100644 app/logical/storage_manager/s3.rb diff --git a/Gemfile b/Gemfile index 72819d68a..6887ab8d3 100644 --- a/Gemfile +++ b/Gemfile @@ -22,7 +22,6 @@ gem 'streamio-ffmpeg' gem 'rubyzip', :require => "zip" gem 'stripe' gem 'twitter' -gem 'aws-sdk-s3', '~> 1' gem 'aws-sdk-sqs', '~> 1' gem 'responders' gem 'dtext_rb', git: "https://github.com/evazion/dtext_rb.git", require: "dtext" diff --git a/Gemfile.lock b/Gemfile.lock index 78f7548e0..5a223e61c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -86,13 +86,6 @@ GEM aws-partitions (~> 1, >= 1.239.0) aws-sigv4 (~> 1.1) jmespath (~> 1.0) - aws-sdk-kms (1.26.0) - aws-sdk-core (~> 3, >= 3.71.0) - aws-sigv4 (~> 1.1) - aws-sdk-s3 (1.59.0) - aws-sdk-core (~> 3, >= 3.83.0) - aws-sdk-kms (~> 1) - aws-sigv4 (~> 1.1) aws-sdk-sqs (1.23.1) aws-sdk-core (~> 3, >= 3.71.0) aws-sigv4 (~> 1.1) @@ -424,7 +417,6 @@ DEPENDENCIES activemodel-serializers-xml addressable awesome_print - aws-sdk-s3 (~> 1) aws-sdk-sqs (~> 1) bcrypt bootsnap diff --git a/app/logical/storage_manager/s3.rb b/app/logical/storage_manager/s3.rb deleted file mode 100644 index 43020d822..000000000 --- a/app/logical/storage_manager/s3.rb +++ /dev/null @@ -1,47 +0,0 @@ -class StorageManager::S3 < StorageManager - # https://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Client.html#initialize-instance_method - DEFAULT_S3_OPTIONS = { - region: Danbooru.config.aws_region, - credentials: Danbooru.config.aws_credentials, - logger: Rails.logger, - } - - # https://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Client.html#put_object-instance_method - DEFAULT_PUT_OPTIONS = { - acl: "public-read", - storage_class: "STANDARD", # STANDARD, STANDARD_IA, REDUCED_REDUNDANCY - cache_control: "public, max-age=#{1.year.to_i}", - #content_type: "image/jpeg" # XXX should set content type - } - - attr_reader :bucket, :client, :s3_options - - def initialize(bucket, client: nil, s3_options: {}, **options) - @bucket = bucket - @s3_options = DEFAULT_S3_OPTIONS.merge(s3_options) - @client = client || Aws::S3::Client.new(**@s3_options) - super(**options) - end - - def key(path) - path.sub(/^.+?data\//, "") - end - - def store(io, path) - data = io.read - base64_md5 = Digest::MD5.base64digest(data) - client.put_object(bucket: bucket, key: key(path), body: data, content_md5: base64_md5, **DEFAULT_PUT_OPTIONS) - end - - def delete(path) - client.delete_object(bucket: bucket, key: key(path)) - rescue Aws::S3::Errors::NoSuchKey - # ignore - end - - def open(path) - file = Tempfile.new(binmode: true) - client.get_object(bucket: bucket, key: key(path), response_target: file) - file - end -end diff --git a/config/danbooru_default_config.rb b/config/danbooru_default_config.rb index d7d36db88..2ca22898a 100644 --- a/config/danbooru_default_config.rb +++ b/config/danbooru_default_config.rb @@ -204,12 +204,6 @@ module Danbooru # ~/.ssh_config or in the ssh_options param (ref: http://net-ssh.github.io/net-ssh/Net/SSH.html#method-c-start) # StorageManager::SFTP.new("i1.example.com", "i2.example.com", base_dir: "/mnt/backup", hierarchical: false, ssh_options: {}) - # Store files in an S3 bucket. The bucket must already exist and be - # writable by you. Configure your S3 settings in aws_region and - # aws_credentials below, or in the s3_options param (ref: - # https://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Client.html#initialize-instance_method) - # StorageManager::S3.new("my_s3_bucket", base_url: "https://my_s3_bucket.s3.amazonaws.com/", s3_options: {}) - # Select the storage method based on the post's id and type (preview, large, or original). # StorageManager::Hybrid.new do |id, md5, file_ext, type| # ssh_options = { user: "danbooru" } @@ -237,12 +231,6 @@ module Danbooru # Backup files to /mnt/backup on a remote system. Configure SSH settings # in ~/.ssh_config or in the ssh_options param (ref: http://net-ssh.github.io/net-ssh/Net/SSH.html#method-c-start) # StorageManager::SFTP.new("www.example.com", base_dir: "/mnt/backup", ssh_options: {}) - - # Backup files to an S3 bucket. The bucket must already exist and be - # writable by you. Configure your S3 settings in aws_region and - # aws_credentials below, or in the s3_options param (ref: - # https://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Client.html#initialize-instance_method) - # StorageManager::S3.new("my_s3_bucket_name", s3_options: {}) end #TAG CONFIGURATION diff --git a/lib/tasks/images.rake b/lib/tasks/images.rake index aeddf5f40..11524bb41 100644 --- a/lib/tasks/images.rake +++ b/lib/tasks/images.rake @@ -1,104 +1,4 @@ namespace :images do - desc "Distribute posts to all servers via SFTP" - task :distribute, [:min_id, :max_id] => :environment do |t, args| - min_id = args[:min_id] - max_id = args[:max_id] - lsm = StorageManager::Local.new(base_url: "https://danbooru.donmai.us/data", base_dir: "/var/www/danbooru2/shared/data", hierarchical: false) - sftpsm = StorageManager::SFTP.new(*Danbooru.config.all_server_hosts, base_url: "https://danbooru.donmai.us/data") - - Post.where("id between ? and ?", min_id, max_id).find_each do |post| - sftpsm.store_file(lsm.open_file(post, :original), post, :original) - sftpsm.store_file(lsm.open_file(post, :large), post, :large) if post.has_large? - sftpsm.store_file(lsm.open_file(post, :preview), post, :preview) if post.has_preview? - end - end - - desc "Reset S3 + Storage Class" - task :reset_s3, [:min_id, :max_id] => :environment do |t, args| - min_id = args[:min_id] # 1 - max_id = args[:max_id] # 50_000 - - credentials = Aws::Credentials.new(Danbooru.config.aws_access_key_id, Danbooru.config.aws_secret_access_key) - Aws.config.update({ - region: "us-east-1", - credentials: credentials - }) - client = Aws::S3::Client.new - bucket = Danbooru.config.aws_s3_bucket_name - - Post.where("id >= ? and id <= ?", min_id, max_id).find_each do |post| - key = File.basename(post.file_path) - begin - client.copy_object(bucket: bucket, key: key, acl: "public-read", storage_class: "STANDARD", copy_source: "/#{bucket}/#{key}", metadata_directive: "COPY") - puts "copied #{post.id}" - rescue Aws::S3::Errors::InvalidObjectState - puts "invalid state #{post.id}" - rescue Aws::S3::Errors::NoSuchKey - puts "missing #{post.id}" - end - end - end - - desc "restore from glacier" - task :restore_glacier, [:min_id, :max_id] => :environment do |t, args| - min_id = args[:min_id] # 10_001 - max_id = args[:max_id] # 50_000 - - credentials = Aws::Credentials.new(Danbooru.config.aws_access_key_id, Danbooru.config.aws_secret_access_key) - Aws.config.update({ - region: "us-east-1", - credentials: credentials - }) - client = Aws::S3::Client.new - bucket = Danbooru.config.aws_s3_bucket_name - - Post.where("id >= ? and id <= ?", min_id, max_id).find_each do |post| - key = "preview/" + File.basename(post.preview_file_path) - begin - client.restore_object( - bucket: bucket, - key: key, - restore_request: { - days: 1, - glacier_job_parameters: { - tier: "Bulk" - } - } - ) - puts "restored #{post.id}" - rescue Aws::S3::Errors::InvalidObjectState - puts "already glaciered #{post.id}" - rescue Aws::S3::Errors::NoSuchKey - puts "missing #{post.id}" - rescue Aws::S3::Errors::RestoreAlreadyInProgress - puts "already restoring #{post.id}" - end - end - end - - desc "Redownload an image from Pixiv" - task :download_pixiv => :environment do - post_id = ENV["id"] - - if post_id !~ /\d+/ - raise "Usage: regen_img.rb POST_ID" - end - - post = Post.find(post_id) - post.source =~ /(\d{5,})/ - if illust_id = $1 - response = PixivApiClient.new.work(illust_id) - upload = Upload.new - upload.source = response.pages.first - upload.file_ext = post.file_ext - upload.image_width = post.image_width - upload.image_height = post.image_height - upload.md5 = post.md5 - upload.download_from_source(post.file_path) - post.distribute_files - end - end - desc "Regenerates all images for a post id" task :regen => :environment do post_id = ENV["id"]