f235b72b3f
* Export daily public database dumps to BigQuery and Google Cloud Storage. * Only data visible to anonymous users is exported. Some tables have null or missing fields because of this. * The bans table is excluded because some bans have an expires_at timestamp set beyond year 9999, which BigQuery doesn't support. * The favorites table is excluded because it's too slow to dump (it doesn't have an id index, which is needed by find_each). * Version tables are excluded because dumping them every day is inefficient, streaming insertions should be used instead. Links: * https://console.cloud.google.com/bigquery?project=danbooru1 * https://console.cloud.google.com/storage/browser/danbooru_public * https://storage.googleapis.com/danbooru_public/data/posts.json
49 lines
1.3 KiB
Ruby
49 lines
1.3 KiB
Ruby
module DanbooruMaintenance
|
|
module_function
|
|
|
|
def hourly
|
|
safely { Upload.prune! }
|
|
safely { PostPruner.prune! }
|
|
safely { PostAppealForumUpdater.update_forum! }
|
|
safely { RateLimit.prune! }
|
|
safely { regenerate_post_counts! }
|
|
end
|
|
|
|
def daily
|
|
safely { Delayed::Job.where('created_at < ?', 45.days.ago).delete_all }
|
|
safely { PostDisapproval.prune! }
|
|
safely { BulkUpdateRequestPruner.warn_old }
|
|
safely { BulkUpdateRequestPruner.reject_expired }
|
|
safely { Ban.prune! }
|
|
safely { BigqueryExportService.async_export_all! }
|
|
safely { ActiveRecord::Base.connection.execute("vacuum analyze") unless Rails.env.test? }
|
|
end
|
|
|
|
def weekly
|
|
safely { TagRelationshipRetirementService.find_and_retire! }
|
|
safely { ApproverPruner.dmail_inactive_approvers! }
|
|
end
|
|
|
|
def monthly
|
|
safely { ApproverPruner.prune! }
|
|
end
|
|
|
|
def regenerate_post_counts!
|
|
updated_tags = Tag.regenerate_post_counts!
|
|
updated_tags.each do |tag|
|
|
DanbooruLogger.info("Updated tag count", tag.attributes)
|
|
end
|
|
end
|
|
|
|
def safely(&block)
|
|
ActiveRecord::Base.connection.execute("set statement_timeout = 0")
|
|
|
|
CurrentUser.scoped(User.system, "127.0.0.1") do
|
|
yield
|
|
end
|
|
rescue StandardError => exception
|
|
DanbooruLogger.log(exception)
|
|
raise exception if Rails.env.test?
|
|
end
|
|
end
|