From 02edb52569ab85a980bb0106e3d3444c3bca58ce Mon Sep 17 00:00:00 2001 From: evazion Date: Thu, 10 Feb 2022 14:03:13 -0600 Subject: [PATCH] uploads: enable multi-file uploads when uploading from source. Make the upload page automatically detect when a source URL has multiple images and let the user choose which images to post. For example, when uploading a Twitter or Pixiv post with more than one image, we direct the user to a page showing a thumbnail for each image and letting them choose which ones to post. This is similar to the batch upload page, except we actually download each image in the background, instead of just hotlinking or proxying the thumbnails through our servers. This avoids various problems with proxying and makes new features possible, like showing which images in the batch have already been posted. --- .../src/javascripts/file_upload_component.js | 4 +-- app/jobs/process_upload_media_asset_job.rb | 9 +++++++ app/models/media_asset.rb | 4 ++- app/models/upload.rb | 9 +++---- app/models/upload_media_asset.rb | 27 +++++++++++++++++++ 5 files changed, 45 insertions(+), 8 deletions(-) create mode 100644 app/jobs/process_upload_media_asset_job.rb diff --git a/app/javascript/src/javascripts/file_upload_component.js b/app/javascript/src/javascripts/file_upload_component.js index ab1fa0b36..ee4545e7c 100644 --- a/app/javascript/src/javascripts/file_upload_component.js +++ b/app/javascript/src/javascripts/file_upload_component.js @@ -96,12 +96,12 @@ export default class FileUploadComponent { this.$component.find("progress").removeClass("hidden"); this.$component.find("input").attr("disabled", "disabled"); - while (upload.status === "pending" || upload.status === "processing") { + while (upload.media_asset_count <= 1 && upload.status !== "completed" && upload.status !== "error") { await Utility.delay(500); upload = await $.get(`/uploads/${upload.id}.json`); } - if (upload.status === "completed") { + if (upload.media_asset_count > 0) { let params = new URLSearchParams(window.location.search); let isBookmarklet = params.has("url"); params.delete("url"); diff --git a/app/jobs/process_upload_media_asset_job.rb b/app/jobs/process_upload_media_asset_job.rb new file mode 100644 index 000000000..db5f2f1ed --- /dev/null +++ b/app/jobs/process_upload_media_asset_job.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class ProcessUploadMediaAssetJob < ApplicationJob + queue_with_priority -1 + + def perform(upload_media_asset) + upload_media_asset.process_upload! + end +end diff --git a/app/models/media_asset.rb b/app/models/media_asset.rb index 441517a7d..e9dad1980 100644 --- a/app/models/media_asset.rb +++ b/app/models/media_asset.rb @@ -205,10 +205,11 @@ class MediaAsset < ApplicationRecord # # This can't be called inside a transaction because the transaction will # fail if there's a RecordNotUnique error when the asset already exists. - def upload!(media_file) + def upload!(media_file, &block) raise Error, "File is corrupt" if media_file.is_corrupt? media_asset = create!(file: media_file, status: :processing) + yield media_asset if block_given? media_asset.distribute_files!(media_file) media_asset.update!(status: :active) media_asset @@ -218,6 +219,7 @@ class MediaAsset < ApplicationRecord raise if e.is_a?(ActiveRecord::RecordInvalid) && !e.record.errors.of_kind?(:md5, :taken) media_asset = find_by!(md5: media_file.md5, status: [:processing, :active]) + yield media_asset if block_given? # XXX If the asset is still being processed by another thread, wait up # to 30 seconds for it to finish. diff --git a/app/models/upload.rb b/app/models/upload.rb index a431c818c..e574bd392 100644 --- a/app/models/upload.rb +++ b/app/models/upload.rb @@ -95,14 +95,13 @@ class Upload < ApplicationRecord update!(upload_media_assets: [upload_media_asset], status: "completed", media_asset_count: 1) elsif source.present? strategy = Sources::Strategies.find(source, referer_url) - image_url = strategy.image_url page_url = strategy.page_url - media_file = strategy.download_file!(strategy.image_url) - media_asset = MediaAsset.upload!(media_file) - upload_media_asset = UploadMediaAsset.new(media_asset: media_asset, source_url: image_url, page_url: page_url, status: "active") + upload_media_assets = strategy.image_urls.map do |image_url| + UploadMediaAsset.new(source_url: image_url, page_url: page_url, media_asset: nil) + end - update!(upload_media_assets: [upload_media_asset], status: "completed", media_asset_count: 1) + update!(upload_media_assets: upload_media_assets, media_asset_count: upload_media_assets.size) else raise "No file or source given" # Should never happen end diff --git a/app/models/upload_media_asset.rb b/app/models/upload_media_asset.rb index 1b43f3999..911b65725 100644 --- a/app/models/upload_media_asset.rb +++ b/app/models/upload_media_asset.rb @@ -6,6 +6,9 @@ class UploadMediaAsset < ApplicationRecord belongs_to :upload belongs_to :media_asset, optional: true + after_create :async_process_upload! + after_save :update_upload_status, if: :saved_change_to_status? + enum status: { pending: 0, processing: 100, @@ -35,5 +38,29 @@ class UploadMediaAsset < ApplicationRecord Sources::Strategies.find(source_url, page_url) end + def async_process_upload! + ProcessUploadMediaAssetJob.perform_later(self) + end + + def process_upload! + update!(status: :processing) + + strategy = Sources::Strategies.find(source_url) + media_file = strategy.download_file!(source_url) + MediaAsset.upload!(media_file) do |media_asset| + update!(media_asset: media_asset) + end + + update!(status: :active) + rescue Exception => e + update!(status: :failed, error: e.message) + end + + def update_upload_status + upload.with_lock do + upload.update!(status: "completed") if upload.upload_media_assets.all?(&:finished?) + end + end + memoize :source_strategy end