From e7744cb6e3d19e69e6449d6e86afc4752961e45f Mon Sep 17 00:00:00 2001
From: evazion <noizave@gmail.com>
Date: Fri, 4 Feb 2022 15:39:00 -0600
Subject: [PATCH] uploads: generate thumbnails in parallel.

Make uploads faster by generating and saving thumbnails in parallel.

We generate each thumbnail in parallel, then send each thumbnail to the
backend image servers in parallel.

Most images have 5 variants: 'preview' (150x150), 180x180, 360x360,
720x720, and 'sample' (850px width). Plus the original file, that's 6
files we have to save. In production we have 2 image servers, so we have
to save each file twice, to 2 remote servers. Doing all this in parallel
should make uploads significantly faster.
---
 app/logical/storage_manager.rb        | 2 +-
 app/logical/storage_manager/local.rb  | 7 +++----
 app/logical/storage_manager/mirror.rb | 8 ++++----
 app/logical/storage_manager/null.rb   | 2 +-
 app/models/media_asset.rb             | 2 +-
 5 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/app/logical/storage_manager.rb b/app/logical/storage_manager.rb
index 53b314fd0..dac42c41c 100644
--- a/app/logical/storage_manager.rb
+++ b/app/logical/storage_manager.rb
@@ -30,7 +30,7 @@ class StorageManager
   # written, or an error is raised and the original file is left unchanged. The
   # file should never be in a partially written state.
   #
-  # @param io [IO] a file (or a readable IO object)
+  # @param src_file [File] the file to store
   # @param path [String] the remote path where the file should be stored
   def store(io, path)
     raise NotImplementedError, "store not implemented"
diff --git a/app/logical/storage_manager/local.rb b/app/logical/storage_manager/local.rb
index 0bea843b9..2e908c433 100644
--- a/app/logical/storage_manager/local.rb
+++ b/app/logical/storage_manager/local.rb
@@ -12,13 +12,12 @@ class StorageManager::Local < StorageManager
     super(**options)
   end
 
-  def store(io, dest_path)
+  def store(src_file, dest_path)
     temp_path = full_path(dest_path) + "-" + SecureRandom.uuid + ".tmp"
 
     FileUtils.mkdir_p(File.dirname(temp_path))
-    io.rewind
-    bytes_copied = IO.copy_stream(io, temp_path)
-    raise Error, "store failed: #{bytes_copied}/#{io.size} bytes copied" if bytes_copied != io.size
+    bytes_copied = IO.copy_stream(src_file.path, temp_path)
+    raise Error, "store failed: #{bytes_copied}/#{src_file.size} bytes copied" if bytes_copied != src_file.size
 
     FileUtils.chmod(DEFAULT_PERMISSIONS, temp_path)
     File.rename(temp_path, full_path(dest_path))
diff --git a/app/logical/storage_manager/mirror.rb b/app/logical/storage_manager/mirror.rb
index 7554a24a3..0cbb5da18 100644
--- a/app/logical/storage_manager/mirror.rb
+++ b/app/logical/storage_manager/mirror.rb
@@ -9,14 +9,14 @@ class StorageManager::Mirror < StorageManager
     super(**options)
   end
 
-  def store(io, dest_path)
-    services.each do |service|
-      service.store(io, dest_path)
+  def store(src_file, dest_path)
+    Parallel.each(services, in_threads: Etc.nprocessors) do |service|
+      service.store(src_file, dest_path)
     end
   end
 
   def delete(path)
-    services.each do |service|
+    Parallel.each(services, in_threads: Etc.nprocessors) do |service|
       service.delete(path)
     end
   end
diff --git a/app/logical/storage_manager/null.rb b/app/logical/storage_manager/null.rb
index cb887c977..360fde1ab 100644
--- a/app/logical/storage_manager/null.rb
+++ b/app/logical/storage_manager/null.rb
@@ -7,7 +7,7 @@ class StorageManager::Null < StorageManager
     super(base_url: nil)
   end
 
-  def store(io, path)
+  def store(src_file, path)
     # no-op
   end
 
diff --git a/app/models/media_asset.rb b/app/models/media_asset.rb
index b0f602a35..441517a7d 100644
--- a/app/models/media_asset.rb
+++ b/app/models/media_asset.rb
@@ -271,7 +271,7 @@ class MediaAsset < ApplicationRecord
     end
 
     def distribute_files!(media_file)
-      variants.each do |variant|
+      Parallel.each(variants, in_threads: Etc.nprocessors) do |variant|
         variant.store_file!(media_file)
       end
     end