From d7d34274887d5b21f3644d9f6e696bb9d4436e15 Mon Sep 17 00:00:00 2001 From: evazion Date: Fri, 2 Dec 2022 17:57:52 -0600 Subject: [PATCH] Fix #5363: Inconsistent order of files from zip uploads. Upload files in natural order rather than archive order when uploading archive files. Before files were listed in the same order they appeared in the zip file. This could be in non-alphabetical order, or even with files from different directories interleaved between each other. Now files are uploaded in natural order, which is alphabetical order but with numbers sorted properly, so that `file-9.jpg` appears before `file-10.jpg`. --- app/logical/danbooru.rb | 32 +++++++++++++++++++++ app/models/upload.rb | 2 +- config/initializers/core_extensions.rb | 2 ++ test/files/archive/out-of-order.zip | Bin 0 -> 3318 bytes test/functional/uploads_controller_test.rb | 13 +++++++++ 5 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 app/logical/danbooru.rb create mode 100644 test/files/archive/out-of-order.zip diff --git a/app/logical/danbooru.rb b/app/logical/danbooru.rb new file mode 100644 index 000000000..f312afdcd --- /dev/null +++ b/app/logical/danbooru.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +# The Danbooru module contains miscellaneous global helper functions. +module Danbooru + module EnumerableMethods + extend self + + # Sort a list of strings in natural order, e.g. with "file-2.txt" before "file-10.txt". + # + # @see https://en.wikipedia.org/wiki/Natural_sort_order + # @see https://stackoverflow.com/a/15170063 + # + # @param list [Enumerable] The list to sort. + # @return [Array] The sorted list. + def natural_sort(list) + natural_sort_by(list, &:to_s) + end + + # Sort a list of objects in natural order. The block should return a sort key, which is compared in natural order. + # + # @param list [Enumerable] The list to sort. + # @return [Array] The sorted list. + def natural_sort_by(list, &block) + list.sort_by do |element| + # "file-2022-10-01.txt" => ["file-", 2022, "-", 10, "-", 1, ".txt"] + yield(element).to_s.split(/(\d+)/).map { |str| str.match?(/\A\d+\z/) ? str.to_i : str } + end + end + end + + extend EnumerableMethods +end diff --git a/app/models/upload.rb b/app/models/upload.rb index f835ebdbf..d951bfcd3 100644 --- a/app/models/upload.rb +++ b/app/models/upload.rb @@ -206,7 +206,7 @@ class Upload < ApplicationRecord tmpdir, filenames = file.extract! tmpdirs << tmpdir - filenames.map do |filename| + Danbooru.natural_sort(filenames).map do |filename| name = "file://#{original_filename}/#{Pathname.new(filename).relative_path_from(tmpdir)}" # "file://foo.zip/foo/1.jpg" UploadMediaAsset.new(upload: self, file: filename, source_url: name) end diff --git a/config/initializers/core_extensions.rb b/config/initializers/core_extensions.rb index c98797b73..1dbf8c990 100644 --- a/config/initializers/core_extensions.rb +++ b/config/initializers/core_extensions.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require "danbooru" + module Danbooru module Extensions module String diff --git a/test/files/archive/out-of-order.zip b/test/files/archive/out-of-order.zip new file mode 100644 index 0000000000000000000000000000000000000000..b5d1626735471c05365063328951ad6ad74ef189 GIT binary patch literal 3318 zcmWIWW@Zs#U|`^2$Y7c5v;F2=rm2h!3^N!(A~Fn?`i2I2>6vMvA)E}%Q?zbG?GOr0 zD6QaTU}Sm0%)kI9iocg^-ry6G5RhP@bn-yL^mF22vlgX&fAIf5E4P@1oc{0s@1Fe^ z)jrehXOQvhgoy8-`pVn&%*|WYe7f_R+y7jwsDOZgxTJmk@5<|M)~%?O|C@fttG?ju zfddWQjSv6-|IZ`g@KK#xnYme|nXiL+)8xjl^-WRnUeEUQguYgppZn{A;wi1NJv+XA zIJft8?)%T%&+SoqY*Kspy8E;oxu!gJ7L#f5zE6#&1>C%IRdr+3)-`4`HXh7;%;(#7 zY~@;E?P>cy?V7vW;7Q=Bs-BrHw#lOPe#o zT;5?R{;l(jS=Z`Vg;OVw9Xor>(zv`-U~|WhRo{ANZ~Nx)W<%n0f$0eklJ0MsIk)?v z$34H;kHYF-kL2IKvU;CZ;oozzd>o{Y+F>{-{YS)=_uEyp0nrwWw5PzF+2T~N@7IF+e_6oh34z$b-J=9-#MhSSmdHy zdVf!!T<-lDiy~(ob4pK-6w!a2kg|b|0e7@=GRQC(8jQy4aEn(C25@#Y9L=u7D!YQB z)zCoSax_|pRkX5$bL(iV4$r!Ev=APe@yf^~$Be5nEWrQ+OBz8eMAMiR(lkbE76*9a zHjR@3*|h7749G1q(v9OlF>XCG1E}2&O@ox0*2W5U9JHh&-86O-x5cu9+(x!xsNrVJ b33i$x0iUw6fuf6zfsNrRBLf3BH;4xS!$G9i literal 0 HcmV?d00001 diff --git a/test/functional/uploads_controller_test.rb b/test/functional/uploads_controller_test.rb index 7c27de676..85b27eb8e 100644 --- a/test/functional/uploads_controller_test.rb +++ b/test/functional/uploads_controller_test.rb @@ -445,6 +445,19 @@ class UploadsControllerTest < ActionDispatch::IntegrationTest assert_equal(5, upload.upload_media_assets.size) assert_equal("file://ugoira.zip/000000.jpg", upload.upload_media_assets[0].source_url) end + + should "upload the files in filename order" do + upload = assert_successful_upload("test/files/archive/out-of-order.zip", user: @user) + + assert_equal(6, upload.media_asset_count) + assert_equal(6, upload.upload_media_assets.size) + assert_equal("file://out-of-order.zip/9/9.gif", upload.upload_media_assets[0].source_url) + assert_equal("file://out-of-order.zip/9/10.gif", upload.upload_media_assets[1].source_url) + assert_equal("file://out-of-order.zip/9/11.gif", upload.upload_media_assets[2].source_url) + assert_equal("file://out-of-order.zip/10/9.gif", upload.upload_media_assets[3].source_url) + assert_equal("file://out-of-order.zip/10/10.gif", upload.upload_media_assets[4].source_url) + assert_equal("file://out-of-order.zip/10/11.gif", upload.upload_media_assets[5].source_url) + end end context "uploading a .rar file from your computer" do