From c23fee846fe4347c424ba8d3dfacfc720f4d5f98 Mon Sep 17 00:00:00 2001 From: nonamethanks Date: Wed, 17 Jun 2020 07:24:34 +0200 Subject: [PATCH 01/59] Add more tags to post replacements filtering and modqueue highlighting. --- config/danbooru_default_config.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/config/danbooru_default_config.rb b/config/danbooru_default_config.rb index c3f22f050..b82014e0a 100644 --- a/config/danbooru_default_config.rb +++ b/config/danbooru_default_config.rb @@ -317,13 +317,15 @@ module Danbooru # A list of tags that should be removed when a post is replaced. Regexes allowed. def post_replacement_tag_removals %w[replaceme .*_sample resized upscaled downscaled md5_mismatch - jpeg_artifacts corrupted_image source_request non-web_source] + jpeg_artifacts corrupted_image missing_image missing_sample missing_thumbnail + resolution_mismatch source_larger source_smaller source_request non-web_source] end # Posts with these tags will be highlighted in the modqueue. def modqueue_warning_tags %w[hard_translated self_upload nude_filter third-party_edit screencap - duplicate image_sample md5_mismatch resized upscaled downscaled] + duplicate image_sample md5_mismatch resized upscaled downscaled + resolution_mismatch source_larger source_smaller] end def stripe_secret_key From 91beb288b68db2b846563c94d0f2cbb2f354e60f Mon Sep 17 00:00:00 2001 From: evazion Date: Wed, 17 Jun 2020 00:57:55 -0500 Subject: [PATCH 02/59] reportbooru: cache post views endpoint. Cache most viewed posts endpoint for /explore/posts/viewed page. --- app/logical/reportbooru_service.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/logical/reportbooru_service.rb b/app/logical/reportbooru_service.rb index 5da79d724..5ad3cd7f6 100644 --- a/app/logical/reportbooru_service.rb +++ b/app/logical/reportbooru_service.rb @@ -31,7 +31,7 @@ class ReportbooruService def post_view_rankings(date = Date.today, expires_in: 1.minutes) return [] unless enabled? - response = http.get("#{reportbooru_server}/post_views/rank?date=#{date}") + response = http.cache(expires_in).get("#{reportbooru_server}/post_views/rank?date=#{date}") return [] if response.status != 200 JSON.parse(response.to_s.force_encoding("utf-8")) end From b8b5c8d6a0b1eb5eec591615c0062a6e8c970b6d Mon Sep 17 00:00:00 2001 From: evazion Date: Wed, 17 Jun 2020 02:28:16 -0500 Subject: [PATCH 03/59] iqdb: fix error with file uploads. Fix "cannot determine size of body" errors on upload page. Caused by exception during IQDB lookup. We were posting the form data wrong, we need to wrap the file with HTTP::FormData::File and pass it through the `form` parameter. --- app/logical/iqdb_proxy.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/app/logical/iqdb_proxy.rb b/app/logical/iqdb_proxy.rb index e1b1f195f..a61dd3c2c 100644 --- a/app/logical/iqdb_proxy.rb +++ b/app/logical/iqdb_proxy.rb @@ -50,9 +50,12 @@ class IqdbProxy file.try(:close) end - def query(params) + def query(file: nil, url: nil, limit: 20) raise NotImplementedError, "the IQDBs service isn't configured" unless enabled? - response = http.post("#{iqdbs_server}/similar", body: params) + + file = HTTP::FormData::File.new(file) if file + form = { file: file, url: url, limit: limit }.compact + response = http.post("#{iqdbs_server}/similar", form: form) raise Error, "IQDB error: #{response.status}" if response.status != 200 raise Error, "IQDB error: #{response.parse["error"]}" if response.parse.is_a?(Hash) From d5a7eef53d9d521b9dd75e6f927a658ca5c61430 Mon Sep 17 00:00:00 2001 From: evazion Date: Wed, 17 Jun 2020 04:13:16 -0500 Subject: [PATCH 04/59] uploads: fix remote file size not appearing. --- app/logical/upload_service/controller_helper.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/logical/upload_service/controller_helper.rb b/app/logical/upload_service/controller_helper.rb index f074c8069..7342168fd 100644 --- a/app/logical/upload_service/controller_helper.rb +++ b/app/logical/upload_service/controller_helper.rb @@ -13,7 +13,7 @@ class UploadService rescue Exception end - [upload, remote_size] + return [upload, remote_size] end if file From fd6ba473a541996ea570ab5faa29914dc090d9fb Mon Sep 17 00:00:00 2001 From: evazion Date: Wed, 17 Jun 2020 12:30:37 -0500 Subject: [PATCH 05/59] tests: possible fix for images getting nuked by tests. --- test/test_helper.rb | 5 ++-- test/unit/storage_manager_test.rb | 44 ++++++++++++------------------- 2 files changed, 20 insertions(+), 29 deletions(-) diff --git a/test/test_helper.rb b/test/test_helper.rb index 700bf1e6d..7832ec36c 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -45,13 +45,14 @@ class ActiveSupport::TestCase Socket.stubs(:gethostname).returns("www.example.com") WebMock.allow_net_connect! - storage_manager = StorageManager::Local.new(base_dir: Dir.mktmpdir("uploads-test-storage-")) + @temp_dir = Dir.mktmpdir("danbooru-temp-") + storage_manager = StorageManager::Local.new(base_dir: @temp_dir) Danbooru.config.stubs(:storage_manager).returns(storage_manager) Danbooru.config.stubs(:backup_storage_manager).returns(StorageManager::Null.new) end teardown do - FileUtils.rm_rf(Danbooru.config.storage_manager.base_dir) + FileUtils.rm_rf(@temp_dir) Cache.clear end diff --git a/test/unit/storage_manager_test.rb b/test/unit/storage_manager_test.rb index 9759a2918..9d39b03e5 100644 --- a/test/unit/storage_manager_test.rb +++ b/test/unit/storage_manager_test.rb @@ -1,8 +1,6 @@ require 'test_helper' class StorageManagerTest < ActiveSupport::TestCase - BASE_DIR = "#{Rails.root}/tmp/test-storage" - setup do CurrentUser.ip_addr = "127.0.0.1" end @@ -45,25 +43,21 @@ class StorageManagerTest < ActiveSupport::TestCase context "StorageManager::Local" do setup do - @storage_manager = StorageManager::Local.new(base_dir: BASE_DIR, base_url: "/data") - end - - teardown do - FileUtils.rm_rf(BASE_DIR) + @storage_manager = StorageManager::Local.new(base_dir: @temp_dir, base_url: "/data") end context "#store method" do should "store the file" do - @storage_manager.store(StringIO.new("data"), "#{BASE_DIR}/test.txt") + @storage_manager.store(StringIO.new("data"), "#{@temp_dir}/test.txt") - assert("data", File.read("#{BASE_DIR}/test.txt")) + assert("data", File.read("#{@temp_dir}/test.txt")) end should "overwrite the file if it already exists" do - @storage_manager.store(StringIO.new("foo"), "#{BASE_DIR}/test.txt") - @storage_manager.store(StringIO.new("bar"), "#{BASE_DIR}/test.txt") + @storage_manager.store(StringIO.new("foo"), "#{@temp_dir}/test.txt") + @storage_manager.store(StringIO.new("bar"), "#{@temp_dir}/test.txt") - assert("bar", File.read("#{BASE_DIR}/test.txt")) + assert("bar", File.read("#{@temp_dir}/test.txt")) end end @@ -72,7 +66,7 @@ class StorageManagerTest < ActiveSupport::TestCase @storage_manager.store(StringIO.new("data"), "test.txt") @storage_manager.delete("test.txt") - assert_not(File.exist?("#{BASE_DIR}/test.txt")) + assert_not(File.exist?("#{@temp_dir}/test.txt")) end should "not fail if the file doesn't exist" do @@ -88,9 +82,9 @@ class StorageManagerTest < ActiveSupport::TestCase @storage_manager.store_file(StringIO.new("data"), @post, :large) @storage_manager.store_file(StringIO.new("data"), @post, :original) - @file_path = "#{BASE_DIR}/preview/#{@post.md5}.jpg" - @large_file_path = "#{BASE_DIR}/sample/sample-#{@post.md5}.jpg" - @preview_file_path = "#{BASE_DIR}/#{@post.md5}.#{@post.file_ext}" + @file_path = "#{@temp_dir}/preview/#{@post.md5}.jpg" + @large_file_path = "#{@temp_dir}/sample/sample-#{@post.md5}.jpg" + @preview_file_path = "#{@temp_dir}/#{@post.md5}.#{@post.file_ext}" end should "store the files at the correct path" do @@ -134,12 +128,12 @@ class StorageManagerTest < ActiveSupport::TestCase context "when the original_subdir option is used" do should "store original files at the correct path" do @post = FactoryBot.create(:post, file_ext: "png") - @storage_manager = StorageManager::Local.new(base_dir: BASE_DIR, base_url: "/data", original_subdir: "original/") + @storage_manager = StorageManager::Local.new(base_dir: @temp_dir, base_url: "/data", original_subdir: "original/") - assert_equal("#{BASE_DIR}/original/#{@post.md5}.png", @storage_manager.file_path(@post, @post.file_ext, :original)) + assert_equal("#{@temp_dir}/original/#{@post.md5}.png", @storage_manager.file_path(@post, @post.file_ext, :original)) @storage_manager.store_file(StringIO.new("data"), @post, :original) - assert_equal(true, File.exist?("#{BASE_DIR}/original/#{@post.md5}.png")) + assert_equal(true, File.exist?("#{@temp_dir}/original/#{@post.md5}.png")) end end end @@ -151,24 +145,20 @@ class StorageManagerTest < ActiveSupport::TestCase @storage_manager = StorageManager::Hybrid.new do |id, md5, file_ext, type| if id.odd? - StorageManager::Local.new(base_dir: "#{BASE_DIR}/i1", base_url: "/i1") + StorageManager::Local.new(base_dir: "#{@temp_dir}/i1", base_url: "/i1") else - StorageManager::Local.new(base_dir: "#{BASE_DIR}/i2", base_url: "/i2") + StorageManager::Local.new(base_dir: "#{@temp_dir}/i2", base_url: "/i2") end end end - teardown do - FileUtils.rm_rf(BASE_DIR) - end - context "#store_file method" do should "store odd-numbered posts under /i1 and even-numbered posts under /i2" do @storage_manager.store_file(StringIO.new("post1"), @post1, :original) @storage_manager.store_file(StringIO.new("post2"), @post2, :original) - assert(File.exist?("#{BASE_DIR}/i1/#{@post1.md5}.png")) - assert(File.exist?("#{BASE_DIR}/i2/#{@post2.md5}.png")) + assert(File.exist?("#{@temp_dir}/i1/#{@post1.md5}.png")) + assert(File.exist?("#{@temp_dir}/i2/#{@post2.md5}.png")) end end From f790a1aeedae7965e0d75bc9340c2f4c997c4eab Mon Sep 17 00:00:00 2001 From: evazion Date: Thu, 18 Jun 2020 00:56:42 -0500 Subject: [PATCH 06/59] http: increase default timeout to 10 seconds. Three seconds was little tight and might have caused timeouts in source strategies sometimes. --- app/logical/danbooru/http.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/logical/danbooru/http.rb b/app/logical/danbooru/http.rb index 343d8d886..666a67f2c 100644 --- a/app/logical/danbooru/http.rb +++ b/app/logical/danbooru/http.rb @@ -1,6 +1,6 @@ module Danbooru class Http - DEFAULT_TIMEOUT = 3 + DEFAULT_TIMEOUT = 10 MAX_REDIRECTS = 5 attr_writer :cache, :http From 459f67c431699e68bc7ba5aa7274be39098a5658 Mon Sep 17 00:00:00 2001 From: evazion Date: Thu, 18 Jun 2020 00:57:51 -0500 Subject: [PATCH 07/59] iqdb: fix 599 timeout errors. Increase timeout to 30 seconds when uploading files to IQDB. Previously we used the default timeout of 3 seconds, which could cause 599 timeout errors sometimes if the upload took too long. --- app/logical/iqdb_proxy.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/logical/iqdb_proxy.rb b/app/logical/iqdb_proxy.rb index a61dd3c2c..67e27bd90 100644 --- a/app/logical/iqdb_proxy.rb +++ b/app/logical/iqdb_proxy.rb @@ -55,7 +55,7 @@ class IqdbProxy file = HTTP::FormData::File.new(file) if file form = { file: file, url: url, limit: limit }.compact - response = http.post("#{iqdbs_server}/similar", form: form) + response = http.timeout(30).post("#{iqdbs_server}/similar", form: form) raise Error, "IQDB error: #{response.status}" if response.status != 200 raise Error, "IQDB error: #{response.parse["error"]}" if response.parse.is_a?(Hash) From 8a06d3a7442646c118584a3177aab0c967a347cb Mon Sep 17 00:00:00 2001 From: nonamethanks Date: Thu, 18 Jun 2020 08:07:45 +0200 Subject: [PATCH 08/59] Zerochan: Normalize png links --- app/logical/sources/strategies/null.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/logical/sources/strategies/null.rb b/app/logical/sources/strategies/null.rb index 4ce04c30c..4dd02ebe6 100644 --- a/app/logical/sources/strategies/null.rb +++ b/app/logical/sources/strategies/null.rb @@ -47,7 +47,7 @@ module Sources when %r{\Ahttps?://c(?:s|han|[1-4])\.sankakucomplex\.com/data(?:/sample)?/(?:[a-f0-9]{2}/){2}(?:sample-|preview)?([a-f0-9]{32})}i "https://chan.sankakucomplex.com/en/post/show?md5=#{$1}" - when %r{\Ahttps?://(?:www|s(?:tatic|[1-4]))\.zerochan\.net/.+(?:\.|\/)(\d+)(?:\.(?:jpe?g?))?\z}i + when %r{\Ahttps?://(?:www|s(?:tatic|[1-4]))\.zerochan\.net/.+(?:\.|\/)(\d+)(?:\.(?:jpe?g?|png))?\z}i "https://www.zerochan.net/#{$1}#full" when %r{\Ahttps?://static[1-6]?\.minitokyo\.net/(?:downloads|view)/(?:\d{2}/){2}(\d+)}i From 213766fac92eca61ed648467ef7ecd182d74f41f Mon Sep 17 00:00:00 2001 From: evazion Date: Thu, 18 Jun 2020 12:24:32 -0500 Subject: [PATCH 09/59] posts: fix "view original" not working on mobile. ref: https://github.com/danbooru/danbooru/commit/38f385d1ca93dba6d7de9f228f61c1f9a1eb4c27#r40000777 --- app/javascript/src/javascripts/posts.js.erb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/app/javascript/src/javascripts/posts.js.erb b/app/javascript/src/javascripts/posts.js.erb index cdf7b9aaf..9eef63e88 100644 --- a/app/javascript/src/javascripts/posts.js.erb +++ b/app/javascript/src/javascripts/posts.js.erb @@ -300,10 +300,10 @@ Post.initialize_favlist = function() { }); } -Post.view_original = function(e) { +Post.view_original = function(e = null) { if (Utility.test_max_width(660)) { // Do the default behavior (navigate to image) - return false; + return; } var $image = $("#image"); @@ -316,13 +316,13 @@ Post.view_original = function(e) { }); Note.Box.scale_all(); $("body").attr("data-post-current-image-size", "original"); - return false; + e?.preventDefault(); } -Post.view_large = function(e) { +Post.view_large = function(e = null) { if (Utility.test_max_width(660)) { // Do the default behavior (navigate to image) - return false; + return; } var $image = $("#image"); @@ -335,7 +335,7 @@ Post.view_large = function(e) { }); Note.Box.scale_all(); $("body").attr("data-post-current-image-size", "large"); - return false; + e?.preventDefault(); } Post.toggle_fit_window = function(e) { From 0a396c8b95b925e40c2bf87e0666f1613a0fd4fd Mon Sep 17 00:00:00 2001 From: nonamethanks Date: Fri, 19 Jun 2020 08:08:01 +0200 Subject: [PATCH 10/59] Revert "Pixiv: don't blacklist digital tools" This reverts commit e83d07ea7baa4cd03990d2d701a3cd95f1494500. It was worth a try, but unfortunately it seems that once someone sets tools in a Pixiv upload, they become defaults and are applied to all of their subsequent uploads, so we get some posts with two or three different digital tags. --- app/logical/pixiv_api_client.rb | 17 ++++++++++++++++- test/unit/sources/pixiv_test.rb | 6 +++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/app/logical/pixiv_api_client.rb b/app/logical/pixiv_api_client.rb index c742e250e..f1d55c981 100644 --- a/app/logical/pixiv_api_client.rb +++ b/app/logical/pixiv_api_client.rb @@ -8,6 +8,21 @@ class PixivApiClient CLIENT_SECRET = "HP3RmkgAmEGro0gn1x9ioawQE8WMfvLXDz3ZqxpK" CLIENT_HASH_SALT = "28c1fdd170a5204386cb1313c7077b34f83e4aaf4aa829ce78c231e05b0bae2c" + # Tools to not include in the tags list. We don't tag digital media, so + # including these results in bad translated tags suggestions. + TOOLS_BLACKLIST = %w[ + Photoshop Illustrator Fireworks Flash Painter PaintShopPro pixiv\ Sketch + CLIP\ STUDIO\ PAINT IllustStudio ComicStudio RETAS\ STUDIO SAI PhotoStudio + Pixia NekoPaint PictBear openCanvas ArtRage Expression Inkscape GIMP + CGillust COMICWORKS MS_Paint EDGE AzPainter AzPainter2 AzDrawing + PicturePublisher SketchBookPro Processing 4thPaint GraphicsGale mdiapp + Paintgraphic AfterEffects drawr CLIP\ PAINT\ Lab FireAlpaca Pixelmator + AzDrawing2 MediBang\ Paint Krita ibisPaint Procreate Live2D + Lightwave3D Shade Poser STRATA AnimationMaster XSI CARRARA CINEMA4D Maya + 3dsMax Blender ZBrush Metasequoia Sunny3D Bryce Vue Hexagon\ King SketchUp + VistaPro Sculptris Comi\ Po! modo DAZ\ Studio 3D-Coat + ] + class Error < StandardError; end class BadIDError < Error; end @@ -24,7 +39,7 @@ class PixivApiClient @artist_commentary_title = json["title"].to_s @artist_commentary_desc = json["caption"].to_s @tags = json["tags"].reject {|x| x =~ /^http:/} - @tags += json["tools"] + @tags += json["tools"] - TOOLS_BLACKLIST if json["metadata"] if json["metadata"]["zip_urls"] diff --git a/test/unit/sources/pixiv_test.rb b/test/unit/sources/pixiv_test.rb index cb7a3d883..f4778615e 100644 --- a/test/unit/sources/pixiv_test.rb +++ b/test/unit/sources/pixiv_test.rb @@ -249,6 +249,10 @@ module Sources assert_includes(@translated_tags, "foo") end + should "not translate tags for digital media" do + assert_equal(false, @tags.include?("Photoshop")) + end + should "normalize 10users入り tags" do assert_includes(@tags, "風景10users入り") assert_includes(@translated_tags, "scenery") @@ -280,7 +284,7 @@ module Sources should "not translate '1000users入り' to '1'" do FactoryBot.create(:tag, name: "1", post_count: 1) source = get_source("https://www.pixiv.net/member_illust.php?mode=medium&illust_id=60665428") - tags = %w[1000users入り Fate/GrandOrder アルジュナ(Fate) アルトリア・ペンドラゴン イシュタル(Fate) グランブルーファンタジー マシュ・キリエライト マーリン(Fate) 両儀式 手袋 CLIP\ STUDIO\ PAINT Photoshop] + tags = %w[1000users入り Fate/GrandOrder アルジュナ(Fate) アルトリア・ペンドラゴン イシュタル(Fate) グランブルーファンタジー マシュ・キリエライト マーリン(Fate) 両儀式 手袋] assert_equal(tags.sort, source.tags.map(&:first).sort) assert_equal(["fate/grand_order"], source.translated_tags.map(&:name)) From ae7fc7d1bc64f5ec3025778d6623ff7a99b47bae Mon Sep 17 00:00:00 2001 From: evazion Date: Thu, 18 Jun 2020 22:12:47 -0500 Subject: [PATCH 11/59] Update eslint to 7.0, add babel-eslint plugin. Add babel-eslint plugin so that eslint can recognize optional chaining syntax (a?.b). --- .eslintrc.yml | 6 +- package.json | 4 +- yarn.lock | 396 ++++++++++++++++++++++++++------------------------ 3 files changed, 213 insertions(+), 193 deletions(-) diff --git a/.eslintrc.yml b/.eslintrc.yml index 1bb08c0bc..df6581f1c 100644 --- a/.eslintrc.yml +++ b/.eslintrc.yml @@ -8,9 +8,13 @@ parserOptions: globals: $: false require: false +parser: babel-eslint +plugins: + - babel rules: # https://eslint.org/docs/rules/ array-callback-return: error + babel/no-unused-expressions: error block-scoped-var: error consistent-return: error default-case: error @@ -32,7 +36,7 @@ rules: no-sequences: error no-shadow: error no-shadow-restricted-names: error - no-unused-expressions: error + #no-unused-expressions: error no-unused-vars: - error - argsIgnorePattern: "^_" diff --git a/package.json b/package.json index 28fa41a81..e715e9198 100644 --- a/package.json +++ b/package.json @@ -27,8 +27,10 @@ "webpack-cli": "^3.3.0" }, "devDependencies": { - "eslint": "^6.0.0", + "babel-eslint": "^10.1.0", + "eslint": "^7.0.0", "eslint-loader": "^4.0.0", + "eslint-plugin-babel": "^5.3.0", "eslint-plugin-ignore-erb": "^0.1.1", "stylelint": "^13.0.0", "stylelint-config-standard": "^20.0.0", diff --git a/yarn.lock b/yarn.lock index 54b157113..f062816ea 100644 --- a/yarn.lock +++ b/yarn.lock @@ -252,7 +252,7 @@ chalk "^2.0.0" js-tokens "^4.0.0" -"@babel/parser@^7.10.1", "@babel/parser@^7.10.2": +"@babel/parser@^7.10.1", "@babel/parser@^7.10.2", "@babel/parser@^7.7.0": version "7.10.2" resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.10.2.tgz#871807f10442b92ff97e4783b9b54f6a0ca812d0" integrity sha512-PApSXlNMJyB4JiGVhCOlzKIif+TKFTvu0aQAhnTvfP/z3vVSN6ZypH5bfUNwFXXjRQtUEBNFd2PtmCmG2Py3qQ== @@ -775,7 +775,7 @@ "@babel/parser" "^7.10.1" "@babel/types" "^7.10.1" -"@babel/traverse@^7.10.1": +"@babel/traverse@^7.10.1", "@babel/traverse@^7.7.0": version "7.10.1" resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.10.1.tgz#bbcef3031e4152a6c0b50147f4958df54ca0dd27" integrity sha512-C/cTuXeKt85K+p08jN6vMDz8vSV0vZcI0wmQ36o6mjbuo++kPMdpOYw23W2XH04dbRt9/nMEfA4W3eR21CD+TQ== @@ -790,7 +790,7 @@ globals "^11.1.0" lodash "^4.17.13" -"@babel/types@^7.10.1", "@babel/types@^7.10.2", "@babel/types@^7.4.4": +"@babel/types@^7.10.1", "@babel/types@^7.10.2", "@babel/types@^7.4.4", "@babel/types@^7.7.0": version "7.10.2" resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.10.2.tgz#30283be31cad0dbf6fb00bd40641ca0ea675172d" integrity sha512-AD3AwWBSz0AWF0AkCN9VPiWrvldXq+/e3cHa4J89vo4ymjz1XwrBFFVZmkJTsQIPNk+ZVomPSXUJqq8yyjZsng== @@ -805,9 +805,9 @@ integrity sha512-5a6wqoJV/xEdbRNKVo6I4hO3VjyDq//8q2f9I6PBAvMesJHFauXDorcNCsr9RzvsZnaWi5NYCcfyqP1QeFHFbw== "@fortawesome/fontawesome-free@^5.11.2": - version "5.13.0" - resolved "https://registry.yarnpkg.com/@fortawesome/fontawesome-free/-/fontawesome-free-5.13.0.tgz#fcb113d1aca4b471b709e8c9c168674fbd6e06d9" - integrity sha512-xKOeQEl5O47GPZYIMToj6uuA2syyFlq9EMSl2ui0uytjY9xbe8XS0pexNWmxrdcCyNGyDmLyYw5FtKsalBUeOg== + version "5.13.1" + resolved "https://registry.yarnpkg.com/@fortawesome/fontawesome-free/-/fontawesome-free-5.13.1.tgz#c53b4066edae16cd1fd669f687baf031b45fb9d6" + integrity sha512-D819f34FLHeBN/4xvw0HR0u7U2G7RqjPSggXqf7LktsxWQ48VAfGwvMrhcVuaZV2fF069c/619RdgCCms0DHhw== "@nodelib/fs.scandir@2.1.3": version "2.1.3" @@ -1125,7 +1125,7 @@ acorn@^6.4.1: resolved "https://registry.yarnpkg.com/acorn/-/acorn-6.4.1.tgz#531e58ba3f51b9dacb9a6646ca4debf5b14ca474" integrity sha512-ZVA9k326Nwrj3Cj9jlh3wGFutC2ZornPNARZwsNYqQYgN0EsV2d53w5RN/co65Ohn4sUAUtb1rSUAOD6XN9idA== -acorn@^7.1.1: +acorn@^7.2.0: version "7.3.1" resolved "https://registry.yarnpkg.com/acorn/-/acorn-7.3.1.tgz#85010754db53c3fbaf3b9ea3e083aa5c5d147ffd" integrity sha512-tLc0wSnatxAQHVHUapaHdz72pi9KUyHjq5KyHjGg9Y8Ifdc79pTh2XvI6I1/chZbnM7QtNKzh66ooDogPZSleA== @@ -1422,6 +1422,18 @@ aws4@^1.8.0: resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.10.0.tgz#a17b3a8ea811060e74d47d306122400ad4497ae2" integrity sha512-3YDiu347mtVtjpyV3u5kVqQLP242c06zwDOgpeRnybmXlYYsLbtTrUBUm8i8srONt+FWobl5aibnU1030PeeuA== +babel-eslint@^10.1.0: + version "10.1.0" + resolved "https://registry.yarnpkg.com/babel-eslint/-/babel-eslint-10.1.0.tgz#6968e568a910b78fb3779cdd8b6ac2f479943232" + integrity sha512-ifWaTHQ0ce+448CYop8AdrQiBsGrnC+bMgfyKFdi6EsPLTAWG+QfyDeM6OH+FmWnKvEq5NnBMLvlBUPKQZoDSg== + dependencies: + "@babel/code-frame" "^7.0.0" + "@babel/parser" "^7.7.0" + "@babel/traverse" "^7.7.0" + "@babel/types" "^7.7.0" + eslint-visitor-keys "^1.0.0" + resolve "^1.12.0" + babel-loader@^8.1.0: version "8.1.0" resolved "https://registry.yarnpkg.com/babel-loader/-/babel-loader-8.1.0.tgz#c611d5112bd5209abe8b9fa84c3e4da25275f1c3" @@ -1883,9 +1895,9 @@ caniuse-api@^3.0.0: lodash.uniq "^4.5.0" caniuse-lite@^1.0.0, caniuse-lite@^1.0.30000981, caniuse-lite@^1.0.30001043, caniuse-lite@^1.0.30001061: - version "1.0.30001083" - resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001083.tgz#52410c20c6f029f604f0d45eca0439a82e712442" - integrity sha512-CnYJ27awX4h7yj5glfK7r1TOI13LBytpLzEgfj0s4mY75/F8pnQcYjL+oVpmS38FB59+vU0gscQ9D8tc+lIXvA== + version "1.0.30001084" + resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001084.tgz#00e471931eaefbeef54f46aa2203914d3c165669" + integrity sha512-ftdc5oGmhEbLUuMZ/Qp3mOpzfZLCxPYKcvGv6v2dJJ+8EdqcvZRbAGOiLmkM/PV1QGta/uwBs8/nCl6sokDW6w== case-sensitive-paths-webpack-plugin@^2.3.0: version "2.3.0" @@ -1911,15 +1923,6 @@ chalk@2.4.1: escape-string-regexp "^1.0.5" supports-color "^5.3.0" -chalk@2.4.2, chalk@^2.0, chalk@^2.0.0, chalk@^2.0.1, chalk@^2.1.0, chalk@^2.4.1, chalk@^2.4.2: - version "2.4.2" - resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424" - integrity sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ== - dependencies: - ansi-styles "^3.2.1" - escape-string-regexp "^1.0.5" - supports-color "^5.3.0" - chalk@^1.1.1: version "1.1.3" resolved "https://registry.yarnpkg.com/chalk/-/chalk-1.1.3.tgz#a8115c55e4a702fe4d150abd3872822a7e09fc98" @@ -1931,6 +1934,15 @@ chalk@^1.1.1: strip-ansi "^3.0.0" supports-color "^2.0.0" +chalk@^2.0, chalk@^2.0.0, chalk@^2.0.1, chalk@^2.4.1, chalk@^2.4.2: + version "2.4.2" + resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424" + integrity sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ== + dependencies: + ansi-styles "^3.2.1" + escape-string-regexp "^1.0.5" + supports-color "^5.3.0" + chalk@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/chalk/-/chalk-3.0.0.tgz#3f73c2bf526591f574cc492c51e2456349f844e4" @@ -1939,7 +1951,7 @@ chalk@^3.0.0: ansi-styles "^4.1.0" supports-color "^7.1.0" -chalk@^4.0.0: +chalk@^4.0.0, chalk@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.0.tgz#4e14870a618d9e2edd97dd8345fd9d9dc315646a" integrity sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A== @@ -2374,7 +2386,15 @@ create-hmac@^1.1.0, create-hmac@^1.1.4, create-hmac@^1.1.7: safe-buffer "^5.0.1" sha.js "^2.4.8" -cross-spawn@6.0.5, cross-spawn@^6.0.0, cross-spawn@^6.0.5: +cross-spawn@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-3.0.1.tgz#1256037ecb9f0c5f79e3d6ef135e30770184b982" + integrity sha1-ElYDfsufDF9549bvE14wdwGEuYI= + dependencies: + lru-cache "^4.0.1" + which "^1.2.9" + +cross-spawn@^6.0.0, cross-spawn@^6.0.5: version "6.0.5" resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-6.0.5.tgz#4a5ec7c64dfae22c3a14124dbacdee846d80cbc4" integrity sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ== @@ -2385,13 +2405,14 @@ cross-spawn@6.0.5, cross-spawn@^6.0.0, cross-spawn@^6.0.5: shebang-command "^1.2.0" which "^1.2.9" -cross-spawn@^3.0.0: - version "3.0.1" - resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-3.0.1.tgz#1256037ecb9f0c5f79e3d6ef135e30770184b982" - integrity sha1-ElYDfsufDF9549bvE14wdwGEuYI= +cross-spawn@^7.0.2: + version "7.0.3" + resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6" + integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w== dependencies: - lru-cache "^4.0.1" - which "^1.2.9" + path-key "^3.1.0" + shebang-command "^2.0.0" + which "^2.0.1" crypto-browserify@^3.11.0: version "3.12.0" @@ -2623,7 +2644,7 @@ debug@2.6.9, debug@^2.2.0, debug@^2.3.3: dependencies: ms "2.0.0" -debug@^3.0.0, debug@^3.1.1, debug@^3.2.5: +debug@^3.1.1, debug@^3.2.5: version "3.2.6" resolved "https://registry.yarnpkg.com/debug/-/debug-3.2.6.tgz#e83d17de16d8a7efb7717edbe5fb10135eee629b" integrity sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ== @@ -2720,7 +2741,7 @@ deep-equal@^1.0.1: object-keys "^1.1.1" regexp.prototype.flags "^1.2.0" -deep-is@~0.1.3: +deep-is@^0.1.3: version "0.1.3" resolved "https://registry.yarnpkg.com/deep-is/-/deep-is-0.1.3.tgz#b369d6fb5dbc13eecf524f91b070feedc357cf34" integrity sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ= @@ -2935,14 +2956,14 @@ ee-first@1.1.1: integrity sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0= electron-to-chromium@^1.3.413: - version "1.3.474" - resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.3.474.tgz#161af012e11f96795eade84bf03b8ddc039621b9" - integrity sha512-fPkSgT9IBKmVJz02XioNsIpg0WYmkPrvU1lUJblMMJALxyE7/32NGvbJQKKxpNokozPvqfqkuUqVClYsvetcLw== + version "1.3.477" + resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.3.477.tgz#7e6b931d0c1a2572101a6e9a835128c50fd49323" + integrity sha512-81p6DZ/XmHDD7O0ITJMa7ESo9bSCfE+v3Fny3MIYR0y77xmhoriu2ShNOLXcPS4eowF6dkxw6d2QqxTkS3DjBg== elliptic@^6.0.0, elliptic@^6.5.2: - version "6.5.2" - resolved "https://registry.yarnpkg.com/elliptic/-/elliptic-6.5.2.tgz#05c5678d7173c049d8ca433552224a495d0e3762" - integrity sha512-f4x70okzZbIQl/NSRLkI/+tteV/9WqL98zx+SQ69KbXxmVrmjwsNUPn/gYJJ0sHvEak24cZgHIPegRePAtA/xw== + version "6.5.3" + resolved "https://registry.yarnpkg.com/elliptic/-/elliptic-6.5.3.tgz#cb59eb2efdaf73a0bd78ccd7015a62ad6e0f93d6" + integrity sha512-IMqzv5wNQf+E6aHeIqATs0tOLeOTwj1QKbRcS3jBbYkl5oLAserA8yJTT7/VyHUYG91PRmPyeQDObKLPpeS4dw== dependencies: bn.js "^4.4.0" brorand "^1.0.1" @@ -2984,16 +3005,7 @@ end-of-stream@^1.0.0, end-of-stream@^1.1.0: dependencies: once "^1.4.0" -enhanced-resolve@4.1.0: - version "4.1.0" - resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-4.1.0.tgz#41c7e0bfdfe74ac1ffe1e57ad6a5c6c9f3742a7f" - integrity sha512-F/7vkyTtyc/llOIn8oWclcB25KdRaiPBpZYDgJHgh/UHtpgT2p2eldQgtQnLtUvfMKPKxbRaQM/hHkvLHt1Vng== - dependencies: - graceful-fs "^4.1.2" - memory-fs "^0.4.0" - tapable "^1.0.0" - -enhanced-resolve@^4.1.0: +enhanced-resolve@^4.1.0, enhanced-resolve@^4.1.1: version "4.2.0" resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-4.2.0.tgz#5d43bda4a0fd447cb0ebbe71bef8deff8805ad0d" integrity sha512-S7eiFb/erugyd1rLb6mQ3Vuq+EXHv5cpCkNqqIkYkBgN2QdFnyCZzFBleqwGEx4lgNGYij81BWnCrFNK7vxvjQ== @@ -3078,6 +3090,13 @@ eslint-loader@^4.0.0: object-hash "^2.0.3" schema-utils "^2.6.5" +eslint-plugin-babel@^5.3.0: + version "5.3.0" + resolved "https://registry.yarnpkg.com/eslint-plugin-babel/-/eslint-plugin-babel-5.3.0.tgz#2e7f251ccc249326da760c1a4c948a91c32d0023" + integrity sha512-HPuNzSPE75O+SnxHIafbW5QB45r2w78fxqwK3HmjqIUoPfPzVrq6rD+CINU3yzoDSzEhUkX07VUphbF73Lth/w== + dependencies: + eslint-rule-composer "^0.3.0" + eslint-plugin-ignore-erb@^0.1.1: version "0.1.1" resolved "https://registry.yarnpkg.com/eslint-plugin-ignore-erb/-/eslint-plugin-ignore-erb-0.1.1.tgz#951497d935c7d8a713a67f6bbbaa92e29bf0826d" @@ -3085,6 +3104,11 @@ eslint-plugin-ignore-erb@^0.1.1: dependencies: requireindex "~1.1.0" +eslint-rule-composer@^0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/eslint-rule-composer/-/eslint-rule-composer-0.3.0.tgz#79320c927b0c5c0d3d3d2b76c8b4a488f25bbaf9" + integrity sha512-bt+Sh8CtDmn2OajxvNO+BX7Wn4CIWMpTRm3MaiKPCQcnnlm0CS2mhui6QaoeQugs+3Kj2ESKEEGJUdVafwhiCg== + eslint-scope@^4.0.3: version "4.0.3" resolved "https://registry.yarnpkg.com/eslint-scope/-/eslint-scope-4.0.3.tgz#ca03833310f6889a3264781aa82e63eb9cfe7848" @@ -3093,7 +3117,7 @@ eslint-scope@^4.0.3: esrecurse "^4.1.0" estraverse "^4.1.1" -eslint-scope@^5.0.0: +eslint-scope@^5.1.0: version "5.1.0" resolved "https://registry.yarnpkg.com/eslint-scope/-/eslint-scope-5.1.0.tgz#d0f971dfe59c69e0cada684b23d49dbf82600ce5" integrity sha512-iiGRvtxWqgtx5m8EyQUJihBloE4EnYeGE/bz1wSPwJE6tZuJUtHlhqDM4Xj2ukE8Dyy1+HCZ4hE0fzIVMzb58w== @@ -3101,34 +3125,34 @@ eslint-scope@^5.0.0: esrecurse "^4.1.0" estraverse "^4.1.1" -eslint-utils@^1.4.3: - version "1.4.3" - resolved "https://registry.yarnpkg.com/eslint-utils/-/eslint-utils-1.4.3.tgz#74fec7c54d0776b6f67e0251040b5806564e981f" - integrity sha512-fbBN5W2xdY45KulGXmLHZ3c3FHfVYmKg0IrAKGOkT/464PQsx2UeIzfz1RmEci+KLm1bBaAzZAh8+/E+XAeZ8Q== +eslint-utils@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/eslint-utils/-/eslint-utils-2.1.0.tgz#d2de5e03424e707dc10c74068ddedae708741b27" + integrity sha512-w94dQYoauyvlDc43XnGB8lU3Zt713vNChgt4EWwhXAP2XkBvndfxF0AgIqKOOasjPIPzj9JqgwkwbCYD0/V3Zg== dependencies: eslint-visitor-keys "^1.1.0" -eslint-visitor-keys@^1.1.0: +eslint-visitor-keys@^1.0.0, eslint-visitor-keys@^1.1.0, eslint-visitor-keys@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-1.2.0.tgz#74415ac884874495f78ec2a97349525344c981fa" integrity sha512-WFb4ihckKil6hu3Dp798xdzSfddwKKU3+nGniKF6HfeW6OLd2OUDEPP7TcHtB5+QXOKg2s6B2DaMPE1Nn/kxKQ== -eslint@^6.0.0: - version "6.8.0" - resolved "https://registry.yarnpkg.com/eslint/-/eslint-6.8.0.tgz#62262d6729739f9275723824302fb227c8c93ffb" - integrity sha512-K+Iayyo2LtyYhDSYwz5D5QdWw0hCacNzyq1Y821Xna2xSJj7cijoLLYmLxTQgcgZ9mC61nryMy9S7GRbYpI5Ig== +eslint@^7.0.0: + version "7.2.0" + resolved "https://registry.yarnpkg.com/eslint/-/eslint-7.2.0.tgz#d41b2e47804b30dbabb093a967fb283d560082e6" + integrity sha512-B3BtEyaDKC5MlfDa2Ha8/D6DsS4fju95zs0hjS3HdGazw+LNayai38A25qMppK37wWGWNYSPOR6oYzlz5MHsRQ== dependencies: "@babel/code-frame" "^7.0.0" ajv "^6.10.0" - chalk "^2.1.0" - cross-spawn "^6.0.5" + chalk "^4.0.0" + cross-spawn "^7.0.2" debug "^4.0.1" doctrine "^3.0.0" - eslint-scope "^5.0.0" - eslint-utils "^1.4.3" - eslint-visitor-keys "^1.1.0" - espree "^6.1.2" - esquery "^1.0.1" + eslint-scope "^5.1.0" + eslint-utils "^2.0.0" + eslint-visitor-keys "^1.2.0" + espree "^7.1.0" + esquery "^1.2.0" esutils "^2.0.2" file-entry-cache "^5.0.1" functional-red-black-tree "^1.0.1" @@ -3141,36 +3165,35 @@ eslint@^6.0.0: is-glob "^4.0.0" js-yaml "^3.13.1" json-stable-stringify-without-jsonify "^1.0.1" - levn "^0.3.0" + levn "^0.4.1" lodash "^4.17.14" minimatch "^3.0.4" - mkdirp "^0.5.1" natural-compare "^1.4.0" - optionator "^0.8.3" + optionator "^0.9.1" progress "^2.0.0" - regexpp "^2.0.1" - semver "^6.1.2" - strip-ansi "^5.2.0" - strip-json-comments "^3.0.1" + regexpp "^3.1.0" + semver "^7.2.1" + strip-ansi "^6.0.0" + strip-json-comments "^3.1.0" table "^5.2.3" text-table "^0.2.0" v8-compile-cache "^2.0.3" -espree@^6.1.2: - version "6.2.1" - resolved "https://registry.yarnpkg.com/espree/-/espree-6.2.1.tgz#77fc72e1fd744a2052c20f38a5b575832e82734a" - integrity sha512-ysCxRQY3WaXJz9tdbWOwuWr5Y/XrPTGX9Kiz3yoUXwW0VZ4w30HTkQLaGx/+ttFjF8i+ACbArnB4ce68a9m5hw== +espree@^7.1.0: + version "7.1.0" + resolved "https://registry.yarnpkg.com/espree/-/espree-7.1.0.tgz#a9c7f18a752056735bf1ba14cb1b70adc3a5ce1c" + integrity sha512-dcorZSyfmm4WTuTnE5Y7MEN1DyoPYy1ZR783QW1FJoenn7RailyWFsq/UL6ZAAA7uXurN9FIpYyUs3OfiIW+Qw== dependencies: - acorn "^7.1.1" + acorn "^7.2.0" acorn-jsx "^5.2.0" - eslint-visitor-keys "^1.1.0" + eslint-visitor-keys "^1.2.0" esprima@^4.0.0: version "4.0.1" resolved "https://registry.yarnpkg.com/esprima/-/esprima-4.0.1.tgz#13b04cdb3e6c5d19df91ab6987a8695619b0aa71" integrity sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A== -esquery@^1.0.1: +esquery@^1.2.0: version "1.3.1" resolved "https://registry.yarnpkg.com/esquery/-/esquery-1.3.1.tgz#b78b5828aa8e214e29fb74c4d5b752e1c033da57" integrity sha512-olpvt9QG0vniUBZspVRN6lwB7hOZoTRtT+jzR+tS4ffYx2mzbw+z0XCOk44aaLYKApNX5nMm+E+P6o25ip/DHQ== @@ -3374,9 +3397,9 @@ fast-deep-equal@^3.1.1: integrity sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q== fast-glob@^3.1.1: - version "3.2.3" - resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-3.2.3.tgz#64d6bf0e32f1195ab45ac8896e4adbe267ccd798" - integrity sha512-fWSEEcoqcYqlFJrpSH5dJTwv6o0r+2bLAmnlne8OQMbFhpSTQXA8Ngp6q1DGA4B+eewHeuH5ndZeiV2qyXXNsA== + version "3.2.4" + resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-3.2.4.tgz#d20aefbf99579383e7f3cc66529158c9b98554d3" + integrity sha512-kr/Oo6PX51265qeuCYsyGypiO5uJFgBS0jksyG7FUeCyQzNwYnzrNIMR1NXfkZXsMYXYLRAHgISHBz8gQcxKHQ== dependencies: "@nodelib/fs.stat" "^2.0.2" "@nodelib/fs.walk" "^1.2.3" @@ -3390,7 +3413,7 @@ fast-json-stable-stringify@^2.0.0: resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz#874bf69c6f404c2b5d99c481341399fd55892633" integrity sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw== -fast-levenshtein@~2.0.6: +fast-levenshtein@^2.0.6: version "2.0.6" resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917" integrity sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc= @@ -3548,7 +3571,7 @@ find-up@^4.0.0, find-up@^4.1.0: locate-path "^5.0.0" path-exists "^4.0.0" -findup-sync@3.0.0: +findup-sync@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/findup-sync/-/findup-sync-3.0.0.tgz#17b108f9ee512dfb7a5c7f3c8b27ea9e1a9c08d1" integrity sha512-YbffarhcicEhOrm4CtrwdKBdCuz576RLdhJDsIfvNtxUuhdRet1qZcsMjqbePtAseKdAnDyM/IyXbu7PRPRLYg== @@ -3586,11 +3609,9 @@ flush-write-stream@^1.0.0: readable-stream "^2.3.6" follow-redirects@^1.0.0: - version "1.11.0" - resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.11.0.tgz#afa14f08ba12a52963140fe43212658897bc0ecb" - integrity sha512-KZm0V+ll8PfBrKwMzdo5D13b1bur9Iq9Zd/RMmAoQQcl2PxxFml8cxXPaaPYVbV0RjNjq1CU7zIzAOqtUPudmA== - dependencies: - debug "^3.0.0" + version "1.12.1" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.12.1.tgz#de54a6205311b93d60398ebc01cf7015682312b6" + integrity sha512-tmRv0AVuR7ZyouUHLeNSiO6pqulF7dYa3s19c6t+wz9LD69/uSzdMxJ2S91nTI9U3rt/IldxpzMOFejp6f0hjg== for-in@^1.0.2: version "1.0.2" @@ -3828,7 +3849,7 @@ global-modules@1.0.0, global-modules@^1.0.0: is-windows "^1.0.1" resolve-dir "^1.0.0" -global-modules@2.0.0, global-modules@^2.0.0: +global-modules@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/global-modules/-/global-modules-2.0.0.tgz#997605ad2345f27f51539bea26574421215c7780" integrity sha512-NGbfmJBp9x8IxyJSd1P+otYK8vonoJactOogrVfFRIAEY1ukil8RSKDz2Yo7wh1oihl51l/r6W4epkeKJHqL8A== @@ -4255,7 +4276,7 @@ import-lazy@^4.0.0: resolved "https://registry.yarnpkg.com/import-lazy/-/import-lazy-4.0.0.tgz#e8eb627483a0a43da3c03f3e35548be5cb0cc153" integrity sha512-rKtvo6a868b5Hu3heneU+L4yEQ4jYKLtjpnPeUdK7h0yzXGmyBTypknlkCvHFBqfX9YlorEiMM6Dnq/5atfHkw== -import-local@2.0.0, import-local@^2.0.0: +import-local@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/import-local/-/import-local-2.0.0.tgz#55070be38a5993cf18ef6db7e961f5bee5c5a09d" integrity sha512-b6s04m3O+s3CGSbqDIyP4R6aAwAeYlVq9+WUWep6iHa8ETRf9yei1U48C5MmfJmV9AiLYYBKPMq/W+/WRpQmCQ== @@ -4350,12 +4371,7 @@ internal-ip@^4.3.0: default-gateway "^4.2.0" ipaddr.js "^1.9.0" -interpret@1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/interpret/-/interpret-1.2.0.tgz#d5061a6224be58e8083985f5014d844359576296" - integrity sha512-mT34yGKMNceBQUoVn7iCDKDntA7SC6gycMAWzGx1z/CMCTV7b2AAtXlo3nRyHZ1FelRkQbQjprHSYGwzLtkVbw== - -interpret@^1.0.0: +interpret@^1.0.0, interpret@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/interpret/-/interpret-1.4.0.tgz#665ab8bc4da27a774a40584e812e3e0fa45b1a1e" integrity sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA== @@ -4932,13 +4948,13 @@ levenary@^1.1.1: dependencies: leven "^3.1.0" -levn@^0.3.0, levn@~0.3.0: - version "0.3.0" - resolved "https://registry.yarnpkg.com/levn/-/levn-0.3.0.tgz#3b09924edf9f083c0490fdd4c0bc4421e04764ee" - integrity sha1-OwmSTt+fCDwEkP3UwLxEIeBHZO4= +levn@^0.4.1: + version "0.4.1" + resolved "https://registry.yarnpkg.com/levn/-/levn-0.4.1.tgz#ae4562c007473b932a6200d403268dd2fffc6ade" + integrity sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ== dependencies: - prelude-ls "~1.1.2" - type-check "~0.3.2" + prelude-ls "^1.2.1" + type-check "~0.4.0" lines-and-columns@^1.1.6: version "1.1.6" @@ -4961,15 +4977,6 @@ loader-runner@^2.4.0: resolved "https://registry.yarnpkg.com/loader-runner/-/loader-runner-2.4.0.tgz#ed47066bfe534d7e84c4c7b9998c2a75607d9357" integrity sha512-Jsmr89RcXGIwivFY21FcRrisYZfvLMTWx5kOLc+JTxtpBOG6xML0vzbc6SEQG2FO9/4Fc3wW4LVcB5DmGflaRw== -loader-utils@1.2.3: - version "1.2.3" - resolved "https://registry.yarnpkg.com/loader-utils/-/loader-utils-1.2.3.tgz#1ff5dc6911c9f0a062531a4c04b609406108c2c7" - integrity sha512-fkpz8ejdnEMG3s37wGL07iSBDg99O9D5yflE9RGNH3hRdx9SOwYfnGYdZOUIZitN8E+E2vkq3MUMYMvPYl5ZZA== - dependencies: - big.js "^5.2.2" - emojis-list "^2.0.0" - json5 "^1.0.1" - loader-utils@^0.2.12: version "0.2.17" resolved "https://registry.yarnpkg.com/loader-utils/-/loader-utils-0.2.17.tgz#f86e6374d43205a6e6c60e9196f17c0299bfb348" @@ -5233,7 +5240,7 @@ mem@^4.0.0: mimic-fn "^2.0.0" p-is-promise "^2.0.0" -memory-fs@^0.4.0, memory-fs@^0.4.1: +memory-fs@^0.4.1: version "0.4.1" resolved "https://registry.yarnpkg.com/memory-fs/-/memory-fs-0.4.1.tgz#3a9a20b8462523e447cfbc7e8bb80ed667bfc552" integrity sha1-OpoguEYlI+RHz7x+i7gO1me/xVI= @@ -5902,17 +5909,17 @@ optimize-css-assets-webpack-plugin@^5.0.3: cssnano "^4.1.10" last-call-webpack-plugin "^3.0.0" -optionator@^0.8.3: - version "0.8.3" - resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.8.3.tgz#84fa1d036fe9d3c7e21d99884b601167ec8fb495" - integrity sha512-+IW9pACdk3XWmmTXG8m3upGUJst5XRGzxMRjXzAuJ1XnIFNvfhjjIuYkDvysnPQ7qzqVzLt78BCruntqRhWQbA== +optionator@^0.9.1: + version "0.9.1" + resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.9.1.tgz#4f236a6373dae0566a6d43e1326674f50c291499" + integrity sha512-74RlY5FCnhq4jRxVUPKDaRwrVNXMqsGsiW6AJw4XK8hmtm10wC0ypZBLw5IIp85NZMr91+qd1RvvENwg7jjRFw== dependencies: - deep-is "~0.1.3" - fast-levenshtein "~2.0.6" - levn "~0.3.0" - prelude-ls "~1.1.2" - type-check "~0.3.2" - word-wrap "~1.2.3" + deep-is "^0.1.3" + fast-levenshtein "^2.0.6" + levn "^0.4.1" + prelude-ls "^1.2.1" + type-check "^0.4.0" + word-wrap "^1.2.3" original@^1.0.0: version "1.0.2" @@ -5931,7 +5938,7 @@ os-homedir@^1.0.0: resolved "https://registry.yarnpkg.com/os-homedir/-/os-homedir-1.0.2.tgz#ffbc4988336e0e833de0c168c7ef152121aa7fb3" integrity sha1-/7xJiDNuDoM94MFox+8VISGqf7M= -os-locale@^3.0.0, os-locale@^3.1.0: +os-locale@^3.0.0: version "3.1.0" resolved "https://registry.yarnpkg.com/os-locale/-/os-locale-3.1.0.tgz#a802a6ee17f24c10483ab9935719cef4ed16bf1a" integrity sha512-Z8l3R4wYWM40/52Z+S265okfFj8Kt2cC2MKY+xNi3kFs+XGI7WXu/I309QQQYbRW4ijiZ+yxs9pqEhJh0DqW3Q== @@ -6164,6 +6171,11 @@ path-key@^2.0.0, path-key@^2.0.1: resolved "https://registry.yarnpkg.com/path-key/-/path-key-2.0.1.tgz#411cadb574c5a140d3a4b1910d40d80cc9f40b40" integrity sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A= +path-key@^3.1.0: + version "3.1.1" + resolved "https://registry.yarnpkg.com/path-key/-/path-key-3.1.1.tgz#581f6ade658cbba65a0d3380de7753295054f375" + integrity sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q== + path-parse@^1.0.6: version "1.0.6" resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.6.tgz#d62dbb5679405d72c4737ec58600e9ddcf06d24c" @@ -6981,10 +6993,10 @@ postcss@^7.0.0, postcss@^7.0.1, postcss@^7.0.14, postcss@^7.0.16, postcss@^7.0.1 source-map "^0.6.1" supports-color "^6.1.0" -prelude-ls@~1.1.2: - version "1.1.2" - resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.1.2.tgz#21932a549f5e52ffd9a827f570e04be62a97da54" - integrity sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ= +prelude-ls@^1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.2.1.tgz#debc6489d7a6e6b0e7611888cec880337d316396" + integrity sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g== prepend-http@^1.0.0: version "1.0.4" @@ -7335,10 +7347,10 @@ regexp.prototype.flags@^1.2.0: define-properties "^1.1.3" es-abstract "^1.17.0-next.1" -regexpp@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/regexpp/-/regexpp-2.0.1.tgz#8d19d31cf632482b589049f8281f93dbcba4d07f" - integrity sha512-lv0M6+TkDVniA3aD1Eg0DVpfU/booSu7Eev3TDO/mZKHBfVjgCGTV4t4buppESEYDtkArYFOxTJWv6S5C+iaNw== +regexpp@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/regexpp/-/regexpp-3.1.0.tgz#206d0ad0a5648cffbdb8ae46438f3dc51c9f78e2" + integrity sha512-ZOIzd8yVsQQA7j8GCSlPGXwg5PfmA1mrq0JP4nGhh54LaKN3xdai/vHUDu74pKwV8OxseMS65u2NImosQcSD0Q== regexpu-core@^4.7.0: version "4.7.0" @@ -7757,11 +7769,16 @@ semver@7.0.0: resolved "https://registry.yarnpkg.com/semver/-/semver-7.0.0.tgz#5f3ca35761e47e05b206c6daff2cf814f0316b8e" integrity sha512-+GB6zVA9LWh6zovYQLALHwv5rb2PHGlJi3lfiqIHxR0uuwCgefcOJc59v9fv1w8GbStwxuuqqAjI9NMAOOgq1A== -semver@^6.0.0, semver@^6.1.2, semver@^6.3.0: +semver@^6.0.0, semver@^6.3.0: version "6.3.0" resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.0.tgz#ee0a64c8af5e8ceea67687b133761e1becbd1d3d" integrity sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw== +semver@^7.2.1: + version "7.3.2" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.2.tgz#604962b052b81ed0786aae84389ffba70ffd3938" + integrity sha512-OrOb32TeeambH6UrhtShmF7CRDqhL6/5XpPNp2DuRH6+9QLw/orhp72j87v8Qa1ScDkvrrBNpZcDejAirJmfXQ== + semver@~5.3.0: version "5.3.0" resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" @@ -7873,11 +7890,23 @@ shebang-command@^1.2.0: dependencies: shebang-regex "^1.0.0" +shebang-command@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/shebang-command/-/shebang-command-2.0.0.tgz#ccd0af4f8835fbdc265b82461aaf0c36663f34ea" + integrity sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA== + dependencies: + shebang-regex "^3.0.0" + shebang-regex@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-1.0.0.tgz#da42f49740c0b42db2ca9728571cb190c98efea3" integrity sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM= +shebang-regex@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-3.0.0.tgz#ae16f1644d873ecad843b0307b143362d4c42172" + integrity sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A== + shelljs@^0.8.1: version "0.8.4" resolved "https://registry.yarnpkg.com/shelljs/-/shelljs-0.8.4.tgz#de7684feeb767f8716b326078a8a00875890e3c2" @@ -8343,7 +8372,7 @@ strip-indent@^3.0.0: dependencies: min-indent "^1.0.0" -strip-json-comments@^3.0.1: +strip-json-comments@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-3.1.0.tgz#7638d31422129ecf4457440009fba03f9f9ac180" integrity sha512-e6/d0eBu7gHtdCqFt0xJr642LdToM5/cN4Qb9DbHjVx1CP5RyeM+zH7pbecEmDv/lBqb0QH+6Uqq75rxFPkM0w== @@ -8388,15 +8417,15 @@ stylelint-config-standard@^20.0.0: stylelint-config-recommended "^3.0.0" stylelint@^13.0.0: - version "13.6.0" - resolved "https://registry.yarnpkg.com/stylelint/-/stylelint-13.6.0.tgz#3528bc402a71f2af2a3de32fa4e9f1c24e49666d" - integrity sha512-55gG2pNjVr183JJM/tlr3KAua6vTVX7Ho/lgKKuCIWszTZ1gmrXjX4Wok53SI8wRYFPbwKAcJGULQ77OJxTcNw== + version "13.6.1" + resolved "https://registry.yarnpkg.com/stylelint/-/stylelint-13.6.1.tgz#cc1d76338116d55e8ff2be94c4a4386c1239b878" + integrity sha512-XyvKyNE7eyrqkuZ85Citd/Uv3ljGiuYHC6UiztTR6sWS9rza8j3UeQv/eGcQS9NZz/imiC4GKdk1EVL3wst5vw== dependencies: "@stylelint/postcss-css-in-js" "^0.37.1" "@stylelint/postcss-markdown" "^0.36.1" autoprefixer "^9.8.0" balanced-match "^1.0.0" - chalk "^4.0.0" + chalk "^4.1.0" cosmiconfig "^6.0.0" debug "^4.1.1" execall "^2.0.0" @@ -8448,13 +8477,6 @@ sugarss@^2.0.0: dependencies: postcss "^7.0.2" -supports-color@6.1.0, supports-color@^6.1.0: - version "6.1.0" - resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-6.1.0.tgz#0764abc69c63d5ac842dd4867e8d025e880df8f3" - integrity sha512-qe1jfm1Mg7Nq/NSh6XE24gPXROEVsWHxC1LIx//XNlD9iw7YZQGjZNjYN7xGaEG6iKdA8EtNFW6R0gjnVXp+wQ== - dependencies: - has-flag "^3.0.0" - supports-color@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-2.0.0.tgz#535d045ce6b6363fa40117084629995e9df324c7" @@ -8467,6 +8489,13 @@ supports-color@^5.3.0: dependencies: has-flag "^3.0.0" +supports-color@^6.1.0: + version "6.1.0" + resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-6.1.0.tgz#0764abc69c63d5ac842dd4867e8d025e880df8f3" + integrity sha512-qe1jfm1Mg7Nq/NSh6XE24gPXROEVsWHxC1LIx//XNlD9iw7YZQGjZNjYN7xGaEG6iKdA8EtNFW6R0gjnVXp+wQ== + dependencies: + has-flag "^3.0.0" + supports-color@^7.0.0, supports-color@^7.1.0: version "7.1.0" resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-7.1.0.tgz#68e32591df73e25ad1c4b49108a2ec507962bfd1" @@ -8566,9 +8595,9 @@ terser-webpack-plugin@^2.3.5: webpack-sources "^1.4.3" terser@^4.1.2, terser@^4.6.12: - version "4.7.0" - resolved "https://registry.yarnpkg.com/terser/-/terser-4.7.0.tgz#15852cf1a08e3256a80428e865a2fa893ffba006" - integrity sha512-Lfb0RiZcjRDXCC3OSHJpEkxJ9Qeqs6mp2v4jf2MHfy8vGERmVDuvjXdd/EnP5Deme5F2yBRBymKmKHCBg2echw== + version "4.8.0" + resolved "https://registry.yarnpkg.com/terser/-/terser-4.8.0.tgz#63056343d7c70bb29f3af665865a46fe03a0df17" + integrity sha512-EAPipTNeWsb/3wLPeup1tVPaXfIaU68xMnVdPafIL1TV05OhASArYyIfFvnvJCNrR2NIOvDVNNTFRa+Re2MWyw== dependencies: commander "^2.20.0" source-map "~0.6.1" @@ -8587,9 +8616,9 @@ thenify-all@^1.0.0: thenify ">= 3.1.0 < 4" "thenify@>= 3.1.0 < 4": - version "3.3.0" - resolved "https://registry.yarnpkg.com/thenify/-/thenify-3.3.0.tgz#e69e38a1babe969b0108207978b9f62b88604839" - integrity sha1-5p44obq+lpsBCCB5eLn2K4hgSDk= + version "3.3.1" + resolved "https://registry.yarnpkg.com/thenify/-/thenify-3.3.1.tgz#8932e686a4066038a016dd9e2ca46add9838a95f" + integrity sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw== dependencies: any-promise "^1.0.0" @@ -8762,12 +8791,12 @@ tweetnacl@^0.14.3, tweetnacl@~0.14.0: resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64" integrity sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q= -type-check@~0.3.2: - version "0.3.2" - resolved "https://registry.yarnpkg.com/type-check/-/type-check-0.3.2.tgz#5884cab512cf1d355e3fb784f30804b2b520db72" - integrity sha1-WITKtRLPHTVeP7eE8wgEsrUg23I= +type-check@^0.4.0, type-check@~0.4.0: + version "0.4.0" + resolved "https://registry.yarnpkg.com/type-check/-/type-check-0.4.0.tgz#07b8203bfa7056c0657050e3ccd2c37730bab8f1" + integrity sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew== dependencies: - prelude-ls "~1.1.2" + prelude-ls "^1.2.1" type-fest@^0.11.0: version "0.11.0" @@ -9085,11 +9114,6 @@ uuid@^3.3.2, uuid@^3.4.0: resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee" integrity sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A== -v8-compile-cache@2.0.3: - version "2.0.3" - resolved "https://registry.yarnpkg.com/v8-compile-cache/-/v8-compile-cache-2.0.3.tgz#00f7494d2ae2b688cfe2899df6ed2c54bef91dbe" - integrity sha512-CNmdbwQMBjwr9Gsmohvm0pbL954tJrNzf6gWL3K+QMQf00PF7ERGrEiLgjuU3mKreLC2MeGhUsNV9ybTbLgd3w== - v8-compile-cache@^2.0.3, v8-compile-cache@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/v8-compile-cache/-/v8-compile-cache-2.1.1.tgz#54bc3cdd43317bca91e35dcaf305b1a7237de745" @@ -9190,21 +9214,21 @@ webpack-assets-manifest@^3.1.1: webpack-sources "^1.0.0" webpack-cli@^3.3.0, webpack-cli@^3.3.11: - version "3.3.11" - resolved "https://registry.yarnpkg.com/webpack-cli/-/webpack-cli-3.3.11.tgz#3bf21889bf597b5d82c38f215135a411edfdc631" - integrity sha512-dXlfuml7xvAFwYUPsrtQAA9e4DOe58gnzSxhgrO/ZM/gyXTBowrsYeubyN4mqGhYdpXMFNyQ6emjJS9M7OBd4g== + version "3.3.12" + resolved "https://registry.yarnpkg.com/webpack-cli/-/webpack-cli-3.3.12.tgz#94e9ada081453cd0aa609c99e500012fd3ad2d4a" + integrity sha512-NVWBaz9k839ZH/sinurM+HcDvJOTXwSjYp1ku+5XKeOC03z8v5QitnK/x+lAxGXFyhdayoIf/GOpv85z3/xPag== dependencies: - chalk "2.4.2" - cross-spawn "6.0.5" - enhanced-resolve "4.1.0" - findup-sync "3.0.0" - global-modules "2.0.0" - import-local "2.0.0" - interpret "1.2.0" - loader-utils "1.2.3" - supports-color "6.1.0" - v8-compile-cache "2.0.3" - yargs "13.2.4" + chalk "^2.4.2" + cross-spawn "^6.0.5" + enhanced-resolve "^4.1.1" + findup-sync "^3.0.0" + global-modules "^2.0.0" + import-local "^2.0.0" + interpret "^1.4.0" + loader-utils "^1.4.0" + supports-color "^6.1.0" + v8-compile-cache "^2.1.1" + yargs "^13.3.2" webpack-dev-middleware@^3.7.2: version "3.7.2" @@ -9334,6 +9358,13 @@ which@1, which@^1.2.14, which@^1.2.9, which@^1.3.1: dependencies: isexe "^2.0.0" +which@^2.0.1: + version "2.0.2" + resolved "https://registry.yarnpkg.com/which/-/which-2.0.2.tgz#7c6a8dd0a636a0327e10b59c9286eee93f3f51b1" + integrity sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA== + dependencies: + isexe "^2.0.0" + wide-align@^1.1.0: version "1.1.3" resolved "https://registry.yarnpkg.com/wide-align/-/wide-align-1.1.3.tgz#ae074e6bdc0c14a431e804e624549c633b000457" @@ -9341,7 +9372,7 @@ wide-align@^1.1.0: dependencies: string-width "^1.0.2 || 2" -word-wrap@~1.2.3: +word-wrap@^1.2.3: version "1.2.3" resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.3.tgz#610636f6b1f703891bd34771ccb17fb93b47079c" integrity sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ== @@ -9437,7 +9468,7 @@ yargs-parser@^11.1.1: camelcase "^5.0.0" decamelize "^1.2.0" -yargs-parser@^13.1.0, yargs-parser@^13.1.2: +yargs-parser@^13.1.2: version "13.1.2" resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-13.1.2.tgz#130f09702ebaeef2650d54ce6e3e5706f7a4fb38" integrity sha512-3lbsNRf/j+A4QuSZfDRA7HRSfWrzO0YjqTJd5kjAq37Zep1CEgaYmrH9Q3GwPiB9cHyd1Y1UwggGhJGoxipbzg== @@ -9471,23 +9502,6 @@ yargs@12.0.5: y18n "^3.2.1 || ^4.0.0" yargs-parser "^11.1.1" -yargs@13.2.4: - version "13.2.4" - resolved "https://registry.yarnpkg.com/yargs/-/yargs-13.2.4.tgz#0b562b794016eb9651b98bd37acf364aa5d6dc83" - integrity sha512-HG/DWAJa1PAnHT9JAhNa8AbAv3FPaiLzioSjCcmuXXhP8MlpHO5vwls4g4j6n30Z74GVQj8Xa62dWVx1QCGklg== - dependencies: - cliui "^5.0.0" - find-up "^3.0.0" - get-caller-file "^2.0.1" - os-locale "^3.1.0" - require-directory "^2.1.1" - require-main-filename "^2.0.0" - set-blocking "^2.0.0" - string-width "^3.0.0" - which-module "^2.0.0" - y18n "^4.0.0" - yargs-parser "^13.1.0" - yargs@^13.3.2: version "13.3.2" resolved "https://registry.yarnpkg.com/yargs/-/yargs-13.3.2.tgz#ad7ffefec1aa59565ac915f82dccb38a9c31a2dd" From 491a53ffe3848060090dd6e439401aee5c86531b Mon Sep 17 00:00:00 2001 From: evazion Date: Thu, 18 Jun 2020 22:07:43 -0500 Subject: [PATCH 12/59] Update mock_redis. --- Gemfile.lock | 2 +- app/models/saved_search.rb | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index b9cc72ac4..c671469e6 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -221,7 +221,7 @@ GEM minitest (>= 5.0) ruby-progressbar mocha (1.11.2) - mock_redis (0.23.0) + mock_redis (0.24.0) msgpack (1.3.3) msgpack (1.3.3-x64-mingw32) multi_json (1.14.1) diff --git a/app/models/saved_search.rb b/app/models/saved_search.rb index 36478284f..722f47eb1 100644 --- a/app/models/saved_search.rb +++ b/app/models/saved_search.rb @@ -18,8 +18,7 @@ class SavedSearch < ApplicationRecord post_ids = Set.new queries.each do |query| redis_key = "search:#{query}" - # XXX change to `exists?` (ref: https://github.com-sds/mock_redis/pull/188 - if redis.exists(redis_key) + if redis.exists?(redis_key) sub_ids = redis.smembers(redis_key).map(&:to_i) post_ids.merge(sub_ids) else From 440bbbb2889770e107209cb610d71af8a1913c3e Mon Sep 17 00:00:00 2001 From: evazion Date: Thu, 18 Jun 2020 14:24:41 -0500 Subject: [PATCH 13/59] Update nokogiri gem. Fix gem version conflicts described in 20abd8a5f. Nokogiri couldn't be upgraded past 1.10.9 because 1.11.0 causes a build failure in Nokogumbo 2.0.2, but we couldn't stay on 1.10.9 either because it has a hard requirement on Ruby <2.7 and we require Ruby >=2.7. This made `bundle update` fail with a Gemfile conflict. The fix is to disable libxml2 support when building Nokogumbo. Nokogumbo wants to use the same version of libxml2 as Nokogiri, but Nokogiri 1.11.0 changed how it reports which version of libxml2 it's using, which causes Nokogumbo's build to fail. Disabling libxml2 may reduce performance of Nokogumbo ([1]). While we're at it, we also make Nokogiri use the system version of libxml2 instead of its own bundled version. Nokogiri really wants us to use its own patched version of libxml2 instead of the system version, but the patches it applies look relatively minor and don't seem relevant to us ([2]). Using the system version reduces build time during CI. This adds libxml2 and libxslt as OS-level dependencies of Danbooru. You may need to do `sudo apt-get install libxml2-dev libxslt-dev` to install these libraries after this commit. [1]: https://github.com/rubys/nokogumbo#flavors-of-nokogumbo [2]: https://github.com/sparklemotion/nokogiri/tree/master/patches/libxml2 --- .bundle/config | 3 +++ .github/workflows/test.yaml | 2 +- .gitignore | 1 - Gemfile | 4 +--- Gemfile.lock | 11 +++++------ config/docker/Dockerfile.danbooru | 10 +++++++++- 6 files changed, 19 insertions(+), 12 deletions(-) create mode 100644 .bundle/config diff --git a/.bundle/config b/.bundle/config new file mode 100644 index 000000000..b64ad2111 --- /dev/null +++ b/.bundle/config @@ -0,0 +1,3 @@ +--- +BUNDLE_BUILD__NOKOGIRI: "--use-system-libraries" +BUNDLE_BUILD__NOKOGUMBO: "--without-libxml2" diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 53485a29a..592dac48b 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -64,7 +64,7 @@ jobs: - name: Install OS dependencies run: | apt-get update - apt-get -y install --no-install-recommends build-essential ruby ruby-dev ruby-bundler git nodejs yarnpkg webpack ffmpeg mkvtoolnix libvips-dev libxml2-dev postgresql-server-dev-all wget + apt-get -y install --no-install-recommends build-essential ruby ruby-dev ruby-bundler git nodejs yarnpkg webpack ffmpeg mkvtoolnix libvips-dev libxml2-dev libxslt-dev zlib1g-dev postgresql-server-dev-all wget ln -sf /usr/bin/yarnpkg /usr/bin/yarn - name: Install Ruby dependencies diff --git a/.gitignore b/.gitignore index 8f4292733..a51a3e83b 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,6 @@ .yarn-integrity .gitconfig .git/ -.bundle/ config/database.yml config/danbooru_local_config.rb config/deploy/*.rb diff --git a/Gemfile b/Gemfile index ad30bc238..4304f26cd 100644 --- a/Gemfile +++ b/Gemfile @@ -47,9 +47,7 @@ gem 'http' gem 'activerecord-hierarchical_query' gem 'pundit' gem 'mail' - -# locked to 1.10.9 to workaround an incompatibility with nokogumbo 2.0.2. -gem 'nokogiri', '~> 1.10.9' +gem 'nokogiri' group :production, :staging do gem 'unicorn', :platforms => :ruby diff --git a/Gemfile.lock b/Gemfile.lock index c671469e6..338a3929b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -211,7 +211,7 @@ GEM mime-types-data (3.2020.0512) mimemagic (0.3.5) mini_mime (1.0.2) - mini_portile2 (2.4.0) + mini_portile2 (2.5.0) minitest (5.14.1) minitest-ci (3.4.0) minitest (>= 5.0.6) @@ -239,10 +239,9 @@ GEM net-ssh (6.1.0) newrelic_rpm (6.11.0.365) nio4r (2.5.2) - nokogiri (1.10.9) - mini_portile2 (~> 2.4.0) - nokogiri (1.10.9-x64-mingw32) - mini_portile2 (~> 2.4.0) + nokogiri (1.11.0.rc2) + mini_portile2 (~> 2.5.0) + nokogiri (1.11.0.rc2-x64-mingw32) nokogumbo (2.0.2) nokogiri (~> 1.8, >= 1.8.4) ntlm-http (0.1.1) @@ -465,7 +464,7 @@ DEPENDENCIES mock_redis net-sftp newrelic_rpm - nokogiri (~> 1.10.9) + nokogiri oauth2 pg pry-byebug diff --git a/config/docker/Dockerfile.danbooru b/config/docker/Dockerfile.danbooru index df2af68bd..013644a89 100644 --- a/config/docker/Dockerfile.danbooru +++ b/config/docker/Dockerfile.danbooru @@ -13,14 +13,20 @@ RUN \ webpack \ libvips-dev \ libxml2-dev \ + libxslt-dev \ + zlib1g-dev \ postgresql-server-dev-all && \ # webpacker expects the binary to be called `yarn`, but debian/ubuntu installs it as `yarnpkg`. ln -sf /usr/bin/yarnpkg /usr/bin/yarn WORKDIR /build +COPY .bundle .bundle COPY Gemfile Gemfile.lock ./ -RUN BUNDLE_DEPLOYMENT=true bundle install --jobs 4 +RUN \ + bundle config set deployment true --local && \ + bundle config set path vendor/bundle && \ + bundle install --jobs 4 COPY package.json yarn.lock ./ RUN yarn install @@ -44,6 +50,8 @@ RUN \ mkvtoolnix \ libvips \ libxml2 \ + libxslt1.1 \ + zlib1g \ postgresql-client USER danbooru From cd9e3e7f3d2319b603bb9a7b6608e6e14c9feeb5 Mon Sep 17 00:00:00 2001 From: evazion Date: Fri, 19 Jun 2020 03:14:25 -0500 Subject: [PATCH 14/59] Update Rails to 6.0.3.2. Fixes CVE-2020-8185: Untrusted users able to run pending migrations in production. --- Gemfile.lock | 102 +++++++++++++++++++++++++-------------------------- 1 file changed, 51 insertions(+), 51 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 338a3929b..7933ca1f5 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -8,63 +8,63 @@ GIT GEM remote: https://rubygems.org/ specs: - actioncable (6.0.3.1) - actionpack (= 6.0.3.1) + actioncable (6.0.3.2) + actionpack (= 6.0.3.2) nio4r (~> 2.0) websocket-driver (>= 0.6.1) - actionmailbox (6.0.3.1) - actionpack (= 6.0.3.1) - activejob (= 6.0.3.1) - activerecord (= 6.0.3.1) - activestorage (= 6.0.3.1) - activesupport (= 6.0.3.1) + actionmailbox (6.0.3.2) + actionpack (= 6.0.3.2) + activejob (= 6.0.3.2) + activerecord (= 6.0.3.2) + activestorage (= 6.0.3.2) + activesupport (= 6.0.3.2) mail (>= 2.7.1) - actionmailer (6.0.3.1) - actionpack (= 6.0.3.1) - actionview (= 6.0.3.1) - activejob (= 6.0.3.1) + actionmailer (6.0.3.2) + actionpack (= 6.0.3.2) + actionview (= 6.0.3.2) + activejob (= 6.0.3.2) mail (~> 2.5, >= 2.5.4) rails-dom-testing (~> 2.0) - actionpack (6.0.3.1) - actionview (= 6.0.3.1) - activesupport (= 6.0.3.1) + actionpack (6.0.3.2) + actionview (= 6.0.3.2) + activesupport (= 6.0.3.2) rack (~> 2.0, >= 2.0.8) rack-test (>= 0.6.3) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.0, >= 1.2.0) - actiontext (6.0.3.1) - actionpack (= 6.0.3.1) - activerecord (= 6.0.3.1) - activestorage (= 6.0.3.1) - activesupport (= 6.0.3.1) + actiontext (6.0.3.2) + actionpack (= 6.0.3.2) + activerecord (= 6.0.3.2) + activestorage (= 6.0.3.2) + activesupport (= 6.0.3.2) nokogiri (>= 1.8.5) - actionview (6.0.3.1) - activesupport (= 6.0.3.1) + actionview (6.0.3.2) + activesupport (= 6.0.3.2) builder (~> 3.1) erubi (~> 1.4) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.1, >= 1.2.0) - activejob (6.0.3.1) - activesupport (= 6.0.3.1) + activejob (6.0.3.2) + activesupport (= 6.0.3.2) globalid (>= 0.3.6) - activemodel (6.0.3.1) - activesupport (= 6.0.3.1) + activemodel (6.0.3.2) + activesupport (= 6.0.3.2) activemodel-serializers-xml (1.0.2) activemodel (> 5.x) activesupport (> 5.x) builder (~> 3.1) - activerecord (6.0.3.1) - activemodel (= 6.0.3.1) - activesupport (= 6.0.3.1) + activerecord (6.0.3.2) + activemodel (= 6.0.3.2) + activesupport (= 6.0.3.2) activerecord-hierarchical_query (1.2.3) activerecord (>= 5.0, < 6.1) pg (>= 0.21, < 1.3) - activestorage (6.0.3.1) - actionpack (= 6.0.3.1) - activejob (= 6.0.3.1) - activerecord (= 6.0.3.1) + activestorage (6.0.3.2) + actionpack (= 6.0.3.2) + activejob (= 6.0.3.2) + activerecord (= 6.0.3.2) marcel (~> 0.3.1) - activesupport (6.0.3.1) + activesupport (6.0.3.2) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 0.7, < 2) minitest (~> 5.1) @@ -184,7 +184,7 @@ GEM listen (3.2.1) rb-fsevent (~> 0.10, >= 0.10.3) rb-inotify (~> 0.9, >= 0.9.10) - loofah (2.5.0) + loofah (2.6.0) crass (~> 1.0.2) nokogiri (>= 1.5.9) mail (2.7.1) @@ -280,29 +280,29 @@ GEM rack rack-test (1.1.0) rack (>= 1.0, < 3) - rails (6.0.3.1) - actioncable (= 6.0.3.1) - actionmailbox (= 6.0.3.1) - actionmailer (= 6.0.3.1) - actionpack (= 6.0.3.1) - actiontext (= 6.0.3.1) - actionview (= 6.0.3.1) - activejob (= 6.0.3.1) - activemodel (= 6.0.3.1) - activerecord (= 6.0.3.1) - activestorage (= 6.0.3.1) - activesupport (= 6.0.3.1) + rails (6.0.3.2) + actioncable (= 6.0.3.2) + actionmailbox (= 6.0.3.2) + actionmailer (= 6.0.3.2) + actionpack (= 6.0.3.2) + actiontext (= 6.0.3.2) + actionview (= 6.0.3.2) + activejob (= 6.0.3.2) + activemodel (= 6.0.3.2) + activerecord (= 6.0.3.2) + activestorage (= 6.0.3.2) + activesupport (= 6.0.3.2) bundler (>= 1.3.0) - railties (= 6.0.3.1) + railties (= 6.0.3.2) sprockets-rails (>= 2.0.0) rails-dom-testing (2.0.3) activesupport (>= 4.2.0) nokogiri (>= 1.6) rails-html-sanitizer (1.3.0) loofah (~> 2.3) - railties (6.0.3.1) - actionpack (= 6.0.3.1) - activesupport (= 6.0.3.1) + railties (6.0.3.2) + actionpack (= 6.0.3.2) + activesupport (= 6.0.3.2) method_source rake (>= 0.8.7) thor (>= 0.20.3, < 2.0) From 05f78f9beff248600ab16088dad1b948e765f497 Mon Sep 17 00:00:00 2001 From: evazion Date: Fri, 19 Jun 2020 12:13:56 -0500 Subject: [PATCH 15/59] saved searches: fix call to redis.exists? https://github.com/redis/redis-rb/blob/master/CHANGELOG.md#421 --- app/models/saved_search.rb | 2 +- test/unit/post_query_builder_test.rb | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/models/saved_search.rb b/app/models/saved_search.rb index 722f47eb1..d992bf11d 100644 --- a/app/models/saved_search.rb +++ b/app/models/saved_search.rb @@ -115,7 +115,7 @@ class SavedSearch < ApplicationRecord def populate(query, timeout: 10_000) redis_key = "search:#{query}" - return if redis.exists(redis_key) + return if redis.exists?(redis_key) post_ids = Post.with_timeout(timeout, [], query: query) do Post.system_tag_match(query).limit(QUERY_LIMIT).pluck(:id) diff --git a/test/unit/post_query_builder_test.rb b/test/unit/post_query_builder_test.rb index d74b08e8e..055e6e765 100644 --- a/test/unit/post_query_builder_test.rb +++ b/test/unit/post_query_builder_test.rb @@ -757,8 +757,8 @@ class PostQueryBuilderTest < ActiveSupport::TestCase create(:saved_search, query: "aaa", labels: ["zzz"], user: CurrentUser.user) create(:saved_search, query: "bbb", user: CurrentUser.user) - Redis.any_instance.stubs(:exists).with("search:aaa").returns(true) - Redis.any_instance.stubs(:exists).with("search:bbb").returns(true) + Redis.any_instance.stubs(:exists?).with("search:aaa").returns(true) + Redis.any_instance.stubs(:exists?).with("search:bbb").returns(true) Redis.any_instance.stubs(:smembers).with("search:aaa").returns([@post1.id]) Redis.any_instance.stubs(:smembers).with("search:bbb").returns([@post2.id]) From 67a52dbc2d9ca82962a470d073c00d821836e4b3 Mon Sep 17 00:00:00 2001 From: evazion Date: Fri, 19 Jun 2020 12:43:41 -0500 Subject: [PATCH 16/59] tumblr: support new va.media.tumblr.com urls. --- app/logical/sources/strategies/tumblr.rb | 2 +- test/unit/sources/tumblr_test.rb | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/logical/sources/strategies/tumblr.rb b/app/logical/sources/strategies/tumblr.rb index 6123f9f8b..b82a1f77d 100644 --- a/app/logical/sources/strategies/tumblr.rb +++ b/app/logical/sources/strategies/tumblr.rb @@ -23,7 +23,7 @@ module Sources::Strategies OLD_IMAGE = %r{\Ahttps?://#{DOMAIN}/(?#{MD5}/)?#{FILENAME}_(?\w+)\.#{EXT}\z}i IMAGE = %r{\Ahttps?://#{DOMAIN}/}i - VIDEO = %r{\Ahttps?://(?:vtt|ve\.media)\.tumblr\.com/}i + VIDEO = %r{\Ahttps?://(?:vtt|ve|va\.media)\.tumblr\.com/}i POST = %r{\Ahttps?://(?[^.]+)\.tumblr\.com/(?:post|image)/(?\d+)}i def self.enabled? diff --git a/test/unit/sources/tumblr_test.rb b/test/unit/sources/tumblr_test.rb index 3b774f934..fdba14560 100644 --- a/test/unit/sources/tumblr_test.rb +++ b/test/unit/sources/tumblr_test.rb @@ -169,7 +169,7 @@ module Sources context "The source for a 'http://ve.media.tumblr.com/*' video post with inline images" do setup do - @url = "https://ve.media.tumblr.com/tumblr_os31dkexhK1wsfqep.mp4" + @url = "https://va.media.tumblr.com/tumblr_os31dkexhK1wsfqep.mp4" @ref = "https://noizave.tumblr.com/post/162222617101" end @@ -177,7 +177,7 @@ module Sources should "get the video and inline images" do site = Sources::Strategies.find(@url, @ref) urls = %w[ - https://ve.media.tumblr.com/tumblr_os31dkexhK1wsfqep.mp4 + https://va.media.tumblr.com/tumblr_os31dkexhK1wsfqep.mp4 https://media.tumblr.com/afed9f5b3c33c39dc8c967e262955de2/tumblr_inline_os31dclyCR1v11u29_1280.png ] From 7a1efc2744f0f9ba3e21632b5c7779f29116982b Mon Sep 17 00:00:00 2001 From: evazion Date: Fri, 19 Jun 2020 13:49:22 -0500 Subject: [PATCH 17/59] Fix #4522: Sidebar doesn't show most searched tags at certain times of day. Revert back to previous workaround of fetching previous day if current day returns no result. A terrible hack, really we should convert dates to Reportbooru's timezone, but that has other complications. --- app/controllers/explore/posts_controller.rb | 4 +-- app/logical/reportbooru_service.rb | 29 ++++++++++--------- .../explore/posts/missed_searches.html.erb | 2 +- app/views/explore/posts/searches.html.erb | 2 +- test/unit/reportbooru_service_test.rb | 8 ++--- 5 files changed, 23 insertions(+), 22 deletions(-) diff --git a/app/controllers/explore/posts_controller.rb b/app/controllers/explore/posts_controller.rb index e40d6c5b3..a3a47423c 100644 --- a/app/controllers/explore/posts_controller.rb +++ b/app/controllers/explore/posts_controller.rb @@ -28,11 +28,11 @@ module Explore def searches @date, @scale, @min_date, @max_date = parse_date(params) - @search_service = ReportbooruService.new + @searches = ReportbooruService.new.popular_searches(@date) end def missed_searches - @search_service = ReportbooruService.new + @missed_searches = ReportbooruService.new.missed_search_rankings end private diff --git a/app/logical/reportbooru_service.rb b/app/logical/reportbooru_service.rb index 5ad3cd7f6..9ae703367 100644 --- a/app/logical/reportbooru_service.rb +++ b/app/logical/reportbooru_service.rb @@ -20,29 +20,30 @@ class ReportbooruService body.lines.map(&:split).map { [_1, _2.to_i] } end - def post_search_rankings(date = Date.today, expires_in: 1.minutes) - return [] unless enabled? - - response = http.cache(expires_in).get("#{reportbooru_server}/post_searches/rank?date=#{date}") - return [] if response.status != 200 - JSON.parse(response.to_s.force_encoding("utf-8")) + def post_search_rankings(date, expires_in: 1.minutes) + request("#{reportbooru_server}/post_searches/rank?date=#{date}", expires_in) end - def post_view_rankings(date = Date.today, expires_in: 1.minutes) - return [] unless enabled? - - response = http.cache(expires_in).get("#{reportbooru_server}/post_views/rank?date=#{date}") - return [] if response.status != 200 - JSON.parse(response.to_s.force_encoding("utf-8")) + def post_view_rankings(date, expires_in: 1.minutes) + request("#{reportbooru_server}/post_views/rank?date=#{date}", expires_in) end - def popular_searches(date = Date.today, limit: 100) + def popular_searches(date, limit: 100) ranking = post_search_rankings(date) ranking.take(limit).map(&:first) end - def popular_posts(date = Date.today, limit: 100) + def popular_posts(date, limit: 100) ranking = post_view_rankings(date) + ranking = post_view_rankings(date.yesterday) if ranking.blank? ranking.take(limit).map { |x| Post.find(x[0]) } end + + def request(url, expires_in) + return [] unless enabled? + + response = http.cache(expires_in).get(url) + return [] if response.status != 200 + JSON.parse(response.to_s.force_encoding("utf-8")) + end end diff --git a/app/views/explore/posts/missed_searches.html.erb b/app/views/explore/posts/missed_searches.html.erb index 171017b87..329b66c43 100644 --- a/app/views/explore/posts/missed_searches.html.erb +++ b/app/views/explore/posts/missed_searches.html.erb @@ -15,7 +15,7 @@ - <% @search_service.missed_search_rankings.each do |tags, count| %> + <% @missed_searches.each do |tags, count| %> <%= link_to tags, posts_path(:tags => tags) %> diff --git a/app/views/explore/posts/searches.html.erb b/app/views/explore/posts/searches.html.erb index fb9e3446e..5256427eb 100644 --- a/app/views/explore/posts/searches.html.erb +++ b/app/views/explore/posts/searches.html.erb @@ -13,7 +13,7 @@ - <% @search_service.post_search_rankings(@date).each do |tags, count| %> + <% @searches.each do |tags, count| %> <%= link_to tags, posts_path(:tags => tags) %> <%= count.to_i %> diff --git a/test/unit/reportbooru_service_test.rb b/test/unit/reportbooru_service_test.rb index 3d795d130..d31e340f6 100644 --- a/test/unit/reportbooru_service_test.rb +++ b/test/unit/reportbooru_service_test.rb @@ -4,20 +4,20 @@ class ReportbooruServiceTest < ActiveSupport::TestCase def setup @service = ReportbooruService.new(reportbooru_server: "http://localhost:1234") @post = create(:post) - @date = "2000-01-01" + @date = Date.parse("2000-01-01") end context "#popular_posts" do should "return the list of popular posts on success" do - body = "[[#{@post.id},100.0]]" - @service.http.expects(:get).with("http://localhost:1234/post_views/rank?date=#{@date}").returns(HTTP::Response.new(status: 200, body: body, version: "1.1")) + mock_post_view_rankings(@date, [[@post.id, 100]]) posts = @service.popular_posts(@date) assert_equal([@post], posts) end should "return nothing on failure" do - @service.http.expects(:get).with("http://localhost:1234/post_views/rank?date=#{@date}").returns(HTTP::Response.new(status: 500, body: "", version: "1.1")) + Danbooru::Http.any_instance.expects(:get).with("http://localhost:1234/post_views/rank?date=#{@date}").returns(HTTP::Response.new(status: 500, body: "", version: "1.1")) + Danbooru::Http.any_instance.expects(:get).with("http://localhost:1234/post_views/rank?date=#{@date.yesterday}").returns(HTTP::Response.new(status: 500, body: "", version: "1.1")) assert_equal([], @service.popular_posts(@date)) end From 10b7a534499e88c90086c3b0ba53c07dde17e50a Mon Sep 17 00:00:00 2001 From: evazion Date: Fri, 19 Jun 2020 15:09:43 -0500 Subject: [PATCH 18/59] unicorn: increase socket backlog. --- config/unicorn/production.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/unicorn/production.rb b/config/unicorn/production.rb index e53d8a24f..7ac586706 100644 --- a/config/unicorn/production.rb +++ b/config/unicorn/production.rb @@ -6,7 +6,7 @@ worker_processes 20 timeout 180 # listen "127.0.0.1:9000", :tcp_nopush => true -listen "/tmp/.unicorn.sock", :backlog => 512 +listen "/tmp/.unicorn.sock", backlog: 1024 # Spawn unicorn master worker for user apps (group: apps) user 'danbooru', 'danbooru' From 26ad844bbe9be28eead133701f42036f7fd8bb62 Mon Sep 17 00:00:00 2001 From: evazion Date: Mon, 15 Jun 2020 04:41:42 -0500 Subject: [PATCH 19/59] downloads: refactor Downloads::File into Danbooru::Http. Remove the Downloads::File class. Move download methods to Danbooru::Http instead. This means that: * HTTParty has been replaced with http.rb for downloading files. * Downloading is no longer tightly coupled to source strategies. Before Downloads::File tried to automatically look up the source and download the full size image instead if we gave it a sample url. Now we can do plain downloads without source strategies altering the url. * The Cloudflare Polish check has been changed from checking for a Cloudflare IP to checking for the CF-Polished header. Looking up the list of Cloudflare IPs was slow and flaky during testing. * The SSRF protection code has been factored out so it can be used for normal http requests, not just for downloads. * The Webmock gem can be removed, since it was only used for stubbing out certain HTTParty requests in the download tests. The Webmock gem is buggy and caused certain tests to fail during CI. * The retriable gem can be removed, since we no longer autoretry failed downloads. We assume that if a download fails once then retrying probably won't help. --- app/logical/cloudflare_service.rb | 9 -- app/logical/danbooru/http.rb | 66 +++++++++- app/logical/downloads/file.rb | 123 ------------------ app/logical/iqdb_proxy.rb | 7 +- app/logical/sources/strategies/base.rb | 20 ++- .../upload_service/controller_helper.rb | 7 +- app/logical/upload_service/utils.rb | 8 +- app/logical/validating_socket.rb | 27 ++++ test/test_helpers/download_test_helper.rb | 6 +- test/unit/cloudflare_service_test.rb | 6 - test/unit/danbooru_http_test.rb | 48 ++++++- test/unit/downloads/art_station_test.rb | 34 +---- test/unit/downloads/file_test.rb | 81 ------------ test/unit/downloads/pixiv_test.rb | 14 +- test/unit/upload_service_test.rb | 2 +- 15 files changed, 173 insertions(+), 285 deletions(-) delete mode 100644 app/logical/downloads/file.rb create mode 100644 app/logical/validating_socket.rb delete mode 100644 test/unit/downloads/file_test.rb diff --git a/app/logical/cloudflare_service.rb b/app/logical/cloudflare_service.rb index 6a1161502..8cc1be195 100644 --- a/app/logical/cloudflare_service.rb +++ b/app/logical/cloudflare_service.rb @@ -9,15 +9,6 @@ class CloudflareService api_token.present? && zone.present? end - def ips(expiry: 24.hours) - response = Danbooru::Http.cache(expiry).get("https://api.cloudflare.com/client/v4/ips") - return [] if response.code != 200 - - result = response.parse["result"] - ips = result["ipv4_cidrs"] + result["ipv6_cidrs"] - ips.map { |ip| IPAddr.new(ip) } - end - def purge_cache(urls) return unless enabled? diff --git a/app/logical/danbooru/http.rb b/app/logical/danbooru/http.rb index 666a67f2c..b87f5e60c 100644 --- a/app/logical/danbooru/http.rb +++ b/app/logical/danbooru/http.rb @@ -1,18 +1,25 @@ module Danbooru class Http + class DownloadError < StandardError; end + class FileTooLargeError < StandardError; end + DEFAULT_TIMEOUT = 10 MAX_REDIRECTS = 5 - attr_writer :cache, :http + attr_writer :cache, :max_size, :http class << self - delegate :get, :put, :post, :delete, :cache, :follow, :timeout, :auth, :basic_auth, :headers, to: :new + delegate :get, :head, :put, :post, :delete, :cache, :follow, :max_size, :timeout, :auth, :basic_auth, :headers, :public_only, :download_media, to: :new end def get(url, **options) request(:get, url, **options) end + def head(url, **options) + request(:head, url, **options) + end + def put(url, **options) request(:get, url, **options) end @@ -33,6 +40,10 @@ module Danbooru dup.tap { |o| o.http = o.http.follow(*args) } end + def max_size(size) + dup.tap { |o| o.max_size = size } + end + def timeout(*args) dup.tap { |o| o.http = o.http.timeout(*args) } end @@ -49,6 +60,46 @@ module Danbooru dup.tap { |o| o.http = o.http.headers(*args) } end + # allow requests only to public IPs, not to local or private networks. + def public_only + dup.tap do |o| + o.http = o.http.dup.tap do |http| + http.default_options = http.default_options.with_socket_class(ValidatingSocket) + end + end + end + + concerning :DownloadMethods do + def download_media(url, no_polish: true, **options) + url = Addressable::URI.heuristic_parse(url) + response = headers(Referer: url.origin).get(url) + + # prevent Cloudflare Polish from modifying images. + if no_polish && response.headers["CF-Polished"].present? + url.query_values = url.query_values.to_h.merge(danbooru_no_polish: SecureRandom.uuid) + return download_media(url, no_polish: false) + end + + file = download_response(response, **options) + [response, MediaFile.open(file)] + end + + def download_response(response, file: Tempfile.new("danbooru-download-", binmode: true)) + raise DownloadError, response if response.status != 200 + raise FileTooLargeError, response if @max_size && response.content_length.to_i > @max_size + + size = 0 + response.body.each do |chunk| + size += chunk.size + raise FileTooLargeError if @max_size && size > @max_size + file.write(chunk) + end + + file.rewind + file + end + end + protected def request(method, url, **options) @@ -57,11 +108,12 @@ module Danbooru else raw_request(method, url, **options) end + rescue ValidatingSocket::ProhibitedIpError + fake_response(597, "") rescue HTTP::Redirector::TooManyRedirectsError - ::HTTP::Response.new(status: 598, body: "", version: "1.1") + fake_response(598, "") rescue HTTP::TimeoutError - # return a synthetic http error on connection timeouts - ::HTTP::Response.new(status: 599, body: "", version: "1.1") + fake_response(599, "") end def cached_request(method, url, **options) @@ -79,6 +131,10 @@ module Danbooru http.send(method, url, **options) end + def fake_response(status, body) + ::HTTP::Response.new(status: status, version: "1.1", body: ::HTTP::Response::Body.new(body)) + end + def http @http ||= ::HTTP. follow(strict: false, max_hops: MAX_REDIRECTS). diff --git a/app/logical/downloads/file.rb b/app/logical/downloads/file.rb deleted file mode 100644 index 8bbab504b..000000000 --- a/app/logical/downloads/file.rb +++ /dev/null @@ -1,123 +0,0 @@ -require 'resolv' - -module Downloads - class File - include ActiveModel::Validations - class Error < StandardError; end - - RETRIABLE_ERRORS = [Errno::ECONNRESET, Errno::ETIMEDOUT, Errno::EIO, Errno::EHOSTUNREACH, Errno::ECONNREFUSED, Timeout::Error, IOError] - - delegate :data, to: :strategy - attr_reader :url, :referer - - validate :validate_url - - def initialize(url, referer = nil) - @url = Addressable::URI.parse(url) rescue nil - @referer = referer - validate! - end - - def size - res = HTTParty.head(uncached_url, **httparty_options, timeout: 3) - - if res.success? - res.content_length - else - raise HTTParty::ResponseError.new(res) - end - end - - def download!(url: uncached_url, tries: 3, **options) - Retriable.retriable(on: RETRIABLE_ERRORS, tries: tries, base_interval: 0) do - file = http_get_streaming(url, headers: strategy.headers, **options) - return [file, strategy] - end - end - - def validate_url - errors[:base] << "URL must not be blank" if url.blank? - errors[:base] << "'#{url}' is not a valid url" if !url.host.present? - errors[:base] << "'#{url}' is not a valid url. Did you mean 'http://#{url}'?" if !url.scheme.in?(%w[http https]) - end - - def http_get_streaming(url, file: Tempfile.new(binmode: true), headers: {}, max_size: Danbooru.config.max_file_size) - size = 0 - - res = HTTParty.get(url, httparty_options) do |chunk| - next if chunk.code == 302 - - size += chunk.size - raise Error.new("File is too large (max size: #{max_size})") if size > max_size && max_size > 0 - - file.write(chunk) - end - - if res.success? - file.rewind - return file - else - raise Error.new("HTTP error code: #{res.code} #{res.message}") - end - end - - # Prevent Cloudflare from potentially mangling the image. See issue #3528. - def uncached_url - return file_url unless is_cloudflare?(file_url) - - url = file_url.dup - url.query_values = url.query_values.to_h.merge(danbooru_no_cache: SecureRandom.uuid) - url - end - - def preview_url - @preview_url ||= Addressable::URI.parse(strategy.preview_url) - end - - def file_url - @file_url ||= Addressable::URI.parse(strategy.image_url) - end - - def strategy - @strategy ||= Sources::Strategies.find(url.to_s, referer) - end - - def httparty_options - { - timeout: 10, - stream_body: true, - headers: strategy.headers, - connection_adapter: ValidatingConnectionAdapter - }.deep_merge(Danbooru.config.httparty_options) - end - - def is_cloudflare?(url) - ip_addr = IPAddr.new(Resolv.getaddress(url.hostname)) - CloudflareService.new.ips.any? { |subnet| subnet.include?(ip_addr) } - end - - def self.banned_ip?(ip) - ip = IPAddress.parse(ip.to_s) unless ip.is_a?(IPAddress) - - if ip.ipv4? - ip.loopback? || ip.link_local? || ip.multicast? || ip.private? - elsif ip.ipv6? - ip.loopback? || ip.link_local? || ip.unique_local? || ip.unspecified? - end - end - end - - # Hook into HTTParty to validate the IP before following redirects. - # https://www.rubydoc.info/github/jnunemaker/httparty/HTTParty/ConnectionAdapter - class ValidatingConnectionAdapter < HTTParty::ConnectionAdapter - def self.call(uri, options) - ip_addr = IPAddress.parse(::Resolv.getaddress(uri.hostname)) - - if Downloads::File.banned_ip?(ip_addr) - raise Downloads::File::Error, "Downloads from #{ip_addr} are not allowed" - end - - super(uri, options) - end - end -end diff --git a/app/logical/iqdb_proxy.rb b/app/logical/iqdb_proxy.rb index 67e27bd90..41dc5cd63 100644 --- a/app/logical/iqdb_proxy.rb +++ b/app/logical/iqdb_proxy.rb @@ -12,8 +12,9 @@ class IqdbProxy end def download(url, type) - download = Downloads::File.new(url) - file, strategy = download.download!(url: download.send(type)) + strategy = Sources::Strategies.find(url) + download_url = strategy.send(type) + file = strategy.download_file!(download_url) file end @@ -32,7 +33,7 @@ class IqdbProxy file = download(params[:image_url], :url) results = query(file: file, limit: limit) elsif params[:file_url].present? - file = download(params[:file_url], :file_url) + file = download(params[:file_url], :image_url) results = query(file: file, limit: limit) elsif params[:post_id].present? url = Post.find(params[:post_id]).preview_file_url diff --git a/app/logical/sources/strategies/base.rb b/app/logical/sources/strategies/base.rb index 0ee78f365..fcc0ba6e6 100644 --- a/app/logical/sources/strategies/base.rb +++ b/app/logical/sources/strategies/base.rb @@ -14,6 +14,8 @@ module Sources module Strategies class Base + class DownloadError < StandardError; end + attr_reader :url, :referer_url, :urls, :parsed_url, :parsed_referer, :parsed_urls extend Memoist @@ -35,8 +37,8 @@ module Sources # referrer_url so the strategy can discover the HTML # page and other information. def initialize(url, referer_url = nil) - @url = url - @referer_url = referer_url + @url = url.to_s + @referer_url = referer_url&.to_s @urls = [url, referer_url].select(&:present?) @parsed_url = Addressable::URI.heuristic_parse(url) rescue nil @@ -144,10 +146,22 @@ module Sources # Returns the size of the image resource without actually downloading the file. def size - Downloads::File.new(image_url).size + http.head(image_url).content_length.to_i end memoize :size + # Download the file at the given url, or at the main image url by default. + def download_file!(download_url = image_url) + response, file = http.download_media(download_url) + raise DownloadError, "Download failed: #{download_url} returned error #{response.status}" if response.status != 200 + file + end + + def http + Danbooru::Http.public_only.timeout(30).max_size(Danbooru.config.max_file_size) + end + memoize :http + # The url to use for artist finding purposes. This will be stored in the # artist entry. Normally this will be the profile url. def normalize_for_artist_finder diff --git a/app/logical/upload_service/controller_helper.rb b/app/logical/upload_service/controller_helper.rb index 7342168fd..9eeb6d442 100644 --- a/app/logical/upload_service/controller_helper.rb +++ b/app/logical/upload_service/controller_helper.rb @@ -7,11 +7,8 @@ class UploadService # this gets called from UploadsController#new so we need to preprocess async UploadPreprocessorDelayedStartJob.perform_later(url, ref, CurrentUser.user) - begin - download = Downloads::File.new(url, ref) - remote_size = download.size - rescue Exception - end + strategy = Sources::Strategies.find(url, ref) + remote_size = strategy.size return [upload, remote_size] end diff --git a/app/logical/upload_service/utils.rb b/app/logical/upload_service/utils.rb index 7482d43b6..e4316cd90 100644 --- a/app/logical/upload_service/utils.rb +++ b/app/logical/upload_service/utils.rb @@ -71,13 +71,13 @@ class UploadService return file if file.present? raise "No file or source URL provided" if upload.source_url.blank? - download = Downloads::File.new(upload.source_url, upload.referer_url) - file, strategy = download.download! + strategy = Sources::Strategies.find(upload.source_url, upload.referer_url) + file = strategy.download_file! - if download.data[:ugoira_frame_data].present? + if strategy.data[:ugoira_frame_data].present? upload.context = { "ugoira" => { - "frame_data" => download.data[:ugoira_frame_data], + "frame_data" => strategy.data[:ugoira_frame_data], "content_type" => "image/jpeg" } } diff --git a/app/logical/validating_socket.rb b/app/logical/validating_socket.rb new file mode 100644 index 000000000..446609073 --- /dev/null +++ b/app/logical/validating_socket.rb @@ -0,0 +1,27 @@ +# A TCPSocket wrapper that disallows connections to local or private IPs. Used for SSRF protection. +# https://owasp.org/www-community/attacks/Server_Side_Request_Forgery + +require "resolv" + +class ValidatingSocket < TCPSocket + class ProhibitedIpError < StandardError; end + + def initialize(hostname, port) + ip = validate_hostname!(hostname) + super(ip, port) + end + + def validate_hostname!(hostname) + ip = IPAddress.parse(::Resolv.getaddress(hostname)) + raise ProhibitedIpError, "Connection to #{hostname} failed; #{ip} is a prohibited IP" if prohibited_ip?(ip) + ip.to_s + end + + def prohibited_ip?(ip) + if ip.ipv4? + ip.loopback? || ip.link_local? || ip.multicast? || ip.private? + elsif ip.ipv6? + ip.loopback? || ip.link_local? || ip.unique_local? || ip.unspecified? + end + end +end diff --git a/test/test_helpers/download_test_helper.rb b/test/test_helpers/download_test_helper.rb index 72805cf9b..eea711588 100644 --- a/test/test_helpers/download_test_helper.rb +++ b/test/test_helpers/download_test_helper.rb @@ -1,8 +1,8 @@ module DownloadTestHelper def assert_downloaded(expected_filesize, source, referer = nil) - download = Downloads::File.new(source, referer) - tempfile, strategy = download.download! - assert_equal(expected_filesize, tempfile.size, "Tested source URL: #{source}") + strategy = Sources::Strategies.find(source, referer) + file = strategy.download_file! + assert_equal(expected_filesize, file.size, "Tested source URL: #{source}") rescue Net::OpenTimeout skip "Remote connection to #{source} failed" end diff --git a/test/unit/cloudflare_service_test.rb b/test/unit/cloudflare_service_test.rb index faced9207..146068a6d 100644 --- a/test/unit/cloudflare_service_test.rb +++ b/test/unit/cloudflare_service_test.rb @@ -14,10 +14,4 @@ class CloudflareServiceTest < ActiveSupport::TestCase assert_requested(:delete, "https://api.cloudflare.com/client/v4/zones/123/purge_cache", times: 1) end end - - context "#ips" do - should "work" do - refute_empty(@cloudflare.ips) - end - end end diff --git a/test/unit/danbooru_http_test.rb b/test/unit/danbooru_http_test.rb index 42e585bef..f7529491b 100644 --- a/test/unit/danbooru_http_test.rb +++ b/test/unit/danbooru_http_test.rb @@ -4,7 +4,7 @@ class DanbooruHttpTest < ActiveSupport::TestCase context "Danbooru::Http" do context "#get method" do should "work for all basic methods" do - %i[get put post delete].each do |method| + %i[get head put post delete].each do |method| response = Danbooru::Http.send(method, "https://httpbin.org/status/200") assert_equal(200, response.status) end @@ -26,8 +26,10 @@ class DanbooruHttpTest < ActiveSupport::TestCase end should "fail if the request takes too long to download" do - response = Danbooru::Http.timeout(1).get("https://httpbin.org/drip?duration=5&numbytes=5") - assert_equal(599, response.status) + # XXX should return status 599 instead + assert_raises(HTTP::TimeoutError) do + response = Danbooru::Http.timeout(1).get("https://httpbin.org/drip?duration=10&numbytes=10").flush + end end should "automatically decompress gzipped responses" do @@ -45,5 +47,45 @@ class DanbooruHttpTest < ActiveSupport::TestCase assert_equal(response2.body, response1.body) end end + + context "#download method" do + should "download files" do + response, file = Danbooru::Http.download_media("https://httpbin.org/bytes/1000") + + assert_equal(200, response.status) + assert_equal(1000, file.size) + end + + should "follow redirects when downloading files" do + response, file = Danbooru::Http.download_media("https://httpbin.org/redirect-to?url=https://httpbin.org/bytes/1000") + + assert_equal(200, response.status) + assert_equal(1000, file.size) + end + + should "fail if the url points to a private IP" do + assert_raises(Danbooru::Http::DownloadError) do + Danbooru::Http.public_only.download_media("https://127.0.0.1.xip.io") + end + end + + should "fail if the url redirects to a private IP" do + assert_raises(Danbooru::Http::DownloadError) do + Danbooru::Http.public_only.download_media("https://httpbin.org/redirect-to?url=https://127.0.0.1.xip.io") + end + end + + should "fail if a download is too large" do + assert_raises(Danbooru::Http::FileTooLargeError) do + response, file = Danbooru::Http.max_size(500).download_media("https://httpbin.org/bytes/1000") + end + end + + should "fail if a streaming download is too large" do + assert_raises(Danbooru::Http::FileTooLargeError) do + response, file = Danbooru::Http.max_size(500).download_media("https://httpbin.org/stream-bytes/1000") + end + end + end end end diff --git a/test/unit/downloads/art_station_test.rb b/test/unit/downloads/art_station_test.rb index 0ebacd585..83b1710ca 100644 --- a/test/unit/downloads/art_station_test.rb +++ b/test/unit/downloads/art_station_test.rb @@ -3,53 +3,27 @@ require 'test_helper' module Downloads class ArtStationTest < ActiveSupport::TestCase context "a download for a (small) artstation image" do - setup do - @asset = "https://cdnb3.artstation.com/p/assets/images/images/003/716/071/small/aoi-ogata-hate-city.jpg?1476754974" - @download = Downloads::File.new(@asset) - end - should "download the /4k/ image instead" do - file, strategy = @download.download! - assert_equal(1_880_910, ::File.size(file.path)) + assert_downloaded(1_880_910, "https://cdnb3.artstation.com/p/assets/images/images/003/716/071/small/aoi-ogata-hate-city.jpg?1476754974") end end context "for an image where an original does not exist" do - setup do - @asset = "https://cdna.artstation.com/p/assets/images/images/004/730/278/large/mendel-oh-dragonll.jpg" - @download = Downloads::File.new(@asset) - end - should "not try to download the original" do - file, strategy = @download.download! - assert_equal(483_192, ::File.size(file.path)) + assert_downloaded(483_192, "https://cdna.artstation.com/p/assets/images/images/004/730/278/large/mendel-oh-dragonll.jpg") end end context "a download for an ArtStation image hosted on CloudFlare" do - setup do - @asset = "https://cdnb.artstation.com/p/assets/images/images/003/716/071/large/aoi-ogata-hate-city.jpg?1476754974" - end - should "return the original file, not the polished file" do + @asset = "https://cdnb.artstation.com/p/assets/images/images/003/716/071/large/aoi-ogata-hate-city.jpg?1476754974" assert_downloaded(1_880_910, @asset) end - - should "return the original filesize, not the polished filesize" do - assert_equal(1_880_910, Downloads::File.new(@asset).size) - end end context "a download for a https://$artist.artstation.com/projects/$id page" do - setup do - @source = "https://dantewontdie.artstation.com/projects/YZK5q" - @download = Downloads::File.new(@source) - end - should "download the original image instead" do - file, strategy = @download.download! - - assert_equal(247_350, ::File.size(file.path)) + assert_downloaded(247_350, "https://dantewontdie.artstation.com/projects/YZK5q") end end end diff --git a/test/unit/downloads/file_test.rb b/test/unit/downloads/file_test.rb deleted file mode 100644 index 7e37787dd..000000000 --- a/test/unit/downloads/file_test.rb +++ /dev/null @@ -1,81 +0,0 @@ -require 'test_helper' - -module Downloads - class FileTest < ActiveSupport::TestCase - context "A post download" do - setup do - @source = "http://www.google.com/intl/en_ALL/images/logo.gif" - @download = Downloads::File.new(@source) - end - - context "for a banned IP" do - setup do - Resolv.expects(:getaddress).returns("127.0.0.1").at_least_once - end - - should "not try to download the file" do - assert_raise(Downloads::File::Error) { Downloads::File.new("http://evil.com").download! } - end - - should "not try to fetch the size" do - assert_raise(Downloads::File::Error) { Downloads::File.new("http://evil.com").size } - end - - should "not follow redirects to banned IPs" do - url = "http://httpbin.org/redirect-to?url=http://127.0.0.1" - stub_request(:get, url).to_return(status: 301, headers: { "Location": "http://127.0.0.1" }) - - assert_raise(Downloads::File::Error) { Downloads::File.new(url).download! } - end - - should "not follow redirects that resolve to a banned IP" do - url = "http://httpbin.org/redirect-to?url=http://127.0.0.1.nip.io" - stub_request(:get, url).to_return(status: 301, headers: { "Location": "http://127.0.0.1.xip.io" }) - - assert_raise(Downloads::File::Error) { Downloads::File.new(url).download! } - end - end - - context "that fails" do - should "retry three times before giving up" do - HTTParty.expects(:get).times(3).raises(Errno::ETIMEDOUT) - assert_raises(Errno::ETIMEDOUT) { @download.download! } - end - - should "return an uncorrupted file on the second try" do - bomb = stub("bomb") - bomb.stubs(:code).raises(IOError) - resp = stub("resp", success?: true) - - chunk = stub("a") - chunk.stubs(:code).returns(200) - chunk.stubs(:size).returns(1) - chunk.stubs(:to_s).returns("a") - - HTTParty.expects(:get).twice.multiple_yields(chunk, bomb).then.multiple_yields(chunk, chunk).returns(resp) - @download.stubs(:is_cloudflare?).returns(false) - tempfile, _strategy = @download.download! - - assert_equal("aa", tempfile.read) - end - end - - should "throw an exception when the file is larger than the maximum" do - assert_raise(Downloads::File::Error) do - @download.download!(max_size: 1) - end - end - - should "store the file in the tempfile path" do - tempfile, strategy = @download.download! - assert_operator(tempfile.size, :>, 0, "should have data") - end - - should "correctly save the file when following 302 redirects" do - download = Downloads::File.new("https://yande.re/post/show/578014") - file, strategy = download.download!(url: download.preview_url) - assert_equal(19134, file.size) - end - end - end -end diff --git a/test/unit/downloads/pixiv_test.rb b/test/unit/downloads/pixiv_test.rb index a22f8a700..ddc32e14d 100644 --- a/test/unit/downloads/pixiv_test.rb +++ b/test/unit/downloads/pixiv_test.rb @@ -136,18 +136,14 @@ module Downloads end context "An ugoira site for pixiv" do - setup do - @download = Downloads::File.new("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364") - @tempfile, strategy = @download.download! - @tempfile.close! - end - should "capture the data" do - assert_equal(2, @download.data[:ugoira_frame_data].size) - if @download.data[:ugoira_frame_data][0]["file"] + @strategy = Sources::Strategies.find("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364") + + assert_equal(2, @strategy.data[:ugoira_frame_data].size) + if @strategy.data[:ugoira_frame_data][0]["file"] assert_equal([{"file" => "000000.jpg", "delay" => 125}, {"file" => "000001.jpg", "delay" => 125}], @download.data[:ugoira_frame_data]) else - assert_equal([{"delay_msec" => 125}, {"delay_msec" => 125}], @download.data[:ugoira_frame_data]) + assert_equal([{"delay_msec" => 125}, {"delay_msec" => 125}], @strategy.data[:ugoira_frame_data]) end end end diff --git a/test/unit/upload_service_test.rb b/test/unit/upload_service_test.rb index b6b7bbf15..b15a399a5 100644 --- a/test/unit/upload_service_test.rb +++ b/test/unit/upload_service_test.rb @@ -212,7 +212,7 @@ class UploadServiceTest < ActiveSupport::TestCase context "on timeout errors" do setup do @source = "https://cdn.donmai.us/original/93/f4/93f4dd66ef1eb11a89e56d31f9adc8d0.jpg" - HTTParty.stubs(:get).raises(Net::ReadTimeout) + Danbooru::Http.any_instance.stubs(:get).raises(HTTP::TimeoutError) end should "leave the upload in an error state" do From f730951e7f454ee22720665d1d8ab5e493b30ad2 Mon Sep 17 00:00:00 2001 From: evazion Date: Mon, 15 Jun 2020 13:26:20 -0500 Subject: [PATCH 20/59] gems: drop webmock. --- Gemfile | 1 - Gemfile.lock | 8 -------- test/test_helper.rb | 1 - test/test_helpers/reportbooru_helper.rb | 4 ++-- test/unit/cloudflare_service_test.rb | 8 ++++---- 5 files changed, 6 insertions(+), 16 deletions(-) diff --git a/Gemfile b/Gemfile index 4304f26cd..70656ec6b 100644 --- a/Gemfile +++ b/Gemfile @@ -84,7 +84,6 @@ group :test do gem "mocha", require: "mocha/minitest" gem "ffaker" gem "simplecov", "~> 0.17.0", require: false - gem "webmock", require: "webmock/minitest" gem "minitest-ci" gem "minitest-reporters", require: "minitest/reporters" gem "mock_redis" diff --git a/Gemfile.lock b/Gemfile.lock index 7933ca1f5..222928350 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -123,8 +123,6 @@ GEM coderay (1.1.3) concurrent-ruby (1.1.6) connection_pool (2.2.3) - crack (0.4.3) - safe_yaml (~> 1.0.0) crass (1.0.6) daemons (1.3.1) delayed_job (4.1.8) @@ -156,7 +154,6 @@ GEM ffi (~> 1.0) globalid (0.4.2) activesupport (>= 4.2.0) - hashdiff (1.0.1) http (4.4.1) addressable (~> 2.3) http-cookie (~> 1.0) @@ -402,10 +399,6 @@ GEM unicorn-worker-killer (0.4.4) get_process_mem (~> 0) unicorn (>= 4, < 6) - webmock (3.8.3) - addressable (>= 2.3.6) - crack (>= 0.3.2) - hashdiff (>= 0.4.0, < 2.0.0) webpacker (5.1.1) activesupport (>= 5.2) rack-proxy (>= 0.6.1) @@ -497,7 +490,6 @@ DEPENDENCIES stripe unicorn unicorn-worker-killer - webmock webpacker (>= 4.0.x) whenever diff --git a/test/test_helper.rb b/test/test_helper.rb index 7832ec36c..1d81bdb12 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -43,7 +43,6 @@ class ActiveSupport::TestCase setup do Socket.stubs(:gethostname).returns("www.example.com") - WebMock.allow_net_connect! @temp_dir = Dir.mktmpdir("danbooru-temp-") storage_manager = StorageManager::Local.new(base_dir: @temp_dir) diff --git a/test/test_helpers/reportbooru_helper.rb b/test/test_helpers/reportbooru_helper.rb index 6a99dc5d4..3ec926202 100644 --- a/test/test_helpers/reportbooru_helper.rb +++ b/test/test_helpers/reportbooru_helper.rb @@ -1,7 +1,7 @@ module ReportbooruHelper - def mock_request(url, method: :get, status: 200, body: nil, http: Danbooru::Http.any_instance) + def mock_request(url, method: :get, status: 200, body: nil, http: Danbooru::Http.any_instance, **options) response = HTTP::Response.new(status: status, body: body, version: "1.1") - http.stubs(method).with(url).returns(response) + http.stubs(method).with(url, **options).returns(response) end def mock_post_search_rankings(date = Date.today, rankings) diff --git a/test/unit/cloudflare_service_test.rb b/test/unit/cloudflare_service_test.rb index 146068a6d..cb9bc86be 100644 --- a/test/unit/cloudflare_service_test.rb +++ b/test/unit/cloudflare_service_test.rb @@ -1,5 +1,4 @@ require 'test_helper' -require 'webmock/minitest' class CloudflareServiceTest < ActiveSupport::TestCase def setup @@ -8,10 +7,11 @@ class CloudflareServiceTest < ActiveSupport::TestCase context "#purge_cache" do should "make calls to cloudflare's api" do - stub_request(:any, "api.cloudflare.com") - @cloudflare.purge_cache(["http://localhost/file.txt"]) + url = "http://www.example.com/file.jpg" + mock_request("https://api.cloudflare.com/client/v4/zones/123/purge_cache", method: :delete, json: { files: [url] }) - assert_requested(:delete, "https://api.cloudflare.com/client/v4/zones/123/purge_cache", times: 1) + response = @cloudflare.purge_cache([url]) + assert_equal(200, response.status) end end end From 11d0539ea17d6b63a36f2164a3f5075adf6daf10 Mon Sep 17 00:00:00 2001 From: BrokenEagle Date: Sat, 20 Jun 2020 05:29:16 +0000 Subject: [PATCH 21/59] Prevent creators from voting on their own BURs --- app/policies/forum_post_policy.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/policies/forum_post_policy.rb b/app/policies/forum_post_policy.rb index 14e315e9c..546991f09 100644 --- a/app/policies/forum_post_policy.rb +++ b/app/policies/forum_post_policy.rb @@ -24,7 +24,7 @@ class ForumPostPolicy < ApplicationPolicy end def votable? - unbanned? && show? && record.bulk_update_request.present? && record.bulk_update_request.is_pending? + unbanned? && show? && record.bulk_update_request.present? && record.bulk_update_request.is_pending? && record.bulk_update_request.user_id != user.id end def reportable? From 33577e2186944cc9d58362ca546bef2a4816bf10 Mon Sep 17 00:00:00 2001 From: BrokenEagle Date: Sat, 20 Jun 2020 08:53:19 +0000 Subject: [PATCH 22/59] Don't display artist wiki when it is deleted, and vice-versa --- app/views/artists/_show.html.erb | 2 +- app/views/wiki_pages/show.html.erb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/views/artists/_show.html.erb b/app/views/artists/_show.html.erb index 3e7a65bec..5352fc333 100644 --- a/app/views/artists/_show.html.erb +++ b/app/views/artists/_show.html.erb @@ -8,7 +8,7 @@ <% if @artist.is_banned? && !policy(@artist).can_view_banned? %>

The artist requested removal of this page.

<% else %> - <% if @artist.wiki_page.present? %> + <% if @artist.wiki_page.present? && !@artist.wiki_page.is_deleted %>
<%= format_text(@artist.wiki_page.body, :disable_mentions => true) %>
diff --git a/app/views/wiki_pages/show.html.erb b/app/views/wiki_pages/show.html.erb index de2863565..9596e045b 100644 --- a/app/views/wiki_pages/show.html.erb +++ b/app/views/wiki_pages/show.html.erb @@ -28,7 +28,7 @@ <%= format_text(@wiki_page.body) %> <% end %> - <% if @wiki_page.artist %> + <% if @wiki_page.artist.present? && !@wiki_page.artist.is_deleted %>

<%= link_to "View artist", @wiki_page.artist %>

<% end %> From 79a59e52ece855b2d636894376d537a8a744340b Mon Sep 17 00:00:00 2001 From: nonamethanks Date: Sat, 20 Jun 2020 15:08:49 +0200 Subject: [PATCH 23/59] Twitter: don't get api without a status id --- app/logical/sources/strategies/twitter.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/logical/sources/strategies/twitter.rb b/app/logical/sources/strategies/twitter.rb index a51e162ea..503a96df0 100644 --- a/app/logical/sources/strategies/twitter.rb +++ b/app/logical/sources/strategies/twitter.rb @@ -200,7 +200,7 @@ module Sources::Strategies end def api_response - return {} unless self.class.enabled? + return {} unless self.class.enabled? && status_id.present? api_client.status(status_id) end From a929f3134e97c502cc09b37696255d60ddec8f97 Mon Sep 17 00:00:00 2001 From: evazion Date: Sat, 20 Jun 2020 16:45:50 -0500 Subject: [PATCH 24/59] danbooru::http: parse html responses. --- app/logical/danbooru/http.rb | 3 +++ app/logical/danbooru/http/html_adapter.rb | 12 ++++++++++++ app/logical/danbooru/http/xml_adapter.rb | 12 ++++++++++++ test/unit/danbooru_http_test.rb | 13 +++++++++++++ 4 files changed, 40 insertions(+) create mode 100644 app/logical/danbooru/http/html_adapter.rb create mode 100644 app/logical/danbooru/http/xml_adapter.rb diff --git a/app/logical/danbooru/http.rb b/app/logical/danbooru/http.rb index b87f5e60c..d6bad229c 100644 --- a/app/logical/danbooru/http.rb +++ b/app/logical/danbooru/http.rb @@ -1,3 +1,6 @@ +require "danbooru/http/html_adapter" +require "danbooru/http/xml_adapter" + module Danbooru class Http class DownloadError < StandardError; end diff --git a/app/logical/danbooru/http/html_adapter.rb b/app/logical/danbooru/http/html_adapter.rb new file mode 100644 index 000000000..733208e17 --- /dev/null +++ b/app/logical/danbooru/http/html_adapter.rb @@ -0,0 +1,12 @@ +module Danbooru + class Http + class HtmlAdapter < HTTP::MimeType::Adapter + HTTP::MimeType.register_adapter "text/html", self + HTTP::MimeType.register_alias "text/html", :html + + def decode(str) + Nokogiri::HTML5(str) + end + end + end +end diff --git a/app/logical/danbooru/http/xml_adapter.rb b/app/logical/danbooru/http/xml_adapter.rb new file mode 100644 index 000000000..70c901c07 --- /dev/null +++ b/app/logical/danbooru/http/xml_adapter.rb @@ -0,0 +1,12 @@ +module Danbooru + class Http + class XmlAdapter < HTTP::MimeType::Adapter + HTTP::MimeType.register_adapter "application/xml", self + HTTP::MimeType.register_alias "application/xml", :xml + + def decode(str) + Hash.from_xml(str).with_indifferent_access + end + end + end +end diff --git a/test/unit/danbooru_http_test.rb b/test/unit/danbooru_http_test.rb index f7529491b..50fc67d32 100644 --- a/test/unit/danbooru_http_test.rb +++ b/test/unit/danbooru_http_test.rb @@ -38,6 +38,19 @@ class DanbooruHttpTest < ActiveSupport::TestCase assert_equal(true, response.parse["gzipped"]) end + should "automatically parse html responses" do + response = Danbooru::Http.get("https://httpbin.org/html") + assert_equal(200, response.status) + assert_instance_of(Nokogiri::HTML5::Document, response.parse) + assert_equal("Herman Melville - Moby-Dick", response.parse.css("h1").text) + end + + should "automatically parse xml responses" do + response = Danbooru::Http.get("https://httpbin.org/xml") + assert_equal(200, response.status) + assert_equal(true, response.parse[:slideshow].present?) + end + should "cache requests" do response1 = Danbooru::Http.cache(1.minute).get("https://httpbin.org/uuid") assert_equal(200, response1.status) From 87ed882234ea4639d07614eee202f432631af715 Mon Sep 17 00:00:00 2001 From: evazion Date: Sat, 20 Jun 2020 22:38:58 -0500 Subject: [PATCH 25/59] danbooru::http: support automatically retrying 429 errors. --- app/logical/danbooru/http.rb | 14 ++--- .../danbooru/http/application_client.rb | 31 +++++++++++ app/logical/danbooru/http/retriable.rb | 52 +++++++++++++++++++ test/unit/danbooru_http_test.rb | 29 +++++++++++ 4 files changed, 120 insertions(+), 6 deletions(-) create mode 100644 app/logical/danbooru/http/application_client.rb create mode 100644 app/logical/danbooru/http/retriable.rb diff --git a/app/logical/danbooru/http.rb b/app/logical/danbooru/http.rb index d6bad229c..c57f85f0b 100644 --- a/app/logical/danbooru/http.rb +++ b/app/logical/danbooru/http.rb @@ -1,5 +1,6 @@ require "danbooru/http/html_adapter" require "danbooru/http/xml_adapter" +require "danbooru/http/retriable" module Danbooru class Http @@ -139,12 +140,13 @@ module Danbooru end def http - @http ||= ::HTTP. - follow(strict: false, max_hops: MAX_REDIRECTS). - timeout(DEFAULT_TIMEOUT). - use(:auto_inflate). - headers(Danbooru.config.http_headers). - headers("Accept-Encoding" => "gzip") + @http ||= + ::Danbooru::Http::ApplicationClient.new + .follow(strict: false, max_hops: MAX_REDIRECTS) + .timeout(DEFAULT_TIMEOUT) + .use(:auto_inflate) + .headers(Danbooru.config.http_headers) + .headers("Accept-Encoding" => "gzip") end end end diff --git a/app/logical/danbooru/http/application_client.rb b/app/logical/danbooru/http/application_client.rb new file mode 100644 index 000000000..27ac6a6d1 --- /dev/null +++ b/app/logical/danbooru/http/application_client.rb @@ -0,0 +1,31 @@ +# An extension to HTTP::Client that lets us write Rack-style middlewares that +# hook into the request/response cycle and override how requests are made. This +# works by extending http.rb's concept of features (HTTP::Feature) to give them +# a `perform` method that takes a http request and returns a http response. +# This can be used to intercept and modify requests and return arbitrary responses. + +module Danbooru + class Http + class ApplicationClient < HTTP::Client + # Override `perform` to call the `perform` method on features first. + def perform(request, options) + features = options.features.values.reverse.select do |feature| + feature.respond_to?(:perform) + end + + perform = proc { |req| super(req, options) } + callback_chain = features.reduce(perform) do |callback_chain, feature| + proc { |req| feature.perform(req, &callback_chain) } + end + + callback_chain.call(request) + end + + # Override `branch` to return an ApplicationClient instead of a + # HTTP::Client so that chaining works. + def branch(...) + ApplicationClient.new(...) + end + end + end +end diff --git a/app/logical/danbooru/http/retriable.rb b/app/logical/danbooru/http/retriable.rb new file mode 100644 index 000000000..681dd8bbe --- /dev/null +++ b/app/logical/danbooru/http/retriable.rb @@ -0,0 +1,52 @@ +# A HTTP::Feature that automatically retries requests that return a 429 error +# or a Retry-After header. Usage: `Danbooru::Http.use(:retriable).get(url)`. +# +# https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/429 +# https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After + +module Danbooru + class Http + class Retriable < HTTP::Feature + HTTP::Options.register_feature :retriable, self + + attr_reader :max_retries, :max_delay + + def initialize(max_retries: 2, max_delay: 5.seconds) + @max_retries = max_retries + @max_delay = max_delay + end + + def perform(request, &block) + response = yield request + + retries = max_retries + while retriable?(response) && retries > 0 && retry_delay(response) <= max_delay + retries -= 1 + sleep(retry_delay(response)) + response = yield request + end + + response + end + + def retriable?(response) + response.status == 429 || response.headers["Retry-After"].present? + end + + def retry_delay(response, current_time: Time.zone.now) + retry_after = response.headers["Retry-After"] + + if retry_after.blank? + 0.seconds + elsif retry_after =~ /\A\d+\z/ + retry_after.to_i.seconds + else + retry_at = Time.zone.parse(retry_after) + return 0.seconds if retry_at.blank? + + [retry_at - current_time, 0].max.seconds + end + end + end + end +end diff --git a/test/unit/danbooru_http_test.rb b/test/unit/danbooru_http_test.rb index 50fc67d32..ae347bfd8 100644 --- a/test/unit/danbooru_http_test.rb +++ b/test/unit/danbooru_http_test.rb @@ -61,6 +61,35 @@ class DanbooruHttpTest < ActiveSupport::TestCase end end + context "retriable feature" do + should "retry immediately if no Retry-After header is sent" do + response_429 = ::HTTP::Response.new(status: 429, version: "1.1", body: "") + response_200 = ::HTTP::Response.new(status: 200, version: "1.1", body: "") + HTTP::Client.any_instance.expects(:perform).times(2).returns(response_429, response_200) + + response = Danbooru::Http.use(:retriable).get("https://httpbin.org/status/429") + assert_equal(200, response.status) + end + + should "retry if the Retry-After header is an integer" do + response_503 = ::HTTP::Response.new(status: 503, version: "1.1", headers: { "Retry-After": "1" }, body: "") + response_200 = ::HTTP::Response.new(status: 200, version: "1.1", body: "") + HTTP::Client.any_instance.expects(:perform).times(2).returns(response_503, response_200) + + response = Danbooru::Http.use(:retriable).get("https://httpbin.org/status/503") + assert_equal(200, response.status) + end + + should "retry if the Retry-After header is a date" do + response_503 = ::HTTP::Response.new(status: 503, version: "1.1", headers: { "Retry-After": 2.seconds.from_now.httpdate }, body: "") + response_200 = ::HTTP::Response.new(status: 200, version: "1.1", body: "") + HTTP::Client.any_instance.expects(:perform).times(2).returns(response_503, response_200) + + response = Danbooru::Http.use(:retriable).get("https://httpbin.org/status/503") + assert_equal(200, response.status) + end + end + context "#download method" do should "download files" do response, file = Danbooru::Http.download_media("https://httpbin.org/bytes/1000") From 71b0bc6c0f03b714bbd4cc9ae73a94a878431971 Mon Sep 17 00:00:00 2001 From: evazion Date: Sat, 20 Jun 2020 16:48:00 -0500 Subject: [PATCH 26/59] danbooru::http: support tracking cookies between requests. Allow cookies to be saved and sent back when making several requests in a row. Usage: http = Danbooru::Http.use(:session) # saves the foo=42 cookie sent by the response. http.get("https://httpbin.org/cookies/set/foo/42") # sends back the foo=42 cookie from the previous request. http.get("https://httpbin.org/cookies") --- app/logical/danbooru/http.rb | 12 ++++++++- app/logical/danbooru/http/session.rb | 37 ++++++++++++++++++++++++++++ test/unit/danbooru_http_test.rb | 12 +++++++++ 3 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 app/logical/danbooru/http/session.rb diff --git a/app/logical/danbooru/http.rb b/app/logical/danbooru/http.rb index c57f85f0b..8fe06a0d1 100644 --- a/app/logical/danbooru/http.rb +++ b/app/logical/danbooru/http.rb @@ -1,6 +1,7 @@ require "danbooru/http/html_adapter" require "danbooru/http/xml_adapter" require "danbooru/http/retriable" +require "danbooru/http/session" module Danbooru class Http @@ -13,7 +14,7 @@ module Danbooru attr_writer :cache, :max_size, :http class << self - delegate :get, :head, :put, :post, :delete, :cache, :follow, :max_size, :timeout, :auth, :basic_auth, :headers, :public_only, :download_media, to: :new + delegate :get, :head, :put, :post, :delete, :cache, :follow, :max_size, :timeout, :auth, :basic_auth, :headers, :cookies, :use, :public_only, :download_media, to: :new end def get(url, **options) @@ -64,6 +65,14 @@ module Danbooru dup.tap { |o| o.http = o.http.headers(*args) } end + def cookies(*args) + dup.tap { |o| o.http = o.http.cookies(*args) } + end + + def use(*args) + dup.tap { |o| o.http = o.http.use(*args) } + end + # allow requests only to public IPs, not to local or private networks. def public_only dup.tap do |o| @@ -143,6 +152,7 @@ module Danbooru @http ||= ::Danbooru::Http::ApplicationClient.new .follow(strict: false, max_hops: MAX_REDIRECTS) + .use(:session) .timeout(DEFAULT_TIMEOUT) .use(:auto_inflate) .headers(Danbooru.config.http_headers) diff --git a/app/logical/danbooru/http/session.rb b/app/logical/danbooru/http/session.rb new file mode 100644 index 000000000..3bb532928 --- /dev/null +++ b/app/logical/danbooru/http/session.rb @@ -0,0 +1,37 @@ +module Danbooru + class Http + class Session < HTTP::Feature + HTTP::Options.register_feature :session, self + + attr_reader :cookie_jar + + def initialize(cookie_jar: HTTP::CookieJar.new) + @cookie_jar = cookie_jar + end + + def perform(request) + add_cookies(request) + response = yield request + save_cookies(response) + response + end + + def add_cookies(request) + cookies = cookies_for_request(request) + request.headers["Cookie"] = cookies if cookies.present? + end + + def cookies_for_request(request) + saved_cookies = cookie_jar.each(request.uri).map { |c| [c.name, c.value] }.to_h + request_cookies = HTTP::Cookie.cookie_value_to_hash(request.headers["Cookie"].to_s) + saved_cookies.merge(request_cookies).map { |name, value| "#{name}=#{value}" }.join("; ") + end + + def save_cookies(response) + response.cookies.each do |cookie| + cookie_jar.add(cookie) + end + end + end + end +end diff --git a/test/unit/danbooru_http_test.rb b/test/unit/danbooru_http_test.rb index ae347bfd8..b6ef81e7c 100644 --- a/test/unit/danbooru_http_test.rb +++ b/test/unit/danbooru_http_test.rb @@ -51,6 +51,18 @@ class DanbooruHttpTest < ActiveSupport::TestCase assert_equal(true, response.parse[:slideshow].present?) end + should "track cookies between requests" do + http = Danbooru::Http.use(:session) + + resp1 = http.get("https://httpbin.org/cookies/set/abc/1") + resp2 = http.get("https://httpbin.org/cookies/set/def/2") + resp3 = http.get("https://httpbin.org/cookies") + assert_equal({ abc: "1", def: "2" }, resp3.parse["cookies"].symbolize_keys) + + resp4 = http.cookies(def: 3, ghi: 4).get("https://httpbin.org/cookies") + assert_equal({ abc: "1", def: "3", ghi: "4" }, resp4.parse["cookies"].symbolize_keys) + end + should "cache requests" do response1 = Danbooru::Http.cache(1.minute).get("https://httpbin.org/uuid") assert_equal(200, response1.status) From 05d7355ebbfa7ef8139578c8359827abea832f2e Mon Sep 17 00:00:00 2001 From: evazion Date: Sun, 21 Jun 2020 03:57:14 -0500 Subject: [PATCH 27/59] danbooru::http: support automatically following redirects. Replace http.rb's builtin redirect following option with our own redirect follower. This fixes an issue with http.rb losing cookies after following a redirect. --- app/logical/danbooru/http.rb | 26 ++++++++-------- app/logical/danbooru/http/redirector.rb | 40 +++++++++++++++++++++++++ app/logical/nico_seiga_api_client.rb | 5 ++-- 3 files changed, 57 insertions(+), 14 deletions(-) create mode 100644 app/logical/danbooru/http/redirector.rb diff --git a/app/logical/danbooru/http.rb b/app/logical/danbooru/http.rb index 8fe06a0d1..dfe6558f8 100644 --- a/app/logical/danbooru/http.rb +++ b/app/logical/danbooru/http.rb @@ -1,5 +1,6 @@ require "danbooru/http/html_adapter" require "danbooru/http/xml_adapter" +require "danbooru/http/redirector" require "danbooru/http/retriable" require "danbooru/http/session" @@ -11,12 +12,24 @@ module Danbooru DEFAULT_TIMEOUT = 10 MAX_REDIRECTS = 5 - attr_writer :cache, :max_size, :http + attr_accessor :cache, :max_size, :http class << self delegate :get, :head, :put, :post, :delete, :cache, :follow, :max_size, :timeout, :auth, :basic_auth, :headers, :cookies, :use, :public_only, :download_media, to: :new end + def initialize + @http ||= + ::Danbooru::Http::ApplicationClient.new + .timeout(DEFAULT_TIMEOUT) + .headers(Danbooru.config.http_headers) + .headers("Accept-Encoding" => "gzip") + .use(:auto_inflate) + .use(:retriable) + .use(redirector: { max_redirects: MAX_REDIRECTS }) + .use(:session) + end + def get(url, **options) request(:get, url, **options) end @@ -147,16 +160,5 @@ module Danbooru def fake_response(status, body) ::HTTP::Response.new(status: status, version: "1.1", body: ::HTTP::Response::Body.new(body)) end - - def http - @http ||= - ::Danbooru::Http::ApplicationClient.new - .follow(strict: false, max_hops: MAX_REDIRECTS) - .use(:session) - .timeout(DEFAULT_TIMEOUT) - .use(:auto_inflate) - .headers(Danbooru.config.http_headers) - .headers("Accept-Encoding" => "gzip") - end end end diff --git a/app/logical/danbooru/http/redirector.rb b/app/logical/danbooru/http/redirector.rb new file mode 100644 index 000000000..26331226e --- /dev/null +++ b/app/logical/danbooru/http/redirector.rb @@ -0,0 +1,40 @@ +# A HTTP::Feature that automatically follows HTTP redirects. +# https://developer.mozilla.org/en-US/docs/Web/HTTP/Redirections + +module Danbooru + class Http + class Redirector < HTTP::Feature + HTTP::Options.register_feature :redirector, self + + attr_reader :max_redirects + + def initialize(max_redirects: 5) + @max_redirects = max_redirects + end + + def perform(request, &block) + response = yield request + + redirects = max_redirects + while response.status.redirect? + raise HTTP::Redirector::TooManyRedirectsError if redirects <= 0 + + response = yield build_redirect(request, response) + redirects -= 1 + end + + response + end + + def build_redirect(request, response) + location = response.headers["Location"].to_s + uri = HTTP::URI.parse(location) + + verb = request.verb + verb = :get if response.status == 303 && !request.verb.in?([:get, :head]) + + request.redirect(uri, verb) + end + end + end +end diff --git a/app/logical/nico_seiga_api_client.rb b/app/logical/nico_seiga_api_client.rb index f5418ac76..2b67bbde0 100644 --- a/app/logical/nico_seiga_api_client.rb +++ b/app/logical/nico_seiga_api_client.rb @@ -5,10 +5,11 @@ class NicoSeigaApiClient attr_reader :http # XXX temp disable following redirects. - def initialize(work_id:, type:, http: Danbooru::Http.follow(nil)) + def initialize(work_id:, type:) @work_id = work_id @work_type = type - @http = http + @http = Danbooru::Http.new + @http.http.default_options.features.reject! { |name, _| name == :redirector } end def image_ids From 6e6ce6e62f79f0c97251d4e1012009f8c138e021 Mon Sep 17 00:00:00 2001 From: evazion Date: Sat, 20 Jun 2020 23:08:38 -0500 Subject: [PATCH 28/59] nijie: replace Mechanize with Danbooru::Http. The Nijie login process works like this: * First we submit our `email` and `password` to `https://nijie.info/login_int.php`. * Then we save the NIJIEIEID session cookie from the response. * We optionally retry if login failed. Nijie returns 429 errors with a `Retry-After: 5` header if we send too many login requests. This can happen during parallel testing. * We cache the login cookies for only 1 hour so we don't have to worry about them becoming invalid if we cache them too long. Cookies and retrying errors on failure are handled transparently by Danbooru::Http. --- app/logical/sources/strategies/nijie.rb | 52 ++++--------------------- test/unit/sources/nijie_test.rb | 2 - 2 files changed, 8 insertions(+), 46 deletions(-) diff --git a/app/logical/sources/strategies/nijie.rb b/app/logical/sources/strategies/nijie.rb index b24605168..f3a15b576 100644 --- a/app/logical/sources/strategies/nijie.rb +++ b/app/logical/sources/strategies/nijie.rb @@ -178,54 +178,18 @@ module Sources def page return nil if page_url.blank? - doc = agent.get(page_url) + http = Danbooru::Http.new + form = { email: Danbooru.config.nijie_login, password: Danbooru.config.nijie_password } + response = http.cache(1.hour).post("https://nijie.info/login_int.php", form: form) + return nil unless response.status == 200 - if doc.search("div#header-login-container").any? - # Session cache is invalid, clear it and log in normally. - Cache.delete("nijie-session") - doc = agent.get(page_url) - end + response = http.cookies(R18: 1).cache(1.minute).get(page_url) + return nil unless response.status == 200 - doc - rescue Mechanize::ResponseCodeError => e - return nil if e.response_code.to_i == 404 - raise + response&.parse end + memoize :page - - def agent - mech = Mechanize.new - - session = Cache.get("nijie-session") - if session - cookie = Mechanize::Cookie.new("NIJIEIJIEID", session) - cookie.domain = ".nijie.info" - cookie.path = "/" - mech.cookie_jar.add(cookie) - else - mech.get("https://nijie.info/login.php") do |page| - page.form_with(:action => "/login_int.php") do |form| - form['email'] = Danbooru.config.nijie_login - form['password'] = Danbooru.config.nijie_password - end.click_button - end - session = mech.cookie_jar.cookies.select {|c| c.name == "NIJIEIJIEID"}.first - Cache.put("nijie-session", session.value, 1.day) if session - end - - # This cookie needs to be set to allow viewing of adult works while anonymous - cookie = Mechanize::Cookie.new("R18", "1") - cookie.domain = ".nijie.info" - cookie.path = "/" - mech.cookie_jar.add(cookie) - - mech - rescue Mechanize::ResponseCodeError => e - raise unless e.response_code.to_i == 429 - sleep(5) - retry - end - memoize :agent end end end diff --git a/test/unit/sources/nijie_test.rb b/test/unit/sources/nijie_test.rb index 796bcfd46..7ca695c90 100644 --- a/test/unit/sources/nijie_test.rb +++ b/test/unit/sources/nijie_test.rb @@ -187,8 +187,6 @@ module Sources desc = <<-EOS.strip_heredoc.chomp foo [b]bold[/b] [i]italics[/i] [s]strike[/s] red - - EOS From 2da8174ce239510531a8d3ce8c66b9a4b72f8035 Mon Sep 17 00:00:00 2001 From: evazion Date: Sun, 21 Jun 2020 05:12:09 -0500 Subject: [PATCH 29/59] hentai foundry: replace HTTParty with Danbooru::Http. --- app/logical/sources/strategies/hentai_foundry.rb | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/app/logical/sources/strategies/hentai_foundry.rb b/app/logical/sources/strategies/hentai_foundry.rb index 74f570288..041fb8df2 100644 --- a/app/logical/sources/strategies/hentai_foundry.rb +++ b/app/logical/sources/strategies/hentai_foundry.rb @@ -64,11 +64,10 @@ module Sources def page return nil if page_url.blank? - doc = Cache.get("hentai-foundry:#{page_url}", 1.minute) do - HTTParty.get("#{page_url}?enterAgree=1").body - end + response = Danbooru::Http.new.cache(1.minute).get("#{page_url}?enterAgree=1") + return nil unless response.status == 200 - Nokogiri::HTML(doc) + response.parse end def tags From 5604ab0079c3d9b0dcfae339bcf2050aaa21cb20 Mon Sep 17 00:00:00 2001 From: evazion Date: Sun, 21 Jun 2020 11:56:37 -0500 Subject: [PATCH 30/59] pixiv: remove fanbox support. This is broken and it needs to be rewritten as a separate source strategy anyway. --- app/logical/pixiv_api_client.rb | 62 --------------------- app/logical/pixiv_web_agent.rb | 74 ------------------------- app/logical/sources/strategies/pixiv.rb | 64 +-------------------- test/unit/downloads/pixiv_test.rb | 11 ---- test/unit/sources/pixiv_test.rb | 11 ---- 5 files changed, 2 insertions(+), 220 deletions(-) delete mode 100644 app/logical/pixiv_web_agent.rb diff --git a/app/logical/pixiv_api_client.rb b/app/logical/pixiv_api_client.rb index c742e250e..a28010f21 100644 --- a/app/logical/pixiv_api_client.rb +++ b/app/logical/pixiv_api_client.rb @@ -99,50 +99,6 @@ class PixivApiClient end end - class FanboxResponse - attr_reader :json - - def initialize(json) - @json = json - end - - def name - json["body"]["user"]["name"] - end - - def user_id - json["body"]["user"]["userId"] - end - - def moniker - "" - end - - def page_count - json["body"]["body"]["images"].size - end - - def artist_commentary_title - json["body"]["title"] - end - - def artist_commentary_desc - json["body"]["body"]["text"] - end - - def tags - [] - end - - def pages - if json["body"]["body"] - json["body"]["body"]["images"].map {|x| x["originalUrl"]} - else - [] - end - end - end - def work(illust_id) headers = Danbooru.config.http_headers.merge( "Referer" => "http://www.pixiv.net", @@ -169,19 +125,6 @@ class PixivApiClient raise Error.new("Pixiv API call failed (status=#{response.code} body=#{response.body})") end - def fanbox(fanbox_id) - url = "https://www.pixiv.net/ajax/fanbox/post?postId=#{fanbox_id.to_i}" - resp = agent.get(url) - json = JSON.parse(resp.body) - if resp.code == "200" - FanboxResponse.new(json) - elsif json["status"] == "failure" - raise Error.new("Pixiv API call failed (status=#{resp.code} body=#{body})") - end - rescue JSON::ParserError - raise Error.new("Pixiv API call failed (status=#{resp.code} body=#{body})") - end - def novel(novel_id) headers = Danbooru.config.http_headers.merge( "Referer" => "http://www.pixiv.net", @@ -237,9 +180,4 @@ class PixivApiClient access_token end end - - def agent - PixivWebAgent.build - end - memoize :agent end diff --git a/app/logical/pixiv_web_agent.rb b/app/logical/pixiv_web_agent.rb deleted file mode 100644 index 1a181672b..000000000 --- a/app/logical/pixiv_web_agent.rb +++ /dev/null @@ -1,74 +0,0 @@ -class PixivWebAgent - SESSION_CACHE_KEY = "pixiv-phpsessid" - COMIC_SESSION_CACHE_KEY = "pixiv-comicsessid" - SESSION_COOKIE_KEY = "PHPSESSID" - COMIC_SESSION_COOKIE_KEY = "_pixiv-comic_session" - - def self.phpsessid(agent) - agent.cookies.select { |cookie| cookie.name == SESSION_COOKIE_KEY }.first.try(:value) - end - - def self.build - mech = Mechanize.new - mech.keep_alive = false - - phpsessid = Cache.get(SESSION_CACHE_KEY) - comicsessid = Cache.get(COMIC_SESSION_CACHE_KEY) - - if phpsessid - cookie = Mechanize::Cookie.new(SESSION_COOKIE_KEY, phpsessid) - cookie.domain = ".pixiv.net" - cookie.path = "/" - mech.cookie_jar.add(cookie) - - if comicsessid - cookie = Mechanize::Cookie.new(COMIC_SESSION_COOKIE_KEY, comicsessid) - cookie.domain = ".pixiv.net" - cookie.path = "/" - mech.cookie_jar.add(cookie) - end - else - headers = { - "Origin" => "https://accounts.pixiv.net", - "Referer" => "https://accounts.pixiv.net/login?lang=en^source=pc&view_type=page&ref=wwwtop_accounts_index" - } - - params = { - pixiv_id: Danbooru.config.pixiv_login, - password: Danbooru.config.pixiv_password, - captcha: nil, - g_captcha_response: nil, - source: "pc", - post_key: nil - } - - mech.get("https://accounts.pixiv.net/login?lang=en&source=pc&view_type=page&ref=wwwtop_accounts_index") do |page| - json = page.search("input#init-config").first.attr("value") - if json =~ /pixivAccount\.postKey":"([a-f0-9]+)/ - params[:post_key] = $1 - end - end - - mech.post("https://accounts.pixiv.net/api/login?lang=en", params, headers) - if mech.current_page.body =~ /"error":false/ - cookie = mech.cookies.select {|x| x.name == SESSION_COOKIE_KEY}.first - if cookie - Cache.put(SESSION_CACHE_KEY, cookie.value, 1.week) - end - end - - begin - mech.get("https://comic.pixiv.net") do - cookie = mech.cookies.select {|x| x.name == COMIC_SESSION_COOKIE_KEY}.first - if cookie - Cache.put(COMIC_SESSION_CACHE_KEY, cookie.value, 1.week) - end - end - rescue Net::HTTPServiceUnavailable - # ignore - end - end - - mech - end -end diff --git a/app/logical/sources/strategies/pixiv.rb b/app/logical/sources/strategies/pixiv.rb index ad7cd7eb1..53475f150 100644 --- a/app/logical/sources/strategies/pixiv.rb +++ b/app/logical/sources/strategies/pixiv.rb @@ -64,9 +64,6 @@ module Sources ORIG_IMAGE = %r{#{PXIMG}/img-original/img/#{DATE}/(?\d+)_p(?\d+)\.#{EXT}\z}i STACC_PAGE = %r{\A#{WEB}/stacc/#{MONIKER}/?\z}i NOVEL_PAGE = %r{(?:\Ahttps?://www\.pixiv\.net/novel/show\.php\?id=(\d+))} - FANBOX_ACCOUNT = %r{(?:\Ahttps?://www\.pixiv\.net/fanbox/creator/\d+\z)} - FANBOX_IMAGE = %r{(?:\Ahttps?://fanbox\.pixiv\.net/images/post/(\d+))} - FANBOX_PAGE = %r{(?:\Ahttps?://www\.pixiv\.net/fanbox/creator/\d+/post/(\d+))} def self.to_dtext(text) if text.nil? @@ -127,14 +124,6 @@ module Sources return "https://www.pixiv.net/novel/show.php?id=#{novel_id}&mode=cover" end - if fanbox_id.present? - return "https://www.pixiv.net/fanbox/creator/#{metadata.user_id}/post/#{fanbox_id}" - end - - if fanbox_account_id.present? - return "https://www.pixiv.net/fanbox/creator/#{fanbox_account_id}" - end - if illust_id.present? return "https://www.pixiv.net/artworks/#{illust_id}" end @@ -192,17 +181,7 @@ module Sources end def headers - if fanbox_id.present? - # need the session to download fanbox images - return { - "Referer" => "https://www.pixiv.net/fanbox", - "Cookie" => HTTP::Cookie.cookie_value(agent.cookies) - } - end - - { - "Referer" => "https://www.pixiv.net" - } + { "Referer" => "https://www.pixiv.net" } end def normalize_for_source @@ -242,10 +221,6 @@ module Sources end def image_urls_sub - if url =~ FANBOX_IMAGE - return [url] - end - # there's too much normalization bullshit we have to deal with # raw urls, so just fetch the canonical url from the api every # time. @@ -265,7 +240,7 @@ module Sources # even though it makes sense to reference page_url here, it will only look # at (url, referer_url). def illust_id - return nil if novel_id.present? || fanbox_id.present? + return nil if novel_id.present? parsed_urls.each do |url| # http://www.pixiv.net/member_illust.php?mode=medium&illust_id=18557054 @@ -328,46 +303,11 @@ module Sources end memoize :novel_id - def fanbox_id - [url, referer_url].each do |x| - if x =~ FANBOX_PAGE - return $1 - end - - if x =~ FANBOX_IMAGE - return $1 - end - end - - nil - end - memoize :fanbox_id - - def fanbox_account_id - [url, referer_url].each do |x| - if x =~ FANBOX_ACCOUNT - return x - end - end - - nil - end - memoize :fanbox_account_id - - def agent - PixivWebAgent.build - end - memoize :agent - def metadata if novel_id.present? return PixivApiClient.new.novel(novel_id) end - if fanbox_id.present? - return PixivApiClient.new.fanbox(fanbox_id) - end - PixivApiClient.new.work(illust_id) end memoize :metadata diff --git a/test/unit/downloads/pixiv_test.rb b/test/unit/downloads/pixiv_test.rb index ddc32e14d..02d923c02 100644 --- a/test/unit/downloads/pixiv_test.rb +++ b/test/unit/downloads/pixiv_test.rb @@ -122,17 +122,6 @@ module Downloads assert_downloaded(@file_size, @file_url, @ref) end end - - context "downloading a pixiv fanbox image" do - should_eventually "work" do - @source = "https://www.pixiv.net/fanbox/creator/12491073/post/82406" - @file_url = "https://fanbox.pixiv.net/images/post/82406/D833IKA7FIesJXL8xx39rrG0.jpeg" - @file_size = 873_387 - - assert_not_rewritten(@file_url, @source) - assert_downloaded(@file_size, @file_url, @source) - end - end end context "An ugoira site for pixiv" do diff --git a/test/unit/sources/pixiv_test.rb b/test/unit/sources/pixiv_test.rb index cb7a3d883..a268fa640 100644 --- a/test/unit/sources/pixiv_test.rb +++ b/test/unit/sources/pixiv_test.rb @@ -73,17 +73,6 @@ module Sources end end - context "A https://www.pixiv.net/fanbox/creator/*/post/* source" do - should_eventually "work" do - @site = Sources::Strategies.find("http://www.pixiv.net/fanbox/creator/554149/post/82555") - - assert_equal("TYONE(お仕事募集中)", @site.artist_name) - assert_equal("https://www.pixiv.net/member.php?id=554149", @site.profile_url) - assert_equal("https://fanbox.pixiv.net/images/post/82555/Lyyeb6dDLcQZmy09nqLZapuS.jpeg", @site.image_url) - assert_nothing_raised { @site.to_h } - end - end - context "A https://www.pixiv.net/*/artworks/* source" do should "work" do @site = Sources::Strategies.find("https://www.pixiv.net/en/artworks/64476642") From 3ad8c708c5466757ad50abcaaed5e8d502984667 Mon Sep 17 00:00:00 2001 From: evazion Date: Sun, 21 Jun 2020 12:49:48 -0500 Subject: [PATCH 31/59] pixiv: replace HTTParty with Danbooru::Http. --- app/logical/pixiv_api_client.rb | 91 ++++++++++++++------------------- 1 file changed, 39 insertions(+), 52 deletions(-) diff --git a/app/logical/pixiv_api_client.rb b/app/logical/pixiv_api_client.rb index a28010f21..7d3d00156 100644 --- a/app/logical/pixiv_api_client.rb +++ b/app/logical/pixiv_api_client.rb @@ -1,5 +1,3 @@ -require 'resolv-replace' - class PixivApiClient extend Memoist @@ -100,21 +98,12 @@ class PixivApiClient end def work(illust_id) - headers = Danbooru.config.http_headers.merge( - "Referer" => "http://www.pixiv.net", - "Content-Type" => "application/x-www-form-urlencoded", - "Authorization" => "Bearer #{access_token}" - ) - params = { - "image_sizes" => "large", - "include_stats" => "true" - } - + params = { image_sizes: "large", include_stats: "true" } url = "https://public-api.secure.pixiv.net/v#{API_VERSION}/works/#{illust_id.to_i}.json" - response = Danbooru::Http.cache(1.minute).headers(headers).get(url, params: params) + response = api_client.cache(1.minute).get(url, params: params) json = response.parse - if response.code == 200 + if response.status == 200 WorkResponse.new(json["response"][0]) elsif json["status"] == "failure" && json.dig("errors", "system", "message") =~ /対象のイラストは見つかりませんでした。/ raise BadIDError.new("Pixiv ##{illust_id} not found: work was deleted, made private, or ID is invalid.") @@ -126,18 +115,11 @@ class PixivApiClient end def novel(novel_id) - headers = Danbooru.config.http_headers.merge( - "Referer" => "http://www.pixiv.net", - "Content-Type" => "application/x-www-form-urlencoded", - "Authorization" => "Bearer #{access_token}" - ) - url = "https://public-api.secure.pixiv.net/v#{API_VERSION}/novels/#{novel_id.to_i}.json" - resp = HTTParty.get(url, Danbooru.config.httparty_options.deep_merge(headers: headers)) - body = resp.body.force_encoding("utf-8") - json = JSON.parse(body) + resp = api_client.cache(1.minute).get(url) + json = resp.parse - if resp.success? + if resp.status == 200 NovelResponse.new(json["response"][0]) elsif json["status"] == "failure" && json.dig("errors", "system", "message") =~ /対象のイラストは見つかりませんでした。/ raise Error.new("Pixiv API call failed (status=#{resp.code} body=#{body})") @@ -147,37 +129,42 @@ class PixivApiClient end def access_token - Cache.get("pixiv-papi-access-token", 3000) do - access_token = nil + # truncate timestamp to 1-hour resolution so that it doesn't break caching. + client_time = Time.zone.now.utc.change(min: 0).rfc3339 + client_hash = Digest::MD5.hexdigest(client_time + CLIENT_HASH_SALT) - client_time = Time.now.rfc3339 - client_hash = Digest::MD5.hexdigest(client_time + CLIENT_HASH_SALT) + headers = { + "Referer": "http://www.pixiv.net", + "X-Client-Time": client_time, + "X-Client-Hash": client_hash + } - headers = { - "Referer": "http://www.pixiv.net", - "X-Client-Time": client_time, - "X-Client-Hash": client_hash - } - params = { - username: Danbooru.config.pixiv_login, - password: Danbooru.config.pixiv_password, - grant_type: "password", - client_id: CLIENT_ID, - client_secret: CLIENT_SECRET - } - url = "https://oauth.secure.pixiv.net/auth/token" + params = { + username: Danbooru.config.pixiv_login, + password: Danbooru.config.pixiv_password, + grant_type: "password", + client_id: CLIENT_ID, + client_secret: CLIENT_SECRET + } - resp = HTTParty.post(url, Danbooru.config.httparty_options.deep_merge(body: params, headers: headers)) - body = resp.body.force_encoding("utf-8") + resp = http.headers(headers).cache(1.hour).post("https://oauth.secure.pixiv.net/auth/token", form: params) + return nil unless resp.status == 200 - if resp.success? - json = JSON.parse(body) - access_token = json["response"]["access_token"] - else - raise Error.new("Pixiv API access token call failed (status=#{resp.code} body=#{body})") - end - - access_token - end + resp.parse.dig("response", "access_token") end + + def api_client + http.headers( + **Danbooru.config.http_headers, + "Referer": "http://www.pixiv.net", + "Content-Type": "application/x-www-form-urlencoded", + "Authorization": "Bearer #{access_token}" + ) + end + + def http + Danbooru::Http.new + end + + memoize :access_token, :api_client, :http end From 7e471fe223a26f4df727a3bcc1bbcc0fd5a1456c Mon Sep 17 00:00:00 2001 From: evazion Date: Sun, 21 Jun 2020 13:10:16 -0500 Subject: [PATCH 32/59] sources: replace HTTParty with Danbooru::Http in `http_exists?`. --- app/logical/sources/strategies/art_station.rb | 2 +- app/logical/sources/strategies/base.rb | 5 ++--- app/logical/sources/strategies/tumblr.rb | 2 +- test/test_helpers/download_test_helper.rb | 15 --------------- test/unit/sources/nijie_test.rb | 8 ++++---- 5 files changed, 8 insertions(+), 24 deletions(-) diff --git a/app/logical/sources/strategies/art_station.rb b/app/logical/sources/strategies/art_station.rb index 6ab7445ca..1c4d32b56 100644 --- a/app/logical/sources/strategies/art_station.rb +++ b/app/logical/sources/strategies/art_station.rb @@ -147,7 +147,7 @@ module Sources::Strategies urls = urls.reverse end - chosen_url = urls.find { |url| http_exists?(url, headers) } + chosen_url = urls.find { |url| http_exists?(url) } chosen_url || url end end diff --git a/app/logical/sources/strategies/base.rb b/app/logical/sources/strategies/base.rb index fcc0ba6e6..e4f3d1a6a 100644 --- a/app/logical/sources/strategies/base.rb +++ b/app/logical/sources/strategies/base.rb @@ -288,9 +288,8 @@ module Sources to_h.to_json end - def http_exists?(url, headers) - res = HTTParty.head(url, Danbooru.config.httparty_options.deep_merge(headers: headers)) - res.success? + def http_exists?(url, headers = {}) + http.headers(headers).head(url).status.success? end # Convert commentary to dtext by stripping html tags. Sites can override diff --git a/app/logical/sources/strategies/tumblr.rb b/app/logical/sources/strategies/tumblr.rb index b82a1f77d..e3440c910 100644 --- a/app/logical/sources/strategies/tumblr.rb +++ b/app/logical/sources/strategies/tumblr.rb @@ -168,7 +168,7 @@ module Sources::Strategies end candidates.find do |candidate| - http_exists?(candidate, headers) + http_exists?(candidate) end end diff --git a/test/test_helpers/download_test_helper.rb b/test/test_helpers/download_test_helper.rb index eea711588..c3f70c75d 100644 --- a/test/test_helpers/download_test_helper.rb +++ b/test/test_helpers/download_test_helper.rb @@ -16,19 +16,4 @@ module DownloadTestHelper def assert_not_rewritten(source, referer = nil) assert_rewritten(source, source, referer) end - - def assert_http_exists(url, headers: {}) - res = HTTParty.head(url, Danbooru.config.httparty_options.deep_merge(headers: headers)) - assert_equal(true, res.success?) - end - - def assert_http_status(code, url, headers: {}) - res = HTTParty.head(url, Danbooru.config.httparty_options.deep_merge(headers: headers)) - assert_equal(code, res.code) - end - - def assert_http_size(size, url, headers: {}) - res = HTTParty.head(url, Danbooru.config.httparty_options.deep_merge(headers: headers)) - assert_equal(size, res.content_length) - end end diff --git a/test/unit/sources/nijie_test.rb b/test/unit/sources/nijie_test.rb index 7ca695c90..c4f7b88c8 100644 --- a/test/unit/sources/nijie_test.rb +++ b/test/unit/sources/nijie_test.rb @@ -43,7 +43,7 @@ module Sources should "get the image url" do assert_equal("https://pic.nijie.net/03/nijie_picture/728995_20170505014820_0.jpg", @site.image_url) - assert_http_size(132_555, @site.image_url) + assert_downloaded(132_555, @site.image_url) end should "get the canonical url" do @@ -53,7 +53,7 @@ module Sources should "get the preview url" do assert_equal("https://pic.nijie.net/03/__rs_l170x170/nijie_picture/728995_20170505014820_0.jpg", @site.preview_url) assert_equal([@site.preview_url], @site.preview_urls) - assert_http_exists(@site.preview_url) + assert_downloaded(132_555, @site.preview_url) end should "get the profile" do @@ -205,8 +205,8 @@ module Sources assert_equal("https://nijie.info/members.php?id=236014", site.profile_url) assert_nothing_raised { site.to_h } - assert_http_size(3619, site.image_url) - assert_http_exists(site.preview_url) + assert_downloaded(3619, site.image_url) + assert_downloaded(3619, site.preview_url) end end From 29a5f7dfc83aa5bced43acb6570f4445b35dd23f Mon Sep 17 00:00:00 2001 From: evazion Date: Sun, 21 Jun 2020 14:01:38 -0500 Subject: [PATCH 33/59] image proxy: replace HTTParty with Danbooru::Http. --- app/controllers/uploads_controller.rb | 2 +- app/logical/image_proxy.rb | 15 +++++++-------- test/functional/uploads_controller_test.rb | 11 +++++++++++ 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/app/controllers/uploads_controller.rb b/app/controllers/uploads_controller.rb index d99d58125..97f5090dc 100644 --- a/app/controllers/uploads_controller.rb +++ b/app/controllers/uploads_controller.rb @@ -21,7 +21,7 @@ class UploadsController < ApplicationController def image_proxy authorize Upload resp = ImageProxy.get_image(params[:url]) - send_data resp.body, :type => resp.content_type, :disposition => "inline" + send_data resp.body, type: resp.mime_type, disposition: "inline" end def index diff --git a/app/logical/image_proxy.rb b/app/logical/image_proxy.rb index 80982c7cc..9c2acfe42 100644 --- a/app/logical/image_proxy.rb +++ b/app/logical/image_proxy.rb @@ -1,4 +1,6 @@ class ImageProxy + class Error < StandardError; end + def self.needs_proxy?(url) fake_referer_for(url).present? end @@ -8,16 +10,13 @@ class ImageProxy end def self.get_image(url) - if url.blank? - raise "Must specify url" - end + raise Error, "URL not present" unless url.present? + raise Error, "Proxy not allowed for this url (url=#{url})" unless needs_proxy?(url) - if !needs_proxy?(url) - raise "Proxy not allowed for this site" - end + referer = fake_referer_for(url) + response = Danbooru::Http.headers(Referer: referer).get(url) + raise Error, "Couldn't proxy image (code=#{response.status}, url=#{url})" unless response.status.success? - response = HTTParty.get(url, Danbooru.config.httparty_options.deep_merge(headers: {"Referer" => fake_referer_for(url)})) - raise "HTTP error code: #{response.code} #{response.message}" unless response.success? response end end diff --git a/test/functional/uploads_controller_test.rb b/test/functional/uploads_controller_test.rb index 9df37a1b4..e19949fca 100644 --- a/test/functional/uploads_controller_test.rb +++ b/test/functional/uploads_controller_test.rb @@ -18,6 +18,17 @@ class UploadsControllerTest < ActionDispatch::IntegrationTest mock_iqdb_service! end + context "image proxy action" do + should "work" do + url = "https://i.pximg.net/img-original/img/2017/11/21/17/06/44/65985331_p0.png" + get_auth image_proxy_uploads_path, @user, params: { url: url } + + assert_response :success + assert_equal("image/png", response.media_type) + assert_equal(15_573, response.body.size) + end + end + context "batch action" do context "for twitter galleries" do should "render" do From 5c7843bd3da0add6afd12c0baf98ad160721afac Mon Sep 17 00:00:00 2001 From: evazion Date: Sun, 21 Jun 2020 15:02:14 -0500 Subject: [PATCH 34/59] scripts: refactor mocked services. Replace the mocked services in scripts/mocked_services with Rails-level mocked services. The scripts in scripts/mocked_services were a set of stub Sinatra servers used to mock the Reportbooru, Recommender, and IQDBs services during development. They return fake data so you can test pages that use these services. Implementing these services in Rails makes it easier to run them. It also lets us drop a dependency on Sinatra and drop a use of HTTParty. To use these services, set the following configuration in danbooru_local_config.rb or .env.local: * reportbooru_server: http://localhost:3000/mock/reportbooru * recommender_server: http://localhost:3000/mock/recommender * iqdbs_server: http://localhost:3000/mock/iqdb where `http://localhost:300` is the url for your local Danbooru server (may need to be changed depending on your configuration). --- app/controllers/mock_services_controller.rb | 48 +++++++++++++++++++ config/danbooru_default_config.rb | 14 +++++- config/routes.rb | 8 ++++ script/mock_services/README.md | 7 --- script/mock_services/iqdbs.rb | 14 ------ script/mock_services/mock_service_helper.rb | 22 --------- script/mock_services/recommender.rb | 19 -------- script/mock_services/reportbooru.rb | 22 --------- .../mock_services_controller_test.rb | 28 +++++++++++ 9 files changed, 96 insertions(+), 86 deletions(-) create mode 100644 app/controllers/mock_services_controller.rb delete mode 100644 script/mock_services/README.md delete mode 100644 script/mock_services/iqdbs.rb delete mode 100644 script/mock_services/mock_service_helper.rb delete mode 100644 script/mock_services/recommender.rb delete mode 100644 script/mock_services/reportbooru.rb create mode 100644 test/functional/mock_services_controller_test.rb diff --git a/app/controllers/mock_services_controller.rb b/app/controllers/mock_services_controller.rb new file mode 100644 index 000000000..67d387fb6 --- /dev/null +++ b/app/controllers/mock_services_controller.rb @@ -0,0 +1,48 @@ +class MockServicesController < ApplicationController + skip_forgery_protection + respond_to :json + + before_action do + raise User::PrivilegeError if Rails.env.production? + end + + def recommender_recommend + @data = posts.map { |post| [post.id, rand(0.0..1.0)] } + render json: @data + end + + def recommender_similar + @data = posts.map { |post| [post.id, rand(0.0..1.0)] } + render json: @data + end + + def reportbooru_missed_searches + @data = tags.map { |tag| "#{tag.name} #{rand(1.0..1000.0)}" }.join("\n") + render json: @data + end + + def reportbooru_post_searches + @data = tags.map { |tag| [tag.name, rand(1..1000)] } + render json: @data + end + + def reportbooru_post_views + @data = posts.map { |post| [post.id, rand(1..1000)] } + render json: @data + end + + def iqdbs_similar + @data = posts.map { |post| { post_id: post.id, score: rand(0..100)} } + render json: @data + end + + private + + def posts(limit = 10) + Post.last(limit) + end + + def tags(limit = 10) + Tag.order(post_count: :desc).limit(limit) + end +end diff --git a/config/danbooru_default_config.rb b/config/danbooru_default_config.rb index b82014e0a..e52e6a998 100644 --- a/config/danbooru_default_config.rb +++ b/config/danbooru_default_config.rb @@ -376,14 +376,20 @@ module Danbooru false end - # reportbooru options - see https://github.com/r888888888/reportbooru + # The URL for the Reportbooru server (https://github.com/evazion/reportbooru). + # Optional. Used for tracking post views, popular searches, and missed searches. + # Set to http://localhost/mock/reportbooru to enable a fake reportbooru + # server for development purposes. def reportbooru_server end def reportbooru_key end - # iqdbs options - see https://github.com/r888888888/iqdbs + # The URL for the IQDBs server (https://github.com/evazion/iqdbs). + # Optional. Used for dupe detection and reverse image searches. + # Set to http://localhost/mock/iqdbs to enable a fake iqdb server for + # development purposes. def iqdbs_server end @@ -461,6 +467,10 @@ module Danbooru def cloudflare_zone end + # The URL for the recommender server (https://github.com/evazion/recommender). + # Optional. Used to generate post recommendations. + # Set to http://localhost/mock/recommender to enable a fake recommender + # server for development purposes. def recommender_server end diff --git a/config/routes.rb b/config/routes.rb index 95392b1e1..9b8ff2e6e 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -372,6 +372,14 @@ Rails.application.routes.draw do get "/static/contact" => "static#contact", :as => "contact" get "/static/dtext_help" => "static#dtext_help", :as => "dtext_help" + get "/mock/recommender/recommend/:user_id" => "mock_services#recommender_recommend", as: "mock_recommender_recommend" + get "/mock/recommender/similiar/:post_id" => "mock_services#recommender_similar", as: "mock_recommender_similar" + get "/mock/reportbooru/missed_searches" => "mock_services#reportbooru_missed_searches", as: "mock_reportbooru_missed_searches" + get "/mock/reportbooru/post_searches/rank" => "mock_services#reportbooru_post_searches", as: "mock_reportbooru_post_searches" + get "/mock/reportbooru/post_views/rank" => "mock_services#reportbooru_post_views", as: "mock_reportbooru_post_views" + get "/mock/iqdbs/similar" => "mock_services#iqdbs_similar", as: "mock_iqdbs_similar" + post "/mock/iqdbs/similar" => "mock_services#iqdbs_similar" + root :to => "posts#index" get "*other", :to => "static#not_found" diff --git a/script/mock_services/README.md b/script/mock_services/README.md deleted file mode 100644 index 6f8f84e8a..000000000 --- a/script/mock_services/README.md +++ /dev/null @@ -1,7 +0,0 @@ -These are mocked services to be used for development purposes. - -- danbooru: port 3000 -- recommender: port 3001 -- iqdbs: port 3002 -- reportbooru: port 3003 - diff --git a/script/mock_services/iqdbs.rb b/script/mock_services/iqdbs.rb deleted file mode 100644 index a3cca08c3..000000000 --- a/script/mock_services/iqdbs.rb +++ /dev/null @@ -1,14 +0,0 @@ -require 'sinatra' -require 'json' -require_relative './mock_service_helper' - -set :port, 3002 - -configure do - POST_IDS = MockServiceHelper.fetch_post_ids -end - -get '/similar' do - content_type :json - POST_IDS[0..10].map {|x| {post_id: x}}.to_json -end diff --git a/script/mock_services/mock_service_helper.rb b/script/mock_services/mock_service_helper.rb deleted file mode 100644 index 2b259bc4c..000000000 --- a/script/mock_services/mock_service_helper.rb +++ /dev/null @@ -1,22 +0,0 @@ -require 'socket' -require 'timeout' -require 'httparty' - -module MockServiceHelper - module_function - - DANBOORU_PORT = 3000 - - def fetch_post_ids - begin - s = TCPSocket.new("localhost", DANBOORU_PORT) - s.close - rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH - sleep 1 - retry - end - - json = HTTParty.get("http://localhost:#{DANBOORU_PORT}/posts.json?random=true&limit=10").body - return JSON.parse(json).map {|x| x["id"]} - end -end diff --git a/script/mock_services/recommender.rb b/script/mock_services/recommender.rb deleted file mode 100644 index cb5f2b317..000000000 --- a/script/mock_services/recommender.rb +++ /dev/null @@ -1,19 +0,0 @@ -require 'sinatra' -require 'json' -require_relative './mock_service_helper' - -set :port, 3001 - -configure do - POST_IDS = MockServiceHelper.fetch_post_ids -end - -get '/recommend/:user_id' do - content_type :json - POST_IDS[0..10].map {|x| [x, "1.000"]}.to_json -end - -get '/similar/:post_id' do - content_type :json - POST_IDS[0..6].map {|x| [x, "1.000"]}.to_json -end diff --git a/script/mock_services/reportbooru.rb b/script/mock_services/reportbooru.rb deleted file mode 100644 index 3aa40edb3..000000000 --- a/script/mock_services/reportbooru.rb +++ /dev/null @@ -1,22 +0,0 @@ -require 'sinatra' -require 'json' - -set :port, 3003 - -get '/missed_searches' do - content_type :text - return "abcdefg 10.0\nblahblahblah 20.0\n" -end - -get '/post_searches/rank' do - content_type :json - return [["abc", 100], ["def", 200]].to_json -end - -get '/reports/user_similarity' do - # todo -end - -post '/post_views' do - # todo -end diff --git a/test/functional/mock_services_controller_test.rb b/test/functional/mock_services_controller_test.rb new file mode 100644 index 000000000..5f33cc691 --- /dev/null +++ b/test/functional/mock_services_controller_test.rb @@ -0,0 +1,28 @@ +require 'test_helper' + +class MockServicesControllerTest < ActionDispatch::IntegrationTest + context "The mock services controller" do + setup do + create(:post) + create(:tag) + end + + context "for all actions" do + should "work" do + paths = [ + mock_recommender_recommend_path(42), + mock_recommender_similar_path(42), + mock_reportbooru_missed_searches_path, + mock_reportbooru_post_searches_path, + mock_reportbooru_post_views_path, + mock_iqdbs_similar_path, + ] + + paths.each do |path| + get path + assert_response :success + end + end + end + end +end From a4efeb22607d2a78a43030ce01e702fdeeb0dc00 Mon Sep 17 00:00:00 2001 From: evazion Date: Sun, 21 Jun 2020 12:00:57 -0500 Subject: [PATCH 35/59] gems: drop Mechanize, HTTParty, and Sinatra gems. --- Gemfile | 3 -- Gemfile.lock | 36 --------------- app/logical/danbooru/http.rb | 2 +- app/logical/pixiv_api_client.rb | 1 - app/logical/sources/strategies/base.rb | 2 +- config/danbooru_default_config.rb | 16 ------- config/initializers/mechanize_patch.rb | 64 -------------------------- 7 files changed, 2 insertions(+), 122 deletions(-) delete mode 100644 config/initializers/mechanize_patch.rb diff --git a/Gemfile b/Gemfile index 70656ec6b..86bb62c2d 100644 --- a/Gemfile +++ b/Gemfile @@ -7,7 +7,6 @@ gem "pg" gem "delayed_job" gem "delayed_job_active_record" gem "simple_form" -gem "mechanize" gem "whenever", :require => false gem "sanitize" gem 'ruby-vips' @@ -28,7 +27,6 @@ gem 'daemons' gem 'oauth2' gem 'bootsnap' gem 'addressable' -gem 'httparty' gem 'rakismet' gem 'recaptcha', require: "recaptcha/rails" gem 'activemodel-serializers-xml' @@ -63,7 +61,6 @@ end group :development do gem 'rubocop' gem 'rubocop-rails' - gem 'sinatra' gem 'meta_request' gem 'rack-mini-profiler' gem 'stackprof' diff --git a/Gemfile.lock b/Gemfile.lock index 222928350..a8fa40f17 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -122,7 +122,6 @@ GEM chronic (0.10.2) coderay (1.1.3) concurrent-ruby (1.1.6) - connection_pool (2.2.3) crass (1.0.6) daemons (1.3.1) delayed_job (4.1.8) @@ -164,9 +163,6 @@ GEM http-form_data (2.3.0) http-parser (1.2.1) ffi-compiler (>= 1.0, < 2.0) - httparty (0.18.1) - mime-types (~> 3.0) - multi_xml (>= 0.5.2) i18n (1.8.3) concurrent-ruby (~> 1.0) ipaddress_2 (0.13.0) @@ -188,24 +184,12 @@ GEM mini_mime (>= 0.1.1) marcel (0.3.3) mimemagic (~> 0.3.2) - mechanize (2.7.6) - domain_name (~> 0.5, >= 0.5.1) - http-cookie (~> 1.0) - mime-types (>= 1.17.2) - net-http-digest_auth (~> 1.1, >= 1.1.1) - net-http-persistent (>= 2.5.2) - nokogiri (~> 1.6) - ntlm-http (~> 0.1, >= 0.1.1) - webrobots (>= 0.0.9, < 0.2) memoist (0.16.2) memory_profiler (0.9.14) meta_request (0.7.2) rack-contrib (>= 1.1, < 3) railties (>= 3.0.0, < 7) method_source (1.0.0) - mime-types (3.3.1) - mime-types-data (~> 3.2015) - mime-types-data (3.2020.0512) mimemagic (0.3.5) mini_mime (1.0.2) mini_portile2 (2.5.0) @@ -224,11 +208,6 @@ GEM multi_json (1.14.1) multi_xml (0.6.0) multipart-post (2.1.1) - mustermann (1.1.1) - ruby2_keywords (~> 0.0.1) - net-http-digest_auth (1.4.1) - net-http-persistent (4.0.0) - connection_pool (~> 2.2) net-scp (3.0.0) net-ssh (>= 2.6.5, < 7.0.0) net-sftp (3.0.0) @@ -241,7 +220,6 @@ GEM nokogiri (1.11.0.rc2-x64-mingw32) nokogumbo (2.0.2) nokogiri (~> 1.8, >= 1.8.4) - ntlm-http (0.1.1) oauth2 (1.4.4) faraday (>= 0.8, < 2.0) jwt (>= 1.0, < 3.0) @@ -271,8 +249,6 @@ GEM rack (~> 2.0) rack-mini-profiler (2.0.2) rack (>= 1.2.0) - rack-protection (2.0.8.1) - rack rack-proxy (0.6.5) rack rack-test (1.1.0) @@ -339,9 +315,7 @@ GEM ruby-progressbar (1.10.1) ruby-vips (2.0.17) ffi (~> 1.9) - ruby2_keywords (0.0.2) rubyzip (2.3.0) - safe_yaml (1.0.5) sanitize (5.2.1) crass (~> 1.0.2) nokogiri (>= 1.8.0) @@ -364,11 +338,6 @@ GEM json (>= 1.8, < 3) simplecov-html (~> 0.10.0) simplecov-html (0.10.2) - sinatra (2.0.8.1) - mustermann (~> 1.0) - rack (~> 2.0) - rack-protection (= 2.0.8.1) - tilt (~> 2.0) sprockets (4.0.2) concurrent-ruby (~> 1.0) rack (> 1, < 3) @@ -385,7 +354,6 @@ GEM stripe (5.22.0) thor (1.0.1) thread_safe (0.3.6) - tilt (2.0.10) tzinfo (1.2.7) thread_safe (~> 0.1) unf (0.1.4) @@ -404,7 +372,6 @@ GEM rack-proxy (>= 0.6.1) railties (>= 5.2) semantic_range (>= 2.3.0) - webrobots (0.1.2) websocket-driver (0.7.2) websocket-extensions (>= 0.1.0) websocket-extensions (0.1.5) @@ -442,12 +409,10 @@ DEPENDENCIES ffaker flamegraph http - httparty ipaddress_2 jquery-rails listen mail - mechanize memoist memory_profiler meta_request @@ -484,7 +449,6 @@ DEPENDENCIES shoulda-matchers simple_form simplecov (~> 0.17.0) - sinatra stackprof streamio-ffmpeg stripe diff --git a/app/logical/danbooru/http.rb b/app/logical/danbooru/http.rb index dfe6558f8..590271fe0 100644 --- a/app/logical/danbooru/http.rb +++ b/app/logical/danbooru/http.rb @@ -22,8 +22,8 @@ module Danbooru @http ||= ::Danbooru::Http::ApplicationClient.new .timeout(DEFAULT_TIMEOUT) - .headers(Danbooru.config.http_headers) .headers("Accept-Encoding" => "gzip") + .headers("User-Agent": "#{Danbooru.config.canonical_app_name}/#{Rails.application.config.x.git_hash}") .use(:auto_inflate) .use(:retriable) .use(redirector: { max_redirects: MAX_REDIRECTS }) diff --git a/app/logical/pixiv_api_client.rb b/app/logical/pixiv_api_client.rb index 7d3d00156..ca4f06ae6 100644 --- a/app/logical/pixiv_api_client.rb +++ b/app/logical/pixiv_api_client.rb @@ -155,7 +155,6 @@ class PixivApiClient def api_client http.headers( - **Danbooru.config.http_headers, "Referer": "http://www.pixiv.net", "Content-Type": "application/x-www-form-urlencoded", "Authorization": "Bearer #{access_token}" diff --git a/app/logical/sources/strategies/base.rb b/app/logical/sources/strategies/base.rb index e4f3d1a6a..f7e4f4b76 100644 --- a/app/logical/sources/strategies/base.rb +++ b/app/logical/sources/strategies/base.rb @@ -141,7 +141,7 @@ module Sources # Subclasses should merge in any required headers needed to access resources # on the site. def headers - Danbooru.config.http_headers + {} end # Returns the size of the image resource without actually downloading the file. diff --git a/config/danbooru_default_config.rb b/config/danbooru_default_config.rb index e52e6a998..f9570949c 100644 --- a/config/danbooru_default_config.rb +++ b/config/danbooru_default_config.rb @@ -340,22 +340,6 @@ module Danbooru def twitter_api_secret end - # The default headers to be sent with outgoing http requests. Some external - # services will fail if you don't set a valid User-Agent. - def http_headers - { - "User-Agent" => "#{Danbooru.config.canonical_app_name}/#{Rails.application.config.x.git_hash}" - } - end - - def httparty_options - # proxy example: - # {http_proxyaddr: "", http_proxyport: "", http_proxyuser: nil, http_proxypass: nil} - { - headers: Danbooru.config.http_headers - } - end - # you should override this def email_key "zDMSATq0W3hmA5p3rKTgD" diff --git a/config/initializers/mechanize_patch.rb b/config/initializers/mechanize_patch.rb deleted file mode 100644 index 875c28b9d..000000000 --- a/config/initializers/mechanize_patch.rb +++ /dev/null @@ -1,64 +0,0 @@ -require 'mechanize' - -if Rails.env.test? - # something about the root certs on the travis ci image causes Mechanize - # to intermittently fail. this is a monkey patch to reset the connection - # after every request to avoid dealing wtiht he issue. - # - # from http://scottwb.com/blog/2013/11/09/defeating-the-infamous-mechanize-too-many-connection-resets-bug/ - class Mechanize::HTTP::Agent - MAX_RESET_RETRIES = 10 - - # We need to replace the core Mechanize HTTP method: - # - # Mechanize::HTTP::Agent#fetch - # - # with a wrapper that handles the infamous "too many connection resets" - # Mechanize bug that is described here: - # - # https://github.com/sparklemotion/mechanize/issues/123 - # - # The wrapper shuts down the persistent HTTP connection when it fails with - # this error, and simply tries again. In practice, this only ever needs to - # be retried once, but I am going to let it retry a few times - # (MAX_RESET_RETRIES), just in case. - # - def fetch_with_retry( - uri, - method = :get, - headers = {}, - params = [], - referer = current_page, - redirects = 0 - ) - action = "#{method.to_s.upcase} #{uri}" - retry_count = 0 - - begin - fetch_without_retry(uri, method, headers, params, referer, redirects) - rescue Net::HTTP::Persistent::Error => e - # Pass on any other type of error. - raise unless e.message =~ /too many connection resets/ - - # Pass on the error if we've tried too many times. - if retry_count >= MAX_RESET_RETRIES - print "R" - # puts "**** WARN: Mechanize retried connection reset #{MAX_RESET_RETRIES} times and never succeeded: #{action}" - raise - end - - # Otherwise, shutdown the persistent HTTP connection and try again. - print "R" - # puts "**** WARN: Mechanize retrying connection reset error: #{action}" - retry_count += 1 - self.http.shutdown - retry - end - end - - # Alias so #fetch actually uses our new #fetch_with_retry to wrap the - # old one aliased as #fetch_without_retry. - alias fetch_without_retry fetch - alias fetch fetch_with_retry - end -end From f020070b7d7871558f37999bcaf87e4145f3ccfb Mon Sep 17 00:00:00 2001 From: evazion Date: Sun, 21 Jun 2020 15:15:47 -0500 Subject: [PATCH 36/59] Add .editorconfig. --- .editorconfig | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 .editorconfig diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 000000000..2f5475e44 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,14 @@ +root = true + +[**.{js,rb,css,erb,md,json,yml}] +indent_style = space +indent_size = 2 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true + +[**.html.erb] +indent_size = unset + +[app/javascript/vendor/**] +indent_size = unset From 4acd89838b034fa23d9ca9181668b75e630c0ac9 Mon Sep 17 00:00:00 2001 From: BrokenEagle Date: Sun, 21 Jun 2020 22:37:59 +0000 Subject: [PATCH 37/59] Add ability to search on status of parent/child --- app/javascript/src/javascripts/autocomplete.js.erb | 13 ++++--------- app/logical/post_query_builder.rb | 4 ++++ test/unit/post_query_builder_test.rb | 13 +++++++++++++ 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/app/javascript/src/javascripts/autocomplete.js.erb b/app/javascript/src/javascripts/autocomplete.js.erb index 28f44f989..49e9abee4 100644 --- a/app/javascript/src/javascripts/autocomplete.js.erb +++ b/app/javascript/src/javascripts/autocomplete.js.erb @@ -9,6 +9,7 @@ Autocomplete.ORDER_METATAGS = <%= PostQueryBuilder::ORDER_METATAGS.to_json.html_ Autocomplete.DISAPPROVAL_REASONS = <%= PostDisapproval::REASONS.to_json.html_safe %>; /* eslint-enable */ +Autocomplete.MISC_STATUSES = ["deleted", "active", "pending", "flagged", "banned", "modqueue", "unmoderated"]; Autocomplete.TAG_PREFIXES = "-|~|" + Object.keys(Autocomplete.TAG_CATEGORIES).map(category => category + ":").join("|"); Autocomplete.METATAGS_REGEX = Autocomplete.METATAGS.concat(Object.keys(Autocomplete.TAG_CATEGORIES)).join("|"); Autocomplete.TERM_REGEX = new RegExp(`([-~]*)(?:(${Autocomplete.METATAGS_REGEX}):)?(\\S*)$`, "i"); @@ -268,9 +269,7 @@ Autocomplete.render_item = function(list, item) { Autocomplete.static_metatags = { order: Autocomplete.ORDER_METATAGS, - status: [ - "any", "deleted", "active", "pending", "flagged", "banned", "modqueue", "unmoderated" - ], + status: ["any"].concat(Autocomplete.MISC_STATUSES), rating: [ "safe", "questionable", "explicit" ], @@ -280,12 +279,8 @@ Autocomplete.static_metatags = { embedded: [ "true", "false" ], - child: [ - "any", "none" - ], - parent: [ - "any", "none" - ], + child: ["any", "none"].concat(Autocomplete.MISC_STATUSES), + parent: ["any", "none"].concat(Autocomplete.MISC_STATUSES), filetype: [ "jpg", "png", "gif", "swf", "zip", "webm", "mp4" ], diff --git a/app/logical/post_query_builder.rb b/app/logical/post_query_builder.rb index 8d3b61d66..0829ce3ba 100644 --- a/app/logical/post_query_builder.rb +++ b/app/logical/post_query_builder.rb @@ -307,6 +307,8 @@ class PostQueryBuilder Post.where(parent: nil) when "any" Post.where.not(parent: nil) + when /pending|flagged|modqueue|deleted|banned|active|unmoderated/ + Post.where.not(parent: nil).where(parent: status_matches(parent)) when /\A\d+\z/ Post.where(id: parent).or(Post.where(parent: parent)) else @@ -320,6 +322,8 @@ class PostQueryBuilder Post.where(has_children: false) when "any" Post.where(has_children: true) + when /pending|flagged|modqueue|deleted|banned|active|unmoderated/ + Post.where(has_children: true).where(children: status_matches(child)) else Post.none end diff --git a/test/unit/post_query_builder_test.rb b/test/unit/post_query_builder_test.rb index d74b08e8e..8323f72e0 100644 --- a/test/unit/post_query_builder_test.rb +++ b/test/unit/post_query_builder_test.rb @@ -277,6 +277,19 @@ class PostQueryBuilderTest < ActiveSupport::TestCase assert_tag_match([child, parent], "-child:garbage") end + should "return posts when using the status of the parent/child" do + parent_of_deleted = create(:post) + deleted = create(:post, is_deleted: true, tag_string: "parent:#{parent_of_deleted.id}") + child_of_deleted = create(:post, tag_string: "parent:#{deleted.id}") + all = [child_of_deleted, deleted, parent_of_deleted] + + assert_tag_match([child_of_deleted], "parent:deleted") + assert_tag_match(all - [child_of_deleted], "-parent:deleted") + + assert_tag_match([parent_of_deleted], "child:deleted") + assert_tag_match(all - [parent_of_deleted], "-child:deleted") + end + should "return posts for the favgroup: metatag" do post1 = create(:post) post2 = create(:post) From f85eef9bcdc35c69395ce457393e5c11868ae024 Mon Sep 17 00:00:00 2001 From: evazion Date: Sun, 21 Jun 2020 18:03:00 -0500 Subject: [PATCH 38/59] nijie: fix bug with retries returning cached responses. Bug: if a Nijie login failed with a 429 Too Many Requests error, the error would get cached, so when we retried the request, we would just get our own cached response back every time. The 429 error would eventually be passed up to the Nijie strategy, which caused random methods to fail because they couldn't get the html page. Fix: add the `retriable` feature *after* the `cache` feature so that retries don't go through the cache. This is a hack. We want retries to go at the bottom of the stack, below caching, but we can't enforce this ordering. --- app/logical/danbooru/http.rb | 1 - app/logical/danbooru/http/retriable.rb | 2 ++ app/logical/sources/strategies/nijie.rb | 5 ++++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/app/logical/danbooru/http.rb b/app/logical/danbooru/http.rb index 590271fe0..8e8655965 100644 --- a/app/logical/danbooru/http.rb +++ b/app/logical/danbooru/http.rb @@ -25,7 +25,6 @@ module Danbooru .headers("Accept-Encoding" => "gzip") .headers("User-Agent": "#{Danbooru.config.canonical_app_name}/#{Rails.application.config.x.git_hash}") .use(:auto_inflate) - .use(:retriable) .use(redirector: { max_redirects: MAX_REDIRECTS }) .use(:session) end diff --git a/app/logical/danbooru/http/retriable.rb b/app/logical/danbooru/http/retriable.rb index 681dd8bbe..23d5e865a 100644 --- a/app/logical/danbooru/http/retriable.rb +++ b/app/logical/danbooru/http/retriable.rb @@ -21,6 +21,8 @@ module Danbooru retries = max_retries while retriable?(response) && retries > 0 && retry_delay(response) <= max_delay + DanbooruLogger.info "Retrying url=#{request.uri} status=#{response.status} retries=#{retries} delay=#{retry_delay(response)}" + retries -= 1 sleep(retry_delay(response)) response = yield request diff --git a/app/logical/sources/strategies/nijie.rb b/app/logical/sources/strategies/nijie.rb index f3a15b576..b2a6fcc7a 100644 --- a/app/logical/sources/strategies/nijie.rb +++ b/app/logical/sources/strategies/nijie.rb @@ -180,7 +180,10 @@ module Sources http = Danbooru::Http.new form = { email: Danbooru.config.nijie_login, password: Danbooru.config.nijie_password } - response = http.cache(1.hour).post("https://nijie.info/login_int.php", form: form) + + # XXX `retriable` must come after `cache` so that retries don't return cached error responses. + response = http.cache(1.hour).use(:retriable).post("https://nijie.info/login_int.php", form: form) + DanbooruLogger.info "Nijie login failed (#{url}, #{response.status})" if response.status != 200 return nil unless response.status == 200 response = http.cookies(R18: 1).cache(1.minute).get(page_url) From bd25be95f52f99e69e573e44d295751478bfed63 Mon Sep 17 00:00:00 2001 From: evazion Date: Sun, 21 Jun 2020 18:27:32 -0500 Subject: [PATCH 39/59] danbooru::http: factor out cache feature. Fixes a bug with cookies stored by the `session` feature not being sent with cached requests. --- app/logical/danbooru/http.rb | 32 +++++++----------------------- app/logical/danbooru/http/cache.rb | 30 ++++++++++++++++++++++++++++ test/unit/danbooru_http_test.rb | 31 +++++++++++++++++++++++++---- 3 files changed, 64 insertions(+), 29 deletions(-) create mode 100644 app/logical/danbooru/http/cache.rb diff --git a/app/logical/danbooru/http.rb b/app/logical/danbooru/http.rb index 8e8655965..6bbc7db87 100644 --- a/app/logical/danbooru/http.rb +++ b/app/logical/danbooru/http.rb @@ -1,5 +1,6 @@ require "danbooru/http/html_adapter" require "danbooru/http/xml_adapter" +require "danbooru/http/cache" require "danbooru/http/redirector" require "danbooru/http/retriable" require "danbooru/http/session" @@ -12,7 +13,7 @@ module Danbooru DEFAULT_TIMEOUT = 10 MAX_REDIRECTS = 5 - attr_accessor :cache, :max_size, :http + attr_accessor :max_size, :http class << self delegate :get, :head, :put, :post, :delete, :cache, :follow, :max_size, :timeout, :auth, :basic_auth, :headers, :cookies, :use, :public_only, :download_media, to: :new @@ -49,10 +50,6 @@ module Danbooru request(:delete, url, **options) end - def cache(expiry) - dup.tap { |o| o.cache = expiry.to_i } - end - def follow(*args) dup.tap { |o| o.http = o.http.follow(*args) } end @@ -85,6 +82,10 @@ module Danbooru dup.tap { |o| o.http = o.http.use(*args) } end + def cache(expires_in) + use(cache: { expires_in: expires_in }) + end + # allow requests only to public IPs, not to local or private networks. def public_only dup.tap do |o| @@ -128,11 +129,7 @@ module Danbooru protected def request(method, url, **options) - if @cache.present? - cached_request(method, url, **options) - else - raw_request(method, url, **options) - end + http.send(method, url, **options) rescue ValidatingSocket::ProhibitedIpError fake_response(597, "") rescue HTTP::Redirector::TooManyRedirectsError @@ -141,21 +138,6 @@ module Danbooru fake_response(599, "") end - def cached_request(method, url, **options) - key = Cache.hash({ method: method, url: url, headers: http.default_options.headers.to_h, **options }.to_json) - - cached_response = Cache.get(key, @cache) do - response = raw_request(method, url, **options) - { status: response.status, body: response.to_s, headers: response.headers.to_h, version: "1.1" } - end - - ::HTTP::Response.new(**cached_response) - end - - def raw_request(method, url, **options) - http.send(method, url, **options) - end - def fake_response(status, body) ::HTTP::Response.new(status: status, version: "1.1", body: ::HTTP::Response::Body.new(body)) end diff --git a/app/logical/danbooru/http/cache.rb b/app/logical/danbooru/http/cache.rb new file mode 100644 index 000000000..43932bd28 --- /dev/null +++ b/app/logical/danbooru/http/cache.rb @@ -0,0 +1,30 @@ +module Danbooru + class Http + class Cache < HTTP::Feature + HTTP::Options.register_feature :cache, self + + attr_reader :expires_in + + def initialize(expires_in:) + @expires_in = expires_in + end + + def perform(request, &block) + ::Cache.get(cache_key(request), expires_in) do + response = yield request + + # XXX hack to remove connection state from response body so we can serialize it for caching. + response.flush + response.body.instance_variable_set(:@connection, nil) + response.body.instance_variable_set(:@stream, nil) + + response + end + end + + def cache_key(request) + "http:" + ::Cache.hash({ method: request.verb, url: request.uri.to_s, headers: request.headers.sort }.to_json) + end + end + end +end diff --git a/test/unit/danbooru_http_test.rb b/test/unit/danbooru_http_test.rb index b6ef81e7c..aa73976ea 100644 --- a/test/unit/danbooru_http_test.rb +++ b/test/unit/danbooru_http_test.rb @@ -62,14 +62,37 @@ class DanbooruHttpTest < ActiveSupport::TestCase resp4 = http.cookies(def: 3, ghi: 4).get("https://httpbin.org/cookies") assert_equal({ abc: "1", def: "3", ghi: "4" }, resp4.parse["cookies"].symbolize_keys) end + end - should "cache requests" do - response1 = Danbooru::Http.cache(1.minute).get("https://httpbin.org/uuid") + context "cache feature" do + should "cache multiple requests to the same url" do + http = Danbooru::Http.cache(1.hour) + + response1 = http.get("https://httpbin.org/uuid") assert_equal(200, response1.status) - response2 = Danbooru::Http.cache(1.minute).get("https://httpbin.org/uuid") + response2 = http.get("https://httpbin.org/uuid") assert_equal(200, response2.status) - assert_equal(response2.body, response1.body) + assert_equal(response2.to_s, response1.to_s) + end + + should "cache cookies correctly" do + http = Danbooru::Http.cache(1.hour) + + resp1 = http.get("https://httpbin.org/cookies") + resp2 = http.get("https://httpbin.org/cookies/set/abc/1") + resp3 = http.get("https://httpbin.org/cookies/set/def/2") + resp4 = http.get("https://httpbin.org/cookies") + + assert_equal(200, resp1.status) + assert_equal(200, resp2.status) + assert_equal(200, resp3.status) + assert_equal(200, resp4.status) + + assert_equal({}, resp1.parse["cookies"].symbolize_keys) + assert_equal({ abc: "1" }, resp2.parse["cookies"].symbolize_keys) + assert_equal({ abc: "1", def: "2" }, resp3.parse["cookies"].symbolize_keys) + assert_equal({ abc: "1", def: "2" }, resp4.parse["cookies"].symbolize_keys) end end From 504edff14be0801a335f12fc480d889285180b08 Mon Sep 17 00:00:00 2001 From: evazion Date: Sun, 21 Jun 2020 18:49:08 -0500 Subject: [PATCH 40/59] Update ruby gems and yarn packages. --- Gemfile.lock | 18 +- yarn.lock | 587 +++++++++++++++++++++------------------------------ 2 files changed, 253 insertions(+), 352 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index a8fa40f17..9ae511510 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -77,7 +77,7 @@ GEM ansi (1.5.0) ast (2.4.1) aws-eventstream (1.1.0) - aws-partitions (1.329.0) + aws-partitions (1.332.0) aws-sdk-core (3.100.0) aws-eventstream (~> 1, >= 1.0.2) aws-partitions (~> 1, >= 1.239.0) @@ -86,8 +86,8 @@ GEM aws-sdk-sqs (1.27.1) aws-sdk-core (~> 3, >= 3.99.0) aws-sigv4 (~> 1.1) - aws-sigv4 (1.1.4) - aws-eventstream (~> 1.0, >= 1.0.2) + aws-sigv4 (1.2.0) + aws-eventstream (~> 1, >= 1.0.2) bcrypt (3.1.13) bootsnap (1.4.6) msgpack (~> 1.0) @@ -110,7 +110,7 @@ GEM sshkit (~> 1.3) capistrano3-unicorn (0.2.1) capistrano (~> 3.1, >= 3.1.0) - capybara (3.32.2) + capybara (3.33.0) addressable mini_mime (>= 0.1.3) nokogiri (~> 1.8) @@ -138,8 +138,8 @@ GEM dotenv (= 2.7.5) railties (>= 3.2, < 6.1) erubi (1.9.0) - factory_bot (5.2.0) - activesupport (>= 4.2.0) + factory_bot (6.0.2) + activesupport (>= 5.0.0) faraday (1.0.1) multipart-post (>= 1.2, < 3) ffaker (2.15.0) @@ -226,9 +226,9 @@ GEM multi_json (~> 1.3) multi_xml (~> 0.5) rack (>= 1.2, < 3) - parallel (1.19.1) - parser (2.7.1.3) - ast (~> 2.4.0) + parallel (1.19.2) + parser (2.7.1.4) + ast (~> 2.4.1) pg (1.2.3) pg (1.2.3-x64-mingw32) pry (0.13.1) diff --git a/yarn.lock b/yarn.lock index f062816ea..c8abadcae 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,35 +2,35 @@ # yarn lockfile v1 -"@babel/code-frame@^7.0.0", "@babel/code-frame@^7.10.1": - version "7.10.1" - resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.10.1.tgz#d5481c5095daa1c57e16e54c6f9198443afb49ff" - integrity sha512-IGhtTmpjGbYzcEDOw7DcQtbQSXcG9ftmAXtWTu9V936vDye4xjjekktFAtgZsWpzTj/X01jocB46mTywm/4SZw== +"@babel/code-frame@^7.0.0", "@babel/code-frame@^7.10.3": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.10.3.tgz#324bcfd8d35cd3d47dae18cde63d752086435e9a" + integrity sha512-fDx9eNW0qz0WkUeqL6tXEXzVlPh6Y5aCDEZesl0xBGA8ndRukX91Uk44ZqnkECp01NAZUdCAl+aiQNGi0k88Eg== dependencies: - "@babel/highlight" "^7.10.1" + "@babel/highlight" "^7.10.3" -"@babel/compat-data@^7.10.1": - version "7.10.1" - resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.10.1.tgz#b1085ffe72cd17bf2c0ee790fc09f9626011b2db" - integrity sha512-CHvCj7So7iCkGKPRFUfryXIkU2gSBw7VSZFYLsqVhrS47269VK2Hfi9S/YcublPMW8k1u2bQBlbDruoQEm4fgw== +"@babel/compat-data@^7.10.1", "@babel/compat-data@^7.10.3": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.10.3.tgz#9af3e033f36e8e2d6e47570db91e64a846f5d382" + integrity sha512-BDIfJ9uNZuI0LajPfoYV28lX8kyCPMHY6uY4WH1lJdcicmAfxCK5ASzaeV0D/wsUaRH/cLk+amuxtC37sZ8TUg== dependencies: browserslist "^4.12.0" invariant "^2.2.4" semver "^5.5.0" "@babel/core@>=7.9.0", "@babel/core@^7.9.0": - version "7.10.2" - resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.10.2.tgz#bd6786046668a925ac2bd2fd95b579b92a23b36a" - integrity sha512-KQmV9yguEjQsXqyOUGKjS4+3K8/DlOCE2pZcq4augdQmtTy5iv5EHtmMSJ7V4c1BIPjuwtZYqYLCq9Ga+hGBRQ== + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.10.3.tgz#73b0e8ddeec1e3fdd7a2de587a60e17c440ec77e" + integrity sha512-5YqWxYE3pyhIi84L84YcwjeEgS+fa7ZjK6IBVGTjDVfm64njkR2lfDhVR5OudLk8x2GK59YoSyVv+L/03k1q9w== dependencies: - "@babel/code-frame" "^7.10.1" - "@babel/generator" "^7.10.2" + "@babel/code-frame" "^7.10.3" + "@babel/generator" "^7.10.3" "@babel/helper-module-transforms" "^7.10.1" "@babel/helpers" "^7.10.1" - "@babel/parser" "^7.10.2" - "@babel/template" "^7.10.1" - "@babel/traverse" "^7.10.1" - "@babel/types" "^7.10.2" + "@babel/parser" "^7.10.3" + "@babel/template" "^7.10.3" + "@babel/traverse" "^7.10.3" + "@babel/types" "^7.10.3" convert-source-map "^1.7.0" debug "^4.1.0" gensync "^1.0.0-beta.1" @@ -40,12 +40,12 @@ semver "^5.4.1" source-map "^0.5.0" -"@babel/generator@^7.10.1", "@babel/generator@^7.10.2": - version "7.10.2" - resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.10.2.tgz#0fa5b5b2389db8bfdfcc3492b551ee20f5dd69a9" - integrity sha512-AxfBNHNu99DTMvlUPlt1h2+Hn7knPpH5ayJ8OqDWSeLld+Fi2AYBTC/IejWDM9Edcii4UzZRCsbUt0WlSDsDsA== +"@babel/generator@^7.10.3": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.10.3.tgz#32b9a0d963a71d7a54f5f6c15659c3dbc2a523a5" + integrity sha512-drt8MUHbEqRzNR0xnF8nMehbY11b1SDkRw03PSNH/3Rb2Z35oxkddVSi3rcaak0YJQ86PCuE7Qx1jSFhbLNBMA== dependencies: - "@babel/types" "^7.10.2" + "@babel/types" "^7.10.3" jsesc "^2.5.1" lodash "^4.17.13" source-map "^0.5.0" @@ -58,12 +58,12 @@ "@babel/types" "^7.10.1" "@babel/helper-builder-binary-assignment-operator-visitor@^7.10.1": - version "7.10.1" - resolved "https://registry.yarnpkg.com/@babel/helper-builder-binary-assignment-operator-visitor/-/helper-builder-binary-assignment-operator-visitor-7.10.1.tgz#0ec7d9be8174934532661f87783eb18d72290059" - integrity sha512-cQpVq48EkYxUU0xozpGCLla3wlkdRRqLWu1ksFMXA9CM5KQmyyRpSEsYXbao7JUkOw/tAaYKCaYyZq6HOFYtyw== + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/helper-builder-binary-assignment-operator-visitor/-/helper-builder-binary-assignment-operator-visitor-7.10.3.tgz#4e9012d6701bef0030348d7f9c808209bd3e8687" + integrity sha512-lo4XXRnBlU6eRM92FkiZxpo1xFLmv3VsPFk61zJKMm7XYJfwqXHsYJTY6agoc4a3L8QPw1HqWehO18coZgbT6A== dependencies: - "@babel/helper-explode-assignable-expression" "^7.10.1" - "@babel/types" "^7.10.1" + "@babel/helper-explode-assignable-expression" "^7.10.3" + "@babel/types" "^7.10.3" "@babel/helper-compilation-targets@^7.10.2": version "7.10.2" @@ -77,14 +77,14 @@ semver "^5.5.0" "@babel/helper-create-class-features-plugin@^7.10.1": - version "7.10.2" - resolved "https://registry.yarnpkg.com/@babel/helper-create-class-features-plugin/-/helper-create-class-features-plugin-7.10.2.tgz#7474295770f217dbcf288bf7572eb213db46ee67" - integrity sha512-5C/QhkGFh1vqcziq1vAL6SI9ymzUp8BCYjFpvYVhWP4DlATIb3u5q3iUd35mvlyGs8fO7hckkW7i0tmH+5+bvQ== + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/helper-create-class-features-plugin/-/helper-create-class-features-plugin-7.10.3.tgz#2783daa6866822e3d5ed119163b50f0fc3ae4b35" + integrity sha512-iRT9VwqtdFmv7UheJWthGc/h2s7MqoweBF9RUj77NFZsg9VfISvBTum3k6coAhJ8RWv2tj3yUjA03HxPd0vfpQ== dependencies: - "@babel/helper-function-name" "^7.10.1" - "@babel/helper-member-expression-to-functions" "^7.10.1" - "@babel/helper-optimise-call-expression" "^7.10.1" - "@babel/helper-plugin-utils" "^7.10.1" + "@babel/helper-function-name" "^7.10.3" + "@babel/helper-member-expression-to-functions" "^7.10.3" + "@babel/helper-optimise-call-expression" "^7.10.3" + "@babel/helper-plugin-utils" "^7.10.3" "@babel/helper-replace-supers" "^7.10.1" "@babel/helper-split-export-declaration" "^7.10.1" @@ -97,59 +97,59 @@ "@babel/helper-regex" "^7.10.1" regexpu-core "^4.7.0" -"@babel/helper-define-map@^7.10.1": - version "7.10.1" - resolved "https://registry.yarnpkg.com/@babel/helper-define-map/-/helper-define-map-7.10.1.tgz#5e69ee8308648470dd7900d159c044c10285221d" - integrity sha512-+5odWpX+OnvkD0Zmq7panrMuAGQBu6aPUgvMzuMGo4R+jUOvealEj2hiqI6WhxgKrTpFoFj0+VdsuA8KDxHBDg== +"@babel/helper-define-map@^7.10.3": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/helper-define-map/-/helper-define-map-7.10.3.tgz#d27120a5e57c84727b30944549b2dfeca62401a8" + integrity sha512-bxRzDi4Sin/k0drWCczppOhov1sBSdBvXJObM1NLHQzjhXhwRtn7aRWGvLJWCYbuu2qUk3EKs6Ci9C9ps8XokQ== dependencies: - "@babel/helper-function-name" "^7.10.1" - "@babel/types" "^7.10.1" + "@babel/helper-function-name" "^7.10.3" + "@babel/types" "^7.10.3" lodash "^4.17.13" -"@babel/helper-explode-assignable-expression@^7.10.1": - version "7.10.1" - resolved "https://registry.yarnpkg.com/@babel/helper-explode-assignable-expression/-/helper-explode-assignable-expression-7.10.1.tgz#e9d76305ee1162ca467357ae25df94f179af2b7e" - integrity sha512-vcUJ3cDjLjvkKzt6rHrl767FeE7pMEYfPanq5L16GRtrXIoznc0HykNW2aEYkcnP76P0isoqJ34dDMFZwzEpJg== +"@babel/helper-explode-assignable-expression@^7.10.3": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/helper-explode-assignable-expression/-/helper-explode-assignable-expression-7.10.3.tgz#9dc14f0cfa2833ea830a9c8a1c742b6e7461b05e" + integrity sha512-0nKcR64XrOC3lsl+uhD15cwxPvaB6QKUDlD84OT9C3myRbhJqTMYir69/RWItUvHpharv0eJ/wk7fl34ONSwZw== dependencies: - "@babel/traverse" "^7.10.1" - "@babel/types" "^7.10.1" + "@babel/traverse" "^7.10.3" + "@babel/types" "^7.10.3" -"@babel/helper-function-name@^7.10.1": - version "7.10.1" - resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.10.1.tgz#92bd63829bfc9215aca9d9defa85f56b539454f4" - integrity sha512-fcpumwhs3YyZ/ttd5Rz0xn0TpIwVkN7X0V38B9TWNfVF42KEkhkAAuPCQ3oXmtTRtiPJrmZ0TrfS0GKF0eMaRQ== +"@babel/helper-function-name@^7.10.1", "@babel/helper-function-name@^7.10.3": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.10.3.tgz#79316cd75a9fa25ba9787ff54544307ed444f197" + integrity sha512-FvSj2aiOd8zbeqijjgqdMDSyxsGHaMt5Tr0XjQsGKHD3/1FP3wksjnLAWzxw7lvXiej8W1Jt47SKTZ6upQNiRw== dependencies: - "@babel/helper-get-function-arity" "^7.10.1" - "@babel/template" "^7.10.1" - "@babel/types" "^7.10.1" + "@babel/helper-get-function-arity" "^7.10.3" + "@babel/template" "^7.10.3" + "@babel/types" "^7.10.3" -"@babel/helper-get-function-arity@^7.10.1": - version "7.10.1" - resolved "https://registry.yarnpkg.com/@babel/helper-get-function-arity/-/helper-get-function-arity-7.10.1.tgz#7303390a81ba7cb59613895a192b93850e373f7d" - integrity sha512-F5qdXkYGOQUb0hpRaPoetF9AnsXknKjWMZ+wmsIRsp5ge5sFh4c3h1eH2pRTTuy9KKAA2+TTYomGXAtEL2fQEw== +"@babel/helper-get-function-arity@^7.10.1", "@babel/helper-get-function-arity@^7.10.3": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/helper-get-function-arity/-/helper-get-function-arity-7.10.3.tgz#3a28f7b28ccc7719eacd9223b659fdf162e4c45e" + integrity sha512-iUD/gFsR+M6uiy69JA6fzM5seno8oE85IYZdbVVEuQaZlEzMO2MXblh+KSPJgsZAUx0EEbWXU0yJaW7C9CdAVg== dependencies: - "@babel/types" "^7.10.1" + "@babel/types" "^7.10.3" -"@babel/helper-hoist-variables@^7.10.1": - version "7.10.1" - resolved "https://registry.yarnpkg.com/@babel/helper-hoist-variables/-/helper-hoist-variables-7.10.1.tgz#7e77c82e5dcae1ebf123174c385aaadbf787d077" - integrity sha512-vLm5srkU8rI6X3+aQ1rQJyfjvCBLXP8cAGeuw04zeAM2ItKb1e7pmVmLyHb4sDaAYnLL13RHOZPLEtcGZ5xvjg== +"@babel/helper-hoist-variables@^7.10.3": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/helper-hoist-variables/-/helper-hoist-variables-7.10.3.tgz#d554f52baf1657ffbd7e5137311abc993bb3f068" + integrity sha512-9JyafKoBt5h20Yv1+BXQMdcXXavozI1vt401KBiRc2qzUepbVnd7ogVNymY1xkQN9fekGwfxtotH2Yf5xsGzgg== dependencies: - "@babel/types" "^7.10.1" + "@babel/types" "^7.10.3" -"@babel/helper-member-expression-to-functions@^7.10.1": - version "7.10.1" - resolved "https://registry.yarnpkg.com/@babel/helper-member-expression-to-functions/-/helper-member-expression-to-functions-7.10.1.tgz#432967fd7e12a4afef66c4687d4ca22bc0456f15" - integrity sha512-u7XLXeM2n50gb6PWJ9hoO5oO7JFPaZtrh35t8RqKLT1jFKj9IWeD1zrcrYp1q1qiZTdEarfDWfTIP8nGsu0h5g== +"@babel/helper-member-expression-to-functions@^7.10.1", "@babel/helper-member-expression-to-functions@^7.10.3": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/helper-member-expression-to-functions/-/helper-member-expression-to-functions-7.10.3.tgz#bc3663ac81ac57c39148fef4c69bf48a77ba8dd6" + integrity sha512-q7+37c4EPLSjNb2NmWOjNwj0+BOyYlssuQ58kHEWk1Z78K5i8vTUsteq78HMieRPQSl/NtpQyJfdjt3qZ5V2vw== dependencies: - "@babel/types" "^7.10.1" + "@babel/types" "^7.10.3" -"@babel/helper-module-imports@^7.10.1": - version "7.10.1" - resolved "https://registry.yarnpkg.com/@babel/helper-module-imports/-/helper-module-imports-7.10.1.tgz#dd331bd45bccc566ce77004e9d05fe17add13876" - integrity sha512-SFxgwYmZ3HZPyZwJRiVNLRHWuW2OgE5k2nrVs6D9Iv4PPnXVffuEHy83Sfx/l4SqF+5kyJXjAyUmrG7tNm+qVg== +"@babel/helper-module-imports@^7.10.1", "@babel/helper-module-imports@^7.10.3": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/helper-module-imports/-/helper-module-imports-7.10.3.tgz#766fa1d57608e53e5676f23ae498ec7a95e1b11a" + integrity sha512-Jtqw5M9pahLSUWA+76nhK9OG8nwYXzhQzVIGFoNaHnXF/r4l7kz4Fl0UAW7B6mqC5myoJiBP5/YQlXQTMfHI9w== dependencies: - "@babel/types" "^7.10.1" + "@babel/types" "^7.10.3" "@babel/helper-module-transforms@^7.10.1": version "7.10.1" @@ -164,17 +164,17 @@ "@babel/types" "^7.10.1" lodash "^4.17.13" -"@babel/helper-optimise-call-expression@^7.10.1": - version "7.10.1" - resolved "https://registry.yarnpkg.com/@babel/helper-optimise-call-expression/-/helper-optimise-call-expression-7.10.1.tgz#b4a1f2561870ce1247ceddb02a3860fa96d72543" - integrity sha512-a0DjNS1prnBsoKx83dP2falChcs7p3i8VMzdrSbfLhuQra/2ENC4sbri34dz/rWmDADsmF1q5GbfaXydh0Jbjg== +"@babel/helper-optimise-call-expression@^7.10.1", "@babel/helper-optimise-call-expression@^7.10.3": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/helper-optimise-call-expression/-/helper-optimise-call-expression-7.10.3.tgz#f53c4b6783093195b0f69330439908841660c530" + integrity sha512-kT2R3VBH/cnSz+yChKpaKRJQJWxdGoc6SjioRId2wkeV3bK0wLLioFpJROrX0U4xr/NmxSSAWT/9Ih5snwIIzg== dependencies: - "@babel/types" "^7.10.1" + "@babel/types" "^7.10.3" -"@babel/helper-plugin-utils@^7.0.0", "@babel/helper-plugin-utils@^7.10.1", "@babel/helper-plugin-utils@^7.8.0": - version "7.10.1" - resolved "https://registry.yarnpkg.com/@babel/helper-plugin-utils/-/helper-plugin-utils-7.10.1.tgz#ec5a5cf0eec925b66c60580328b122c01230a127" - integrity sha512-fvoGeXt0bJc7VMWZGCAEBEMo/HAjW2mP8apF5eXK0wSqwLAVHAISCWRoLMBMUs2kqeaG77jltVqu4Hn8Egl3nA== +"@babel/helper-plugin-utils@^7.0.0", "@babel/helper-plugin-utils@^7.10.1", "@babel/helper-plugin-utils@^7.10.3", "@babel/helper-plugin-utils@^7.8.0": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/helper-plugin-utils/-/helper-plugin-utils-7.10.3.tgz#aac45cccf8bc1873b99a85f34bceef3beb5d3244" + integrity sha512-j/+j8NAWUTxOtx4LKHybpSClxHoq6I91DQ/mKgAXn5oNUPIUiGppjPIX3TDtJWPrdfP9Kfl7e4fgVMiQR9VE/g== "@babel/helper-regex@^7.10.1": version "7.10.1" @@ -183,16 +183,16 @@ dependencies: lodash "^4.17.13" -"@babel/helper-remap-async-to-generator@^7.10.1": - version "7.10.1" - resolved "https://registry.yarnpkg.com/@babel/helper-remap-async-to-generator/-/helper-remap-async-to-generator-7.10.1.tgz#bad6aaa4ff39ce8d4b82ccaae0bfe0f7dbb5f432" - integrity sha512-RfX1P8HqsfgmJ6CwaXGKMAqbYdlleqglvVtht0HGPMSsy2V6MqLlOJVF/0Qyb/m2ZCi2z3q3+s6Pv7R/dQuZ6A== +"@babel/helper-remap-async-to-generator@^7.10.1", "@babel/helper-remap-async-to-generator@^7.10.3": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/helper-remap-async-to-generator/-/helper-remap-async-to-generator-7.10.3.tgz#18564f8a6748be466970195b876e8bba3bccf442" + integrity sha512-sLB7666ARbJUGDO60ZormmhQOyqMX/shKBXZ7fy937s+3ID8gSrneMvKSSb+8xIM5V7Vn6uNVtOY1vIm26XLtA== dependencies: "@babel/helper-annotate-as-pure" "^7.10.1" "@babel/helper-wrap-function" "^7.10.1" - "@babel/template" "^7.10.1" - "@babel/traverse" "^7.10.1" - "@babel/types" "^7.10.1" + "@babel/template" "^7.10.3" + "@babel/traverse" "^7.10.3" + "@babel/types" "^7.10.3" "@babel/helper-replace-supers@^7.10.1": version "7.10.1" @@ -219,10 +219,10 @@ dependencies: "@babel/types" "^7.10.1" -"@babel/helper-validator-identifier@^7.10.1": - version "7.10.1" - resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.10.1.tgz#5770b0c1a826c4f53f5ede5e153163e0318e94b5" - integrity sha512-5vW/JXLALhczRCWP0PnFDMCJAchlBvM7f4uk/jXritBnIa6E1KmqmtrS3yn1LAnxFBypQ3eneLuXjsnfQsgILw== +"@babel/helper-validator-identifier@^7.10.3": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.10.3.tgz#60d9847f98c4cea1b279e005fdb7c28be5412d15" + integrity sha512-bU8JvtlYpJSBPuj1VUmKpFGaDZuLxASky3LhaKj3bmpSTY6VWooSM8msk+Z0CZoErFye2tlABF6yDkT3FOPAXw== "@babel/helper-wrap-function@^7.10.1": version "7.10.1" @@ -243,27 +243,27 @@ "@babel/traverse" "^7.10.1" "@babel/types" "^7.10.1" -"@babel/highlight@^7.10.1": - version "7.10.1" - resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.10.1.tgz#841d098ba613ba1a427a2b383d79e35552c38ae0" - integrity sha512-8rMof+gVP8mxYZApLF/JgNDAkdKa+aJt3ZYxF8z6+j/hpeXL7iMsKCPHa2jNMHu/qqBwzQF4OHNoYi8dMA/rYg== +"@babel/highlight@^7.10.3": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.10.3.tgz#c633bb34adf07c5c13156692f5922c81ec53f28d" + integrity sha512-Ih9B/u7AtgEnySE2L2F0Xm0GaM729XqqLfHkalTsbjXGyqmf/6M0Cu0WpvqueUlW+xk88BHw9Nkpj49naU+vWw== dependencies: - "@babel/helper-validator-identifier" "^7.10.1" + "@babel/helper-validator-identifier" "^7.10.3" chalk "^2.0.0" js-tokens "^4.0.0" -"@babel/parser@^7.10.1", "@babel/parser@^7.10.2", "@babel/parser@^7.7.0": - version "7.10.2" - resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.10.2.tgz#871807f10442b92ff97e4783b9b54f6a0ca812d0" - integrity sha512-PApSXlNMJyB4JiGVhCOlzKIif+TKFTvu0aQAhnTvfP/z3vVSN6ZypH5bfUNwFXXjRQtUEBNFd2PtmCmG2Py3qQ== +"@babel/parser@^7.10.3", "@babel/parser@^7.7.0": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.10.3.tgz#7e71d892b0d6e7d04a1af4c3c79d72c1f10f5315" + integrity sha512-oJtNJCMFdIMwXGmx+KxuaD7i3b8uS7TTFYW/FNG2BT8m+fmGHoiPYoH0Pe3gya07WuFmM5FCDIr1x0irkD/hyA== -"@babel/plugin-proposal-async-generator-functions@^7.10.1": - version "7.10.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-async-generator-functions/-/plugin-proposal-async-generator-functions-7.10.1.tgz#6911af5ba2e615c4ff3c497fe2f47b35bf6d7e55" - integrity sha512-vzZE12ZTdB336POZjmpblWfNNRpMSua45EYnRigE2XsZxcXcIyly2ixnTJasJE4Zq3U7t2d8rRF7XRUuzHxbOw== +"@babel/plugin-proposal-async-generator-functions@^7.10.3": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-async-generator-functions/-/plugin-proposal-async-generator-functions-7.10.3.tgz#5a02453d46e5362e2073c7278beab2e53ad7d939" + integrity sha512-WUUWM7YTOudF4jZBAJIW9D7aViYC/Fn0Pln4RIHlQALyno3sXSjqmTA4Zy1TKC2D49RCR8Y/Pn4OIUtEypK3CA== dependencies: - "@babel/helper-plugin-utils" "^7.10.1" - "@babel/helper-remap-async-to-generator" "^7.10.1" + "@babel/helper-plugin-utils" "^7.10.3" + "@babel/helper-remap-async-to-generator" "^7.10.3" "@babel/plugin-syntax-async-generators" "^7.8.0" "@babel/plugin-proposal-class-properties@^7.10.1", "@babel/plugin-proposal-class-properties@^7.8.3": @@ -306,12 +306,12 @@ "@babel/helper-plugin-utils" "^7.10.1" "@babel/plugin-syntax-numeric-separator" "^7.10.1" -"@babel/plugin-proposal-object-rest-spread@^7.10.1", "@babel/plugin-proposal-object-rest-spread@^7.9.0": - version "7.10.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-object-rest-spread/-/plugin-proposal-object-rest-spread-7.10.1.tgz#cba44908ac9f142650b4a65b8aa06bf3478d5fb6" - integrity sha512-Z+Qri55KiQkHh7Fc4BW6o+QBuTagbOp9txE+4U1i79u9oWlf2npkiDx+Rf3iK3lbcHBuNy9UOkwuR5wOMH3LIQ== +"@babel/plugin-proposal-object-rest-spread@^7.10.3", "@babel/plugin-proposal-object-rest-spread@^7.9.0": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-object-rest-spread/-/plugin-proposal-object-rest-spread-7.10.3.tgz#b8d0d22f70afa34ad84b7a200ff772f9b9fce474" + integrity sha512-ZZh5leCIlH9lni5bU/wB/UcjtcVLgR8gc+FAgW2OOY+m9h1II3ItTO1/cewNUcsIDZSYcSaz/rYVls+Fb0ExVQ== dependencies: - "@babel/helper-plugin-utils" "^7.10.1" + "@babel/helper-plugin-utils" "^7.10.3" "@babel/plugin-syntax-object-rest-spread" "^7.8.0" "@babel/plugin-transform-parameters" "^7.10.1" @@ -323,12 +323,12 @@ "@babel/helper-plugin-utils" "^7.10.1" "@babel/plugin-syntax-optional-catch-binding" "^7.8.0" -"@babel/plugin-proposal-optional-chaining@^7.10.1": - version "7.10.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-optional-chaining/-/plugin-proposal-optional-chaining-7.10.1.tgz#15f5d6d22708629451a91be28f8facc55b0e818c" - integrity sha512-dqQj475q8+/avvok72CF3AOSV/SGEcH29zT5hhohqqvvZ2+boQoOr7iGldBG5YXTO2qgCgc2B3WvVLUdbeMlGA== +"@babel/plugin-proposal-optional-chaining@^7.10.3": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-optional-chaining/-/plugin-proposal-optional-chaining-7.10.3.tgz#9a726f94622b653c0a3a7a59cdce94730f526f7c" + integrity sha512-yyG3n9dJ1vZ6v5sfmIlMMZ8azQoqx/5/nZTSWX1td6L1H1bsjzA8TInDChpafCZiJkeOFzp/PtrfigAQXxI1Ng== dependencies: - "@babel/helper-plugin-utils" "^7.10.1" + "@babel/helper-plugin-utils" "^7.10.3" "@babel/plugin-syntax-optional-chaining" "^7.8.0" "@babel/plugin-proposal-private-methods@^7.10.1": @@ -448,26 +448,26 @@ "@babel/helper-plugin-utils" "^7.10.1" lodash "^4.17.13" -"@babel/plugin-transform-classes@^7.10.1": - version "7.10.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-classes/-/plugin-transform-classes-7.10.1.tgz#6e11dd6c4dfae70f540480a4702477ed766d733f" - integrity sha512-P9V0YIh+ln/B3RStPoXpEQ/CoAxQIhRSUn7aXqQ+FZJ2u8+oCtjIXR3+X0vsSD8zv+mb56K7wZW1XiDTDGiDRQ== +"@babel/plugin-transform-classes@^7.10.3": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-classes/-/plugin-transform-classes-7.10.3.tgz#8d9a656bc3d01f3ff69e1fccb354b0f9d72ac544" + integrity sha512-irEX0ChJLaZVC7FvvRoSIxJlmk0IczFLcwaRXUArBKYHCHbOhe57aG8q3uw/fJsoSXvZhjRX960hyeAGlVBXZw== dependencies: "@babel/helper-annotate-as-pure" "^7.10.1" - "@babel/helper-define-map" "^7.10.1" - "@babel/helper-function-name" "^7.10.1" - "@babel/helper-optimise-call-expression" "^7.10.1" - "@babel/helper-plugin-utils" "^7.10.1" + "@babel/helper-define-map" "^7.10.3" + "@babel/helper-function-name" "^7.10.3" + "@babel/helper-optimise-call-expression" "^7.10.3" + "@babel/helper-plugin-utils" "^7.10.3" "@babel/helper-replace-supers" "^7.10.1" "@babel/helper-split-export-declaration" "^7.10.1" globals "^11.1.0" -"@babel/plugin-transform-computed-properties@^7.10.1": - version "7.10.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-computed-properties/-/plugin-transform-computed-properties-7.10.1.tgz#59aa399064429d64dce5cf76ef9b90b7245ebd07" - integrity sha512-mqSrGjp3IefMsXIenBfGcPXxJxweQe2hEIwMQvjtiDQ9b1IBvDUjkAtV/HMXX47/vXf14qDNedXsIiNd1FmkaQ== +"@babel/plugin-transform-computed-properties@^7.10.3": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-computed-properties/-/plugin-transform-computed-properties-7.10.3.tgz#d3aa6eef67cb967150f76faff20f0abbf553757b" + integrity sha512-GWzhaBOsdbjVFav96drOz7FzrcEW6AP5nax0gLIpstiFaI3LOb2tAg06TimaWU6YKOfUACK3FVrxPJ4GSc5TgA== dependencies: - "@babel/helper-plugin-utils" "^7.10.1" + "@babel/helper-plugin-utils" "^7.10.3" "@babel/plugin-transform-destructuring@^7.10.1", "@babel/plugin-transform-destructuring@^7.8.8": version "7.10.1" @@ -547,14 +547,14 @@ "@babel/helper-simple-access" "^7.10.1" babel-plugin-dynamic-import-node "^2.3.3" -"@babel/plugin-transform-modules-systemjs@^7.10.1": - version "7.10.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-modules-systemjs/-/plugin-transform-modules-systemjs-7.10.1.tgz#9962e4b0ac6aaf2e20431ada3d8ec72082cbffb6" - integrity sha512-ewNKcj1TQZDL3YnO85qh9zo1YF1CHgmSTlRQgHqe63oTrMI85cthKtZjAiZSsSNjPQ5NCaYo5QkbYqEw1ZBgZA== +"@babel/plugin-transform-modules-systemjs@^7.10.3": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-modules-systemjs/-/plugin-transform-modules-systemjs-7.10.3.tgz#004ae727b122b7b146b150d50cba5ffbff4ac56b" + integrity sha512-GWXWQMmE1GH4ALc7YXW56BTh/AlzvDWhUNn9ArFF0+Cz5G8esYlVbXfdyHa1xaD1j+GnBoCeoQNlwtZTVdiG/A== dependencies: - "@babel/helper-hoist-variables" "^7.10.1" + "@babel/helper-hoist-variables" "^7.10.3" "@babel/helper-module-transforms" "^7.10.1" - "@babel/helper-plugin-utils" "^7.10.1" + "@babel/helper-plugin-utils" "^7.10.3" babel-plugin-dynamic-import-node "^2.3.3" "@babel/plugin-transform-modules-umd@^7.10.1": @@ -565,10 +565,10 @@ "@babel/helper-module-transforms" "^7.10.1" "@babel/helper-plugin-utils" "^7.10.1" -"@babel/plugin-transform-named-capturing-groups-regex@^7.8.3": - version "7.8.3" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-named-capturing-groups-regex/-/plugin-transform-named-capturing-groups-regex-7.8.3.tgz#a2a72bffa202ac0e2d0506afd0939c5ecbc48c6c" - integrity sha512-f+tF/8UVPU86TrCb06JoPWIdDpTNSGGcAtaD9mLP0aYGA0OS0j7j7DHJR0GTFrUZPUU6loZhbsVZgTh0N+Qdnw== +"@babel/plugin-transform-named-capturing-groups-regex@^7.10.3": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-named-capturing-groups-regex/-/plugin-transform-named-capturing-groups-regex-7.10.3.tgz#a4f8444d1c5a46f35834a410285f2c901c007ca6" + integrity sha512-I3EH+RMFyVi8Iy/LekQm948Z4Lz4yKT7rK+vuCAeRm0kTa6Z5W7xuhRxDNJv0FPya/her6AUgrDITb70YHtTvA== dependencies: "@babel/helper-create-regexp-features-plugin" "^7.8.3" @@ -602,10 +602,10 @@ dependencies: "@babel/helper-plugin-utils" "^7.10.1" -"@babel/plugin-transform-regenerator@^7.10.1", "@babel/plugin-transform-regenerator@^7.8.7": - version "7.10.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-regenerator/-/plugin-transform-regenerator-7.10.1.tgz#10e175cbe7bdb63cc9b39f9b3f823c5c7c5c5490" - integrity sha512-B3+Y2prScgJ2Bh/2l9LJxKbb8C8kRfsG4AdPT+n7ixBHIxJaIG8bi8tgjxUMege1+WqSJ+7gu1YeoMVO3gPWzw== +"@babel/plugin-transform-regenerator@^7.10.3", "@babel/plugin-transform-regenerator@^7.8.7": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-regenerator/-/plugin-transform-regenerator-7.10.3.tgz#6ec680f140a5ceefd291c221cb7131f6d7e8cb6d" + integrity sha512-H5kNeW0u8mbk0qa1jVIVTeJJL6/TJ81ltD4oyPx0P499DhMJrTmmIFCmJ3QloGpQG8K9symccB7S7SJpCKLwtw== dependencies: regenerator-transform "^0.14.2" @@ -617,12 +617,12 @@ "@babel/helper-plugin-utils" "^7.10.1" "@babel/plugin-transform-runtime@^7.9.0": - version "7.10.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-runtime/-/plugin-transform-runtime-7.10.1.tgz#fd1887f749637fb2ed86dc278e79eb41df37f4b1" - integrity sha512-4w2tcglDVEwXJ5qxsY++DgWQdNJcCCsPxfT34wCUwIf2E7dI7pMpH8JczkMBbgBTNzBX62SZlNJ9H+De6Zebaw== + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-runtime/-/plugin-transform-runtime-7.10.3.tgz#3b287b06acc534a7cb6e6c71d6b1d88b1922dd6c" + integrity sha512-b5OzMD1Hi8BBzgQdRHyVVaYrk9zG0wset1it2o3BgonkPadXfOv0aXRqd7864DeOIu3FGKP/h6lr15FE5mahVw== dependencies: - "@babel/helper-module-imports" "^7.10.1" - "@babel/helper-plugin-utils" "^7.10.1" + "@babel/helper-module-imports" "^7.10.3" + "@babel/helper-plugin-utils" "^7.10.3" resolve "^1.8.1" semver "^5.5.1" @@ -648,13 +648,13 @@ "@babel/helper-plugin-utils" "^7.10.1" "@babel/helper-regex" "^7.10.1" -"@babel/plugin-transform-template-literals@^7.10.1": - version "7.10.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-template-literals/-/plugin-transform-template-literals-7.10.1.tgz#914c7b7f4752c570ea00553b4284dad8070e8628" - integrity sha512-t7B/3MQf5M1T9hPCRG28DNGZUuxAuDqLYS03rJrIk2prj/UV7Z6FOneijhQhnv/Xa039vidXeVbvjK2SK5f7Gg== +"@babel/plugin-transform-template-literals@^7.10.3": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-template-literals/-/plugin-transform-template-literals-7.10.3.tgz#69d39b3d44b31e7b4864173322565894ce939b25" + integrity sha512-yaBn9OpxQra/bk0/CaA4wr41O0/Whkg6nqjqApcinxM7pro51ojhX6fv1pimAnVjVfDy14K0ULoRL70CA9jWWA== dependencies: "@babel/helper-annotate-as-pure" "^7.10.1" - "@babel/helper-plugin-utils" "^7.10.1" + "@babel/helper-plugin-utils" "^7.10.3" "@babel/plugin-transform-typeof-symbol@^7.10.1": version "7.10.1" @@ -679,23 +679,23 @@ "@babel/helper-plugin-utils" "^7.10.1" "@babel/preset-env@^7.9.0": - version "7.10.2" - resolved "https://registry.yarnpkg.com/@babel/preset-env/-/preset-env-7.10.2.tgz#715930f2cf8573b0928005ee562bed52fb65fdfb" - integrity sha512-MjqhX0RZaEgK/KueRzh+3yPSk30oqDKJ5HP5tqTSB1e2gzGS3PLy7K0BIpnp78+0anFuSwOeuCf1zZO7RzRvEA== + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/preset-env/-/preset-env-7.10.3.tgz#3e58c9861bbd93b6a679987c7e4bd365c56c80c9" + integrity sha512-jHaSUgiewTmly88bJtMHbOd1bJf2ocYxb5BWKSDQIP5tmgFuS/n0gl+nhSrYDhT33m0vPxp+rP8oYYgPgMNQlg== dependencies: - "@babel/compat-data" "^7.10.1" + "@babel/compat-data" "^7.10.3" "@babel/helper-compilation-targets" "^7.10.2" - "@babel/helper-module-imports" "^7.10.1" - "@babel/helper-plugin-utils" "^7.10.1" - "@babel/plugin-proposal-async-generator-functions" "^7.10.1" + "@babel/helper-module-imports" "^7.10.3" + "@babel/helper-plugin-utils" "^7.10.3" + "@babel/plugin-proposal-async-generator-functions" "^7.10.3" "@babel/plugin-proposal-class-properties" "^7.10.1" "@babel/plugin-proposal-dynamic-import" "^7.10.1" "@babel/plugin-proposal-json-strings" "^7.10.1" "@babel/plugin-proposal-nullish-coalescing-operator" "^7.10.1" "@babel/plugin-proposal-numeric-separator" "^7.10.1" - "@babel/plugin-proposal-object-rest-spread" "^7.10.1" + "@babel/plugin-proposal-object-rest-spread" "^7.10.3" "@babel/plugin-proposal-optional-catch-binding" "^7.10.1" - "@babel/plugin-proposal-optional-chaining" "^7.10.1" + "@babel/plugin-proposal-optional-chaining" "^7.10.3" "@babel/plugin-proposal-private-methods" "^7.10.1" "@babel/plugin-proposal-unicode-property-regex" "^7.10.1" "@babel/plugin-syntax-async-generators" "^7.8.0" @@ -712,8 +712,8 @@ "@babel/plugin-transform-async-to-generator" "^7.10.1" "@babel/plugin-transform-block-scoped-functions" "^7.10.1" "@babel/plugin-transform-block-scoping" "^7.10.1" - "@babel/plugin-transform-classes" "^7.10.1" - "@babel/plugin-transform-computed-properties" "^7.10.1" + "@babel/plugin-transform-classes" "^7.10.3" + "@babel/plugin-transform-computed-properties" "^7.10.3" "@babel/plugin-transform-destructuring" "^7.10.1" "@babel/plugin-transform-dotall-regex" "^7.10.1" "@babel/plugin-transform-duplicate-keys" "^7.10.1" @@ -724,24 +724,24 @@ "@babel/plugin-transform-member-expression-literals" "^7.10.1" "@babel/plugin-transform-modules-amd" "^7.10.1" "@babel/plugin-transform-modules-commonjs" "^7.10.1" - "@babel/plugin-transform-modules-systemjs" "^7.10.1" + "@babel/plugin-transform-modules-systemjs" "^7.10.3" "@babel/plugin-transform-modules-umd" "^7.10.1" - "@babel/plugin-transform-named-capturing-groups-regex" "^7.8.3" + "@babel/plugin-transform-named-capturing-groups-regex" "^7.10.3" "@babel/plugin-transform-new-target" "^7.10.1" "@babel/plugin-transform-object-super" "^7.10.1" "@babel/plugin-transform-parameters" "^7.10.1" "@babel/plugin-transform-property-literals" "^7.10.1" - "@babel/plugin-transform-regenerator" "^7.10.1" + "@babel/plugin-transform-regenerator" "^7.10.3" "@babel/plugin-transform-reserved-words" "^7.10.1" "@babel/plugin-transform-shorthand-properties" "^7.10.1" "@babel/plugin-transform-spread" "^7.10.1" "@babel/plugin-transform-sticky-regex" "^7.10.1" - "@babel/plugin-transform-template-literals" "^7.10.1" + "@babel/plugin-transform-template-literals" "^7.10.3" "@babel/plugin-transform-typeof-symbol" "^7.10.1" "@babel/plugin-transform-unicode-escapes" "^7.10.1" "@babel/plugin-transform-unicode-regex" "^7.10.1" "@babel/preset-modules" "^0.1.3" - "@babel/types" "^7.10.2" + "@babel/types" "^7.10.3" browserslist "^4.12.0" core-js-compat "^3.6.2" invariant "^2.2.2" @@ -760,42 +760,42 @@ esutils "^2.0.2" "@babel/runtime@^7.7.2", "@babel/runtime@^7.8.4", "@babel/runtime@^7.9.2": - version "7.10.2" - resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.10.2.tgz#d103f21f2602497d38348a32e008637d506db839" - integrity sha512-6sF3uQw2ivImfVIl62RZ7MXhO2tap69WeWK57vAaimT6AZbE4FbqjdEJIN1UqoD6wI6B+1n9UiagafH1sxjOtg== + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.10.3.tgz#670d002655a7c366540c67f6fd3342cd09500364" + integrity sha512-RzGO0RLSdokm9Ipe/YD+7ww8X2Ro79qiXZF3HU9ljrM+qnJmH1Vqth+hbiQZy761LnMJTMitHDuKVYTk3k4dLw== dependencies: regenerator-runtime "^0.13.4" -"@babel/template@^7.10.1": - version "7.10.1" - resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.10.1.tgz#e167154a94cb5f14b28dc58f5356d2162f539811" - integrity sha512-OQDg6SqvFSsc9A0ej6SKINWrpJiNonRIniYondK2ViKhB06i3c0s+76XUft71iqBEe9S1OKsHwPAjfHnuvnCig== +"@babel/template@^7.10.1", "@babel/template@^7.10.3": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.10.3.tgz#4d13bc8e30bf95b0ce9d175d30306f42a2c9a7b8" + integrity sha512-5BjI4gdtD+9fHZUsaxPHPNpwa+xRkDO7c7JbhYn2afvrkDu5SfAAbi9AIMXw2xEhO/BR35TqiW97IqNvCo/GqA== dependencies: - "@babel/code-frame" "^7.10.1" - "@babel/parser" "^7.10.1" - "@babel/types" "^7.10.1" + "@babel/code-frame" "^7.10.3" + "@babel/parser" "^7.10.3" + "@babel/types" "^7.10.3" -"@babel/traverse@^7.10.1", "@babel/traverse@^7.7.0": - version "7.10.1" - resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.10.1.tgz#bbcef3031e4152a6c0b50147f4958df54ca0dd27" - integrity sha512-C/cTuXeKt85K+p08jN6vMDz8vSV0vZcI0wmQ36o6mjbuo++kPMdpOYw23W2XH04dbRt9/nMEfA4W3eR21CD+TQ== +"@babel/traverse@^7.10.1", "@babel/traverse@^7.10.3", "@babel/traverse@^7.7.0": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.10.3.tgz#0b01731794aa7b77b214bcd96661f18281155d7e" + integrity sha512-qO6623eBFhuPm0TmmrUFMT1FulCmsSeJuVGhiLodk2raUDFhhTECLd9E9jC4LBIWziqt4wgF6KuXE4d+Jz9yug== dependencies: - "@babel/code-frame" "^7.10.1" - "@babel/generator" "^7.10.1" - "@babel/helper-function-name" "^7.10.1" + "@babel/code-frame" "^7.10.3" + "@babel/generator" "^7.10.3" + "@babel/helper-function-name" "^7.10.3" "@babel/helper-split-export-declaration" "^7.10.1" - "@babel/parser" "^7.10.1" - "@babel/types" "^7.10.1" + "@babel/parser" "^7.10.3" + "@babel/types" "^7.10.3" debug "^4.1.0" globals "^11.1.0" lodash "^4.17.13" -"@babel/types@^7.10.1", "@babel/types@^7.10.2", "@babel/types@^7.4.4", "@babel/types@^7.7.0": - version "7.10.2" - resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.10.2.tgz#30283be31cad0dbf6fb00bd40641ca0ea675172d" - integrity sha512-AD3AwWBSz0AWF0AkCN9VPiWrvldXq+/e3cHa4J89vo4ymjz1XwrBFFVZmkJTsQIPNk+ZVomPSXUJqq8yyjZsng== +"@babel/types@^7.10.1", "@babel/types@^7.10.3", "@babel/types@^7.4.4", "@babel/types@^7.7.0": + version "7.10.3" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.10.3.tgz#6535e3b79fea86a6b09e012ea8528f935099de8e" + integrity sha512-nZxaJhBXBQ8HVoIcGsf9qWep3Oh3jCENK54V4mRF7qaJabVsAYdbTtmSD8WmAp1R6ytPiu5apMwSXyxB1WlaBA== dependencies: - "@babel/helper-validator-identifier" "^7.10.1" + "@babel/helper-validator-identifier" "^7.10.3" lodash "^4.17.13" to-fast-properties "^2.0.0" @@ -1144,9 +1144,9 @@ ajv-errors@^1.0.0: integrity sha512-DCRfO/4nQ+89p/RK43i8Ezd41EqdGIU4ld7nGF8OQ14oc/we5rEntLCUa7+jrn3nn83BosfwZA0wb4pon2o8iQ== ajv-keywords@^3.1.0, ajv-keywords@^3.4.1: - version "3.4.1" - resolved "https://registry.yarnpkg.com/ajv-keywords/-/ajv-keywords-3.4.1.tgz#ef916e271c64ac12171fd8384eaae6b2345854da" - integrity sha512-RO1ibKvd27e6FEShVFfPALuHI3WjSVNeK5FIsmme/LYRNxjKuNj+Dt7bucLa6NdSv3JcVTyMlm9kGR84z1XpaQ== + version "3.5.0" + resolved "https://registry.yarnpkg.com/ajv-keywords/-/ajv-keywords-3.5.0.tgz#5c894537098785926d71e696114a53ce768ed773" + integrity sha512-eyoaac3btgU8eJlvh01En8OCKzRqlLe2G5jDsCr3RiE2uLGMEEB1aaGwVVpwR8M95956tGH6R+9edC++OvzaVw== ajv@^6.1.0, ajv@^6.10.0, ajv@^6.10.2, ajv@^6.12.2, ajv@^6.5.5: version "6.12.2" @@ -1168,18 +1168,11 @@ amdefine@>=0.0.4: resolved "https://registry.yarnpkg.com/amdefine/-/amdefine-1.0.1.tgz#4a5282ac164729e93619bcfd3ad151f817ce91f5" integrity sha1-SlKCrBZHKek2Gbz9OtFR+BfOkfU= -ansi-colors@^3.0.0: +ansi-colors@^3.0.0, ansi-colors@^3.2.1: version "3.2.4" resolved "https://registry.yarnpkg.com/ansi-colors/-/ansi-colors-3.2.4.tgz#e3a3da4bfbae6c86a9c285625de124a234026fbf" integrity sha512-hHUXGagefjN2iRrID63xckIvotOXOojhQKWIPUZ4mNUZ9nLZW+7FMNoE1lOkEhNWYsx/7ysGIuJYCiMAA9FnrA== -ansi-escapes@^4.2.1: - version "4.3.1" - resolved "https://registry.yarnpkg.com/ansi-escapes/-/ansi-escapes-4.3.1.tgz#a5c47cc43181f1f38ffd7076837700d395522a61" - integrity sha512-JWF7ocqNrp8u9oqpgV+wH5ftbt+cfvv+PTjOvKLT3AdYly/LmORARfEVT1iyjwN+4MqE5UmVKoAdIBqeoCHgLA== - dependencies: - type-fest "^0.11.0" - ansi-html@0.0.7: version "0.0.7" resolved "https://registry.yarnpkg.com/ansi-html/-/ansi-html-0.0.7.tgz#813584021962a9e9e6fd039f940d12f56ca7859e" @@ -1400,16 +1393,16 @@ atob@^2.1.2: integrity sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg== autoprefixer@^9.6.1, autoprefixer@^9.8.0: - version "9.8.0" - resolved "https://registry.yarnpkg.com/autoprefixer/-/autoprefixer-9.8.0.tgz#68e2d2bef7ba4c3a65436f662d0a56a741e56511" - integrity sha512-D96ZiIHXbDmU02dBaemyAg53ez+6F5yZmapmgKcjm35yEe1uVDYI8hGW3VYoGRaG290ZFf91YxHrR518vC0u/A== + version "9.8.2" + resolved "https://registry.yarnpkg.com/autoprefixer/-/autoprefixer-9.8.2.tgz#7347396ee576b18687041bfbacd76d78e27baa56" + integrity sha512-9UwMMU8Rg7Fj0c55mbOpXrr/2WrRqoOwOlLNTyyYt+nhiyQdIBWipp5XWzt+Lge8r3DK5y+EHMc1OBf8VpZA6Q== dependencies: browserslist "^4.12.0" - caniuse-lite "^1.0.30001061" - chalk "^2.4.2" + caniuse-lite "^1.0.30001084" + kleur "^4.0.1" normalize-range "^0.1.2" num2fraction "^1.2.2" - postcss "^7.0.30" + postcss "^7.0.32" postcss-value-parser "^4.1.0" aws-sign2@~0.7.0: @@ -1894,10 +1887,10 @@ caniuse-api@^3.0.0: lodash.memoize "^4.1.2" lodash.uniq "^4.5.0" -caniuse-lite@^1.0.0, caniuse-lite@^1.0.30000981, caniuse-lite@^1.0.30001043, caniuse-lite@^1.0.30001061: - version "1.0.30001084" - resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001084.tgz#00e471931eaefbeef54f46aa2203914d3c165669" - integrity sha512-ftdc5oGmhEbLUuMZ/Qp3mOpzfZLCxPYKcvGv6v2dJJ+8EdqcvZRbAGOiLmkM/PV1QGta/uwBs8/nCl6sokDW6w== +caniuse-lite@^1.0.0, caniuse-lite@^1.0.30000981, caniuse-lite@^1.0.30001043, caniuse-lite@^1.0.30001084: + version "1.0.30001085" + resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001085.tgz#bed28bd51ff7425d33ee23e730c7f3b703711db6" + integrity sha512-x0YRFRE0pmOD90z+9Xk7jwO58p4feVNXP+U8kWV+Uo/HADyrgESlepzIkUqPgaXkpyceZU6siM1gsK7sHgplqA== case-sensitive-paths-webpack-plugin@^2.3.0: version "2.3.0" @@ -1943,14 +1936,6 @@ chalk@^2.0, chalk@^2.0.0, chalk@^2.0.1, chalk@^2.4.1, chalk@^2.4.2: escape-string-regexp "^1.0.5" supports-color "^5.3.0" -chalk@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/chalk/-/chalk-3.0.0.tgz#3f73c2bf526591f574cc492c51e2456349f844e4" - integrity sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg== - dependencies: - ansi-styles "^4.1.0" - supports-color "^7.1.0" - chalk@^4.0.0, chalk@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.0.tgz#4e14870a618d9e2edd97dd8345fd9d9dc315646a" @@ -1979,11 +1964,6 @@ character-reference-invalid@^1.0.0: resolved "https://registry.yarnpkg.com/character-reference-invalid/-/character-reference-invalid-1.1.4.tgz#083329cda0eae272ab3dbbf37e9a382c13af1560" integrity sha512-mKKUkUbhPpQlCOfIuZkvSEgktjPFIsZKRRbC6KWVEMvlzblj3i3asQv5ODsrwt0N3pHAEvjP8KTQPHkp0+6jOg== -chardet@^0.7.0: - version "0.7.0" - resolved "https://registry.yarnpkg.com/chardet/-/chardet-0.7.0.tgz#90094849f0937f2eedc2425d0d28a9e5f0cbad9e" - integrity sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA== - chokidar@^2.1.8: version "2.1.8" resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-2.1.8.tgz#804b3a7b6a99358c3c5c61e71d8728f041cff917" @@ -2053,18 +2033,6 @@ clean-stack@^2.0.0: resolved "https://registry.yarnpkg.com/clean-stack/-/clean-stack-2.2.0.tgz#ee8472dbb129e727b31e8a10a427dee9dfe4008b" integrity sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A== -cli-cursor@^3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/cli-cursor/-/cli-cursor-3.1.0.tgz#264305a7ae490d1d03bf0c9ba7c925d1753af307" - integrity sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw== - dependencies: - restore-cursor "^3.1.0" - -cli-width@^2.0.0: - version "2.2.1" - resolved "https://registry.yarnpkg.com/cli-width/-/cli-width-2.2.1.tgz#b0433d0b4e9c847ef18868a4ef16fd5fc8271c48" - integrity sha512-GRMWDxpOB6Dgk2E5Uo+3eEBvtOOlimMmpbFiKuLFnQzYDavtLFY3K5ona41jgN/WdRZtG7utuVSVTL4HbZHGkw== - cliui@^4.0.0: version "4.1.0" resolved "https://registry.yarnpkg.com/cliui/-/cliui-4.1.0.tgz#348422dbe82d800b3022eef4f6ac10bf2e4d1b49" @@ -2956,9 +2924,9 @@ ee-first@1.1.1: integrity sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0= electron-to-chromium@^1.3.413: - version "1.3.477" - resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.3.477.tgz#7e6b931d0c1a2572101a6e9a835128c50fd49323" - integrity sha512-81p6DZ/XmHDD7O0ITJMa7ESo9bSCfE+v3Fny3MIYR0y77xmhoriu2ShNOLXcPS4eowF6dkxw6d2QqxTkS3DjBg== + version "1.3.480" + resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.3.480.tgz#190ae45074578349a4c4f336fba29e76b20e9ef5" + integrity sha512-wnuUfQCBMAdzu5Xe+F4FjaRK+6ToG6WvwG72s8k/3E6b+hoGVYGiQE7JD1NhiCMcqF3+wV+c2vAnaLGRSSWVqA== elliptic@^6.0.0, elliptic@^6.5.2: version "6.5.3" @@ -3014,6 +2982,13 @@ enhanced-resolve@^4.1.0, enhanced-resolve@^4.1.1: memory-fs "^0.5.0" tapable "^1.0.0" +enquirer@^2.3.5: + version "2.3.5" + resolved "https://registry.yarnpkg.com/enquirer/-/enquirer-2.3.5.tgz#3ab2b838df0a9d8ab9e7dff235b0e8712ef92381" + integrity sha512-BNT1C08P9XD0vNg3J475yIUG+mVdp9T6towYFHUv897X0KoHBjB1shyrNmhmtHWKP17iSWgo7Gqh7BBuzLZMSA== + dependencies: + ansi-colors "^3.2.1" + entities@^1.1.1: version "1.1.2" resolved "https://registry.yarnpkg.com/entities/-/entities-1.1.2.tgz#bdfa735299664dfafd34529ed4f8522a275fea56" @@ -3133,14 +3108,14 @@ eslint-utils@^2.0.0: eslint-visitor-keys "^1.1.0" eslint-visitor-keys@^1.0.0, eslint-visitor-keys@^1.1.0, eslint-visitor-keys@^1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-1.2.0.tgz#74415ac884874495f78ec2a97349525344c981fa" - integrity sha512-WFb4ihckKil6hu3Dp798xdzSfddwKKU3+nGniKF6HfeW6OLd2OUDEPP7TcHtB5+QXOKg2s6B2DaMPE1Nn/kxKQ== + version "1.3.0" + resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz#30ebd1ef7c2fdff01c3a4f151044af25fab0523e" + integrity sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ== eslint@^7.0.0: - version "7.2.0" - resolved "https://registry.yarnpkg.com/eslint/-/eslint-7.2.0.tgz#d41b2e47804b30dbabb093a967fb283d560082e6" - integrity sha512-B3BtEyaDKC5MlfDa2Ha8/D6DsS4fju95zs0hjS3HdGazw+LNayai38A25qMppK37wWGWNYSPOR6oYzlz5MHsRQ== + version "7.3.0" + resolved "https://registry.yarnpkg.com/eslint/-/eslint-7.3.0.tgz#f9f1fc3dc1227985d0db88769f2bbac7b4b875d7" + integrity sha512-dJMVXwfU5PT1cj2Nv2VPPrKahKTGdX+5Dh0Q3YuKt+Y2UhdL2YbzsVaBMyG9HC0tBismlv/r1+eZqs6SMIV38Q== dependencies: "@babel/code-frame" "^7.0.0" ajv "^6.10.0" @@ -3148,6 +3123,7 @@ eslint@^7.0.0: cross-spawn "^7.0.2" debug "^4.0.1" doctrine "^3.0.0" + enquirer "^2.3.5" eslint-scope "^5.1.0" eslint-utils "^2.0.0" eslint-visitor-keys "^1.2.0" @@ -3161,7 +3137,6 @@ eslint@^7.0.0: ignore "^4.0.6" import-fresh "^3.0.0" imurmurhash "^0.1.4" - inquirer "^7.0.0" is-glob "^4.0.0" js-yaml "^3.13.1" json-stable-stringify-without-jsonify "^1.0.1" @@ -3358,15 +3333,6 @@ extend@^3.0.0, extend@~3.0.2: resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.2.tgz#f8b1136b4071fbd8eb140aff858b1019ec2915fa" integrity sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g== -external-editor@^3.0.3: - version "3.1.0" - resolved "https://registry.yarnpkg.com/external-editor/-/external-editor-3.1.0.tgz#cb03f740befae03ea4d283caed2741a83f335495" - integrity sha512-hMQ4CX1p1izmuLYyZqLMO/qGNw10wSv9QDCPfzXfyFrOaCSSoRfqE1Kf1s5an66J5JZC62NewG+mK49jOCtQew== - dependencies: - chardet "^0.7.0" - iconv-lite "^0.4.24" - tmp "^0.0.33" - extglob@^2.0.4: version "2.0.4" resolved "https://registry.yarnpkg.com/extglob/-/extglob-2.0.4.tgz#ad00fe4dc612a9232e8718711dc5cb5ab0285543" @@ -3451,13 +3417,6 @@ figgy-pudding@^3.5.1: resolved "https://registry.yarnpkg.com/figgy-pudding/-/figgy-pudding-3.5.2.tgz#b4eee8148abb01dcf1d1ac34367d59e12fa61d6e" integrity sha512-0btnI/H8f2pavGMN8w40mlSKOfTK2SVJmBfBeVIj3kNw0swwgzyRq0d5TJVOwodFmtvpPeWPN/MCcfuWF0Ezbw== -figures@^3.0.0: - version "3.2.0" - resolved "https://registry.yarnpkg.com/figures/-/figures-3.2.0.tgz#625c18bd293c604dc4a8ddb2febf0c88341746af" - integrity sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg== - dependencies: - escape-string-regexp "^1.0.5" - file-entry-cache@^5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/file-entry-cache/-/file-entry-cache-5.0.1.tgz#ca0f6efa6dd3d561333fb14515065c2fafdf439c" @@ -4200,7 +4159,7 @@ https-browserify@^1.0.0: resolved "https://registry.yarnpkg.com/https-browserify/-/https-browserify-1.0.0.tgz#ec06c10e0a34c0f2faf199f7fd7fc78fffd03c73" integrity sha1-7AbBDgo0wPL68Zn3/X/Hj//QPHM= -iconv-lite@0.4.24, iconv-lite@^0.4.24: +iconv-lite@0.4.24: version "0.4.24" resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.24.tgz#2022b4b25fbddc21d2f524974a474aafe733908b" integrity sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA== @@ -4344,25 +4303,6 @@ ini@^1.3.4, ini@^1.3.5: resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.5.tgz#eee25f56db1c9ec6085e0c22778083f596abf927" integrity sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw== -inquirer@^7.0.0: - version "7.2.0" - resolved "https://registry.yarnpkg.com/inquirer/-/inquirer-7.2.0.tgz#63ce99d823090de7eb420e4bb05e6f3449aa389a" - integrity sha512-E0c4rPwr9ByePfNlTIB8z51kK1s2n6jrHuJeEHENl/sbq2G/S1auvibgEwNR4uSyiU+PiYHqSwsgGiXjG8p5ZQ== - dependencies: - ansi-escapes "^4.2.1" - chalk "^3.0.0" - cli-cursor "^3.1.0" - cli-width "^2.0.0" - external-editor "^3.0.3" - figures "^3.0.0" - lodash "^4.17.15" - mute-stream "0.0.8" - run-async "^2.4.0" - rxjs "^6.5.3" - string-width "^4.1.0" - strip-ansi "^6.0.0" - through "^2.3.6" - internal-ip@^4.3.0: version "4.3.0" resolved "https://registry.yarnpkg.com/internal-ip/-/internal-ip-4.3.0.tgz#845452baad9d2ca3b69c635a137acb9a0dad0907" @@ -4781,9 +4721,9 @@ jquery@>=1.6.0: integrity sha512-XwIBPqcMn57FxfT+Go5pzySnm4KWkT1Tv7gjrpT1srtf8Weynl6R273VJ5GjkRb51IzMp5nbaPjJXMWeju2MKg== js-base64@^2.1.8: - version "2.5.2" - resolved "https://registry.yarnpkg.com/js-base64/-/js-base64-2.5.2.tgz#313b6274dda718f714d00b3330bbae6e38e90209" - integrity sha512-Vg8czh0Q7sFBSUMWWArX/miJeBWYBPpdU/3M/DKSaekLMqrqVPaedp+5mZhie/r0lgrcaYBfwXatEew6gwgiQQ== + version "2.6.1" + resolved "https://registry.yarnpkg.com/js-base64/-/js-base64-2.6.1.tgz#c328374225d2e65569791ded73c258e2c59334c7" + integrity sha512-G5x2saUTupU9D/xBY9snJs3TxvwX8EkpLFiYlPpDt/VmMHOXprnSU1nxiTmFbijCX4BLF/cMRIfAcC5BiMYgFQ== "js-tokens@^3.0.0 || ^4.0.0", js-tokens@^4.0.0: version "4.0.0" @@ -4916,6 +4856,11 @@ kind-of@^6.0.0, kind-of@^6.0.2, kind-of@^6.0.3: resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-6.0.3.tgz#07c05034a6c349fa06e24fa35aa76db4580ce4dd" integrity sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw== +kleur@^4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/kleur/-/kleur-4.0.1.tgz#3d4948534b666e2578f93b6fafb62108e64f05ef" + integrity sha512-Qs6SqCLm63rd0kNVh+wO4XsWLU6kgfwwaPYsLiClWf0Tewkzsa6MvB21bespb8cz+ANS+2t3So1ge3gintzhlw== + known-css-properties@^0.19.0: version "0.19.0" resolved "https://registry.yarnpkg.com/known-css-properties/-/known-css-properties-0.19.0.tgz#5d92b7fa16c72d971bda9b7fe295bdf61836ee5b" @@ -5368,7 +5313,7 @@ mime@^2.4.4: resolved "https://registry.yarnpkg.com/mime/-/mime-2.4.6.tgz#e5b407c90db442f2beb5b162373d07b69affa4d1" integrity sha512-RZKhC3EmpBchfTGBVb8fb+RL2cWyw/32lshnsETttkBAyAUXSGHxbEJWWRXc751DrIxG1q04b8QwMbAwkRPpUA== -mimic-fn@^2.0.0, mimic-fn@^2.1.0: +mimic-fn@^2.0.0: version "2.1.0" resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-2.1.0.tgz#7ed2c2ccccaf84d3ffcb7a69b57711fc2083401b" integrity sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg== @@ -5518,11 +5463,6 @@ multicast-dns@^6.0.1: dns-packet "^1.3.1" thunky "^1.0.2" -mute-stream@0.0.8: - version "0.0.8" - resolved "https://registry.yarnpkg.com/mute-stream/-/mute-stream-0.0.8.tgz#1630c42b2251ff81e2a283de96a5497ea92e5e0d" - integrity sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA== - mz@2.7.0: version "2.7.0" resolved "https://registry.yarnpkg.com/mz/-/mz-2.7.0.tgz#95008057a56cafadc2bc63dde7f9ff6955948e32" @@ -5804,9 +5744,9 @@ object-hash@^2.0.3: integrity sha512-JPKn0GMu+Fa3zt3Bmr66JhokJU5BaNBIh4ZeTlaCBzrBsOeXzwcKKAK1tbLiPKgvwmPXsDvvLHoWh5Bm7ofIYg== object-inspect@^1.7.0: - version "1.7.0" - resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.7.0.tgz#f4f6bd181ad77f006b5ece60bd0b6f398ff74a67" - integrity sha512-a7pEHdh1xKIAgTySUGgLMx/xwDZskN1Ud6egYYN3EdRW4ZMPNEDUTF+hwy2LUC+Bl+SyLXANnwz/jyh/qutKUw== + version "1.8.0" + resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.8.0.tgz#df807e5ecf53a609cc6bfe93eac3cc7be5b3a9d0" + integrity sha512-jLdtEOB112fORuypAyl/50VRVIBIdVQOSUUGQHzJ4xBSbit81zRarz7GThkEFZy1RceYrWYcPcBFPQwHyAc1gA== object-is@^1.0.1: version "1.1.2" @@ -5887,13 +5827,6 @@ once@^1.3.0, once@^1.3.1, once@^1.4.0: dependencies: wrappy "1" -onetime@^5.1.0: - version "5.1.0" - resolved "https://registry.yarnpkg.com/onetime/-/onetime-5.1.0.tgz#fff0f3c91617fe62bb50189636e99ac8a6df7be5" - integrity sha512-5NcSkPHhwTVFIQN+TUqXoS5+dlElHXdpAWu9I0HP20YOtIi+aZ0Ct82jdlILDxjLEAWwvm+qj1m6aEtsDVmm6Q== - dependencies: - mimic-fn "^2.1.0" - opn@^5.5.0: version "5.5.0" resolved "https://registry.yarnpkg.com/opn/-/opn-5.5.0.tgz#fc7164fab56d235904c51c3b27da6758ca3b9bfc" @@ -5947,7 +5880,7 @@ os-locale@^3.0.0: lcid "^2.0.0" mem "^4.0.0" -os-tmpdir@^1.0.0, os-tmpdir@~1.0.2: +os-tmpdir@^1.0.0: version "1.0.2" resolved "https://registry.yarnpkg.com/os-tmpdir/-/os-tmpdir-1.0.2.tgz#bbe67406c79aa85c5cfec766fe5734555dfa1274" integrity sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ= @@ -6984,7 +6917,7 @@ postcss-values-parser@^2.0.0, postcss-values-parser@^2.0.1: indexes-of "^1.0.1" uniq "^1.0.1" -postcss@^7.0.0, postcss@^7.0.1, postcss@^7.0.14, postcss@^7.0.16, postcss@^7.0.17, postcss@^7.0.2, postcss@^7.0.21, postcss@^7.0.26, postcss@^7.0.27, postcss@^7.0.30, postcss@^7.0.32, postcss@^7.0.5, postcss@^7.0.6, postcss@^7.0.7: +postcss@^7.0.0, postcss@^7.0.1, postcss@^7.0.14, postcss@^7.0.16, postcss@^7.0.17, postcss@^7.0.2, postcss@^7.0.21, postcss@^7.0.26, postcss@^7.0.27, postcss@^7.0.32, postcss@^7.0.5, postcss@^7.0.6, postcss@^7.0.7: version "7.0.32" resolved "https://registry.yarnpkg.com/postcss/-/postcss-7.0.32.tgz#4310d6ee347053da3433db2be492883d62cec59d" integrity sha512-03eXong5NLnNCD05xscnGKGDZ98CyzoqPSMjOe6SuoQY7Z2hIj0Ld1g/O/UQRuOle2aRtiIRDg9tDcTGAkLfKw== @@ -7573,14 +7506,6 @@ resolve@^1.1.6, resolve@^1.1.7, resolve@^1.10.0, resolve@^1.12.0, resolve@^1.3.2 dependencies: path-parse "^1.0.6" -restore-cursor@^3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/restore-cursor/-/restore-cursor-3.1.0.tgz#39f67c54b3a7a58cea5236d95cf0034239631f7e" - integrity sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA== - dependencies: - onetime "^5.1.0" - signal-exit "^3.0.2" - ret@~0.1.10: version "0.1.15" resolved "https://registry.yarnpkg.com/ret/-/ret-0.1.15.tgz#b8a4825d5bdb1fc3f6f53c2bc33f81388681c7bc" @@ -7635,11 +7560,6 @@ ripemd160@^2.0.0, ripemd160@^2.0.1: hash-base "^3.0.0" inherits "^2.0.1" -run-async@^2.4.0: - version "2.4.1" - resolved "https://registry.yarnpkg.com/run-async/-/run-async-2.4.1.tgz#8440eccf99ea3e70bd409d49aab88e10c189a455" - integrity sha512-tvVnVv01b8c1RrA6Ep7JkStj85Guv/YrMcwqYQnwjsAS2cTmmPGBBjAjpCW7RrSodNSoE2/qg9O4bceNvUuDgQ== - run-parallel@^1.1.9: version "1.1.9" resolved "https://registry.yarnpkg.com/run-parallel/-/run-parallel-1.1.9.tgz#c9dd3a7cf9f4b2c4b6244e173a6ed866e61dd679" @@ -7652,13 +7572,6 @@ run-queue@^1.0.0, run-queue@^1.0.3: dependencies: aproba "^1.1.1" -rxjs@^6.5.3: - version "6.5.5" - resolved "https://registry.yarnpkg.com/rxjs/-/rxjs-6.5.5.tgz#c5c884e3094c8cfee31bf27eb87e54ccfc87f9ec" - integrity sha512-WfQI+1gohdf0Dai/Bbmk5L5ItH5tYqm3ki2c5GdWhKjalzjg93N3avFjVStyZZz+A2Em+ZxKH5bNghw9UeylGQ== - dependencies: - tslib "^1.9.0" - safe-buffer@5.1.2, safe-buffer@~5.1.0, safe-buffer@~5.1.1: version "5.1.2" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" @@ -8254,7 +8167,7 @@ string-width@^3.0.0, string-width@^3.1.0: is-fullwidth-code-point "^2.0.0" strip-ansi "^5.1.0" -string-width@^4.1.0, string-width@^4.2.0: +string-width@^4.2.0: version "4.2.0" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.0.tgz#952182c46cc7b2c313d1596e623992bd163b72b5" integrity sha512-zUz5JD+tgqtuDjMhwIg5uFVV3dtqZ9yQJlZVfq4I01/K5Paj5UHj7VyrQOJvzawSVlKpObApbfD0Ed6yJc+1eg== @@ -8630,7 +8543,7 @@ through2@^2.0.0: readable-stream "~2.3.6" xtend "~4.0.1" -through@^2.3.6, through@^2.3.8: +through@^2.3.8: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" integrity sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU= @@ -8652,13 +8565,6 @@ timsort@^0.3.0: resolved "https://registry.yarnpkg.com/timsort/-/timsort-0.3.0.tgz#405411a8e7e6339fe64db9a234de11dc31e02bd4" integrity sha1-QFQRqOfmM5/mTbmiNN4R3DHgK9Q= -tmp@^0.0.33: - version "0.0.33" - resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.0.33.tgz#6d34335889768d21b2bcda0aa277ced3b1bfadf9" - integrity sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw== - dependencies: - os-tmpdir "~1.0.2" - to-arraybuffer@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/to-arraybuffer/-/to-arraybuffer-1.0.1.tgz#7d229b1fcc637e466ca081180836a7aabff83f43" @@ -8798,11 +8704,6 @@ type-check@^0.4.0, type-check@~0.4.0: dependencies: prelude-ls "^1.2.1" -type-fest@^0.11.0: - version "0.11.0" - resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.11.0.tgz#97abf0872310fed88a5c466b25681576145e33f1" - integrity sha512-OdjXJxnCN1AvyLSzeKIgXTXxV+99ZuXl3Hpo9XpJAv9MBcHrrJOQ5kV7ypXOuQie+AmWG25hLbiKdwYTifzcfQ== - type-fest@^0.13.1: version "0.13.1" resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.13.1.tgz#0172cb5bce80b0bd542ea348db50c7e21834d934" From 209350de5bbe18212759db8d174a7c1ad8c7fc6b Mon Sep 17 00:00:00 2001 From: evazion Date: Sun, 21 Jun 2020 18:56:42 -0500 Subject: [PATCH 41/59] scripts: remove unused danbooru-specific config. Danbooru-specific configuration now lives in https://github.com/danbooru/danbooru-ansible. --- INSTALL.debian | 14 --- script/install/app_server.sh | 101 ------------------ script/install/distribute_new_pubkey.sh | 10 -- script/install/nginx.service.d-local.conf | 2 - script/install/nginx/conf.d/common.conf | 92 ---------------- script/install/nginx/nginx.conf | 82 -------------- .../install/nginx/sites-enabled/danbooru.conf | 68 ------------ script/install/vrack-cfg.yaml | 10 -- 8 files changed, 379 deletions(-) delete mode 100644 script/install/app_server.sh delete mode 100644 script/install/distribute_new_pubkey.sh delete mode 100644 script/install/nginx.service.d-local.conf delete mode 100644 script/install/nginx/conf.d/common.conf delete mode 100644 script/install/nginx/nginx.conf delete mode 100644 script/install/nginx/sites-enabled/danbooru.conf delete mode 100644 script/install/vrack-cfg.yaml diff --git a/INSTALL.debian b/INSTALL.debian index a6f3d6736..c435d7748 100644 --- a/INSTALL.debian +++ b/INSTALL.debian @@ -32,9 +32,6 @@ if [[ -z "$HOSTNAME" ]] ; then exit 1 fi -echo -n "* Enter the VLAN IP address for this server (ex: 172.16.0.1, enter nothing to skip): " -read VLAN_IP_ADDR - # Install packages echo "* Installing packages..." @@ -52,17 +49,6 @@ apt-get -y install $LIBSSL_DEV_PKG build-essential automake libxml2-dev libxslt- apt-get -y install libpq-dev postgresql-client apt-get -y install liblcms2-dev $LIBJPEG_TURBO_DEV_PKG libexpat1-dev libgif-dev libpng-dev libexif-dev -# vrack specific stuff -if [ -n "$VLAN_IP_ADDR" ] ; then - apt-get -y install vlan - modprobe 8021q - echo "8021q" >> /etc/modules - vconfig add eno2 99 - ip addr add $VLAN_IP_ADDR/24 dev eno2.99 - ip link set up eno2.99 - curl -L -s $GITHUB_INSTALL_SCRIPTS/vrack-cfg.yaml -o /etc/netplan/01-netcfg.yaml -fi - curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list curl -sSL https://deb.nodesource.com/setup_10.x | sudo -E bash - diff --git a/script/install/app_server.sh b/script/install/app_server.sh deleted file mode 100644 index a5a14d8ac..000000000 --- a/script/install/app_server.sh +++ /dev/null @@ -1,101 +0,0 @@ -#!/bin/bash - -# this is a version of the install script designed to be run on -# app servers (that is, they won't install PostgreSQL server). -# -# Run: curl -L -s https://raw.githubusercontent.com/r888888888/danbooru/master/script/install/app_server.sh | sh - -export RUBY_VERSION=2.6.3 -export GITHUB_INSTALL_SCRIPTS=https://raw.githubusercontent.com/r888888888/danbooru/master/script/install -export VIPS_VERSION=8.7.0 - -if [[ "$(whoami)" != "root" ]] ; then - echo "You must run this script as root" - exit 1 -fi - -echo "* DANBOORU INSTALLATION SCRIPT" -echo "*" -echo "* This script will install all the necessary packages to run Danbooru on an" -echo "* Ubuntu server." -echo - -echo -n "* Enter the VLAN IP address for this server: " -read VLAN_IP_ADDR - -# Install packages -echo "* Installing packages..." - -apt-get update -apt-get -y install libssl-dev build-essential automake libxml2-dev libxslt-dev ncurses-dev sudo libreadline-dev flex bison ragel redis git curl libcurl4-openssl-dev sendmail-bin sendmail nginx ssh coreutils ffmpeg mkvtoolnix -apt-get -y install libpq-dev postgresql-client -apt-get -y install liblcms2-dev libjpeg-turbo8-dev libexpat1-dev libgif-dev libpng-dev libexif-dev - -# vrack specific stuff -apt-get -y install vlan -modprobe 8021q -echo "8021q" >> /etc/modules -vconfig add eno2 99 -ip addr add $VLAN_IP_ADDR/24 dev eno2.99 -ip link set up eno2.99 -curl -L -s $GITHUB_INSTALL_SCRIPTS/vrack-cfg.yaml -o /etc/netplan/01-netcfg.yaml - -curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - -echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list -curl -sSL https://deb.nodesource.com/setup_10.x | sudo -E bash - -apt-get update -apt-get -y install nodejs yarn -apt-get remove cmdtest - -# compile and install libvips (the version in apt is too old) -cd /tmp -wget -q https://github.com/libvips/libvips/releases/download/v$VIPS_VERSION/vips-$VIPS_VERSION.tar.gz -tar xzf vips-$VIPS_VERSION.tar.gz -cd vips-$VIPS_VERSION -./configure --prefix=/usr -make install -ldconfig - -# Create user account -useradd -m danbooru -chsh -s /bin/bash danbooru -usermod -G danbooru,sudo danbooru - -# Set up Postgres -git clone https://github.com/r888888888/test_parser.git /tmp/test_parser -cd /tmp/test_parser -make install - -# Install rbenv -echo "* Installing rbenv..." -cd /tmp -sudo -u danbooru git clone git://github.com/sstephenson/rbenv.git ~danbooru/.rbenv -sudo -u danbooru touch ~danbooru/.bash_profile -echo 'export PATH="$HOME/.rbenv/bin:$PATH"' >> ~danbooru/.bash_profile -echo 'eval "$(rbenv init -)"' >> ~danbooru/.bash_profile -sudo -u danbooru mkdir -p ~danbooru/.rbenv/plugins -sudo -u danbooru git clone git://github.com/sstephenson/ruby-build.git ~danbooru/.rbenv/plugins/ruby-build -sudo -u danbooru bash -l -c "rbenv install $RUBY_VERSION" -sudo -u danbooru bash -l -c "rbenv global $RUBY_VERSION" - -# Install gems -echo "* Installing gems..." -sudo -u danbooru bash -l -c 'gem install --no-ri --no-rdoc bundler' - -# Setup danbooru account -echo "* Enter a new password for the danbooru account" -passwd danbooru - -echo "* Setting up SSH keys for the danbooru account" -sudo -u danbooru ssh-keygen -sudo -u danbooru cat ~danbooru/.ssh/id_rsa.pub >> ~danbooru/.ssh/authorized_keys - -echo "* TODO:" -echo "on kagamihara:" -echo "script/install/distribute_new_pubkey.sh" -echo -echo "on this server:" -echo "rsync -av kagamihara:/etc/nginx/nginx.conf /etc/nginx" -echo "rsync -av kagamihara:/etc/nginx/conf.d /etc/nginx" -echo "rsync -av kagamihara:/etc/nginx/sites-enabled /etc/nginx" -echo "rsync -av kagamihara:/etc/logrotate.d /etc/logrotate.d" diff --git a/script/install/distribute_new_pubkey.sh b/script/install/distribute_new_pubkey.sh deleted file mode 100644 index 52feaa245..000000000 --- a/script/install/distribute_new_pubkey.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh - -HOSTS="kagamihara shima saitou" - -echo "Enter new SSH pubkey: " -read $key - -for host in $HOSTS ; do - ssh danbooru@$host echo $key >> .ssh/authorized_keys -done diff --git a/script/install/nginx.service.d-local.conf b/script/install/nginx.service.d-local.conf deleted file mode 100644 index 01c1b16bc..000000000 --- a/script/install/nginx.service.d-local.conf +++ /dev/null @@ -1,2 +0,0 @@ -[Service] -LimitNOFILE=10000 diff --git a/script/install/nginx/conf.d/common.conf b/script/install/nginx/conf.d/common.conf deleted file mode 100644 index 677a14da7..000000000 --- a/script/install/nginx/conf.d/common.conf +++ /dev/null @@ -1,92 +0,0 @@ -ssl_protocols TLSv1 TLSv1.1 TLSv1.2; -ssl_ciphers 'ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS'; -ssl_prefer_server_ciphers on; -ssl_session_timeout 4h; -ssl_session_cache shared:SSL:20m; -ssl_session_tickets off; -ssl_stapling on; -ssl_stapling_verify on; -ssl_dhparam /etc/nginx/ssl/dhparam.pem; -resolver 8.8.8.8 8.8.4.4; - -root /var/www/danbooru/current/public; -index index.html; -access_log off; -error_log /var/www/danbooru/shared/log/server.error.log; -try_files $uri/index.html $uri.html $uri @app; -client_max_body_size 35m; -error_page 503 @maintenance; -error_page 404 /404.html; -error_page 500 502 503 504 /500.html; - -location /assets { - expires max; - break; -} - -location /data/preview { - expires max; - break; -} - -location /posts/mobile { - return 404; -} - -location /users { - limit_req zone=users burst=5; - limit_req_status 429; - try_files $uri @app_server; -} - -location /posts { - limit_req zone=posts burst=20; - limit_req_status 429; - try_files $uri @app_server; -} - -location /data { - valid_referers none *.donmai.us donmai.us ~\.google\. ~\.bing\. ~\.yahoo\.; - - if ($invalid_referer) { - return 403; - } - - rewrite ^/data/sample/__.+?__(.+) /data/sample/$1 last; - rewrite ^/data/__.+?__(.+) /data/$1 last; - - expires max; - break; -} - -location /maintenance.html { - expires 10; -} - -if (-f $document_root/maintenance.html) { - return 503; -} - -if ($http_user_agent ~ (WinHttp\.WinHttpRequest\.5) ) { - return 403; -} - -location @maintenance { - rewrite ^(.*)$ /maintenance.html last; -} - -location @app_server { - proxy_pass http://app_server; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Real-IP $remote_addr; - proxy_redirect off; - proxy_set_header Host $http_host; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_buffer_size 128k; - proxy_buffers 4 256k; - proxy_busy_buffers_size 256k; -} - -location / { - try_files $uri @app_server; -} diff --git a/script/install/nginx/nginx.conf b/script/install/nginx/nginx.conf deleted file mode 100644 index f8d9b7e99..000000000 --- a/script/install/nginx/nginx.conf +++ /dev/null @@ -1,82 +0,0 @@ -user www-data; -worker_processes auto; -pid /var/run/nginx.pid; - -events { - use epoll; - worker_connections 10000; - multi_accept on; - accept_mutex on; -} - -http { - limit_req_zone $binary_remote_addr zone=users:10m rate=5r/s; - limit_req_zone $binary_remote_addr zone=posts:100m rate=10r/s; - - ## - # Basic Settings - ## - - sendfile on; - tcp_nopush on; - tcp_nodelay on; - keepalive_timeout 5; - types_hash_max_size 2048; - # server_tokens off; - - server_names_hash_bucket_size 128; - # server_name_in_redirect off; - - include /etc/nginx/mime.types; - default_type application/octet-stream; - - ## - # Logging Settings - ## - - access_log off; - error_log /var/log/nginx/error.log; - - ## - # Gzip Settings - ## - - gzip on; - gzip_disable "msie6"; - - gzip_http_version 1.1; - gzip_vary on; - gzip_comp_level 5; - gzip_proxied any; - gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/rss+xml text/javascript application/atom+xml; - - # curl https://www.cloudflare.com/ips-v4 | sort - set_real_ip_from 103.21.244.0/22; - set_real_ip_from 103.22.200.0/22; - set_real_ip_from 103.31.4.0/22; - set_real_ip_from 104.16.0.0/12; - set_real_ip_from 108.162.192.0/18; - set_real_ip_from 131.0.72.0/22; - set_real_ip_from 141.101.64.0/18; - set_real_ip_from 162.158.0.0/15; - set_real_ip_from 172.64.0.0/13; - set_real_ip_from 173.245.48.0/20; - set_real_ip_from 188.114.96.0/20; - set_real_ip_from 190.93.240.0/20; - set_real_ip_from 197.234.240.0/22; - set_real_ip_from 198.41.128.0/17; - set_real_ip_from 199.27.128.0/21; - - # curl https://www.cloudflare.com/ips-v4 | sort - set_real_ip_from 2400:cb00::/32; - set_real_ip_from 2606:4700::/32; - set_real_ip_from 2803:f800::/32; - set_real_ip_from 2405:b500::/32; - set_real_ip_from 2405:8100::/32; - set_real_ip_from 2a06:98c0::/29; - set_real_ip_from 2c0f:f248::/32; - - real_ip_header CF-Connecting-IP; - - include /etc/nginx/sites-enabled/*.conf; -} diff --git a/script/install/nginx/sites-enabled/danbooru.conf b/script/install/nginx/sites-enabled/danbooru.conf deleted file mode 100644 index 30d19cda3..000000000 --- a/script/install/nginx/sites-enabled/danbooru.conf +++ /dev/null @@ -1,68 +0,0 @@ -server { - listen 443 ssl http2 default_server; - server_name danbooru.donmai.us; - - ssl_certificate /etc/nginx/ssl/danbooru.chain.pem; - ssl_certificate_key /etc/nginx/ssl/danbooru.key; - - include /etc/nginx/conf.d/common.conf; -} - -server { - listen 443 ssl http2; - server_name safebooru.donmai.us; - - ssl_certificate /etc/nginx/ssl/safebooru.chain.pem; - ssl_certificate_key /etc/nginx/ssl/safebooru.key; - - include /etc/nginx/conf.d/common.conf; -} - -server { - listen 443 ssl http2; - server_name kagamihara.donmai.us; - - ssl_certificate /etc/letsencrypt/live/kagamihara.donmai.us/fullchain.pem; # managed by Certbot - ssl_certificate_key /etc/letsencrypt/live/kagamihara.donmai.us/privkey.pem; # managed by Certbot - - include /etc/nginx/conf.d/common.conf; -} - -server { - listen 443 ssl http2; - server_name saitou.donmai.us; - - ssl_certificate /etc/letsencrypt/live/saitou.donmai.us/fullchain.pem; # managed by Certbot - ssl_certificate_key /etc/letsencrypt/live/saitou.donmai.us/privkey.pem; # managed by Certbot - - include /etc/nginx/conf.d/common.conf; -} - -server { - listen 443 ssl http2; - server_name shima.donmai.us; - - ssl_certificate /etc/letsencrypt/live/shima.donmai.us/fullchain.pem; # managed by Certbot - ssl_certificate_key /etc/letsencrypt/live/shima.donmai.us/privkey.pem; # managed by Certbot - - include /etc/nginx/conf.d/common.conf; -} - -# redirect HTTP to HTTPS. -server { - listen 80; - server_name safebooru.donmai.us danbooru.donmai.us kagamihara.donmai.us saitou.donmai.us shima.donmai.us; - return 301 https://$host$request_uri; -} - -# redirect donmai.us and www.donmai.us to danbooru.donmai.us. -server { - listen 80; - listen 443 ssl; - server_name donmai.us www.donmai.us; - return 301 https://danbooru.donmai.us$request_uri; -} - -upstream app_server { - server unix:/tmp/.unicorn.sock fail_timeout=0; -} diff --git a/script/install/vrack-cfg.yaml b/script/install/vrack-cfg.yaml deleted file mode 100644 index cbd7434a3..000000000 --- a/script/install/vrack-cfg.yaml +++ /dev/null @@ -1,10 +0,0 @@ -network: - version: 2 - renderer: networkd - ethernets: - eno2: {} - vlans: - eno2.99: - id: 99 - link: eno2 - addresses: [172.16.0.1] From b904c01d69ef9685f3dc23497f217b3806e9a13e Mon Sep 17 00:00:00 2001 From: evazion Date: Sun, 21 Jun 2020 19:01:12 -0500 Subject: [PATCH 42/59] config: remove unused config files. --- config/cable.yml | 10 ---------- config/secrets.yml | 26 -------------------------- config/storage.yml | 34 ---------------------------------- 3 files changed, 70 deletions(-) delete mode 100644 config/cable.yml delete mode 100644 config/secrets.yml delete mode 100644 config/storage.yml diff --git a/config/cable.yml b/config/cable.yml deleted file mode 100644 index feea78f65..000000000 --- a/config/cable.yml +++ /dev/null @@ -1,10 +0,0 @@ -development: - adapter: async - -test: - adapter: test - -production: - adapter: redis - url: <%= ENV.fetch("REDIS_URL") { "redis://localhost:6379/1" } %> - channel_prefix: danbooru_production diff --git a/config/secrets.yml b/config/secrets.yml deleted file mode 100644 index a3d89cfec..000000000 --- a/config/secrets.yml +++ /dev/null @@ -1,26 +0,0 @@ -# Be sure to restart your server when you modify this file. - -# Your secret key is used for verifying the integrity of signed cookies. -# If you change this key, all old signed cookies will become invalid! - -# Make sure the secret is at least 30 characters and all random, -# no regular words or you'll be exposed to dictionary attacks. -# You can use `rake secret` to generate a secure secret key. - -# Make sure the secrets in this file are kept private -# if you're sharing your code publicly. - -development: - secret_key_base: bcc62a512b9c055c292c17742f1e65bd6d88fa37f4d01c8475103809f3ac4c03e3e98605c47d55cd8801333010ea98920a61b722770629926759624bce732539 - -test: - secret_key_base: 60e32a818af77bdfc40bca866e3b4d7b88d7ba767057ffc9e4532279358af8c67d42f2b99c084b700727303ce25b812a592b52723ebc1e3b812fd09a1f969435 - -# Do not keep production secrets in the repository, -# instead read values from the environment. -production: - secret_key_base: <%= ENV["SECRET_KEY_BASE"] %> - -staging: - secret_key_base: <%= ENV["SECRET_KEY_BASE"] %> - \ No newline at end of file diff --git a/config/storage.yml b/config/storage.yml deleted file mode 100644 index d32f76e8f..000000000 --- a/config/storage.yml +++ /dev/null @@ -1,34 +0,0 @@ -test: - service: Disk - root: <%= Rails.root.join("tmp/storage") %> - -local: - service: Disk - root: <%= Rails.root.join("storage") %> - -# Use rails credentials:edit to set the AWS secrets (as aws:access_key_id|secret_access_key) -# amazon: -# service: S3 -# access_key_id: <%= Rails.application.credentials.dig(:aws, :access_key_id) %> -# secret_access_key: <%= Rails.application.credentials.dig(:aws, :secret_access_key) %> -# region: us-east-1 -# bucket: your_own_bucket - -# Remember not to checkin your GCS keyfile to a repository -# google: -# service: GCS -# project: your_project -# credentials: <%= Rails.root.join("path/to/gcs.keyfile") %> -# bucket: your_own_bucket - -# Use rails credentials:edit to set the Azure Storage secret (as azure_storage:storage_access_key) -# microsoft: -# service: AzureStorage -# storage_account_name: your_account_name -# storage_access_key: <%= Rails.application.credentials.dig(:azure_storage, :storage_access_key) %> -# container: your_container_name - -# mirror: -# service: Mirror -# primary: local -# mirrors: [ amazon, google, microsoft ] From db3407caa31e1da1c3b6cd634aac1a1cfadb4881 Mon Sep 17 00:00:00 2001 From: evazion Date: Mon, 22 Jun 2020 15:32:48 -0500 Subject: [PATCH 43/59] uploads: fix uploading from source not working. ref: https://github.com/danbooru/danbooru/commit/26ad844bbe9be28eead133701f42036f7fd8bb62#r40077579. --- app/logical/sources/strategies/base.rb | 3 +- test/functional/uploads_controller_test.rb | 96 ++++++++++++++++------ 2 files changed, 71 insertions(+), 28 deletions(-) diff --git a/app/logical/sources/strategies/base.rb b/app/logical/sources/strategies/base.rb index f7e4f4b76..e2f490657 100644 --- a/app/logical/sources/strategies/base.rb +++ b/app/logical/sources/strategies/base.rb @@ -39,7 +39,7 @@ module Sources def initialize(url, referer_url = nil) @url = url.to_s @referer_url = referer_url&.to_s - @urls = [url, referer_url].select(&:present?) + @urls = [@url, @referer_url].select(&:present?) @parsed_url = Addressable::URI.heuristic_parse(url) rescue nil @parsed_referer = Addressable::URI.heuristic_parse(referer_url) rescue nil @@ -152,6 +152,7 @@ module Sources # Download the file at the given url, or at the main image url by default. def download_file!(download_url = image_url) + raise DownloadError, "Download failed: couldn't find download url for #{url}" if download_url.blank? response, file = http.download_media(download_url) raise DownloadError, "Download failed: #{download_url} returned error #{response.status}" if response.status != 200 file diff --git a/test/functional/uploads_controller_test.rb b/test/functional/uploads_controller_test.rb index e19949fca..c3b71f3fd 100644 --- a/test/functional/uploads_controller_test.rb +++ b/test/functional/uploads_controller_test.rb @@ -1,15 +1,30 @@ require 'test_helper' class UploadsControllerTest < ActionDispatch::IntegrationTest - def assert_uploaded(file_path, user, **upload_params) - file = Rack::Test::UploadedFile.new("#{Rails.root}/#{file_path}") + def self.should_upload_successfully(source) + should "upload successfully from #{source}" do + assert_successful_upload(source, user: create(:user, created_at: 1.month.ago)) + end + end - assert_difference(["Upload.count", "Post.count"]) do - post_auth uploads_path, user, params: { upload: { file: file, **upload_params }} - assert_redirected_to Upload.last + def assert_successful_upload(source_or_file_path, user: @user, **params) + if source_or_file_path =~ %r{\Ahttps?://}i + source = { source: source_or_file_path } + else + file = Rack::Test::UploadedFile.new(Rails.root.join(source_or_file_path)) + source = { file: file } end - Upload.last + assert_difference(["Upload.count"]) do + post_auth uploads_path, user, params: { upload: { tag_string: "abc", rating: "e", **source, **params }} + end + + upload = Upload.last + assert_response :redirect + assert_redirected_to upload + assert_equal("completed", upload.status) + assert_equal(Post.last, upload.post) + assert_equal(upload.post.md5, upload.md5) end context "The uploads controller" do @@ -270,32 +285,59 @@ class UploadsControllerTest < ActionDispatch::IntegrationTest end context "uploading a file from your computer" do - should "work for a jpeg file" do - upload = assert_uploaded("test/files/test.jpg", @user, tag_string: "aaa", rating: "e", source: "aaa") + should_upload_successfully("test/files/test.jpg") + should_upload_successfully("test/files/test.png") + should_upload_successfully("test/files/test-static-32x32.gif") + should_upload_successfully("test/files/test-animated-86x52.gif") + should_upload_successfully("test/files/test-300x300.mp4") + should_upload_successfully("test/files/test-512x512.webm") + should_upload_successfully("test/files/compressed.swf") + end - assert_equal("jpg", upload.post.file_ext) - assert_equal("aaa", upload.post.source) - assert_equal(500, upload.post.image_width) - assert_equal(335, upload.post.image_height) - end + context "uploading a file from a source" do + should_upload_successfully("https://www.artstation.com/artwork/04XA4") + should_upload_successfully("https://dantewontdie.artstation.com/projects/YZK5q") + should_upload_successfully("https://cdna.artstation.com/p/assets/images/images/006/029/978/large/amama-l-z.jpg") - should "work for a webm file" do - upload = assert_uploaded("test/files/test-512x512.webm", @user, tag_string: "aaa", rating: "e", source: "aaa") + should_upload_successfully("https://www.deviantart.com/aeror404/art/Holiday-Elincia-424551484") + should_upload_successfully("https://noizave.deviantart.com/art/test-no-download-697415967") + should_upload_successfully("https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/8b472d70-a0d6-41b5-9a66-c35687090acc/d23jbr4-8a06af02-70cb-46da-8a96-42a6ba73cdb4.jpg/v1/fill/w_786,h_1017,q_70,strp/silverhawks_quicksilver_by_edsfox_d23jbr4-pre.jpg") - assert_equal("webm", upload.post.file_ext) - assert_equal("aaa", upload.post.source) - assert_equal(512, upload.post.image_width) - assert_equal(512, upload.post.image_height) - end + should_upload_successfully("https://www.hentai-foundry.com/pictures/user/Afrobull/795025/kuroeda") + should_upload_successfully("https://pictures.hentai-foundry.com/a/Afrobull/795025/Afrobull-795025-kuroeda.png") - should "work for a flash file" do - upload = assert_uploaded("test/files/compressed.swf", @user, tag_string: "aaa", rating: "e", source: "aaa") + should_upload_successfully("https://yande.re/post/show/482880") + should_upload_successfully("https://files.yande.re/image/7ecfdead705d7b956b26b1d37b98d089/yande.re%20482880.jpg") - assert_equal("swf", upload.post.file_ext) - assert_equal("aaa", upload.post.source) - assert_equal(607, upload.post.image_width) - assert_equal(756, upload.post.image_height) - end + should_upload_successfully("https://konachan.com/post/show/270916") + should_upload_successfully("https://konachan.com/image/ca12cdb79a66d242e95a6f958341bf05/Konachan.com%20-%20270916.png") + + should_upload_successfully("http://lohas.nicoseiga.jp/o/910aecf08e542285862954017f8a33a8c32a8aec/1433298801/4937663") + should_upload_successfully("http://seiga.nicovideo.jp/seiga/im4937663") + should_upload_successfully("https://seiga.nicovideo.jp/image/source/9146749") + should_upload_successfully("https://seiga.nicovideo.jp/watch/mg389884") + should_upload_successfully("https://www.nicovideo.jp/watch/sm36465441") + should_upload_successfully("https://dic.nicovideo.jp/oekaki/52833.png") + + should_upload_successfully("http://nijie.info/view.php?id=213043") + should_upload_successfully("https://nijie.info/view_popup.php?id=213043") + should_upload_successfully("https://pic.nijie.net/03/nijie_picture/728995_20170505014820_0.jpg") + + should_upload_successfully("https://pawoo.net/web/statuses/1202176") + should_upload_successfully("https://img.pawoo.net/media_attachments/files/000/128/953/original/4c0a06087b03343f.png") + + should_upload_successfully("https://www.pixiv.net/en/artworks/64476642") + should_upload_successfully("https://i.pximg.net/img-original/img/2017/08/18/00/09/21/64476642_p0.jpg") + + should_upload_successfully("https://noizave.tumblr.com/post/162206271767") + should_upload_successfully("https://media.tumblr.com/3bbfcbf075ddf969c996641b264086fd/tumblr_os2buiIOt51wsfqepo1_1280.png") + + should_upload_successfully("https://twitter.com/noizave/status/875768175136317440") + should_upload_successfully("https://pbs.twimg.com/media/DCdZ_FhUIAAYKFN?format=jpg&name=medium") + should_upload_successfully("https://video.twimg.com/tweet_video/EWHWVrmVcAAp4Vw.mp4") + + should_upload_successfully("https://www.weibo.com/5501756072/J2UNKfbqV") + should_upload_successfully("https://wx1.sinaimg.cn/mw690/0060kO5aly1gezsyt5xvhj30ok0sgtc9.jpg") end end end From 95fee75d9ac5ecdf57db9e5bfb13ac642692a37f Mon Sep 17 00:00:00 2001 From: evazion Date: Mon, 22 Jun 2020 16:53:50 -0500 Subject: [PATCH 44/59] nicoseiga: fix uploads not working for certain direct image urls. Fix Nicoseiga strategy to work with certain direct image urls that we can't otherwise extract any information from. Examples: * https://dic.nicovideo.jp/oekaki/52833.png --- app/logical/sources/strategies/nico_seiga.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/logical/sources/strategies/nico_seiga.rb b/app/logical/sources/strategies/nico_seiga.rb index ee4efd3d5..e3ca777ba 100644 --- a/app/logical/sources/strategies/nico_seiga.rb +++ b/app/logical/sources/strategies/nico_seiga.rb @@ -73,8 +73,7 @@ module Sources end def image_url - return if image_urls.blank? - return url if api_client.blank? + return url if image_urls.blank? || api_client.blank? img = case url when DIRECT || CDN_DIRECT then "https://seiga.nicovideo.jp/image/source/#{image_id_from_url(url)}" From 8c6759bbd714c65f20c1e17b850c5ca6d3526e0a Mon Sep 17 00:00:00 2001 From: evazion Date: Mon, 22 Jun 2020 18:35:19 -0500 Subject: [PATCH 45/59] nicoseiga: fix login endpoint. * Update the login endpoint. The old endpoint returns 404 now. POST https://account.nicovideo.jp/api/v1/login -> POST https://account.nicovideo.jp/login/redirector?site=seiga * Let Danbooru::Http cache the login request instead of caching it manually. * Let Danbooru::Http automatically follow redirects instead of dealing with the Location header manually. --- app/logical/nico_seiga_api_client.rb | 31 +++++++------------- app/logical/sources/strategies/nico_seiga.rb | 8 ++--- test/functional/uploads_controller_test.rb | 9 ++++-- 3 files changed, 21 insertions(+), 27 deletions(-) diff --git a/app/logical/nico_seiga_api_client.rb b/app/logical/nico_seiga_api_client.rb index 2b67bbde0..b9f4f00d7 100644 --- a/app/logical/nico_seiga_api_client.rb +++ b/app/logical/nico_seiga_api_client.rb @@ -4,12 +4,10 @@ class NicoSeigaApiClient attr_reader :http - # XXX temp disable following redirects. - def initialize(work_id:, type:) + def initialize(work_id:, type:, http: Danbooru::Http.new) @work_id = work_id @work_type = type - @http = Danbooru::Http.new - @http.http.default_options.features.reject! { |name, _| name == :redirector } + @http = http end def image_ids @@ -81,28 +79,19 @@ class NicoSeigaApiClient end def get(url) - cookie_header = Cache.get("nicoseiga-cookie-header") || regenerate_cookie_header - - resp = http.headers({Cookie: cookie_header}).cache(1.minute).get(url) - - if resp.headers["Location"] =~ %r{seiga\.nicovideo\.jp/login/}i - cookie_header = regenerate_cookie_header - resp = http.headers({Cookie: cookie_header}).cache(1.minute).get(url) - end - - resp - end - - def regenerate_cookie_header form = { mail_tel: Danbooru.config.nico_seiga_login, password: Danbooru.config.nico_seiga_password } - resp = http.post("https://account.nicovideo.jp/api/v1/login", form: form) - cookies = resp.cookies.map { |c| c.name + "=" + c.value } - cookies << "accept_fetish_warning=2" - Cache.put("nicoseiga-cookie-header", cookies.join(";"), 1.week) + # XXX should fail gracefully instead of raising exception + resp = http.cache(1.hour).post("https://account.nicovideo.jp/login/redirector?site=seiga", form: form) + raise RuntimeError, "NicoSeiga login failed (status=#{resp.status} url=#{url})" if resp.status != 200 + + resp = http.headers(accept_fetish_warning: 2).cache(1.minute).get(url) + #raise RuntimeError, "NicoSeiga get failed (status=#{resp.status} url=#{url})" if resp.status != 200 + + resp end memoize :api_response, :manga_api_response, :user_api_response diff --git a/app/logical/sources/strategies/nico_seiga.rb b/app/logical/sources/strategies/nico_seiga.rb index e3ca777ba..d4af63084 100644 --- a/app/logical/sources/strategies/nico_seiga.rb +++ b/app/logical/sources/strategies/nico_seiga.rb @@ -82,7 +82,7 @@ module Sources end resp = api_client.get(img) - if resp.headers["Location"] =~ %r{https?://.+/(\w+/\d+/\d+)\z}i + if resp.uri.to_s =~ %r{https?://.+/(\w+/\d+/\d+)\z}i "https://lohas.nicoseiga.jp/priv/#{$1}" else img @@ -180,12 +180,12 @@ module Sources def api_client if illust_id.present? - NicoSeigaApiClient.new(work_id: illust_id, type: "illust") + NicoSeigaApiClient.new(work_id: illust_id, type: "illust", http: http) elsif manga_id.present? - NicoSeigaApiClient.new(work_id: manga_id, type: "manga") + NicoSeigaApiClient.new(work_id: manga_id, type: "manga", http: http) elsif image_id.present? # We default to illust to attempt getting the api anyway - NicoSeigaApiClient.new(work_id: image_id, type: "illust") + NicoSeigaApiClient.new(work_id: image_id, type: "illust", http: http) end end memoize :api_client diff --git a/test/functional/uploads_controller_test.rb b/test/functional/uploads_controller_test.rb index c3b71f3fd..1ff08e658 100644 --- a/test/functional/uploads_controller_test.rb +++ b/test/functional/uploads_controller_test.rb @@ -316,8 +316,13 @@ class UploadsControllerTest < ActionDispatch::IntegrationTest should_upload_successfully("http://seiga.nicovideo.jp/seiga/im4937663") should_upload_successfully("https://seiga.nicovideo.jp/image/source/9146749") should_upload_successfully("https://seiga.nicovideo.jp/watch/mg389884") - should_upload_successfully("https://www.nicovideo.jp/watch/sm36465441") should_upload_successfully("https://dic.nicovideo.jp/oekaki/52833.png") + should_upload_successfully("https://lohas.nicoseiga.jp/o/971eb8af9bbcde5c2e51d5ef3a2f62d6d9ff5552/1589933964/3583893") + should_upload_successfully("http://lohas.nicoseiga.jp/priv/3521156?e=1382558156&h=f2e089256abd1d453a455ec8f317a6c703e2cedf") + should_upload_successfully("http://lohas.nicoseiga.jp/priv/b80f86c0d8591b217e7513a9e175e94e00f3c7a1/1384936074/3583893") + should_upload_successfully("http://lohas.nicoseiga.jp/material/5746c5/4459092") + # XXX should_upload_successfully("https://dcdn.cdn.nimg.jp/priv/62a56a7f67d3d3746ae5712db9cac7d465f4a339/1592186183/10466669") + # XXX should_upload_successfully("https://dcdn.cdn.nimg.jp/nicoseiga/lohas/o/8ba0a9b2ea34e1ef3b5cc50785bd10cd63ec7e4a/1592187477/10466669") should_upload_successfully("http://nijie.info/view.php?id=213043") should_upload_successfully("https://nijie.info/view_popup.php?id=213043") @@ -334,7 +339,7 @@ class UploadsControllerTest < ActionDispatch::IntegrationTest should_upload_successfully("https://twitter.com/noizave/status/875768175136317440") should_upload_successfully("https://pbs.twimg.com/media/DCdZ_FhUIAAYKFN?format=jpg&name=medium") - should_upload_successfully("https://video.twimg.com/tweet_video/EWHWVrmVcAAp4Vw.mp4") + # XXX should_upload_successfully("https://video.twimg.com/tweet_video/EWHWVrmVcAAp4Vw.mp4") should_upload_successfully("https://www.weibo.com/5501756072/J2UNKfbqV") should_upload_successfully("https://wx1.sinaimg.cn/mw690/0060kO5aly1gezsyt5xvhj30ok0sgtc9.jpg") From a6994cd4d78cc752b26403bcb402ab88d55cf498 Mon Sep 17 00:00:00 2001 From: evazion Date: Mon, 22 Jun 2020 16:59:28 -0500 Subject: [PATCH 46/59] media file: fix exception on empty files. This may happen if a user uploads from a source that returns an error HTTP response with no data. --- app/logical/danbooru/http.rb | 2 -- app/logical/media_file.rb | 2 ++ test/files/test-empty.bin | 0 test/unit/media_file_test.rb | 4 ++++ 4 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 test/files/test-empty.bin diff --git a/app/logical/danbooru/http.rb b/app/logical/danbooru/http.rb index 6bbc7db87..17f45ec21 100644 --- a/app/logical/danbooru/http.rb +++ b/app/logical/danbooru/http.rb @@ -7,7 +7,6 @@ require "danbooru/http/session" module Danbooru class Http - class DownloadError < StandardError; end class FileTooLargeError < StandardError; end DEFAULT_TIMEOUT = 10 @@ -111,7 +110,6 @@ module Danbooru end def download_response(response, file: Tempfile.new("danbooru-download-", binmode: true)) - raise DownloadError, response if response.status != 200 raise FileTooLargeError, response if @max_size && response.content_length.to_i > @max_size size = 0 diff --git a/app/logical/media_file.rb b/app/logical/media_file.rb index c28bf25a9..334de8f05 100644 --- a/app/logical/media_file.rb +++ b/app/logical/media_file.rb @@ -43,6 +43,8 @@ class MediaFile else :bin end + rescue EOFError + :bin end def self.videos_enabled? diff --git a/test/files/test-empty.bin b/test/files/test-empty.bin new file mode 100644 index 000000000..e69de29bb diff --git a/test/unit/media_file_test.rb b/test/unit/media_file_test.rb index a7516198d..4c77cf5b3 100644 --- a/test/unit/media_file_test.rb +++ b/test/unit/media_file_test.rb @@ -98,6 +98,10 @@ class MediaFileTest < ActiveSupport::TestCase should "determine the correct extension for a flash file" do assert_equal(:swf, MediaFile.open("test/files/compressed.swf").file_ext) end + + should "not fail for empty files" do + assert_equal(:bin, MediaFile.open("test/files/test-empty.bin").file_ext) + end end should "determine the correct md5 for a jpeg file" do From 31802fb666c1a86987b4f57f37a7a13149f14b07 Mon Sep 17 00:00:00 2001 From: evazion Date: Mon, 22 Jun 2020 22:21:17 -0500 Subject: [PATCH 47/59] nijie: fix parallel test failures. Nijie tests fail often under parallel testing. This is because every test needs to login to Nijie first, but Nijie rate-limits the login endpoint, so eventually we hit the limit and tests start failing. This is made worse by a thundering herd problem. Eight test processes try to login to Nijie at the same time, but only one succeeds, so the rest sleep and try again, but they all wakeup and try again at the same time, hitting the rate limits again. The workaround is to set the retry limit ridiculously high, higher than we would ideally like in production. Another workaround would be to serialize the Nijie tests in the test suite. This can be done with lockfiles and flock(2). This helps, but we can still hit the rate limit even under serialized execution. --- app/logical/sources/strategies/nijie.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/logical/sources/strategies/nijie.rb b/app/logical/sources/strategies/nijie.rb index b2a6fcc7a..4556f274a 100644 --- a/app/logical/sources/strategies/nijie.rb +++ b/app/logical/sources/strategies/nijie.rb @@ -182,7 +182,7 @@ module Sources form = { email: Danbooru.config.nijie_login, password: Danbooru.config.nijie_password } # XXX `retriable` must come after `cache` so that retries don't return cached error responses. - response = http.cache(1.hour).use(:retriable).post("https://nijie.info/login_int.php", form: form) + response = http.cache(1.hour).use(retriable: { max_retries: 20 }).post("https://nijie.info/login_int.php", form: form) DanbooruLogger.info "Nijie login failed (#{url}, #{response.status})" if response.status != 200 return nil unless response.status == 200 From 7f5e87568a6d322d44b2d82d101913c3f1ab4898 Mon Sep 17 00:00:00 2001 From: evazion Date: Mon, 22 Jun 2020 22:51:36 -0500 Subject: [PATCH 48/59] danbooru::http: raise exception on failed downloads. Restore behavior from a6994cd4d, it breaks tests when they try to the response body from a fake 599 response. --- app/logical/danbooru/http.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/logical/danbooru/http.rb b/app/logical/danbooru/http.rb index 17f45ec21..ed25aa804 100644 --- a/app/logical/danbooru/http.rb +++ b/app/logical/danbooru/http.rb @@ -7,6 +7,7 @@ require "danbooru/http/session" module Danbooru class Http + class DownloadError < StandardError; end class FileTooLargeError < StandardError; end DEFAULT_TIMEOUT = 10 @@ -110,6 +111,7 @@ module Danbooru end def download_response(response, file: Tempfile.new("danbooru-download-", binmode: true)) + raise DownloadError, "Downloading #{response.uri} failed with code #{response.status}" if response.status != 200 raise FileTooLargeError, response if @max_size && response.content_length.to_i > @max_size size = 0 From 6ea2c934fd0197f4e13c4d3a316125afb5e7c732 Mon Sep 17 00:00:00 2001 From: evazion Date: Mon, 22 Jun 2020 22:54:41 -0500 Subject: [PATCH 49/59] tests: temp skip failing danbooru::http redirect tests. Skip for now until https://github.com/postmanlabs/httpbin/issues/617 is fixed. --- test/unit/danbooru_http_test.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/unit/danbooru_http_test.rb b/test/unit/danbooru_http_test.rb index aa73976ea..79ba9e0d5 100644 --- a/test/unit/danbooru_http_test.rb +++ b/test/unit/danbooru_http_test.rb @@ -11,11 +11,13 @@ class DanbooruHttpTest < ActiveSupport::TestCase end should "follow redirects" do + skip "Skipping test (https://github.com/postmanlabs/httpbin/issues/617)" response = Danbooru::Http.get("https://httpbin.org/absolute-redirect/3") assert_equal(200, response.status) end should "fail if redirected too many times" do + skip "Skipping test (https://github.com/postmanlabs/httpbin/issues/617)" response = Danbooru::Http.get("https://httpbin.org/absolute-redirect/10") assert_equal(598, response.status) end @@ -134,6 +136,7 @@ class DanbooruHttpTest < ActiveSupport::TestCase end should "follow redirects when downloading files" do + skip "Skipping test (https://github.com/postmanlabs/httpbin/issues/617)" response, file = Danbooru::Http.download_media("https://httpbin.org/redirect-to?url=https://httpbin.org/bytes/1000") assert_equal(200, response.status) From 50740e302fcc21ab2e2d67253b528abf9a542346 Mon Sep 17 00:00:00 2001 From: BrokenEagle Date: Mon, 22 Jun 2020 16:07:11 +0000 Subject: [PATCH 50/59] Add option to search for wikis that don't link to a specific wiki Also add inputs on the search page for both the linked_to and the not_linked_to search parameters. Additionally, normalize the title first since autocomplete adds trailing spaces. The search query was also simplified a bit by taking advantage of Rails associations. --- app/models/wiki_page.rb | 11 ++++++++++- app/views/wiki_pages/search.html.erb | 2 ++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/app/models/wiki_page.rb b/app/models/wiki_page.rb index efef59f3a..73f7f8ced 100644 --- a/app/models/wiki_page.rb +++ b/app/models/wiki_page.rb @@ -53,7 +53,11 @@ class WikiPage < ApplicationRecord end def linked_to(title) - where(id: DtextLink.wiki_page.wiki_link.where(link_target: title).select(:model_id)) + where(dtext_links: DtextLink.wiki_page.wiki_link.where(link_target: normalize_title(title))) + end + + def not_linked_to(title) + where.not(dtext_links: DtextLink.wiki_page.wiki_link.where(link_target: normalize_title(title))) end def default_order @@ -82,6 +86,10 @@ class WikiPage < ApplicationRecord q = q.linked_to(params[:linked_to]) end + if params[:not_linked_to].present? + q = q.not_linked_to(params[:not_linked_to]) + end + if params[:hide_deleted].to_s.truthy? q = q.where("is_deleted = false") end @@ -146,6 +154,7 @@ class WikiPage < ApplicationRecord end def self.normalize_title(title) + return if title.blank? title.downcase.delete_prefix("~").gsub(/[[:space:]]+/, "_").gsub(/__/, "_").gsub(/\A_|_\z/, "") end diff --git a/app/views/wiki_pages/search.html.erb b/app/views/wiki_pages/search.html.erb index 4fb24c4fb..3297d9695 100644 --- a/app/views/wiki_pages/search.html.erb +++ b/app/views/wiki_pages/search.html.erb @@ -4,6 +4,8 @@ <%= f.input :title_normalize, label: "Title", hint: "Use * for wildcard searches", input_html: { "data-autocomplete": "wiki-page" } %> <%= f.input :other_names_match, label: "Other names", hint: "Use * for wildcard searches" %> <%= f.input :body_matches, label: "Body" %> + <%= f.input :linked_to, hint: "Which wikis link to the specified wiki.", input_html: { "data-autocomplete": "wiki-page" } %> + <%= f.input :not_linked_to, hint: "Which wikis do not link to the specified wiki.", input_html: { "data-autocomplete": "wiki-page" } %> <%= f.input :other_names_present, as: :select %> <%= f.input :hide_deleted, as: :select, include_blank: false %> <%= f.input :order, collection: [%w[Name title], %w[Date time], %w[Posts post_count]], include_blank: false %> From 5dbe08372bc1733ab83842561ba589da993da0ee Mon Sep 17 00:00:00 2001 From: evazion Date: Tue, 23 Jun 2020 00:49:38 -0500 Subject: [PATCH 51/59] nicoseiga: remove accept_fetish_warning. This should have been a cookie, but it doesn't matter because the strategy still works without it. --- app/logical/nico_seiga_api_client.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/logical/nico_seiga_api_client.rb b/app/logical/nico_seiga_api_client.rb index b9f4f00d7..d471836d6 100644 --- a/app/logical/nico_seiga_api_client.rb +++ b/app/logical/nico_seiga_api_client.rb @@ -88,7 +88,7 @@ class NicoSeigaApiClient resp = http.cache(1.hour).post("https://account.nicovideo.jp/login/redirector?site=seiga", form: form) raise RuntimeError, "NicoSeiga login failed (status=#{resp.status} url=#{url})" if resp.status != 200 - resp = http.headers(accept_fetish_warning: 2).cache(1.minute).get(url) + resp = http.cache(1.minute).get(url) #raise RuntimeError, "NicoSeiga get failed (status=#{resp.status} url=#{url})" if resp.status != 200 resp From 1a879357643717acad82125d8db6d73af05a6b2b Mon Sep 17 00:00:00 2001 From: BrokenEagle Date: Tue, 23 Jun 2020 05:46:25 +0000 Subject: [PATCH 52/59] Fix the tag matches option The split function was mistakenly used instead of the match function. --- app/models/post_version.rb | 2 +- test/functional/post_versions_controller_test.rb | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/app/models/post_version.rb b/app/models/post_version.rb index a6be23df2..a38ee2779 100644 --- a/app/models/post_version.rb +++ b/app/models/post_version.rb @@ -33,7 +33,7 @@ class PostVersion < ApplicationRecord end def tag_matches(string) - tag = string.split(/\S+/)[0] + tag = string.match(/\S+/)[0] return all if tag.nil? tag = "*#{tag}*" unless tag =~ /\*/ where_ilike(:tags, tag) diff --git a/test/functional/post_versions_controller_test.rb b/test/functional/post_versions_controller_test.rb index a57914cff..521d99adb 100644 --- a/test/functional/post_versions_controller_test.rb +++ b/test/functional/post_versions_controller_test.rb @@ -41,6 +41,13 @@ class PostVersionsControllerTest < ActionDispatch::IntegrationTest assert_response :success assert_equal @post.versions[1].id, response.parsed_body[0]["id"].to_i end + + should "list all versions for search[tag_matches]" do + get post_versions_path, as: :json, params: { search: { tag_matches: "tagme" }} + assert_response :success + assert_equal @post.versions[0].id, response.parsed_body[0]["id"].to_i + assert_equal 1, response.parsed_body.length + end end context "undo action" do From 73506bac3314d60827eb4c3d00f27f7c44eb0c0a Mon Sep 17 00:00:00 2001 From: evazion Date: Tue, 23 Jun 2020 02:37:21 -0500 Subject: [PATCH 53/59] twitter: add tests for uploading profile banners (#4520). --- test/functional/uploads_controller_test.rb | 1 + test/unit/sources/twitter_test.rb | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/test/functional/uploads_controller_test.rb b/test/functional/uploads_controller_test.rb index 1ff08e658..10fab5471 100644 --- a/test/functional/uploads_controller_test.rb +++ b/test/functional/uploads_controller_test.rb @@ -339,6 +339,7 @@ class UploadsControllerTest < ActionDispatch::IntegrationTest should_upload_successfully("https://twitter.com/noizave/status/875768175136317440") should_upload_successfully("https://pbs.twimg.com/media/DCdZ_FhUIAAYKFN?format=jpg&name=medium") + should_upload_successfully("https://pbs.twimg.com/profile_banners/1225702850002468864/1588597370/1500x500") # XXX should_upload_successfully("https://video.twimg.com/tweet_video/EWHWVrmVcAAp4Vw.mp4") should_upload_successfully("https://www.weibo.com/5501756072/J2UNKfbqV") diff --git a/test/unit/sources/twitter_test.rb b/test/unit/sources/twitter_test.rb index a3b9728f8..96e187ae2 100644 --- a/test/unit/sources/twitter_test.rb +++ b/test/unit/sources/twitter_test.rb @@ -244,6 +244,14 @@ module Sources end end + context "A profile banner image" do + should "work" do + @site = Sources::Strategies.find("https://pbs.twimg.com/profile_banners/1225702850002468864/1588597370/1500x500") + assert_equal(@site.image_url, @site.url) + assert_nothing_raised { @site.to_h } + end + end + context "A tweet containing non-normalized Unicode text" do should "be normalized to nfkc" do site = Sources::Strategies.find("https://twitter.com/aprilarcus/status/367557195186970624") From 0276792b35700bc9575dccbe8b1bd9fec0d54951 Mon Sep 17 00:00:00 2001 From: evazion Date: Tue, 23 Jun 2020 02:43:12 -0500 Subject: [PATCH 54/59] BURs: add test for users voting on their own BURs (#4527) --- test/functional/forum_post_votes_controller_test.rb | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/functional/forum_post_votes_controller_test.rb b/test/functional/forum_post_votes_controller_test.rb index 7143c038d..5d693d26b 100644 --- a/test/functional/forum_post_votes_controller_test.rb +++ b/test/functional/forum_post_votes_controller_test.rb @@ -36,6 +36,13 @@ class ForumPostVotesControllerTest < ActionDispatch::IntegrationTest assert_response 403 end end + + should "not allow creators to vote on their own BURs" do + assert_difference("ForumPostVote.count", 0) do + post_auth forum_post_votes_path(format: :js), @bulk_update_request.user, params: { forum_post_id: @forum_post.id, forum_post_vote: { score: 1 }} + assert_response 403 + end + end end context "destroy action" do From be4bdfc1368023ae7b68bdd0b08955168e332c88 Mon Sep 17 00:00:00 2001 From: evazion Date: Tue, 23 Jun 2020 02:57:30 -0500 Subject: [PATCH 55/59] artists: add test for hiding deleted wikis on artist pages (#4526). --- app/views/artists/_show.html.erb | 12 +++++++----- app/views/wiki_pages/show.html.erb | 2 +- test/functional/artists_controller_test.rb | 16 ++++++++++++++++ 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/app/views/artists/_show.html.erb b/app/views/artists/_show.html.erb index 5352fc333..8a00e1bf6 100644 --- a/app/views/artists/_show.html.erb +++ b/app/views/artists/_show.html.erb @@ -8,12 +8,14 @@ <% if @artist.is_banned? && !policy(@artist).can_view_banned? %>

The artist requested removal of this page.

<% else %> - <% if @artist.wiki_page.present? && !@artist.wiki_page.is_deleted %> -
- <%= format_text(@artist.wiki_page.body, :disable_mentions => true) %> -
+ <% if @artist.wiki_page.present? && !@artist.wiki_page.is_deleted? %> +
+
+ <%= format_text(@artist.wiki_page.body, :disable_mentions => true) %> +
-

<%= link_to "View wiki page", @artist.wiki_page %>

+

<%= link_to "View wiki page", @artist.wiki_page %>

+
<% end %> <%= yield %> diff --git a/app/views/wiki_pages/show.html.erb b/app/views/wiki_pages/show.html.erb index 9596e045b..bd3b0bd22 100644 --- a/app/views/wiki_pages/show.html.erb +++ b/app/views/wiki_pages/show.html.erb @@ -28,7 +28,7 @@ <%= format_text(@wiki_page.body) %> <% end %> - <% if @wiki_page.artist.present? && !@wiki_page.artist.is_deleted %> + <% if @wiki_page.artist.present? && !@wiki_page.artist.is_deleted? %>

<%= link_to "View artist", @wiki_page.artist %>

<% end %> diff --git a/test/functional/artists_controller_test.rb b/test/functional/artists_controller_test.rb index 2b24f3cc7..2907aa2a9 100644 --- a/test/functional/artists_controller_test.rb +++ b/test/functional/artists_controller_test.rb @@ -54,6 +54,22 @@ class ArtistsControllerTest < ActionDispatch::IntegrationTest get artist_path(@artist.id) assert_response :success end + + should "show active wikis" do + as(@user) { create(:wiki_page, title: @artist.name) } + get artist_path(@artist.id) + + assert_response :success + assert_select ".artist-wiki", count: 1 + end + + should "not show deleted wikis" do + as(@user) { create(:wiki_page, title: @artist.name, is_deleted: true) } + get artist_path(@artist.id) + + assert_response :success + assert_select ".artist-wiki", count: 0 + end end context "new action" do From 83a8468ee968f6de4a86a8434fd90ca831b62785 Mon Sep 17 00:00:00 2001 From: evazion Date: Tue, 23 Jun 2020 03:09:22 -0500 Subject: [PATCH 56/59] tests: remove unnecessary rescueing of Net::OpenTimeout errors. These exceptions are no longer thrown now that we've switched from HTTParty to http.rb. Swallowing unexpected exceptions during testing was a bad practice anyway. --- test/functional/artists_controller_test.rb | 9 +- test/test_helpers/download_test_helper.rb | 2 - test/unit/artist_test.rb | 18 +-- test/unit/sources/pixiv_test.rb | 3 - test/unit/upload_service_test.rb | 171 +++++++++------------ 5 files changed, 77 insertions(+), 126 deletions(-) diff --git a/test/functional/artists_controller_test.rb b/test/functional/artists_controller_test.rb index 2907aa2a9..2ae68d78e 100644 --- a/test/functional/artists_controller_test.rb +++ b/test/functional/artists_controller_test.rb @@ -4,11 +4,8 @@ class ArtistsControllerTest < ActionDispatch::IntegrationTest def assert_artist_found(expected_artist, source_url = nil) if source_url get_auth artists_path(format: "json", search: { url_matches: source_url }), @user - if response.body =~ /Net::OpenTimeout/ - skip "Remote connection to #{source_url} failed" - return - end end + assert_response :success json = JSON.parse(response.body) assert_equal(1, json.size, "Testing URL: #{source_url}") @@ -17,10 +14,6 @@ class ArtistsControllerTest < ActionDispatch::IntegrationTest def assert_artist_not_found(source_url) get_auth artists_path(format: "json", search: { url_matches: source_url }), @user - if response.body =~ /Net::OpenTimeout/ - skip "Remote connection to #{source_url} failed" - return - end assert_response :success json = JSON.parse(response.body) diff --git a/test/test_helpers/download_test_helper.rb b/test/test_helpers/download_test_helper.rb index c3f70c75d..68e14abc6 100644 --- a/test/test_helpers/download_test_helper.rb +++ b/test/test_helpers/download_test_helper.rb @@ -3,8 +3,6 @@ module DownloadTestHelper strategy = Sources::Strategies.find(source, referer) file = strategy.download_file! assert_equal(expected_filesize, file.size, "Tested source URL: #{source}") - rescue Net::OpenTimeout - skip "Remote connection to #{source} failed" end def assert_rewritten(expected_source, test_source, test_referer = nil) diff --git a/test/unit/artist_test.rb b/test/unit/artist_test.rb index b9dc3a90c..44beba428 100644 --- a/test/unit/artist_test.rb +++ b/test/unit/artist_test.rb @@ -6,15 +6,11 @@ class ArtistTest < ActiveSupport::TestCase assert_equal(1, artists.size) assert_equal(expected_name, artists.first.name, "Testing URL: #{source_url}") - rescue Net::OpenTimeout, PixivApiClient::Error - skip "Remote connection failed for #{source_url}" end def assert_artist_not_found(source_url) artists = ArtistFinder.find_artists(source_url).to_a assert_equal(0, artists.size, "Testing URL: #{source_url}") - rescue Net::OpenTimeout - skip "Remote connection failed for #{source_url}" end context "An artist" do @@ -172,15 +168,11 @@ class ArtistTest < ActiveSupport::TestCase a2 = FactoryBot.create(:artist, :name => "subway", :url_string => "http://subway.com/x/test.jpg") a3 = FactoryBot.create(:artist, :name => "minko", :url_string => "https://minko.com/x/test.jpg") - begin - assert_artist_found("rembrandt", "http://rembrandt.com/x/test.jpg") - assert_artist_found("rembrandt", "http://rembrandt.com/x/another.jpg") - assert_artist_not_found("http://nonexistent.com/test.jpg") - assert_artist_found("minko", "https://minko.com/x/test.jpg") - assert_artist_found("minko", "http://minko.com/x/test.jpg") - rescue Net::OpenTimeout - skip "network failure" - end + assert_artist_found("rembrandt", "http://rembrandt.com/x/test.jpg") + assert_artist_found("rembrandt", "http://rembrandt.com/x/another.jpg") + assert_artist_not_found("http://nonexistent.com/test.jpg") + assert_artist_found("minko", "https://minko.com/x/test.jpg") + assert_artist_found("minko", "http://minko.com/x/test.jpg") end should "be case-insensitive to domains when finding matches by url" do diff --git a/test/unit/sources/pixiv_test.rb b/test/unit/sources/pixiv_test.rb index 323e40a00..82f03f5ba 100644 --- a/test/unit/sources/pixiv_test.rb +++ b/test/unit/sources/pixiv_test.rb @@ -15,10 +15,7 @@ module Sources def get_source(source) @site = Sources::Strategies.find(source) - @site - rescue Net::OpenTimeout - skip "Remote connection to #{source} failed" end context "in all cases" do diff --git a/test/unit/upload_service_test.rb b/test/unit/upload_service_test.rb index b15a399a5..d44ecdf44 100644 --- a/test/unit/upload_service_test.rb +++ b/test/unit/upload_service_test.rb @@ -131,12 +131,9 @@ class UploadServiceTest < ActiveSupport::TestCase end should "download the file" do - begin - @service = UploadService::Preprocessor.new(source: @source, referer_url: @ref) - @upload = @service.start! - rescue Net::OpenTimeout - skip "network failure" - end + @service = UploadService::Preprocessor.new(source: @source, referer_url: @ref) + @upload = @service.start! + assert_equal("preprocessed", @upload.status) assert_equal(294591, @upload.file_size) assert_equal("jpg", @upload.file_ext) @@ -155,11 +152,8 @@ class UploadServiceTest < ActiveSupport::TestCase skip unless MediaFile::Ugoira.videos_enabled? @service = UploadService::Preprocessor.new(source: @source) - begin - @upload = @service.start! - rescue Net::OpenTimeout - skip "network problems" - end + @upload = @service.start! + assert_equal("preprocessed", @upload.status) assert_equal(2804, @upload.file_size) assert_equal("zip", @upload.file_ext) @@ -176,11 +170,8 @@ class UploadServiceTest < ActiveSupport::TestCase should "download the file" do @service = UploadService::Preprocessor.new(source: @source) - begin - @upload = @service.start! - rescue Net::OpenTimeout - skip "network problems" - end + @upload = @service.start! + assert_equal("preprocessed", @upload.status) assert_equal(181309, @upload.file_size) assert_equal("jpg", @upload.file_ext) @@ -512,36 +503,28 @@ class UploadServiceTest < ActiveSupport::TestCase context "a post with a pixiv html source" do should "replace with the full size image" do - begin - as(@user) do - @post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350") - end - - assert_equal(80, @post.image_width) - assert_equal(82, @post.image_height) - assert_equal(16275, @post.file_size) - assert_equal("png", @post.file_ext) - assert_equal("4ceadc314938bc27f3574053a3e1459a", @post.md5) - assert_equal("4ceadc314938bc27f3574053a3e1459a", Digest::MD5.file(@post.file).hexdigest) - assert_equal("https://i.pximg.net/img-original/img/2017/04/04/08/54/15/62247350_p0.png", @post.replacements.last.replacement_url) - assert_equal("https://i.pximg.net/img-original/img/2017/04/04/08/54/15/62247350_p0.png", @post.source) - rescue Net::OpenTimeout - skip "Remote connection to Pixiv failed" + as(@user) do + @post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350") end + + assert_equal(80, @post.image_width) + assert_equal(82, @post.image_height) + assert_equal(16275, @post.file_size) + assert_equal("png", @post.file_ext) + assert_equal("4ceadc314938bc27f3574053a3e1459a", @post.md5) + assert_equal("4ceadc314938bc27f3574053a3e1459a", Digest::MD5.file(@post.file).hexdigest) + assert_equal("https://i.pximg.net/img-original/img/2017/04/04/08/54/15/62247350_p0.png", @post.replacements.last.replacement_url) + assert_equal("https://i.pximg.net/img-original/img/2017/04/04/08/54/15/62247350_p0.png", @post.source) end should "delete the old files after thirty days" do - begin - @post.unstub(:queue_delete_files) - FileUtils.expects(:rm_f).times(3) + @post.unstub(:queue_delete_files) + FileUtils.expects(:rm_f).times(3) - as(@user) { @post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350") } + as(@user) { @post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350") } - travel_to((PostReplacement::DELETION_GRACE_PERIOD + 1).days.from_now) do - perform_enqueued_jobs - end - rescue Net::OpenTimeout - skip "Remote connection to Pixiv failed" + travel_to((PostReplacement::DELETION_GRACE_PERIOD + 1).days.from_now) do + perform_enqueued_jobs end end end @@ -568,34 +551,30 @@ class UploadServiceTest < ActiveSupport::TestCase context "a post that is replaced to another file then replaced back to the original file" do should "not delete the original files" do - begin - skip unless MediaFile::Ugoira.videos_enabled? - @post.unstub(:queue_delete_files) + skip unless MediaFile::Ugoira.videos_enabled? + @post.unstub(:queue_delete_files) - # this is called thrice to delete the file for 62247364 - FileUtils.expects(:rm_f).times(3) + # this is called thrice to delete the file for 62247364 + FileUtils.expects(:rm_f).times(3) - as(@user) do - @post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350") - @post.reload - @post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364") - @post.reload - Upload.destroy_all - @post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350") - end - - assert_nothing_raised { @post.file(:original) } - assert_nothing_raised { @post.file(:preview) } - - assert_enqueued_jobs 3, only: DeletePostFilesJob - travel PostReplacement::DELETION_GRACE_PERIOD + 1.day - assert_raise(Post::DeletionError) { perform_enqueued_jobs } - - assert_nothing_raised { @post.file(:original) } - assert_nothing_raised { @post.file(:preview) } - rescue Net::OpenTimeout - skip "Remote connection to Pixiv failed" + as(@user) do + @post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350") + @post.reload + @post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364") + @post.reload + Upload.destroy_all + @post.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350") end + + assert_nothing_raised { @post.file(:original) } + assert_nothing_raised { @post.file(:preview) } + + assert_enqueued_jobs 3, only: DeletePostFilesJob + travel PostReplacement::DELETION_GRACE_PERIOD + 1.day + assert_raise(Post::DeletionError) { perform_enqueued_jobs } + + assert_nothing_raised { @post.file(:original) } + assert_nothing_raised { @post.file(:preview) } end end @@ -609,39 +588,35 @@ class UploadServiceTest < ActiveSupport::TestCase should "not delete the still active files" do # swap the images between @post1 and @post2. - begin - as(@user) do - skip unless MediaFile::Ugoira.videos_enabled? + as(@user) do + skip unless MediaFile::Ugoira.videos_enabled? - @post1.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350") - @post2.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364") - assert_equal("4ceadc314938bc27f3574053a3e1459a", @post1.md5) - assert_equal("cad1da177ef309bf40a117c17b8eecf5", @post2.md5) + @post1.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350") + @post2.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364") + assert_equal("4ceadc314938bc27f3574053a3e1459a", @post1.md5) + assert_equal("cad1da177ef309bf40a117c17b8eecf5", @post2.md5) - @post2.reload - @post2.replace!(replacement_url: "https://cdn.donmai.us/original/d3/4e/d34e4cf0a437a5d65f8e82b7bcd02606.jpg") - assert_equal("d34e4cf0a437a5d65f8e82b7bcd02606", @post2.md5) - Upload.destroy_all - @post1.reload - @post2.reload + @post2.reload + @post2.replace!(replacement_url: "https://cdn.donmai.us/original/d3/4e/d34e4cf0a437a5d65f8e82b7bcd02606.jpg") + assert_equal("d34e4cf0a437a5d65f8e82b7bcd02606", @post2.md5) + Upload.destroy_all + @post1.reload + @post2.reload - @post1.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364") - @post2.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350") - assert_equal("cad1da177ef309bf40a117c17b8eecf5", @post1.md5) - assert_equal("4ceadc314938bc27f3574053a3e1459a", @post2.md5) - end - - travel_to (PostReplacement::DELETION_GRACE_PERIOD + 1).days.from_now do - assert_raise(Post::DeletionError) do - perform_enqueued_jobs - end - end - - assert_nothing_raised { @post1.file(:original) } - assert_nothing_raised { @post2.file(:original) } - rescue Net::OpenTimeout - skip "Remote connection to Pixiv failed" + @post1.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364") + @post2.replace!(replacement_url: "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247350") + assert_equal("cad1da177ef309bf40a117c17b8eecf5", @post1.md5) + assert_equal("4ceadc314938bc27f3574053a3e1459a", @post2.md5) end + + travel_to (PostReplacement::DELETION_GRACE_PERIOD + 1).days.from_now do + assert_raise(Post::DeletionError) do + perform_enqueued_jobs + end + end + + assert_nothing_raised { @post1.file(:original) } + assert_nothing_raised { @post2.file(:original) } end end @@ -885,12 +860,8 @@ class UploadServiceTest < ActiveSupport::TestCase end should "record the canonical source" do - begin - post = subject.new({}).create_post_from_upload(@upload) - assert_equal(@source, post.source) - rescue Net::OpenTimeout - skip "network failure" - end + post = subject.new({}).create_post_from_upload(@upload) + assert_equal(@source, post.source) end end From 2b969646bc2586fdcc71ecaa3264c91e7332454d Mon Sep 17 00:00:00 2001 From: evazion Date: Tue, 23 Jun 2020 23:32:14 -0500 Subject: [PATCH 57/59] tests: add `respond_to_search` test helper. Add a `respond_to_search` test helper for concisely testing that a controller's index action correctly responds to a search. Usage: # Tests that `/tags.json?search[name]=touhou` returns the `touhou` tag. setup { @touhou = create(:tag, name: "touhou") } should respond_to_search(name: "touhou").with { @touhou } --- test/test_helper.rb | 2 ++ test/test_helpers/controller_helper.rb | 47 ++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 test/test_helpers/controller_helper.rb diff --git a/test/test_helper.rb b/test/test_helper.rb index 1d81bdb12..892046f7e 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -61,6 +61,8 @@ class ActiveSupport::TestCase end class ActionDispatch::IntegrationTest + extend ControllerHelper + register_encoder :xml, response_parser: ->(body) { Nokogiri.XML(body) } def method_authenticated(method_name, url, user, **options) diff --git a/test/test_helpers/controller_helper.rb b/test/test_helpers/controller_helper.rb new file mode 100644 index 000000000..716bfaa97 --- /dev/null +++ b/test/test_helpers/controller_helper.rb @@ -0,0 +1,47 @@ +module ControllerHelper + # A custom Shoulda matcher that tests that a controller's index endpoint + # responds to a search correctly. See https://thoughtbot.com/blog/shoulda-matchers. + # + # Usage: + # + # # Tests that `/tags.json?search[name]=touhou` returns the `touhou` tag. + # subject { TagsController } + # setup { @touhou = create(:tag, name: "touhou") } + # should respond_to_search(name: "touhou").with { @touhou } + # + def respond_to_search(search_params) + RespondToSearchMatcher.new(search_params) + end + + class RespondToSearchMatcher < Struct.new(:params) + def description + "should respond to a search for #{params}" + end + + def matches?(subject, &block) + search_params = { search: params } + expected_items = @test_case.instance_eval(&@expected) + + @test_case.instance_eval do + # calls e.g. "wiki_pages_path" if we're in WikiPagesControllerTest. + index_url = send("#{subject.controller_path}_path") + get index_url, as: :json, params: search_params + + expected_ids = Array(expected_items).map(&:id) + responded_ids = response.parsed_body.map { |item| item["id"] } + + assert_response :success + assert_equal(expected_ids, responded_ids) + end + end + + def with(&block) + @expected = block + self + end + + def in_context(test_case) + @test_case = test_case + end + end +end From 5f3c41416ea947a47b13c8029368aaf2b5cda2bc Mon Sep 17 00:00:00 2001 From: evazion Date: Tue, 23 Jun 2020 23:36:16 -0500 Subject: [PATCH 58/59] tests: add more wiki page controller search tests. --- test/functional/wiki_pages_controller_test.rb | 37 +++++++++++-------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/test/functional/wiki_pages_controller_test.rb b/test/functional/wiki_pages_controller_test.rb index 119483201..2ea1619c6 100644 --- a/test/functional/wiki_pages_controller_test.rb +++ b/test/functional/wiki_pages_controller_test.rb @@ -10,8 +10,11 @@ class WikiPagesControllerTest < ActionDispatch::IntegrationTest context "index action" do setup do as(@user) do - @wiki_page_abc = create(:wiki_page, :title => "abc") - @wiki_page_def = create(:wiki_page, :title => "def") + @tagme = create(:wiki_page, title: "tagme") + @deleted = create(:wiki_page, title: "deleted", is_deleted: true) + @vocaloid = create(:wiki_page, title: "vocaloid") + @miku = create(:wiki_page, title: "hatsune_miku", other_names: ["初音ミク"], body: "miku is a [[vocaloid]]") + create(:tag, name: "hatsune_miku", category: Tag.categories.character) end end @@ -20,22 +23,24 @@ class WikiPagesControllerTest < ActionDispatch::IntegrationTest assert_response :success end - should "list all wiki_pages (with search)" do - get wiki_pages_path, params: {:search => {:title => "abc"}} - assert_response :success - assert_select "tr td:first-child", text: "abc" - end - - should "list wiki_pages without tags with order=post_count" do - get wiki_pages_path, params: {:search => {:title => "abc", :order => "post_count"}} - assert_response :success - assert_select "tr td:first-child", text: "abc" - end - should "redirect the legacy title param to the show page" do - get wiki_pages_path(title: "abc") - assert_redirected_to wiki_pages_path(search: { title_normalize: "abc" }, redirect: true) + get wiki_pages_path(title: "tagme") + assert_redirected_to wiki_pages_path(search: { title_normalize: "tagme" }, redirect: true) end + + should respond_to_search(title: "tagme").with { @tagme } + should respond_to_search(title: "tagme", order: "post_count").with { @tagme } + should respond_to_search(title_normalize: "TAGME ").with { @tagme } + + should respond_to_search(tag: { category: Tag.categories.character }).with { @miku } + should respond_to_search(hide_deleted: "true").with { [@miku, @vocaloid, @tagme] } + should respond_to_search(linked_to: "vocaloid").with { @miku } + should respond_to_search(not_linked_to: "vocaloid").with { [@vocaloid, @deleted, @tagme] } + + should respond_to_search(other_names_match: "初音ミク").with { @miku } + should respond_to_search(other_names_match: "初*").with { @miku } + should respond_to_search(other_names_present: "true").with { @miku } + should respond_to_search(other_names_present: "false").with { [@vocaloid, @deleted, @tagme] } end context "search action" do From bb765f55d5ad89e1f87821147e7fac7b2fdc21ea Mon Sep 17 00:00:00 2001 From: evazion Date: Tue, 23 Jun 2020 23:37:56 -0500 Subject: [PATCH 59/59] Eliminate misc dead code. --- app/logical/current_user.rb | 9 --------- app/logical/tag_relationship_retirement_service.rb | 8 -------- app/models/moderation_report.rb | 2 +- 3 files changed, 1 insertion(+), 18 deletions(-) diff --git a/app/logical/current_user.rb b/app/logical/current_user.rb index 292bf4a72..4c8d80349 100644 --- a/app/logical/current_user.rb +++ b/app/logical/current_user.rb @@ -24,15 +24,6 @@ class CurrentUser scoped(user, &block) end - def self.as_system(&block) - if block_given? - scoped(::User.system, "127.0.0.1", &block) - else - self.user = User.system - self.ip_addr = "127.0.0.1" - end - end - def self.user RequestStore[:current_user] end diff --git a/app/logical/tag_relationship_retirement_service.rb b/app/logical/tag_relationship_retirement_service.rb index 0f11ee931..0aa79cc6b 100644 --- a/app/logical/tag_relationship_retirement_service.rb +++ b/app/logical/tag_relationship_retirement_service.rb @@ -11,14 +11,6 @@ module TagRelationshipRetirementService "This topic deals with tag relationships created two or more years ago that have not been used since. They will be retired. This topic will be updated as an automated system retires expired relationships." end - def dry_run - [TagAlias, TagImplication].each do |model| - each_candidate(model) do |rel| - puts "#{rel.relationship} #{rel.antecedent_name} -> #{rel.consequent_name} retired" - end - end - end - def forum_topic topic = ForumTopic.where(title: forum_topic_title).first if topic.nil? diff --git a/app/models/moderation_report.rb b/app/models/moderation_report.rb index 17083458b..b45a6d86c 100644 --- a/app/models/moderation_report.rb +++ b/app/models/moderation_report.rb @@ -34,7 +34,7 @@ class ModerationReport < ApplicationRecord def forum_topic topic = ForumTopic.find_by_title(forum_topic_title) if topic.nil? - CurrentUser.as_system do + CurrentUser.scoped(User.system) do topic = ForumTopic.create!(creator: User.system, title: forum_topic_title, category_id: 0, min_level: User::Levels::MODERATOR) forum_post = ForumPost.create!(creator: User.system, body: forum_topic_body, topic: topic) end