jaculabilis/danbooru
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

replacements: backfill images in parallel.

Browse Source
This commit is contained in:
evazion
2022-11-08 21:41:59 -06:00
parent 153d5f3e67
commit 83d14a281f
1 changed files with 7 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
script/fixes/127_download_old_replacements.rb
View File

@@ -4,8 +4,12 @@ require_relative "base"
FIX = ENV.fetch("FIX", "false").truthy? FIX = ENV.fetch("FIX", "false").truthy?
COND = ENV.fetch("COND", "TRUE") COND = ENV.fetch("COND", "TRUE")
WORKERS = ENV.fetch("WORKERS", 5).to_i
def download(replacement, md5) def download(replacement, md5)
# url = "https://b2.donmai.us/file/danbooru/original/#{md5[0..1]}/#{md5[2..3}/#{md5}.jpg"
# image_url = url if Danbooru::Http.head(url)&.status == 200
url = "https://gelbooru.com/index.php?page=post&s=list&md5=#{md5}" url = "https://gelbooru.com/index.php?page=post&s=list&md5=#{md5}"
image_url = Source::Extractor.find(url).image_urls.first image_url = Source::Extractor.find(url).image_urls.first
@@ -17,10 +21,10 @@ rescue Timeout::Error
puts ({ error: "upload timed out", replacement: replacement.id, upload: upload&.id, image_url:, }).to_json puts ({ error: "upload timed out", replacement: replacement.id, upload: upload&.id, image_url:, }).to_json
end end
PostReplacement.where(COND).find_each do |replacement| PostReplacement.where(COND).parallel_each(in_processes: WORKERS) do |replacement|
old_media_asset = MediaAsset.active.find_by(md5: replacement.old_md5) old_media_asset = MediaAsset.active.find_by(md5: replacement.old_md5)
new_media_asset = MediaAsset.active.find_by(md5: replacement.md5) new_media_asset = MediaAsset.active.find_by(md5: replacement.md5)
download(replacement, replacement.old_md5) if old_media_asset.nil? download(replacement, replacement.old_md5) if old_media_asset.nil? && replacement.old_md5.present?
download(replacement, replacement.md5) if new_media_asset.nil? download(replacement, replacement.md5) if new_media_asset.nil? && replacement.md5.present?
end end