jaculabilis/danbooru
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Newgrounds: support video uploads

Browse Source
This commit is contained in:
nonamethanks
2022-11-09 15:01:28 +01:00
parent d7d35cb16f
commit 35bfcbc3bd
3 changed files with 76 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
app/logical/source/extractor/newgrounds.rb
View File

@@ -9,12 +9,16 @@ module Source
end
def image_urls
if parsed_url.image_url?
if parsed_url.full_image_url.present?
[parsed_url.full_image_url]
elsif parsed_url.image_url?
[url]
elsif video_data.present?
[video_data.dig("sources", "1080p", 0, "src")&.sub(".1080p.", ".")].compact
else
urls = []
urls += page&.css(".image img").to_a.map { |img| img["src"] }
urls += page&.css(".image img").to_a.pluck("src")
urls += page&.css("#author_comments img[data-user-image='1']").to_a.map { |img| img["data-smartload-src"] || img["src"] }
urls.compact
@@ -22,24 +26,12 @@ module Source
end
def page_url
return nil if illust_title.blank? || user_name.blank?
"https://www.newgrounds.com/art/view/#{user_name}/#{illust_title}"
parsed_url.page_url || parsed_referer&.page_url
end
def page
return nil if page_url.blank?
response = http.cookies(vmkIdu5l8m: Danbooru.config.newgrounds_session_cookie).cache(1.minute).get(page_url)
return nil if response.status == 404
response.parse
end
memoize :page
def tags
page&.css("#sidestats .tags a").to_a.map do |tag|
[tag.text, "https://www.newgrounds.com/search/conduct/art?match=tags&tags=" + tag.text]
[tag.text, "https://www.newgrounds.com/search/conduct/art?match=tags&tags=#{tag.text}"]
end
end
@@ -83,6 +75,38 @@ module Source
def illust_title
parsed_url.work_title || parsed_referer&.work_title
end
def video_id
parsed_url.video_id || parsed_referer&.video_id
end
def http
super.cookies(vmkIdu5l8m: Danbooru.config.newgrounds_session_cookie)
end
def page
return nil if page_url.blank?
response = http.cache(1.minute).get(page_url)
return nil if response.status == 404
response.parse
end
def video_data
# flash files return {"error"=>{"code"=>404, "msg"=>"The submission you are looking for does not have a video."}}
return {} unless video_id.present?
response = http.headers("X-Requested-With": "XMLHttpRequest").cache(1.minute).get("https://www.newgrounds.com/portal/video/#{video_id}")
return {} unless response.status == 200
JSON.parse(response).with_indifferent_access
rescue JSON::ParserError
{}
end
memoize :page, :video_data
end
end
end

31
app/logical/source/url/newgrounds.rb
View File

@@ -2,21 +2,11 @@
# Unsupported:
#
# Video URLs
#
# * https://www.newgrounds.com/portal/view/825916 (page)
# * https://picon.ngfiles.com/825000/flash_825916_card.png?f1639666239 (poster)
# * https://uploads.ungrounded.net/alternate/1801000/1801343_alternate_165104.1080p.mp4?1639666238
# * https://uploads.ungrounded.net/alternate/1801000/1801343_alternate_165104.720p.mp4?1639666238
# * https://uploads.ungrounded.net/alternate/1801000/1801343_alternate_165104.360p.mp4?1639666238
#
# Flash URLs
#
# * https://www.newgrounds.com/portal/view/225625 (page)
# * https://uploads.ungrounded.net/225000/225625_colormedressup.swf?1111143751 (file)
#
# * https://www.newgrounds.com/portal/view/225625 (flash page)
# * https://uploads.ungrounded.net/225000/225625_colormedressup.swf?1111143751 (flash file)
class Source::URL::Newgrounds < Source::URL
attr_reader :username, :work_id, :work_title, :video_id
attr_reader :username, :work_id, :work_title, :video_id, :full_image_url
def self.match?(url)
url.domain.in?(["newgrounds.com", "ngfiles.com", "ungrounded.net"])
@@ -36,20 +26,29 @@ class Source::URL::Newgrounds < Source::URL
in "www.newgrounds.com", "portal", ("view" | "video"), video_id
@video_id = video_id
# https://uploads.ungrounded.net/alternate/1801000/1801343_alternate_165104.1080p.mp4?1639666238
# https://uploads.ungrounded.net/alternate/1801000/1801343_alternate_165104.720p.mp4?1639666238
# https://uploads.ungrounded.net/alternate/1801000/1801343_alternate_165104.360p.mp4?1639666238
in "uploads.ungrounded.net", "alternate", /^\d+$/ => subdir, /^\d+_alternate_\d+(?:\.\d+p)?\.mp4/ => file
max_file = file.sub(/\.\d+p\./, ".")
@full_image_url = "https://uploads.ungrounded.net/alternate/#{subdir}/#{max_file}"
# https://art.ngfiles.com/images/1254000/1254722_natthelich_pandora.jpg
# https://art.ngfiles.com/images/1033000/1033622_natthelich_fire-emblem-marth-plus-progress-pic.png?f1569487181
in "art.ngfiles.com", "images", _, /^(\d+)_([^_]+)_(.*)\.\w+$/
@work_id = $1
@username = $2
@work_title = $3
@full_image_url = original_url
# https://art.ngfiles.com/thumbnails/1254000/1254985.png?f1588263349
in "art.ngfiles.com", "thumbnails", _, /^(\d+)\.\w+$/
@work_id = $1
@full_image_url = original_url
# https://art.ngfiles.com/comments/57000/iu_57615_7115981.jpg
in "art.ngfiles.com", "comments", _, /^iu/
nil
@full_image_url = original_url
# https://natthelich.newgrounds.com
# https://natthelich.newgrounds.com/art/
@@ -62,7 +61,7 @@ class Source::URL::Newgrounds < Source::URL
end
def image_url?
url.host == "art.ngfiles.com"
url.host.in? ["art.ngfiles.com", "uploads.ungrounded.net"]
end
def page_url

21
test/unit/sources/newgrounds_test.rb
View File

@@ -34,6 +34,26 @@ module Sources
)
end
context "A newgrounds video post" do
strategy_should_work(
"https://www.newgrounds.com/portal/view/536659",
image_urls: [%r{https://uploads\.ungrounded\.net/alternate/167000/167280_alternate_602\.mp4}],
profile_url: "https://jenjamik.newgrounds.com",
artist_name: "jenjamik",
page_url: "https://www.newgrounds.com/portal/view/536659",
artist_commentary_title: "Link's Barrel Beat",
dtext_artist_commentary_desc: /Long time no see!/
)
end
context "A newgrounds direct video url" do
strategy_should_work(
"https://uploads.ungrounded.net/alternate/1801000/1801343_alternate_165104.360p.mp4?1639666238",
image_urls: ["https://uploads.ungrounded.net/alternate/1801000/1801343_alternate_165104.mp4"],
download_size: 75_605_846
)
end
context "A multi-image post" do
strategy_should_work(
"https://www.newgrounds.com/art/view/natthelich/weaver",
@@ -75,6 +95,7 @@ module Sources
assert(Source::URL.image_url?("https://art.ngfiles.com/images/1254000/1254722_natthelich_pandora.jpg"))
assert(Source::URL.image_url?("https://art.ngfiles.com/comments/57000/iu_57615_7115981.jpg"))
assert(Source::URL.image_url?("https://art.ngfiles.com/thumbnails/1254000/1254985.png?f1588263349"))
assert(Source::URL.image_url?("https://uploads.ungrounded.net/alternate/1801000/1801343_alternate_165104.mp4?1639666238"))
assert(Source::URL.page_url?("https://www.newgrounds.com/art/view/puddbytes/costanza-at-bat"))
assert(Source::URL.page_url?("https://www.newgrounds.com/portal/view/830293"))