From 35bfcbc3bd56cccec80901a985e13943e1dd7f63 Mon Sep 17 00:00:00 2001 From: nonamethanks Date: Wed, 9 Nov 2022 15:01:28 +0100 Subject: [PATCH] Newgrounds: support video uploads --- app/logical/source/extractor/newgrounds.rb | 56 +++++++++++++++------- app/logical/source/url/newgrounds.rb | 31 ++++++------ test/unit/sources/newgrounds_test.rb | 21 ++++++++ 3 files changed, 76 insertions(+), 32 deletions(-) diff --git a/app/logical/source/extractor/newgrounds.rb b/app/logical/source/extractor/newgrounds.rb index 070db1735..1eb4764e3 100644 --- a/app/logical/source/extractor/newgrounds.rb +++ b/app/logical/source/extractor/newgrounds.rb @@ -9,12 +9,16 @@ module Source end def image_urls - if parsed_url.image_url? + if parsed_url.full_image_url.present? + [parsed_url.full_image_url] + elsif parsed_url.image_url? [url] + elsif video_data.present? + [video_data.dig("sources", "1080p", 0, "src")&.sub(".1080p.", ".")].compact else urls = [] - urls += page&.css(".image img").to_a.map { |img| img["src"] } + urls += page&.css(".image img").to_a.pluck("src") urls += page&.css("#author_comments img[data-user-image='1']").to_a.map { |img| img["data-smartload-src"] || img["src"] } urls.compact @@ -22,24 +26,12 @@ module Source end def page_url - return nil if illust_title.blank? || user_name.blank? - - "https://www.newgrounds.com/art/view/#{user_name}/#{illust_title}" + parsed_url.page_url || parsed_referer&.page_url end - def page - return nil if page_url.blank? - - response = http.cookies(vmkIdu5l8m: Danbooru.config.newgrounds_session_cookie).cache(1.minute).get(page_url) - return nil if response.status == 404 - - response.parse - end - memoize :page - def tags page&.css("#sidestats .tags a").to_a.map do |tag| - [tag.text, "https://www.newgrounds.com/search/conduct/art?match=tags&tags=" + tag.text] + [tag.text, "https://www.newgrounds.com/search/conduct/art?match=tags&tags=#{tag.text}"] end end @@ -83,6 +75,38 @@ module Source def illust_title parsed_url.work_title || parsed_referer&.work_title end + + def video_id + parsed_url.video_id || parsed_referer&.video_id + end + + def http + super.cookies(vmkIdu5l8m: Danbooru.config.newgrounds_session_cookie) + end + + def page + return nil if page_url.blank? + + response = http.cache(1.minute).get(page_url) + return nil if response.status == 404 + + response.parse + end + + def video_data + # flash files return {"error"=>{"code"=>404, "msg"=>"The submission you are looking for does not have a video."}} + + return {} unless video_id.present? + + response = http.headers("X-Requested-With": "XMLHttpRequest").cache(1.minute).get("https://www.newgrounds.com/portal/video/#{video_id}") + return {} unless response.status == 200 + + JSON.parse(response).with_indifferent_access + rescue JSON::ParserError + {} + end + + memoize :page, :video_data end end end diff --git a/app/logical/source/url/newgrounds.rb b/app/logical/source/url/newgrounds.rb index 3ff4e04b3..eb2209121 100644 --- a/app/logical/source/url/newgrounds.rb +++ b/app/logical/source/url/newgrounds.rb @@ -2,21 +2,11 @@ # Unsupported: # -# Video URLs -# -# * https://www.newgrounds.com/portal/view/825916 (page) -# * https://picon.ngfiles.com/825000/flash_825916_card.png?f1639666239 (poster) -# * https://uploads.ungrounded.net/alternate/1801000/1801343_alternate_165104.1080p.mp4?1639666238 -# * https://uploads.ungrounded.net/alternate/1801000/1801343_alternate_165104.720p.mp4?1639666238 -# * https://uploads.ungrounded.net/alternate/1801000/1801343_alternate_165104.360p.mp4?1639666238 -# -# Flash URLs -# -# * https://www.newgrounds.com/portal/view/225625 (page) -# * https://uploads.ungrounded.net/225000/225625_colormedressup.swf?1111143751 (file) -# +# * https://www.newgrounds.com/portal/view/225625 (flash page) +# * https://uploads.ungrounded.net/225000/225625_colormedressup.swf?1111143751 (flash file) + class Source::URL::Newgrounds < Source::URL - attr_reader :username, :work_id, :work_title, :video_id + attr_reader :username, :work_id, :work_title, :video_id, :full_image_url def self.match?(url) url.domain.in?(["newgrounds.com", "ngfiles.com", "ungrounded.net"]) @@ -36,20 +26,29 @@ class Source::URL::Newgrounds < Source::URL in "www.newgrounds.com", "portal", ("view" | "video"), video_id @video_id = video_id + # https://uploads.ungrounded.net/alternate/1801000/1801343_alternate_165104.1080p.mp4?1639666238 + # https://uploads.ungrounded.net/alternate/1801000/1801343_alternate_165104.720p.mp4?1639666238 + # https://uploads.ungrounded.net/alternate/1801000/1801343_alternate_165104.360p.mp4?1639666238 + in "uploads.ungrounded.net", "alternate", /^\d+$/ => subdir, /^\d+_alternate_\d+(?:\.\d+p)?\.mp4/ => file + max_file = file.sub(/\.\d+p\./, ".") + @full_image_url = "https://uploads.ungrounded.net/alternate/#{subdir}/#{max_file}" + # https://art.ngfiles.com/images/1254000/1254722_natthelich_pandora.jpg # https://art.ngfiles.com/images/1033000/1033622_natthelich_fire-emblem-marth-plus-progress-pic.png?f1569487181 in "art.ngfiles.com", "images", _, /^(\d+)_([^_]+)_(.*)\.\w+$/ @work_id = $1 @username = $2 @work_title = $3 + @full_image_url = original_url # https://art.ngfiles.com/thumbnails/1254000/1254985.png?f1588263349 in "art.ngfiles.com", "thumbnails", _, /^(\d+)\.\w+$/ @work_id = $1 + @full_image_url = original_url # https://art.ngfiles.com/comments/57000/iu_57615_7115981.jpg in "art.ngfiles.com", "comments", _, /^iu/ - nil + @full_image_url = original_url # https://natthelich.newgrounds.com # https://natthelich.newgrounds.com/art/ @@ -62,7 +61,7 @@ class Source::URL::Newgrounds < Source::URL end def image_url? - url.host == "art.ngfiles.com" + url.host.in? ["art.ngfiles.com", "uploads.ungrounded.net"] end def page_url diff --git a/test/unit/sources/newgrounds_test.rb b/test/unit/sources/newgrounds_test.rb index 05535edb0..b8b5c0a38 100644 --- a/test/unit/sources/newgrounds_test.rb +++ b/test/unit/sources/newgrounds_test.rb @@ -34,6 +34,26 @@ module Sources ) end + context "A newgrounds video post" do + strategy_should_work( + "https://www.newgrounds.com/portal/view/536659", + image_urls: [%r{https://uploads\.ungrounded\.net/alternate/167000/167280_alternate_602\.mp4}], + profile_url: "https://jenjamik.newgrounds.com", + artist_name: "jenjamik", + page_url: "https://www.newgrounds.com/portal/view/536659", + artist_commentary_title: "Link's Barrel Beat", + dtext_artist_commentary_desc: /Long time no see!/ + ) + end + + context "A newgrounds direct video url" do + strategy_should_work( + "https://uploads.ungrounded.net/alternate/1801000/1801343_alternate_165104.360p.mp4?1639666238", + image_urls: ["https://uploads.ungrounded.net/alternate/1801000/1801343_alternate_165104.mp4"], + download_size: 75_605_846 + ) + end + context "A multi-image post" do strategy_should_work( "https://www.newgrounds.com/art/view/natthelich/weaver", @@ -75,6 +95,7 @@ module Sources assert(Source::URL.image_url?("https://art.ngfiles.com/images/1254000/1254722_natthelich_pandora.jpg")) assert(Source::URL.image_url?("https://art.ngfiles.com/comments/57000/iu_57615_7115981.jpg")) assert(Source::URL.image_url?("https://art.ngfiles.com/thumbnails/1254000/1254985.png?f1588263349")) + assert(Source::URL.image_url?("https://uploads.ungrounded.net/alternate/1801000/1801343_alternate_165104.mp4?1639666238")) assert(Source::URL.page_url?("https://www.newgrounds.com/art/view/puddbytes/costanza-at-bat")) assert(Source::URL.page_url?("https://www.newgrounds.com/portal/view/830293"))