Newgrounds: support video uploads

app/logical/source/extractor/newgrounds.rb

@@ -9,12 +9,16 @@ module Source
       end
 
       def image_urls
-        if parsed_url.image_url?
+        if parsed_url.full_image_url.present?
+          [parsed_url.full_image_url]
+        elsif parsed_url.image_url?
           [url]
+        elsif video_data.present?
+          [video_data.dig("sources", "1080p", 0, "src")&.sub(".1080p.", ".")].compact
         else
           urls = []
 
-          urls += page&.css(".image img").to_a.map { |img| img["src"] }
+          urls += page&.css(".image img").to_a.pluck("src")
           urls += page&.css("#author_comments img[data-user-image='1']").to_a.map { |img| img["data-smartload-src"] || img["src"] }
 
           urls.compact
@@ -22,24 +26,12 @@ module Source
       end
 
       def page_url
-        return nil if illust_title.blank? || user_name.blank?
+        parsed_url.page_url || parsed_referer&.page_url
-
-        "https://www.newgrounds.com/art/view/#{user_name}/#{illust_title}"
       end
 
-      def page
-        return nil if page_url.blank?
-
-        response = http.cookies(vmkIdu5l8m: Danbooru.config.newgrounds_session_cookie).cache(1.minute).get(page_url)
-        return nil if response.status == 404
-
-        response.parse
-      end
-      memoize :page
-
       def tags
         page&.css("#sidestats .tags a").to_a.map do |tag|
-          [tag.text, "https://www.newgrounds.com/search/conduct/art?match=tags&tags=" + tag.text]
+          [tag.text, "https://www.newgrounds.com/search/conduct/art?match=tags&tags=#{tag.text}"]
         end
       end
 
@@ -83,6 +75,38 @@ module Source
       def illust_title
         parsed_url.work_title || parsed_referer&.work_title
       end
+
+      def video_id
+        parsed_url.video_id || parsed_referer&.video_id
+      end
+
+      def http
+        super.cookies(vmkIdu5l8m: Danbooru.config.newgrounds_session_cookie)
+      end
+
+      def page
+        return nil if page_url.blank?
+
+        response = http.cache(1.minute).get(page_url)
+        return nil if response.status == 404
+
+        response.parse
+      end
+
+      def video_data
+        # flash files return {"error"=>{"code"=>404, "msg"=>"The submission you are looking for does not have a video."}}
+
+        return {} unless video_id.present?
+
+        response = http.headers("X-Requested-With": "XMLHttpRequest").cache(1.minute).get("https://www.newgrounds.com/portal/video/#{video_id}")
+        return {} unless response.status == 200
+
+        JSON.parse(response).with_indifferent_access
+      rescue JSON::ParserError
+        {}
+      end
+
+      memoize :page, :video_data
     end
   end
 end

app/logical/source/url/newgrounds.rb

@@ -2,21 +2,11 @@
 
 # Unsupported:
 #
-# Video URLs
+# * https://www.newgrounds.com/portal/view/225625 (flash page)
-#
+# * https://uploads.ungrounded.net/225000/225625_colormedressup.swf?1111143751 (flash file)
-# * https://www.newgrounds.com/portal/view/825916 (page)
+
-# * https://picon.ngfiles.com/825000/flash_825916_card.png?f1639666239 (poster)
-# * https://uploads.ungrounded.net/alternate/1801000/1801343_alternate_165104.1080p.mp4?1639666238
-# * https://uploads.ungrounded.net/alternate/1801000/1801343_alternate_165104.720p.mp4?1639666238
-# * https://uploads.ungrounded.net/alternate/1801000/1801343_alternate_165104.360p.mp4?1639666238
-#
-# Flash URLs
-#
-# * https://www.newgrounds.com/portal/view/225625 (page)
-# * https://uploads.ungrounded.net/225000/225625_colormedressup.swf?1111143751 (file)
-#
 class Source::URL::Newgrounds < Source::URL
-  attr_reader :username, :work_id, :work_title, :video_id
+  attr_reader :username, :work_id, :work_title, :video_id, :full_image_url
 
   def self.match?(url)
     url.domain.in?(["newgrounds.com", "ngfiles.com", "ungrounded.net"])
@@ -36,20 +26,29 @@ class Source::URL::Newgrounds < Source::URL
     in "www.newgrounds.com", "portal", ("view" | "video"), video_id
       @video_id = video_id
 
+    # https://uploads.ungrounded.net/alternate/1801000/1801343_alternate_165104.1080p.mp4?1639666238
+    # https://uploads.ungrounded.net/alternate/1801000/1801343_alternate_165104.720p.mp4?1639666238
+    # https://uploads.ungrounded.net/alternate/1801000/1801343_alternate_165104.360p.mp4?1639666238
+    in "uploads.ungrounded.net", "alternate", /^\d+$/ => subdir, /^\d+_alternate_\d+(?:\.\d+p)?\.mp4/ => file
+      max_file = file.sub(/\.\d+p\./, ".")
+      @full_image_url = "https://uploads.ungrounded.net/alternate/#{subdir}/#{max_file}"
+
     # https://art.ngfiles.com/images/1254000/1254722_natthelich_pandora.jpg
     # https://art.ngfiles.com/images/1033000/1033622_natthelich_fire-emblem-marth-plus-progress-pic.png?f1569487181
     in "art.ngfiles.com", "images", _, /^(\d+)_([^_]+)_(.*)\.\w+$/
       @work_id = $1
       @username = $2
       @work_title = $3
+      @full_image_url = original_url
 
     # https://art.ngfiles.com/thumbnails/1254000/1254985.png?f1588263349
     in "art.ngfiles.com", "thumbnails", _, /^(\d+)\.\w+$/
       @work_id = $1
+      @full_image_url = original_url
 
     # https://art.ngfiles.com/comments/57000/iu_57615_7115981.jpg
     in "art.ngfiles.com", "comments", _, /^iu/
-      nil
+      @full_image_url = original_url
 
     # https://natthelich.newgrounds.com
     # https://natthelich.newgrounds.com/art/
@@ -62,7 +61,7 @@ class Source::URL::Newgrounds < Source::URL
   end
 
   def image_url?
-    url.host == "art.ngfiles.com"
+    url.host.in? ["art.ngfiles.com", "uploads.ungrounded.net"]
   end
 
   def page_url

test/unit/sources/newgrounds_test.rb

@@ -34,6 +34,26 @@ module Sources
       )
     end
 
+    context "A newgrounds video post" do
+      strategy_should_work(
+        "https://www.newgrounds.com/portal/view/536659",
+        image_urls: [%r{https://uploads\.ungrounded\.net/alternate/167000/167280_alternate_602\.mp4}],
+        profile_url: "https://jenjamik.newgrounds.com",
+        artist_name: "jenjamik",
+        page_url: "https://www.newgrounds.com/portal/view/536659",
+        artist_commentary_title: "Link's Barrel Beat",
+        dtext_artist_commentary_desc: /Long time no see!/
+      )
+    end
+
+    context "A newgrounds direct video url" do
+      strategy_should_work(
+        "https://uploads.ungrounded.net/alternate/1801000/1801343_alternate_165104.360p.mp4?1639666238",
+        image_urls: ["https://uploads.ungrounded.net/alternate/1801000/1801343_alternate_165104.mp4"],
+        download_size: 75_605_846
+      )
+    end
+
     context "A multi-image post" do
       strategy_should_work(
         "https://www.newgrounds.com/art/view/natthelich/weaver",
@@ -75,6 +95,7 @@ module Sources
       assert(Source::URL.image_url?("https://art.ngfiles.com/images/1254000/1254722_natthelich_pandora.jpg"))
       assert(Source::URL.image_url?("https://art.ngfiles.com/comments/57000/iu_57615_7115981.jpg"))
       assert(Source::URL.image_url?("https://art.ngfiles.com/thumbnails/1254000/1254985.png?f1588263349"))
+      assert(Source::URL.image_url?("https://uploads.ungrounded.net/alternate/1801000/1801343_alternate_165104.mp4?1639666238"))
 
       assert(Source::URL.page_url?("https://www.newgrounds.com/art/view/puddbytes/costanza-at-bat"))
       assert(Source::URL.page_url?("https://www.newgrounds.com/portal/view/830293"))