foundation.app: fix scraping of image urls.

Foundation changed their HTML page format and we can no longer scrape
the image URL directly from the page. Instead we have to build it based
on API data.
This commit is contained in:
evazion
2022-08-23 21:26:47 -05:00
parent c8917684b8
commit 2c36e02810
3 changed files with 30 additions and 3 deletions

View File

@@ -11,8 +11,24 @@ module Source
def image_urls def image_urls
if parsed_url.full_image_url.present? if parsed_url.full_image_url.present?
[parsed_url.full_image_url] [parsed_url.full_image_url]
elsif (image_url = page&.at(".fullscreen img, .fullscreen video")&.attr(:src)) elsif api_response.dig("props", "pageProps", "artwork").present?
[Source::URL.parse(image_url).full_image_url].compact artwork = api_response.dig("props", "pageProps", "artwork")
asset_id = artwork["assetId"]
# Reverse engineered from the Foundation.app Javascript; look for buildVideoUrl in utils/assets.ts.
if artwork["mimeType"].starts_with?("video/")
if artwork["assetVersion"] == 5
url = "#{artwork["assetScheme"]}#{artwork["assetHost"]}#{artwork["assetPath"]}/nft.mp4"
elsif artwork["assetVersion"] == 3
url = "https://assets.foundation.app/#{asset_id[-4..-3]}/#{asset_id[-2..-1]}/#{asset_id}/nft_q4.mp4"
else
url = "https://assets.foundation.app/#{asset_id[-4..-3]}/#{asset_id[-2..-1]}/#{asset_id}/nft.mp4"
end
else
url = "#{artwork["assetScheme"]}#{artwork["assetHost"]}/#{artwork["assetPath"]}"
end
[Source::URL.parse(url).full_image_url].compact
else else
[] []
end end

View File

@@ -96,7 +96,7 @@ module SourceTestHelper
actual_values = actual_value.sort actual_values = actual_value.sort
expected_value.sort.each_with_index { |each_value, index| assert_match(each_value, actual_values[index]) } expected_value.sort.each_with_index { |each_value, index| assert_match(each_value, actual_values[index]) }
else else
assert_equal(expected_value.sort, actual_value) assert_equal(expected_value.sort, actual_value.sort)
end end
elsif expected_value.nil? elsif expected_value.nil?
assert_nil(actual_value) assert_nil(actual_value)

View File

@@ -23,6 +23,17 @@ module Sources
) )
end end
context "A foundation gif" do
strategy_should_work(
"https://foundation.app/@patch_oxxo/shine/1",
artist_name: "patch_oxxo",
profile_url: "https://foundation.app/@patch_oxxo",
image_urls: ["https://f8n-ipfs-production.imgix.net/QmWQpt9opcue5F7Q2vTT5P5jPYo1xHhEs7RDxgXzWFHcfK/nft.gif"],
download_size: 52_352_138,
tags: %w[2d animation],
)
end
context "A foundation.app/@username/foo-bar-1234 URL" do context "A foundation.app/@username/foo-bar-1234 URL" do
strategy_should_work( strategy_should_work(
"https://foundation.app/@asuka111art/dinner-with-cats-82426", "https://foundation.app/@asuka111art/dinner-with-cats-82426",