foundation.app: fix scraping of image urls.

Foundation changed their HTML page format and we can no longer scrape
the image URL directly from the page. Instead we have to build it based
on API data.
This commit is contained in:
evazion
2022-08-23 21:26:47 -05:00
parent c8917684b8
commit 2c36e02810
3 changed files with 30 additions and 3 deletions

View File

@@ -11,8 +11,24 @@ module Source
def image_urls
if parsed_url.full_image_url.present?
[parsed_url.full_image_url]
elsif (image_url = page&.at(".fullscreen img, .fullscreen video")&.attr(:src))
[Source::URL.parse(image_url).full_image_url].compact
elsif api_response.dig("props", "pageProps", "artwork").present?
artwork = api_response.dig("props", "pageProps", "artwork")
asset_id = artwork["assetId"]
# Reverse engineered from the Foundation.app Javascript; look for buildVideoUrl in utils/assets.ts.
if artwork["mimeType"].starts_with?("video/")
if artwork["assetVersion"] == 5
url = "#{artwork["assetScheme"]}#{artwork["assetHost"]}#{artwork["assetPath"]}/nft.mp4"
elsif artwork["assetVersion"] == 3
url = "https://assets.foundation.app/#{asset_id[-4..-3]}/#{asset_id[-2..-1]}/#{asset_id}/nft_q4.mp4"
else
url = "https://assets.foundation.app/#{asset_id[-4..-3]}/#{asset_id[-2..-1]}/#{asset_id}/nft.mp4"
end
else
url = "#{artwork["assetScheme"]}#{artwork["assetHost"]}/#{artwork["assetPath"]}"
end
[Source::URL.parse(url).full_image_url].compact
else
[]
end

View File

@@ -96,7 +96,7 @@ module SourceTestHelper
actual_values = actual_value.sort
expected_value.sort.each_with_index { |each_value, index| assert_match(each_value, actual_values[index]) }
else
assert_equal(expected_value.sort, actual_value)
assert_equal(expected_value.sort, actual_value.sort)
end
elsif expected_value.nil?
assert_nil(actual_value)

View File

@@ -23,6 +23,17 @@ module Sources
)
end
context "A foundation gif" do
strategy_should_work(
"https://foundation.app/@patch_oxxo/shine/1",
artist_name: "patch_oxxo",
profile_url: "https://foundation.app/@patch_oxxo",
image_urls: ["https://f8n-ipfs-production.imgix.net/QmWQpt9opcue5F7Q2vTT5P5jPYo1xHhEs7RDxgXzWFHcfK/nft.gif"],
download_size: 52_352_138,
tags: %w[2d animation],
)
end
context "A foundation.app/@username/foo-bar-1234 URL" do
strategy_should_work(
"https://foundation.app/@asuka111art/dinner-with-cats-82426",