jaculabilis/danbooru
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

include whitecube source/rewrite strategies

Browse Source
This commit is contained in:
r888888888
2016-09-23 15:29:49 -07:00
parent 34d1f95cbf
commit 4fa2741ed2
9 changed files with 234 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
app/logical/downloads/rewrite_strategies/pixiv.rb
View File

@@ -53,10 +53,11 @@ module Downloads
# http://www.pixiv.net/member_illust.php?mode=big&illust_id=18557054
# http://www.pixiv.net/member_illust.php?mode=manga&illust_id=18557054
# http://www.pixiv.net/member_illust.php?mode=manga_big&illust_id=18557054&page=1
# http://www.pixiv.net/whitecube/user/277898/illust/59182257
# Plus this:
# i2.pixiv.net/img-inf/img/2014/09/25/00/57/24/46170939_64x64.jpg
def rewrite_html_pages(url, headers)
if url =~ /illust_id=\d+/i || url =~ %r!pixiv\.net/img-inf/img/!i
if url =~ /illust_id=\d+/i || url =~ %r!pixiv\.net/img-inf/img/!i || url =~ /illust\/\d+/
return [source.file_url, headers]
else
return [url, headers]

5
app/logical/pixiv_api_client.rb
View File

@@ -6,7 +6,7 @@ class PixivApiClient
class Error < Exception ; end
class WorksResponse
attr_reader :json, :pages, :moniker, :page_count
attr_reader :json, :pages, :name, :moniker, :user_id, :page_count, :tags
attr_reader :artist_commentary_title, :artist_commentary_desc
def initialize(json)
@@ -90,10 +90,13 @@ class PixivApiClient
# }
@json = json
@name = json["user"]["name"]
@user_id = json["user"]["id"]
@moniker = json["user"]["account"]
@page_count = json["page_count"].to_i
@artist_commentary_title = json["title"]
@artist_commentary_desc = json["caption"]
@tags = json["tags"]
if page_count > 1
@pages = json["metadata"]["pages"].map {|x| x["image_urls"]["large"]}

2
app/logical/pixiv_ugoira_converter.rb
View File

@@ -31,7 +31,7 @@ class PixivUgoiraConverter
f.write("# timecode format v2\n")
frame_data.each do |img|
f.write("#{delay_sum}\n")
delay_sum += img["delay"]
delay_sum += (img["delay"] || img["delay_msec"])
end
f.write("#{delay_sum}\n")
f.write("#{delay_sum}\n")

2
app/logical/sources/site.rb
View File

@@ -10,7 +10,7 @@ module Sources
:artist_commentary_desc, :to => :strategy
def self.strategies
[Strategies::Pixiv, Strategies::NicoSeiga, Strategies::DeviantArt, Strategies::Nijie, Strategies::Twitter, Strategies::Tumblr]
[Strategies::PixivWhitecube, Strategies::Pixiv, Strategies::NicoSeiga, Strategies::DeviantArt, Strategies::Nijie, Strategies::Twitter, Strategies::Tumblr]
end
def initialize(url, options = {})

3
app/logical/sources/strategies/pixiv.rb
View File

@@ -11,12 +11,13 @@ module Sources
TIMESTAMP = '(?:[0-9]{4}/[0-9]{2}/[0-9]{2}/[0-9]{2}/[0-9]{2}/[0-9]{2})'
EXT = "(?:jpg|jpeg|png|gif)"
WHITECUBE = /^https?:\/\/wwww\.pixiv\.net\/whitecube/i
WEB = "^(?:https?://)?www\\.pixiv\\.net"
I12 = "^(?:https?://)?i[0-9]+\\.pixiv\\.net"
IMG = "^(?:https?://)?img[0-9]*\\.pixiv\\.net"
def self.url_match?(url)
url =~ /#{WEB}|#{IMG}|#{I12}/i
url !~ WHITECUBE && url =~ /#{WEB}|#{IMG}|#{I12}/i
end
def referer_url

83
app/logical/sources/strategies/pixiv_whitecube.rb Normal file
View File

@@ -0,0 +1,83 @@
module Sources
module Strategies
class PixivWhitecube < Base
# sample: http://www.pixiv.net/whitecube/user/277898/illust/59182257
WHITECUBE_ILLUST = %r!^https?://www\.pixiv\.net/whitecube/user/\d+/illust/\d+!i
def self.url_match?(url)
url =~ WHITECUBE_ILLUST
end
def referer_url
@url
end
def site_name
"Pixiv Whitecube"
end
def unique_id
@pixiv_moniker
end
def fake_referer
"http://www.pixiv.net"
end
def has_artist_commentary?
@artist_commentary_desc.present?
end
def normalizable_for_artist_finder?
true
end
def normalize_for_artist_finder!
"http://img.pixiv.net/img/#{@moniker}/"
end
def get
@illust_id = illust_id_from_url
@data = query_pixiv_api(@illust_id)
@artist_name = @data.name
@profile_url = "https://www.pixiv.net/whitecube/user/" + @data.user_id.to_s
@pixiv_moniker = @data.moniker
@zip_url, @ugoira_frame_data = get_zip_url_from_api(@data)
@tags = @data.tags
@page_count = @data.page_count
@artist_commentary_title = @data.artist_commentary_title
@artist_commentary_desc = @data.artist_commentary_desc
is_manga = @page_count > 1
if !@zip_url
@image_url = @data.pages.first
end
end
def illust_id_from_url
# http://www.pixiv.net/whitecube/user/277898/illust/59182257
if url =~ %r!/whitecube/user/\d+/illust/(\d+)!
$1
else
raise Sources::Error.new("Couldn't get illust ID from URL: #{url}")
end
end
def query_pixiv_api(illust_id)
@data ||= PixivApiClient.new.works(illust_id)
end
def get_zip_url_from_api(data)
if data.json["metadata"] && data.json["metadata"]["zip_urls"]
zip_url = data.json["metadata"]["zip_urls"]["ugoira600x600"].sub("_ugoira600x600.zip", "_ugoira1920x1080.zip")
frame_data = data.json["metadata"]["frames"]
return [zip_url, frame_data]
end
end
end
end
end

8
config/danbooru_default_config.rb
View File

@@ -267,6 +267,14 @@ module Danbooru
nil
end
def pixiv_whitecube_login
nil
end
def pixiv_whitecube_password
nil
end
def tinami_login
nil
end

117
test/fixtures/vcr_cassettes/pixiv-whitecube-ilust.yml vendored Normal file
View File

@@ -0,0 +1,117 @@
---
http_interactions:
- request:
method: post
uri: https://oauth.secure.pixiv.net/auth/token
body:
encoding: US-ASCII
string: username=SENSITIVE&password=SENSITIVE&grant_type=password&client_id=bYGKuGVw91e0NMfPGp44euvGt59s&client_secret=HP3RmkgAmEGro0gn1x9ioawQE8WMfvLXDz3ZqxpK
headers:
Referer:
- http://www.pixiv.net
Accept-Encoding:
- gzip;q=1.0,deflate;q=0.6,identity;q=0.3
Accept:
- "*/*"
User-Agent:
- Ruby
response:
status:
code: 200
message: OK
headers:
Server:
- nginx
Date:
- Tue, 27 Sep 2016 22:47:51 GMT
Content-Type:
- application/json
Content-Length:
- '352'
Connection:
- keep-alive
Set-Cookie:
- PHPSESSID=14613426_55e806b338f5a162453aee474b0d653f; expires=Tue, 27-Sep-2016
23:47:51 GMT; Max-Age=3600; path=/; domain=.pixiv.net; secure
Expires:
- Thu, 19 Nov 1981 08:52:00 GMT
Cache-Control:
- no-store, no-cache, must-revalidate, post-check=0, pre-check=0
Pragma:
- no-cache
X-Content-Type-Options:
- nosniff
Vary:
- Accept-Encoding
body:
encoding: UTF-8
string: '{"response":{"access_token":"Q6cRks2WRIlQI88HSW17xc1QdGvC-lp2rCLjtZT96VI","expires_in":3600,"token_type":"bearer","scope":"unlimited","refresh_token":"-SGkey31OzOEWAj8B2lrCPlaFkAEja95Ie5Cj6-DSVM","user":{"profile_image_urls":{"px_16x16":"http:\/\/source.pixiv.net\/common\/images\/no_profile_ss.png","px_50x50":"http:\/\/source.pixiv.net\/common\/images\/no_profile_s.png","px_170x170":"http:\/\/source.pixiv.net\/common\/images\/no_profile.png"},"id":"14613426","name":"SENSITIVE","account":"SENSITIVE","is_premium":false,"x_restrict":2,"is_mail_authorized":true}}}'
http_version:
recorded_at: Tue, 27 Sep 2016 22:47:51 GMT
- request:
method: get
uri: https://public-api.secure.pixiv.net/v1/works/59182257.json?image_sizes=large&include_stats=true
body:
encoding: US-ASCII
string: ''
headers:
Referer:
- http://www.pixiv.net
User-Agent:
- dior/2.105.0
Content-Type:
- application/x-www-form-urlencoded
Authorization:
- Bearer Q6cRks2WRIlQI88HSW17xc1QdGvC-lp2rCLjtZT96VI
Accept-Encoding:
- gzip;q=1.0,deflate;q=0.6,identity;q=0.3
Accept:
- "*/*"
response:
status:
code: 200
message: OK
headers:
Server:
- nginx
Date:
- Tue, 27 Sep 2016 22:47:52 GMT
Content-Type:
- application/json
Content-Length:
- '622'
Connection:
- keep-alive
X-Content-Type-Options:
- nosniff
Vary:
- Accept-Encoding
body:
encoding: ASCII-8BIT
string: !binary |-
eyJzdGF0dXMiOiJzdWNjZXNzIiwicmVzcG9uc2UiOlt7ImlkIjo1OTE4MjI1
NywidGl0bGUiOiLvvLPvvKTntKsiLCJjYXB0aW9uIjoi57WQ55WM44OB44O8
44OgIiwidGFncyI6WyLmnbHmlrkiLCLlhavpm7LntKsiXSwidG9vbHMiOlsi
Q0xJUCBTVFVESU8gUEFJTlQiLCJBelBhaW50ZXIyIiwiQXpEcmF3aW5nMiJd
LCJpbWFnZV91cmxzIjp7ImxhcmdlIjoiaHR0cDovL2kyLnBpeGl2Lm5ldC9p
bWctb3JpZ2luYWwvaW1nLzIwMTYvMDkvMjYvMjEvMzAvNDEvNTkxODIyNTdf
cDAuanBnIn0sIndpZHRoIjo4MDAsImhlaWdodCI6ODAwLCJzdGF0cyI6eyJz
Y29yZWRfY291bnQiOjQwLCJzY29yZSI6Mzk5LCJ2aWV3c19jb3VudCI6Mzk2
LCJmYXZvcml0ZWRfY291bnQiOnsicHVibGljIjoxOSwicHJpdmF0ZSI6Mn0s
ImNvbW1lbnRlZF9jb3VudCI6NX0sInB1YmxpY2l0eSI6MCwiYWdlX2xpbWl0
IjoiYWxsLWFnZSIsImNyZWF0ZWRfdGltZSI6IjIwMTYtMDktMjYgMjE6MzA6
NDEiLCJyZXVwbG9hZGVkX3RpbWUiOiIyMDE2LTA5LTI2IDIxOjMwOjQxIiwi
dXNlciI6eyJpZCI6Mjc3ODk4LCJhY2NvdW50IjoibW9ub2NhdGllbnVzIiwi
bmFtZSI6ImNhdG8iLCJpc19mb2xsb3dpbmciOmZhbHNlLCJpc19mb2xsb3dl
ciI6ZmFsc2UsImlzX2ZyaWVuZCI6ZmFsc2UsImlzX3ByZW1pdW0iOm51bGws
InByb2ZpbGVfaW1hZ2VfdXJscyI6eyJweF81MHg1MCI6Imh0dHA6Ly9pMi5w
aXhpdi5uZXQvdXNlci1wcm9maWxlL2ltZy8yMDE2LzA5LzA3LzIyLzQ2LzIx
LzExNDY1NzAxXzY4YmRkZTIyMDI2M2U5NzgwZGE4MjQ1YzkzMzQ4Yzc4XzUw
LmpwZyJ9LCJzdGF0cyI6bnVsbCwicHJvZmlsZSI6bnVsbH0sImlzX21hbmdh
IjpmYWxzZSwiaXNfbGlrZWQiOmZhbHNlLCJmYXZvcml0ZV9pZCI6MCwicGFn
ZV9jb3VudCI6MSwiYm9va19zdHlsZSI6Im5vbmUiLCJ0eXBlIjoiaWxsdXN0
cmF0aW9uIiwibWV0YWRhdGEiOm51bGwsImNvbnRlbnRfdHlwZSI6bnVsbH1d
LCJjb3VudCI6MX0=
http_version:
recorded_at: Tue, 27 Sep 2016 22:47:52 GMT
recorded_with: VCR 2.9.3

18
test/unit/sources/pixiv_test.rb
View File

@@ -18,9 +18,23 @@ module Sources
end
context "in all cases" do
context "A whitecube page" do
setup do
VCR.use_cassette("pixiv-whitecube-ilust", :record => :none) do
@site = Sources::Site.new("https://www.pixiv.net/whitecube/user/277898/illust/59182257")
@site.get
@image_urls = @site.image_urls
end
end
should "1234 get all the image urls" do
assert_equal(["http://i2.pixiv.net/img-original/img/2016/09/26/21/30/41/59182257_p0.jpg"], @image_urls)
end
end
context "A gallery page" do
setup do
VCR.use_cassette("pixiv-gallery", :record => :once) do
VCR.use_cassette("pixiv-gallery", :record => :none) do
@site = Sources::Site.new("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=49270482")
@site.get
@image_urls = @site.image_urls
@@ -34,7 +48,7 @@ module Sources
context "An ugoira source site for pixiv" do
setup do
VCR.use_cassette("ugoira-converter", :record => :once) do
VCR.use_cassette("ugoira-converter", :record => :none) do
@site = Sources::Site.new("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=46378654")
@site.get
end