pixiv: fix API breakage.

Fix the Pixiv API no longer working by rewriting the Pixiv strategy to
use the Ajax API instead of the mobile API.

Before we could authenticate in the mobile API by using the OAuth 2.0
grant_type=password authentication flow. This no longer works. Now it
requires logging in through a HTML page, which is protected by Google
reCaptcha. This makes using the mobile API infeasible.

Instead we switch to the Ajax API, which only needs a PHPSESSID to
authenticate. This can be obtained by logging in manually and using the
devtools to extract the cookie.

This also temporarily removes support for Pixiv novels. This should be
moved to a separate source strategy.
This commit is contained in:
evazion
2021-02-09 05:48:02 -06:00
parent 7520c4db49
commit 39cc3ed5cf
8 changed files with 458 additions and 317 deletions

View File

@@ -32,8 +32,7 @@ jobs:
DANBOORU_AWS_SQS_ENABLED: false
DANBOORU_TWITTER_API_KEY: ${{ secrets.DANBOORU_TWITTER_API_KEY }}
DANBOORU_TWITTER_API_SECRET: ${{ secrets.DANBOORU_TWITTER_API_SECRET }}
DANBOORU_PIXIV_LOGIN: ${{ secrets.DANBOORU_PIXIV_LOGIN }}
DANBOORU_PIXIV_PASSWORD: ${{ secrets.DANBOORU_PIXIV_PASSWORD }}
DANBOORU_PIXIV_PHPSESSID: ${{ secrets.DANBOORU_PIXIV_PHPSESSID }}
DANBOORU_NIJIE_LOGIN: ${{ secrets.DANBOORU_NIJIE_LOGIN }}
DANBOORU_NIJIE_PASSWORD: ${{ secrets.DANBOORU_NIJIE_PASSWORD }}
DANBOORU_NICO_SEIGA_LOGIN: ${{ secrets.DANBOORU_NICO_SEIGA_LOGIN }}

View File

@@ -0,0 +1,393 @@
# Notes on Pixiv's AJAX API:
#
# * The user agent must be spoofed as a browser user agent, otherwise the API
# will return an error.
# * You must authenticate by passing a PHPSESSID cookie, otherwise the API will
# return nothing for R-18 works.
# * The PHPSESSID cookie is obtained by logging in manually and getting the
# cookie from the devtools.
# * The /ajax/illust/$id endpoint only returns the first image for multi-image
# works. The /ajax/illust/$id/pages API must be used to get the full list of images.
# * Ugoiras need to use the /ajax/illust/$id/ugoira_meta API to get the frame metadata.
#
# Endpoints:
#
# * https://www.pixiv.net/ajax/illust/87598468
# * https://www.pixiv.net/ajax/user/883091/illusts?ids[]=87598468
# * https://www.pixiv.net/ajax/illust/87598468/pages
# * https://www.pixiv.net/ajax/illust/74932152/ugoira_meta
#
# https://www.pixiv.net/ajax/illust/87598468
#
# => {
# "error": false,
# "message": "",
# "body": {
# "illustId": "87598468",
# "illustTitle": "Billow",
# "illustComment": "須田景凪さんの1st Full Album「Billow」のイラストを担当いたしました。<br />ジャケットはデザイナーさんがイラストをぐにゃっとして作ってくれました。<br /><br />クロスフェード<br />・niconico<br /><a href=\"/jump.php?https%3A%2F%2Fwww.nicovideo.jp%2Fwatch%2Fsm38153587\" target=\"_blank\">https://www.nicovideo.jp/watch/sm38153587</a><br />・youtube<br /><a href=\"/jump.php?https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DTQnwTiFXSgc\" target=\"_blank\">https://www.youtube.com/watch?v=TQnwTiFXSgc</a><br /><br />アートディレクター<br />クロスフェード制作<br />吉良進太郎",
# "id": "87598468",
# "title": "Billow",
# "description": "須田景凪さんの1st Full Album「Billow」のイラストを担当いたしました。<br />ジャケットはデザイナーさんがイラストをぐにゃっとして作ってくれました。<br /><br />クロスフェード<br />・niconico<br /><a href=\"/jump.php?https%3A%2F%2Fwww.nicovideo.jp%2Fwatch%2Fsm38153587\" target=\"_blank\">https://www.nicovideo.jp/watch/sm38153587</a><br />・youtube<br /><a href=\"/jump.php?https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DTQnwTiFXSgc\" target=\"_blank\">https://www.youtube.com/watch?v=TQnwTiFXSgc</a><br /><br />アートディレクター<br />クロスフェード制作<br />吉良進太郎",
# "illustType": 0,
# "createDate": "2021-02-07T09:02:03+00:00",
# "uploadDate": "2021-02-07T09:02:03+00:00",
# "restrict": 0,
# "xRestrict": 0,
# "sl": 2,
# "urls": {
# "mini": "https://i.pximg.net/c/48x48/img-master/img/2021/02/07/18/02/03/87598468_p0_square1200.jpg",
# "thumb": "https://i.pximg.net/c/250x250_80_a2/img-master/img/2021/02/07/18/02/03/87598468_p0_square1200.jpg",
# "small": "https://i.pximg.net/c/540x540_70/img-master/img/2021/02/07/18/02/03/87598468_p0_master1200.jpg",
# "regular": "https://i.pximg.net/img-master/img/2021/02/07/18/02/03/87598468_p0_master1200.jpg",
# "original": "https://i.pximg.net/img-original/img/2021/02/07/18/02/03/87598468_p0.jpg"
# },
# "tags": {
# "authorId": "883091",
# "isLocked": false,
# "tags": [
# {
# "tag": "オリジナル",
# "locked": true,
# "deletable": false,
# "userId": "883091",
# "romaji": "orijinaru",
# "translation": {
# "en": "original"
# },
# "userName": "アボガド6"
# },
# {
# "tag": "須田景凪",
# "locked": true,
# "deletable": false,
# "userId": "883091",
# "romaji": "sudakeina",
# "userName": "アボガド6"
# }
# ],
# "writable": true
# },
# "alt": "original / Billow / February 7th, 2021",
# "storableTags": [
# "RTJMXD26Ak",
# "bG5P5jsGQR"
# ],
# "userId": "883091",
# "userName": "アボガド6",
# "userAccount": "avogado602",
# "userIllusts": {
# ...
# },
# "likeData": false,
# "width": 4491,
# "height": 4435,
# "pageCount": 16,
# "bookmarkCount": 3843,
# "likeCount": 3386,
# "commentCount": 11,
# "responseCount": 0,
# "viewCount": 62936,
# "isHowto": false,
# "isOriginal": true,
# "imageResponseOutData": [],
# "imageResponseData": [],
# "imageResponseCount": 0,
# "pollData": null,
# "seriesNavData": null,
# "descriptionBoothId": null,
# "descriptionYoutubeId": "TQnwTiFXSgc",
# "comicPromotion": null,
# "fanboxPromotion": null,
# "contestBanners": [],
# "isBookmarkable": true,
# "bookmarkData": null,
# "contestData": null,
# "zoneConfig": {
# ...
# },
# "extraData": {
# "meta": {
# "title": "original / Billow / February 7th, 2021 - pixiv",
# "description": "この作品 「Billow」 は 「オリジナル」「須田景凪」 のタグがつけられた「アボガド6」さんのイラストです。 「須田景凪さんの1st Full Album「Billow」のイラストを担当いたしました。ジャケットはデザイナーさんがイラストをぐにゃっとして作ってくれました。クロ…",
# "canonical": "https://www.pixiv.net/en/artworks/87598468",
# "alternateLanguages": {
# "ja": "https://www.pixiv.net/artworks/87598468",
# "en": "https://www.pixiv.net/en/artworks/87598468"
# },
# "descriptionHeader": "original are the most prominent tags for this work posted on February 7th, 2021.",
# "ogp": {
# "description": "須田景凪さんの1st Full Album「Billow」のイラストを担当いたしました。ジャケットはデザイナーさんがイラストをぐにゃっとして作ってくれました。クロスフェード・niconicohttps",
# "image": "https://embed.pixiv.net/decorate.php?illust_id=87598468",
# "title": "original / Billow / February 7th, 2021 - pixiv",
# "type": "article"
# },
# "twitter": {
# "description": "須田景凪さんの1st Full Album「Billow」のイラストを担当いたしました。\r\nジャケットはデザイナーさんがイラストをぐにゃっとして作ってくれました。\r\n\r\nクロスフェード\r\n・niconico\r\nhttps://www.nicovideo.jp/watch/sm38153587\r\n・youtube\r\nhttps://www.youtube.com/watch?v=TQnwTiFXSgc\r\n\r\nアートディレクター\r\nクロスフェード制作\r\n吉良進太郎",
# "image": "https://embed.pixiv.net/decorate.php?illust_id=87598468",
# "title": "Billow",
# "card": "summary_large_image"
# }
# }
# },
# "titleCaptionTranslation": {
# "workTitle": "",
# "workCaption": ""
# },
# "isUnlisted": false,
# "request": null
# }
# }
#
# https://www.pixiv.net/ajax/illust/87598468/pages?lang=en
#
# => {
# "error": false,
# "message": "",
# "body": [
# {
# "urls": {
# "thumb_mini": "https://i.pximg.net/c/128x128/img-master/img/2021/02/07/18/02/03/87598468_p0_square1200.jpg",
# "small": "https://i.pximg.net/c/540x540_70/img-master/img/2021/02/07/18/02/03/87598468_p0_master1200.jpg",
# "regular": "https://i.pximg.net/img-master/img/2021/02/07/18/02/03/87598468_p0_master1200.jpg",
# "original": "https://i.pximg.net/img-original/img/2021/02/07/18/02/03/87598468_p0.jpg"
# },
# "width": 4491,
# "height": 4435
# },
# {
# "urls": {
# "thumb_mini": "https://i.pximg.net/c/128x128/img-master/img/2021/02/07/18/02/03/87598468_p1_square1200.jpg",
# "small": "https://i.pximg.net/c/540x540_70/img-master/img/2021/02/07/18/02/03/87598468_p1_master1200.jpg",
# "regular": "https://i.pximg.net/img-master/img/2021/02/07/18/02/03/87598468_p1_master1200.jpg",
# "original": "https://i.pximg.net/img-original/img/2021/02/07/18/02/03/87598468_p1.jpg"
# },
# "width": 5000,
# "height": 5000
# },
# {
# "urls": {
# "thumb_mini": "https://i.pximg.net/c/128x128/img-master/img/2021/02/07/18/02/03/87598468_p2_square1200.jpg",
# "small": "https://i.pximg.net/c/540x540_70/img-master/img/2021/02/07/18/02/03/87598468_p2_master1200.jpg",
# "regular": "https://i.pximg.net/img-master/img/2021/02/07/18/02/03/87598468_p2_master1200.jpg",
# "original": "https://i.pximg.net/img-original/img/2021/02/07/18/02/03/87598468_p2.jpg"
# },
# "width": 5000,
# "height": 5000
# },
# {
# "urls": {
# "thumb_mini": "https://i.pximg.net/c/128x128/img-master/img/2021/02/07/18/02/03/87598468_p3_square1200.jpg",
# "small": "https://i.pximg.net/c/540x540_70/img-master/img/2021/02/07/18/02/03/87598468_p3_master1200.jpg",
# "regular": "https://i.pximg.net/img-master/img/2021/02/07/18/02/03/87598468_p3_master1200.jpg",
# "original": "https://i.pximg.net/img-original/img/2021/02/07/18/02/03/87598468_p3.jpg"
# },
# "width": 5000,
# "height": 5000
# },
# {
# "urls": {
# "thumb_mini": "https://i.pximg.net/c/128x128/img-master/img/2021/02/07/18/02/03/87598468_p4_square1200.jpg",
# "small": "https://i.pximg.net/c/540x540_70/img-master/img/2021/02/07/18/02/03/87598468_p4_master1200.jpg",
# "regular": "https://i.pximg.net/img-master/img/2021/02/07/18/02/03/87598468_p4_master1200.jpg",
# "original": "https://i.pximg.net/img-original/img/2021/02/07/18/02/03/87598468_p4.jpg"
# },
# "width": 5000,
# "height": 5000
# },
# {
# "urls": {
# "thumb_mini": "https://i.pximg.net/c/128x128/img-master/img/2021/02/07/18/02/03/87598468_p5_square1200.jpg",
# "small": "https://i.pximg.net/c/540x540_70/img-master/img/2021/02/07/18/02/03/87598468_p5_master1200.jpg",
# "regular": "https://i.pximg.net/img-master/img/2021/02/07/18/02/03/87598468_p5_master1200.jpg",
# "original": "https://i.pximg.net/img-original/img/2021/02/07/18/02/03/87598468_p5.jpg"
# },
# "width": 5000,
# "height": 5000
# },
# {
# "urls": {
# "thumb_mini": "https://i.pximg.net/c/128x128/img-master/img/2021/02/07/18/02/03/87598468_p6_square1200.jpg",
# "small": "https://i.pximg.net/c/540x540_70/img-master/img/2021/02/07/18/02/03/87598468_p6_master1200.jpg",
# "regular": "https://i.pximg.net/img-master/img/2021/02/07/18/02/03/87598468_p6_master1200.jpg",
# "original": "https://i.pximg.net/img-original/img/2021/02/07/18/02/03/87598468_p6.jpg"
# },
# "width": 5000,
# "height": 5000
# },
# {
# "urls": {
# "thumb_mini": "https://i.pximg.net/c/128x128/img-master/img/2021/02/07/18/02/03/87598468_p7_square1200.jpg",
# "small": "https://i.pximg.net/c/540x540_70/img-master/img/2021/02/07/18/02/03/87598468_p7_master1200.jpg",
# "regular": "https://i.pximg.net/img-master/img/2021/02/07/18/02/03/87598468_p7_master1200.jpg",
# "original": "https://i.pximg.net/img-original/img/2021/02/07/18/02/03/87598468_p7.jpg"
# },
# "width": 5000,
# "height": 5000
# },
# {
# "urls": {
# "thumb_mini": "https://i.pximg.net/c/128x128/img-master/img/2021/02/07/18/02/03/87598468_p8_square1200.jpg",
# "small": "https://i.pximg.net/c/540x540_70/img-master/img/2021/02/07/18/02/03/87598468_p8_master1200.jpg",
# "regular": "https://i.pximg.net/img-master/img/2021/02/07/18/02/03/87598468_p8_master1200.jpg",
# "original": "https://i.pximg.net/img-original/img/2021/02/07/18/02/03/87598468_p8.jpg"
# },
# "width": 5000,
# "height": 5000
# },
# {
# "urls": {
# "thumb_mini": "https://i.pximg.net/c/128x128/img-master/img/2021/02/07/18/02/03/87598468_p9_square1200.jpg",
# "small": "https://i.pximg.net/c/540x540_70/img-master/img/2021/02/07/18/02/03/87598468_p9_master1200.jpg",
# "regular": "https://i.pximg.net/img-master/img/2021/02/07/18/02/03/87598468_p9_master1200.jpg",
# "original": "https://i.pximg.net/img-original/img/2021/02/07/18/02/03/87598468_p9.jpg"
# },
# "width": 5000,
# "height": 5000
# },
# {
# "urls": {
# "thumb_mini": "https://i.pximg.net/c/128x128/img-master/img/2021/02/07/18/02/03/87598468_p10_square1200.jpg",
# "small": "https://i.pximg.net/c/540x540_70/img-master/img/2021/02/07/18/02/03/87598468_p10_master1200.jpg",
# "regular": "https://i.pximg.net/img-master/img/2021/02/07/18/02/03/87598468_p10_master1200.jpg",
# "original": "https://i.pximg.net/img-original/img/2021/02/07/18/02/03/87598468_p10.jpg"
# },
# "width": 5000,
# "height": 5000
# },
# {
# "urls": {
# "thumb_mini": "https://i.pximg.net/c/128x128/img-master/img/2021/02/07/18/02/03/87598468_p11_square1200.jpg",
# "small": "https://i.pximg.net/c/540x540_70/img-master/img/2021/02/07/18/02/03/87598468_p11_master1200.jpg",
# "regular": "https://i.pximg.net/img-master/img/2021/02/07/18/02/03/87598468_p11_master1200.jpg",
# "original": "https://i.pximg.net/img-original/img/2021/02/07/18/02/03/87598468_p11.jpg"
# },
# "width": 5000,
# "height": 5000
# },
# {
# "urls": {
# "thumb_mini": "https://i.pximg.net/c/128x128/img-master/img/2021/02/07/18/02/03/87598468_p12_square1200.jpg",
# "small": "https://i.pximg.net/c/540x540_70/img-master/img/2021/02/07/18/02/03/87598468_p12_master1200.jpg",
# "regular": "https://i.pximg.net/img-master/img/2021/02/07/18/02/03/87598468_p12_master1200.jpg",
# "original": "https://i.pximg.net/img-original/img/2021/02/07/18/02/03/87598468_p12.jpg"
# },
# "width": 5000,
# "height": 5000
# },
# {
# "urls": {
# "thumb_mini": "https://i.pximg.net/c/128x128/img-master/img/2021/02/07/18/02/03/87598468_p13_square1200.jpg",
# "small": "https://i.pximg.net/c/540x540_70/img-master/img/2021/02/07/18/02/03/87598468_p13_master1200.jpg",
# "regular": "https://i.pximg.net/img-master/img/2021/02/07/18/02/03/87598468_p13_master1200.jpg",
# "original": "https://i.pximg.net/img-original/img/2021/02/07/18/02/03/87598468_p13.jpg"
# },
# "width": 5000,
# "height": 5000
# },
# {
# "urls": {
# "thumb_mini": "https://i.pximg.net/c/128x128/img-master/img/2021/02/07/18/02/03/87598468_p14_square1200.jpg",
# "small": "https://i.pximg.net/c/540x540_70/img-master/img/2021/02/07/18/02/03/87598468_p14_master1200.jpg",
# "regular": "https://i.pximg.net/img-master/img/2021/02/07/18/02/03/87598468_p14_master1200.jpg",
# "original": "https://i.pximg.net/img-original/img/2021/02/07/18/02/03/87598468_p14.jpg"
# },
# "width": 5000,
# "height": 5000
# },
# {
# "urls": {
# "thumb_mini": "https://i.pximg.net/c/128x128/img-master/img/2021/02/07/18/02/03/87598468_p15_square1200.jpg",
# "small": "https://i.pximg.net/c/540x540_70/img-master/img/2021/02/07/18/02/03/87598468_p15_master1200.jpg",
# "regular": "https://i.pximg.net/img-master/img/2021/02/07/18/02/03/87598468_p15_master1200.jpg",
# "original": "https://i.pximg.net/img-original/img/2021/02/07/18/02/03/87598468_p15.jpg"
# },
# "width": 5000,
# "height": 5000
# }
# ]
# }
#
# https://www.pixiv.net/ajax/illust/74932152/ugoira_meta?lang=en
#
# => {
# "error": false,
# "message": "",
# "body": {
# "src": "https://i.pximg.net/img-zip-ugoira/img/2019/05/27/17/59/33/74932152_ugoira600x600.zip",
# "originalSrc": "https://i.pximg.net/img-zip-ugoira/img/2019/05/27/17/59/33/74932152_ugoira1920x1080.zip",
# "mime_type": "image/jpeg",
# "frames": [
# {
# "file": "000000.jpg",
# "delay": 60
# },
# {
# "file": "000001.jpg",
# "delay": 60
# },
# {
# "file": "000002.jpg",
# "delay": 60
# },
# {
# "file": "000003.jpg",
# "delay": 60
# },
# {
# "file": "000004.jpg",
# "delay": 60
# },
# {
# "file": "000005.jpg",
# "delay": 60
# },
# {
# "file": "000006.jpg",
# "delay": 60
# },
# {
# "file": "000007.jpg",
# "delay": 60
# }
# ]
# }
# }
class PixivAjaxClient
USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36"
attr_reader :phpsessid, :http
def initialize(phpsessid, http: Danbooru::Http.new)
@phpsessid = phpsessid
@http = http
end
def illust(illust_id)
get("https://www.pixiv.net/ajax/illust/#{illust_id}").with_indifferent_access
end
def pages(illust_id)
get("https://www.pixiv.net/ajax/illust/#{illust_id}/pages")
end
def ugoira_meta(illust_id)
get("https://www.pixiv.net/ajax/illust/#{illust_id}/ugoira_meta").with_indifferent_access
end
def get(url)
response = client.cache(1.minute).get(url)
if response.code == 200
response.parse["body"]
else
DanbooruLogger.info("Pixiv API call failed (url=#{url} status=#{response.code} body=#{response.body})")
{}
end
end
def client
@client ||= http.headers("User-Agent": USER_AGENT).cookies(PHPSESSID: phpsessid)
end
end

View File

@@ -1,184 +0,0 @@
class PixivApiClient
extend Memoist
API_VERSION = "1"
CLIENT_ID = "bYGKuGVw91e0NMfPGp44euvGt59s"
CLIENT_SECRET = "HP3RmkgAmEGro0gn1x9ioawQE8WMfvLXDz3ZqxpK"
CLIENT_HASH_SALT = "28c1fdd170a5204386cb1313c7077b34f83e4aaf4aa829ce78c231e05b0bae2c"
# Tools to not include in the tags list. We don't tag digital media, so
# including these results in bad translated tags suggestions.
TOOLS_BLACKLIST = %w[
Photoshop Illustrator Fireworks Flash Painter PaintShopPro pixiv\ Sketch
CLIP\ STUDIO\ PAINT IllustStudio ComicStudio RETAS\ STUDIO SAI PhotoStudio
Pixia NekoPaint PictBear openCanvas ArtRage Expression Inkscape GIMP
CGillust COMICWORKS MS_Paint EDGE AzPainter AzPainter2 AzDrawing
PicturePublisher SketchBookPro Processing 4thPaint GraphicsGale mdiapp
Paintgraphic AfterEffects drawr CLIP\ PAINT\ Lab FireAlpaca Pixelmator
AzDrawing2 MediBang\ Paint Krita ibisPaint Procreate Live2D
Lightwave3D Shade Poser STRATA AnimationMaster XSI CARRARA CINEMA4D Maya
3dsMax Blender ZBrush Metasequoia Sunny3D Bryce Vue Hexagon\ King SketchUp
VistaPro Sculptris Comi\ Po! modo DAZ\ Studio 3D-Coat
]
class Error < StandardError; end
class BadIDError < Error; end
class WorkResponse
attr_reader :json, :pages, :name, :moniker, :user_id, :page_count, :tags
attr_reader :artist_commentary_title, :artist_commentary_desc
def initialize(json)
@json = json
@name = json["user"]["name"]
@user_id = json["user"]["id"]
@moniker = json["user"]["account"]
@page_count = json["page_count"].to_i
@artist_commentary_title = json["title"].to_s
@artist_commentary_desc = json["caption"].to_s
@tags = json["tags"].reject {|x| x =~ /^http:/}
@tags += json["tools"] - TOOLS_BLACKLIST
if json["metadata"]
if json["metadata"]["zip_urls"]
@pages = json["metadata"]["zip_urls"]
elsif page_count > 1
@pages = json["metadata"]["pages"].map {|x| x["image_urls"]["large"]}
end
end
if @pages.nil? && json["image_urls"]
@pages = [json["image_urls"]["large"]]
end
end
end
class NovelResponse
extend Memoist
attr_reader :json
def initialize(json)
@json = json
end
def name
json["user"]["name"]
end
def user_id
json["user"]["id"]
end
def moniker
json["user"]["account"]
end
def page_count
json["page_count"].to_i
end
def artist_commentary_title
json["title"]
end
def artist_commentary_desc
json["caption"]
end
def tags
json["tags"]
end
def pages
# ex:
# https://i.pximg.net/c/150x150_80/novel-cover-master/img/2017/07/27/23/14/17/8465454_80685d10e6df4d7d53ad347ddc18a36b_master1200.jpg (6096b)
# =>
# https://i.pximg.net/novel-cover-original/img/2017/07/27/23/14/17/8465454_80685d10e6df4d7d53ad347ddc18a36b.jpg (532129b)
[find_original(json["image_urls"]["small"])]
end
memoize :pages
public
PXIMG = %r!\Ahttps?://i\.pximg\.net/c/\d+x\d+_\d+/novel-cover-master/img/(?<timestamp>\d+/\d+/\d+/\d+/\d+/\d+)/(?<filename>\d+_[a-f0-9]+)_master\d+\.(?<ext>jpg|jpeg|png|gif)!i
def find_original(x)
if x =~ PXIMG
return "https://i.pximg.net/novel-cover-original/img/#{$~[:timestamp]}/#{$~[:filename]}.#{$~[:ext]}"
end
return x
end
end
def work(illust_id)
params = { image_sizes: "large", include_stats: "true" }
url = "https://public-api.secure.pixiv.net/v#{API_VERSION}/works/#{illust_id.to_i}.json"
response = api_client.cache(1.minute).get(url, params: params)
json = response.parse
if response.status == 200
WorkResponse.new(json["response"][0])
elsif json["status"] == "failure" && json.dig("errors", "system", "message") =~ /対象のイラストは見つかりませんでした。/
raise BadIDError.new("Pixiv ##{illust_id} not found: work was deleted, made private, or ID is invalid.")
else
raise Error.new("Pixiv API call failed (status=#{response.code} body=#{response.body})")
end
rescue JSON::ParserError
raise Error.new("Pixiv API call failed (status=#{response.code} body=#{response.body})")
end
def novel(novel_id)
url = "https://public-api.secure.pixiv.net/v#{API_VERSION}/novels/#{novel_id.to_i}.json"
resp = api_client.cache(1.minute).get(url)
json = resp.parse
if resp.status == 200
NovelResponse.new(json["response"][0])
elsif json["status"] == "failure" && json.dig("errors", "system", "message") =~ /対象のイラストは見つかりませんでした。/
raise Error.new("Pixiv API call failed (status=#{resp.code} body=#{body})")
end
rescue JSON::ParserError
raise Error.new("Pixiv API call failed (status=#{resp.code} body=#{body})")
end
def access_token
# truncate timestamp to 1-hour resolution so that it doesn't break caching.
client_time = Time.zone.now.utc.change(min: 0).rfc3339
client_hash = Digest::MD5.hexdigest(client_time + CLIENT_HASH_SALT)
headers = {
"Referer": "http://www.pixiv.net",
"X-Client-Time": client_time,
"X-Client-Hash": client_hash
}
params = {
username: Danbooru.config.pixiv_login,
password: Danbooru.config.pixiv_password,
grant_type: "password",
client_id: CLIENT_ID,
client_secret: CLIENT_SECRET
}
resp = http.headers(headers).cache(1.hour).post("https://oauth.secure.pixiv.net/auth/token", form: params)
return nil unless resp.status == 200
resp.parse.dig("response", "access_token")
end
def api_client
http.headers(
"Referer": "http://www.pixiv.net",
"Content-Type": "application/x-www-form-urlencoded",
"Authorization": "Bearer #{access_token}"
)
end
def http
Danbooru::Http.new
end
memoize :access_token, :api_client, :http
end

View File

@@ -2,7 +2,7 @@ module Sources
module Strategies
def self.all
[
#Strategies::Pixiv,
Strategies::Pixiv,
Strategies::Fanbox,
Strategies::NicoSeiga,
Strategies::Twitter,

View File

@@ -48,27 +48,22 @@ module Sources
I12 = %r{(?:\A(?:https?://)?i[0-9]+\.pixiv\.net)}
IMG = %r{(?:\A(?:https?://)?img[0-9]*\.pixiv\.net)}
PXIMG = %r{(?:\A(?:https?://)?[^.]+\.pximg\.net)}
TOUCH = %r{(?:\A(?:https?://)?touch\.pixiv\.net)}
UGOIRA = %r{#{PXIMG}/img-zip-ugoira/img/#{DATE}/(?<illust_id>\d+)_ugoira1920x1080\.zip\z}i
ORIG_IMAGE = %r{#{PXIMG}/img-original/img/#{DATE}/(?<illust_id>\d+)_p(?<page>\d+)\.#{EXT}\z}i
STACC_PAGE = %r{\A#{WEB}/stacc/#{MONIKER}/?\z}i
NOVEL_PAGE = %r{(?:\Ahttps?://www\.pixiv\.net/novel/show\.php\?id=(\d+))}
def self.enabled?
Danbooru.config.pixiv_login.present? && Danbooru.config.pixiv_password.present?
Danbooru.config.pixiv_phpsessid.present?
end
def self.to_dtext(text)
if text.nil?
return nil
end
return nil if text.nil?
text = text.gsub(%r{https?://www\.pixiv\.net/member_illust\.php\?mode=medium&illust_id=([0-9]+)}i) do |_match|
text = text.gsub(%r{<a href="https?://www\.pixiv\.net/en/artworks/([0-9]+)">illust/[0-9]+</a>}i) do |_match|
pixiv_id = $1
%(pixiv ##{pixiv_id} "»":[#{Routes.posts_path(tags: "pixiv:#{pixiv_id}")}])
end
text = text.gsub(%r{https?://www\.pixiv\.net/member\.php\?id=([0-9]+)}i) do |_match|
text = text.gsub(%r{<a href="https?://www\.pixiv\.net/en/users/([0-9]+)">user/[0-9]+</a>}i) do |_match|
member_id = $1
profile_url = "https://www.pixiv.net/users/#{member_id}"
artist_search_url = Routes.artists_path(search: { url_matches: profile_url })
@@ -76,7 +71,6 @@ module Sources
%("user/#{member_id}":[#{profile_url}] "»":[#{artist_search_url}])
end
text = text.gsub(/\r\n|\r|\n/, "<br>")
DText.from_html(text)
end
@@ -95,9 +89,19 @@ module Sources
end
def image_urls
image_urls_sub
rescue PixivApiClient::BadIDError
[url]
if is_ugoira?
[api_ugoira[:originalSrc]]
elsif manga_page.present? && original_urls.present?
[original_urls[manga_page]]
elsif original_urls.present?
original_urls
else
[url]
end
end
def original_urls
api_pages.map { |page| page.dig("urls", "original") }
end
def preview_urls
@@ -114,17 +118,8 @@ module Sources
end
def page_url
if novel_id.present?
return "https://www.pixiv.net/novel/show.php?id=#{novel_id}&mode=cover"
end
if illust_id.present?
return "https://www.pixiv.net/artworks/#{illust_id}"
end
url
rescue PixivApiClient::BadIDError
nil
return nil if illust_id.blank?
"https://www.pixiv.net/artworks/#{illust_id}"
end
def canonical_url
@@ -132,15 +127,15 @@ module Sources
end
def profile_url
[url, referer_url].each do |x|
if x =~ PROFILE
return x
end
end
url = urls.find { |url| url.match?(PROFILE) }
"https://www.pixiv.net/users/#{metadata.user_id}"
rescue PixivApiClient::BadIDError
nil
if url.present?
url
elsif api_illust[:userId].present?
"https://www.pixiv.net/users/#{api_illust[:userId]}"
else
nil
end
end
def stacc_url
@@ -153,9 +148,7 @@ module Sources
end
def artist_name
metadata.name
rescue PixivApiClient::BadIDError
nil
api_illust[:userName]
end
def other_names
@@ -163,15 +156,11 @@ module Sources
end
def artist_commentary_title
metadata.artist_commentary_title
rescue PixivApiClient::BadIDError
nil
api_illust[:title]
end
def artist_commentary_desc
metadata.artist_commentary_desc
rescue PixivApiClient::BadIDError
nil
api_illust[:description]
end
def headers
@@ -179,8 +168,7 @@ module Sources
end
def normalize_for_source
return if illust_id.blank?
return nil if illust_id.blank?
"https://www.pixiv.net/artworks/#{illust_id}"
end
@@ -189,11 +177,10 @@ module Sources
end
def tags
metadata.tags.map do |tag|
api_illust.dig(:tags, :tags).to_a.map do |item|
tag = item[:tag]
[tag, "https://www.pixiv.net/search.php?s_mode=s_tag_full&#{{word: tag}.to_param}"]
end
rescue PixivApiClient::BadIDError
[]
end
def normalize_tag(tag)
@@ -214,28 +201,12 @@ module Sources
illust_id.present? ? "pixiv:#{illust_id}" : "source:#{canonical_url}"
end
def image_urls_sub
# there's too much normalization bullshit we have to deal with
# raw urls, so just fetch the canonical url from the api every
# time.
if manga_page.present?
return [metadata.pages[manga_page]]
end
if metadata.pages.is_a?(Hash)
return [ugoira_zip_url]
end
metadata.pages
def is_ugoira?
# https://i.pximg.net/img-original/img/2019/05/27/17/59/33/74932152_ugoira0.jpg
url.match?(UGOIRA) || api_illust.dig(:urls, :original)&.match?(/ugoira/)
end
# in order to prevent recursive loops, this method should not make any
# api calls and only try to extract the illust_id from the url. therefore,
# even though it makes sense to reference page_url here, it will only look
# at (url, referer_url).
def illust_id
return nil if novel_id.present?
parsed_urls.each do |url|
# http://www.pixiv.net/member_illust.php?mode=medium&illust_id=18557054
# http://www.pixiv.net/member_illust.php?mode=big&illust_id=18557054
@@ -284,27 +255,22 @@ module Sources
nil
end
memoize :illust_id
def novel_id
[url, referer_url].each do |x|
if x =~ NOVEL_PAGE
return $1
end
end
nil
def api_client
PixivAjaxClient.new(Danbooru.config.pixiv_phpsessid)
end
memoize :novel_id
def metadata
if novel_id.present?
return PixivApiClient.new.novel(novel_id)
end
PixivApiClient.new.work(illust_id)
def api_illust
api_client.illust(illust_id)
end
def api_pages
api_client.pages(illust_id)
end
def api_ugoira
api_client.ugoira_meta(illust_id)
end
memoize :metadata
def moniker
# we can sometimes get the moniker from the url
@@ -315,44 +281,17 @@ module Sources
elsif url =~ %r{#{WEB}/stacc/(#{MONIKER})/?$}i
$1
else
metadata.moniker
api_illust[:userAccount]
end
rescue PixivApiClient::BadIDError
nil
end
memoize :moniker
def data
{ ugoira_frame_data: ugoira_frame_data }
{ ugoira_frame_data: api_ugoira[:frames] }
end
def ugoira_zip_url
if metadata.pages.is_a?(Hash) && metadata.pages["ugoira600x600"]
metadata.pages["ugoira600x600"].sub("_ugoira600x600.zip", "_ugoira1920x1080.zip")
end
end
memoize :ugoira_zip_url
def ugoira_frame_data
metadata.json.dig("metadata", "frames")
rescue PixivApiClient::BadIDError
nil
end
memoize :ugoira_frame_data
def ugoira_content_type
case metadata.json["image_urls"].to_s
when /\.jpg/
"image/jpeg"
when /\.png/
"image/png"
when /\.gif/
"image/gif"
else
raise Sources::Error, "content type not found for (#{url}, #{referer_url})"
end
api_ugoira[:mime_type]
end
memoize :ugoira_content_type
# Returns the current page number of the manga. This will not
# make any api calls and only looks at (url, referer_url).
@@ -373,7 +312,8 @@ module Sources
nil
end
memoize :manga_page
memoize :illust_id, :api_client, :api_illust, :api_pages, :api_ugoira
end
end
end

View File

@@ -289,11 +289,10 @@ module Danbooru
[]
end
def pixiv_login
nil
end
def pixiv_password
# Your Pixiv PHPSESSID cookie. Get this by logging in to Pixiv and using
# the devtools to find the PHPSESSID cookie. This is need for Pixiv upload
# support.
def pixiv_phpsessid
nil
end

View File

@@ -129,11 +129,7 @@ module Downloads
@strategy = Sources::Strategies.find("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364")
assert_equal(2, @strategy.data[:ugoira_frame_data].size)
if @strategy.data[:ugoira_frame_data][0]["file"]
assert_equal([{"file" => "000000.jpg", "delay" => 125}, {"file" => "000001.jpg", "delay" => 125}], @download.data[:ugoira_frame_data])
else
assert_equal([{"delay_msec" => 125}, {"delay_msec" => 125}], @strategy.data[:ugoira_frame_data])
end
assert_equal([{"file" => "000000.jpg", "delay" => 125}, {"file" => "000001.jpg", "delay" => 125}], @strategy.data[:ugoira_frame_data])
end
end
end

View File

@@ -44,12 +44,10 @@ module Sources
end
should "capture the frame data" do
assert_equal(2, @site.ugoira_frame_data.size)
if @site.ugoira_frame_data[0]["file"]
assert_equal([{"file" => "000000.jpg", "delay" => 125}, {"file" => "000001.jpg", "delay" => 125}], @site.ugoira_frame_data)
else
assert_equal([{"delay_msec" => 125}, {"delay_msec" => 125}], @site.ugoira_frame_data)
end
ugoira_frame_data = @site.data[:ugoira_frame_data]
assert_equal(2, ugoira_frame_data.size)
assert_equal([{"file" => "000000.jpg", "delay" => 125}, {"file" => "000001.jpg", "delay" => 125}], ugoira_frame_data)
end
end
@@ -195,7 +193,7 @@ module Sources
should "convert illust links and member links to dtext" do
get_source("https://www.pixiv.net/member_illust.php?mode=medium&illust_id=63421642")
dtext_desc = %(foo 【pixiv #46337015 "»":[/posts?tags=pixiv%3A46337015]】bar 【pixiv #14901720 "»":[/posts?tags=pixiv%3A14901720]】\n\nbaz【"user/83739":[https://www.pixiv.net/users/83739] "»":[/artists?search%5Burl_matches%5D=https%3A%2F%2Fwww.pixiv.net%2Fusers%2F83739]】)
dtext_desc = %(foo 【[b]pixiv #46337015 "»":[/posts?tags=pixiv%3A46337015][/b]】bar 【[b]pixiv #14901720 "»":[/posts?tags=pixiv%3A14901720][/b]\n\nbaz【[b]"user/83739":[https://www.pixiv.net/users/83739] "»":[/artists?search%5Burl_matches%5D=https%3A%2F%2Fwww.pixiv.net%2Fusers%2F83739][/b])
assert_equal(dtext_desc, @site.dtext_artist_commentary_desc)
end
end