From 39cc3ed5cf913499093d2f641d70d7682a14fa42 Mon Sep 17 00:00:00 2001 From: evazion Date: Tue, 9 Feb 2021 05:48:02 -0600 Subject: [PATCH] pixiv: fix API breakage. Fix the Pixiv API no longer working by rewriting the Pixiv strategy to use the Ajax API instead of the mobile API. Before we could authenticate in the mobile API by using the OAuth 2.0 grant_type=password authentication flow. This no longer works. Now it requires logging in through a HTML page, which is protected by Google reCaptcha. This makes using the mobile API infeasible. Instead we switch to the Ajax API, which only needs a PHPSESSID to authenticate. This can be obtained by logging in manually and using the devtools to extract the cookie. This also temporarily removes support for Pixiv novels. This should be moved to a separate source strategy. --- .github/workflows/test.yaml | 3 +- app/logical/pixiv_ajax_client.rb | 393 ++++++++++++++++++++++++ app/logical/pixiv_api_client.rb | 184 ----------- app/logical/sources/strategies.rb | 2 +- app/logical/sources/strategies/pixiv.rb | 166 ++++------ config/danbooru_default_config.rb | 9 +- test/unit/downloads/pixiv_test.rb | 6 +- test/unit/sources/pixiv_test.rb | 12 +- 8 files changed, 458 insertions(+), 317 deletions(-) create mode 100644 app/logical/pixiv_ajax_client.rb delete mode 100644 app/logical/pixiv_api_client.rb diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 74d78e2bb..719dac0f4 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -32,8 +32,7 @@ jobs: DANBOORU_AWS_SQS_ENABLED: false DANBOORU_TWITTER_API_KEY: ${{ secrets.DANBOORU_TWITTER_API_KEY }} DANBOORU_TWITTER_API_SECRET: ${{ secrets.DANBOORU_TWITTER_API_SECRET }} - DANBOORU_PIXIV_LOGIN: ${{ secrets.DANBOORU_PIXIV_LOGIN }} - DANBOORU_PIXIV_PASSWORD: ${{ secrets.DANBOORU_PIXIV_PASSWORD }} + DANBOORU_PIXIV_PHPSESSID: ${{ secrets.DANBOORU_PIXIV_PHPSESSID }} DANBOORU_NIJIE_LOGIN: ${{ secrets.DANBOORU_NIJIE_LOGIN }} DANBOORU_NIJIE_PASSWORD: ${{ secrets.DANBOORU_NIJIE_PASSWORD }} DANBOORU_NICO_SEIGA_LOGIN: ${{ secrets.DANBOORU_NICO_SEIGA_LOGIN }} diff --git a/app/logical/pixiv_ajax_client.rb b/app/logical/pixiv_ajax_client.rb new file mode 100644 index 000000000..9b79525c9 --- /dev/null +++ b/app/logical/pixiv_ajax_client.rb @@ -0,0 +1,393 @@ +# Notes on Pixiv's AJAX API: +# +# * The user agent must be spoofed as a browser user agent, otherwise the API +# will return an error. +# * You must authenticate by passing a PHPSESSID cookie, otherwise the API will +# return nothing for R-18 works. +# * The PHPSESSID cookie is obtained by logging in manually and getting the +# cookie from the devtools. +# * The /ajax/illust/$id endpoint only returns the first image for multi-image +# works. The /ajax/illust/$id/pages API must be used to get the full list of images. +# * Ugoiras need to use the /ajax/illust/$id/ugoira_meta API to get the frame metadata. +# +# Endpoints: +# +# * https://www.pixiv.net/ajax/illust/87598468 +# * https://www.pixiv.net/ajax/user/883091/illusts?ids[]=87598468 +# * https://www.pixiv.net/ajax/illust/87598468/pages +# * https://www.pixiv.net/ajax/illust/74932152/ugoira_meta +# +# https://www.pixiv.net/ajax/illust/87598468 +# +# => { +# "error": false, +# "message": "", +# "body": { +# "illustId": "87598468", +# "illustTitle": "Billow", +# "illustComment": "須田景凪さんの1st Full Album「Billow」のイラストを担当いたしました。
ジャケットはデザイナーさんがイラストをぐにゃっとして作ってくれました。

クロスフェード
・niconico
https://www.nicovideo.jp/watch/sm38153587
・youtube
https://www.youtube.com/watch?v=TQnwTiFXSgc

アートディレクター
クロスフェード制作
吉良進太郎", +# "id": "87598468", +# "title": "Billow", +# "description": "須田景凪さんの1st Full Album「Billow」のイラストを担当いたしました。
ジャケットはデザイナーさんがイラストをぐにゃっとして作ってくれました。

クロスフェード
・niconico
https://www.nicovideo.jp/watch/sm38153587
・youtube
https://www.youtube.com/watch?v=TQnwTiFXSgc

アートディレクター
クロスフェード制作
吉良進太郎", +# "illustType": 0, +# "createDate": "2021-02-07T09:02:03+00:00", +# "uploadDate": "2021-02-07T09:02:03+00:00", +# "restrict": 0, +# "xRestrict": 0, +# "sl": 2, +# "urls": { +# "mini": "https://i.pximg.net/c/48x48/img-master/img/2021/02/07/18/02/03/87598468_p0_square1200.jpg", +# "thumb": "https://i.pximg.net/c/250x250_80_a2/img-master/img/2021/02/07/18/02/03/87598468_p0_square1200.jpg", +# "small": "https://i.pximg.net/c/540x540_70/img-master/img/2021/02/07/18/02/03/87598468_p0_master1200.jpg", +# "regular": "https://i.pximg.net/img-master/img/2021/02/07/18/02/03/87598468_p0_master1200.jpg", +# "original": "https://i.pximg.net/img-original/img/2021/02/07/18/02/03/87598468_p0.jpg" +# }, +# "tags": { +# "authorId": "883091", +# "isLocked": false, +# "tags": [ +# { +# "tag": "オリジナル", +# "locked": true, +# "deletable": false, +# "userId": "883091", +# "romaji": "orijinaru", +# "translation": { +# "en": "original" +# }, +# "userName": "アボガド6" +# }, +# { +# "tag": "須田景凪", +# "locked": true, +# "deletable": false, +# "userId": "883091", +# "romaji": "sudakeina", +# "userName": "アボガド6" +# } +# ], +# "writable": true +# }, +# "alt": "original / Billow / February 7th, 2021", +# "storableTags": [ +# "RTJMXD26Ak", +# "bG5P5jsGQR" +# ], +# "userId": "883091", +# "userName": "アボガド6", +# "userAccount": "avogado602", +# "userIllusts": { +# ... +# }, +# "likeData": false, +# "width": 4491, +# "height": 4435, +# "pageCount": 16, +# "bookmarkCount": 3843, +# "likeCount": 3386, +# "commentCount": 11, +# "responseCount": 0, +# "viewCount": 62936, +# "isHowto": false, +# "isOriginal": true, +# "imageResponseOutData": [], +# "imageResponseData": [], +# "imageResponseCount": 0, +# "pollData": null, +# "seriesNavData": null, +# "descriptionBoothId": null, +# "descriptionYoutubeId": "TQnwTiFXSgc", +# "comicPromotion": null, +# "fanboxPromotion": null, +# "contestBanners": [], +# "isBookmarkable": true, +# "bookmarkData": null, +# "contestData": null, +# "zoneConfig": { +# ... +# }, +# "extraData": { +# "meta": { +# "title": "original / Billow / February 7th, 2021 - pixiv", +# "description": "この作品 「Billow」 は 「オリジナル」「須田景凪」 のタグがつけられた「アボガド6」さんのイラストです。 「須田景凪さんの1st Full Album「Billow」のイラストを担当いたしました。ジャケットはデザイナーさんがイラストをぐにゃっとして作ってくれました。クロ…", +# "canonical": "https://www.pixiv.net/en/artworks/87598468", +# "alternateLanguages": { +# "ja": "https://www.pixiv.net/artworks/87598468", +# "en": "https://www.pixiv.net/en/artworks/87598468" +# }, +# "descriptionHeader": "original are the most prominent tags for this work posted on February 7th, 2021.", +# "ogp": { +# "description": "須田景凪さんの1st Full Album「Billow」のイラストを担当いたしました。ジャケットはデザイナーさんがイラストをぐにゃっとして作ってくれました。クロスフェード・niconicohttps", +# "image": "https://embed.pixiv.net/decorate.php?illust_id=87598468", +# "title": "original / Billow / February 7th, 2021 - pixiv", +# "type": "article" +# }, +# "twitter": { +# "description": "須田景凪さんの1st Full Album「Billow」のイラストを担当いたしました。\r\nジャケットはデザイナーさんがイラストをぐにゃっとして作ってくれました。\r\n\r\nクロスフェード\r\n・niconico\r\nhttps://www.nicovideo.jp/watch/sm38153587\r\n・youtube\r\nhttps://www.youtube.com/watch?v=TQnwTiFXSgc\r\n\r\nアートディレクター\r\nクロスフェード制作\r\n吉良進太郎", +# "image": "https://embed.pixiv.net/decorate.php?illust_id=87598468", +# "title": "Billow", +# "card": "summary_large_image" +# } +# } +# }, +# "titleCaptionTranslation": { +# "workTitle": "", +# "workCaption": "" +# }, +# "isUnlisted": false, +# "request": null +# } +# } +# +# https://www.pixiv.net/ajax/illust/87598468/pages?lang=en +# +# => { +# "error": false, +# "message": "", +# "body": [ +# { +# "urls": { +# "thumb_mini": "https://i.pximg.net/c/128x128/img-master/img/2021/02/07/18/02/03/87598468_p0_square1200.jpg", +# "small": "https://i.pximg.net/c/540x540_70/img-master/img/2021/02/07/18/02/03/87598468_p0_master1200.jpg", +# "regular": "https://i.pximg.net/img-master/img/2021/02/07/18/02/03/87598468_p0_master1200.jpg", +# "original": "https://i.pximg.net/img-original/img/2021/02/07/18/02/03/87598468_p0.jpg" +# }, +# "width": 4491, +# "height": 4435 +# }, +# { +# "urls": { +# "thumb_mini": "https://i.pximg.net/c/128x128/img-master/img/2021/02/07/18/02/03/87598468_p1_square1200.jpg", +# "small": "https://i.pximg.net/c/540x540_70/img-master/img/2021/02/07/18/02/03/87598468_p1_master1200.jpg", +# "regular": "https://i.pximg.net/img-master/img/2021/02/07/18/02/03/87598468_p1_master1200.jpg", +# "original": "https://i.pximg.net/img-original/img/2021/02/07/18/02/03/87598468_p1.jpg" +# }, +# "width": 5000, +# "height": 5000 +# }, +# { +# "urls": { +# "thumb_mini": "https://i.pximg.net/c/128x128/img-master/img/2021/02/07/18/02/03/87598468_p2_square1200.jpg", +# "small": "https://i.pximg.net/c/540x540_70/img-master/img/2021/02/07/18/02/03/87598468_p2_master1200.jpg", +# "regular": "https://i.pximg.net/img-master/img/2021/02/07/18/02/03/87598468_p2_master1200.jpg", +# "original": "https://i.pximg.net/img-original/img/2021/02/07/18/02/03/87598468_p2.jpg" +# }, +# "width": 5000, +# "height": 5000 +# }, +# { +# "urls": { +# "thumb_mini": "https://i.pximg.net/c/128x128/img-master/img/2021/02/07/18/02/03/87598468_p3_square1200.jpg", +# "small": "https://i.pximg.net/c/540x540_70/img-master/img/2021/02/07/18/02/03/87598468_p3_master1200.jpg", +# "regular": "https://i.pximg.net/img-master/img/2021/02/07/18/02/03/87598468_p3_master1200.jpg", +# "original": "https://i.pximg.net/img-original/img/2021/02/07/18/02/03/87598468_p3.jpg" +# }, +# "width": 5000, +# "height": 5000 +# }, +# { +# "urls": { +# "thumb_mini": "https://i.pximg.net/c/128x128/img-master/img/2021/02/07/18/02/03/87598468_p4_square1200.jpg", +# "small": "https://i.pximg.net/c/540x540_70/img-master/img/2021/02/07/18/02/03/87598468_p4_master1200.jpg", +# "regular": "https://i.pximg.net/img-master/img/2021/02/07/18/02/03/87598468_p4_master1200.jpg", +# "original": "https://i.pximg.net/img-original/img/2021/02/07/18/02/03/87598468_p4.jpg" +# }, +# "width": 5000, +# "height": 5000 +# }, +# { +# "urls": { +# "thumb_mini": "https://i.pximg.net/c/128x128/img-master/img/2021/02/07/18/02/03/87598468_p5_square1200.jpg", +# "small": "https://i.pximg.net/c/540x540_70/img-master/img/2021/02/07/18/02/03/87598468_p5_master1200.jpg", +# "regular": "https://i.pximg.net/img-master/img/2021/02/07/18/02/03/87598468_p5_master1200.jpg", +# "original": "https://i.pximg.net/img-original/img/2021/02/07/18/02/03/87598468_p5.jpg" +# }, +# "width": 5000, +# "height": 5000 +# }, +# { +# "urls": { +# "thumb_mini": "https://i.pximg.net/c/128x128/img-master/img/2021/02/07/18/02/03/87598468_p6_square1200.jpg", +# "small": "https://i.pximg.net/c/540x540_70/img-master/img/2021/02/07/18/02/03/87598468_p6_master1200.jpg", +# "regular": "https://i.pximg.net/img-master/img/2021/02/07/18/02/03/87598468_p6_master1200.jpg", +# "original": "https://i.pximg.net/img-original/img/2021/02/07/18/02/03/87598468_p6.jpg" +# }, +# "width": 5000, +# "height": 5000 +# }, +# { +# "urls": { +# "thumb_mini": "https://i.pximg.net/c/128x128/img-master/img/2021/02/07/18/02/03/87598468_p7_square1200.jpg", +# "small": "https://i.pximg.net/c/540x540_70/img-master/img/2021/02/07/18/02/03/87598468_p7_master1200.jpg", +# "regular": "https://i.pximg.net/img-master/img/2021/02/07/18/02/03/87598468_p7_master1200.jpg", +# "original": "https://i.pximg.net/img-original/img/2021/02/07/18/02/03/87598468_p7.jpg" +# }, +# "width": 5000, +# "height": 5000 +# }, +# { +# "urls": { +# "thumb_mini": "https://i.pximg.net/c/128x128/img-master/img/2021/02/07/18/02/03/87598468_p8_square1200.jpg", +# "small": "https://i.pximg.net/c/540x540_70/img-master/img/2021/02/07/18/02/03/87598468_p8_master1200.jpg", +# "regular": "https://i.pximg.net/img-master/img/2021/02/07/18/02/03/87598468_p8_master1200.jpg", +# "original": "https://i.pximg.net/img-original/img/2021/02/07/18/02/03/87598468_p8.jpg" +# }, +# "width": 5000, +# "height": 5000 +# }, +# { +# "urls": { +# "thumb_mini": "https://i.pximg.net/c/128x128/img-master/img/2021/02/07/18/02/03/87598468_p9_square1200.jpg", +# "small": "https://i.pximg.net/c/540x540_70/img-master/img/2021/02/07/18/02/03/87598468_p9_master1200.jpg", +# "regular": "https://i.pximg.net/img-master/img/2021/02/07/18/02/03/87598468_p9_master1200.jpg", +# "original": "https://i.pximg.net/img-original/img/2021/02/07/18/02/03/87598468_p9.jpg" +# }, +# "width": 5000, +# "height": 5000 +# }, +# { +# "urls": { +# "thumb_mini": "https://i.pximg.net/c/128x128/img-master/img/2021/02/07/18/02/03/87598468_p10_square1200.jpg", +# "small": "https://i.pximg.net/c/540x540_70/img-master/img/2021/02/07/18/02/03/87598468_p10_master1200.jpg", +# "regular": "https://i.pximg.net/img-master/img/2021/02/07/18/02/03/87598468_p10_master1200.jpg", +# "original": "https://i.pximg.net/img-original/img/2021/02/07/18/02/03/87598468_p10.jpg" +# }, +# "width": 5000, +# "height": 5000 +# }, +# { +# "urls": { +# "thumb_mini": "https://i.pximg.net/c/128x128/img-master/img/2021/02/07/18/02/03/87598468_p11_square1200.jpg", +# "small": "https://i.pximg.net/c/540x540_70/img-master/img/2021/02/07/18/02/03/87598468_p11_master1200.jpg", +# "regular": "https://i.pximg.net/img-master/img/2021/02/07/18/02/03/87598468_p11_master1200.jpg", +# "original": "https://i.pximg.net/img-original/img/2021/02/07/18/02/03/87598468_p11.jpg" +# }, +# "width": 5000, +# "height": 5000 +# }, +# { +# "urls": { +# "thumb_mini": "https://i.pximg.net/c/128x128/img-master/img/2021/02/07/18/02/03/87598468_p12_square1200.jpg", +# "small": "https://i.pximg.net/c/540x540_70/img-master/img/2021/02/07/18/02/03/87598468_p12_master1200.jpg", +# "regular": "https://i.pximg.net/img-master/img/2021/02/07/18/02/03/87598468_p12_master1200.jpg", +# "original": "https://i.pximg.net/img-original/img/2021/02/07/18/02/03/87598468_p12.jpg" +# }, +# "width": 5000, +# "height": 5000 +# }, +# { +# "urls": { +# "thumb_mini": "https://i.pximg.net/c/128x128/img-master/img/2021/02/07/18/02/03/87598468_p13_square1200.jpg", +# "small": "https://i.pximg.net/c/540x540_70/img-master/img/2021/02/07/18/02/03/87598468_p13_master1200.jpg", +# "regular": "https://i.pximg.net/img-master/img/2021/02/07/18/02/03/87598468_p13_master1200.jpg", +# "original": "https://i.pximg.net/img-original/img/2021/02/07/18/02/03/87598468_p13.jpg" +# }, +# "width": 5000, +# "height": 5000 +# }, +# { +# "urls": { +# "thumb_mini": "https://i.pximg.net/c/128x128/img-master/img/2021/02/07/18/02/03/87598468_p14_square1200.jpg", +# "small": "https://i.pximg.net/c/540x540_70/img-master/img/2021/02/07/18/02/03/87598468_p14_master1200.jpg", +# "regular": "https://i.pximg.net/img-master/img/2021/02/07/18/02/03/87598468_p14_master1200.jpg", +# "original": "https://i.pximg.net/img-original/img/2021/02/07/18/02/03/87598468_p14.jpg" +# }, +# "width": 5000, +# "height": 5000 +# }, +# { +# "urls": { +# "thumb_mini": "https://i.pximg.net/c/128x128/img-master/img/2021/02/07/18/02/03/87598468_p15_square1200.jpg", +# "small": "https://i.pximg.net/c/540x540_70/img-master/img/2021/02/07/18/02/03/87598468_p15_master1200.jpg", +# "regular": "https://i.pximg.net/img-master/img/2021/02/07/18/02/03/87598468_p15_master1200.jpg", +# "original": "https://i.pximg.net/img-original/img/2021/02/07/18/02/03/87598468_p15.jpg" +# }, +# "width": 5000, +# "height": 5000 +# } +# ] +# } +# +# https://www.pixiv.net/ajax/illust/74932152/ugoira_meta?lang=en +# +# => { +# "error": false, +# "message": "", +# "body": { +# "src": "https://i.pximg.net/img-zip-ugoira/img/2019/05/27/17/59/33/74932152_ugoira600x600.zip", +# "originalSrc": "https://i.pximg.net/img-zip-ugoira/img/2019/05/27/17/59/33/74932152_ugoira1920x1080.zip", +# "mime_type": "image/jpeg", +# "frames": [ +# { +# "file": "000000.jpg", +# "delay": 60 +# }, +# { +# "file": "000001.jpg", +# "delay": 60 +# }, +# { +# "file": "000002.jpg", +# "delay": 60 +# }, +# { +# "file": "000003.jpg", +# "delay": 60 +# }, +# { +# "file": "000004.jpg", +# "delay": 60 +# }, +# { +# "file": "000005.jpg", +# "delay": 60 +# }, +# { +# "file": "000006.jpg", +# "delay": 60 +# }, +# { +# "file": "000007.jpg", +# "delay": 60 +# } +# ] +# } +# } + +class PixivAjaxClient + USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36" + + attr_reader :phpsessid, :http + + def initialize(phpsessid, http: Danbooru::Http.new) + @phpsessid = phpsessid + @http = http + end + + def illust(illust_id) + get("https://www.pixiv.net/ajax/illust/#{illust_id}").with_indifferent_access + end + + def pages(illust_id) + get("https://www.pixiv.net/ajax/illust/#{illust_id}/pages") + end + + def ugoira_meta(illust_id) + get("https://www.pixiv.net/ajax/illust/#{illust_id}/ugoira_meta").with_indifferent_access + end + + def get(url) + response = client.cache(1.minute).get(url) + + if response.code == 200 + response.parse["body"] + else + DanbooruLogger.info("Pixiv API call failed (url=#{url} status=#{response.code} body=#{response.body})") + {} + end + end + + def client + @client ||= http.headers("User-Agent": USER_AGENT).cookies(PHPSESSID: phpsessid) + end +end diff --git a/app/logical/pixiv_api_client.rb b/app/logical/pixiv_api_client.rb deleted file mode 100644 index 22e6c9955..000000000 --- a/app/logical/pixiv_api_client.rb +++ /dev/null @@ -1,184 +0,0 @@ -class PixivApiClient - extend Memoist - - API_VERSION = "1" - CLIENT_ID = "bYGKuGVw91e0NMfPGp44euvGt59s" - CLIENT_SECRET = "HP3RmkgAmEGro0gn1x9ioawQE8WMfvLXDz3ZqxpK" - CLIENT_HASH_SALT = "28c1fdd170a5204386cb1313c7077b34f83e4aaf4aa829ce78c231e05b0bae2c" - - # Tools to not include in the tags list. We don't tag digital media, so - # including these results in bad translated tags suggestions. - TOOLS_BLACKLIST = %w[ - Photoshop Illustrator Fireworks Flash Painter PaintShopPro pixiv\ Sketch - CLIP\ STUDIO\ PAINT IllustStudio ComicStudio RETAS\ STUDIO SAI PhotoStudio - Pixia NekoPaint PictBear openCanvas ArtRage Expression Inkscape GIMP - CGillust COMICWORKS MS_Paint EDGE AzPainter AzPainter2 AzDrawing - PicturePublisher SketchBookPro Processing 4thPaint GraphicsGale mdiapp - Paintgraphic AfterEffects drawr CLIP\ PAINT\ Lab FireAlpaca Pixelmator - AzDrawing2 MediBang\ Paint Krita ibisPaint Procreate Live2D - Lightwave3D Shade Poser STRATA AnimationMaster XSI CARRARA CINEMA4D Maya - 3dsMax Blender ZBrush Metasequoia Sunny3D Bryce Vue Hexagon\ King SketchUp - VistaPro Sculptris Comi\ Po! modo DAZ\ Studio 3D-Coat - ] - - class Error < StandardError; end - class BadIDError < Error; end - - class WorkResponse - attr_reader :json, :pages, :name, :moniker, :user_id, :page_count, :tags - attr_reader :artist_commentary_title, :artist_commentary_desc - - def initialize(json) - @json = json - @name = json["user"]["name"] - @user_id = json["user"]["id"] - @moniker = json["user"]["account"] - @page_count = json["page_count"].to_i - @artist_commentary_title = json["title"].to_s - @artist_commentary_desc = json["caption"].to_s - @tags = json["tags"].reject {|x| x =~ /^http:/} - @tags += json["tools"] - TOOLS_BLACKLIST - - if json["metadata"] - if json["metadata"]["zip_urls"] - @pages = json["metadata"]["zip_urls"] - elsif page_count > 1 - @pages = json["metadata"]["pages"].map {|x| x["image_urls"]["large"]} - end - end - - if @pages.nil? && json["image_urls"] - @pages = [json["image_urls"]["large"]] - end - end - end - - class NovelResponse - extend Memoist - - attr_reader :json - - def initialize(json) - @json = json - end - - def name - json["user"]["name"] - end - - def user_id - json["user"]["id"] - end - - def moniker - json["user"]["account"] - end - - def page_count - json["page_count"].to_i - end - - def artist_commentary_title - json["title"] - end - - def artist_commentary_desc - json["caption"] - end - - def tags - json["tags"] - end - - def pages - # ex: - # https://i.pximg.net/c/150x150_80/novel-cover-master/img/2017/07/27/23/14/17/8465454_80685d10e6df4d7d53ad347ddc18a36b_master1200.jpg (6096b) - # => - # https://i.pximg.net/novel-cover-original/img/2017/07/27/23/14/17/8465454_80685d10e6df4d7d53ad347ddc18a36b.jpg (532129b) - [find_original(json["image_urls"]["small"])] - end - memoize :pages - - public - - PXIMG = %r!\Ahttps?://i\.pximg\.net/c/\d+x\d+_\d+/novel-cover-master/img/(?\d+/\d+/\d+/\d+/\d+/\d+)/(?\d+_[a-f0-9]+)_master\d+\.(?jpg|jpeg|png|gif)!i - - def find_original(x) - if x =~ PXIMG - return "https://i.pximg.net/novel-cover-original/img/#{$~[:timestamp]}/#{$~[:filename]}.#{$~[:ext]}" - end - - return x - end - end - - def work(illust_id) - params = { image_sizes: "large", include_stats: "true" } - url = "https://public-api.secure.pixiv.net/v#{API_VERSION}/works/#{illust_id.to_i}.json" - response = api_client.cache(1.minute).get(url, params: params) - json = response.parse - - if response.status == 200 - WorkResponse.new(json["response"][0]) - elsif json["status"] == "failure" && json.dig("errors", "system", "message") =~ /対象のイラストは見つかりませんでした。/ - raise BadIDError.new("Pixiv ##{illust_id} not found: work was deleted, made private, or ID is invalid.") - else - raise Error.new("Pixiv API call failed (status=#{response.code} body=#{response.body})") - end - rescue JSON::ParserError - raise Error.new("Pixiv API call failed (status=#{response.code} body=#{response.body})") - end - - def novel(novel_id) - url = "https://public-api.secure.pixiv.net/v#{API_VERSION}/novels/#{novel_id.to_i}.json" - resp = api_client.cache(1.minute).get(url) - json = resp.parse - - if resp.status == 200 - NovelResponse.new(json["response"][0]) - elsif json["status"] == "failure" && json.dig("errors", "system", "message") =~ /対象のイラストは見つかりませんでした。/ - raise Error.new("Pixiv API call failed (status=#{resp.code} body=#{body})") - end - rescue JSON::ParserError - raise Error.new("Pixiv API call failed (status=#{resp.code} body=#{body})") - end - - def access_token - # truncate timestamp to 1-hour resolution so that it doesn't break caching. - client_time = Time.zone.now.utc.change(min: 0).rfc3339 - client_hash = Digest::MD5.hexdigest(client_time + CLIENT_HASH_SALT) - - headers = { - "Referer": "http://www.pixiv.net", - "X-Client-Time": client_time, - "X-Client-Hash": client_hash - } - - params = { - username: Danbooru.config.pixiv_login, - password: Danbooru.config.pixiv_password, - grant_type: "password", - client_id: CLIENT_ID, - client_secret: CLIENT_SECRET - } - - resp = http.headers(headers).cache(1.hour).post("https://oauth.secure.pixiv.net/auth/token", form: params) - return nil unless resp.status == 200 - - resp.parse.dig("response", "access_token") - end - - def api_client - http.headers( - "Referer": "http://www.pixiv.net", - "Content-Type": "application/x-www-form-urlencoded", - "Authorization": "Bearer #{access_token}" - ) - end - - def http - Danbooru::Http.new - end - - memoize :access_token, :api_client, :http -end diff --git a/app/logical/sources/strategies.rb b/app/logical/sources/strategies.rb index ca77191ec..fddc1fdd3 100644 --- a/app/logical/sources/strategies.rb +++ b/app/logical/sources/strategies.rb @@ -2,7 +2,7 @@ module Sources module Strategies def self.all [ - #Strategies::Pixiv, + Strategies::Pixiv, Strategies::Fanbox, Strategies::NicoSeiga, Strategies::Twitter, diff --git a/app/logical/sources/strategies/pixiv.rb b/app/logical/sources/strategies/pixiv.rb index 00d5ff86b..0a670351d 100644 --- a/app/logical/sources/strategies/pixiv.rb +++ b/app/logical/sources/strategies/pixiv.rb @@ -48,27 +48,22 @@ module Sources I12 = %r{(?:\A(?:https?://)?i[0-9]+\.pixiv\.net)} IMG = %r{(?:\A(?:https?://)?img[0-9]*\.pixiv\.net)} PXIMG = %r{(?:\A(?:https?://)?[^.]+\.pximg\.net)} - TOUCH = %r{(?:\A(?:https?://)?touch\.pixiv\.net)} UGOIRA = %r{#{PXIMG}/img-zip-ugoira/img/#{DATE}/(?\d+)_ugoira1920x1080\.zip\z}i ORIG_IMAGE = %r{#{PXIMG}/img-original/img/#{DATE}/(?\d+)_p(?\d+)\.#{EXT}\z}i - STACC_PAGE = %r{\A#{WEB}/stacc/#{MONIKER}/?\z}i - NOVEL_PAGE = %r{(?:\Ahttps?://www\.pixiv\.net/novel/show\.php\?id=(\d+))} def self.enabled? - Danbooru.config.pixiv_login.present? && Danbooru.config.pixiv_password.present? + Danbooru.config.pixiv_phpsessid.present? end def self.to_dtext(text) - if text.nil? - return nil - end + return nil if text.nil? - text = text.gsub(%r{https?://www\.pixiv\.net/member_illust\.php\?mode=medium&illust_id=([0-9]+)}i) do |_match| + text = text.gsub(%r{illust/[0-9]+}i) do |_match| pixiv_id = $1 %(pixiv ##{pixiv_id} "»":[#{Routes.posts_path(tags: "pixiv:#{pixiv_id}")}]) end - text = text.gsub(%r{https?://www\.pixiv\.net/member\.php\?id=([0-9]+)}i) do |_match| + text = text.gsub(%r{user/[0-9]+}i) do |_match| member_id = $1 profile_url = "https://www.pixiv.net/users/#{member_id}" artist_search_url = Routes.artists_path(search: { url_matches: profile_url }) @@ -76,7 +71,6 @@ module Sources %("user/#{member_id}":[#{profile_url}] "»":[#{artist_search_url}]) end - text = text.gsub(/\r\n|\r|\n/, "
") DText.from_html(text) end @@ -95,9 +89,19 @@ module Sources end def image_urls - image_urls_sub - rescue PixivApiClient::BadIDError - [url] + if is_ugoira? + [api_ugoira[:originalSrc]] + elsif manga_page.present? && original_urls.present? + [original_urls[manga_page]] + elsif original_urls.present? + original_urls + else + [url] + end + end + + def original_urls + api_pages.map { |page| page.dig("urls", "original") } end def preview_urls @@ -114,17 +118,8 @@ module Sources end def page_url - if novel_id.present? - return "https://www.pixiv.net/novel/show.php?id=#{novel_id}&mode=cover" - end - - if illust_id.present? - return "https://www.pixiv.net/artworks/#{illust_id}" - end - - url - rescue PixivApiClient::BadIDError - nil + return nil if illust_id.blank? + "https://www.pixiv.net/artworks/#{illust_id}" end def canonical_url @@ -132,15 +127,15 @@ module Sources end def profile_url - [url, referer_url].each do |x| - if x =~ PROFILE - return x - end - end + url = urls.find { |url| url.match?(PROFILE) } - "https://www.pixiv.net/users/#{metadata.user_id}" - rescue PixivApiClient::BadIDError - nil + if url.present? + url + elsif api_illust[:userId].present? + "https://www.pixiv.net/users/#{api_illust[:userId]}" + else + nil + end end def stacc_url @@ -153,9 +148,7 @@ module Sources end def artist_name - metadata.name - rescue PixivApiClient::BadIDError - nil + api_illust[:userName] end def other_names @@ -163,15 +156,11 @@ module Sources end def artist_commentary_title - metadata.artist_commentary_title - rescue PixivApiClient::BadIDError - nil + api_illust[:title] end def artist_commentary_desc - metadata.artist_commentary_desc - rescue PixivApiClient::BadIDError - nil + api_illust[:description] end def headers @@ -179,8 +168,7 @@ module Sources end def normalize_for_source - return if illust_id.blank? - + return nil if illust_id.blank? "https://www.pixiv.net/artworks/#{illust_id}" end @@ -189,11 +177,10 @@ module Sources end def tags - metadata.tags.map do |tag| + api_illust.dig(:tags, :tags).to_a.map do |item| + tag = item[:tag] [tag, "https://www.pixiv.net/search.php?s_mode=s_tag_full&#{{word: tag}.to_param}"] end - rescue PixivApiClient::BadIDError - [] end def normalize_tag(tag) @@ -214,28 +201,12 @@ module Sources illust_id.present? ? "pixiv:#{illust_id}" : "source:#{canonical_url}" end - def image_urls_sub - # there's too much normalization bullshit we have to deal with - # raw urls, so just fetch the canonical url from the api every - # time. - if manga_page.present? - return [metadata.pages[manga_page]] - end - - if metadata.pages.is_a?(Hash) - return [ugoira_zip_url] - end - - metadata.pages + def is_ugoira? + # https://i.pximg.net/img-original/img/2019/05/27/17/59/33/74932152_ugoira0.jpg + url.match?(UGOIRA) || api_illust.dig(:urls, :original)&.match?(/ugoira/) end - # in order to prevent recursive loops, this method should not make any - # api calls and only try to extract the illust_id from the url. therefore, - # even though it makes sense to reference page_url here, it will only look - # at (url, referer_url). def illust_id - return nil if novel_id.present? - parsed_urls.each do |url| # http://www.pixiv.net/member_illust.php?mode=medium&illust_id=18557054 # http://www.pixiv.net/member_illust.php?mode=big&illust_id=18557054 @@ -284,27 +255,22 @@ module Sources nil end - memoize :illust_id - def novel_id - [url, referer_url].each do |x| - if x =~ NOVEL_PAGE - return $1 - end - end - - nil + def api_client + PixivAjaxClient.new(Danbooru.config.pixiv_phpsessid) end - memoize :novel_id - def metadata - if novel_id.present? - return PixivApiClient.new.novel(novel_id) - end - - PixivApiClient.new.work(illust_id) + def api_illust + api_client.illust(illust_id) + end + + def api_pages + api_client.pages(illust_id) + end + + def api_ugoira + api_client.ugoira_meta(illust_id) end - memoize :metadata def moniker # we can sometimes get the moniker from the url @@ -315,44 +281,17 @@ module Sources elsif url =~ %r{#{WEB}/stacc/(#{MONIKER})/?$}i $1 else - metadata.moniker + api_illust[:userAccount] end - rescue PixivApiClient::BadIDError - nil end - memoize :moniker def data - { ugoira_frame_data: ugoira_frame_data } + { ugoira_frame_data: api_ugoira[:frames] } end - def ugoira_zip_url - if metadata.pages.is_a?(Hash) && metadata.pages["ugoira600x600"] - metadata.pages["ugoira600x600"].sub("_ugoira600x600.zip", "_ugoira1920x1080.zip") - end - end - memoize :ugoira_zip_url - - def ugoira_frame_data - metadata.json.dig("metadata", "frames") - rescue PixivApiClient::BadIDError - nil - end - memoize :ugoira_frame_data - def ugoira_content_type - case metadata.json["image_urls"].to_s - when /\.jpg/ - "image/jpeg" - when /\.png/ - "image/png" - when /\.gif/ - "image/gif" - else - raise Sources::Error, "content type not found for (#{url}, #{referer_url})" - end + api_ugoira[:mime_type] end - memoize :ugoira_content_type # Returns the current page number of the manga. This will not # make any api calls and only looks at (url, referer_url). @@ -373,7 +312,8 @@ module Sources nil end - memoize :manga_page + + memoize :illust_id, :api_client, :api_illust, :api_pages, :api_ugoira end end end diff --git a/config/danbooru_default_config.rb b/config/danbooru_default_config.rb index 526891393..9e890c39f 100644 --- a/config/danbooru_default_config.rb +++ b/config/danbooru_default_config.rb @@ -289,11 +289,10 @@ module Danbooru [] end - def pixiv_login - nil - end - - def pixiv_password + # Your Pixiv PHPSESSID cookie. Get this by logging in to Pixiv and using + # the devtools to find the PHPSESSID cookie. This is need for Pixiv upload + # support. + def pixiv_phpsessid nil end diff --git a/test/unit/downloads/pixiv_test.rb b/test/unit/downloads/pixiv_test.rb index 7b2ea08b7..11dcf78c2 100644 --- a/test/unit/downloads/pixiv_test.rb +++ b/test/unit/downloads/pixiv_test.rb @@ -129,11 +129,7 @@ module Downloads @strategy = Sources::Strategies.find("http://www.pixiv.net/member_illust.php?mode=medium&illust_id=62247364") assert_equal(2, @strategy.data[:ugoira_frame_data].size) - if @strategy.data[:ugoira_frame_data][0]["file"] - assert_equal([{"file" => "000000.jpg", "delay" => 125}, {"file" => "000001.jpg", "delay" => 125}], @download.data[:ugoira_frame_data]) - else - assert_equal([{"delay_msec" => 125}, {"delay_msec" => 125}], @strategy.data[:ugoira_frame_data]) - end + assert_equal([{"file" => "000000.jpg", "delay" => 125}, {"file" => "000001.jpg", "delay" => 125}], @strategy.data[:ugoira_frame_data]) end end end diff --git a/test/unit/sources/pixiv_test.rb b/test/unit/sources/pixiv_test.rb index a69106c3d..480fe19f2 100644 --- a/test/unit/sources/pixiv_test.rb +++ b/test/unit/sources/pixiv_test.rb @@ -44,12 +44,10 @@ module Sources end should "capture the frame data" do - assert_equal(2, @site.ugoira_frame_data.size) - if @site.ugoira_frame_data[0]["file"] - assert_equal([{"file" => "000000.jpg", "delay" => 125}, {"file" => "000001.jpg", "delay" => 125}], @site.ugoira_frame_data) - else - assert_equal([{"delay_msec" => 125}, {"delay_msec" => 125}], @site.ugoira_frame_data) - end + ugoira_frame_data = @site.data[:ugoira_frame_data] + + assert_equal(2, ugoira_frame_data.size) + assert_equal([{"file" => "000000.jpg", "delay" => 125}, {"file" => "000001.jpg", "delay" => 125}], ugoira_frame_data) end end @@ -195,7 +193,7 @@ module Sources should "convert illust links and member links to dtext" do get_source("https://www.pixiv.net/member_illust.php?mode=medium&illust_id=63421642") - dtext_desc = %(foo 【pixiv #46337015 "»":[/posts?tags=pixiv%3A46337015]】bar 【pixiv #14901720 "»":[/posts?tags=pixiv%3A14901720]】\n\nbaz【"user/83739":[https://www.pixiv.net/users/83739] "»":[/artists?search%5Burl_matches%5D=https%3A%2F%2Fwww.pixiv.net%2Fusers%2F83739]】) + dtext_desc = %(foo 【[b]pixiv #46337015 "»":[/posts?tags=pixiv%3A46337015][/b]】bar 【[b]pixiv #14901720 "»":[/posts?tags=pixiv%3A14901720][/b]】\n\nbaz【[b]"user/83739":[https://www.pixiv.net/users/83739] "»":[/artists?search%5Burl_matches%5D=https%3A%2F%2Fwww.pixiv.net%2Fusers%2F83739][/b]】) assert_equal(dtext_desc, @site.dtext_artist_commentary_desc) end end