Merge pull request #3210 from evazion/fix-translated-tags

Fix #3208: Fix translated tag suggestions for Pixiv.

app/logical/pixiv_api_client.rb

@@ -3,6 +3,21 @@ class PixivApiClient
   CLIENT_ID = "bYGKuGVw91e0NMfPGp44euvGt59s"
   CLIENT_SECRET = "HP3RmkgAmEGro0gn1x9ioawQE8WMfvLXDz3ZqxpK"
 
+  # Tools to not include in the tags list. We don't tag digital media, so
+  # including these results in bad translated tags suggestions.
+  TOOLS_BLACKLIST = %w[
+    Photoshop Illustrator Fireworks Flash Painter PaintShopPro pixiv\ Sketch
+    CLIP\ STUDIO\ PAINT IllustStudio ComicStudio RETAS\ STUDIO SAI PhotoStudio
+    Pixia NekoPaint PictBear openCanvas ArtRage Expression Inkscape GIMP
+    CGillust COMICWORKS MS_Paint EDGE AzPainter AzPainter2 AzDrawing
+    PicturePublisher SketchBookPro Processing 4thPaint GraphicsGale mdiapp
+    Paintgraphic AfterEffects drawr CLIP\ PAINT\ Lab FireAlpaca Pixelmator
+    AzDrawing2 MediBang\ Paint Krita ibisPaint Procreate Live2D
+    Lightwave3D Shade Poser STRATA AnimationMaster XSI CARRARA CINEMA4D Maya
+    3dsMax Blender ZBrush Metasequoia Sunny3D Bryce Vue Hexagon\ King SketchUp
+    VistaPro Sculptris Comi\ Po! modo DAZ\ Studio 3D-Coat
+  ]
+
   class Error < Exception ; end
 
   class WorksResponse
@@ -96,7 +111,8 @@ class PixivApiClient
       @page_count = json["page_count"].to_i
       @artist_commentary_title = json["title"].to_s
       @artist_commentary_desc = json["caption"].to_s
-      @tags = [json["tags"], json["tools"]].flatten.compact.reject {|x| x =~ /^http:/}
+      @tags = json["tags"].reject {|x| x =~ /^http:/}
+      @tags += json["tools"] - TOOLS_BLACKLIST
 
       if page_count > 1
         @pages = json["metadata"]["pages"].map {|x| x["image_urls"]["large"]}

app/logical/sources/site.rb

@@ -8,7 +8,7 @@ module Sources
       :file_url, :ugoira_frame_data, :ugoira_content_type, :image_urls,
       :artist_commentary_title, :artist_commentary_desc,
       :dtext_artist_commentary_title, :dtext_artist_commentary_desc,
-      :rewrite_thumbnails, :illust_id_from_url, :to => :strategy
+      :rewrite_thumbnails, :illust_id_from_url, :translate_tag, :translated_tags, :to => :strategy
 
     def self.strategies
       [Strategies::PixivWhitecube, Strategies::Pixiv, Strategies::NicoSeiga, Strategies::DeviantArt, Strategies::ArtStation, Strategies::Nijie, Strategies::Twitter, Strategies::Tumblr, Strategies::Pawoo]
@@ -43,23 +43,6 @@ module Sources
       url
     end
 
-    def translated_tags
-      untranslated_tags = tags
-      untranslated_tags = untranslated_tags.map(&:first)
-      untranslated_tags += untranslated_tags.grep(/\//).map {|x| x.split(/\//)}.flatten
-      untranslated_tags = untranslated_tags.map do |tag|
-        if tag =~ /\A(\S+?)_?\d+users入り\Z/
-          $1
-        else
-          tag
-        end
-      end
-      untranslated_tags.reject! {|x| x.blank?}
-      wikis = WikiPage.title_in(untranslated_tags)
-      wikis += WikiPage.other_names_equal(untranslated_tags)
-      wikis.uniq.map{|wiki_page| [wiki_page.title, wiki_page.category_name]}
-    end
-
     def to_h
       return {
         :artist_name => artist_name,

app/logical/sources/strategies/base.rb

@@ -80,6 +80,23 @@ module Sources
         (@tags || []).uniq
       end
 
+      def translated_tags
+        translated_tags = tags.map(&:first).flat_map(&method(:translate_tag)).uniq.sort
+        translated_tags.map { |tag| [tag.name, tag.category] }
+      end
+
+      # Given a tag from the source site, should return an array of corresponding Danbooru tags.
+      def translate_tag(untranslated_tag)
+        translated_tags = Tag.where(name: WikiPage.active.other_names_equal([untranslated_tag]).uniq.select(:title))
+
+        if translated_tags.empty?
+          normalized_name = TagAlias.to_aliased([Tag.normalize_name(untranslated_tag)])
+          translated_tags = Tag.nonempty.where(name: normalized_name)
+        end
+
+        translated_tags
+      end
+
       # Should be set to a url for sites that prevent hotlinking, or left nil for sites that don't.
       def fake_referer
         nil

app/logical/sources/strategies/pixiv.rb

@@ -56,6 +56,17 @@ module Sources
         "http://www.pixiv.net/member.php?id=#{@metadata.user_id}/"
       end
 
+      def translate_tag(tag)
+        normalized_tag = tag.gsub(/\A(\S+?)_?\d+users入り\Z/i, '\1')
+
+        translated_tags = super(normalized_tag)
+        if translated_tags.empty? && normalized_tag.include?("/")
+          translated_tags = normalized_tag.split("/").flat_map { |tag| super(tag) }
+        end
+
+        translated_tags
+      end
+
       def get
         return unless illust_id_from_url
         @illust_id = illust_id_from_url

app/models/tag.rb

@@ -787,6 +787,10 @@ class Tag < ApplicationRecord
   end
 
   module SearchMethods
+    def nonempty
+      where("tags.post_count > 0")
+    end
+
     def name_matches(name)
       where("tags.name LIKE ? ESCAPE E'\\\\'", name.mb_chars.downcase.to_escaped_for_sql_like)
     end