Fix #4112: Colorize tags in DText.

DText is processed in three phases: a preprocessing phase, the regular
parsing phases, and a postprocessing phase.

In the preprocessing phase we extract all the wiki links from all the
dtext messages on the page (more precisely, we do this in forum threads
and on comment pages, because these are the main places with lots of
dtext). This is so we can lookup all the tags and wiki pages in one
query, which is necessary because in the worst case (in certain forum
threads and in certain list_of_* wiki pages) there can be hundreds of
tags per page.

In the postprocessing phase we fixup the html generated by the ragel
parser to add CSS classes to wiki links. We do this in a postprocessing
step because it's easier than doing it in the ragel parser itself.

app/helpers/application_helper.rb

@@ -33,9 +33,7 @@ module ApplicationHelper
   end
 
   def format_text(text, **options)
-    raw DTextRagel.parse(text, **options)
-  rescue DTextRagel::Error => e
-    raw ""
+    raw DText.format_text(text, **options)
   end
 
   def strip_dtext(text)

app/javascript/src/styles/common/dtext.scss

@@ -108,6 +108,10 @@ div.prose {
     padding: 0 0.2em 0 0.3em;
     vertical-align: 1px;
   }
+
+  a.dtext-wiki-does-not-exist, a.dtext-tag-does-not-exist {
+    text-decoration: dotted underline;
+  }
 }
 
 // avoid empty gaps beneath dtext blocks in table rows.

app/logical/d_text.rb

@@ -4,6 +4,54 @@ require 'uri'
 class DText
   MENTION_REGEXP = /(?<=^| )@\S+/
 
+  def self.format_text(text, data: nil, **options)
+    data = preprocess([text]) if data.nil?
+    html = DTextRagel.parse(text, **options)
+    html = postprocess(html, *data)
+    html
+  rescue DTextRagel::Error => e
+    ""
+  end
+
+  def self.preprocess(dtext_messages)
+    names = dtext_messages.map { |message| parse_wiki_titles(message) }.flatten.uniq
+    wiki_pages = WikiPage.where(title: names)
+    tags = Tag.where(name: names)
+
+    [wiki_pages, tags]
+  end
+
+  def self.postprocess(html, wiki_pages, tags)
+    fragment = Nokogiri::HTML.fragment(html)
+
+    fragment.css("a.dtext-wiki-link").each do |node|
+      name = node["href"][%r!\A/wiki_pages/show_or_new\?title=(.*)\z!i, 1]
+      name = CGI.unescape(name)
+      name = WikiPage.normalize_title(name)
+      wiki = wiki_pages.find { |wiki| wiki.title == name }
+      tag = tags.find { |tag| tag.name == name }
+
+      if wiki.blank?
+        node["class"] += " dtext-wiki-does-not-exist"
+        node["title"] = "This wiki page does not exist"
+      end
+
+      if WikiPage.is_meta_wiki?(name)
+        # skip (meta wikis aren't expected to have a tag)
+      elsif tag.blank?
+        node["class"] += " dtext-tag-does-not-exist"
+        node["title"] = "This wiki page does not have a tag"
+      elsif tag.post_count <= 0
+        node["class"] += " dtext-tag-empty"
+        node["title"] = "This wiki page does not have a tag"
+      else
+        node["class"] += " tag-type-#{tag.category}"
+      end
+    end
+
+    fragment.to_s
+  end
+
   def self.quote(message, creator_name)
     stripped_body = DText.strip_blocks(message, "quote")
     "[quote]\n#{creator_name} said:\n\n#{stripped_body}\n[/quote]\n\n"

app/models/wiki_page.rb

@@ -1,6 +1,8 @@
 class WikiPage < ApplicationRecord
   class RevertError < Exception ; end
 
+  META_WIKIS = ["list_of_", "tag_group:", "pool_group:", "howto:", "about:", "help:", "template:"]
+
   before_save :normalize_title
   before_save :normalize_other_names
   after_save :create_version
@@ -155,6 +157,10 @@ class WikiPage < ApplicationRecord
     title.tr("_", " ")
   end
 
+  def self.is_meta_wiki?(title)
+    title.starts_with?(*META_WIKIS)
+  end
+
   def wiki_page_changed?
     saved_change_to_title? || saved_change_to_body? || saved_change_to_is_locked? || saved_change_to_is_deleted? || saved_change_to_other_names?
   end

app/views/comments/_index_by_comment.html.erb

@@ -8,7 +8,7 @@
               <%= link_to(image_tag(comment.post.preview_file_url), post_path(comment.post)) %>
             <% end %>
           </div>
-          <%= render :partial => "comments/partials/show/comment", :collection => [comment] %>
+          <%= render partial: "comments/partials/show/comment", collection: [comment], locals: { dtext_data: DText.preprocess(@comments.map(&:body)) } %>
         <% end %>
       <% end %>
     <% end %>

app/views/comments/index_for_post.js.erb

@@ -1,5 +1,5 @@
 $("#threshold-comments-notice-for-<%= @post.id %>").hide();
 
 var current_comment_section = $("div.comments-for-post[data-post-id=<%= @post.id %>] div.list-of-comments");
-current_comment_section.html("<%= j(render(:partial => 'comments/partials/show/comment', :collection => @comments))%>");
+current_comment_section.html("<%= j(render(partial: 'comments/partials/show/comment', collection: @comments, locals: { dtext_data: DText.preprocess(@comments.map(&:body)) })) %>");
 $(window).trigger("danbooru:index_for_post", [<%= @post.id %>]);

app/views/comments/partials/index/_list.html.erb

@@ -13,7 +13,7 @@
 
   <div class="list-of-comments list-of-messages">
     <% if comments.present? %>
-      <%= render partial: "comments/partials/show/comment", collection: comments %>
+      <%= render partial: "comments/partials/show/comment", collection: comments, locals: { dtext_data: DText.preprocess(comments.map(&:body)) } %>
     <% elsif post.last_commented_at.present? %>
       <p>There are no visible comments.</p>
     <% else %>

app/views/comments/partials/show/_comment.html.erb

@@ -22,7 +22,7 @@
     </div>
     <div class="content">
       <div class="body prose">
-        <%= format_text(comment.body) %>
+        <%= format_text(comment.body, data: dtext_data) %>
       </div>
       <%= render "application/update_notice", record: comment %>
       

app/views/forum_posts/_forum_post.html.erb

@@ -11,7 +11,7 @@
     </div>
     <div class="content">
       <div class="prose">
-        <%= format_text(parse_embedded_tag_request_text(forum_post.body)) %>
+        <%= format_text(parse_embedded_tag_request_text(forum_post.body), data: dtext_data) %>
       </div>
       <%= render "application/update_notice", record: forum_post %>
       <menu>

app/views/forum_posts/_listing.html.erb

@@ -1,8 +1,9 @@
 <%- # forum_post %>
 <%- # original_forum_post_id %>
+<%- # dtext_data %>
 
 <div class="list-of-forum-posts list-of-messages">
   <% forum_posts.each do |forum_post| %>
-    <%= render "forum_posts/forum_post", :forum_post => forum_post, :original_forum_post_id => original_forum_post_id %>
+    <%= render "forum_posts/forum_post", forum_post: forum_post, original_forum_post_id: original_forum_post_id, dtext_data: dtext_data %>
   <% end %>
 </div>

app/views/forum_topics/show.html.erb

@@ -20,7 +20,7 @@
       </div>
     <% end %>
 
-    <%= render "forum_posts/listing", :forum_posts => @forum_posts, :original_forum_post_id => @forum_topic.original_post.id %>
+    <%= render "forum_posts/listing", forum_posts: @forum_posts, original_forum_post_id: @forum_topic.original_post.id, dtext_data: DText.preprocess(@forum_posts.map(&:body)) %>
 
     <% if CurrentUser.is_member? %>
       <% if CurrentUser.is_moderator? || !@forum_topic.is_locked? %>

test/unit/d_text_test.rb

@@ -38,5 +38,23 @@ class DTextTest < ActiveSupport::TestCase
         assert_strip_dtext("one two three\nfour\n\nfive six", "one [b]two[/b] three\nfour\n\nfive six")
       end
     end
+
+    context "#format_text" do
+      should "add tag types to wiki links" do
+        create(:tag, name: "bkub", category: Tag.categories.artist, post_count: 42)
+        assert_match(/tag-type-#{Tag.categories.artist}/, DText.format_text("[[bkub]]"))
+      end
+
+      should "mark links to nonexistent tags or wikis" do
+        create(:tag, name: "no_wiki", post_count: 42)
+        create(:tag, name: "empty_tag", post_count: 0)
+
+        assert_match(/dtext-wiki-does-not-exist/, DText.format_text("[[no wiki]]"))
+        assert_match(/dtext-tag-does-not-exist/, DText.format_text("[[no tag]]"))
+        assert_match(/dtext-tag-empty/, DText.format_text("[[empty tag]]"))
+
+        refute_match(/dtext-tag-does-not-exist/, DText.format_text("[[help:nothing]]"))
+      end
+    end
   end
 end