dtext links: add table for tracking links between wikis.

Add a dtext_links table for tracking links between wiki pages. This is
to allow for broken link detection and "what links here" searches, among
other uses.

app/logical/d_text.rb

@@ -77,6 +77,21 @@ class DText
       title = WikiPage.normalize_title(title)
       title
     end
+
+    titles.uniq
+  end
+
+  def self.parse_external_links(text)
+    html = DTextRagel.parse(text)
+    fragment = Nokogiri::HTML.fragment(html)
+
+    links = fragment.css("a.dtext-external-link").map { |node| node["href"] }
+    links.uniq
+  end
+
+  def self.dtext_links_differ?(a, b)
+    Set.new(parse_wiki_titles(a)) != Set.new(parse_wiki_titles(b)) ||
+    Set.new(parse_external_links(a)) != Set.new(parse_external_links(b))
   end
 
   def self.strip_blocks(string, tag)

app/models/dtext_link.rb

@@ -0,0 +1,27 @@
+class DtextLink < ApplicationRecord
+  belongs_to :model, polymorphic: true
+  enum link_type: [:wiki_link, :external_link]
+
+  before_validation :normalize_link_target
+  validates :link_target, uniqueness: { scope: [:model_type, :model_id] }
+
+  def self.new_from_dtext(dtext)
+    links = []
+
+    links += DText.parse_wiki_titles(dtext).map do |link|
+      DtextLink.new(link_type: :wiki_link, link_target: link)
+    end
+
+    links += DText.parse_external_links(dtext).map do |link|
+      DtextLink.new(link_type: :external_link, link_target: link)
+    end
+
+    links
+  end
+
+  def normalize_link_target
+    if wiki_link?
+      self.link_target = WikiPage.normalize_title(link_target)
+    end
+  end
+end

app/models/wiki_page.rb

@@ -5,6 +5,7 @@ class WikiPage < ApplicationRecord
 
   before_save :normalize_title
   before_save :normalize_other_names
+  before_save :update_dtext_links, if: :dtext_links_changed?
   after_save :create_version
   validates_uniqueness_of :title, :case_sensitive => false
   validates_presence_of :title
@@ -19,6 +20,7 @@ class WikiPage < ApplicationRecord
   has_one :tag, :foreign_key => "name", :primary_key => "title"
   has_one :artist, -> {where(:is_active => true)}, :foreign_key => "name", :primary_key => "title"
   has_many :versions, -> {order("wiki_page_versions.id ASC")}, :class_name => "WikiPageVersion", :dependent => :destroy
+  has_many :dtext_links, as: :model, dependent: :destroy
 
   api_attributes including: [:creator_name, :category_name]
 
@@ -197,6 +199,14 @@ class WikiPage < ApplicationRecord
     end
   end
 
+  def dtext_links_changed?
+    body_changed? && DText.dtext_links_differ?(body, body_was)
+  end
+
+  def update_dtext_links
+    self.dtext_links = DtextLink.new_from_dtext(body)
+  end
+
   def post_set
     @post_set ||= PostSets::WikiPage.new(title, 1, 4)
   end

db/migrate/20191023191749_create_dtext_links.rb

@@ -0,0 +1,13 @@
+class CreateDtextLinks < ActiveRecord::Migration[6.0]
+  def change
+    create_table :dtext_links do |t|
+      t.timestamps
+      t.references :model, polymorphic: true, null: false
+      t.integer :link_type, null: false
+      t.string :link_target, null: false
+
+      t.index :link_type
+      t.index :link_target, opclass: "text_pattern_ops"
+    end
+  end
+end

db/structure.sql

@@ -880,6 +880,40 @@ CREATE SEQUENCE public.dmails_id_seq
 ALTER SEQUENCE public.dmails_id_seq OWNED BY public.dmails.id;
 
 
+--
+-- Name: dtext_links; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.dtext_links (
+    id bigint NOT NULL,
+    created_at timestamp(6) without time zone NOT NULL,
+    updated_at timestamp(6) without time zone NOT NULL,
+    model_type character varying NOT NULL,
+    model_id bigint NOT NULL,
+    link_type integer NOT NULL,
+    link_target character varying NOT NULL
+);
+
+
+--
+-- Name: dtext_links_id_seq; Type: SEQUENCE; Schema: public; Owner: -
+--
+
+CREATE SEQUENCE public.dtext_links_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
+--
+-- Name: dtext_links_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
+--
+
+ALTER SEQUENCE public.dtext_links_id_seq OWNED BY public.dtext_links.id;
+
+
 --
 -- Name: favorite_groups; Type: TABLE; Schema: public; Owner: -
 --
@@ -3295,6 +3329,13 @@ ALTER TABLE ONLY public.dmail_filters ALTER COLUMN id SET DEFAULT nextval('publi
 ALTER TABLE ONLY public.dmails ALTER COLUMN id SET DEFAULT nextval('public.dmails_id_seq'::regclass);
 
 
+--
+-- Name: dtext_links id; Type: DEFAULT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.dtext_links ALTER COLUMN id SET DEFAULT nextval('public.dtext_links_id_seq'::regclass);
+
+
 --
 -- Name: favorite_groups id; Type: DEFAULT; Schema: public; Owner: -
 --
@@ -4352,6 +4393,14 @@ ALTER TABLE ONLY public.dmails
     ADD CONSTRAINT dmails_pkey PRIMARY KEY (id);
 
 
+--
+-- Name: dtext_links dtext_links_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.dtext_links
+    ADD CONSTRAINT dtext_links_pkey PRIMARY KEY (id);
+
+
 --
 -- Name: favorite_groups favorite_groups_pkey; Type: CONSTRAINT; Schema: public; Owner: -
 --
@@ -4926,6 +4975,27 @@ CREATE INDEX index_dmails_on_message_index ON public.dmails USING gin (message_i
 CREATE INDEX index_dmails_on_owner_id ON public.dmails USING btree (owner_id);
 
 
+--
+-- Name: index_dtext_links_on_link_target; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX index_dtext_links_on_link_target ON public.dtext_links USING btree (link_target text_pattern_ops);
+
+
+--
+-- Name: index_dtext_links_on_link_type; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX index_dtext_links_on_link_type ON public.dtext_links USING btree (link_type);
+
+
+--
+-- Name: index_dtext_links_on_model_type_and_model_id; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX index_dtext_links_on_model_type_and_model_id ON public.dtext_links USING btree (model_type, model_id);
+
+
 --
 -- Name: index_favorite_groups_on_creator_id; Type: INDEX; Schema: public; Owner: -
 --
@@ -7337,6 +7407,7 @@ INSERT INTO "schema_migrations" (version) VALUES
 ('20190908035317'),
 ('20190919175836'),
 ('20190923071044'),
-('20190926000912');
+('20190926000912'),
+('20191023191749');
 
 

test/unit/d_text_test.rb

@@ -56,5 +56,31 @@ class DTextTest < ActiveSupport::TestCase
         refute_match(/dtext-tag-does-not-exist/, DText.format_text("[[help:nothing]]"))
       end
     end
+
+    context "#parse_wiki_titles" do
+      should "parse wiki links in dtext" do
+        assert_equal(["foo"], DText.parse_wiki_titles("[[foo]] [[FOO]"))
+      end
+    end
+
+    context "#parse_external_links" do
+      should "parse external links in dtext" do
+        dtext = <<~EOS
+          * https://test1.com
+          * <https://test2.com>
+          * "test":https://test3.com
+          * "test":[https://test4.com]
+          * [https://test5.com](test)
+          * <a href="https://test6.com">test</a>
+        EOS
+
+        links = %w[
+          https://test1.com https://test2.com https://test3.com
+          https://test4.com https://test5.com https://test6.com
+        ]
+
+        assert_equal(links, DText.parse_external_links(dtext))
+      end
+    end
   end
 end

test/unit/wiki_page_test.rb

@@ -110,6 +110,21 @@ class WikiPageTest < ActiveSupport::TestCase
         version = WikiPageVersion.last
         assert_not_equal(@wiki_page.creator_id, version.updater_id)
       end
+
+      should "update its dtext links" do
+        @wiki_page.update!(body: "[[long hair]]")
+        assert_equal(1, @wiki_page.dtext_links.size)
+        assert_equal("wiki_link", @wiki_page.dtext_links.first.link_type)
+        assert_equal("long_hair", @wiki_page.dtext_links.first.link_target)
+
+        @wiki_page.update!(body: "http://www.google.com")
+        assert_equal(1, @wiki_page.dtext_links.size)
+        assert_equal("external_link", @wiki_page.dtext_links.first.link_type)
+        assert_equal("http://www.google.com", @wiki_page.dtext_links.first.link_target)
+
+        @wiki_page.update!(body: "nothing")
+        assert_equal(0, @wiki_page.dtext_links.size)
+      end
     end
   end
 end