diff --git a/app/logical/d_text.rb b/app/logical/d_text.rb index 0ac557ee3..2e7ef8714 100644 --- a/app/logical/d_text.rb +++ b/app/logical/d_text.rb @@ -369,6 +369,53 @@ class DText s end + def self.from_html(text, &block) + html = Nokogiri::HTML.fragment(text) + + dtext = html.children.map do |element| + block.call(element) if block.present? + + case element.name + when "text" + element.content + when "br" + "\n" + when "p" + from_html(element.inner_html, &block) + "\n\n" + when "blockquote" + "[quote]#{from_html(element.inner_html, &block)}[/quote]" if element.inner_html.present? + when "small", "sub" + "[tn]#{from_html(element.inner_html, &block)}[/tn]" if element.inner_html.present? + when "b", "strong" + "[b]#{from_html(element.inner_html, &block)}[/b]" if element.inner_html.present? + when "i", "em" + "[i]#{from_html(element.inner_html, &block)}[/i]" if element.inner_html.present? + when "u" + "[u]#{from_html(element.inner_html, &block)}[/u]" if element.inner_html.present? + when "s", "strike" + "[s]#{from_html(element.inner_html, &block)}[/s]" if element.inner_html.present? + when "li" + "* #{from_html(element.inner_html, &block)}" if element.inner_html.present? + when "h1", "h2", "h3", "h4", "h5", "h6" + hN = element.name + title = from_html(element.inner_html, &block) + "#{hN}. #{title}\n" + when "a" + title = from_html(element.inner_html, &block) + url = element["href"] + %("#{title}":[#{url}]) if title.present? && url.present? + when "img" + element.attributes["title"] || element.attributes["alt"] || "" + when "comment" + # ignored + else + from_html(element.inner_html, &block) + end + end.join + + dtext + end + # extract the first paragraph `needle` occurs in. def self.excerpt(dtext, needle) dtext = dtext.gsub(/\r\n|\r|\n/, "\n") diff --git a/app/logical/sources/strategies/deviant_art.rb b/app/logical/sources/strategies/deviant_art.rb index 2e9ea50b0..27e7f10f1 100644 --- a/app/logical/sources/strategies/deviant_art.rb +++ b/app/logical/sources/strategies/deviant_art.rb @@ -36,47 +36,11 @@ module Sources end def self.to_dtext(text) - html = Nokogiri::HTML.fragment(text) - - dtext = html.children.map do |element| - case element.name - when "text" - element.content - when "br" - "\n" - when "blockquote" - "[quote]#{to_dtext(element.inner_html)}[/quote]" if element.inner_html.present? - when "small", "sub" - "[tn]#{to_dtext(element.inner_html)}[/tn]" if element.inner_html.present? - when "b" - "[b]#{to_dtext(element.inner_html)}[/b]" if element.inner_html.present? - when "i" - "[i]#{to_dtext(element.inner_html)}[/i]" if element.inner_html.present? - when "u" - "[u]#{to_dtext(element.inner_html)}[/u]" if element.inner_html.present? - when "strike" - "[s]#{to_dtext(element.inner_html)}[/s]" if element.inner_html.present? - when "li" - "* #{to_dtext(element.inner_html)}" if element.inner_html.present? - when "h1", "h2", "h3", "h4", "h5", "h6" - hN = element.name - title = to_dtext(element.inner_html) - "#{hN}. #{title}\n" - when "a" - title = to_dtext(element.inner_html) - url = element.attributes["href"].value - url = url.gsub(%r!\Ahttps?://www\.deviantart\.com/users/outgoing\?!i, "") - %("#{title}":[#{url}]) if title.present? - when "img" - element.attributes["title"] || element.attributes["alt"] || "" - when "comment" - # ignored - else - to_dtext(element.inner_html) + DText.from_html(text) do |element| + if element.name == "a" && element["href"].present? + element["href"] = element["href"].gsub(%r!\Ahttps?://www\.deviantart\.com/users/outgoing\?!i, "") end - end.join - - dtext + end end protected diff --git a/app/logical/sources/strategies/nijie.rb b/app/logical/sources/strategies/nijie.rb index 71de9869d..53e32d21e 100644 --- a/app/logical/sources/strategies/nijie.rb +++ b/app/logical/sources/strategies/nijie.rb @@ -50,26 +50,9 @@ module Sources protected - # XXX: duplicated from strategies/deviant_art.rb. def self.to_dtext(text) - html = Nokogiri::HTML.fragment(text) - - dtext = html.children.map do |element| - case element.name - when "text" - element.content - when "strong" - "[b]#{to_dtext(element.inner_html)}[/b]" if element.inner_html.present? - when "i" - "[i]#{to_dtext(element.inner_html)}[/i]" if element.inner_html.present? - when "s" - "[s]#{to_dtext(element.inner_html)}[/s]" if element.inner_html.present? - else - to_dtext(element.inner_html) - end - end.join - - dtext + text = text.gsub(/\r\n|\r/, "
") + DText.from_html(text).strip end def get_commentary_from_page(page) diff --git a/app/logical/sources/strategies/pawoo.rb b/app/logical/sources/strategies/pawoo.rb index 1605332e0..6c27eea80 100644 --- a/app/logical/sources/strategies/pawoo.rb +++ b/app/logical/sources/strategies/pawoo.rb @@ -55,32 +55,13 @@ module Sources::Strategies end def dtext_artist_commentary_desc - to_dtext(artist_commentary_desc) - end - - def to_dtext(text) - html = Nokogiri::HTML.fragment(text) - - dtext = html.children.map do |element| - case element.name - when "text" - element.content - when "p" - to_dtext(element.inner_html) + "\n\n" - when "a" + DText.from_html(artist_commentary_desc) do |element| + if element.name == "a" # don't include links to the toot itself. media_urls = api_response.json["media_attachments"].map { |attr| attr["text_url"] } - next if element.attribute("href").value.in?(media_urls) - - title = to_dtext(element.inner_html) - url = element.attributes["href"].value - %("#{title}":[#{url}]) - else - to_dtext(element.inner_html) + element["href"] = nil if element["href"].in?(media_urls) end - end.join.strip - - dtext + end.strip end end end diff --git a/test/unit/sources/nijie_test.rb b/test/unit/sources/nijie_test.rb index 39dafb344..6e1528116 100644 --- a/test/unit/sources/nijie_test.rb +++ b/test/unit/sources/nijie_test.rb @@ -92,8 +92,8 @@ module Sources should "get the dtext-ified commentary" do desc = <<-EOS.strip_heredoc.chomp - foo [b]bold[/b] [i]italics[/i] [s]strike[/s] red\r - \r + foo [b]bold[/b] [i]italics[/i] [s]strike[/s] red + http://nijie.info/view.php?id=218944 EOS