twitter: convert commentary to dtext.

* Convert hashtags and mentions to dtext links.
* Replace http://t.co urls to the actual url.
* Strip the http://t.co url linking to the tweet itself.

app/logical/sources/strategies/twitter.rb

@@ -46,6 +46,23 @@ module Sources::Strategies
       true
     end
 
+    def dtext_artist_commentary_desc
+      url_replacements = Array(api_response.attrs[:entities][:urls]).map do |url:, expanded_url:, **attrs|
+        [url, expanded_url]
+      end
+      url_replacements += Array(api_response.attrs[:entities][:media]).map do |url:, expanded_url:, **attrs|
+        [url, ""]
+      end
+      url_replacements = url_replacements.to_h
+
+      desc = artist_commentary_desc
+      desc = CGI::unescapeHTML(desc)
+      desc = desc.gsub(%r!https?://t\.co/[^[:space:]]+!i, url_replacements)
+      desc = desc.gsub(%r!#([^[:space:]]+)!, '"#\\1":[https://twitter.com/hashtag/\\1]')
+      desc = desc.gsub(%r!@([a-zA-Z0-9_]+)!, '"@\\1":[https://twitter.com/\\1]')
+      desc.strip
+    end
+
     def status_id_from_url(url)
       if url =~ %r{^https?://(?:mobile\.)?twitter\.com/\w+/status/(\d+)}
         $1.to_i

test/unit/sources/twitter_test.rb

@@ -89,5 +89,17 @@ module Sources
         end
       end
     end
+
+    context "A tweet" do
+      setup do
+        @site = Sources::Site.new("https://twitter.com/noizave/status/875768175136317440")
+        @site.get
+      end
+
+      should "convert urls, hashtags, and mentions to dtext" do
+        desc = 'test "#foo":[https://twitter.com/hashtag/foo] "#ホワイトデー":[https://twitter.com/hashtag/ホワイトデー] "@noizave":[https://twitter.com/noizave]\'s blah http://www.example.com <>& 😀'
+        assert_equal(desc, @site.dtext_artist_commentary_desc)
+      end
+    end
   end
 end