refactored nico seiga sources

Gemfile

@@ -4,7 +4,7 @@ group :test do
   gem "shoulda"
   gem "factory_girl"
   gem "mocha", :require => "mocha/setup"
-  gem "ffaker", :git => "http://github.com/EmmanuelOga/ffaker.git"
+  gem "ffaker"
   gem "simplecov", :require => false
   gem "pry"
   gem "vcr"
@@ -25,10 +25,10 @@ gem "dalli"
 gem "delayed_job"
 gem "delayed_job_active_record"
 gem "simple_form"
-gem "mechanize", :git => 'git://github.com/caribio/mechanize.git'
+gem "mechanize"
 gem "nokogiri"
 gem "whenever", :require => false
-gem "sanitize", :git => "git://github.com/rgrove/sanitize.git"
+gem "sanitize"
 gem 'rmagick', :require => "RMagick"
 gem 'daemons'
 gem 'net-ssh'

Gemfile.lock

@@ -1,28 +1,3 @@
-GIT
-  remote: git://github.com/caribio/mechanize.git
-  revision: 602a453f25c376681317e1f1af7229d39ccfff3d
-  specs:
-    mechanize (2.1.1)
-      domain_name (~> 0.5, >= 0.5.1)
-      net-http-digest_auth (~> 1.1, >= 1.1.1)
-      net-http-persistent (~> 2.3, >= 2.3.2)
-      nokogiri (~> 1.4)
-      ntlm-http (~> 0.1, >= 0.1.1)
-      webrobots (~> 0.0, >= 0.0.9)
-
-GIT
-  remote: git://github.com/rgrove/sanitize.git
-  revision: 82f132a9521b3c81421faf77886d82ebbe24783b
-  specs:
-    sanitize (2.0.4)
-      nokogiri (>= 1.4.4, < 1.6)
-
-GIT
-  remote: http://github.com/EmmanuelOga/ffaker.git
-  revision: ff12b3e71f51c71893e3fd593957463fa09f1f5d
-  specs:
-    ffaker (1.15.0)
-
 GIT
   remote: https://github.com/r888888888/ruby-imagespec.git
   revision: 2dab9811f4abb4fbaeea66feb42e388ba545b2d8
@@ -60,75 +35,88 @@ GEM
     activesupport (3.2.12)
       i18n (~> 0.6)
       multi_json (~> 1.0)
-    addressable (2.3.4)
+    addressable (2.3.5)
     arel (3.0.2)
     awesome_print (1.1.0)
     aws-s3 (0.6.3)
       builder
       mime-types
       xml-simple
-    bcrypt-ruby (3.0.1)
+    bcrypt-ruby (3.1.1)
-    bourne (1.4.0)
-      mocha (~> 0.13.2)
     builder (3.0.4)
-    capistrano (2.14.2)
+    capistrano (2.15.5)
       highline
       net-scp (>= 1.0.0)
       net-sftp (>= 2.0.0)
       net-ssh (>= 2.0.14)
       net-ssh-gateway (>= 1.1.0)
-    capistrano-unicorn (0.1.7)
+    capistrano-unicorn (0.1.10)
       capistrano
     chronic (0.9.1)
     coderay (1.0.9)
-    crack (0.3.2)
+    crack (0.4.1)
+      safe_yaml (~> 0.9.0)
     daemons (1.1.9)
     dalli (2.6.4)
-    delayed_job (3.0.5)
+    delayed_job (4.0.0)
-      activesupport (~> 3.0)
+      activesupport (>= 3.0, < 4.1)
-    delayed_job_active_record (0.4.4)
+    delayed_job_active_record (4.0.0)
-      activerecord (>= 2.1.0, < 4)
+      activerecord (>= 3.0, < 4.1)
-      delayed_job (~> 3.0)
+      delayed_job (>= 3.0, < 4.1)
-    diff-lcs (1.2.2)
+    diff-lcs (1.2.4)
-    domain_name (0.5.10)
+    domain_name (0.5.12)
       unf (>= 0.0.5, < 1.0.0)
     erubis (2.7.0)
     execjs (1.4.0)
       multi_json (~> 1.0)
     factory_girl (4.2.0)
       activesupport (>= 3.0.0)
-    highline (1.6.18)
+    ffaker (1.18.0)
-    hike (1.2.1)
+    highline (1.6.19)
+    hike (1.2.3)
+    http-cookie (1.0.1)
+      domain_name (~> 0.5)
     i18n (0.6.4)
     journey (1.0.4)
-    json (1.7.7)
+    json (1.8.0)
     kgio (2.8.0)
     libv8 (3.11.8.17)
     mail (2.4.4)
       i18n (>= 0.4.0)
       mime-types (~> 1.16)
       treetop (~> 1.4.8)
+    mechanize (2.7.1)
+      domain_name (~> 0.5, >= 0.5.1)
+      http-cookie (~> 1.0.0)
+      mime-types (~> 1.17, >= 1.17.2)
+      net-http-digest_auth (~> 1.1, >= 1.1.1)
+      net-http-persistent (~> 2.5, >= 2.5.2)
+      nokogiri (~> 1.4)
+      ntlm-http (~> 0.1, >= 0.1.1)
+      webrobots (>= 0.0.9, < 0.2)
     metaclass (0.0.1)
-    method_source (0.8.1)
+    method_source (0.8.2)
-    mime-types (1.22)
+    mime-types (1.23)
-    mocha (0.13.3)
+    mini_portile (0.5.1)
+    mocha (0.14.0)
       metaclass (~> 0.0.1)
-    multi_json (1.7.2)
+    multi_json (1.7.8)
-    net-http-digest_auth (1.3)
+    net-http-digest_auth (1.4)
-    net-http-persistent (2.8)
+    net-http-persistent (2.9)
-    net-scp (1.1.0)
+    net-scp (1.1.2)
       net-ssh (>= 2.6.5)
-    net-sftp (2.1.1)
+    net-sftp (2.1.2)
       net-ssh (>= 2.6.5)
-    net-ssh (2.6.6)
+    net-ssh (2.6.8)
     net-ssh-gateway (1.2.0)
       net-ssh (>= 2.6.5)
-    newrelic_rpm (3.6.0.78)
+    newrelic_rpm (3.6.6.147)
-    nokogiri (1.5.9)
+    nokogiri (1.6.0)
+      mini_portile (~> 0.5.0)
     ntlm-http (0.1.1)
     pg (0.12.2)
     polyglot (0.3.3)
-    pry (0.9.12)
+    pry (0.9.12.2)
       coderay (~> 1.0.5)
       method_source (~> 0.8)
       slop (~> 3.4)
@@ -154,25 +142,27 @@ GEM
       rake (>= 0.8.7)
       rdoc (~> 3.4)
       thor (>= 0.14.6, < 2.0)
-    raindrops (0.10.0)
+    raindrops (0.11.0)
-    rake (10.0.4)
+    rake (10.1.0)
     rdoc (3.12.2)
       json (~> 1.4)
-    ref (1.0.4)
+    ref (1.0.5)
     rmagick (2.13.2)
     ruby-prof (0.13.0)
-    sass (3.2.7)
+    safe_yaml (0.9.5)
+    sanitize (2.0.6)
+      nokogiri (>= 1.4.4)
+    sass (3.2.10)
     sass-rails (3.2.6)
       railties (~> 3.2.0)
       sass (>= 3.1.10)
       tilt (~> 1.3)
-    shoulda (3.4.0)
+    shoulda (3.5.0)
       shoulda-context (~> 1.0, >= 1.0.1)
-      shoulda-matchers (~> 1.0, >= 1.4.1)
+      shoulda-matchers (>= 1.4.1, < 3.0)
-    shoulda-context (1.1.0)
+    shoulda-context (1.1.4)
-    shoulda-matchers (1.5.6)
+    shoulda-matchers (2.2.0)
       activesupport (>= 3.0.0)
-      bourne (~> 1.3)
     simple_form (2.1.0)
       actionpack (~> 3.0)
       activemodel (~> 3.0)
@@ -180,40 +170,42 @@ GEM
       multi_json (~> 1.0)
       simplecov-html (~> 0.7.1)
     simplecov-html (0.7.1)
-    slop (3.4.4)
+    slop (3.4.6)
     sprockets (2.2.2)
       hike (~> 1.2)
       multi_json (~> 1.0)
       rack (~> 1.0)
       tilt (~> 1.1, != 1.3.0)
     statistics2 (0.54)
-    term-ansicolor (1.1.4)
+    term-ansicolor (1.2.2)
+      tins (~> 0.8)
     therubyracer (0.11.4)
       libv8 (~> 3.11.8.12)
       ref
     thor (0.18.1)
-    tilt (1.3.6)
+    tilt (1.4.1)
-    timecop (0.6.1)
+    timecop (0.6.3)
-    treetop (1.4.12)
+    tins (0.8.3)
+    treetop (1.4.14)
       polyglot
       polyglot (>= 0.3.1)
     tzinfo (0.3.37)
-    uglifier (1.3.0)
+    uglifier (2.1.2)
       execjs (>= 0.3.0)
       multi_json (~> 1.0, >= 1.0.2)
     unf (0.1.1)
       unf_ext
     unf_ext (0.0.6)
-    unicorn (4.6.2)
+    unicorn (4.6.3)
       kgio (~> 2.6)
       rack
       raindrops (~> 0.7)
-    vcr (2.4.0)
+    vcr (2.5.0)
-    webmock (1.11.0)
+    webmock (1.13.0)
       addressable (>= 2.2.7)
       crack (>= 0.3.2)
     webrobots (0.1.1)
-    whenever (0.8.2)
+    whenever (0.8.4)
       activesupport (>= 2.3.4)
       chronic (>= 0.6.3)
     xml-simple (1.1.2)
@@ -232,9 +224,9 @@ DEPENDENCIES
   delayed_job_active_record
   diff-lcs
   factory_girl
-  ffaker!
+  ffaker
   kgio
-  mechanize!
+  mechanize
   mocha
   net-sftp
   net-ssh
@@ -246,7 +238,7 @@ DEPENDENCIES
   rmagick
   ruby-imagespec!
   ruby-prof
-  sanitize!
+  sanitize
   sass-rails
   shoulda
   simple_form

app/logical/sources/strategies/nico_seiga.rb

@@ -25,7 +25,7 @@ module Sources
     protected
 
       def get_profile_from_page(page)
-        links = page.search("div.illust_user_name a")
+        links = page.search("li a").select {|x| x["href"] =~ /user\/illust/}
 
         if links.any?
           profile_url = "http://seiga.nicovideo.jp" + links[0]["href"]
@@ -39,17 +39,17 @@ module Sources
       end
 
       def get_image_url_from_page(page)
-        links = page.search("a#illust_link")
+        meta = page.search("meta[property='og:image']")
 
-        if links.any?
+        if meta.any?
-          "http://seiga.nicovideo.jp" + links[0]["href"]
+          meta[0]["content"]
         else
           nil
         end
       end
 
       def get_tags_from_page(page)
-        links = page.search("div#tag_block nobr a.tag")
+        links = page.search("a.tag")
 
         links.map do |node|
           [node.text, "http://seiga.nicovideo.jp" + node.attr("href")]

test/unit/sources/nico_seiga_test.rb

@@ -6,35 +6,45 @@ module Sources
   class NicoSeigaTest < ActiveSupport::TestCase
     context "The source site for nico seiga" do
       setup do
-        # Sources::Strategies::NicoSeiga.new("http://lohas.nicoseiga.jp/priv/2853566?e=1361296671&h=794b3686b02edfd64c22ed2f99a4c55650371854")
+        VCR.use_cassette("source-nico-seiga-unit-test-1", :record => :new_episodes) do
+          @site_1 = Sources::Site.new("http://lohas.nicoseiga.jp/priv/3329388?e=1375906127&h=dc01a9bfc7d1745d700aa8022314b9c3e8c145dd")
+        end
 
-        VCR.use_cassette("source-nico-seiga-unit-test", :record => :new_episodes) do
+        VCR.use_cassette("source-nico-seiga-unit-test-2", :record => :new_episodes) do
-          @site = Sources::Site.new("http://seiga.nicovideo.jp/seiga/im1464351?track=ranking")
+          @site_2 = Sources::Site.new("http://seiga.nicovideo.jp/seiga/im3324796")
         end
       end
 
       should "get the profile" do
-        assert_equal("http://seiga.nicovideo.jp/user/illust/20446930?target=seiga", @site.profile_url)
+        assert_equal("http://seiga.nicovideo.jp/user/illust/17586868", @site_1.profile_url)
+        assert_equal("http://seiga.nicovideo.jp/user/illust/341034", @site_2.profile_url)
       end
 
       should "get the artist name" do
-        assert_equal("rariemonn", @site.artist_name)
+        assert(@site_1.artist_name =~ /./)
+        assert(@site_2.artist_name =~ /./)
       end
 
       should "get the image url" do
-        assert_equal("http://seiga.nicovideo.jp/image/source?id=1464351", @site.image_url)
+        assert_equal("http://lohas.nicoseiga.jp/thumb/3329388i?", @site_1.image_url)
+        assert_equal("http://lohas.nicoseiga.jp/thumb/3324796i?", @site_2.image_url)
       end
 
       should "get the tags" do
-        assert(@site.tags.size > 0)
+        assert(@site_1.tags.size > 0)
-        first_tag = @site.tags.first
+        first_tag = @site_1.tags.first
         assert_equal(2, first_tag.size)
         assert(first_tag[0] =~ /./)
+
+        assert(@site_2.tags.size > 0)
       end
 
       should "convert a page into a json representation" do
         assert_nothing_raised do
-          @site.to_json
+          @site_1.to_json
+        end
+        assert_nothing_raised do
+          @site_2.to_json
         end
       end
     end