support pool version archive

.gitignore

@@ -20,3 +20,4 @@ danbooru.db
 iqdb.db
 danbooru.sublime-project
 danbooru.sublime-workspace
+script/scratch.rb

README.md

@@ -43,3 +43,7 @@ IQDB integration is now delegated to the [IQDBS service](https://github.com/r888
 
 You will need to install your own copy and enable the appropriate 
 configuration settings.
+
+### Archive Service
+
+In order to access versioned data for pools (and eventually posts) you will need to install and configure the [Archive service](https://github.com/r888888888/archives).

app/controllers/pool_versions_controller.rb

@@ -1,16 +1,35 @@
 class PoolVersionsController < ApplicationController
   respond_to :html, :xml, :json
+  before_filter :check_availabililty
 
   def index
     if params[:search] && params[:search][:pool_id].present?
       @pool = Pool.find(params[:search][:pool_id])
     end
 
-    @pool_versions = PoolVersion.search(params[:search]).order("updated_at desc").paginate(params[:page], :limit => params[:limit], :search_count => params[:search])
+    @pool_versions = PoolArchive.search(params[:search]).order("updated_at desc").paginate(params[:page], :limit => params[:limit], :search_count => params[:search])
     respond_with(@pool_versions) do |format|
       format.xml do
         render :xml => @pool_versions.to_xml(:root => "pool-versions")
       end
     end
   end
+
+private
+
+  def check_availabililty
+    if !PoolArchive.enabled?
+      respond_to do |format|
+        format.html do
+          flash[:notice] = "Archive service is not configured. Pool versions are not saved."
+          redirect_to :back
+        end
+        format.json do
+          render json: {success: false, reason: "Archive service is not configured"}.to_json, status: 501
+        end
+      end
+
+      return false
+    end
+  end
 end

app/helpers/pool_versions_helper.rb

@@ -2,13 +2,13 @@ module PoolVersionsHelper
   def pool_version_diff(pool_version)
     html = ""
 
-    html << pool_version.changes[:added_posts].map do |post_id|
+    html << pool_version.added_post_ids.map do |post_id|
       '<ins><a href="/posts/' + post_id.to_s + '">' + post_id.to_s + '</a></ins>'
     end.join(" ")
 
     html << " "
 
-    html << pool_version.changes[:removed_posts].map do |post_id|
+    html << pool_version.removed_post_ids.map do |post_id|
       '<del><a href="/posts/' + post_id.to_s + '">' + post_id.to_s + '</a></del>'
     end.join(" ")
 

app/models/pool.rb

@@ -11,7 +11,6 @@ class Pool < ActiveRecord::Base
   validate :updater_can_remove_posts
   belongs_to :creator, :class_name => "User"
   belongs_to :updater, :class_name => "User"
-  has_many :versions, lambda {order("pool_versions.id ASC")}, :class_name => "PoolVersion", :dependent => :destroy
   before_validation :normalize_post_ids
   before_validation :normalize_name
   before_validation :initialize_is_active, :on => :create
@@ -158,6 +157,14 @@ class Pool < ActiveRecord::Base
     end
   end
 
+  def versions
+    if PoolArchive.enabled?
+      PoolArchive.where("pool_id = ?", id).order("id asc")
+    else
+      raise "Archive service not configured"
+    end
+  end
+
   def is_series?
     category == "series"
   end
@@ -200,8 +207,10 @@ class Pool < ActiveRecord::Base
       raise RevertError.new("You cannot revert to a previous version of another pool.")
     end
 
-    self.post_ids = version.post_ids
+    self.post_ids = version.post_ids.join(" ")
     self.name = version.name
+    self.description = version.description
+
     synchronize!
   end
 
@@ -294,7 +303,7 @@ class Pool < ActiveRecord::Base
 
   def synchronize!
     synchronize
-    save
+    save if post_ids_changed?
   end
 
   def post_id_array
@@ -336,17 +345,10 @@ class Pool < ActiveRecord::Base
   end
 
   def create_version(force = false)
-    if post_ids_changed? || name_changed? || description_changed? || is_active_changed? || is_deleted_changed? || category_changed? || force
+    if PoolArchive.enabled?
-      last_version = versions.last
+      PoolArchive.queue(self)
-
+    else
-      if last_version && last_version.updater_ip_addr == CurrentUser.ip_addr && CurrentUser.user.id == last_version.updater_id && last_version.created_at > 1.hour.ago
+      Rails.logger.warn("Archive service is not configured. Pool versions will not be saved.")
-        # merge
-        last_version.update_column(:post_ids, post_ids)
-        last_version.update_column(:name, name)
-      else
-        # create
-        versions.create(:post_ids => post_ids, :name => name)
-      end
     end
   end
 

app/models/pool_archive.rb

@@ -0,0 +1,77 @@
+class PoolArchive < ActiveRecord::Base
+  establish_connection "archive_#{Rails.env}".to_sym
+  self.table_name = "pool_versions"
+
+  module SearchMethods
+    def for_user(user_id)
+      where("updater_id = ?", user_id)
+    end
+
+    def search(params)
+      q = where("true")
+      return q if params.blank?
+
+      if params[:updater_id].present?
+        q = q.for_user(params[:updater_id].to_i)
+      end
+
+      if params[:updater_name].present?
+        q = q.where("updater_id = ?", User.name_to_id(params[:updater_name]))
+      end
+
+      if params[:pool_id].present?
+        q = q.where("pool_id = ?", params[:pool_id].to_i)
+      end
+
+      q
+    end
+  end
+
+  extend SearchMethods
+
+  def self.enabled?
+    Danbooru.config.aws_sqs_archives_url.present?
+  end
+
+  def self.sqs_service
+    SqsService.new(Danbooru.config.aws_sqs_archives_url)
+  end
+
+  def self.queue(pool)
+    # queue updates to sqs so that if archives goes down for whatever reason it won't
+    # block pool updates
+    raise "Archive service is not configured" if !enabled?
+
+    json = {
+      pool_id: pool.id,
+      post_ids: pool.post_ids.scan(/\d+/).map(&:to_i),
+      updater_id: CurrentUser.id,
+      updater_ip_addr: CurrentUser.ip_addr.to_s,
+      created_at: pool.created_at.try(:iso8601),
+      updated_at: pool.updated_at.try(:iso8601),
+      description: pool.description,
+      name: pool.name,
+      is_active: pool.is_active?,
+      is_deleted: pool.is_deleted?,
+      category: pool.category
+    }
+    msg = "add pool version\n#{json.to_json}"
+    sqs_service.send_message(msg)
+  end
+
+  def pool
+    Pool.find(pool_id)
+  end
+
+  def updater
+    User.find(updater_id)
+  end
+
+  def updater_name
+    User.id_to_name(updater_id)
+  end
+
+  def pretty_name
+    name.tr("_", " ")
+  end
+end

app/models/pool_version.rb

@@ -34,6 +34,46 @@ class PoolVersion < ActiveRecord::Base
 
   extend SearchMethods
 
+  def self.export_to_archives(starting_version_id = 0)
+    raise "SQS URL not setup" if Danbooru.config.aws_sqs_archives_url.nil?
+
+    credentials = Aws::Credentials.new(
+      Danbooru.config.aws_access_key_id,
+      Danbooru.config.aws_secret_access_key
+    )
+    sqs = Aws::SQS::Client.new(
+      credentials: credentials,
+      region: Danbooru.config.aws_sqs_region
+    )
+    last_version_id = 0
+
+    where("id > ?", starting_version_id).find_each do |version|
+      last_version_id = version.id
+
+      json = {
+        id: version.id,
+        pool_id: version.pool_id,
+        post_ids: version.post_ids.scan(/\d+/).map(&:to_i),
+        updater_id: version.updater_id,
+        updater_ip_addr: version.updater_ip_addr.to_s,
+        created_at: version.created_at.try(:iso8601),
+        updated_at: version.updated_at.try(:iso8601),
+        description: version.pool.description,
+        name: version.name,
+        is_active: version.pool.is_active?,
+        is_deleted: version.pool.is_deleted?,
+        category: version.pool.category
+      }
+      msg = "add pool version\n#{json.to_json}"
+      sqs.send_message(
+        message_body: msg,
+        queue_url: Danbooru.config.aws_sqs_archives_url
+      )
+    end
+
+    puts "last version id: #{last_version_id}"
+  end
+
   def updater_name
     User.id_to_name(updater_id)
   end

app/presenters/user_presenter.rb

@@ -178,7 +178,11 @@ class UserPresenter
   end
 
   def pool_version_count(template)
-    template.link_to(user.pool_version_count, template.pool_versions_path(:search => {:updater_id => user.id}))
+    if PoolArchive.enabled?
+      template.link_to(user.pool_version_count, template.pool_versions_path(:search => {:updater_id => user.id}))
+    else
+      "N/A"
+    end
   end
 
   def inviter(template)

app/views/pool_versions/index.html.erb

@@ -22,9 +22,9 @@
         <% @pool_versions.each do |pool_version| %>
           <tr>
             <td><%= link_to pool_version.pretty_name, pool_path(pool_version.pool_id), :class => "pool-category-#{pool_version.pool.category}" %></td>
-            <td><%= link_to pool_version.post_id_array.size, pool_versions_path(:search => {:pool_id => pool_version.pool_id}) %></td>
+            <td><%= link_to pool_version.post_ids.size, pool_versions_path(:search => {:pool_id => pool_version.pool_id}) %></td>
             <td><%= pool_version_diff(pool_version) %></td>
-            <td><%= link_to_user pool_version.updater%></td>
+            <td><%= link_to_user pool_version.updater %></td>
             <% if CurrentUser.is_moderator? %>
               <td>
                 <%= pool_version.updater_ip_addr %>

app/views/pools/_secondary_links.html.erb

@@ -19,7 +19,9 @@
           <li><%= link_to "Delete", pool_path(@pool), :method => :delete, :data => {:confirm => "Are you sure you want to delete this pool?"}, :remote => true %></li>
         <% end %>
       <% end %>
-      <li><%= link_to "History", pool_versions_path(:search => {:pool_id => @pool.id}) %></li>
+      <% if PoolArchive.enabled? %>
+        <li><%= link_to "History", pool_versions_path(:search => {:pool_id => @pool.id}) %></li>
+      <% end %>
       <% if @pool.post_count <= 100 && CurrentUser.is_member? %>
         <li><%= link_to "Order", edit_pool_order_path(@pool) %></li>
       <% end %>

app/views/static/site_map.html.erb

@@ -53,7 +53,9 @@
         <li><h1>Pools</h1></li>
         <li><%= link_to("Help", wiki_pages_path(:title => "help:pools")) %></li>
         <li><%= link_to("Listing", pools_path) %></li>
-        <li><%= link_to("Changes", pool_versions_path) %></li>
+        <% if PoolArchive.enabled? %>
+          <li><%= link_to("Changes", pool_versions_path) %></li>
+        <% end %>
       </ul>
       <ul>
         <li><h1>Reports</h1></li>

app/views/users/_statistics.html.erb

@@ -84,10 +84,12 @@
         <td><%= presenter.artist_commentary_version_count(self) %></td>
       </tr>
 
-      <tr>
+      <% if PoolArchive.enabled? %>
-        <th>Pool Changes</th>
+        <tr>
-        <td><%= presenter.pool_version_count(self) %></td>
+          <th>Pool Changes</th>
-      </tr>
+          <td><%= presenter.pool_version_count(self) %></td>
+        </tr>
+      <% end %>
 
       <tr>
         <th>Forum Posts</th>

config/danbooru_default_config.rb

@@ -494,5 +494,8 @@ module Danbooru
 
     def aws_sqs_iqdb_url
     end
+
+    def aws_sqs_archives_url
+    end
   end
 end

test/functional/pool_elements_controller_test.rb

@@ -1,8 +1,13 @@
 require 'test_helper'
+require 'helpers/pool_archive_test_helper'
 
 class PoolElementsControllerTest < ActionController::TestCase
+  include PoolArchiveTestHelper
+
   context "The pools posts controller" do
     setup do
+      mock_pool_archive_service!
+      start_pool_archive_transaction
       @user = Timecop.travel(1.month.ago) {FactoryGirl.create(:user)}
       @mod = FactoryGirl.create(:moderator_user)
       CurrentUser.user = @user
@@ -12,6 +17,7 @@ class PoolElementsControllerTest < ActionController::TestCase
     end
 
     teardown do
+      rollback_pool_archive_transaction
       CurrentUser.user = nil
     end
 

test/functional/pool_versions_controller_test.rb

@@ -1,14 +1,20 @@
 require 'test_helper'
+require 'helpers/pool_archive_test_helper'
 
 class PoolVersionsControllerTest < ActionController::TestCase
+  include PoolArchiveTestHelper
+
   context "The pool versions controller" do
     setup do
+      mock_pool_archive_service!
+      start_pool_archive_transaction
       @user = FactoryGirl.create(:user)
       CurrentUser.user = @user
       CurrentUser.ip_addr = "127.0.0.1"
     end
 
     teardown do
+      rollback_pool_archive_transaction
       CurrentUser.user = nil
       CurrentUser.ip_addr = nil
     end

test/functional/pools_controller_test.rb

@@ -1,6 +1,9 @@
 require 'test_helper'
+require 'helpers/pool_archive_test_helper'
 
 class PoolsControllerTest < ActionController::TestCase
+  include PoolArchiveTestHelper
+
   context "The pools controller" do
     setup do
       Timecop.travel(1.month.ago) do
@@ -10,9 +13,12 @@ class PoolsControllerTest < ActionController::TestCase
       CurrentUser.user = @user
       CurrentUser.ip_addr = "127.0.0.1"
       @post = FactoryGirl.create(:post)
+      mock_pool_archive_service!
+      start_pool_archive_transaction
     end
 
     teardown do
+      rollback_pool_archive_transaction
       CurrentUser.user = nil
     end
 
@@ -99,10 +105,10 @@ class PoolsControllerTest < ActionController::TestCase
       end
 
       should "revert to a previous version" do
-        assert_equal(2, PoolVersion.count)
+        assert_equal(2, PoolArchive.count)
         @pool.reload
         version = @pool.versions.first
-        assert_equal("#{@post.id}", version.post_ids)
+        assert_equal([@post.id], version.post_ids)
         post :revert, {:id => @pool.id, :version_id => version.id}, {:user_id => @mod.id}
         @pool.reload
         assert_equal([@post.id], @pool.post_id_array)
@@ -111,7 +117,7 @@ class PoolsControllerTest < ActionController::TestCase
       should "not allow reverting to a previous version of another pool" do
         @pool2 = FactoryGirl.create(:pool)
 
-        post :revert, { :id => @pool.id, :version_id => @pool2.versions(true).first.id }, {:user_id => @user.id}
+        post :revert, { :id => @pool.id, :version_id => @pool2.versions.first.id }, {:user_id => @user.id}
         @pool.reload
 
         assert_not_equal(@pool.name, @pool2.name)

test/helpers/bulk_update_requests_helper_test.rb

@@ -1,4 +0,0 @@
-require 'test_helper'
-
-class BulkUpdateRequestsHelperTest < ActionView::TestCase
-end

test/helpers/pool_archive_test_helper.rb

@@ -0,0 +1,26 @@
+module PoolArchiveTestHelper
+  def mock_pool_archive_service!
+    mock_sqs_service = Class.new do
+      def send_message(msg)
+        _, json = msg.split(/\n/)
+        json = JSON.parse(json)
+        prev = PoolArchive.where(pool_id: json["pool_id"]).order("id desc").first
+        if prev && prev.updater_ip_addr.to_s == json["updater_ip_addr"]
+          prev.update_columns(json)
+        else
+          PoolArchive.create(json)
+        end
+      end
+    end
+
+    PoolArchive.stubs(:sqs_service).returns(mock_sqs_service.new)
+  end
+
+  def start_pool_archive_transaction
+    PoolArchive.connection.begin_transaction joinable: false
+  end
+
+  def rollback_pool_archive_transaction
+    PoolArchive.connection.rollback_transaction
+  end
+end

test/helpers/saved_searches_helper_test.rb

@@ -1,4 +0,0 @@
-require 'test_helper'
-
-class SavedSearchesHelperTest < ActionView::TestCase
-end

test/unit/pool_test.rb

@@ -1,18 +1,26 @@
 # encoding: utf-8
 
 require 'test_helper'
+require 'helpers/pool_archive_test_helper'
 
 class PoolTest < ActiveSupport::TestCase
+  include PoolArchiveTestHelper
+
   setup do
     Timecop.travel(1.month.ago) do
-      user = FactoryGirl.create(:user)
+      @user = FactoryGirl.create(:user)
-      CurrentUser.user = user
+      CurrentUser.user = @user
     end
+
     CurrentUser.ip_addr = "127.0.0.1"
     MEMCACHE.flush_all
+
+    mock_pool_archive_service!
+    start_pool_archive_transaction
   end
 
   teardown do
+    rollback_pool_archive_transaction
     CurrentUser.user = nil
     CurrentUser.ip_addr = nil
   end
@@ -53,33 +61,41 @@ class PoolTest < ActiveSupport::TestCase
 
   context "Reverting a pool" do
     setup do
+      PoolArchive.stubs(:enabled?).returns(true)
+
       @pool = FactoryGirl.create(:pool)
       @p1 = FactoryGirl.create(:post)
       @p2 = FactoryGirl.create(:post)
       @p3 = FactoryGirl.create(:post)
-      CurrentUser.ip_addr = "1.2.3.4"
+      CurrentUser.scoped(@user, "1.2.3.4") do
-      @pool.add!(@p1)
+        @pool.add!(@p1)
-      CurrentUser.ip_addr = "1.2.3.5"
+      end
-      @pool.add!(@p2)
+      CurrentUser.scoped(@user, "1.2.3.5") do
-      CurrentUser.ip_addr = "1.2.3.6"
+        @pool.add!(@p2)
-      @pool.add!(@p3)
+      end
-      CurrentUser.ip_addr = "1.2.3.7"
+      CurrentUser.scoped(@user, "1.2.3.6") do
-      @pool.remove!(@p1)
+        @pool.add!(@p3)
-      CurrentUser.ip_addr = "1.2.3.8"
+      end
-      @pool.revert_to!(@pool.versions.all[1])
+      CurrentUser.scoped(@user, "1.2.3.7") do
+        @pool.remove!(@p1)
+      end
+      CurrentUser.scoped(@user, "1.2.3.8") do
+        version = @pool.versions[1]
+        @pool.revert_to!(version)
+      end
     end
 
     should "have the correct versions" do
       assert_equal(6, @pool.versions.size)
-      assert_equal("", @pool.versions.all[0].post_ids)
+      assert_equal([], @pool.versions.all[0].post_ids)
-      assert_equal("#{@p1.id}", @pool.versions.all[1].post_ids)
+      assert_equal([@p1.id], @pool.versions.all[1].post_ids)
-      assert_equal("#{@p1.id} #{@p2.id}", @pool.versions.all[2].post_ids)
+      assert_equal([@p1.id, @p2.id], @pool.versions.all[2].post_ids)
-      assert_equal("#{@p1.id} #{@p2.id} #{@p3.id}", @pool.versions.all[3].post_ids)
+      assert_equal([@p1.id, @p2.id, @p3.id], @pool.versions.all[3].post_ids)
-      assert_equal("#{@p2.id} #{@p3.id}", @pool.versions.all[4].post_ids)
+      assert_equal([@p2.id, @p3.id], @pool.versions.all[4].post_ids)
     end
 
     should "update its post_ids" do
-      assert_equal("#{@p1.id}", @pool.post_ids)
+      assert_equal([@p1.id], @pool.post_id_array)
     end
 
     should "update any old posts that were removed" do

test/unit/post_sets/pool_test.rb

@@ -1,7 +1,10 @@
 require 'test_helper'
+require 'helpers/pool_archive_test_helper'
 
 module PostSets
   class PoolTest < ActiveSupport::TestCase
+    include PoolArchiveTestHelper
+
     context "In all cases" do
       setup do
         @user = FactoryGirl.create(:user)
@@ -9,6 +12,9 @@ module PostSets
         CurrentUser.ip_addr = "127.0.0.1"
         MEMCACHE.flush_all
 
+        mock_pool_archive_service!
+        start_pool_archive_transaction
+
         @post_1 = FactoryGirl.create(:post)
         @post_2 = FactoryGirl.create(:post)
         @post_3 = FactoryGirl.create(:post)
@@ -19,6 +25,7 @@ module PostSets
       end
 
       teardown do
+        rollback_pool_archive_transaction
         CurrentUser.user = nil
         CurrentUser.ip_addr = nil
       end

test/unit/post_test.rb

@@ -1,6 +1,9 @@
 require 'test_helper'
+require 'helpers/pool_archive_test_helper'
 
 class PostTest < ActiveSupport::TestCase
+  include PoolArchiveTestHelper
+
   setup do
     Timecop.travel(2.weeks.ago) do
       @user = FactoryGirl.create(:user)
@@ -35,6 +38,7 @@ class PostTest < ActiveSupport::TestCase
 
       context "that belongs to a pool" do
         setup do
+          SqsService.any_instance.stubs(:send_message)
           @pool = FactoryGirl.create(:pool)
           @pool.add!(@post)
           @post.reload
@@ -590,6 +594,15 @@ class PostTest < ActiveSupport::TestCase
         end
 
         context "for a pool" do
+          setup do
+            mock_pool_archive_service!
+            start_pool_archive_transaction
+          end
+
+          teardown do
+            rollback_pool_archive_transaction
+          end
+
           context "on creation" do
             setup do
               @pool = FactoryGirl.create(:pool)
@@ -1143,11 +1156,11 @@ class PostTest < ActiveSupport::TestCase
           should "clear the pixiv id" do
             @post.pixiv_id = 1234
             @post.update(source: "http://fc06.deviantart.net/fs71/f/2013/295/d/7/you_are_already_dead__by_mar11co-d6rgm0e.jpg")
-            assert_equal(nil, @post.pixiv_id)
+            assert_nil(@post.pixiv_id)
 
             @post.pixiv_id = 1234
             @post.update(source: "http://pictures.hentai-foundry.com//a/AnimeFlux/219123.jpg")
-            assert_equal(nil, @post.pixiv_id)
+            assert_nil(@post.pixiv_id)
           end
         end
 
@@ -1232,19 +1245,19 @@ class PostTest < ActiveSupport::TestCase
             should "not save the pixiv id" do
               @post.pixiv_id = 1234
               @post.update(source: "http://i1.pixiv.net/novel-cover-original/img/2016/11/03/20/10/58/7436075_f75af69f3eacd1656d3733c72aa959cf.jpg")
-              assert_equal(nil, @post.pixiv_id)
+              assert_nil(@post.pixiv_id)
 
               @post.pixiv_id = 1234
               @post.update(source: "http://i2.pixiv.net/background/img/2016/10/30/12/27/30/7059005_da9946b806c10d391a81ed1117cd33d6.jpg")
-              assert_equal(nil, @post.pixiv_id)
+              assert_nil(@post.pixiv_id)
 
               @post.pixiv_id = 1234
               @post.update(source: "http://i1.pixiv.net/img15/img/omega777/novel/2612734.jpg")
-              assert_equal(nil, @post.pixiv_id)
+              assert_nil(@post.pixiv_id)
 
               @post.pixiv_id = 1234
               @post.update(source: "http://img08.pixiv.net/profile/nice/1408837.jpg")
-              assert_equal(nil, @post.pixiv_id)
+              assert_nil(@post.pixiv_id)
             end
           end
         end
@@ -1440,6 +1453,10 @@ class PostTest < ActiveSupport::TestCase
   end
 
   context "Pools:" do
+    setup do
+      SqsService.any_instance.stubs(:send_message)
+    end
+
     context "Removing a post from a pool" do
       should "update the post's pool string" do
         post = FactoryGirl.create(:post)
@@ -1615,6 +1632,8 @@ class PostTest < ActiveSupport::TestCase
     end
 
     should "return posts for the <pool> metatag" do
+      SqsService.any_instance.stubs(:send_message)
+
       post1 = FactoryGirl.create(:post)
       post2 = FactoryGirl.create(:post)
       post3 = FactoryGirl.create(:post)
@@ -1626,6 +1645,8 @@ class PostTest < ActiveSupport::TestCase
     end
 
     should "return posts for the <pool> metatag with a wildcard" do
+      SqsService.any_instance.stubs(:send_message)
+
       post1 = FactoryGirl.create(:post)
       post2 = FactoryGirl.create(:post)
       post3 = FactoryGirl.create(:post)
@@ -2020,7 +2041,7 @@ class PostTest < ActiveSupport::TestCase
 
         should "correctly revert all fields" do
           assert_equal("aaa bbb ccc ddd", @post.tag_string)
-          assert_equal(nil, @post.source)
+          assert_nil(@post.source)
           assert_equal("q", @post.rating)
         end
       end