jaculabilis/danbooru
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'post-archive'

Browse Source
This commit is contained in:
r888888888
2017-02-27 10:45:13 -08:00
parent 2ebdfd7ab3 6bd03ee4db
commit 398f58684f
20 changed files with 392 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
app/assets/stylesheets/specific/forum.css.scss
View File

@@ -9,6 +9,10 @@ div.list-of-forum-posts {
margin-bottom: 3em;
word-wrap: break-word;
&:target {
background-color: #FFC;
}
div.author {
width: 12em;
float: left;

4
app/controllers/post_versions_controller.rb
View File

@@ -3,7 +3,7 @@ class PostVersionsController < ApplicationController
rescue_from ActiveRecord::StatementInvalid, :with => :rescue_exception
def index
@post_versions = PostVersion.search(params[:search]).order("updated_at desc, id desc").paginate(params[:page], :limit => params[:limit], :search_count => params[:search])
@post_versions = PostArchive.search(params[:search]).order("updated_at desc, id desc").paginate(params[:page], :limit => params[:limit], :search_count => params[:search])
respond_with(@post_versions) do |format|
format.xml do
render :xml => @post_versions.to_xml(:root => "post-versions")
@@ -15,7 +15,7 @@ class PostVersionsController < ApplicationController
end
def undo
@post_version = PostVersion.find(params[:id])
@post_version = PostArchive.find(params[:id])
if @post_version.post.visible?
@post_version.undo!

2
app/logical/bulk_revert.rb
View File

@@ -27,7 +27,7 @@ class BulkRevert
end
def find_post_versions
q = PostVersion.where("true")
q = PostArchive.where("true")
if constraints[:user_name]
constraints[:user_id] = User.find_by_name(constraints[:user_name]).try(:id)

4
app/logical/moderator/ip_addr_search.rb
View File

@@ -34,7 +34,7 @@ module Moderator
add_row(sums, Hash[User.where(last_ip_addr: ip_addrs).collect { |user| [user, 1] }])
add_row_id(sums, PoolArchive.where(updater_ip_addr: ip_addrs).group(:updater_id).count) if PoolArchive.enabled?
add_row_id(sums, PostVersion.where(updater_ip_addr: ip_addrs).group(:updater_id).count)
add_row_id(sums, PostArchive.where(updater_ip_addr: ip_addrs).group(:updater_id).count) if PostArchive.enabled?
sums
end
@@ -52,7 +52,7 @@ module Moderator
add_row(sums, ArtistVersion.where(updater: users).group(:updater_ip_addr).count)
add_row(sums, NoteVersion.where(updater: users).group(:updater_ip_addr).count)
add_row(sums, PoolArchive.where(updater_id: users.map(&:id)).group(:updater_ip_addr).count) if PoolArchive.enabled?
add_row(sums, PostVersion.where(updater_id: users.map(&:id)).group(:updater_ip_addr).count)
add_row(sums, PostArchive.where(updater_id: users.map(&:id)).group(:updater_ip_addr).count) if PostArchive.enabled?
add_row(sums, WikiPageVersion.where(updater: users).group(:updater_ip_addr).count)
add_row(sums, Comment.where(creator: users).group(:ip_addr).count)
add_row(sums, Dmail.where(from: users).group(:creator_ip_addr).count)

2
app/logical/reports/post_versions.rb
View File

@@ -13,7 +13,7 @@ module Reports
end
def mock_version(row)
PostVersion.new.tap do |x|
PostArchive.new.tap do |x|
x.id = row["f"][0]["v"]
x.post_id = row["f"][1]["v"]
x.updated_at = Time.at(row["f"][2]["v"].to_f)

4
app/logical/user_revert.rb
View File

@@ -15,13 +15,13 @@ class UserRevert
end
def validate!
if PostVersion.where(updater_id: user_id).count > THRESHOLD
if PostArchive.where(updater_id: user_id).count > THRESHOLD
raise TooManyChangesError.new("This user has too many changes to be reverted")
end
end
def revert_post_changes
PostVersion.where(updater_id: user_id).find_each do |x|
PostArchive.where(updater_id: user_id).find_each do |x|
x.undo!
end
end

4
app/models/forum_topic.rb
View File

@@ -154,6 +154,10 @@ class ForumTopic < ActiveRecord::Base
self.updater_id = CurrentUser.id
end
def page_for(post_id)
(posts.where("id < ?", post_id).count / Danbooru.config.posts_per_page.to_f).ceil
end
def last_page
(response_count / Danbooru.config.posts_per_page.to_f).ceil
end

27
app/models/post.rb
View File

@@ -42,7 +42,6 @@ class Post < ActiveRecord::Base
has_one :pixiv_ugoira_frame_data, :class_name => "PixivUgoiraFrameData", :dependent => :destroy
has_many :flags, :class_name => "PostFlag", :dependent => :destroy
has_many :appeals, :class_name => "PostAppeal", :dependent => :destroy
has_many :versions, lambda {order("post_versions.updated_at ASC, post_versions.id ASC")}, :class_name => "PostVersion", :dependent => :destroy
has_many :votes, :class_name => "PostVote", :dependent => :destroy
has_many :notes, :dependent => :destroy
has_many :comments, lambda {includes(:creator, :updater).order("comments.id")}, :dependent => :destroy
@@ -1414,12 +1413,16 @@ class Post < ActiveRecord::Base
end
module VersionMethods
def versions
if PostArchive.enabled?
PostArchive.where(post_id: id).order("updated_at ASC, id asc")
else
raise "Archive service not configured"
end
end
def create_version(force = false)
if new_record? || rating_changed? || source_changed? || parent_id_changed? || tag_string_changed? || force
if merge_version?
delete_previous_version
end
create_new_version
end
end
@@ -1431,19 +1434,7 @@ class Post < ActiveRecord::Base
def create_new_version
User.where(id: CurrentUser.id).update_all("post_update_count = post_update_count + 1")
CurrentUser.reload
versions.create(
:rating => rating,
:source => source,
:tags => tag_string,
:parent_id => parent_id
)
end
def delete_previous_version
prev = versions.last
prev.destroy
PostArchive.queue(self) if PostArchive.enabled?
end
def revert_to(target)

316
app/models/post_archive.rb
View File

@@ -1,4 +1,6 @@
class PostArchive < ActiveRecord::Base
extend Memoist
def self.enabled?
Danbooru.config.aws_sqs_archives_url.present?
end
@@ -6,49 +8,283 @@ class PostArchive < ActiveRecord::Base
establish_connection (ENV["ARCHIVE_DATABASE_URL"] || "archive_#{Rails.env}".to_sym) if enabled?
self.table_name = "post_versions"
def self.calculate_version(post_id, updated_at, version_id)
if updated_at.to_i == Time.zone.parse("2007-03-14T19:38:12Z").to_i
# Old post versions which didn't have updated_at set correctly
1 + PostVersion.where("post_id = ? and updated_at = ? and id < ?", post_id, updated_at, version_id).count
else
1 + PostVersion.where("post_id = ? and updated_at < ?", post_id, updated_at).count
end
end
def self.export(version_id = 9096768)
PostVersion.where("id > ?", version_id).find_each do |version|
previous = version.previous
tags = version.tags.scan(/\S+/)
if previous
prev_tags = previous.tags.scan(/\S+/)
added_tags = tags - previous.tags.scan(/\S+/)
removed_tags = previous.tags.scan(/\S+/) - tags
module SearchMethods
def for_user(user_id)
if user_id
where("updater_id = ?", user_id)
else
added_tags = tags
removed_tags = []
where("false")
end
end
def for_user_name(name)
user_id = User.name_to_id(name)
for_user(user_id)
end
def search(params)
q = where("true")
params = {} if params.blank?
if params[:updater_name].present?
q = q.for_user_name(params[:updater_name])
end
rating_changed = previous.nil? || version.rating != previous.try(:rating)
parent_changed = previous.nil? || version.parent_id != previous.try(:parent_id)
source_changed = previous.nil? || version.source != previous.try(:source)
create(
post_id: version.post_id,
tags: version.tags,
added_tags: added_tags,
removed_tags: removed_tags,
updater_id: version.updater_id,
updater_ip_addr: version.updater_ip_addr.to_s,
updated_at: version.updated_at,
version: calculate_version(version.post_id, version.updated_at, version.id),
rating: version.rating,
rating_changed: rating_changed,
parent_id: version.parent_id,
parent_changed: parent_changed,
source: version.source,
source_changed: source_changed
)
puts "inserted #{version.id}"
if params[:updater_id].present?
q = q.where("updater_id = ?", params[:updater_id].to_i)
end
if params[:post_id].present?
q = q.where("post_id = ?", params[:post_id].to_i)
end
if params[:start_id].present?
q = q.where("id <= ?", params[:start_id].to_i)
end
q
end
end
module ArchiveServiceMethods
extend ActiveSupport::Concern
class_methods do
def sqs_service
SqsService.new(Danbooru.config.aws_sqs_archives_url)
end
def queue(post)
# queue updates to sqs so that if archives goes down for whatever reason it won't
# block post updates
raise "Archive service is not configured" if !enabled?
json = {
"post_id" => post.id,
"rating" => post.rating,
"parent_id" => post.parent_id,
"source" => post.source,
"updater_id" => CurrentUser.id,
"updater_ip_addr" => CurrentUser.ip_addr.to_s,
"updated_at" => post.updated_at.try(:iso8601),
"created_at" => post.created_at.try(:iso8601),
"tags" => post.tag_string
}
msg = "add post version\n#{json.to_json}"
sqs_service.send_message(msg)
end
def export_to_archives(version_id = 4394763)
PostVersion.where("id > ?", version_id).find_each do |version|
previous = version.previous
tags = version.tags.scan(/\S+/)
version_number = if version.updated_at.to_i == Time.zone.parse("2007-03-14T19:38:12Z").to_i
# Old post versions which didn't have updated_at set correctly
1 + PostVersion.where("post_id = ? and updated_at = ? and id < ?", version.post_id, version.updated_at, version.id).count
else
1 + PostVersion.where("post_id = ? and updated_at < ?", version.post_id, version.updated_at).count
end
if previous
prev_tags = previous.tags.scan(/\S+/)
added_tags = tags - prev_tags
removed_tags = prev_tags - tags
else
added_tags = tags
removed_tags = []
end
rating_changed = previous.nil? || version.rating != previous.rating
parent_changed = previous.nil? || version.parent_id != previous.parent_id
source_changed = previous.nil? || version.source != previous.source
create(
post_id: version.post_id,
tags: version.tags,
added_tags: added_tags,
removed_tags: removed_tags,
updater_id: version.updater_id,
updater_ip_addr: version.updater_ip_addr.to_s,
updated_at: version.updated_at,
version: version_number,
rating: version.rating,
rating_changed: rating_changed,
parent_id: version.parent_id,
parent_changed: parent_changed,
source: version.source,
source_changed: source_changed
)
puts "inserted #{version.id}"
end
end
end
end
extend SearchMethods
include ArchiveServiceMethods
def tag_array
tags.scan(/\S+/)
end
def presenter
PostVersionPresenter.new(self)
end
def reload
flush_cache
super
end
def post
Post.where(id: post_id).first
end
def previous
PostArchive.where("post_id = ? and version < ?", post_id, version).order("version desc").first
end
def updater
User.find(updater_id)
end
def diff(version = nil)
if post.nil?
latest_tags = tag_array
else
latest_tags = post.tag_array
latest_tags << "rating:#{post.rating}" if post.rating.present?
latest_tags << "parent:#{post.parent_id}" if post.parent_id.present?
latest_tags << "source:#{post.source}" if post.source.present?
end
new_tags = tag_array
new_tags << "rating:#{rating}" if rating.present?
new_tags << "parent:#{parent_id}" if parent_id.present?
new_tags << "source:#{source}" if source.present?
old_tags = version.present? ? version.tag_array : []
if version.present?
old_tags << "rating:#{version.rating}" if version.rating.present?
old_tags << "parent:#{version.parent_id}" if version.parent_id.present?
old_tags << "source:#{version.source}" if version.source.present?
end
added_tags = new_tags - old_tags
removed_tags = old_tags - new_tags
return {
:added_tags => added_tags,
:removed_tags => removed_tags,
:obsolete_added_tags => added_tags - latest_tags,
:obsolete_removed_tags => removed_tags & latest_tags,
:unchanged_tags => new_tags & old_tags,
}
end
def changes
delta = {
:added_tags => added_tags,
:removed_tags => removed_tags
}
latest_tags = post.tag_array
latest_tags << "rating:#{post.rating}" if post.rating.present?
latest_tags << "parent:#{post.parent_id}" if post.parent_id.present?
latest_tags << "source:#{post.source}" if post.source.present?
if parent_changed?
delta[:added_tags] << "parent:#{parent_id}"
if previous
delta[:removed_tags] << "parent:#{previous.parent_id}"
end
end
if rating_changed?
delta[:added_tags] << "rating:#{rating}"
if previous
delta[:removed_tags] << "rating:#{previous.rating}"
end
end
if source_changed?
delta[:added_tags] << "source:#{source}"
if previous
delta[:removed_tags] << "source:#{previous.source}"
end
end
delta[:obsolete_added_tags] = delta[:added_tags] - latest_tags
delta[:obsolete_removed_tags] = delta[:removed_tags] & latest_tags
if previous
delta[:unchanged_tags] = tag_array & previous.tag_array
end
delta
end
def added_tags_with_fields
changes[:added_tags].join(" ")
end
def removed_tags_with_fields
changes[:removed_tags].join(" ")
end
def obsolete_added_tags
changes[:obsolete_added_tags].join(" ")
end
def obsolete_removed_tags
changes[:obsolete_removed_tags].join(" ")
end
def unchanged_tags
changes[:unchanged_tags].join(" ")
end
def truncated_source
source.gsub(/^http:\/\//, "").sub(/\/.+/, "")
end
def undo
added = changes[:added_tags_with_fields] - changes[:obsolete_added_tags]
removed = changes[:removed_tags_with_fields] - changes[:obsolete_removed_tags]
added.each do |tag|
if tag =~ /^source:/
post.source = ""
elsif tag =~ /^parent:/
post.parent_id = nil
else
escaped_tag = Regexp.escape(tag)
post.tag_string = post.tag_string.sub(/(?:\A| )#{escaped_tag}(?:\Z| )/, " ").strip
end
end
removed.each do |tag|
if tag =~ /^source:(.+)$/
post.source = $1
else
post.tag_string = "#{post.tag_string} #{tag}".strip
end
end
end
def undo!
undo
post.save!
end
def updater
User.find_by_id(updater_id)
end
def method_attributes
super + [:added_tags, :removed_tags, :obsolete_added_tags, :obsolete_removed_tags, :unchanged_tags, :updater_name]
end
memoize :previous, :post, :tag_array, :changes, :added_tags_with_fields, :removed_tags_with_fields, :obsolete_removed_tags, :obsolete_added_tags, :unchanged_tags
end

11
app/models/post_version.rb
View File

@@ -61,17 +61,6 @@ class PostVersion < ActiveRecord::Base
super
end
def sequence_for_post
versions = PostVersion.where(:post_id => post_id).order("updated_at desc, id desc")
diffs = []
versions.each_index do |i|
if i < versions.size - 1
diffs << versions[i].diff(versions[i + 1])
end
end
return diffs
end
def diff(version)
latest_tags = post.tag_array
latest_tags << "rating:#{post.rating}" if post.rating.present?