jaculabilis/danbooru
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Remove mod-only bulk revert system (#4178).

Browse Source 
The mass undo system added in #4178 is a replacement for the mod-only
bulk revert system.
This commit is contained in:
evazion
2019-09-27 20:51:56 -05:00
parent 1341dbb262
commit f2dccf8cf1
12 changed files with 1 additions and 353 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
Gemfile
View File

@@ -27,7 +27,6 @@ gem 'responders'
gem 'dtext_rb', :git => "https://github.com/r888888888/dtext_rb.git", :require => "dtext" gem 'dtext_rb', :git => "https://github.com/r888888888/dtext_rb.git", :require => "dtext"
gem 'google-api-client' gem 'google-api-client'
gem 'cityhash' gem 'cityhash'
gem 'bigquery', :git => "https://github.com/abronte/BigQuery.git", :ref => "b92b4e0b54574e3fde7ad910f39a67538ed387ad"
gem 'memoist' gem 'memoist'
gem 'daemons' gem 'daemons'
gem 'oauth2' gem 'oauth2'

10
Gemfile.lock
View File

@@ -1,12 +1,3 @@
GIT
remote: https://github.com/abronte/BigQuery.git
revision: b92b4e0b54574e3fde7ad910f39a67538ed387ad
ref: b92b4e0b54574e3fde7ad910f39a67538ed387ad
specs:
bigquery (0.9.0)
google-api-client (~> 0.9.3)
googleauth (~> 0.5.0)
GIT GIT
remote: https://github.com/r888888888/dtext_rb.git remote: https://github.com/r888888888/dtext_rb.git
revision: 073b369bf90217ab86fdef3d0f88a96e10343d37 revision: 073b369bf90217ab86fdef3d0f88a96e10343d37
@@ -465,7 +456,6 @@ DEPENDENCIES
awesome_print awesome_print
aws-sdk (~> 2) aws-sdk (~> 2)
bcrypt bcrypt
bigquery!
bootsnap bootsnap
builder builder
capistrano (~> 3.10) capistrano (~> 3.10)

35
app/controllers/moderator/bulk_reverts_controller.rb
View File

@@ -1,35 +0,0 @@
module Moderator
class BulkRevertsController < ApplicationController
before_action :moderator_only
before_action :init_constraints
helper PostVersionsHelper
rescue_from BulkRevert::ConstraintTooGeneralError, :with => :tag_constraint_too_general
def new
end
def create
@bulk_revert = BulkRevert.new
if params[:commit] == "Test"
@bulk_revert.preview
render action: "new"
else
ProcessBulkRevertJob.perform_later(CurrentUser.user, @constraints)
flash[:notice] = "Reverts queued"
redirect_to new_moderator_bulk_revert_path
end
end
private
def init_constraints
@constraints = params[:constraints] || {}
end
def tag_constraint_too_general
flash[:notice] = "Your tag constraints are too general; try adding min and max version ids"
render action: "new"
end
end
end

8
app/jobs/process_bulk_revert_job.rb
View File

@@ -1,8 +0,0 @@
class ProcessBulkRevertJob < ApplicationJob
queue_as :default
queue_with_priority 20
def perform(creator, constraints)
BulkRevert.new.process(creator, constraints)
end
end

67
app/logical/bulk_revert.rb
View File

@@ -1,67 +0,0 @@
class BulkRevert
BIG_QUERY_LIMIT = 5_000
attr_reader :constraints
class ConstraintTooGeneralError < Exception ; end
def process(creator, constraints = {})
@constraints = constraints
ModAction.log("Processed bulk revert for #{constraints.inspect} by #{creator.name}",:bulk_revert)
CurrentUser.scoped(creator) do
ApplicationRecord.without_timeout do
find_post_versions.order("updated_at, id").each do |version|
version.undo!
end
end
end
end
def initialize
@constraints = {}
end
def preview
@_preview ||= find_post_versions
end
def query_gbq(user_id, added_tags, removed_tags, min_version_id, max_version_id)
GoogleBigQuery::PostVersion.new.find(user_id, added_tags, removed_tags, min_version_id, max_version_id, BIG_QUERY_LIMIT)
end
def find_post_versions
q = PostArchive.where("true")
if constraints[:user_name]
constraints[:user_id] = User.find_by_name(constraints[:user_name]).try(:id)
end
if constraints[:user_id]
q = q.where("post_versions.updater_id = ?", constraints[:user_id])
end
if constraints[:added_tags] || constraints[:removed_tags]
hash = CityHash.hash64("#{constraints[:added_tags]} #{constraints{removed_tags}} #{constraints[:min_version_id]} #{constraints[:max_version_id]}").to_s(36)
sub_ids = Cache.get("br/fpv/#{hash}", 300) do
query_gbq(constraints[:user_id], constraints[:added_tags], constraints[:removed_tags], constraints[:min_version_id], constraints[:max_version_id])
end
if sub_ids.size >= BIG_QUERY_LIMIT
raise ConstraintTooGeneralError.new
end
q = q.where("post_versions.id in (?)", sub_ids)
end
if constraints[:min_version_id].present?
q = q.where("post_versions.id >= ?", constraints[:min_version_id])
end
if constraints[:max_version_id].present?
q = q.where("post_versions.id <= ?", constraints[:max_version_id])
end
q
end
end

44
app/logical/google_big_query/base.rb
View File

@@ -1,44 +0,0 @@
require "big_query"
module GoogleBigQuery
class Base
def self.enabled?
File.exists?(Danbooru.config.google_api_json_key_path)
end
def initialize
raise NotImplementedError.new("Google Big Query is not configured.") unless GoogleBigQuery::Base.enabled?
end
def query(q)
client.query(q)
end
def escape(s)
Regexp.escape(s).gsub(/\\/, '\0\0').gsub(/['"]/, '\\\\\0')
end
def client
@_client ||= BigQuery::Client.new(
"json_key" => client_options[:google_key_path],
"project_id" => google_config["project_id"],
"dataset" => client_options[:google_data_set]
)
end
def client_options
@_client_options ||= {
google_key_path: Danbooru.config.google_api_json_key_path,
google_data_set: "danbooru_#{Rails.env}"
}
end
def google_config
@_google_config ||= JSON.parse(File.read(client_options[:google_key_path]))
end
def data_set
client_options[:google_data_set]
end
end
end

66
app/logical/google_big_query/post_version.rb
View File

@@ -1,66 +0,0 @@
module GoogleBigQuery
class PostVersion < Base
def find_removed(tag, limit = 1_000)
limit = limit.to_i
query("select id, post_id, updated_at, updater_id, updater_ip_addr, tags, added_tags, removed_tags, parent_id, rating, source from [#{data_set}.post_versions] where #{remove_tag_condition(tag)} order by updated_at desc limit #{limit}")
end
def find_added(tag, limit = 1_000)
limit = limit.to_i
query("select id, post_id, updated_at, updater_id, updater_ip_addr, tags, added_tags, removed_tags, parent_id, rating, source from [#{data_set}.post_versions] where #{add_tag_condition(tag)} order by updated_at desc limit #{limit}")
end
def add_tag_condition(t)
es = escape(t)
"regexp_match(added_tags, \"(?:^| )#{es}(?:$| )\")"
end
def remove_tag_condition(t)
es = escape(t)
"regexp_match(removed_tags, \"(?:^| )#{es}(?:$| )\")"
end
def find_for_post(post_id, created_at)
post_id = post_id.to_i
btime = created_at.strftime("%Y-%m-%d 00:00:00", created_at)
etime = 1.day.from(created_at).strftime("%Y-%m-%d 00:00:00")
"select updater_id, added_tag from [danbooru_#{Rails.env}].post_versions_flat_part where _partitiontime >= #{btime} and _partitiontime <= #{etime} and post_id = #{post_id}"
end
def find(user_id, added_tags, removed_tags, min_version_id, max_version_id, limit = 1_000)
constraints = []
constraints << "updater_id = #{user_id.to_i}"
if added_tags
added_tags.split.each do |tag|
constraints << add_tag_condition(tag)
end
end
if removed_tags
removed_tags.split.each do |tag|
constraints << remove_tag_condition(tag)
end
end
if min_version_id
constraints << "id >= #{min_version_id.to_i}"
end
if max_version_id
constraints << "id <= #{max_version_id.to_i}"
end
limit = limit.to_i
sql = "select id from [#{data_set}.post_versions] where " + constraints.join(" and ") + " order by updated_at desc limit #{limit}"
result = query(sql)
if result["rows"]
result["rows"].map {|x| x["f"][0]["v"].to_i}
else
[]
end
end
end
end

2
app/models/mod_action.rb
View File

@@ -50,7 +50,7 @@ class ModAction < ApplicationRecord
ip_ban_create: 160, ip_ban_create: 160,
ip_ban_delete: 162, ip_ban_delete: 162,
mass_update: 1000, mass_update: 1000,
bulk_revert: 1001, bulk_revert: 1001, # XXX unused
other: 2000 other: 2000
} }

99
app/views/moderator/bulk_reverts/new.html.erb
View File

@@ -1,99 +0,0 @@
<div id="c-moderator-bulk-reverts">
<div id="a-new">
<h1>New Bulk Revert</h1>
<p>You must preview a bulk revert before processing it. Previews may take several minutes to generate.</p>
<%= form_tag(moderator_bulk_revert_path, :class => "simple_form") do %>
<div class="input">
<label>User Name</label>
<%= text_field :constraints, :user_name, :value => @constraints[:user_name], :data => { autocomplete: "user" } %>
</div>
<div class="input">
<label>Min Version Id</label>
<%= text_field :constraints, :min_version_id, :value => @constraints[:min_version_id] %>
</div>
<div class="input">
<label>Max Version Id</label>
<%= text_field :constraints, :max_version_id, :value => @constraints[:max_version_id] %>
</div>
<div class="input" id="added-tags-input">
<label>Added Tags</label>
<%= text_field :constraints, :added_tags, :value => @constraints[:added_tags], :data => { :autocomplete => "tag" } %>
<p class="hint">You must specify a user to add tags</p>
</div>
<div class="input" id="removed-tags-input">
<label>Removed Tags</label>
<%= text_field :constraints, :removed_tags, :value => @constraints[:removed_tags], :data => { :autocomplete => "tag" } %>
<p class="hint">You must specify a user to add tags</p>
</div>
<%= submit_tag "Test" %>
<% if params[:commit] == "Test" %>
<%= submit_tag %>
<% end %>
<% end %>
<% if @bulk_revert %>
<div>
<p>Preview limited to 200 changes</p>
<table class="striped" width="100%">
<thead>
<tr>
<th>Post</th>
<th>Date</th>
<th>User</th>
<th>Changes</th>
</tr>
</thead>
<tbody>
<% @bulk_revert.preview.limit(200).each do |post_version| %>
<tr>
<td><%= link_to("#{post_version.post_id}.#{post_version.id}", post_path(post_version.post_id)) %></td>
<td><%= compact_time(post_version.updated_at) %></td>
<td>
<% if post_version.updater %>
<%= link_to_user(post_version.updater) %>
<% end %>
</td>
<td><%= post_version_diff(post_version) %></td>
</tr>
<% end %>
</tbody>
</table>
</div>
<% end %>
</div>
</div>
<% content_for(:page_title) do %>
New Bulk Revert - <%= Danbooru.config.app_name %>
<% end %>
<% content_for(:html_header) do %>
<%= javascript_tag do %>
$(function() {
if (!$("#constraints_user_name").val().length) {
$("#added-tags-input input").prop("disabled", true);
$("#removed-tags-input input").prop("disabled", true);
}
$("#constraints_user_name").keyup(function() {
if ($("#constraints_user_name").val().length) {
$("#added-tags-input input").prop("disabled", false);
$("#removed-tags-input input").prop("disabled", false);
} else {
$("#added-tags-input input").prop("disabled", true);
$("#removed-tags-input input").prop("disabled", true);
}
});
});
<% end %>
<% end %>

1
app/views/static/site_map.html.erb
View File

@@ -157,7 +157,6 @@
<% if CurrentUser.is_moderator? %> <% if CurrentUser.is_moderator? %>
<li><%= link_to("IP Bans", ip_bans_path) %></li> <li><%= link_to("IP Bans", ip_bans_path) %></li>
<li><%= link_to("Bulk Revert", new_moderator_bulk_revert_path) %></li>
<% end %> <% end %>
<% if CurrentUser.is_admin? %> <% if CurrentUser.is_admin? %>

1
config/routes.rb
View File

@@ -6,7 +6,6 @@ Rails.application.routes.draw do
resource :dashboard, :only => [:show] resource :dashboard, :only => [:show]
end end
namespace :moderator do namespace :moderator do
resource :bulk_revert, :only => [:new, :create]
resource :dashboard, :only => [:show] resource :dashboard, :only => [:show]
resources :ip_addrs, :only => [:index] do resources :ip_addrs, :only => [:index] do
collection do collection do

20
test/unit/bulk_revert_test.rb
View File

@@ -1,20 +0,0 @@
require 'test_helper'
class BulkRevertTest < ActiveSupport::TestCase
context "#find_post_versions" do
subject do
@user = FactoryBot.create(:user)
BulkRevert.new
end
setup do
subject.stubs(:constraints).returns({added_tags: ["a"]})
subject.expects(:query_gbq).returns([1,2,3])
end
should "revert all changes found in a search" do
q = subject.find_post_versions
assert_match(/post_versions\.id in \(1,2,3\)/, q.to_sql)
end
end
end