From f2dccf8cf19a1c7e17bdae55fee2a233cdf94e4d Mon Sep 17 00:00:00 2001 From: evazion Date: Fri, 27 Sep 2019 20:51:56 -0500 Subject: [PATCH] Remove mod-only bulk revert system (#4178). The mass undo system added in #4178 is a replacement for the mod-only bulk revert system. --- Gemfile | 1 - Gemfile.lock | 10 -- .../moderator/bulk_reverts_controller.rb | 35 ------- app/jobs/process_bulk_revert_job.rb | 8 -- app/logical/bulk_revert.rb | 67 ------------- app/logical/google_big_query/base.rb | 44 --------- app/logical/google_big_query/post_version.rb | 66 ------------- app/models/mod_action.rb | 2 +- app/views/moderator/bulk_reverts/new.html.erb | 99 ------------------- app/views/static/site_map.html.erb | 1 - config/routes.rb | 1 - test/unit/bulk_revert_test.rb | 20 ---- 12 files changed, 1 insertion(+), 353 deletions(-) delete mode 100644 app/controllers/moderator/bulk_reverts_controller.rb delete mode 100644 app/jobs/process_bulk_revert_job.rb delete mode 100644 app/logical/bulk_revert.rb delete mode 100644 app/logical/google_big_query/base.rb delete mode 100644 app/logical/google_big_query/post_version.rb delete mode 100644 app/views/moderator/bulk_reverts/new.html.erb delete mode 100644 test/unit/bulk_revert_test.rb diff --git a/Gemfile b/Gemfile index bf6444978..4ab61db1e 100644 --- a/Gemfile +++ b/Gemfile @@ -27,7 +27,6 @@ gem 'responders' gem 'dtext_rb', :git => "https://github.com/r888888888/dtext_rb.git", :require => "dtext" gem 'google-api-client' gem 'cityhash' -gem 'bigquery', :git => "https://github.com/abronte/BigQuery.git", :ref => "b92b4e0b54574e3fde7ad910f39a67538ed387ad" gem 'memoist' gem 'daemons' gem 'oauth2' diff --git a/Gemfile.lock b/Gemfile.lock index b7a7bd9c2..d6be1904d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,12 +1,3 @@ -GIT - remote: https://github.com/abronte/BigQuery.git - revision: b92b4e0b54574e3fde7ad910f39a67538ed387ad - ref: b92b4e0b54574e3fde7ad910f39a67538ed387ad - specs: - bigquery (0.9.0) - google-api-client (~> 0.9.3) - googleauth (~> 0.5.0) - GIT remote: https://github.com/r888888888/dtext_rb.git revision: 073b369bf90217ab86fdef3d0f88a96e10343d37 @@ -465,7 +456,6 @@ DEPENDENCIES awesome_print aws-sdk (~> 2) bcrypt - bigquery! bootsnap builder capistrano (~> 3.10) diff --git a/app/controllers/moderator/bulk_reverts_controller.rb b/app/controllers/moderator/bulk_reverts_controller.rb deleted file mode 100644 index d441bd892..000000000 --- a/app/controllers/moderator/bulk_reverts_controller.rb +++ /dev/null @@ -1,35 +0,0 @@ -module Moderator - class BulkRevertsController < ApplicationController - before_action :moderator_only - before_action :init_constraints - helper PostVersionsHelper - rescue_from BulkRevert::ConstraintTooGeneralError, :with => :tag_constraint_too_general - - def new - end - - def create - @bulk_revert = BulkRevert.new - - if params[:commit] == "Test" - @bulk_revert.preview - render action: "new" - else - ProcessBulkRevertJob.perform_later(CurrentUser.user, @constraints) - flash[:notice] = "Reverts queued" - redirect_to new_moderator_bulk_revert_path - end - end - - private - - def init_constraints - @constraints = params[:constraints] || {} - end - - def tag_constraint_too_general - flash[:notice] = "Your tag constraints are too general; try adding min and max version ids" - render action: "new" - end - end -end diff --git a/app/jobs/process_bulk_revert_job.rb b/app/jobs/process_bulk_revert_job.rb deleted file mode 100644 index 8aa5387ca..000000000 --- a/app/jobs/process_bulk_revert_job.rb +++ /dev/null @@ -1,8 +0,0 @@ -class ProcessBulkRevertJob < ApplicationJob - queue_as :default - queue_with_priority 20 - - def perform(creator, constraints) - BulkRevert.new.process(creator, constraints) - end -end diff --git a/app/logical/bulk_revert.rb b/app/logical/bulk_revert.rb deleted file mode 100644 index 591c97010..000000000 --- a/app/logical/bulk_revert.rb +++ /dev/null @@ -1,67 +0,0 @@ -class BulkRevert - BIG_QUERY_LIMIT = 5_000 - attr_reader :constraints - - class ConstraintTooGeneralError < Exception ; end - - def process(creator, constraints = {}) - @constraints = constraints - - ModAction.log("Processed bulk revert for #{constraints.inspect} by #{creator.name}",:bulk_revert) - - CurrentUser.scoped(creator) do - ApplicationRecord.without_timeout do - find_post_versions.order("updated_at, id").each do |version| - version.undo! - end - end - end - end - - def initialize - @constraints = {} - end - - def preview - @_preview ||= find_post_versions - end - - def query_gbq(user_id, added_tags, removed_tags, min_version_id, max_version_id) - GoogleBigQuery::PostVersion.new.find(user_id, added_tags, removed_tags, min_version_id, max_version_id, BIG_QUERY_LIMIT) - end - - def find_post_versions - q = PostArchive.where("true") - - if constraints[:user_name] - constraints[:user_id] = User.find_by_name(constraints[:user_name]).try(:id) - end - - if constraints[:user_id] - q = q.where("post_versions.updater_id = ?", constraints[:user_id]) - end - - if constraints[:added_tags] || constraints[:removed_tags] - hash = CityHash.hash64("#{constraints[:added_tags]} #{constraints{removed_tags}} #{constraints[:min_version_id]} #{constraints[:max_version_id]}").to_s(36) - sub_ids = Cache.get("br/fpv/#{hash}", 300) do - query_gbq(constraints[:user_id], constraints[:added_tags], constraints[:removed_tags], constraints[:min_version_id], constraints[:max_version_id]) - end - - if sub_ids.size >= BIG_QUERY_LIMIT - raise ConstraintTooGeneralError.new - end - - q = q.where("post_versions.id in (?)", sub_ids) - end - - if constraints[:min_version_id].present? - q = q.where("post_versions.id >= ?", constraints[:min_version_id]) - end - - if constraints[:max_version_id].present? - q = q.where("post_versions.id <= ?", constraints[:max_version_id]) - end - - q - end -end diff --git a/app/logical/google_big_query/base.rb b/app/logical/google_big_query/base.rb deleted file mode 100644 index 59ea23fd6..000000000 --- a/app/logical/google_big_query/base.rb +++ /dev/null @@ -1,44 +0,0 @@ -require "big_query" - -module GoogleBigQuery - class Base - def self.enabled? - File.exists?(Danbooru.config.google_api_json_key_path) - end - - def initialize - raise NotImplementedError.new("Google Big Query is not configured.") unless GoogleBigQuery::Base.enabled? - end - - def query(q) - client.query(q) - end - - def escape(s) - Regexp.escape(s).gsub(/\\/, '\0\0').gsub(/['"]/, '\\\\\0') - end - - def client - @_client ||= BigQuery::Client.new( - "json_key" => client_options[:google_key_path], - "project_id" => google_config["project_id"], - "dataset" => client_options[:google_data_set] - ) - end - - def client_options - @_client_options ||= { - google_key_path: Danbooru.config.google_api_json_key_path, - google_data_set: "danbooru_#{Rails.env}" - } - end - - def google_config - @_google_config ||= JSON.parse(File.read(client_options[:google_key_path])) - end - - def data_set - client_options[:google_data_set] - end - end -end diff --git a/app/logical/google_big_query/post_version.rb b/app/logical/google_big_query/post_version.rb deleted file mode 100644 index 7d30e8e63..000000000 --- a/app/logical/google_big_query/post_version.rb +++ /dev/null @@ -1,66 +0,0 @@ -module GoogleBigQuery - class PostVersion < Base - def find_removed(tag, limit = 1_000) - limit = limit.to_i - query("select id, post_id, updated_at, updater_id, updater_ip_addr, tags, added_tags, removed_tags, parent_id, rating, source from [#{data_set}.post_versions] where #{remove_tag_condition(tag)} order by updated_at desc limit #{limit}") - end - - def find_added(tag, limit = 1_000) - limit = limit.to_i - query("select id, post_id, updated_at, updater_id, updater_ip_addr, tags, added_tags, removed_tags, parent_id, rating, source from [#{data_set}.post_versions] where #{add_tag_condition(tag)} order by updated_at desc limit #{limit}") - end - - def add_tag_condition(t) - es = escape(t) - "regexp_match(added_tags, \"(?:^| )#{es}(?:$| )\")" - end - - def remove_tag_condition(t) - es = escape(t) - "regexp_match(removed_tags, \"(?:^| )#{es}(?:$| )\")" - end - - def find_for_post(post_id, created_at) - post_id = post_id.to_i - btime = created_at.strftime("%Y-%m-%d 00:00:00", created_at) - etime = 1.day.from(created_at).strftime("%Y-%m-%d 00:00:00") - "select updater_id, added_tag from [danbooru_#{Rails.env}].post_versions_flat_part where _partitiontime >= #{btime} and _partitiontime <= #{etime} and post_id = #{post_id}" - end - - def find(user_id, added_tags, removed_tags, min_version_id, max_version_id, limit = 1_000) - constraints = [] - - constraints << "updater_id = #{user_id.to_i}" - - if added_tags - added_tags.split.each do |tag| - constraints << add_tag_condition(tag) - end - end - - if removed_tags - removed_tags.split.each do |tag| - constraints << remove_tag_condition(tag) - end - end - - if min_version_id - constraints << "id >= #{min_version_id.to_i}" - end - - if max_version_id - constraints << "id <= #{max_version_id.to_i}" - end - - limit = limit.to_i - sql = "select id from [#{data_set}.post_versions] where " + constraints.join(" and ") + " order by updated_at desc limit #{limit}" - result = query(sql) - - if result["rows"] - result["rows"].map {|x| x["f"][0]["v"].to_i} - else - [] - end - end - end -end diff --git a/app/models/mod_action.rb b/app/models/mod_action.rb index 698bd9b14..3d5fa8d5e 100644 --- a/app/models/mod_action.rb +++ b/app/models/mod_action.rb @@ -50,7 +50,7 @@ class ModAction < ApplicationRecord ip_ban_create: 160, ip_ban_delete: 162, mass_update: 1000, - bulk_revert: 1001, + bulk_revert: 1001, # XXX unused other: 2000 } diff --git a/app/views/moderator/bulk_reverts/new.html.erb b/app/views/moderator/bulk_reverts/new.html.erb deleted file mode 100644 index 1efa2d70b..000000000 --- a/app/views/moderator/bulk_reverts/new.html.erb +++ /dev/null @@ -1,99 +0,0 @@ -
-
-

New Bulk Revert

- -

You must preview a bulk revert before processing it. Previews may take several minutes to generate.

- - <%= form_tag(moderator_bulk_revert_path, :class => "simple_form") do %> -
- - <%= text_field :constraints, :user_name, :value => @constraints[:user_name], :data => { autocomplete: "user" } %> -
- -
- - <%= text_field :constraints, :min_version_id, :value => @constraints[:min_version_id] %> -
- -
- - <%= text_field :constraints, :max_version_id, :value => @constraints[:max_version_id] %> -
- -
- - <%= text_field :constraints, :added_tags, :value => @constraints[:added_tags], :data => { :autocomplete => "tag" } %> -

You must specify a user to add tags

-
- -
- - <%= text_field :constraints, :removed_tags, :value => @constraints[:removed_tags], :data => { :autocomplete => "tag" } %> -

You must specify a user to add tags

-
- - <%= submit_tag "Test" %> - - <% if params[:commit] == "Test" %> - <%= submit_tag %> - <% end %> - <% end %> - - <% if @bulk_revert %> -
-

Preview limited to 200 changes

- - - - - - - - - - - - - <% @bulk_revert.preview.limit(200).each do |post_version| %> - - - - - - - <% end %> - -
PostDateUserChanges
<%= link_to("#{post_version.post_id}.#{post_version.id}", post_path(post_version.post_id)) %><%= compact_time(post_version.updated_at) %> - <% if post_version.updater %> - <%= link_to_user(post_version.updater) %> - <% end %> - <%= post_version_diff(post_version) %>
-
- <% end %> -
-
- -<% content_for(:page_title) do %> - New Bulk Revert - <%= Danbooru.config.app_name %> -<% end %> - -<% content_for(:html_header) do %> - <%= javascript_tag do %> - $(function() { - if (!$("#constraints_user_name").val().length) { - $("#added-tags-input input").prop("disabled", true); - $("#removed-tags-input input").prop("disabled", true); - } - - $("#constraints_user_name").keyup(function() { - if ($("#constraints_user_name").val().length) { - $("#added-tags-input input").prop("disabled", false); - $("#removed-tags-input input").prop("disabled", false); - } else { - $("#added-tags-input input").prop("disabled", true); - $("#removed-tags-input input").prop("disabled", true); - } - }); - }); - <% end %> -<% end %> diff --git a/app/views/static/site_map.html.erb b/app/views/static/site_map.html.erb index 1894c21d0..83569d083 100644 --- a/app/views/static/site_map.html.erb +++ b/app/views/static/site_map.html.erb @@ -157,7 +157,6 @@ <% if CurrentUser.is_moderator? %>
  • <%= link_to("IP Bans", ip_bans_path) %>
    • -
  • <%= link_to("Bulk Revert", new_moderator_bulk_revert_path) %>
    • <% end %> <% if CurrentUser.is_admin? %> diff --git a/config/routes.rb b/config/routes.rb index 38f5962e5..99d5ff75b 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -6,7 +6,6 @@ Rails.application.routes.draw do resource :dashboard, :only => [:show] end namespace :moderator do - resource :bulk_revert, :only => [:new, :create] resource :dashboard, :only => [:show] resources :ip_addrs, :only => [:index] do collection do diff --git a/test/unit/bulk_revert_test.rb b/test/unit/bulk_revert_test.rb deleted file mode 100644 index a001bce36..000000000 --- a/test/unit/bulk_revert_test.rb +++ /dev/null @@ -1,20 +0,0 @@ -require 'test_helper' - -class BulkRevertTest < ActiveSupport::TestCase - context "#find_post_versions" do - subject do - @user = FactoryBot.create(:user) - BulkRevert.new - end - - setup do - subject.stubs(:constraints).returns({added_tags: ["a"]}) - subject.expects(:query_gbq).returns([1,2,3]) - end - - should "revert all changes found in a search" do - q = subject.find_post_versions - assert_match(/post_versions\.id in \(1,2,3\)/, q.to_sql) - end - end -end