From d1d0fe9bc52b740621e075ab27cfd3d84db2497d Mon Sep 17 00:00:00 2001 From: evazion Date: Sat, 18 Mar 2017 18:21:13 -0500 Subject: [PATCH 1/3] /moderator/dashboard: optimize sql queries * Converts queries to use active record instead of raw sql. This ensures that user objects are loaded by rails in the join, so that we don't have to issue `User.find` calls to load users one-by-one. * Use `.includes` to preload associations used in the view, to avoid additional N+1 query problems (primarily, calls to link_to_user also causing users to be loaded one-by-one). --- .../moderator/dashboard/queries/artist.rb | 30 +++++----------- .../moderator/dashboard/queries/comment.rb | 35 ++++++------------- .../moderator/dashboard/queries/mod_action.rb | 2 +- .../moderator/dashboard/queries/note.rb | 30 +++++----------- .../dashboard/queries/post_appeal.rb | 27 ++++---------- .../moderator/dashboard/queries/tag.rb | 8 +---- .../moderator/dashboard/queries/upload.rb | 30 +++++----------- .../dashboard/queries/user_feedback.rb | 2 +- .../moderator/dashboard/queries/wiki_page.rb | 30 +++++----------- .../dashboards/_activity_appeal.html.erb | 12 +++---- 10 files changed, 62 insertions(+), 144 deletions(-) diff --git a/app/logical/moderator/dashboard/queries/artist.rb b/app/logical/moderator/dashboard/queries/artist.rb index 842f00716..326dfce31 100644 --- a/app/logical/moderator/dashboard/queries/artist.rb +++ b/app/logical/moderator/dashboard/queries/artist.rb @@ -1,28 +1,16 @@ module Moderator module Dashboard module Queries - class Artist - attr_reader :user, :count - + class Artist < ::Struct.new(:user, :count) def self.all(min_date, max_level) - sql = <<-EOS - SELECT artist_versions.updater_id AS updater_id, count(*) - FROM artist_versions - JOIN users ON users.id = artist_versions.updater_id - WHERE - artist_versions.created_at > ? - AND users.level <= ? - GROUP BY artist_versions.updater_id - ORDER BY count(*) DESC - LIMIT 10 - EOS - - ActiveRecord::Base.select_all_sql(sql, min_date, max_level).map {|x| new(x)} - end - - def initialize(hash) - @user = ::User.find(hash["updater_id"]) - @count = hash["count"] + ::ArtistVersion.joins(:updater) + .where("artist_versions.created_at > ?", min_date) + .where("users.level <= ?", max_level) + .group(:updater) + .order("count(*) desc") + .limit(10) + .count + .map { |user, count| new(user, count) } end end end diff --git a/app/logical/moderator/dashboard/queries/comment.rb b/app/logical/moderator/dashboard/queries/comment.rb index b86d189f0..9a9a06e90 100644 --- a/app/logical/moderator/dashboard/queries/comment.rb +++ b/app/logical/moderator/dashboard/queries/comment.rb @@ -1,31 +1,18 @@ module Moderator module Dashboard module Queries - class Comment - attr_reader :comment, :count - + class Comment < ::Struct.new(:comment, :count) def self.all(min_date, max_level) - sql = <<-EOS - SELECT comment_votes.comment_id, count(*) - FROM comment_votes - JOIN comments ON comments.id = comment_id - JOIN users ON users.id = comments.creator_id - WHERE - comment_votes.created_at > ? - AND comments.score < 0 - AND users.level <= ? - GROUP BY comment_votes.comment_id - HAVING count(*) >= 3 - ORDER BY count(*) DESC - LIMIT 10 - EOS - - ActiveRecord::Base.select_all_sql(sql, min_date, max_level).map {|x| new(x)} - end - - def initialize(hash) - @comment = ::Comment.find(hash["comment_id"]) - @count = hash["count"] + ::CommentVote.joins(comment: [:creator]) + .where("comments.score < 0") + .where("comment_votes.created_at > ?", min_date) + .where("users.level <= ?", max_level) + .group(:comment) + .having("count(*) >= 3") + .order("count(*) desc") + .limit(10) + .count + .map { |comment, count| new(comment, count) } end end end diff --git a/app/logical/moderator/dashboard/queries/mod_action.rb b/app/logical/moderator/dashboard/queries/mod_action.rb index 42917834a..9cd2d57dc 100644 --- a/app/logical/moderator/dashboard/queries/mod_action.rb +++ b/app/logical/moderator/dashboard/queries/mod_action.rb @@ -3,7 +3,7 @@ module Moderator module Queries class ModAction def self.all - ::ModAction.order("id desc").limit(10) + ::ModAction.includes(:creator).order("id desc").limit(10) end end end diff --git a/app/logical/moderator/dashboard/queries/note.rb b/app/logical/moderator/dashboard/queries/note.rb index 3e58257f1..f9129694a 100644 --- a/app/logical/moderator/dashboard/queries/note.rb +++ b/app/logical/moderator/dashboard/queries/note.rb @@ -1,28 +1,16 @@ module Moderator module Dashboard module Queries - class Note - attr_reader :user, :count - + class Note < ::Struct.new(:user, :count) def self.all(min_date, max_level) - sql = <<-EOS - SELECT note_versions.updater_id, count(*) - FROM note_versions - JOIN users ON users.id = note_versions.updater_id - WHERE - note_versions.created_at > ? - AND users.level <= ? - GROUP BY note_versions.updater_id - ORDER BY count(*) DESC - LIMIT 10 - EOS - - ActiveRecord::Base.select_all_sql(sql, min_date, max_level).map {|x| new(x)} - end - - def initialize(hash) - @user = ::User.find(hash["updater_id"]) - @count = hash["count"] + ::NoteVersion.joins(:updater) + .where("note_versions.created_at > ?", min_date) + .where("users.level <= ?", max_level) + .group(:updater) + .order("count(*) desc") + .limit(10) + .count + .map { |user, count| new(user, count) } end end end diff --git a/app/logical/moderator/dashboard/queries/post_appeal.rb b/app/logical/moderator/dashboard/queries/post_appeal.rb index c84ed0d9c..f6ae5937e 100644 --- a/app/logical/moderator/dashboard/queries/post_appeal.rb +++ b/app/logical/moderator/dashboard/queries/post_appeal.rb @@ -2,28 +2,13 @@ module Moderator module Dashboard module Queries class PostAppeal - attr_reader :post, :count - def self.all(min_date) - sql = <<-EOS - SELECT post_appeals.post_id, count(*) - FROM post_appeals - JOIN posts ON posts.id = post_appeals.post_id - WHERE - post_appeals.created_at > ? - and posts.is_deleted = true - and posts.is_pending = false - GROUP BY post_appeals.post_id - ORDER BY count(*) DESC - LIMIT 10 - EOS - - ActiveRecord::Base.select_all_sql(sql, min_date).map {|x| new(x)} - end - - def initialize(hash) - @post = ::Post.find(hash["post_id"]) - @count = hash["count"] + ::Post.joins(:appeals).includes(:uploader, :flags, appeals: [:creator]) + .deleted + .where("post_appeals.created_at > ?", min_date) + .group(:id) + .order("count(*) desc") + .limit(10) end end end diff --git a/app/logical/moderator/dashboard/queries/tag.rb b/app/logical/moderator/dashboard/queries/tag.rb index f87434fac..20144c607 100644 --- a/app/logical/moderator/dashboard/queries/tag.rb +++ b/app/logical/moderator/dashboard/queries/tag.rb @@ -1,9 +1,7 @@ module Moderator module Dashboard module Queries - class Tag - attr_reader :user, :count - + class Tag < ::Struct.new(:user, :count) def self.all(min_date, max_level) return [] unless PostArchive.enabled? @@ -13,10 +11,6 @@ module Moderator records.select { |rec| rec.user.level <= max_level }.sort_by(&:count).reverse.take(10) end - - def initialize(user, count) - @user, @count = user, count - end end end end diff --git a/app/logical/moderator/dashboard/queries/upload.rb b/app/logical/moderator/dashboard/queries/upload.rb index 588ff4169..372878566 100644 --- a/app/logical/moderator/dashboard/queries/upload.rb +++ b/app/logical/moderator/dashboard/queries/upload.rb @@ -1,28 +1,16 @@ module Moderator module Dashboard module Queries - class Upload - attr_reader :user, :count - + class Upload < ::Struct.new(:user, :count) def self.all(min_date, max_level) - sql = <<-EOS - select uploader_id, count(*) - from posts - join users on uploader_id = users.id - where - posts.created_at > ? - and level <= ? - group by posts.uploader_id - order by count(*) desc - limit 10 - EOS - - ActiveRecord::Base.select_all_sql(sql, min_date, max_level).map {|x| new(x)} - end - - def initialize(hash) - @user = ::User.find(hash["uploader_id"]) - @count = hash["count"] + ::Post.joins(:uploader) + .where("posts.created_at > ?", min_date) + .where("users.level <= ?", max_level) + .group(:uploader) + .order("count(*) desc") + .limit(10) + .count + .map { |user, count| new(user, count) } end end end diff --git a/app/logical/moderator/dashboard/queries/user_feedback.rb b/app/logical/moderator/dashboard/queries/user_feedback.rb index 989f0a2a3..81a4a195d 100644 --- a/app/logical/moderator/dashboard/queries/user_feedback.rb +++ b/app/logical/moderator/dashboard/queries/user_feedback.rb @@ -3,7 +3,7 @@ module Moderator module Queries class UserFeedback def self.all - ::UserFeedback.order("id desc").limit(10) + ::UserFeedback.includes(:user).order("id desc").limit(10) end end end diff --git a/app/logical/moderator/dashboard/queries/wiki_page.rb b/app/logical/moderator/dashboard/queries/wiki_page.rb index 9b4c15f36..e32b0ed75 100644 --- a/app/logical/moderator/dashboard/queries/wiki_page.rb +++ b/app/logical/moderator/dashboard/queries/wiki_page.rb @@ -1,28 +1,16 @@ module Moderator module Dashboard module Queries - class WikiPage - attr_reader :user, :count - + class WikiPage < ::Struct.new(:user, :count) def self.all(min_date, max_level) - sql = <<-EOS - SELECT wiki_page_versions.updater_id, count(*) - FROM wiki_page_versions - JOIN users ON users.id = wiki_page_versions.updater_id - WHERE - wiki_page_versions.created_at > ? - AND users.level <= ? - GROUP BY wiki_page_versions.updater_id - ORDER BY count(*) DESC - LIMIT 10 - EOS - - ActiveRecord::Base.select_all_sql(sql, min_date, max_level).map {|x| new(x)} - end - - def initialize(hash) - @user = ::User.find(hash["updater_id"]) - @count = hash["count"] + ::WikiPageVersion.joins(:updater) + .where("wiki_page_versions.created_at > ?", min_date) + .where("users.level <= ?", max_level) + .group(:updater) + .order("count(*) desc") + .limit(10) + .count + .map { |user, count| new(user, count) } end end end diff --git a/app/views/moderator/dashboards/_activity_appeal.html.erb b/app/views/moderator/dashboards/_activity_appeal.html.erb index d330e6f6e..317286b49 100644 --- a/app/views/moderator/dashboards/_activity_appeal.html.erb +++ b/app/views/moderator/dashboards/_activity_appeal.html.erb @@ -10,13 +10,13 @@ - <% @dashboard.appeals.each do |appeal| %> + <% @dashboard.appeals.each do |post| %> - <%= link_to image_tag(appeal.post.preview_file_url), post_path(appeal.post) %> - <%= mod_link_to_user appeal.post.uploader, :negative %> - <%= post_flag_reasons(appeal.post) %> - <%= post_appeal_reasons(appeal.post) %> - <%= appeal.post.score %> + <%= link_to image_tag(post.preview_file_url), post_path(post) %> + <%= mod_link_to_user post.uploader, :negative %> + <%= post_flag_reasons(post) %> + <%= post_appeal_reasons(post) %> + <%= post.score %> <% end %> From 246eb1e8ab25ec29bf013f5a01a97eb36c012445 Mon Sep 17 00:00:00 2001 From: evazion Date: Sat, 18 Mar 2017 20:00:13 -0500 Subject: [PATCH 2/3] db: add created_at indexes on post_appeals, artist/note/wiki_page versions, This optimizes queries on /moderator/dashboard that filter by creation date. --- ...000519_add_created_at_index_to_versions.rb | 10 +++++++ db/structure.sql | 29 +++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 db/migrate/20170319000519_add_created_at_index_to_versions.rb diff --git a/db/migrate/20170319000519_add_created_at_index_to_versions.rb b/db/migrate/20170319000519_add_created_at_index_to_versions.rb new file mode 100644 index 000000000..38b6df765 --- /dev/null +++ b/db/migrate/20170319000519_add_created_at_index_to_versions.rb @@ -0,0 +1,10 @@ +class AddCreatedAtIndexToVersions < ActiveRecord::Migration + def change + ActiveRecord::Base.without_timeout do + add_index :note_versions, :created_at + add_index :artist_versions, :created_at + add_index :wiki_page_versions, :created_at + add_index :post_appeals, :created_at + end + end +end diff --git a/db/structure.sql b/db/structure.sql index db56a9db0..08ac0f916 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -4977,6 +4977,13 @@ CREATE INDEX index_artist_urls_on_url_pattern ON artist_urls USING btree (url te CREATE INDEX index_artist_versions_on_artist_id ON artist_versions USING btree (artist_id); +-- +-- Name: index_artist_versions_on_created_at; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_artist_versions_on_created_at ON artist_versions USING btree (created_at); + + -- -- Name: index_artist_versions_on_name; Type: INDEX; Schema: public; Owner: - -- @@ -6706,6 +6713,13 @@ CREATE UNIQUE INDEX index_key_values_on_key ON key_values USING btree (key); CREATE INDEX index_news_updates_on_created_at ON news_updates USING btree (created_at); +-- +-- Name: index_note_versions_on_created_at; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_note_versions_on_created_at ON note_versions USING btree (created_at); + + -- -- Name: index_note_versions_on_note_id; Type: INDEX; Schema: public; Owner: - -- @@ -6790,6 +6804,13 @@ CREATE INDEX index_pools_on_name ON pools USING btree (name); CREATE INDEX index_pools_on_name_trgm ON pools USING gin (name gin_trgm_ops); +-- +-- Name: index_post_appeals_on_created_at; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_post_appeals_on_created_at ON post_appeals USING btree (created_at); + + -- -- Name: index_post_appeals_on_creator_id; Type: INDEX; Schema: public; Owner: - -- @@ -7147,6 +7168,13 @@ CREATE UNIQUE INDEX index_users_on_name ON users USING btree (lower((name)::text CREATE INDEX index_users_on_name_trgm ON users USING gin (lower((name)::text) gin_trgm_ops); +-- +-- Name: index_wiki_page_versions_on_created_at; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_wiki_page_versions_on_created_at ON wiki_page_versions USING btree (created_at); + + -- -- Name: index_wiki_page_versions_on_updater_ip_addr; Type: INDEX; Schema: public; Owner: - -- @@ -7532,3 +7560,4 @@ INSERT INTO schema_migrations (version) VALUES ('20170314235626'); INSERT INTO schema_migrations (version) VALUES ('20170316224630'); +INSERT INTO schema_migrations (version) VALUES ('20170319000519'); From 66b5dc641c1f0909ac0e7d10df27104c211f0289 Mon Sep 17 00:00:00 2001 From: evazion Date: Sun, 19 Mar 2017 17:20:51 -0500 Subject: [PATCH 3/3] /moderator/dashboard: use PostPresenter.preview for appeal thumbnails. --- app/views/moderator/dashboards/_activity_appeal.html.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/moderator/dashboards/_activity_appeal.html.erb b/app/views/moderator/dashboards/_activity_appeal.html.erb index 317286b49..4d1423a07 100644 --- a/app/views/moderator/dashboards/_activity_appeal.html.erb +++ b/app/views/moderator/dashboards/_activity_appeal.html.erb @@ -12,7 +12,7 @@ <% @dashboard.appeals.each do |post| %> - <%= link_to image_tag(post.preview_file_url), post_path(post) %> + <%= PostPresenter.preview(post, show_deleted: true) %> <%= mod_link_to_user post.uploader, :negative %> <%= post_flag_reasons(post) %> <%= post_appeal_reasons(post) %>