From 7646521d0fbfebf2ef77053a6a4d537dc63a739b Mon Sep 17 00:00:00 2001 From: evazion Date: Thu, 20 Oct 2022 05:20:22 -0500 Subject: [PATCH] Add basic tables and graphs for various tables. Add basic tables and graphs for viewing things like uploads over time, new users over time, comments over time, etc. Located at https://betabooru.donmai.us/reports. The graphing uses Apache ECharts: https://echarts.apache.org/en/index.html. --- app/components/time_series_component.rb | 13 +++ .../time_series_component.html.erb | 43 ++++++++++ .../time_series_component.js | 1 + app/controllers/reports_controller.rb | 81 +++++++++++++++++++ app/javascript/packs/application.js | 2 + .../src/javascripts/current_user.js | 6 ++ .../src/javascripts/time_series_component.js | 72 +++++++++++++++++ app/logical/concerns/aggregatable.rb | 81 +++++++++++++++++++ app/logical/concerns/searchable.rb | 11 ++- app/models/application_record.rb | 1 + app/views/reports/index.html.erb | 24 ++++++ app/views/reports/show.html.erb | 22 +++++ app/views/static/site_map.html.erb | 1 + config/routes.rb | 1 + package.json | 1 + yarn.lock | 33 +++++++- 16 files changed, 384 insertions(+), 9 deletions(-) create mode 100644 app/components/time_series_component.rb create mode 100644 app/components/time_series_component/time_series_component.html.erb create mode 120000 app/components/time_series_component/time_series_component.js create mode 100644 app/controllers/reports_controller.rb create mode 100644 app/javascript/src/javascripts/time_series_component.js create mode 100644 app/logical/concerns/aggregatable.rb create mode 100644 app/views/reports/index.html.erb create mode 100644 app/views/reports/show.html.erb diff --git a/app/components/time_series_component.rb b/app/components/time_series_component.rb new file mode 100644 index 000000000..651ad6129 --- /dev/null +++ b/app/components/time_series_component.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class TimeSeriesComponent < ApplicationComponent + delegate :current_page_path, :search_params, to: :helpers + + attr_reader :results, :columns, :mode + + def initialize(results, columns, mode: :table) + @results = results + @columns = columns + @mode = mode.to_sym + end +end diff --git a/app/components/time_series_component/time_series_component.html.erb b/app/components/time_series_component/time_series_component.html.erb new file mode 100644 index 000000000..e0c10f399 --- /dev/null +++ b/app/components/time_series_component/time_series_component.html.erb @@ -0,0 +1,43 @@ +
+ <% if mode == :table %> + <%= link_to "Chart", current_page_path(search: { **search_params, mode: "chart" }) %> + <% else %> + <%= link_to "Table", current_page_path(search: { **search_params, mode: "table" }) %> + <% end %> +
+ +<% if mode == :table %> + + + + + <% columns.each do |column| %> + <%= tag.th(column.to_s.capitalize, class: ("col-expand" if column == columns.last)) %> + <% end %> + + + + <% results.each do |row| %> + + + + <% columns.each do |column| %> + + <% end %> + + <% end %> + +
Date
<%= row["date"].to_date %><%= row[column.to_s] %>
+<% elsif mode == :chart %> +
+ + +<% end %> diff --git a/app/components/time_series_component/time_series_component.js b/app/components/time_series_component/time_series_component.js new file mode 120000 index 000000000..fd5249567 --- /dev/null +++ b/app/components/time_series_component/time_series_component.js @@ -0,0 +1 @@ +../../javascript/src/javascripts/time_series_component.js \ No newline at end of file diff --git a/app/controllers/reports_controller.rb b/app/controllers/reports_controller.rb new file mode 100644 index 000000000..230249ff0 --- /dev/null +++ b/app/controllers/reports_controller.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +class ReportsController < ApplicationController + respond_to :html, :json, :xml + + def index + end + + def show + @report = params[:id] + @mode = params.dig(:search, :mode) || "chart" + + case @report + when "posts" + @model = Post + @title = "Posts Report" + @columns = { posts: "COUNT(*)", uploaders: "COUNT(distinct uploader_id)" } + when "post_votes" + @model = PostVote + @title = "Post Votes Report" + @columns = { votes: "COUNT(*)", posts: "COUNT(distinct post_id)", voters: "COUNT(distinct user_id)" } + when "pools" + @model = Pool + @title = "Pools Report" + @columns = { series_pools: "COUNT(*) FILTER (WHERE category = 'series')", collection_pools: "COUNT(*) FILTER (WHERE category = 'collection')" } + when "comments" + @model = Comment + @title = "Comments Report" + @columns = { comments: "COUNT(*)", commenters: "COUNT(distinct creator_id)" } + when "comment_votes" + @model = CommentVote + @title = "Comment Votes Report" + @columns = { votes: "COUNT(*)", comments: "COUNT(distinct comment_id)", voters: "COUNT(distinct user_id)" } + when "forum_posts" + @model = ForumPost + @title = "Forum Posts Report" + @columns = { forum_posts: "COUNT(*)", posters: "COUNT(distinct creator_id)" } + when "bulk_update_requests" + @model = BulkUpdateRequest + @title = "Bulk Update Requests Report" + @columns = { requests: "COUNT(*)", requestors: "COUNT(distinct user_id)" } + when "tag_aliases" + @model = TagAlias + @title = "Tag Aliases Report" + @columns = { aliases: "COUNT(*)" } + when "tag_implications" + @model = TagImplication + @title = "Tag Implications Report" + @columns = { aliases: "COUNT(*)" } + when "artist_versions" + @model = ArtistVersion + @title = "Artist Edits Report" + @columns = { artist_edits: "COUNT(*)", artists: "COUNT(distinct artist_id)", editors: "COUNT(distinct updater_id)" } + when "note_versions" + @model = NoteVersion + @title = "Note Edits Report" + @columns = { note_edits: "COUNT(*)", posts: "COUNT(distinct post_id)", editors: "COUNT(distinct updater_id)" } + when "wiki_page_versions" + @model = WikiPageVersion + @title = "Wiki Edits Report" + @columns = { wiki_edits: "COUNT(*)", editors: "COUNT(distinct updater_id)" } + when "users" + @model = User + @title = "New Users Report" + @columns = { users: "COUNT(*)" } + when "bans" + @model = Ban + @title = "Bans Report" + @columns = { bans: "COUNT(*)", banners: "COUNT(DISTINCT banner_id)" } + else + raise ActiveRecord::RecordNotFound + end + + @period = params.dig(:search, :period)&.downcase || "day" + @from = params.dig(:search, :from) || 1.month.ago + @to = params.dig(:search, :to) || Time.zone.now + + @results = @model.search(params[:search], CurrentUser.user).timeseries(period: @period, from: @from, to: @to, columns: @columns) + respond_with(@results) + end +end diff --git a/app/javascript/packs/application.js b/app/javascript/packs/application.js index 3a7878da7..a1abfc62e 100644 --- a/app/javascript/packs/application.js +++ b/app/javascript/packs/application.js @@ -55,6 +55,7 @@ import PreviewSizeMenuComponent from "../src/javascripts/preview_size_menu_compo import RelatedTag from "../src/javascripts/related_tag.js"; import Shortcuts from "../src/javascripts/shortcuts.js"; import TagCounter from "../src/javascripts/tag_counter.js"; +import TimeSeriesComponent from "../src/javascripts/time_series_component.js"; import Upload from "../src/javascripts/uploads.js"; import UserTooltip from "../src/javascripts/user_tooltips.js"; import Utility from "../src/javascripts/utility.js"; @@ -82,6 +83,7 @@ Danbooru.PreviewSizeMenuComponent = PreviewSizeMenuComponent; Danbooru.RelatedTag = RelatedTag; Danbooru.Shortcuts = Shortcuts; Danbooru.TagCounter = TagCounter; +Danbooru.TimeSeriesComponent = TimeSeriesComponent; Danbooru.Upload = Upload; Danbooru.UserTooltip = UserTooltip; Danbooru.Utility = Utility; diff --git a/app/javascript/src/javascripts/current_user.js b/app/javascript/src/javascripts/current_user.js index e38ae191f..a97fa9b7e 100644 --- a/app/javascript/src/javascripts/current_user.js +++ b/app/javascript/src/javascripts/current_user.js @@ -11,4 +11,10 @@ CurrentUser.update = function(settings) { }); }; +CurrentUser.darkMode = function() { + let theme = CurrentUser.data("theme"); + + return theme === "dark" || (theme === "auto" && window.matchMedia("(prefers-color-scheme: dark)").matches); +}; + export default CurrentUser; diff --git a/app/javascript/src/javascripts/time_series_component.js b/app/javascript/src/javascripts/time_series_component.js new file mode 100644 index 000000000..ffaa9b1a8 --- /dev/null +++ b/app/javascript/src/javascripts/time_series_component.js @@ -0,0 +1,72 @@ +import * as echarts from "echarts"; +import startCase from "lodash/startCase"; +import CurrentUser from "./current_user.js"; + +export default class TimeSeriesComponent { + constructor({ container = null, data = [], columns = [], theme = null } = {}) { + this.container = container; + this.data = data; + this.columns = columns; + this.theme = CurrentUser.darkMode() ? "dark" : null; + + this.options = { + dataset: { + dimensions: ["date", ...columns], + source: data, + }, + tooltip: { + trigger: "axis", + axisPointer: { + type: "cross", + label: { + backgroundColor: '#6a7985' + } + } + }, + toolbox: { + feature: { + dataView: {}, + dataZoom: { + yAxisIndex: "none" + }, + magicType: { + type: ["line", "bar"], + }, + restore: {}, + saveAsImage: {} + } + }, + dataZoom: [ + { type: "inside" }, + { type: "slider" } + ], + grid: { + left: "1%", + right: "1%", + containLabel: true + }, + legend: { + data: columns.map(startCase), + }, + xAxis: { type: "time" }, + yAxis: columns.map(name => ({ type: "value" })), + series: columns.map(name => ({ + name: startCase(name), + type: "line", + areaStyle: {}, + emphasis: { + focus: "series" + }, + encode: { + x: "date", + y: name + } + })) + }; + + this.chart = echarts.init(container, this.theme); + this.chart.setOption(this.options); + + $(window).on("resize.danbooru", () => this.chart.resize()); + } +} diff --git a/app/logical/concerns/aggregatable.rb b/app/logical/concerns/aggregatable.rb new file mode 100644 index 000000000..26b3aa9e2 --- /dev/null +++ b/app/logical/concerns/aggregatable.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +module Aggregatable + extend ActiveSupport::Concern + + def timeseries(period: "day", date_column: :created_at, from: first[date_column], to: Time.now.utc, columns: { count: "COUNT(*)" }) + raise ArgumentError, "invalid period: #{period}" if !period.in?(%w[second minute hour day week month quarter year]) + + from = from.to_date + to = to.to_date + + # SELECT + # date_trunc('day', posts.created_at) AS date + # COUNT(*) AS count + # FROM posts + # WHERE posts.created_at BETWEEN from AND to + # GROUP BY date + subquery = select(date_trunc(period, date_column).as("date")).where(date_column => (from..to)).group("date").reorder(nil) + columns.each do |name, sql| + # SELECT COUNT(*) AS count + subquery = subquery.select(Arel.sql(sql).as(name.to_s).to_sql) + end + + # SELECT date_trunc('day', dates) AS date FROM generate_series('2022-01-01', '2022-02-15', '1 day'::interval) AS dates + dates = "SELECT #{date_trunc(period, Arel.sql("dates")).to_sql} AS date FROM #{generate_timeseries(from, to, period).to_sql} AS dates" + + # SELECT + # date_trunc('day', dates.date) AS date, + # COALESCE(subquery.count, 0) AS count + # FROM ( + # SELECT date_trunc('day', dates) AS date + # FROM generate_series(from, to, '1 day'::interval) AS dates + # ) AS dates + # LEFT OUTER JOIN ( + # SELECT + # date_trunc('day', posts.created_at) AS date, + # COUNT(*) AS count + # FROM posts + # WHERE posts.created_at BETWEEN from AND to + # GROUP BY date + # ) AS subquery + # ORDER BY date DESC + query = + unscoped. + select(date_trunc(period, Arel.sql("dates.date")).as("date")). + from("(#{dates}) AS dates"). + joins("LEFT OUTER JOIN (#{subquery.to_sql}) AS subquery ON subquery.date = dates.date"). + order("date DESC") + + columns.each do |name, sql| + # SELECT COALESCE(subquery.count, 0) AS count + query = query.select(coalesce(Arel.sql("subquery.#{connection.quote_column_name(name)}"), 0).as(name.to_s)) + end + + query.select_all + end + + def group_by_period(period = "day", column = :created_at) + select(date_trunc(period, column).as("date")).group("date").order(Arel.sql("date DESC")) + end + + def select_all + connection.select_all(all) + end + + def date_trunc(field, column) + sql_function(:date_trunc, field, column) + end + + def coalesce(column, value) + sql_function(:coalesce, column, value) + end + + def generate_series(from, to, interval) + sql_function(:generate_series, from, to, interval) + end + + def generate_timeseries(from, to, interval) + generate_series(from, to, Arel.sql("#{connection.quote("1 #{interval}")}::interval")) + end +end diff --git a/app/logical/concerns/searchable.rb b/app/logical/concerns/searchable.rb index 33ff16b1a..c1a13a061 100644 --- a/app/logical/concerns/searchable.rb +++ b/app/logical/concerns/searchable.rb @@ -744,16 +744,15 @@ module Searchable end def sql_value(value) - if Arel.arel_node?(value) + case value + in _ if Arel.arel_node?(value) value - elsif value.is_a?(String) - Arel::Nodes.build_quoted(value) - elsif value.is_a?(Symbol) + in Symbol arel_table[value] - elsif value.is_a?(Array) + in Array sql_array(value) else - raise ArgumentError + Arel::Nodes.build_quoted(value) end end diff --git a/app/models/application_record.rb b/app/models/application_record.rb index e1520bbd7..baa3cad66 100644 --- a/app/models/application_record.rb +++ b/app/models/application_record.rb @@ -10,6 +10,7 @@ class ApplicationRecord < ActiveRecord::Base include HasDtextLinks extend HasBitFlags extend Searchable + extend Aggregatable concerning :PaginationMethods do class_methods do diff --git a/app/views/reports/index.html.erb b/app/views/reports/index.html.erb new file mode 100644 index 000000000..216b465dc --- /dev/null +++ b/app/views/reports/index.html.erb @@ -0,0 +1,24 @@ +<% page_title "Reports" %> + +
+
+

Reports

+ +
    +
  • <%= link_to "Posts", report_path("posts") %>
    • +
  • <%= link_to "Post votes", report_path("post_votes") %>
    • +
  • <%= link_to "Pools", report_path("pools") %>
    • +
  • <%= link_to "Comments", report_path("comments") %>
    • +
  • <%= link_to "Comment votes", report_path("comment_votes") %>
    • +
  • <%= link_to "Forum posts", report_path("forum_posts") %>
    • +
  • <%= link_to "Bulk update requests", report_path("bulk_update_requests") %>
    • +
  • <%= link_to "Tag aliases", report_path("tag_aliases") %>
    • +
  • <%= link_to "Tag implications", report_path("tag_implications") %>
    • +
  • <%= link_to "Artist edits", report_path("artist_versions") %>
    • +
  • <%= link_to "Note edits", report_path("note_versions") %>
    • +
  • <%= link_to "Wiki edits", report_path("wiki_page_versions") %>
    • +
  • <%= link_to "New users", report_path("users") %>
    • +
  • <%= link_to "Bans", report_path("bans") %>
    • +
+
+
diff --git a/app/views/reports/show.html.erb b/app/views/reports/show.html.erb new file mode 100644 index 000000000..c219e4d4c --- /dev/null +++ b/app/views/reports/show.html.erb @@ -0,0 +1,22 @@ +<% page_title @title %> + +
+
+

<%= @title %>

+ <%= link_to "« Back", reports_path, class: "text-xs" %> + + <%= search_form_for(current_page_path) do |f| %> + <% if @report == "posts" %> + <%= f.input :tags, label: "Tags", input_html: { value: params[:search][:tags], data: { autocomplete: "tag-query" } } %> + <% end %> + + <%= f.input :from, as: :date, html5: true, input_html: { value: params[:search][:from] || 1.month.ago.to_date } %> + <%= f.input :to, as: :date, html5: true, input_html: { value: params[:search][:to] || Time.zone.now.to_date } %> + <%= f.input :period, collection: %w[Day Week Month Year], selected: params[:search][:period] %> + <%= f.input :mode, as: :hidden, input_html: { value: params[:search][:mode] } %> + <%= f.submit "Search" %> + <% end %> + + <%= render TimeSeriesComponent.new(@results, @columns.keys, mode: @mode) %> +
+
diff --git a/app/views/static/site_map.html.erb b/app/views/static/site_map.html.erb index cd2407b29..b180c1495 100644 --- a/app/views/static/site_map.html.erb +++ b/app/views/static/site_map.html.erb @@ -73,6 +73,7 @@