Add basic tables and graphs for various tables.

Add basic tables and graphs for viewing things like uploads over time, new users
over time, comments over time, etc. Located at https://betabooru.donmai.us/reports.

The graphing uses Apache ECharts: https://echarts.apache.org/en/index.html.

app/components/time_series_component.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class TimeSeriesComponent < ApplicationComponent
+  delegate :current_page_path, :search_params, to: :helpers
+
+  attr_reader :results, :columns, :mode
+
+  def initialize(results, columns, mode: :table)
+    @results = results
+    @columns = columns
+    @mode = mode.to_sym
+  end
+end

app/components/time_series_component/time_series_component.html.erb

@@ -0,0 +1,43 @@
+<div class="flex justify-end text-xs mb-2">
+  <% if mode == :table %>
+    <%= link_to "Chart", current_page_path(search: { **search_params, mode: "chart" }) %>
+  <% else %>
+    <%= link_to "Table", current_page_path(search: { **search_params, mode: "table" }) %>
+  <% end %>
+</div>
+
+<% if mode == :table %>
+  <table class="striped autofit" width="100%">
+    <thead>
+      <th>Date</th>
+
+      <% columns.each do |column| %>
+        <%= tag.th(column.to_s.capitalize, class: ("col-expand" if column == columns.last)) %>
+      <% end %>
+    </thead>
+
+    <tbody>
+      <% results.each do |row| %>
+        <tr>
+          <td><%= row["date"].to_date %></td>
+
+          <% columns.each do |column| %>
+            <td><%= row[column.to_s] %></td>
+          <% end %>
+        <tr>
+      <% end %>
+    </tbody>
+  </table>
+<% elsif mode == :chart %>
+  <div class="line-chart" style="width: 100%; height: 80vh;"></div>
+
+  <script type="text/javascript">
+    var data = <%= raw results.to_a.to_json %>;
+    var columns = <%= raw columns.to_json %>;
+    var chart = new Danbooru.TimeSeriesComponent({
+      container: $(".line-chart").get(0),
+      data: data,
+      columns: columns,
+    });
+  </script>
+<% end %>

app/components/time_series_component/time_series_component.js

@@ -0,0 +1 @@
+../../javascript/src/javascripts/time_series_component.js

app/controllers/reports_controller.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+class ReportsController < ApplicationController
+  respond_to :html, :json, :xml
+
+  def index
+  end
+
+  def show
+    @report = params[:id]
+    @mode = params.dig(:search, :mode) || "chart"
+
+    case @report
+    when "posts"
+      @model = Post
+      @title = "Posts Report"
+      @columns = { posts: "COUNT(*)", uploaders: "COUNT(distinct uploader_id)" }
+    when "post_votes"
+      @model = PostVote
+      @title = "Post Votes Report"
+      @columns = { votes: "COUNT(*)", posts: "COUNT(distinct post_id)", voters: "COUNT(distinct user_id)" }
+    when "pools"
+      @model = Pool
+      @title = "Pools Report"
+      @columns = { series_pools: "COUNT(*) FILTER (WHERE category = 'series')", collection_pools: "COUNT(*) FILTER (WHERE category = 'collection')" }
+    when "comments"
+      @model = Comment
+      @title = "Comments Report"
+      @columns = { comments: "COUNT(*)", commenters: "COUNT(distinct creator_id)" }
+    when "comment_votes"
+      @model = CommentVote
+      @title = "Comment Votes Report"
+      @columns = { votes: "COUNT(*)", comments: "COUNT(distinct comment_id)", voters: "COUNT(distinct user_id)" }
+    when "forum_posts"
+      @model = ForumPost
+      @title = "Forum Posts Report"
+      @columns = { forum_posts: "COUNT(*)", posters: "COUNT(distinct creator_id)" }
+    when "bulk_update_requests"
+      @model = BulkUpdateRequest
+      @title = "Bulk Update Requests Report"
+      @columns = { requests: "COUNT(*)", requestors: "COUNT(distinct user_id)" }
+    when "tag_aliases"
+      @model = TagAlias
+      @title = "Tag Aliases Report"
+      @columns = { aliases: "COUNT(*)" }
+    when "tag_implications"
+      @model = TagImplication
+      @title = "Tag Implications Report"
+      @columns = { aliases: "COUNT(*)" }
+    when "artist_versions"
+      @model = ArtistVersion
+      @title = "Artist Edits Report"
+      @columns = { artist_edits: "COUNT(*)", artists: "COUNT(distinct artist_id)", editors: "COUNT(distinct updater_id)" }
+    when "note_versions"
+      @model = NoteVersion
+      @title = "Note Edits Report"
+      @columns = { note_edits: "COUNT(*)", posts: "COUNT(distinct post_id)", editors: "COUNT(distinct updater_id)" }
+    when "wiki_page_versions"
+      @model = WikiPageVersion
+      @title = "Wiki Edits Report"
+      @columns = { wiki_edits: "COUNT(*)", editors: "COUNT(distinct updater_id)" }
+    when "users"
+      @model = User
+      @title = "New Users Report"
+      @columns = { users: "COUNT(*)" }
+    when "bans"
+      @model = Ban
+      @title = "Bans Report"
+      @columns = { bans: "COUNT(*)", banners: "COUNT(DISTINCT banner_id)" }
+    else
+      raise ActiveRecord::RecordNotFound
+    end
+
+    @period = params.dig(:search, :period)&.downcase || "day"
+    @from = params.dig(:search, :from) || 1.month.ago
+    @to = params.dig(:search, :to) || Time.zone.now
+
+    @results = @model.search(params[:search], CurrentUser.user).timeseries(period: @period, from: @from, to: @to, columns: @columns)
+    respond_with(@results)
+  end
+end

app/javascript/packs/application.js

@@ -55,6 +55,7 @@ import PreviewSizeMenuComponent from "../src/javascripts/preview_size_menu_compo
 import RelatedTag from "../src/javascripts/related_tag.js";
 import Shortcuts from "../src/javascripts/shortcuts.js";
 import TagCounter from "../src/javascripts/tag_counter.js";
+import TimeSeriesComponent from "../src/javascripts/time_series_component.js";
 import Upload from "../src/javascripts/uploads.js";
 import UserTooltip from "../src/javascripts/user_tooltips.js";
 import Utility from "../src/javascripts/utility.js";
@@ -82,6 +83,7 @@ Danbooru.PreviewSizeMenuComponent = PreviewSizeMenuComponent;
 Danbooru.RelatedTag = RelatedTag;
 Danbooru.Shortcuts = Shortcuts;
 Danbooru.TagCounter = TagCounter;
+Danbooru.TimeSeriesComponent = TimeSeriesComponent;
 Danbooru.Upload = Upload;
 Danbooru.UserTooltip = UserTooltip;
 Danbooru.Utility = Utility;

app/javascript/src/javascripts/current_user.js

@@ -11,4 +11,10 @@ CurrentUser.update = function(settings) {
   });
 };
 
+CurrentUser.darkMode = function() {
+  let theme = CurrentUser.data("theme");
+
+  return theme === "dark" || (theme === "auto" && window.matchMedia("(prefers-color-scheme: dark)").matches);
+};
+
 export default CurrentUser;

app/javascript/src/javascripts/time_series_component.js

@@ -0,0 +1,72 @@
+import * as echarts from "echarts";
+import startCase from "lodash/startCase";
+import CurrentUser from "./current_user.js";
+
+export default class TimeSeriesComponent {
+  constructor({ container = null, data = [], columns = [], theme = null } = {}) {
+    this.container = container;
+    this.data = data;
+    this.columns = columns;
+    this.theme = CurrentUser.darkMode() ? "dark" : null;
+
+    this.options = {
+      dataset: {
+        dimensions: ["date", ...columns],
+        source: data,
+      },
+      tooltip: {
+        trigger: "axis",
+        axisPointer: {
+          type: "cross",
+          label: {
+            backgroundColor: '#6a7985'
+          }
+        }
+      },
+      toolbox: {
+        feature: {
+          dataView: {},
+          dataZoom: {
+            yAxisIndex: "none"
+          },
+          magicType: {
+            type: ["line", "bar"],
+          },
+          restore: {},
+          saveAsImage: {}
+        }
+      },
+      dataZoom: [
+        { type: "inside" },
+        { type: "slider" }
+      ],
+      grid: {
+        left: "1%",
+        right: "1%",
+        containLabel: true
+      },
+      legend: {
+        data: columns.map(startCase),
+      },
+      xAxis: { type: "time" },
+      yAxis: columns.map(name => ({ type: "value" })),
+      series: columns.map(name => ({
+        name: startCase(name),
+        type: "line",
+        areaStyle: {},
+        emphasis: {
+          focus: "series"
+        },
+        encode: {
+          x: "date",
+          y: name
+        }
+      }))
+    };
+
+    this.chart = echarts.init(container, this.theme);
+    this.chart.setOption(this.options);
+
+    $(window).on("resize.danbooru", () => this.chart.resize());
+  }
+}

app/logical/concerns/aggregatable.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module Aggregatable
+  extend ActiveSupport::Concern
+
+  def timeseries(period: "day", date_column: :created_at, from: first[date_column], to: Time.now.utc, columns: { count: "COUNT(*)" })
+    raise ArgumentError, "invalid period: #{period}" if !period.in?(%w[second minute hour day week month quarter year])
+
+    from = from.to_date
+    to = to.to_date
+
+    # SELECT
+    #   date_trunc('day', posts.created_at) AS date
+    #   COUNT(*) AS count
+    # FROM posts
+    # WHERE posts.created_at BETWEEN from AND to
+    # GROUP BY date
+    subquery = select(date_trunc(period, date_column).as("date")).where(date_column => (from..to)).group("date").reorder(nil)
+    columns.each do |name, sql|
+      # SELECT COUNT(*) AS count
+      subquery = subquery.select(Arel.sql(sql).as(name.to_s).to_sql)
+    end
+
+    # SELECT date_trunc('day', dates) AS date FROM generate_series('2022-01-01', '2022-02-15', '1 day'::interval) AS dates
+    dates = "SELECT #{date_trunc(period, Arel.sql("dates")).to_sql} AS date FROM #{generate_timeseries(from, to, period).to_sql} AS dates"
+
+    # SELECT
+    #   date_trunc('day', dates.date) AS date,
+    #   COALESCE(subquery.count, 0) AS count
+    # FROM (
+    #   SELECT date_trunc('day', dates) AS date
+    #   FROM generate_series(from, to, '1 day'::interval) AS dates
+    # ) AS dates
+    # LEFT OUTER JOIN (
+    #   SELECT
+    #     date_trunc('day', posts.created_at) AS date,
+    #     COUNT(*) AS count
+    #   FROM posts
+    #   WHERE posts.created_at BETWEEN from AND to
+    #   GROUP BY date
+    # ) AS subquery
+    # ORDER BY date DESC
+    query =
+      unscoped.
+      select(date_trunc(period, Arel.sql("dates.date")).as("date")).
+      from("(#{dates}) AS dates").
+      joins("LEFT OUTER JOIN (#{subquery.to_sql}) AS subquery ON subquery.date = dates.date").
+      order("date DESC")
+
+    columns.each do |name, sql|
+      # SELECT COALESCE(subquery.count, 0) AS count
+      query = query.select(coalesce(Arel.sql("subquery.#{connection.quote_column_name(name)}"), 0).as(name.to_s))
+    end
+
+    query.select_all
+  end
+
+  def group_by_period(period = "day", column = :created_at)
+    select(date_trunc(period, column).as("date")).group("date").order(Arel.sql("date DESC"))
+  end
+
+  def select_all
+    connection.select_all(all)
+  end
+
+  def date_trunc(field, column)
+    sql_function(:date_trunc, field, column)
+  end
+
+  def coalesce(column, value)
+    sql_function(:coalesce, column, value)
+  end
+
+  def generate_series(from, to, interval)
+    sql_function(:generate_series, from, to, interval)
+  end
+
+  def generate_timeseries(from, to, interval)
+    generate_series(from, to, Arel.sql("#{connection.quote("1 #{interval}")}::interval"))
+  end
+end

app/logical/concerns/searchable.rb

@@ -744,16 +744,15 @@ module Searchable
   end
 
   def sql_value(value)
-    if Arel.arel_node?(value)
+    case value
+    in _ if Arel.arel_node?(value)
       value
-    elsif value.is_a?(String)
-      Arel::Nodes.build_quoted(value)
-    elsif value.is_a?(Symbol)
+    in Symbol
       arel_table[value]
-    elsif value.is_a?(Array)
+    in Array
       sql_array(value)
     else
-      raise ArgumentError
+      Arel::Nodes.build_quoted(value)
     end
   end
 

app/models/application_record.rb

@@ -10,6 +10,7 @@ class ApplicationRecord < ActiveRecord::Base
   include HasDtextLinks
   extend HasBitFlags
   extend Searchable
+  extend Aggregatable
 
   concerning :PaginationMethods do
     class_methods do

app/views/reports/index.html.erb

@@ -0,0 +1,24 @@
+<% page_title "Reports" %>
+
+<div id="c-reports">
+  <div id="a-index">
+    <h1>Reports</h1>
+
+    <ul class="list-bulleted">
+      <li><%= link_to "Posts", report_path("posts") %></li>
+      <li><%= link_to "Post votes", report_path("post_votes") %></li>
+      <li><%= link_to "Pools", report_path("pools") %></li>
+      <li><%= link_to "Comments", report_path("comments") %></li>
+      <li><%= link_to "Comment votes", report_path("comment_votes") %></li>
+      <li><%= link_to "Forum posts", report_path("forum_posts") %></li>
+      <li><%= link_to "Bulk update requests", report_path("bulk_update_requests") %></li>
+      <li><%= link_to "Tag aliases", report_path("tag_aliases") %></li>
+      <li><%= link_to "Tag implications", report_path("tag_implications") %></li>
+      <li><%= link_to "Artist edits", report_path("artist_versions") %></li>
+      <li><%= link_to "Note edits", report_path("note_versions") %></li>
+      <li><%= link_to "Wiki edits", report_path("wiki_page_versions") %></li>
+      <li><%= link_to "New users", report_path("users") %></li>
+      <li><%= link_to "Bans", report_path("bans") %></li>
+    </ul>
+  </div>
+</div>

app/views/reports/show.html.erb

@@ -0,0 +1,22 @@
+<% page_title @title %>
+
+<div id="c-reports">
+  <div id="a-show">
+    <h1><%= @title %></h1>
+    <%= link_to "« Back", reports_path, class: "text-xs" %>
+
+    <%= search_form_for(current_page_path) do |f| %>
+      <% if @report == "posts" %>
+        <%= f.input :tags, label: "Tags", input_html: { value: params[:search][:tags], data: { autocomplete: "tag-query" } } %>
+      <% end %>
+
+      <%= f.input :from, as: :date, html5: true, input_html: { value: params[:search][:from] || 1.month.ago.to_date } %>
+      <%= f.input :to, as: :date, html5: true, input_html: { value: params[:search][:to] || Time.zone.now.to_date } %>
+      <%= f.input :period, collection: %w[Day Week Month Year], selected: params[:search][:period] %>
+      <%= f.input :mode, as: :hidden, input_html: { value: params[:search][:mode] } %>
+      <%= f.submit "Search" %>
+    <% end %>
+
+    <%= render TimeSeriesComponent.new(@results, @columns.keys, mode: @mode) %>
+  </div>
+</div>

app/views/static/site_map.html.erb

@@ -73,6 +73,7 @@
 
       <ul>
         <li><h2>Reports</h2></li>
+        <li><%= link_to("Reports", reports_path) %></li>
         <li><%= link_to("User Reports", "https://isshiki.donmai.us/user-reports") %></li>
         <li><%= link_to("Top Searches", searches_explore_posts_path) %></li>
         <li><%= link_to("Missed Searches", missed_searches_explore_posts_path) %></li>