jaculabilis/danbooru
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add basic tables and graphs for various tables.

Browse Source 
Add basic tables and graphs for viewing things like uploads over time, new users
over time, comments over time, etc. Located at https://betabooru.donmai.us/reports.

The graphing uses Apache ECharts: https://echarts.apache.org/en/index.html.
This commit is contained in:
evazion
2022-10-20 05:20:22 -05:00
parent 412b7f2727
commit 7646521d0f
16 changed files with 384 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
app/components/time_series_component.rb Normal file
View File

@@ -0,0 +1,13 @@
# frozen_string_literal: true
class TimeSeriesComponent < ApplicationComponent
delegate :current_page_path, :search_params, to: :helpers
attr_reader :results, :columns, :mode
def initialize(results, columns, mode: :table)
@results = results
@columns = columns
@mode = mode.to_sym
end
end

43
app/components/time_series_component/time_series_component.html.erb Normal file
View File

@@ -0,0 +1,43 @@
<div class="flex justify-end text-xs mb-2">
<% if mode == :table %>
<%= link_to "Chart", current_page_path(search: { **search_params, mode: "chart" }) %>
<% else %>
<%= link_to "Table", current_page_path(search: { **search_params, mode: "table" }) %>
<% end %>
</div>
<% if mode == :table %>
<table class="striped autofit" width="100%">
<thead>
<th>Date</th>
<% columns.each do |column| %>
<%= tag.th(column.to_s.capitalize, class: ("col-expand" if column == columns.last)) %>
<% end %>
</thead>
<tbody>
<% results.each do |row| %>
<tr>
<td><%= row["date"].to_date %></td>
<% columns.each do |column| %>
<td><%= row[column.to_s] %></td>
<% end %>
<tr>
<% end %>
</tbody>
</table>
<% elsif mode == :chart %>
<div class="line-chart" style="width: 100%; height: 80vh;"></div>
<script type="text/javascript">
var data = <%= raw results.to_a.to_json %>;
var columns = <%= raw columns.to_json %>;
var chart = new Danbooru.TimeSeriesComponent({
container: $(".line-chart").get(0),
data: data,
columns: columns,
});
</script>
<% end %>

1
app/components/time_series_component/time_series_component.js Symbolic link
View File

@@ -0,0 +1 @@
../../javascript/src/javascripts/time_series_component.js

81
app/controllers/reports_controller.rb Normal file
View File

@@ -0,0 +1,81 @@
# frozen_string_literal: true
class ReportsController < ApplicationController
respond_to :html, :json, :xml
def index
end
def show
@report = params[:id]
@mode = params.dig(:search, :mode) || "chart"
case @report
when "posts"
@model = Post
@title = "Posts Report"
@columns = { posts: "COUNT(*)", uploaders: "COUNT(distinct uploader_id)" }
when "post_votes"
@model = PostVote
@title = "Post Votes Report"
@columns = { votes: "COUNT(*)", posts: "COUNT(distinct post_id)", voters: "COUNT(distinct user_id)" }
when "pools"
@model = Pool
@title = "Pools Report"
@columns = { series_pools: "COUNT(*) FILTER (WHERE category = 'series')", collection_pools: "COUNT(*) FILTER (WHERE category = 'collection')" }
when "comments"
@model = Comment
@title = "Comments Report"
@columns = { comments: "COUNT(*)", commenters: "COUNT(distinct creator_id)" }
when "comment_votes"
@model = CommentVote
@title = "Comment Votes Report"
@columns = { votes: "COUNT(*)", comments: "COUNT(distinct comment_id)", voters: "COUNT(distinct user_id)" }
when "forum_posts"
@model = ForumPost
@title = "Forum Posts Report"
@columns = { forum_posts: "COUNT(*)", posters: "COUNT(distinct creator_id)" }
when "bulk_update_requests"
@model = BulkUpdateRequest
@title = "Bulk Update Requests Report"
@columns = { requests: "COUNT(*)", requestors: "COUNT(distinct user_id)" }
when "tag_aliases"
@model = TagAlias
@title = "Tag Aliases Report"
@columns = { aliases: "COUNT(*)" }
when "tag_implications"
@model = TagImplication
@title = "Tag Implications Report"
@columns = { aliases: "COUNT(*)" }
when "artist_versions"
@model = ArtistVersion
@title = "Artist Edits Report"
@columns = { artist_edits: "COUNT(*)", artists: "COUNT(distinct artist_id)", editors: "COUNT(distinct updater_id)" }
when "note_versions"
@model = NoteVersion
@title = "Note Edits Report"
@columns = { note_edits: "COUNT(*)", posts: "COUNT(distinct post_id)", editors: "COUNT(distinct updater_id)" }
when "wiki_page_versions"
@model = WikiPageVersion
@title = "Wiki Edits Report"
@columns = { wiki_edits: "COUNT(*)", editors: "COUNT(distinct updater_id)" }
when "users"
@model = User
@title = "New Users Report"
@columns = { users: "COUNT(*)" }
when "bans"
@model = Ban
@title = "Bans Report"
@columns = { bans: "COUNT(*)", banners: "COUNT(DISTINCT banner_id)" }
else
raise ActiveRecord::RecordNotFound
end
@period = params.dig(:search, :period)&.downcase || "day"
@from = params.dig(:search, :from) || 1.month.ago
@to = params.dig(:search, :to) || Time.zone.now
@results = @model.search(params[:search], CurrentUser.user).timeseries(period: @period, from: @from, to: @to, columns: @columns)
respond_with(@results)
end
end

2
app/javascript/packs/application.js
View File

@@ -55,6 +55,7 @@ import PreviewSizeMenuComponent from "../src/javascripts/preview_size_menu_compo
import RelatedTag from "../src/javascripts/related_tag.js";
import Shortcuts from "../src/javascripts/shortcuts.js";
import TagCounter from "../src/javascripts/tag_counter.js";
import TimeSeriesComponent from "../src/javascripts/time_series_component.js";
import Upload from "../src/javascripts/uploads.js";
import UserTooltip from "../src/javascripts/user_tooltips.js";
import Utility from "../src/javascripts/utility.js";
@@ -82,6 +83,7 @@ Danbooru.PreviewSizeMenuComponent = PreviewSizeMenuComponent;
Danbooru.RelatedTag = RelatedTag;
Danbooru.Shortcuts = Shortcuts;
Danbooru.TagCounter = TagCounter;
Danbooru.TimeSeriesComponent = TimeSeriesComponent;
Danbooru.Upload = Upload;
Danbooru.UserTooltip = UserTooltip;
Danbooru.Utility = Utility;

6
app/javascript/src/javascripts/current_user.js
View File

@@ -11,4 +11,10 @@ CurrentUser.update = function(settings) {
});
};
CurrentUser.darkMode = function() {
let theme = CurrentUser.data("theme");
return theme === "dark" || (theme === "auto" && window.matchMedia("(prefers-color-scheme: dark)").matches);
};
export default CurrentUser;

72
app/javascript/src/javascripts/time_series_component.js Normal file
View File

@@ -0,0 +1,72 @@
import * as echarts from "echarts";
import startCase from "lodash/startCase";
import CurrentUser from "./current_user.js";
export default class TimeSeriesComponent {
constructor({ container = null, data = [], columns = [], theme = null } = {}) {
this.container = container;
this.data = data;
this.columns = columns;
this.theme = CurrentUser.darkMode() ? "dark" : null;
this.options = {
dataset: {
dimensions: ["date", ...columns],
source: data,
},
tooltip: {
trigger: "axis",
axisPointer: {
type: "cross",
label: {
backgroundColor: '#6a7985'
}
}
},
toolbox: {
feature: {
dataView: {},
dataZoom: {
yAxisIndex: "none"
},
magicType: {
type: ["line", "bar"],
},
restore: {},
saveAsImage: {}
}
},
dataZoom: [
{ type: "inside" },
{ type: "slider" }
],
grid: {
left: "1%",
right: "1%",
containLabel: true
},
legend: {
data: columns.map(startCase),
},
xAxis: { type: "time" },
yAxis: columns.map(name => ({ type: "value" })),
series: columns.map(name => ({
name: startCase(name),
type: "line",
areaStyle: {},
emphasis: {
focus: "series"
},
encode: {
x: "date",
y: name
}
}))
};
this.chart = echarts.init(container, this.theme);
this.chart.setOption(this.options);
$(window).on("resize.danbooru", () => this.chart.resize());
}
}

81
app/logical/concerns/aggregatable.rb Normal file
View File

@@ -0,0 +1,81 @@
# frozen_string_literal: true
module Aggregatable
extend ActiveSupport::Concern
def timeseries(period: "day", date_column: :created_at, from: first[date_column], to: Time.now.utc, columns: { count: "COUNT(*)" })
raise ArgumentError, "invalid period: #{period}" if !period.in?(%w[second minute hour day week month quarter year])
from = from.to_date
to = to.to_date
# SELECT
# date_trunc('day', posts.created_at) AS date
# COUNT(*) AS count
# FROM posts
# WHERE posts.created_at BETWEEN from AND to
# GROUP BY date
subquery = select(date_trunc(period, date_column).as("date")).where(date_column => (from..to)).group("date").reorder(nil)
columns.each do |name, sql|
# SELECT COUNT(*) AS count
subquery = subquery.select(Arel.sql(sql).as(name.to_s).to_sql)
end
# SELECT date_trunc('day', dates) AS date FROM generate_series('2022-01-01', '2022-02-15', '1 day'::interval) AS dates
dates = "SELECT #{date_trunc(period, Arel.sql("dates")).to_sql} AS date FROM #{generate_timeseries(from, to, period).to_sql} AS dates"
# SELECT
# date_trunc('day', dates.date) AS date,
# COALESCE(subquery.count, 0) AS count
# FROM (
# SELECT date_trunc('day', dates) AS date
# FROM generate_series(from, to, '1 day'::interval) AS dates
# ) AS dates
# LEFT OUTER JOIN (
# SELECT
# date_trunc('day', posts.created_at) AS date,
# COUNT(*) AS count
# FROM posts
# WHERE posts.created_at BETWEEN from AND to
# GROUP BY date
# ) AS subquery
# ORDER BY date DESC
query =
unscoped.
select(date_trunc(period, Arel.sql("dates.date")).as("date")).
from("(#{dates}) AS dates").
joins("LEFT OUTER JOIN (#{subquery.to_sql}) AS subquery ON subquery.date = dates.date").
order("date DESC")
columns.each do |name, sql|
# SELECT COALESCE(subquery.count, 0) AS count
query = query.select(coalesce(Arel.sql("subquery.#{connection.quote_column_name(name)}"), 0).as(name.to_s))
end
query.select_all
end
def group_by_period(period = "day", column = :created_at)
select(date_trunc(period, column).as("date")).group("date").order(Arel.sql("date DESC"))
end
def select_all
connection.select_all(all)
end
def date_trunc(field, column)
sql_function(:date_trunc, field, column)
end
def coalesce(column, value)
sql_function(:coalesce, column, value)
end
def generate_series(from, to, interval)
sql_function(:generate_series, from, to, interval)
end
def generate_timeseries(from, to, interval)
generate_series(from, to, Arel.sql("#{connection.quote("1 #{interval}")}::interval"))
end
end

11
app/logical/concerns/searchable.rb
View File

@@ -744,16 +744,15 @@ module Searchable
end
def sql_value(value)
if Arel.arel_node?(value)
case value
in _ if Arel.arel_node?(value)
value
elsif value.is_a?(String)
Arel::Nodes.build_quoted(value)
elsif value.is_a?(Symbol)
in Symbol
arel_table[value]
elsif value.is_a?(Array)
in Array
sql_array(value)
else
raise ArgumentError
Arel::Nodes.build_quoted(value)
end
end

1
app/models/application_record.rb
View File

@@ -10,6 +10,7 @@ class ApplicationRecord < ActiveRecord::Base
include HasDtextLinks
extend HasBitFlags
extend Searchable
extend Aggregatable
concerning :PaginationMethods do
class_methods do

24
app/views/reports/index.html.erb Normal file
View File

@@ -0,0 +1,24 @@
<% page_title "Reports" %>
<div id="c-reports">
<div id="a-index">
<h1>Reports</h1>
<ul class="list-bulleted">
<li><%= link_to "Posts", report_path("posts") %></li>
<li><%= link_to "Post votes", report_path("post_votes") %></li>
<li><%= link_to "Pools", report_path("pools") %></li>
<li><%= link_to "Comments", report_path("comments") %></li>
<li><%= link_to "Comment votes", report_path("comment_votes") %></li>
<li><%= link_to "Forum posts", report_path("forum_posts") %></li>
<li><%= link_to "Bulk update requests", report_path("bulk_update_requests") %></li>
<li><%= link_to "Tag aliases", report_path("tag_aliases") %></li>
<li><%= link_to "Tag implications", report_path("tag_implications") %></li>
<li><%= link_to "Artist edits", report_path("artist_versions") %></li>
<li><%= link_to "Note edits", report_path("note_versions") %></li>
<li><%= link_to "Wiki edits", report_path("wiki_page_versions") %></li>
<li><%= link_to "New users", report_path("users") %></li>
<li><%= link_to "Bans", report_path("bans") %></li>
</ul>
</div>
</div>

22
app/views/reports/show.html.erb Normal file
View File

@@ -0,0 +1,22 @@
<% page_title @title %>
<div id="c-reports">
<div id="a-show">
<h1><%= @title %></h1>
<%= link_to "« Back", reports_path, class: "text-xs" %>
<%= search_form_for(current_page_path) do |f| %>
<% if @report == "posts" %>
<%= f.input :tags, label: "Tags", input_html: { value: params[:search][:tags], data: { autocomplete: "tag-query" } } %>
<% end %>
<%= f.input :from, as: :date, html5: true, input_html: { value: params[:search][:from] || 1.month.ago.to_date } %>
<%= f.input :to, as: :date, html5: true, input_html: { value: params[:search][:to] || Time.zone.now.to_date } %>
<%= f.input :period, collection: %w[Day Week Month Year], selected: params[:search][:period] %>
<%= f.input :mode, as: :hidden, input_html: { value: params[:search][:mode] } %>
<%= f.submit "Search" %>
<% end %>
<%= render TimeSeriesComponent.new(@results, @columns.keys, mode: @mode) %>
</div>
</div>

1
app/views/static/site_map.html.erb
View File

@@ -73,6 +73,7 @@
<ul>
<li><h2>Reports</h2></li>
<li><%= link_to("Reports", reports_path) %></li>
<li><%= link_to("User Reports", "https://isshiki.donmai.us/user-reports") %></li>
<li><%= link_to("Top Searches", searches_explore_posts_path) %></li>
<li><%= link_to("Missed Searches", missed_searches_explore_posts_path) %></li>

1
config/routes.rb
View File

@@ -230,6 +230,7 @@ Rails.application.routes.draw do
resources :rate_limits, only: [:index]
resource :related_tag, :only => [:show, :update]
resources :recommended_posts, only: [:index]
resources :reports, only: [:index, :show]
resources :robots, only: [:index]
resources :saved_searches, :except => [:show]
resource :session, only: [:new, :create, :destroy] do

1
package.json
View File

@@ -19,6 +19,7 @@
"core-js": "^3.20.3",
"css-loader": "^6.5.1",
"dropzone": "^6.0.0-beta.2",
"echarts": "^5.4.0",
"hammerjs": "^2.0.8",
"jquery": "^3.6.0",
"jquery-hotkeys": "^0.2.2",

33
yarn.lock
View File

@@ -2754,9 +2754,9 @@ __metadata:
linkType: hard
"caniuse-lite@npm:^1.0.30001400, caniuse-lite@npm:^1.0.30001407":
version: 1.0.30001421
resolution: "caniuse-lite@npm:1.0.30001421"
checksum: 184f673e19792e5103a5160cde1f7e1fb4758b55cf877bbf8618f72e7ea8cade8aa4b6bc46717d3fa6cb95186a6b04e32ad4f0ee586164b1c8aa20e57457b520
version: 1.0.30001422
resolution: "caniuse-lite@npm:1.0.30001422"
checksum: 045ad4a3af7629f0c9f5af0823bca5002ca1daa6fd47cca37aa14ae270adf1016b1e7acc6e1b6d72de60aaca004ffd1168628f1d2f6b08283153a8329e054339
languageName: node
linkType: hard
@@ -3291,6 +3291,16 @@ __metadata:
languageName: node
linkType: hard
"echarts@npm:^5.4.0":
version: 5.4.0
resolution: "echarts@npm:5.4.0"
dependencies:
tslib: 2.3.0
zrender: 5.4.0
checksum: b4336ba9fbc579a9e018770e773a9d9daa7eeae402357fb2def6c7e059734504645bbbcab9bfb87def1e9d78ff370a7b40082cbc2f72050b2966565b2564ca35
languageName: node
linkType: hard
"ee-first@npm:1.1.1":
version: 1.1.1
resolution: "ee-first@npm:1.1.1"
@@ -6474,6 +6484,7 @@ fsevents@~2.3.2:
core-js: ^3.20.3
css-loader: ^6.5.1
dropzone: ^6.0.0-beta.2
echarts: ^5.4.0
eslint: ^8.7.0
eslint-plugin-babel: ^5.3.1
eslint-plugin-ignore-erb: ^0.1.1
@@ -7301,6 +7312,13 @@ fsevents@~2.3.2:
languageName: node
linkType: hard
"tslib@npm:2.3.0":
version: 2.3.0
resolution: "tslib@npm:2.3.0"
checksum: 7b4fc9feff0f704743c3760f5d8d708f6417fac6458159e63df3a6b1100f0736e3b99edb9fe370f274ad15160a1f49ff05cb49402534c818ff552c48e38c3e6e
languageName: node
linkType: hard
"type-check@npm:^0.4.0, type-check@npm:~0.4.0":
version: 0.4.0
resolution: "type-check@npm:0.4.0"
@@ -7778,3 +7796,12 @@ fsevents@~2.3.2:
checksum: 096c3b40beb2804659539be1605a35c58eb0c85285f94b77b3e924f42ee265c1a40bf9f4153770039517146b469a964d51742395f35ca8135fc9f7e4982b785d
languageName: node
linkType: hard
"zrender@npm:5.4.0":
version: 5.4.0
resolution: "zrender@npm:5.4.0"
dependencies:
tslib: 2.3.0
checksum: ddd754f92e132939694ad4abf0914f7811696397370f1dfc684bde45dcaa5a5b0e66e7b60b062a3f3a2dbcd54393ec431cae447a4ea7db9c5307a5f70a0ea5c9
languageName: node
linkType: hard