jaculabilis/danbooru
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

continue refactoring savedsearch

Browse Source
This commit is contained in:
Albert Yi
2018-11-13 18:05:05 -08:00
parent df02eb7951
commit 0508b127fd
17 changed files with 171 additions and 186 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
Gemfile
View File

@@ -89,4 +89,5 @@ group :test do
gem "timecop" gem "timecop"
gem "webmock" gem "webmock"
gem "minitest-ci" gem "minitest-ci"
gem "mock_redis"
end end

2
Gemfile.lock
View File

@@ -232,6 +232,7 @@ GEM
minitest (>= 5.0.6) minitest (>= 5.0.6)
mocha (1.5.0) mocha (1.5.0)
metaclass (~> 0.0.1) metaclass (~> 0.0.1)
mock_redis (0.19.0)
msgpack (1.2.4) msgpack (1.2.4)
msgpack (1.2.4-x64-mingw32) msgpack (1.2.4-x64-mingw32)
multi_json (1.13.1) multi_json (1.13.1)
@@ -479,6 +480,7 @@ DEPENDENCIES
memoist memoist
minitest-ci minitest-ci
mocha mocha
mock_redis
net-sftp net-sftp
newrelic_rpm newrelic_rpm
oauth2 oauth2

1
Procfile
View File

@@ -3,4 +3,3 @@ jobs: bin/rake jobs:work
recommender: bundle exec ruby script/mock_services/recommender.rb recommender: bundle exec ruby script/mock_services/recommender.rb
iqdbs: bundle exec ruby script/mock_services/iqdbs.rb iqdbs: bundle exec ruby script/mock_services/iqdbs.rb
reportbooru: bundle exec ruby script/mock_services/reportbooru.rb reportbooru: bundle exec ruby script/mock_services/reportbooru.rb
listbooru: bundle exec ruby script/mock_services/listbooru.rb

5
README.md
View File

@@ -68,11 +68,6 @@ The following features requires a Google API account:
IQDB integration is delegated to the [IQDBS service](https://github.com/r888888888/iqdbs). IQDB integration is delegated to the [IQDBS service](https://github.com/r888888888/iqdbs).
### Listbooru Service
In order to access saved search functionality you will need to install and
configure the [Listbooru service](https://github.com/r888888888/listbooru).
### Archive Service ### Archive Service
In order to access versioned data for pools and posts you will In order to access versioned data for pools and posts you will

2
app/controllers/saved_searches_controller.rb
View File

@@ -50,7 +50,7 @@ class SavedSearchesController < ApplicationController
def check_availability def check_availability
if !SavedSearch.enabled? if !SavedSearch.enabled?
raise NotImplementedError.new("Listbooru service is not configured. Saved searches are not available.") raise NotImplementedError.new("Saved searches are not available.")
end end
end end

4
app/logical/post_query_builder.rb
View File

@@ -71,9 +71,9 @@ class PostQueryBuilder
if SavedSearch.enabled? if SavedSearch.enabled?
saved_searches.each do |saved_search| saved_searches.each do |saved_search|
if saved_search == "all" if saved_search == "all"
post_ids = SavedSearch.post_ids(CurrentUser.id) post_ids = SavedSearch.post_ids_for(CurrentUser.id)
else else
post_ids = SavedSearch.post_ids(CurrentUser.id, saved_search) post_ids = SavedSearch.post_ids_for(CurrentUser.id, label: saved_search)
end end
post_ids = [0] if post_ids.empty? post_ids = [0] if post_ids.empty?

117
app/models/saved_search.rb
View File

@@ -1,30 +1,35 @@
class SavedSearch < ApplicationRecord class SavedSearch < ApplicationRecord
REDIS_EXPIRY = 3600 REDIS_EXPIRY = 3600
QUERY_LIMIT = 500 QUERY_LIMIT = 1000
def self.enabled?
Danbooru.config.redis_url.present?
end
concerning :Redis do concerning :Redis do
class_methods do class_methods do
def enabled? extend Memoist
Danbooru.config.redis_url.present?
def redis
::Redis.new(url: Danbooru.config.redis_url)
end end
memoize :redis
def post_ids_for(user_id, label: nil) def post_ids_for(user_id, label: nil)
redis = Redis.new(url: Danbooru.config.redis_url)
label = normalize_label(label) if label label = normalize_label(label) if label
queries = queries_for(user_id, label: label) queries = queries_for(user_id, label: label)
post_ids = Set.new post_ids = Set.new
queries.each do |query| queries.each do |query|
query_hash = Cache.hash(query) redis_key = "search:#{query}"
redis_key = "search:#{query_hash}"
if redis.exists(redis_key) if redis.exists(redis_key)
sub_ids = redis.smembers(redis_key) sub_ids = redis.smembers(redis_key).map(&:to_i)
post_ids.merge(sub_ids) post_ids.merge(sub_ids)
redis.expire(redis_key, REDIS_EXPIRY) redis.expire(redis_key, REDIS_EXPIRY)
else else
SavedSearch.delay(queue: "default").populate(query) SavedSearch.delay(queue: "default").populate(query)
end end
end end
post_ids.to_a.sort post_ids.to_a.sort.last(QUERY_LIMIT)
end end
end end
end end
@@ -52,24 +57,39 @@ class SavedSearch < ApplicationRecord
labels labels
end end
def self.labels_for(user_id) def labels_for(user_id)
Cache.get(cache_key(user_id)) do SavedSearch.
SavedSearch. where(user_id: user_id).
where(user_id: user_id). order("label").
order("label"). pluck(Arel.sql("distinct unnest(labels) as label"))
pluck(Arel.sql("distinct unnest(labels) as label"))
end
end end
end end
def normalize_labels
self.labels = labels.
map {|x| SavedSearch.normalize_label(x)}.
reject {|x| x.blank?}
end
def label_string
labels.join(" ")
end
def label_string=(val)
self.labels = val.to_s.split(/[[:space:]]+/)
end
def labels=(labels)
labels = labels.map { |label| SavedSearch.normalize_label(label) }
super(labels)
end
end end
concerning :Search do concerning :Search do
class_methods do class_methods do
def populate(query) def populate(query)
CurrentUser.as_system do CurrentUser.as_system do
query_hash = Cache.hash(query) redis_key = "search:#{query}"
redis_key = "search:#{query_hash}"
redis = Redis.new(url: Danbooru.config.redis_url)
return if redis.exists(redis_key) return if redis.exists(redis_key)
post_ids = Post.tag_match(query, true).limit(QUERY_LIMIT).pluck(:id) post_ids = Post.tag_match(query, true).limit(QUERY_LIMIT).pluck(:id)
redis.sadd(redis_key, post_ids) redis.sadd(redis_key, post_ids)
@@ -81,35 +101,37 @@ class SavedSearch < ApplicationRecord
end end
end end
concerning :Queries do
class_methods do
def queries_for(user_id, label: nil, options: {})
SavedSearch.
where(user_id: user_id).
labeled(label).
pluck(:query).
map {|x| Tag.normalize_query(x, sort: true)}.
sort.
uniq
end
end
def normalized_query
Tag.normalize_query(query, sort: true)
end
def normalize_query
self.query = Tag.normalize_query(query, sort: false)
end
end
attr_accessor :disable_labels attr_accessor :disable_labels
belongs_to :user belongs_to :user
validates :query, presence: true validates :query, presence: true
validate :validate_count validate :validate_count
before_create :update_user_on_create before_create :update_user_on_create
after_destroy :update_user_on_destroy after_destroy :update_user_on_destroy
before_validation :normalize before_validation :normalize_query
scope :labeled, ->(label) { where("labels @> string_to_array(?, '~~~~')", label)} before_validation :normalize_labels
scope :labeled, ->(label) { label.present? ? where("labels @> string_to_array(?, '~~~~')", label) : where("true") }
def self.queries_for(user_id, label: nil, options: {})
SavedSearch.
where(user_id: user_id).
tap {|arel| label ? arel.labeled(label) : arel}.
pluck(:query).
map {|x| Tag.normalize_query(x, sort: true)}.
sort.
uniq
end
def normalized_query
Tag.normalize_query(query, sort: true)
end
def normalize
self.query = Tag.normalize_query(query, sort: false)
self.labels = labels.
map {|x| SavedSearch.normalize_label(x)}.
reject {|x| x.blank?}
end
def validate_count def validate_count
if user.saved_searches.count + 1 > user.max_saved_searches if user.saved_searches.count + 1 > user.max_saved_searches
@@ -129,19 +151,6 @@ class SavedSearch < ApplicationRecord
end end
end end
def label_string
labels.join(" ")
end
def label_string=(val)
self.labels = val.to_s.split(/[[:space:]]+/)
end
def labels=(labels)
labels = labels.map { |label| SavedSearch.normalize_label(label) }
super(labels)
end
def disable_labels=(value) def disable_labels=(value)
CurrentUser.update(disable_categorized_saved_searches: true) if value.to_s.truthy? CurrentUser.update(disable_categorized_saved_searches: true) if value.to_s.truthy?
end end

2
app/presenters/post_set_presenters/post.rb
View File

@@ -19,7 +19,7 @@ module PostSetPresenters
if post_set.is_pattern_search? if post_set.is_pattern_search?
pattern_tags pattern_tags
elsif post_set.is_saved_search? elsif post_set.is_saved_search?
SavedSearch.labels_for(CurrentUser.user.id).map {|x| "search:#{x}"} ["search:all"] + SavedSearch.labels_for(CurrentUser.user.id).map {|x| "search:#{x}"}
elsif post_set.is_empty_tag? || post_set.tag_string == "order:rank" elsif post_set.is_empty_tag? || post_set.tag_string == "order:rank"
popular_tags popular_tags
elsif post_set.is_single_tag? elsif post_set.is_single_tag?

7
config/danbooru_default_config.rb
View File

@@ -673,13 +673,6 @@ module Danbooru
def reportbooru_key def reportbooru_key
end end
# listbooru options - see https://github.com/r888888888/listbooru
def listbooru_server
end
def listbooru_auth_key
end
# iqdbs options - see https://github.com/r888888888/iqdbs # iqdbs options - see https://github.com/r888888888/iqdbs
def iqdbs_auth_key def iqdbs_auth_key
end end

53
db/structure.sql
View File

@@ -443,7 +443,6 @@ CREATE TABLE public.advertisement_hits (
-- --
CREATE SEQUENCE public.advertisement_hits_id_seq CREATE SEQUENCE public.advertisement_hits_id_seq
AS integer
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
@@ -482,7 +481,6 @@ CREATE TABLE public.advertisements (
-- --
CREATE SEQUENCE public.advertisements_id_seq CREATE SEQUENCE public.advertisements_id_seq
AS integer
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
@@ -514,7 +512,6 @@ CREATE TABLE public.amazon_backups (
-- --
CREATE SEQUENCE public.amazon_backups_id_seq CREATE SEQUENCE public.amazon_backups_id_seq
AS integer
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
@@ -546,7 +543,6 @@ CREATE TABLE public.anti_voters (
-- --
CREATE SEQUENCE public.anti_voters_id_seq CREATE SEQUENCE public.anti_voters_id_seq
AS integer
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
@@ -579,7 +575,6 @@ CREATE TABLE public.api_keys (
-- --
CREATE SEQUENCE public.api_keys_id_seq CREATE SEQUENCE public.api_keys_id_seq
AS integer
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
@@ -627,7 +622,6 @@ CREATE TABLE public.artist_commentaries (
-- --
CREATE SEQUENCE public.artist_commentaries_id_seq CREATE SEQUENCE public.artist_commentaries_id_seq
AS integer
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
@@ -665,7 +659,6 @@ CREATE TABLE public.artist_commentary_versions (
-- --
CREATE SEQUENCE public.artist_commentary_versions_id_seq CREATE SEQUENCE public.artist_commentary_versions_id_seq
AS integer
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
@@ -700,7 +693,6 @@ CREATE TABLE public.artist_urls (
-- --
CREATE SEQUENCE public.artist_urls_id_seq CREATE SEQUENCE public.artist_urls_id_seq
AS integer
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
@@ -740,7 +732,6 @@ CREATE TABLE public.artist_versions (
-- --
CREATE SEQUENCE public.artist_versions_id_seq CREATE SEQUENCE public.artist_versions_id_seq
AS integer
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
@@ -778,7 +769,6 @@ CREATE TABLE public.artists (
-- --
CREATE SEQUENCE public.artists_id_seq CREATE SEQUENCE public.artists_id_seq
AS integer
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
@@ -813,7 +803,6 @@ CREATE TABLE public.bans (
-- --
CREATE SEQUENCE public.bans_id_seq CREATE SEQUENCE public.bans_id_seq
AS integer
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
@@ -851,7 +840,6 @@ CREATE TABLE public.bulk_update_requests (
-- --
CREATE SEQUENCE public.bulk_update_requests_id_seq CREATE SEQUENCE public.bulk_update_requests_id_seq
AS integer
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
@@ -885,7 +873,6 @@ CREATE TABLE public.comment_votes (
-- --
CREATE SEQUENCE public.comment_votes_id_seq CREATE SEQUENCE public.comment_votes_id_seq
AS integer
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
@@ -927,7 +914,6 @@ CREATE TABLE public.comments (
-- --
CREATE SEQUENCE public.comments_id_seq CREATE SEQUENCE public.comments_id_seq
AS integer
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
@@ -967,7 +953,6 @@ CREATE TABLE public.delayed_jobs (
-- --
CREATE SEQUENCE public.delayed_jobs_id_seq CREATE SEQUENCE public.delayed_jobs_id_seq
AS integer
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
@@ -1000,7 +985,6 @@ CREATE TABLE public.dmail_filters (
-- --
CREATE SEQUENCE public.dmail_filters_id_seq CREATE SEQUENCE public.dmail_filters_id_seq
AS integer
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
@@ -1041,7 +1025,6 @@ CREATE TABLE public.dmails (
-- --
CREATE SEQUENCE public.dmails_id_seq CREATE SEQUENCE public.dmails_id_seq
AS integer
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
@@ -1077,7 +1060,6 @@ CREATE TABLE public.favorite_groups (
-- --
CREATE SEQUENCE public.favorite_groups_id_seq CREATE SEQUENCE public.favorite_groups_id_seq
AS integer
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
@@ -2108,7 +2090,6 @@ INHERITS (public.favorites);
-- --
CREATE SEQUENCE public.favorites_id_seq CREATE SEQUENCE public.favorites_id_seq
AS integer
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
@@ -2178,7 +2159,6 @@ CREATE TABLE public.forum_posts (
-- --
CREATE SEQUENCE public.forum_posts_id_seq CREATE SEQUENCE public.forum_posts_id_seq
AS integer
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
@@ -2211,7 +2191,6 @@ CREATE TABLE public.forum_subscriptions (
-- --
CREATE SEQUENCE public.forum_subscriptions_id_seq CREATE SEQUENCE public.forum_subscriptions_id_seq
AS integer
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
@@ -2245,7 +2224,6 @@ CREATE TABLE public.forum_topic_visits (
-- --
CREATE SEQUENCE public.forum_topic_visits_id_seq CREATE SEQUENCE public.forum_topic_visits_id_seq
AS integer
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
@@ -2286,7 +2264,6 @@ CREATE TABLE public.forum_topics (
-- --
CREATE SEQUENCE public.forum_topics_id_seq CREATE SEQUENCE public.forum_topics_id_seq
AS integer
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
@@ -2320,7 +2297,6 @@ CREATE TABLE public.ip_bans (
-- --
CREATE SEQUENCE public.ip_bans_id_seq CREATE SEQUENCE public.ip_bans_id_seq
AS integer
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
@@ -2355,7 +2331,6 @@ CREATE TABLE public.janitor_trials (
-- --
CREATE SEQUENCE public.janitor_trials_id_seq CREATE SEQUENCE public.janitor_trials_id_seq
AS integer
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
@@ -2389,7 +2364,6 @@ CREATE TABLE public.mod_actions (
-- --
CREATE SEQUENCE public.mod_actions_id_seq CREATE SEQUENCE public.mod_actions_id_seq
AS integer
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
@@ -2423,7 +2397,6 @@ CREATE TABLE public.news_updates (
-- --
CREATE SEQUENCE public.news_updates_id_seq CREATE SEQUENCE public.news_updates_id_seq
AS integer
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
@@ -2465,7 +2438,6 @@ CREATE TABLE public.note_versions (
-- --
CREATE SEQUENCE public.note_versions_id_seq CREATE SEQUENCE public.note_versions_id_seq
AS integer
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
@@ -2506,7 +2478,6 @@ CREATE TABLE public.notes (
-- --
CREATE SEQUENCE public.notes_id_seq CREATE SEQUENCE public.notes_id_seq
AS integer
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
@@ -2538,7 +2509,6 @@ CREATE TABLE public.pixiv_ugoira_frame_data (
-- --
CREATE SEQUENCE public.pixiv_ugoira_frame_data_id_seq CREATE SEQUENCE public.pixiv_ugoira_frame_data_id_seq
AS integer
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
@@ -2576,7 +2546,6 @@ CREATE TABLE public.pools (
-- --
CREATE SEQUENCE public.pools_id_seq CREATE SEQUENCE public.pools_id_seq
AS integer
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
@@ -2611,7 +2580,6 @@ CREATE TABLE public.post_appeals (
-- --
CREATE SEQUENCE public.post_appeals_id_seq CREATE SEQUENCE public.post_appeals_id_seq
AS integer
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
@@ -2644,7 +2612,6 @@ CREATE TABLE public.post_approvals (
-- --
CREATE SEQUENCE public.post_approvals_id_seq CREATE SEQUENCE public.post_approvals_id_seq
AS integer
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
@@ -2679,7 +2646,6 @@ CREATE TABLE public.post_disapprovals (
-- --
CREATE SEQUENCE public.post_disapprovals_id_seq CREATE SEQUENCE public.post_disapprovals_id_seq
AS integer
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
@@ -2715,7 +2681,6 @@ CREATE TABLE public.post_flags (
-- --
CREATE SEQUENCE public.post_flags_id_seq CREATE SEQUENCE public.post_flags_id_seq
AS integer
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
@@ -2760,7 +2725,6 @@ CREATE TABLE public.post_replacements (
-- --
CREATE SEQUENCE public.post_replacements_id_seq CREATE SEQUENCE public.post_replacements_id_seq
AS integer
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
@@ -2803,7 +2767,6 @@ CREATE TABLE public.post_votes (
-- --
CREATE SEQUENCE public.post_votes_id_seq CREATE SEQUENCE public.post_votes_id_seq
AS integer
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
@@ -2874,7 +2837,6 @@ CREATE TABLE public.posts (
-- --
CREATE SEQUENCE public.posts_id_seq CREATE SEQUENCE public.posts_id_seq
AS integer
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
@@ -2908,7 +2870,6 @@ CREATE TABLE public.saved_searches (
-- --
CREATE SEQUENCE public.saved_searches_id_seq CREATE SEQUENCE public.saved_searches_id_seq
AS integer
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
@@ -2949,7 +2910,6 @@ CREATE TABLE public.super_voters (
-- --
CREATE SEQUENCE public.super_voters_id_seq CREATE SEQUENCE public.super_voters_id_seq
AS integer
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
@@ -2989,7 +2949,6 @@ CREATE TABLE public.tag_aliases (
-- --
CREATE SEQUENCE public.tag_aliases_id_seq CREATE SEQUENCE public.tag_aliases_id_seq
AS integer
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
@@ -3029,7 +2988,6 @@ CREATE TABLE public.tag_implications (
-- --
CREATE SEQUENCE public.tag_implications_id_seq CREATE SEQUENCE public.tag_implications_id_seq
AS integer
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
@@ -3067,7 +3025,6 @@ CREATE TABLE public.tag_subscriptions (
-- --
CREATE SEQUENCE public.tag_subscriptions_id_seq CREATE SEQUENCE public.tag_subscriptions_id_seq
AS integer
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
@@ -3104,7 +3061,6 @@ CREATE TABLE public.tags (
-- --
CREATE SEQUENCE public.tags_id_seq CREATE SEQUENCE public.tags_id_seq
AS integer
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
@@ -3169,7 +3125,6 @@ CREATE TABLE public.uploads (
-- --
CREATE SEQUENCE public.uploads_id_seq CREATE SEQUENCE public.uploads_id_seq
AS integer
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
@@ -3204,7 +3159,6 @@ CREATE TABLE public.user_feedback (
-- --
CREATE SEQUENCE public.user_feedback_id_seq CREATE SEQUENCE public.user_feedback_id_seq
AS integer
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
@@ -3242,7 +3196,6 @@ CREATE TABLE public.user_name_change_requests (
-- --
CREATE SEQUENCE public.user_name_change_requests_id_seq CREATE SEQUENCE public.user_name_change_requests_id_seq
AS integer
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
@@ -3275,7 +3228,6 @@ CREATE TABLE public.user_password_reset_nonces (
-- --
CREATE SEQUENCE public.user_password_reset_nonces_id_seq CREATE SEQUENCE public.user_password_reset_nonces_id_seq
AS integer
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
@@ -3334,7 +3286,6 @@ furry -rating:s'::text,
-- --
CREATE SEQUENCE public.users_id_seq CREATE SEQUENCE public.users_id_seq
AS integer
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
@@ -3373,7 +3324,6 @@ CREATE TABLE public.wiki_page_versions (
-- --
CREATE SEQUENCE public.wiki_page_versions_id_seq CREATE SEQUENCE public.wiki_page_versions_id_seq
AS integer
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
@@ -3413,7 +3363,6 @@ CREATE TABLE public.wiki_pages (
-- --
CREATE SEQUENCE public.wiki_pages_id_seq CREATE SEQUENCE public.wiki_pages_id_seq
AS integer
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
@@ -7413,7 +7362,7 @@ CREATE TRIGGER trigger_wiki_pages_on_update_for_other_names BEFORE INSERT OR UPD
-- PostgreSQL database dump complete -- PostgreSQL database dump complete
-- --
SET search_path TO public; SET search_path TO "$user", public;
INSERT INTO "schema_migrations" (version) VALUES INSERT INTO "schema_migrations" (version) VALUES
('20100204211522'), ('20100204211522'),

2
script/mock_services/README.md
View File

@@ -4,4 +4,4 @@ These are mocked services to be used for development purposes.
- recommender: port 3001 - recommender: port 3001
- iqdbs: port 3002 - iqdbs: port 3002
- reportbooru: port 3003 - reportbooru: port 3003
- listbooru: port 3004

8
script/mock_services/listbooru.rb
View File

@@ -1,8 +0,0 @@
require 'sinatra'
require 'json'
set :port, 3004
post '/v2/search' do
# todo
end

1
test/functional/saved_searches_controller_test.rb
View File

@@ -3,6 +3,7 @@ require 'test_helper'
class SavedSearchesControllerTest < ActionDispatch::IntegrationTest class SavedSearchesControllerTest < ActionDispatch::IntegrationTest
context "The saved searches controller" do context "The saved searches controller" do
setup do setup do
SavedSearch.stubs(:enabled?).returns(true)
@user = create(:user) @user = create(:user)
as_user do as_user do
@saved_search = create(:saved_search, user: @user) @saved_search = create(:saved_search, user: @user)

2
test/test_helpers/saved_search_test_helper.rb
View File

@@ -17,7 +17,5 @@ module SavedSearchTestHelper
service = mock_sqs_service.new service = mock_sqs_service.new
SavedSearch.stubs(:sqs_service).returns(service) SavedSearch.stubs(:sqs_service).returns(service)
Danbooru.config.stubs(:aws_sqs_saved_search_url).returns("http://localhost:3002") Danbooru.config.stubs(:aws_sqs_saved_search_url).returns("http://localhost:3002")
Danbooru.config.stubs(:listbooru_auth_key).returns("blahblahblah")
Danbooru.config.stubs(:listbooru_server).returns("http://localhost:3001")
end end
end end

42
test/unit/post_test.rb
View File

@@ -2265,25 +2265,35 @@ class PostTest < ActiveSupport::TestCase
assert_tag_match([post], "pixiv_id:none") assert_tag_match([post], "pixiv_id:none")
end end
# should "return posts for a pixiv novel id search" do context "saved searches" do
# url = "http://www.pixiv.net/novel/show.php?id=2156088" setup do
# post = FactoryBot.create(:post, :source => url) SavedSearch.stubs(:enabled?).returns(true)
# assert_equal(1, Post.tag_match("pixiv_novel_id:2156088").count) @post1 = FactoryBot.create(:post, tag_string: "aaa")
# end @post2 = FactoryBot.create(:post, tag_string: "bbb")
FactoryBot.create(:saved_search, query: "aaa", labels: ["zzz"], user: CurrentUser.user)
FactoryBot.create(:saved_search, query: "bbb", user: CurrentUser.user)
end
should "return posts for a search:<category> metatag" do context "labeled" do
post1 = FactoryBot.create(:post, tag_string: "aaa") should "work" do
post2 = FactoryBot.create(:post, tag_string: "bbb") SavedSearch.expects(:post_ids_for).with(CurrentUser.id, label: "zzz").returns([@post1.id])
FactoryBot.create(:saved_search, query: "aaa", labels: ["zzz"], user: CurrentUser.user) assert_tag_match([@post1], "search:zzz")
FactoryBot.create(:saved_search, query: "bbb", user: CurrentUser.user) end
end
SavedSearch.expects(:post_ids).with(CurrentUser.id, "zzz").returns([post1.id]) context "missing" do
SavedSearch.expects(:post_ids).with(CurrentUser.id, "uncategorized").returns([post2.id]) should "work" do
SavedSearch.expects(:post_ids).with(CurrentUser.id).returns([post1.id, post2.id]) SavedSearch.expects(:post_ids_for).with(CurrentUser.id, label: "uncategorized").returns([@post2.id])
assert_tag_match([@post2], "search:uncategorized")
end
end
assert_tag_match([post1], "search:zzz") context "all" do
assert_tag_match([post2], "search:uncategorized") should "work" do
assert_tag_match([post2, post1], "search:all") SavedSearch.expects(:post_ids_for).with(CurrentUser.id).returns([@post1.id, @post2.id])
assert_tag_match([@post2, @post1], "search:all")
end
end
end end
should "return posts for a rating:<s|q|e> metatag" do should "return posts for a rating:<s|q|e> metatag" do

88
test/unit/saved_search_test.rb
View File

@@ -6,7 +6,8 @@ class SavedSearchTest < ActiveSupport::TestCase
@user = FactoryBot.create(:user) @user = FactoryBot.create(:user)
CurrentUser.user = @user CurrentUser.user = @user
CurrentUser.ip_addr = "127.0.0.1" CurrentUser.ip_addr = "127.0.0.1"
mock_saved_search_service! @mock_redis = MockRedis.new
SavedSearch.stubs(:redis).returns(@mock_redis)
end end
def teardown def teardown
@@ -24,11 +25,6 @@ class SavedSearchTest < ActiveSupport::TestCase
should "fetch the labels used by a user" do should "fetch the labels used by a user" do
assert_equal(%w(blah zah), SavedSearch.labels_for(@user.id)) assert_equal(%w(blah zah), SavedSearch.labels_for(@user.id))
end end
should "expire when a search is updated" do
Cache.expects(:delete).once
FactoryBot.create(:saved_search, user: @user, query: "blah")
end
end end
context ".queries_for" do context ".queries_for" do
@@ -40,47 +36,81 @@ class SavedSearchTest < ActiveSupport::TestCase
end end
should "fetch the queries used by a user for a label" do should "fetch the queries used by a user for a label" do
assert_equal(%w(aaa), SavedSearch.queries_for(@user.id, "blah")) assert_equal(%w(aaa), SavedSearch.queries_for(@user.id, label: "blah"))
end end
should "return fully normalized queries" do should "fetch the queries used by a user without a label" do
assert_equal(["aaa", "aaa ccc"], SavedSearch.queries_for(@user.id)) assert_equal(["aaa", "aaa ccc"], SavedSearch.queries_for(@user.id))
end end
end end
context "Fetching the post ids for a search" do context ".search_labels" do
setup do
FactoryBot.create(:tag_alias, antecedent_name: "bbb", consequent_name: "ccc", creator: @user)
FactoryBot.create(:saved_search, user: @user, label_string: "blah", query: "aaa")
FactoryBot.create(:saved_search, user: @user, label_string: "blahbling", query: "CCC BBB AAA")
FactoryBot.create(:saved_search, user: @user, label_string: "qux", query: " aaa bbb ccc ")
end
should "fetch the queries used by a user for a label" do
assert_equal(%w(blah blahbling), SavedSearch.search_labels(@user.id, label: "blah"))
end
end
context ".post_ids_for" do
context "with a label" do context "with a label" do
setup do setup do
SavedSearch.expects(:queries_for).with(1, "blah").returns(%w(a b c)) SavedSearch.expects(:queries_for).with(1, label: "blah").returns(%w(a b c))
stub_request(:post, "http://localhost:3001/v2/search").to_return(:body => "1 2 3 4")
end end
should "return a list of ids" do context "without a primed cache" do
post_ids = SavedSearch.post_ids(1, "blah") should "delay processing three times" do
assert_equal([1,2,3,4], post_ids) SavedSearch.expects(:populate).times(3)
post_ids = SavedSearch.post_ids_for(1, label: "blah")
assert_equal([], post_ids)
end
end
context "with a primed cached" do
setup do
@mock_redis.sadd("search:a", 1)
@mock_redis.sadd("search:b", 2)
@mock_redis.sadd("search:c", 3)
end
should "fetch the post ids" do
SavedSearch.expects(:delay).never
post_ids = SavedSearch.post_ids_for(1, label: "blah")
assert_equal([1,2,3], post_ids)
end
end end
end end
context "without a label" do context "without a label" do
setup do setup do
SavedSearch.expects(:queries_for).with(1, nil).returns(%w(a b c)) SavedSearch.expects(:queries_for).with(1, label: nil).returns(%w(a b c))
stub_request(:post, "http://localhost:3001/v2/search").to_return(:body => "1 2 3 4")
end end
should "return a list of ids" do context "without a primed cache" do
post_ids = SavedSearch.post_ids(1) should "delay processing three times" do
assert_equal([1,2,3,4], post_ids) SavedSearch.expects(:populate).times(3)
end post_ids = SavedSearch.post_ids_for(1)
end assert_equal([], post_ids)
end
context "with a nonexistent label" do
setup do
SavedSearch.expects(:queries_for).with(1, "empty").returns([])
end end
should "return an empty list of ids" do context "with a primed cache" do
post_ids = SavedSearch.post_ids(1, "empty") setup do
assert_equal([], post_ids) @mock_redis.sadd("search:a", 1)
@mock_redis.sadd("search:b", 2)
@mock_redis.sadd("search:c", 3)
end
should "fetch the post ids" do
SavedSearch.expects(:delay).never
post_ids = SavedSearch.post_ids_for(1)
assert_equal([1,2,3], post_ids)
end
end end
end end
end end
@@ -88,7 +118,7 @@ class SavedSearchTest < ActiveSupport::TestCase
context "Creating a saved search" do context "Creating a saved search" do
setup do setup do
FactoryBot.create(:tag_alias, antecedent_name: "zzz", consequent_name: "yyy", creator: @user) FactoryBot.create(:tag_alias, antecedent_name: "zzz", consequent_name: "yyy", creator: @user)
@saved_search = @user.saved_searches.create(:query => " ZZZ xxx ") @saved_search = @user.saved_searches.create(query: " ZZZ xxx ")
end end
should "update the bitpref on the user" do should "update the bitpref on the user" do

20
test/unit/tag_alias_test.rb
View File

@@ -68,13 +68,19 @@ class TagAliasTest < ActiveSupport::TestCase
assert_nil(Cache.get("ta:#{Cache.hash("aaa")}")) assert_nil(Cache.get("ta:#{Cache.hash("aaa")}"))
end end
should "move saved searches" do context "saved searches" do
tag1 = FactoryBot.create(:tag, :name => "...") setup do
tag2 = FactoryBot.create(:tag, :name => "bbb") SavedSearch.stubs(:enabled?).returns(true)
ss = FactoryBot.create(:saved_search, :query => "123 ... 456", :user => CurrentUser.user) end
ta = FactoryBot.create(:tag_alias, :antecedent_name => "...", :consequent_name => "bbb")
ss.reload should "move saved searches" do
assert_equal(%w(123 456 bbb), ss.query.split.sort) tag1 = FactoryBot.create(:tag, :name => "...")
tag2 = FactoryBot.create(:tag, :name => "bbb")
ss = FactoryBot.create(:saved_search, :query => "123 ... 456", :user => CurrentUser.user)
ta = FactoryBot.create(:tag_alias, :antecedent_name => "...", :consequent_name => "bbb")
ss.reload
assert_equal(%w(123 456 bbb), ss.query.split.sort)
end
end end
should "update any affected posts when saved" do should "update any affected posts when saved" do