From 4f8cd6006c7dd85ca592d84c501e63ce3201fc92 Mon Sep 17 00:00:00 2001 From: albert Date: Fri, 29 Mar 2013 11:28:01 -0400 Subject: [PATCH] fixes for source pattern search, fixed tests --- app/logical/post_query_builder.rb | 5 ++- app/models/artist_version.rb | 4 +- ...39_change_source_pattern_index_on_posts.rb | 24 +++++++----- db/structure.sql | 26 ++++++++----- test/unit/post_test.rb | 39 +++++++++++++++---- 5 files changed, 69 insertions(+), 29 deletions(-) diff --git a/app/logical/post_query_builder.rb b/app/logical/post_query_builder.rb index cc03c74ba..e46e06f20 100644 --- a/app/logical/post_query_builder.rb +++ b/app/logical/post_query_builder.rb @@ -124,6 +124,7 @@ class PostQueryBuilder relation = add_range_relation(q[:copyright_tag_count], "posts.tag_count_copyright", relation) relation = add_range_relation(q[:character_tag_count], "posts.tag_count_character", relation) relation = add_range_relation(q[:post_tag_count], "posts.tag_count", relation) + # relation = add_range_relation(q[:pixiv_id], "substring(posts.source, 'pixiv.net/img.*/([0-9]+)[^/]*$')::integer", relation) if q[:md5] relation = relation.where(["posts.md5 IN (?)", q[:md5]]) @@ -156,8 +157,8 @@ class PostQueryBuilder if q[:source] if q[:source] == "none%" relation = relation.where("(posts.source = '' OR posts.source IS NULL)") - elsif q[:source] =~ /^%\.?pixiv(\.net(\/img)?)?(%\/|(?=%$))(.+)$/ - relation = relation.where("SourcePattern(posts.source) LIKE ? ESCAPE E'\\\\'", "pixiv/" + $5) + elsif q[:source] =~ /^%\.?pixiv(?:\.net(?:\/img)?)?(?:%\/|(?=%$))(.+)$/ + relation = relation.where("SourcePattern(posts.source) LIKE ? ESCAPE E'\\\\'", "pixiv/" + $1) has_constraints! else relation = relation.where("SourcePattern(posts.source) LIKE SourcePattern(?) ESCAPE E'\\\\'", q[:source]) diff --git a/app/models/artist_version.rb b/app/models/artist_version.rb index de50888b8..3677675e6 100644 --- a/app/models/artist_version.rb +++ b/app/models/artist_version.rb @@ -24,11 +24,11 @@ class ArtistVersion < ActiveRecord::Base end def url_array - url_string.scan(/\S+/) + url_string.to_s.scan(/\S+/) end def other_names_array - other_names.scan(/\S+/) + other_names.to_s.scan(/\S+/) end def urls_diff(version) diff --git a/db/migrate/20130328092739_change_source_pattern_index_on_posts.rb b/db/migrate/20130328092739_change_source_pattern_index_on_posts.rb index 523070ce0..1a5fb6131 100644 --- a/db/migrate/20130328092739_change_source_pattern_index_on_posts.rb +++ b/db/migrate/20130328092739_change_source_pattern_index_on_posts.rb @@ -1,24 +1,30 @@ class ChangeSourcePatternIndexOnPosts < ActiveRecord::Migration def up execute "set statement_timeout = 0" - execute "DROP INDEX index_posts_on_pixiv_suffix;" - execute "DROP INDEX index_posts_on_source_pattern;" + begin + execute "DROP INDEX index_posts_on_pixiv_suffix" + execute "DROP INDEX index_posts_on_source_pattern" + rescue Exception + end execute "CREATE FUNCTION SourcePattern(src text) RETURNS text AS $$ BEGIN - RETURN regexp_replace(src, '^[^/]*(//)?[^/]*\.pixiv\.net/img.*(/[^/]*/[^/]*)$', 'pixiv\2'); + RETURN regexp_replace(src, '^[^/]*(//)?[^/]*\.pixiv\.net/img.*(/[^/]*/[^/]*)$', E'pixiv\\\\2'); END; - $$ LANGUAGE plpgsql IMMUTABLE RETURNS NULL ON NULL INPUT;" + $$ LANGUAGE plpgsql IMMUTABLE RETURNS NULL ON NULL INPUT" execute "CREATE INDEX index_posts_on_source_pattern ON posts USING btree - ((SourcePattern(source)) text_pattern_ops);" + ((SourcePattern(source)) text_pattern_ops)" + # execute "CREATE INDEX index_posts_on_pixiv_id ON posts USING btree + # ((substring(source, 'pixiv.net/img.*/([0-9]+)[^/]*$')::integer))" end def down execute "set statement_timeout = 0" - execute "DROP INDEX index_posts_on_source_pattern;" - execute "DROP FUNCTION SourcePattern(text);" + execute "DROP INDEX index_posts_on_source_pattern" + execute "DROP FUNCTION SourcePattern(text)" execute "CREATE INDEX index_posts_on_source_pattern ON posts USING btree - (source text_pattern_ops);" + (source text_pattern_ops)" execute "CREATE INDEX index_posts_on_pixiv_suffix ON posts USING btree - ((substring(source, 'pixiv.net/img.*/([^/]*/[^/]*)$')) text_pattern_ops);" + ((substring(source, 'pixiv.net/img.*/([^/]*/[^/]*)$')) text_pattern_ops)" + # execute "DROP INDEX index_posts_on_pixiv_id" end end diff --git a/db/structure.sql b/db/structure.sql index 30197e5bd..4a038306b 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -331,6 +331,19 @@ CREATE FUNCTION favorites_insert_trigger() RETURNS trigger $$; +-- +-- Name: sourcepattern(text); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION sourcepattern(src text) RETURNS text + LANGUAGE plpgsql IMMUTABLE STRICT + AS $_$ + BEGIN + RETURN regexp_replace(src, '^[^/]*(//)?[^/]*.pixiv.net/img.*(/[^/]*/[^/]*)$', E'pixiv\\2'); + END; + $_$; + + -- -- Name: testprs_end(internal); Type: FUNCTION; Schema: public; Owner: - -- @@ -5929,13 +5942,6 @@ CREATE INDEX index_posts_on_parent_id ON posts USING btree (parent_id); CREATE INDEX index_posts_on_pixiv_id ON posts USING btree ((("substring"((source)::text, 'pixiv.net/img.*/([0-9]+)[^/]*$'::text))::integer)); --- --- Name: index_posts_on_pixiv_suffix; Type: INDEX; Schema: public; Owner: -; Tablespace: --- - -CREATE INDEX index_posts_on_pixiv_suffix ON posts USING btree ("substring"((source)::text, 'pixiv.net/img.*/([^/]*/[^/]*)$'::text) text_pattern_ops); - - -- -- Name: index_posts_on_source; Type: INDEX; Schema: public; Owner: -; Tablespace: -- @@ -5947,7 +5953,7 @@ CREATE INDEX index_posts_on_source ON posts USING btree (source); -- Name: index_posts_on_source_pattern; Type: INDEX; Schema: public; Owner: -; Tablespace: -- -CREATE INDEX index_posts_on_source_pattern ON posts USING btree (source text_pattern_ops); +CREATE INDEX index_posts_on_source_pattern ON posts USING btree (sourcepattern((source)::text) text_pattern_ops); -- @@ -6346,4 +6352,6 @@ INSERT INTO schema_migrations (version) VALUES ('20130322173859'); INSERT INTO schema_migrations (version) VALUES ('20130323160259'); -INSERT INTO schema_migrations (version) VALUES ('20130326035904'); \ No newline at end of file +INSERT INTO schema_migrations (version) VALUES ('20130326035904'); + +INSERT INTO schema_migrations (version) VALUES ('20130328092739'); \ No newline at end of file diff --git a/test/unit/post_test.rb b/test/unit/post_test.rb index 83cb5bce9..40fd61076 100644 --- a/test/unit/post_test.rb +++ b/test/unit/post_test.rb @@ -907,12 +907,6 @@ class PostTest < ActiveSupport::TestCase assert_equal(post1.id, relation.first.id) end - should "return posts for the metatag" do - post = FactoryGirl.create(:post, :source => "http://i1.pixiv.net/img123/img/artist-name/789.png") - assert_equal(1, Post.tag_match("pixiv:789").count) - assert_equal(0, Post.tag_match("pixiv:790").count) - end - should "return posts for a list of md5 hashes" do post1 = FactoryGirl.create(:post, :md5 => "abcd") post2 = FactoryGirl.create(:post) @@ -939,13 +933,44 @@ class PostTest < ActiveSupport::TestCase end should "return posts for a pixiv source search" do - post = FactoryGirl.create(:post, :source => "http://i1.pixiv.net/img123/img/artist-name/789.png") + url = "http://i1.pixiv.net/img123/img/artist-name/789.png" + post = FactoryGirl.create(:post, :source => url) assert_equal(1, Post.tag_match("source:pixiv/artist-name/*").count) assert_equal(0, Post.tag_match("source:pixiv/artist-fake/*").count) assert_equal(1, Post.tag_match("source:*.pixiv.net/img*/artist-name/*").count) assert_equal(0, Post.tag_match("source:*.pixiv.net/img*/artist-fake/*").count) end + should "return posts for a pixiv id search (type 1)" do + url = "http://i1.pixiv.net/img-inf/img/2013/03/14/03/02/36/34228050_s.jpg" + post = FactoryGirl.create(:post, :source => url) + assert_equal(1, Post.tag_match("pixiv_id:34228050").count) + end + + should "return posts for a pixiv id search (type 2)" do + url = "http://i1.pixiv.net/img123/img/artist-name/789.png" + post = FactoryGirl.create(:post, :source => url) + assert_equal(1, Post.tag_match("pixiv_id:789").count) + end + + should "return posts for a pixiv id search (type 3)" do + url = "http://www.pixiv.net/member_illust.php?mode=manga_big&illust_id=19113635&page=0" + post = FactoryGirl.create(:post, :source => url) + assert_equal(1, Post.tag_match("pixiv_id:19113635").count) + end + + should "return posts for a pixiv id search (type 4)" do + url = "http://i2.pixiv.net/img70/img/disappearedstump/34551381_p3.jpg?1364424318" + post = FactoryGirl.create(:post, :source => url) + assert_equal(1, Post.tag_match("pixiv_id:34551381").count) + end + + should "return posts for a pixiv novel id search" do + url = "http://www.pixiv.net/novel/show.php?id=2156088" + post = FactoryGirl.create(:post, :source => url) + assert_equal(1, Post.tag_match("pixiv_novel_id:2156088").count) + end + should "return posts for a tag subscription search" do post1 = FactoryGirl.create(:post, :tag_string => "aaa") sub = FactoryGirl.create(:tag_subscription, :tag_query => "aaa", :name => "zzz")