jaculabilis/danbooru
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

artist urls: stop using normalized_url.

Browse Source 
Stop the last remaining uses of the `artist_urls.normalized_url` column.
It's already no longer used by the artist finder. The only remaining
uses were by API users. Those users should use the `url` column instead.
This commit is contained in:
evazion
2022-04-02 23:27:07 -05:00
parent 86e9cf1d05
commit 0d480eb832
6 changed files with 11 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
app/logical/artist_finder.rb
View File

@@ -218,7 +218,7 @@ module ArtistFinder
def find_artists(url)
return Artist.none if url.blank?
url = ArtistURL.normalize_normalized_url(url)
url = ArtistURL.normalize_url(url)
# First try an exact match
artists = Artist.active.joins(:urls).where(urls: { url: url })

30
app/models/artist_url.rb
View File

@@ -1,6 +1,8 @@
# frozen_string_literal: true
class ArtistURL < ApplicationRecord
self.ignored_columns = [:normalized_url]
normalize :url, :normalize_url
validates :url, presence: true, uniqueness: { scope: :artist_id }
@@ -8,8 +10,6 @@ class ArtistURL < ApplicationRecord
validate :validate_url_is_not_duplicate
belongs_to :artist, :touch => true
scope :url_matches, ->(url) { url_attribute_matches(:url, url) }
scope :normalized_url_matches, ->(url) { url_attribute_matches(:normalized_url, url) }
scope :active, -> { where(is_active: true) }
def self.parse_prefix(url)
@@ -19,23 +19,12 @@ class ArtistURL < ApplicationRecord
[is_active, url]
end
def self.normalize_normalized_url(url)
return nil if url.nil?
url = Source::URL.parse(url)&.profile_url || url
url = url.sub(%r{^https://}, "http://")
url = url.gsub(%r{/+\Z}, "")
url + "/"
end
def self.search(params = {})
q = search_attributes(params, :id, :created_at, :updated_at, :url, :normalized_url, :is_active, :artist)
q = search_attributes(params, :id, :created_at, :updated_at, :url, :is_active, :artist)
q = q.url_matches(params[:url_matches])
q = q.normalized_url_matches(params[:normalized_url_matches])
case params[:order]
when /\A(id|artist_id|url|normalized_url|is_active|created_at|updated_at)(?:_(asc|desc))?\z/i
when /\A(id|artist_id|url|is_active|created_at|updated_at)(?:_(asc|desc))?\z/i
dir = $2 || :desc
q = q.order($1 => dir).order(id: :desc)
else
@@ -45,16 +34,16 @@ class ArtistURL < ApplicationRecord
q
end
def self.url_attribute_matches(attr, url)
def self.url_matches(url)
if url.blank?
all
elsif url =~ %r{\A/(.*)/\z}
where_regex(attr, $1)
where_regex(:url, $1)
elsif url.include?("*")
where_ilike(attr, url)
where_ilike(:url, url)
else
profile_url = Source::Extractor.find(url).profile_url || url
where(attr => normalize_normalized_url(profile_url))
profile_url = Source::Extractor.find(url).profile_url || normalize_url(url)
where(url: profile_url)
end
end
@@ -110,7 +99,6 @@ class ArtistURL < ApplicationRecord
def url=(url)
super(url)
@parsed_url = Source::URL.parse(url)
self.normalized_url = self.class.normalize_normalized_url(self.url)
end
def parsed_url

4
app/views/artist_urls/index.html.erb
View File

@@ -5,7 +5,6 @@
<%= fa.input :name, label: "Artist Name", input_html: { value: params.dig(:search, :artist, :name), "data-autocomplete": "artist" } %>
<% end %>
<%= f.input :url_matches, label: "URL", input_html: { value: params[:search][:url_matches] } %>
<%= f.input :normalized_url_matches, label: "Normalized URL", input_html: { value: params[:search][:normalized_url_matches] } %>
<%= f.input :is_active, label: "Active?", collection: [["Yes", true], ["No", false]], include_blank: true, selected: params[:search][:is_active] %>
<%= f.input :order, collection: [["ID", "id"], ["Created", "created_at"], ["Updated", "updated_at"]], selected: params[:search][:order] %>
<%= f.submit "Search" %>
@@ -19,9 +18,6 @@
<% t.column "URL" do |artist_url| %>
<%= external_link_to(artist_url.url.to_s) %>
<% end %>
<% t.column "Normalized URL" do |artist_url| %>
<%= external_link_to(artist_url.normalized_url) %>
<% end %>
<% t.column "Active?" do |artist_url| %>
<%= artist_url.is_active.to_s %>
<% end %>

1
test/functional/artists_controller_test.rb
View File

@@ -184,7 +184,6 @@ class ArtistsControllerTest < ActionDispatch::IntegrationTest
should respond_to_search(has_urls: "true").with { [@artgerm, @masao] }
should respond_to_search(has_urls: "false").with { [@banned, @deleted, @artist] }
should respond_to_search(urls: {url: "https://www.pixiv.net/users/32777"}).with { @masao }
should respond_to_search(urls: {normalized_url: "http://www.deviantart.com/artgerm/"}).with { @artgerm }
end
end
end

2
test/unit/artist_test.rb
View File

@@ -134,7 +134,7 @@ class ArtistTest < ActiveSupport::TestCase
should "allow fixing invalid urls" do
artist = FactoryBot.build(:artist)
artist.urls << FactoryBot.build(:artist_url, url: "www.example.com", normalized_url: "www.example.com")
artist.urls << FactoryBot.build(:artist_url, url: "www.example.com")
artist.save(validate: false)
artist.update(url_string: "http://www.example.com")

36
test/unit/artist_url_test.rb
View File

@@ -19,7 +19,6 @@ class ArtistURLTest < ActiveSupport::TestCase
should "allow urls to be marked as inactive" do
url = create(:artist_url, url: "http://monet.com", is_active: false)
assert_equal("http://monet.com", url.url)
assert_equal("http://monet.com/", url.normalized_url)
assert_equal("-http://monet.com", url.to_s)
end
@@ -38,172 +37,141 @@ class ArtistURLTest < ActiveSupport::TestCase
should "automatically add http:// if missing" do
url = create(:artist_url, url: "example.com")
assert_equal("http://example.com", url.url)
assert_equal("http://example.com/", url.normalized_url)
end
should "normalize trailing slashes" do
url = create(:artist_url, url: "http://monet.com")
assert_equal("http://monet.com", url.url)
assert_equal("http://monet.com/", url.normalized_url)
url = create(:artist_url, url: "http://monet.com/")
assert_equal("http://monet.com", url.url)
assert_equal("http://monet.com/", url.normalized_url)
end
should "normalise https" do
url = create(:artist_url, url: "https://google.com")
assert_equal("https://google.com", url.url)
assert_equal("http://google.com/", url.normalized_url)
end
should "normalise domains to lowercase" do
url = create(:artist_url, url: "https://ArtistName.example.com")
assert_equal("https://artistname.example.com", url.url)
assert_equal("http://artistname.example.com/", url.normalized_url)
end
should "normalize ArtStation urls" do
url = create(:artist_url, url: "https://artstation.com/koyorin")
assert_equal("https://www.artstation.com/koyorin", url.url)
assert_equal("http://www.artstation.com/koyorin/", url.normalized_url)
url = create(:artist_url, url: "https://koyorin.artstation.com")
assert_equal("https://www.artstation.com/koyorin", url.url)
assert_equal("http://www.artstation.com/koyorin/", url.normalized_url)
url = create(:artist_url, url: "https://www.artstation.com/artist/koyorin/albums/all/")
assert_equal("https://www.artstation.com/koyorin", url.url)
assert_equal("http://www.artstation.com/koyorin/", url.normalized_url)
end
should "normalize fc2 urls" do
url = create(:artist_url, url: "http://silencexs.blog106.fc2.com/")
assert_equal("http://silencexs.blog.fc2.com", url.url)
assert_equal("http://silencexs.blog.fc2.com/", url.normalized_url)
end
should "normalize deviant art artist urls" do
url = create(:artist_url, url: "https://noizave.deviantart.com")
assert_equal("https://www.deviantart.com/noizave", url.url)
assert_equal("http://www.deviantart.com/noizave/", url.normalized_url)
end
should "normalize nico seiga artist urls" do
url = create(:artist_url, url: "http://seiga.nicovideo.jp/user/illust/7017777")
assert_equal("https://seiga.nicovideo.jp/user/illust/7017777", url.url)
assert_equal("http://seiga.nicovideo.jp/user/illust/7017777/", url.normalized_url)
url = create(:artist_url, url: "http://seiga.nicovideo.jp/manga/list?user_id=23839737")
assert_equal("https://seiga.nicovideo.jp/manga/list?user_id=23839737", url.url)
assert_equal("http://seiga.nicovideo.jp/manga/list?user_id=23839737/", url.normalized_url)
url = create(:artist_url, url: "https://www.nicovideo.jp/user/20446930/mylist/28674289")
assert_equal("https://www.nicovideo.jp/user/20446930", url.url)
assert_equal("http://www.nicovideo.jp/user/20446930/", url.normalized_url)
end
should "normalize hentai foundry artist urls" do
url = create(:artist_url, url: "http://www.hentai-foundry.com/user/kajinman/profile")
assert_equal("https://www.hentai-foundry.com/user/kajinman", url.url)
assert_equal("http://www.hentai-foundry.com/user/kajinman/", url.normalized_url)
end
should "normalize pixiv stacc urls" do
url = create(:artist_url, url: "http://www.pixiv.net/stacc/evazion/")
assert_equal("https://www.pixiv.net/stacc/evazion", url.url)
assert_equal("http://www.pixiv.net/stacc/evazion/", url.normalized_url)
end
should "normalize pixiv fanbox account urls" do
url = create(:artist_url, url: "http://www.pixiv.net/fanbox/creator/3113804/post")
assert_equal("https://www.pixiv.net/fanbox/creator/3113804", url.url)
assert_equal("http://www.pixiv.net/fanbox/creator/3113804/", url.normalized_url)
url = create(:artist_url, url: "http://omu001.fanbox.cc/posts/39714")
assert_equal("https://omu001.fanbox.cc", url.url)
assert_equal("http://omu001.fanbox.cc/", url.normalized_url)
end
should "normalize pixiv.net/user/123 urls" do
url = create(:artist_url, url: "http://www.pixiv.net/en/users/123")
assert_equal("https://www.pixiv.net/users/123", url.url)
assert_equal("http://www.pixiv.net/users/123/", url.normalized_url)
end
should "normalize twitter urls" do
url = create(:artist_url, url: "https://twitter.com/aoimanabu/status/892370963630743552")
assert_equal("https://twitter.com/aoimanabu", url.url)
assert_equal("http://twitter.com/aoimanabu/", url.normalized_url)
url = create(:artist_url, url: "https://twitter.com/BLAH")
assert_equal("https://twitter.com/BLAH", url.url)
assert_equal("http://twitter.com/BLAH/", url.normalized_url)
end
should "normalize https://twitter.com/intent/user?user_id=* urls" do
url = create(:artist_url, url: "https://twitter.com/intent/user?user_id=2784590030")
assert_equal("https://twitter.com/intent/user?user_id=2784590030", url.url)
assert_equal("http://twitter.com/intent/user?user_id=2784590030/", url.normalized_url)
end
should "normalize twitpic urls" do
url = create(:artist_url, url: "http://twitpic.com/photos/mirakichi")
assert_equal("http://twitpic.com/photos/mirakichi", url.url)
assert_equal("http://twitpic.com/photos/mirakichi/", url.normalized_url)
end
should "normalize nijie urls" do
url = create(:artist_url, url: "https://pic03.nijie.info/nijie_picture/236014_20170620101426_0.png")
assert_equal("https://nijie.info/members.php?id=236014", url.url)
assert_equal("http://nijie.info/members.php?id=236014/", url.normalized_url)
url = create(:artist_url, url: "http://nijie.info/members.php?id=161703")
assert_equal("https://nijie.info/members.php?id=161703", url.url)
assert_equal("http://nijie.info/members.php?id=161703/", url.normalized_url)
url = create(:artist_url, url: "http://www.nijie.info/members_illust.php?id=161703")
assert_equal("https://nijie.info/members.php?id=161703", url.url)
assert_equal("http://nijie.info/members.php?id=161703/", url.normalized_url)
url = create(:artist_url, url: "http://nijie.info/invalid.php")
assert_equal("http://nijie.info/invalid.php", url.url)
assert_equal("http://nijie.info/invalid.php/", url.normalized_url)
end
should "normalize pawoo.net urls" do
url = create(:artist_url, url: "http://www.pawoo.net/@evazion/19451018")
assert_equal("https://pawoo.net/@evazion", url.url)
assert_equal("http://pawoo.net/@evazion/", url.normalized_url)
url = create(:artist_url, url: "http://www.pawoo.net/users/evazion/media")
assert_equal("https://pawoo.net/@evazion", url.url)
assert_equal("http://pawoo.net/@evazion/", url.normalized_url)
end
should "normalize baraag.net urls" do
url = create(:artist_url, url: "http://baraag.net/@curator/102270656480174153")
assert_equal("https://baraag.net/@curator", url.url)
assert_equal("http://baraag.net/@curator/", url.normalized_url)
end
should "normalize Instagram urls" do
url = create(:artist_url, url: "http://instagram.com/itomugi")
assert_equal("https://www.instagram.com/itomugi/", url.url)
assert_equal("http://www.instagram.com/itomugi/", url.normalized_url)
end
should "normalize Booth.pm urls" do
url = create(:artist_url, url: "http://mesh-mesh.booth.pm/items/746971")
assert_equal("https://mesh-mesh.booth.pm", url.url)
assert_equal("http://mesh-mesh.booth.pm/", url.normalized_url)
end
context "#search method" do
@@ -221,10 +189,6 @@ class ArtistURLTest < ActiveSupport::TestCase
assert_search_equals([@bkub_url], url_matches: "*bkub*")
assert_search_equals([@bkub_url], url_matches: "/^https?://bkub\.com$/")
assert_search_equals([@bkub_url], normalized_url_matches: "*bkub*")
assert_search_equals([@bkub_url], normalized_url_matches: "/^https?://bkub\.com/$/")
assert_search_equals([@bkub_url], normalized_url_matches: "https://bkub.com")
assert_search_equals([@bkub_url], url: "https://bkub.com")
assert_search_equals([@bkub_url], url_eq: "https://bkub.com")
assert_search_equals([@bkub_url], url_not_eq: "https://masao.com")