jaculabilis/danbooru
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

seo: increase sitemap coverage.

Browse Source 
Rework sitemaps to provide more coverage of the site. We want every
important page on the site - including every post, tag, and wiki page -
to be indexed by Google. We do this by generating sitemaps and sitemap
indexes that contain links to every important page on the site.
This commit is contained in:
evazion
2020-07-09 22:38:35 -05:00
parent d88a2a674f
commit 42f0112c38
21 changed files with 187 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
app/views/application/index.sitemap.erb Normal file
View File

@@ -0,0 +1,10 @@
<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
<% @current_item.each do |item| %>
<url>
<loc><%= polymorphic_url(item) %></loc>
<lastmod><%= item.updated_at.iso8601 %></lastmod>
</url>
<% end %>
</urlset>

30
app/views/posts/index.sitemap.erb Normal file
View File

@@ -0,0 +1,30 @@
<?xml version="1.0" encoding="UTF-8"?>
<%# https://support.google.com/webmasters/answer/178636 %>
<%# https://support.google.com/webmasters/answer/80471 %>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
xmlns:image="http://www.google.com/schemas/sitemap-image/1.1"
xmlns:video="http://www.google.com/schemas/sitemap-video/1.1">
<% @posts.each do |post| %>
<url>
<loc><%= post_url(post) %></loc>
<lastmod><%= post.updated_at.iso8601 %></lastmod>
<% if post.visible? %>
<% if post.is_image? %>
<image:image>
<image:loc><%= post.file_url %></image:loc>
</image:image>
<% elsif post.is_video? %>
<video:video>
<video:thumbnail_loc><%= post.preview_file_url %></video:thumbnail_loc>
<video:content_loc><%= post.file_url %></video:content_loc>
<video:publication_date><%= post.created_at.iso8601 %></video:publication_date>
<video:title><%= "Post ##{post.id}" %></video:title>
<video:description><%= post.tag_string %></video:description>
<video:family_friendly><%= post.rating == "s" ? "yes" : "no" %></video:family_friendly>
</video:video>
<% end %>
<% end %>
</url>
<% end %>
</urlset>

19
app/views/robots/index.text.erb
View File

@@ -1,20 +1,19 @@
Sitemap: <%= root_url %>sitemap.xml
User-agent: *
Disallow: /
Allow: /$
<% if Rails.env.production? && Danbooru.config.hostname == request.host %>
<% if !Rails.env.production? || Danbooru.config.hostname == request.host %>
Disallow: /*.atom
Disallow: /*.json
Disallow: /*.xml
Allow: /$
Allow: /artists
Allow: /artist_commentaries
Allow: /comments
Allow: /explore
Allow: /favorite_groups
Allow: /forum_posts
Allow: /forum_topics
Allow: /iqdb_queries
Allow: /login
Allow: /notes
Allow: /pools
@@ -23,7 +22,7 @@ Allow: /sessions
Allow: /static
Allow: /tags
Allow: /uploads
Allow: /user_upgrades
Allow: /user_upgrade
Allow: /users
Allow: /wiki_pages
@@ -51,4 +50,12 @@ Allow: /packs
Allow: /terms_of_service
Allow: /privacy
Allow: /sitemap.xml
Sitemap: <%= sitemap_url(format: :xml, sitemap: "artists") %>
Sitemap: <%= sitemap_url(format: :xml, sitemap: "forum_topics") %>
Sitemap: <%= sitemap_url(format: :xml, sitemap: "pools") %>
Sitemap: <%= sitemap_url(format: :xml, sitemap: "posts") %>
Sitemap: <%= sitemap_url(format: :xml, sitemap: "tags") %>
Sitemap: <%= sitemap_url(format: :xml, sitemap: "users") %>
Sitemap: <%= sitemap_url(format: :xml, sitemap: "wiki_pages") %>
<% end %>

41
app/views/static/sitemap.xml.erb
View File

@@ -1,41 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
xmlns:image="http://www.google.com/schemas/sitemap-image/1.1">
<url>
<loc><%= posts_url %></loc>
<changefreq>daily</changefreq>
</url>
<url>
<loc><%= wiki_pages_url %></loc>
<changefreq>daily</changefreq>
</url>
<url>
<loc><%= pools_url %></loc>
<changefreq>daily</changefreq>
</url>
<% cache("sitemap", :expires_in => 24.hours) do %>
<% @reportbooru_service.post_search_rankings(Date.yesterday) do |tags, count| %>
<url>
<loc><%= posts_url(tags: tags) %></loc>
<lastmod><%= Date.today %></lastmod>
</url>
<% end %>
<% @posts.each do |post| %>
<url>
<loc><%= post_url(post) %></loc>
<image:image>
<image:loc>
<%= post.file_url %>
</image:loc>
<image:caption>
<%= post.presenter.humanized_essential_tag_string %>
</image:caption>
</image:image>
<lastmod><%= post.created_at.to_date %></lastmod>
</url>
<% end %>
<% end %>
</urlset>

21
app/views/static/sitemap_index.xml.erb Normal file
View File

@@ -0,0 +1,21 @@
<?xml version="1.0" encoding="UTF-8"?>
<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
<% 0.upto(@relation.maximum(:id) / @limit) do |page| %>
<% lo = page * @limit %>
<% hi = (page + 1) * @limit %>
<% lastmod = @relation.where(id: lo..hi).maximum(:updated_at).iso8601 %>
<% if @sitemap == "posts" %>
<% loc = posts_url(limit: @limit, format: :sitemap, tags: "id:#{lo}..#{hi}") %>
<% else %>
<% loc = polymorphic_url(@relation.klass, limit: @limit, format: :sitemap, search: { id: "#{lo}..#{hi}", **@search }) %>
<% end %>
<% if lastmod.present? %>
<sitemap>
<loc><%= loc %></loc>
<lastmod><%= lastmod %></lastmod>
</sitemap>
<% end %>
<% end %>
</sitemapindex>

10
app/views/tags/index.sitemap.erb Normal file
View File

@@ -0,0 +1,10 @@
<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
<% @tags.each do |tag| %>
<url>
<loc><%= posts_url(tags: tag.name) %></loc>
<lastmod><%= tag.updated_at.iso8601 %></lastmod>
</url>
<% end %>
</urlset>

9
app/views/users/index.sitemap.erb Normal file
View File

@@ -0,0 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
<% @users.each do |user| %>
<url>
<loc><%= user_url(user) %></loc>
</url>
<% end %>
</urlset>