From 30b73459006d1416e0de84c160de680c6004dc1c Mon Sep 17 00:00:00 2001 From: evazion Date: Sun, 7 Feb 2021 22:30:10 -0600 Subject: [PATCH] robots.txt: disallow crawling version pages. Some of these pages were accidentally crawled because rules like `Allow: /artist` allowed `/artist_versions` to be crawled (Allow rules are prefix matches). --- app/views/robots/index.text.erb | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/app/views/robots/index.text.erb b/app/views/robots/index.text.erb index 8737193bf..866530a79 100644 --- a/app/views/robots/index.text.erb +++ b/app/views/robots/index.text.erb @@ -6,6 +6,14 @@ Allow: /$ Disallow: /*.atom Disallow: /*.json +Disallow: <%= artist_urls_path %> +Disallow: <%= artist_versions_path %> +Disallow: <%= artist_commentary_versions_path %> +Disallow: <%= note_versions_path %> +Disallow: <%= post_versions_path %> +Disallow: <%= pool_versions_path %> +Disallow: <%= wiki_page_versions_path %> + Allow: <%= artists_path %> Allow: <%= artist_commentaries_path %> Allow: <%= comments_path %>