fix #57, disalow robots for some path

ontoportal · Jan 24, 2022 · f60467c · f60467c
1 parent 4e66374
commit f60467c
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 364 deletions.
diff --git a/Gemfile.lock b/Gemfile.lock
diff --git a/config/robots/appliance.txt b/config/robots/appliance.txt
@@ -1,9 +1,15 @@
 User-agent: *
+# This will match '/mappings', as well - due to path value wildcard [
+# https://developers.google.com/search/reference/robots_txt#url-matching-based-on-path-values ].
 Disallow: /*mappings
+Disallow: /projects
+Disallow: /projects/
 Disallow: /login
 Disallow: /virtual
 Disallow: /visualize
-Crawl-delay: 2
+Disallow: /resource_index
+Disallow: /feedback
+Crawl-delay: 5
 
 User-agent: SemrushBot
 Disallow: /
@@ -17,4 +23,6 @@ Disallow: /
 User-agent: MJ12bot
 Disallow: /
 
+User-agent: AhrefsBot
+Disallow: /