feat: Added robots.txt for api,web,embed and allowed /auth/signup and…

… /auth/signin for the crawler
implerhq · Nov 20, 2024 · 3ac7ee7 · 3ac7ee7
1 parent 6047946
commit 3ac7ee7
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 0 deletions.
diff --git a/apps/api/public/robots.txt b/apps/api/public/robots.txt
@@ -0,0 +1,3 @@
+# https://www.robotstxt.org/robotstxt.html
+User-agent: *
+Disallow:
diff --git a/apps/web/public/robots.txt b/apps/web/public/robots.txt
@@ -0,0 +1,7 @@
+# https://www.robotstxt.org/robotstxt.html
+User-agent: *
+Disallow:
+
+User-agent: *
+Allow: /auth/signin
+Allow: /auth/signup
diff --git a/libs/embed/public/robots.txt b/libs/embed/public/robots.txt
@@ -0,0 +1,3 @@
+# https://www.robotstxt.org/robotstxt.html
+User-agent: *
+Disallow: