From 3ac7ee72132db1f735bd7304795f49a22b2d3e29 Mon Sep 17 00:00:00 2001 From: Mayur Date: Wed, 20 Nov 2024 16:50:21 +0530 Subject: [PATCH] feat: Added robots.txt for api,web,embed and allowed /auth/signup and /auth/signin for the crawler --- apps/api/public/robots.txt | 3 +++ apps/web/public/robots.txt | 7 +++++++ libs/embed/public/robots.txt | 3 +++ 3 files changed, 13 insertions(+) create mode 100644 apps/api/public/robots.txt create mode 100644 apps/web/public/robots.txt create mode 100644 libs/embed/public/robots.txt diff --git a/apps/api/public/robots.txt b/apps/api/public/robots.txt new file mode 100644 index 000000000..e9e57dc4d --- /dev/null +++ b/apps/api/public/robots.txt @@ -0,0 +1,3 @@ +# https://www.robotstxt.org/robotstxt.html +User-agent: * +Disallow: diff --git a/apps/web/public/robots.txt b/apps/web/public/robots.txt new file mode 100644 index 000000000..e72170214 --- /dev/null +++ b/apps/web/public/robots.txt @@ -0,0 +1,7 @@ +# https://www.robotstxt.org/robotstxt.html +User-agent: * +Disallow: + +User-agent: * +Allow: /auth/signin +Allow: /auth/signup diff --git a/libs/embed/public/robots.txt b/libs/embed/public/robots.txt new file mode 100644 index 000000000..e9e57dc4d --- /dev/null +++ b/libs/embed/public/robots.txt @@ -0,0 +1,3 @@ +# https://www.robotstxt.org/robotstxt.html +User-agent: * +Disallow: