From 3ac7ee72132db1f735bd7304795f49a22b2d3e29 Mon Sep 17 00:00:00 2001 From: Mayur Date: Wed, 20 Nov 2024 16:50:21 +0530 Subject: [PATCH 1/2] feat: Added robots.txt for api,web,embed and allowed /auth/signup and /auth/signin for the crawler --- apps/api/public/robots.txt | 3 +++ apps/web/public/robots.txt | 7 +++++++ libs/embed/public/robots.txt | 3 +++ 3 files changed, 13 insertions(+) create mode 100644 apps/api/public/robots.txt create mode 100644 apps/web/public/robots.txt create mode 100644 libs/embed/public/robots.txt diff --git a/apps/api/public/robots.txt b/apps/api/public/robots.txt new file mode 100644 index 000000000..e9e57dc4d --- /dev/null +++ b/apps/api/public/robots.txt @@ -0,0 +1,3 @@ +# https://www.robotstxt.org/robotstxt.html +User-agent: * +Disallow: diff --git a/apps/web/public/robots.txt b/apps/web/public/robots.txt new file mode 100644 index 000000000..e72170214 --- /dev/null +++ b/apps/web/public/robots.txt @@ -0,0 +1,7 @@ +# https://www.robotstxt.org/robotstxt.html +User-agent: * +Disallow: + +User-agent: * +Allow: /auth/signin +Allow: /auth/signup diff --git a/libs/embed/public/robots.txt b/libs/embed/public/robots.txt new file mode 100644 index 000000000..e9e57dc4d --- /dev/null +++ b/libs/embed/public/robots.txt @@ -0,0 +1,3 @@ +# https://www.robotstxt.org/robotstxt.html +User-agent: * +Disallow: From 8b94046792f15256b194f00e36a0f9b5ffd73490 Mon Sep 17 00:00:00 2001 From: Mayur Date: Wed, 20 Nov 2024 17:08:28 +0530 Subject: [PATCH 2/2] feat: Removed redundant User-agent which may cause conflict among crawlers --- apps/web/public/robots.txt | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/apps/web/public/robots.txt b/apps/web/public/robots.txt index e72170214..bd401b0fe 100644 --- a/apps/web/public/robots.txt +++ b/apps/web/public/robots.txt @@ -1,7 +1,4 @@ -# https://www.robotstxt.org/robotstxt.html -User-agent: * +User-agent: * Disallow: - -User-agent: * Allow: /auth/signin -Allow: /auth/signup +Allow: /auth/signup \ No newline at end of file