remove unnecessary allow in robots.txt

2025-03-18 19:38:47 -05:00
parent 55a4e5dec2
commit 26ce537dde
1 changed files with 0 additions and 3 deletions
--- a/pages/robots.txt
+++ b/pages/robots.txt
@@ -1,6 +1,3 @@
-User-agent: *
-Allow: /
-
 # Block all known AI crawlers and assistants
 # from using content for training AI models.
 # Source: https://robotstxt.com/ai