SEO tweaks

2025-11-20 23:43:02 -05:00
parent cf010974dd
commit 5305c34e3d
5 changed files with 183 additions and 3 deletions
--- a/web/static/robots.txt
+++ b/web/static/robots.txt
@@ -0,0 +1,21 @@
+# Robots.txt for Maze Generator
+# Allow all search engines to index the site
+
+User-agent: *
+Allow: /
+Disallow: /api/
+
+# Specific crawlers
+User-agent: Googlebot
+Allow: /
+Disallow: /api/
+
+User-agent: Bingbot
+Allow: /
+Disallow: /api/
+
+# Sitemap location
+Sitemap: https://yoursite.com/sitemap.xml
+
+# Crawl delay (optional)
+Crawl-delay: 1