restructuring for better SEO

public/robots.txt

@@ -0,0 +1,22 @@
+User-agent: *
+Allow: /
+
+# Sitemap location
+Sitemap: https://www.keyboardvagabond.com/sitemap.xml
+
+# Disallow crawling of fediverse subdomains to focus on main site
+Disallow: /mastodon/
+Disallow: /piefed/
+Disallow: /pixelfed/
+Disallow: /bookwyrm/
+Disallow: /blog/
+Disallow: /picsur/
+
+# Allow important pages
+Allow: /about.html
+Allow: /css/
+Allow: /scripts/
+Allow: /site-styles/
+
+# Crawl delay to be respectful
+Crawl-delay: 1