Sitemap cleanup

AlphaGameDeveloper · Apr 25, 2024 · e9ed9cb · e9ed9cb
1 parent b70c1fa
commit e9ed9cb
Show file tree

Hide file tree

Showing 3 changed files with 14 additions and 1 deletion.
diff --git a/robots.txt b/robots.txt
@@ -1,5 +1,16 @@
 User-agent: *
-Allow: /
+
+# /.well-known/ contains domain verification
+# stuff like Discord, etc.  Best not to have
+# search engines indexing it, as it is useless!
+Disallow: /.well-known/
+
+# /google*.html is Google Search Console verification
+# files!  We don't need nor want search engines to index them.
+Disallow: /google*.html
+
+# 404 page does not need to be indexed!
+Disallow: /404.html
 
 # Use a sitemap, generated by jekyll-sitemap.
 # Added: April 25, 2024 at 2:15pm

diff --git a/site-verification/discord.md b/site-verification/discord.md
@@ -2,4 +2,5 @@
 layout: baredata
 text: "dh=0d357269ea63b01b470822a5ce459a7afb790acc"
 permalink: /.well-known/discord
+sitemap: false
 ---
diff --git a/site-verification/nusd-google-verification.md b/site-verification/nusd-google-verification.md
@@ -2,4 +2,5 @@
 layout: baredata
 permalink: /google7d7113d411a0fce3.html
 text: "google-site-verification: google7d7113d411a0fce3.html"
+sitemap: false
 ---