deploy: fa52926

mijndert · Dec 31, 2024 · f74ecc7 · f74ecc7
1 parent 727afcf
commit f74ecc7
Show file tree

Hide file tree

Showing 2 changed files with 34 additions and 17 deletions.
diff --git a/robots.txt b/robots.txt
@@ -2,28 +2,45 @@ Sitemap: https://mijndertstuij.nl/sitemap.xml
 
 User-agent: *
 Disallow:
-
-User-agent: AdsBot-Google
+User-agent: AI2Bot
+User-agent: Ai2Bot-Dolma
 User-agent: Amazonbot
 User-agent: anthropic-ai
+User-agent: Applebot
 User-agent: Applebot-Extended
 User-agent: Bytespider
 User-agent: CCBot
 User-agent: ChatGPT-User
-User-agent: ClaudeBot
 User-agent: Claude-Web
+User-agent: ClaudeBot
 User-agent: cohere-ai
 User-agent: Diffbot
+User-agent: DuckAssistBot
 User-agent: FacebookBot
 User-agent: FriendlyCrawler
 User-agent: Google-Extended
 User-agent: GoogleOther
+User-agent: GoogleOther-Image
+User-agent: GoogleOther-Video
 User-agent: GPTBot
+User-agent: iaskspider/2.0
+User-agent: ICC-Crawler
+User-agent: ImagesiftBot
 User-agent: img2dataset
+User-agent: ISSCyberRiskCrawler
+User-agent: Kangaroo Bot
+User-agent: Meta-ExternalAgent
+User-agent: Meta-ExternalFetcher
+User-agent: OAI-SearchBot
 User-agent: omgili
 User-agent: omgilibot
-User-agent: peer39_crawler
-User-agent: peer39_crawler/1.0
+User-agent: PanguBot
 User-agent: PerplexityBot
+User-agent: PetalBot
+User-agent: Scrapy
+User-agent: Sidetrade indexer bot
+User-agent: Timpibot
+User-agent: VelenPublicWebCrawler
+User-agent: Webzio-Extended
 User-agent: YouBot
 Disallow: /
diff --git a/sitemap.xml b/sitemap.xml
@@ -92,62 +92,62 @@
 
         <url>
             <loc>https://mijndertstuij.nl/404.html</loc>
-            <lastmod>2024-12-30T19:42:37.597Z</lastmod>
+            <lastmod>2024-12-31T23:26:06.500Z</lastmod>
         </url>
 
         <url>
             <loc>https://mijndertstuij.nl/ai/</loc>
-            <lastmod>2024-12-30T19:42:37.598Z</lastmod>
+            <lastmod>2024-12-31T23:26:06.501Z</lastmod>
         </url>
 
         <url>
             <loc>https://mijndertstuij.nl/blog/</loc>
-            <lastmod>2024-12-30T19:42:37.598Z</lastmod>
+            <lastmod>2024-12-31T23:26:06.501Z</lastmod>
         </url>
 
         <url>
             <loc>https://mijndertstuij.nl/blogroll/</loc>
-            <lastmod>2024-12-30T19:42:37.599Z</lastmod>
+            <lastmod>2024-12-31T23:26:06.501Z</lastmod>
         </url>
 
         <url>
             <loc>https://mijndertstuij.nl/colophon/</loc>
-            <lastmod>2024-12-30T19:42:37.599Z</lastmod>
+            <lastmod>2024-12-31T23:26:06.501Z</lastmod>
         </url>
 
         <url>
             <loc>https://mijndertstuij.nl/connect/</loc>
-            <lastmod>2024-12-30T19:42:37.599Z</lastmod>
+            <lastmod>2024-12-31T23:26:06.501Z</lastmod>
         </url>
 
         <url>
             <loc>https://mijndertstuij.nl/defaults/</loc>
-            <lastmod>2024-12-30T19:42:37.599Z</lastmod>
+            <lastmod>2024-12-31T23:26:06.502Z</lastmod>
         </url>
 
         <url>
             <loc>https://mijndertstuij.nl/explore/</loc>
-            <lastmod>2024-12-30T19:42:37.599Z</lastmod>
+            <lastmod>2024-12-31T23:26:06.502Z</lastmod>
         </url>
 
         <url>
             <loc>https://mijndertstuij.nl/</loc>
-            <lastmod>2024-12-30T19:42:37.608Z</lastmod>
+            <lastmod>2024-12-31T23:26:06.509Z</lastmod>
         </url>
 
         <url>
             <loc>https://mijndertstuij.nl/now/</loc>
-            <lastmod>2024-12-30T19:42:37.608Z</lastmod>
+            <lastmod>2024-12-31T23:26:06.509Z</lastmod>
         </url>
 
         <url>
             <loc>https://mijndertstuij.nl/uses/</loc>
-            <lastmod>2024-12-30T19:42:37.609Z</lastmod>
+            <lastmod>2024-12-31T23:26:06.511Z</lastmod>
         </url>
 
         <url>
             <loc>https://mijndertstuij.nl/feed.xml</loc>
-            <lastmod>2024-12-30T19:42:42.758Z</lastmod>
+            <lastmod>2024-12-31T23:26:11.578Z</lastmod>
         </url>
 
 </urlset>