User-Agent: *
Disallow: /index.html
Disallow: /blogs?page=*
Disallow: /blogs?category=*

 
# Allow AI & LLM Crawlers

User-Agent: GPTBot
Allow: /
 
User-Agent: ChatGPT-User
Allow: /
 
User-Agent: Google-Extended
Allow: /
 
User-Agent: CCBot
Allow: /
 
User-Agent: anthropic-ai
Allow: /
 
User-Agent: ClaudeBot
Allow: /
 
User-Agent: Amazonbot
Allow: /
 
User-Agent: Applebot
Allow: /
 
User-Agent: Bingbot
Allow: /
 
 
# Sitemap Files
Sitemap: https://msmcoretech.com/sitemap.xml
Sitemap: https://msmcoretech.com/blogs/sitemap_index.xml