# MWNS llms.txt
# This file manages access for AI and LLM crawlers.
# Last updated: 2025-11-21

# --- Allow reputable AI crawlers that respect websites ---

User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: CCBot
Allow: /

User-agent: Google-Extended
Allow: /

# --- Block AI crawlers known for aggressive scraping/training ---

User-agent: PerplexityBot
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: CommonCrawl
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: cohere-ai
Disallow: /

# --- Default rule (safe + flexible) ---

User-agent: *
Allow: /