### robots.txt
#
# Per RFC 9309 a crawler obeys ONLY the group that best matches its own name and
# never merges in the `User-agent: *` group. So every Disallow below has to be
# repeated verbatim inside each named group — dropping it from one group means
# that crawler sees no restrictions at all.
#
# Shared disallow set (keep the named groups below in sync with this list):
#   /api/      — API routes, nothing to index
#   /admin/    — admin routes, nothing to index
#   /cdn-cgi/  — worker-only build artifacts (the merged ~2 MB translation
#                bundles written by scripts/export-messages.ts)
#
# On Cloudflare, /cdn-cgi/* is intercepted at the edge and never reaches Workers
# Assets, so the /cdn-cgi/ rule is only a defensive second layer plus coverage for
# a future non-Cloudflare host. It is NOT access control: robots.txt is a request
# to well-behaved crawlers and constrains `curl` not at all. If this site ever
# moves to a filesystem-backed host (e.g. Vercel), public/cdn-cgi/* becomes
# genuinely downloadable and needs a real fix — see CLAUDE.md.

User-agent: *
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /cdn-cgi/

# Sitemap location
Sitemap: https://caesarcipher.org/sitemap.xml

# LLM documentation
# llms.txt: https://caesarcipher.org/llms.txt
# llms-full.txt: https://caesarcipher.org/llms-full.txt

# Allow AI search crawlers explicitly.
# Each group restates the shared disallow set — see the note at the top.

User-agent: ChatGPT-User
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /cdn-cgi/

User-agent: OAI-SearchBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /cdn-cgi/

User-agent: GPTBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /cdn-cgi/

User-agent: PerplexityBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /cdn-cgi/

User-agent: Claude-Web
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /cdn-cgi/

User-agent: ClaudeBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /cdn-cgi/

User-agent: anthropic-ai
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /cdn-cgi/

User-agent: Google-Extended
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /cdn-cgi/

User-agent: GoogleOther
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /cdn-cgi/

User-agent: Amazonbot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /cdn-cgi/

User-agent: Applebot-Extended
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /cdn-cgi/

User-agent: Meta-ExternalAgent
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /cdn-cgi/

User-agent: cohere-ai
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /cdn-cgi/

User-agent: CCBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /cdn-cgi/

User-agent: PhindBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /cdn-cgi/

User-agent: YouBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /cdn-cgi/

# Block aggressive scrapers
User-agent: Bytespider
Disallow: /