# Keep public pages available to search engines and answer-engine retrieval.
# Internal and admin routes are excluded. Paginated archives stay crawlable
# because they are self-canonical index pages with unique article lists.

User-agent: *
Allow: /
Disallow: /login
Disallow: /consult
Disallow: /performance
Disallow: /admin
Disallow: /api

# Search and user-directed retrieval crawlers. These are distinct from the
# crawlers used to collect content for model training.
User-agent: OAI-SearchBot
Allow: /
Disallow: /login
Disallow: /consult
Disallow: /performance
Disallow: /admin
Disallow: /api

User-agent: ChatGPT-User
Allow: /
Disallow: /login
Disallow: /consult
Disallow: /performance
Disallow: /admin
Disallow: /api

User-agent: Claude-SearchBot
Allow: /
Disallow: /login
Disallow: /consult
Disallow: /performance
Disallow: /admin
Disallow: /api

User-agent: Claude-User
Allow: /
Disallow: /login
Disallow: /consult
Disallow: /performance
Disallow: /admin
Disallow: /api

User-agent: PerplexityBot
Allow: /
Disallow: /login
Disallow: /consult
Disallow: /performance
Disallow: /admin
Disallow: /api

User-agent: Applebot
Allow: /
Disallow: /login
Disallow: /consult
Disallow: /performance
Disallow: /admin
Disallow: /api

# Preserve the site's existing model-training opt-out. Cloudflare also emits
# matching managed directives in production; these source rules prevent a
# later allow rule from contradicting that policy.
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

Sitemap: https://josiahroche.co/sitemap.xml