# Generic crawlers: existing rules, formatting cleaned up.
User-agent: *
Allow: /
Disallow: */privacy-policy/
Disallow: */terms-of-service/
Disallow: */tag/*
Disallow: *.pdf

# User-facing AI bots: explicit allow, including conversion pages.
# These crawl when a human asks an AI a question and the AI cites Gallio.
# Reason: ChatGPT-referred traffic converts at 23% to 42% in our data.
User-agent: ChatGPT-User
Allow: /
User-agent: OAI-SearchBot
Allow: /
User-agent: PerplexityBot
Allow: /
User-agent: Perplexity-User
Allow: /
User-agent: ClaudeBot
Allow: /
User-agent: Claude-User
Allow: /
User-agent: GoogleOther
Allow: /
User-agent: Google-Extended
Allow: /

# AI training bots: 90-day trial allow, review by 2026-09-10.
# Reason: brand presence in future-generation models. Reversible if we see misuse.
User-agent: GPTBot
Allow: /
User-agent: Bytespider
Allow: /
User-agent: Meta-ExternalAgent
Allow: /
User-agent: anthropic-ai
Allow: /
User-agent: CCBot
Allow: /
User-agent: Diffbot
Allow: /

Sitemap: https://gallio.pro/sitemap-index.xml