# https://www.getxapi.com

User-agent: *
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /dashboard/
Disallow: /payment/
Disallow: /signup
Disallow: /login
Disallow: /_next/static/
Disallow: /blogs?*

# Explicit allow for search + AI retrieval/training crawlers. The wildcard
# above already permits them; these named stanzas make the welcome explicit and
# future-proof it against any future Disallow on User-agent:*. Anthropic runs 3
# bots (ClaudeBot=training, Claude-SearchBot=search index, Claude-User=real-time
# retrieval) and OpenAI runs 3 (GPTBot=training, OAI-SearchBot=index,
# ChatGPT-User=retrieval); all are allowed, since blocking the search/retrieval
# bots while meaning to block only training is the #1 silent AEO own-goal.
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Google-Extended
Allow: /

# Content Signals, declare AI/search preferences per https://contentsignals.org
# search: allow indexing in search results
# ai-train: allow use of content in AI model training
# ai-input: allow retrieval for AI-generated answers (RAG / grounding)
Content-Signal: search=yes, ai-train=yes, ai-input=yes

Sitemap: https://www.getxapi.com/sitemap.xml