# X9Elysium robots.txt # Policy: explicit allowlist for well-known search, AI, and social bots. # All other crawlers are disallowed by the catch-all at the bottom. # Sitemap: https://x9elysium.com/sitemap.xml # ============================================================ # Search engines # ============================================================ User-agent: Googlebot Allow: / Disallow: /api/ User-agent: Googlebot-Image Allow: / Disallow: /api/ User-agent: Googlebot-News Allow: / Disallow: /api/ User-agent: Googlebot-Video Allow: / Disallow: /api/ User-agent: AdsBot-Google Allow: / Disallow: /api/ User-agent: Mediapartners-Google Allow: / Disallow: /api/ User-agent: Bingbot Allow: / Disallow: /api/ User-agent: Slurp Allow: / Disallow: /api/ User-agent: DuckDuckBot Allow: / Disallow: /api/ User-agent: Baiduspider Allow: / Disallow: /api/ User-agent: YandexBot Allow: / Disallow: /api/ # ============================================================ # AI / GEO crawlers (training + retrieval + user-triggered) # ============================================================ User-agent: GPTBot Allow: / Disallow: /api/ User-agent: OAI-SearchBot Allow: / Disallow: /api/ User-agent: ChatGPT-User Allow: / Disallow: /api/ User-agent: ClaudeBot Allow: / Disallow: /api/ User-agent: Claude-Web Allow: / Disallow: /api/ User-agent: claude-searchbot Allow: / Disallow: /api/ User-agent: anthropic-ai Allow: / Disallow: /api/ User-agent: PerplexityBot Allow: / Disallow: /api/ User-agent: Perplexity-User Allow: / Disallow: /api/ User-agent: Google-Extended Allow: / Disallow: /api/ User-agent: Applebot Allow: / Disallow: /api/ User-agent: Applebot-Extended Allow: / Disallow: /api/ User-agent: Amazonbot Allow: / Disallow: /api/ User-agent: CCBot Allow: / Disallow: /api/ User-agent: Bytespider Allow: / Disallow: /api/ User-agent: Meta-ExternalAgent Allow: / Disallow: /api/ User-agent: Meta-ExternalFetcher Allow: / Disallow: /api/ User-agent: FacebookBot Allow: / Disallow: /api/ User-agent: Mistral-AI-User Allow: / Disallow: /api/ User-agent: cohere-ai Allow: / Disallow: /api/ User-agent: YouBot Allow: / Disallow: /api/ User-agent: Diffbot Allow: / Disallow: /api/ # ============================================================ # Social link previewers (Open Graph / oEmbed unfurlers) # ============================================================ User-agent: facebookexternalhit Allow: / User-agent: Twitterbot Allow: / User-agent: LinkedInBot Allow: / User-agent: Slackbot Allow: / User-agent: Slackbot-LinkExpanding Allow: / User-agent: Discordbot Allow: / User-agent: TelegramBot Allow: / User-agent: WhatsApp Allow: / User-agent: Pinterestbot Allow: / User-agent: redditbot Allow: / User-agent: Embedly Allow: / # ============================================================ # Web archives (kept allowed for citation durability) # ============================================================ User-agent: ia_archiver Allow: / User-agent: archive.org_bot Allow: / # ============================================================ # Default — block every other crawler we haven't explicitly approved. # This intentionally excludes SEO scrapers (AhrefsBot, SemrushBot, # MJ12bot, DotBot, BLEXBot, PetalBot, DataForSeoBot, etc.) and any # unknown agent. Add a named block above to grant access. # ============================================================ User-agent: * Disallow: / Sitemap: https://x9elysium.com/sitemap.xml