# Suger — robots.txt
# All well-behaved crawlers, including AI assistants and answer engines, are welcome.
# Machine-readable content: /llms.txt (index), /llms-full.txt (full corpus),
# /<path>/index.md (per-page Markdown), and an MCP endpoint at /mcp
# (discovery manifest: /.well-known/mcp.json).

User-agent: Googlebot
Allow: /
# HubSpot-era tracking URLs and CTA-redirect params — no content, pure crawl-budget
# waste in Search Console. Canonical tags already dedupe them; this stops the crawl.
# A named crawler obeys only its most-specific group and ignores `User-agent: *`,
# so the search engines each need their own copy of these rules.
Disallow: /cs/c/
Disallow: /_hcms/
Disallow: /*?*__hstc=
Disallow: /*?*__hssc=
Disallow: /*?*__hsfp=
Disallow: /*?*hsLang=
Disallow: /*?*cta_guid=
Disallow: /*?*hsCtaTracking=

User-agent: Bingbot
Allow: /
Disallow: /cs/c/
Disallow: /_hcms/
Disallow: /*?*__hstc=
Disallow: /*?*__hssc=
Disallow: /*?*__hsfp=
Disallow: /*?*hsLang=
Disallow: /*?*cta_guid=
Disallow: /*?*hsCtaTracking=

User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

# --- AI assistants / answer engines / model crawlers (explicitly allowed) ---

# OpenAI (ChatGPT)
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: OAI-AdsBot
Allow: /

# Anthropic (Claude) — Claude-Web / anthropic-ai are legacy, kept for compatibility
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: anthropic-ai
User-agent: Claude-Web
Allow: /

# Perplexity
User-agent: PerplexityBot
User-agent: Perplexity-User
Allow: /

# Google (Gemini / AI Overviews / Vertex AI)
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: Google-CloudVertexBot
Allow: /

# Apple (Siri / Apple Intelligence)
User-agent: Applebot
User-agent: Applebot-Extended
Allow: /

# Common Crawl
User-agent: CCBot
Allow: /

# Amazon
User-agent: Amazonbot
Allow: /

# Meta (register both cases)
User-agent: Meta-ExternalAgent
User-agent: meta-externalagent
User-agent: Meta-ExternalFetcher
User-agent: meta-externalfetcher
User-agent: FacebookBot
Allow: /

# ByteDance
User-agent: Bytespider
Allow: /

# Cohere
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
Allow: /

# Mistral (Le Chat)
User-agent: MistralAI-User
Allow: /

# You.com / DuckDuckGo
User-agent: YouBot
User-agent: DuckAssistBot
Allow: /

# --- Everything else ---
User-agent: *
Allow: /
Disallow: /cs/c/
Disallow: /_hcms/
Disallow: /*?*__hstc=
Disallow: /*?*__hssc=
Disallow: /*?*__hsfp=
Disallow: /*?*hsLang=
Disallow: /*?*cta_guid=
Disallow: /*?*hsCtaTracking=

Sitemap: https://www.suger.io/sitemap-index.xml
# LLMs index: https://www.suger.io/llms.txt