# Camli — Flight Booking Platform
# https://camli.com

# ─── Canonical Host ───
Host: https://camli.com

# ─── Standard Crawlers ───
User-agent: *
Allow: /

# Private / transactional pages (no SEO value)
Allow: /booking-vs-camli
Disallow: /booking
Disallow: /confirmation
Disallow: /payment
Disallow: /shared
Disallow: /login
# Note: /searching and /results removed from Disallow — they use noindex meta tags instead.
# Blocking + noindex is counterproductive: Google can't see the noindex tag if it can't crawl the page.

# Admin / CRM panel
Disallow: /crmcamli/

# API endpoints (except public SEO/blog endpoints needed for rendering)
Disallow: /api/crm/
Disallow: /api/oauth/
Disallow: /api/track/

# Infrastructure
Disallow: /cdn-cgi/
Disallow: /_core/
Disallow: /user.v1.

# Prevent indexing of query-parameterized duplicates
Disallow: /*?sessionId=
Disallow: /*?utm_
Disallow: /*?ref=
Disallow: /*?fbclid=
Disallow: /*?gclid=
Disallow: /*?msclkid=

# ─── Google ───
User-agent: Googlebot
Allow: /api/trpc/seo.*
Allow: /api/trpc/crm.blog.*
Allow: /api/trpc/crm.sitemapDeals.*
Allow: /booking-vs-camli
Allow: /
Disallow: /booking
Disallow: /confirmation
Disallow: /payment
Disallow: /shared
Disallow: /crmcamli/
Disallow: /api/crm/
Disallow: /api/oauth/
Disallow: /api/track/

# Crawl-budget: prevent bots from crawling analytics and search API endpoints
Disallow: /api/trpc/crm.blogAnalytics.track
Disallow: /api/trpc/search.flights

# ─── Bing ───
User-agent: Bingbot
Allow: /api/trpc/seo.*
Allow: /api/trpc/crm.blog.*
Allow: /api/trpc/crm.sitemapDeals.*
Allow: /booking-vs-camli
Allow: /
Disallow: /booking
Disallow: /confirmation
Disallow: /payment
Disallow: /shared
Disallow: /crmcamli/
Disallow: /api/crm/
Disallow: /api/oauth/
Disallow: /api/track/
# Crawl-budget: prevent bots from crawling analytics and search API endpoints
Disallow: /api/trpc/crm.blogAnalytics.track
Disallow: /api/trpc/search.flights
Crawl-delay: 1

# ─── AI Crawlers (explicitly allowed for AEO discoverability) ───

# OpenAI (ChatGPT browsing, training, search)
User-agent: GPTBot
Allow: /
Disallow: /crmcamli/
Disallow: /api/crm/
Disallow: /api/oauth/

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Anthropic (Claude)
User-agent: ClaudeBot
Allow: /
Disallow: /crmcamli/
Disallow: /api/crm/
Disallow: /api/oauth/

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

# Google AI (Gemini, Bard, Vertex, Google Agent)
User-agent: Google-Extended
Allow: /

User-agent: Google-Agent
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /
Disallow: /crmcamli/
Disallow: /api/crm/
Disallow: /api/oauth/

User-agent: Perplexity-User
Allow: /

# Apple (Apple Intelligence, Siri)
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Microsoft (Copilot)
User-agent: CopilotBot
Allow: /

# Common Crawl (used by multiple LLM training sets)
User-agent: CCBot
Allow: /

# Mistral
User-agent: MistralAI-User
Allow: /

# Meta AI
User-agent: Meta-ExternalAgent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

User-agent: FacebookBot
Allow: /

# Cohere
User-agent: cohere-ai
Allow: /

# You.com
User-agent: YouBot
Allow: /

# AI2 (Allen Institute for AI)
User-agent: AI2Bot
Allow: /

User-agent: Ai2Bot-Dolma
Allow: /

# Brave Search
User-agent: BraveBot
Allow: /

# Amazon / Alexa
User-agent: Amazonbot
Allow: /

# Bytedance / TikTok AI
User-agent: Bytespider
Allow: /

# DeepSeek
User-agent: DeepSeekBot
Allow: /

# ─── Sitemaps ───
Sitemap: https://camli.com/sitemap.xml
Sitemap: https://camli.com/sitemap-images.xml