User-agent: Googlebot
Allow: /
Disallow: /admin
Disallow: /auth
Disallow: /dashboard
Disallow: /report/
Disallow: /ref/
Disallow: /ref1/
Disallow: /affiliate/
Disallow: /affiliate1/
Disallow: /free/
Disallow: /start-cb/

User-agent: Bingbot
Allow: /
Disallow: /admin
Disallow: /auth
Disallow: /dashboard
Disallow: /report/
Disallow: /ref/
Disallow: /ref1/
Disallow: /affiliate/
Disallow: /affiliate1/
Disallow: /free/
Disallow: /start-cb/

User-agent: Twitterbot
Allow: /
Disallow: /admin
Disallow: /auth
Disallow: /dashboard
Disallow: /report/
Disallow: /ref/
Disallow: /ref1/
Disallow: /affiliate/
Disallow: /affiliate1/
Disallow: /free/
Disallow: /start-cb/

User-agent: facebookexternalhit
Allow: /
Disallow: /admin
Disallow: /auth
Disallow: /dashboard
Disallow: /report/
Disallow: /ref/
Disallow: /ref1/
Disallow: /affiliate/
Disallow: /affiliate1/
Disallow: /free/
Disallow: /start-cb/

# ===== AI / LLM crawlers (GEO - Generative Engine Optimization) =====
# Allowed to index public content so the site surfaces in AI answers
# (ChatGPT, Perplexity, Claude, Gemini, Copilot, etc.)

# OpenAI - ChatGPT search indexing
User-agent: OAI-SearchBot
Allow: /
Disallow: /admin
Disallow: /auth
Disallow: /dashboard
Disallow: /report/
Disallow: /ref/
Disallow: /ref1/
Disallow: /affiliate/
Disallow: /affiliate1/
Disallow: /free/
Disallow: /start-cb/

# OpenAI - ChatGPT live browsing on user request
User-agent: ChatGPT-User
Allow: /
Disallow: /admin
Disallow: /auth
Disallow: /dashboard
Disallow: /report/
Disallow: /ref/
Disallow: /ref1/
Disallow: /affiliate/
Disallow: /affiliate1/
Disallow: /free/
Disallow: /start-cb/

# OpenAI - GPTBot (training). Allow to maximise AI visibility.
User-agent: GPTBot
Allow: /
Disallow: /admin
Disallow: /auth
Disallow: /dashboard
Disallow: /report/
Disallow: /ref/
Disallow: /ref1/
Disallow: /affiliate/
Disallow: /affiliate1/
Disallow: /free/
Disallow: /start-cb/

# Google - Gemini / AI Overviews / Vertex AI
User-agent: Google-Extended
Allow: /
Disallow: /admin
Disallow: /auth
Disallow: /dashboard
Disallow: /report/
Disallow: /ref/
Disallow: /ref1/
Disallow: /affiliate/
Disallow: /affiliate1/
Disallow: /free/
Disallow: /start-cb/

# Anthropic - Claude
User-agent: ClaudeBot
Allow: /
Disallow: /admin
Disallow: /auth
Disallow: /dashboard
Disallow: /report/
Disallow: /ref/
Disallow: /ref1/
Disallow: /affiliate/
Disallow: /affiliate1/
Disallow: /free/
Disallow: /start-cb/

User-agent: Claude-Web
Allow: /
Disallow: /admin
Disallow: /auth
Disallow: /dashboard
Disallow: /report/
Disallow: /ref/
Disallow: /ref1/
Disallow: /affiliate/
Disallow: /affiliate1/
Disallow: /free/
Disallow: /start-cb/

User-agent: anthropic-ai
Allow: /
Disallow: /admin
Disallow: /auth
Disallow: /dashboard
Disallow: /report/
Disallow: /ref/
Disallow: /ref1/
Disallow: /affiliate/
Disallow: /affiliate1/
Disallow: /free/
Disallow: /start-cb/

# Perplexity
User-agent: PerplexityBot
Allow: /
Disallow: /admin
Disallow: /auth
Disallow: /dashboard
Disallow: /report/
Disallow: /ref/
Disallow: /ref1/
Disallow: /affiliate/
Disallow: /affiliate1/
Disallow: /free/
Disallow: /start-cb/

User-agent: Perplexity-User
Allow: /
Disallow: /admin
Disallow: /auth
Disallow: /dashboard
Disallow: /report/
Disallow: /ref/
Disallow: /ref1/
Disallow: /affiliate/
Disallow: /affiliate1/
Disallow: /free/
Disallow: /start-cb/

# Microsoft Copilot
User-agent: CCBot
Allow: /
Disallow: /admin
Disallow: /auth
Disallow: /dashboard
Disallow: /report/
Disallow: /ref/
Disallow: /ref1/
Disallow: /affiliate/
Disallow: /affiliate1/
Disallow: /free/
Disallow: /start-cb/

# Apple Intelligence
User-agent: Applebot
Allow: /
Disallow: /admin
Disallow: /auth
Disallow: /dashboard
Disallow: /report/
Disallow: /ref/
Disallow: /ref1/
Disallow: /affiliate/
Disallow: /affiliate1/
Disallow: /free/
Disallow: /start-cb/

User-agent: Applebot-Extended
Allow: /
Disallow: /admin
Disallow: /auth
Disallow: /dashboard
Disallow: /report/
Disallow: /ref/
Disallow: /ref1/
Disallow: /affiliate/
Disallow: /affiliate1/
Disallow: /free/
Disallow: /start-cb/

# Meta AI
User-agent: Meta-ExternalAgent
Allow: /
Disallow: /admin
Disallow: /auth
Disallow: /dashboard
Disallow: /report/
Disallow: /ref/
Disallow: /ref1/
Disallow: /affiliate/
Disallow: /affiliate1/
Disallow: /free/
Disallow: /start-cb/

User-agent: FacebookBot
Allow: /
Disallow: /admin
Disallow: /auth
Disallow: /dashboard
Disallow: /report/
Disallow: /ref/
Disallow: /ref1/
Disallow: /affiliate/
Disallow: /affiliate1/
Disallow: /free/
Disallow: /start-cb/

# Amazon AI
User-agent: Amazonbot
Allow: /
Disallow: /admin
Disallow: /auth
Disallow: /dashboard
Disallow: /report/
Disallow: /ref/
Disallow: /ref1/
Disallow: /affiliate/
Disallow: /affiliate1/
Disallow: /free/
Disallow: /start-cb/

# DuckDuckGo AI
User-agent: DuckAssistBot
Allow: /
Disallow: /admin
Disallow: /auth
Disallow: /dashboard
Disallow: /report/
Disallow: /ref/
Disallow: /ref1/
Disallow: /affiliate/
Disallow: /affiliate1/
Disallow: /free/
Disallow: /start-cb/

# You.com
User-agent: YouBot
Allow: /
Disallow: /admin
Disallow: /auth
Disallow: /dashboard
Disallow: /report/
Disallow: /ref/
Disallow: /ref1/
Disallow: /affiliate/
Disallow: /affiliate1/
Disallow: /free/
Disallow: /start-cb/

# Mistral
User-agent: MistralAI-User
Allow: /
Disallow: /admin
Disallow: /auth
Disallow: /dashboard
Disallow: /report/
Disallow: /ref/
Disallow: /ref1/
Disallow: /affiliate/
Disallow: /affiliate1/
Disallow: /free/
Disallow: /start-cb/

# Cohere
User-agent: cohere-ai
Allow: /
Disallow: /admin
Disallow: /auth
Disallow: /dashboard
Disallow: /report/
Disallow: /ref/
Disallow: /ref1/
Disallow: /affiliate/
Disallow: /affiliate1/
Disallow: /free/
Disallow: /start-cb/

User-agent: cohere-training-data-crawler
Allow: /
Disallow: /admin
Disallow: /auth
Disallow: /dashboard
Disallow: /report/
Disallow: /ref/
Disallow: /ref1/
Disallow: /affiliate/
Disallow: /affiliate1/
Disallow: /free/
Disallow: /start-cb/

# Bytedance / Doubao
User-agent: Bytespider
Allow: /
Disallow: /admin
Disallow: /auth
Disallow: /dashboard
Disallow: /report/
Disallow: /ref/
Disallow: /ref1/
Disallow: /affiliate/
Disallow: /affiliate1/
Disallow: /free/
Disallow: /start-cb/

# Diffbot
User-agent: Diffbot
Allow: /
Disallow: /admin
Disallow: /auth
Disallow: /dashboard
Disallow: /report/
Disallow: /ref/
Disallow: /ref1/
Disallow: /affiliate/
Disallow: /affiliate1/
Disallow: /free/
Disallow: /start-cb/

# Default - all other crawlers
User-agent: *
Allow: /
Disallow: /admin
Disallow: /auth
Disallow: /dashboard
Disallow: /report/
Disallow: /ref/
Disallow: /ref1/
Disallow: /affiliate/
Disallow: /affiliate1/
Disallow: /free/
Disallow: /start-cb/

Sitemap: https://atscvcheck.co.uk/sitemap.xml