# Robots.txt for Ankaforce
# https://ankaforce.com

User-agent: *
Allow: /

# Sitemaps
Sitemap: https://ankaforce.com/sitemap.xml

# IndexNow - Instant URL submission to Bing, Yandex, and AI search engines
# Key verification file: https://ankaforce.com/9e9762a5b788417b91cfa6c7e20e4c05.txt

# AI/LLM Information Files
# llms.txt provides structured product info for AI assistants
# See: https://llmstxt.org/
# LLMs-Txt: https://ankaforce.com/llms.txt

# Disallow admin and private areas
Disallow: /admin/
Disallow: /api/
Disallow: /media/uploads/
Disallow: /static/admin/

# Disallow duplicate content
Disallow: /search?
Disallow: /*?sort=
Disallow: /*?filter=
Disallow: /*?category=

# Allow search engines to access CSS, JS, and media
Allow: /static/*.css
Allow: /static/*.js
Allow: /media/*.jpg
Allow: /media/*.png
Allow: /media/*.webp

# === Search Engines ===

User-agent: Googlebot
Allow: /
Crawl-delay: 0

User-agent: Bingbot
Allow: /
Crawl-delay: 0

User-agent: Slurp
Allow: /
Crawl-delay: 1

# === AI Search Crawlers ===
# These bots power ChatGPT, Claude, Perplexity, and other AI search tools.
# Allowing them is critical for AI visibility.

# OpenAI - ChatGPT Search (appears in ChatGPT search results)
User-agent: OAI-SearchBot
Allow: /

# OpenAI - Model training (helps ChatGPT learn about Ankaforce)
User-agent: GPTBot
Allow: /

# OpenAI - User-initiated browsing in ChatGPT
User-agent: ChatGPT-User
Allow: /

# Anthropic - Claude
User-agent: ClaudeBot
Allow: /
User-agent: anthropic-ai
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# Google AI (Gemini)
User-agent: Google-Extended
Allow: /

# Meta AI
User-agent: FacebookBot
Allow: /

# Apple AI (Siri, Apple Intelligence)
User-agent: Applebot
Allow: /
User-agent: Applebot-Extended
Allow: /

# Cohere AI
User-agent: cohere-ai
Allow: /

# Common Crawl (used by many AI models for training)
User-agent: CCBot
Allow: /

# Microsoft Copilot
User-agent: Copilot
Allow: /

# Anthropic - Claude Search (newer crawler)
User-agent: Claude-SearchBot
Allow: /

# Brave AI Search
User-agent: BraveBot
Allow: /

# You.com AI Search
User-agent: YouBot
Allow: /

# Deepseek AI
User-agent: Deepseek
Allow: /

# Mistral AI
User-agent: MistralBot
Allow: /

# Amazon AI (Alexa, Amazon Q)
User-agent: Amazonbot
Allow: /

# === Social Media Crawlers ===

User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: WhatsApp
Allow: /

# === SEO Tool Crawlers ===
# Allowed so backlink profiles are visible to SEO tools

User-agent: SemrushBot
Allow: /

User-agent: AhrefsBot
Allow: /

# === Block Unwanted Bots ===

User-agent: DotBot
Disallow: /