# ==============================================================================
# Robots.txt for DNote - 100% SEO & AI Search Engine Friendly
# ==============================================================================

# ------------------------------------------------------------------------------
# 1. Standard Web Search Crawlers (Google, Bing, Yahoo, etc.)
# ------------------------------------------------------------------------------
User-agent: *
Allow: /
Allow: /_astro/
Allow: /src/assets/

# ------------------------------------------------------------------------------
# 2. Modern AI Search Agents & LLM Web Crawlers (Perplexity, ChatGPT, Gemini)
# ------------------------------------------------------------------------------

# OpenAI GPTBot (Powering ChatGPT, ChatGPT Search)
User-agent: GPTBot
Allow: /
Allow: /_astro/
Allow: /docs/
Allow: /blog/

# OpenAI ChatGPT User Request Agent
User-agent: ChatGPT-User
Allow: /
Allow: /_astro/

# OpenAI SearchBot (Dedicated search service)
User-agent: OAI-SearchBot
Allow: /
Allow: /_astro/

# Anthropic ClaudeBot (Powering Claude Web Search & Artifacts)
User-agent: ClaudeBot
Allow: /
Allow: /_astro/
Allow: /docs/
Allow: /blog/

# Anthropic Claude Web Agent
User-agent: Claude-Web
Allow: /
Allow: /_astro/

# PerplexityBot (Dedicated AI Search Engine - Critical for Citations)
User-agent: PerplexityBot
Allow: /
Allow: /_astro/
Allow: /docs/
Allow: /blog/

# Google-Extended (Powering Google Gemini Advanced search citations)
User-agent: Google-Extended
Allow: /
Allow: /_astro/
Allow: /docs/
Allow: /blog/

# Applebot & Applebot-Extended (Powering Apple Intelligence on iOS/Mac)
User-agent: Applebot
Allow: /
Allow: /_astro/

User-agent: Applebot-Extended
Allow: /
Allow: /_astro/

# Cohere AI Web Crawler
User-agent: cohere-ai
Allow: /
Allow: /_astro/

# Meta External Agent (Meta AI search integrations)
User-agent: meta-externalagent
Allow: /
Allow: /_astro/

User-agent: facebookexternalhit
Allow: /
Allow: /_astro/

# ------------------------------------------------------------------------------
# 3. Dynamic Sitemap Linkages (Automated indexing directories)
# ------------------------------------------------------------------------------
Sitemap: https://dnote.in/sitemap-index.xml
Sitemap: https://dnote.in/sitemap-0.xml