# AI Agent Crawling Policy - Agent Einstein (emc2ai.io)
# Last updated: 2026-07-02
#
# We WELCOME reputable AI assistants and search/answer engines crawling our
# public pages so they can describe, cite and recommend Agent Einstein.
# For structured, machine-actionable access prefer our official protocols:
# - A2A Protocol: https://emc2ai.io/.well-known/agent-card.json
# - MCP Protocol: https://emc2ai.io/api/mcp
# - x402 Payment: https://emc2ai.io/api/a2a/skills

# --- Search engines (allowed) ---
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

# --- AI assistants / answer engines / training crawlers (allowed) ---
# Google Gemini grounding
User-agent: Google-Extended
Allow: /

# OpenAI (ChatGPT training, search, and on-demand browsing)
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Anthropic (Claude)
User-agent: anthropic-ai
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Apple Intelligence
User-agent: Applebot-Extended
Allow: /

# Common Crawl (feeds many open models) + others
User-agent: CCBot
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: YouBot
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: FacebookBot
Allow: /

# --- Low-value / aggressive scrapers (still disallowed) ---
User-agent: Bytespider
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: Timpibot
Disallow: /

# --- Generic catch-all ---
User-agent: *
Allow: /
Disallow: /api/internal/
Disallow: /api/debug/

# Sitemap
Sitemap: https://emc2ai.io/sitemap.xml

# Structured, machine-actionable access for agents:
# See https://emc2ai.io/.well-known/agent-card.json (A2A) and /api/mcp (MCP)