# Collar Guardrail — robots.txt
# API:  https://api.collarguardrail.com
# Site: https://collarguardrail.com

# =============================================================
# Search & AI crawlers — allowed
#
# We want to be discovered by search engines and AI assistants.
# Everything under the discovery surface (llms.txt, skill.md,
# /.well-known/, methodology, status) is meant to be read.
# =============================================================

User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: Bytespider
Allow: /

User-agent: MCPBot
Allow: /

# =============================================================
# MCP directory collectors
#
# These bots probe MCP servers to index them. We want to be
# indexed, but their live probes are often malformed and hammer
# the transport. Prefer the declarative server card over live
# probing; if you must probe, respect the crawl delay.
# =============================================================

User-agent: rokmcp-collector
Crawl-delay: 10
Allow: /.well-known/mcp-server-card.json
Allow: /.well-known/agent-card.json
Allow: /.well-known/x402.json
Allow: /.well-known/methodology.md
Allow: /llms.txt
Allow: /skill.md
Allow: /methodology.html
Disallow: /mcp-http/
Disallow: /api/

User-agent: rokmcp
Crawl-delay: 10
Allow: /.well-known/
Allow: /llms.txt
Allow: /skill.md
Allow: /methodology.html
Disallow: /mcp-http/
Disallow: /api/

# =============================================================
# Default — discovery open, transport closed
#
# The discovery surface is public on purpose. The transport
# (/mcp-http/) and REST API (/api/) are for live agent calls,
# not for crawlers.
# =============================================================

User-agent: *
Allow: /
Allow: /.well-known/
Allow: /llms.txt
Allow: /skill.md
Allow: /methodology.html
Disallow: /api/
Disallow: /mcp-http/

Sitemap: https://api.collarguardrail.com/sitemap.xml