# ═══════════════════════════════════════════════════════════════════
# Contro1 Platform - AI-Optimized robots.txt
# Generated with Contro1 Semantic Scanner
# Last Updated: 2026-02-05
# ═══════════════════════════════════════════════════════════════════
#
# This robots.txt demonstrates best practices for AI agent discoverability.
# Contro1 helps you generate optimized robots.txt for your own website.
#
# Learn more: https://contro1.com
# ═══════════════════════════════════════════════════════════════════

# ═══════════════════════════════════════════════════════════════════
# General Web Crawlers (Search Engines)
# ═══════════════════════════════════════════════════════════════════

User-agent: *
Allow: /
Allow: /home
Allow: /contact
Allow: /api/scanner/public/
Allow: /api/manifest/
Allow: /api/llms-txt/
Allow: /api/navigation-map/

# Protected routes (require authentication)
Disallow: /dashboard
Disallow: /api/auth/
Disallow: /api/scanner/protected/
Disallow: /login
Disallow: /register

# Static assets (allow but deprioritize)
Allow: /*.css$
Allow: /*.js$
Allow: /*.png$
Allow: /*.jpg$
Allow: /*.svg$
Allow: /*.ico$

Crawl-delay: 1

# ═══════════════════════════════════════════════════════════════════
# AI Agents & LLM Bots (More Permissive)
# ═══════════════════════════════════════════════════════════════════
# AI agents need broader access to understand site structure and workflows

# OpenAI GPT Bot (ChatGPT, GPT-4, etc.)
User-agent: GPTBot
User-agent: ChatGPT-User
Allow: /
Allow: /home
Allow: /api/scanner/public/
Allow: /api/manifest/
Allow: /api/llms-txt/
Allow: /api/navigation-map/
Disallow: /dashboard
Disallow: /api/auth/
Crawl-delay: 0.5

# Anthropic Claude Bot
User-agent: Claude-Web
User-agent: anthropic-ai
Allow: /
Allow: /home
Allow: /api/scanner/public/
Allow: /api/manifest/
Allow: /api/llms-txt/
Allow: /api/navigation-map/
Disallow: /dashboard
Disallow: /api/auth/
Crawl-delay: 0.5

# Google AI (Gemini, Bard)
User-agent: Google-Extended
User-agent: GoogleOther
Allow: /
Allow: /home
Allow: /api/scanner/public/
Allow: /api/manifest/
Allow: /api/llms-txt/
Allow: /api/navigation-map/
Disallow: /dashboard
Disallow: /api/auth/
Crawl-delay: 0.5

# Perplexity AI
User-agent: PerplexityBot
Allow: /
Allow: /home
Allow: /api/scanner/public/
Allow: /api/manifest/
Disallow: /dashboard
Disallow: /api/auth/
Crawl-delay: 0.5

# Meta AI
User-agent: FacebookBot
User-agent: Meta-ExternalAgent
Allow: /
Allow: /home
Disallow: /dashboard
Disallow: /api/auth/
Crawl-delay: 1

# Cohere AI
User-agent: cohere-ai
Allow: /
Allow: /home
Allow: /api/manifest/
Disallow: /dashboard
Disallow: /api/auth/
Crawl-delay: 0.5

# Diffbot (AI-powered web data extraction)
User-agent: Diffbot
Allow: /
Allow: /api/scanner/public/
Disallow: /dashboard
Disallow: /api/auth/
Crawl-delay: 1

# Common Crawler (used by AI training)
User-agent: CCBot
Allow: /
Allow: /home
Disallow: /dashboard
Disallow: /api/auth/
Crawl-delay: 1

# ═══════════════════════════════════════════════════════════════════
# AI Navigation & Discovery Files
# ═══════════════════════════════════════════════════════════════════
# Standard locations for AI agent discovery (RFC-style)

# llms.txt - AI-friendly site documentation
# Learn more: https://llmstxt.org
llms-txt: https://contro1.com/llms.txt

# AI Manifest - Semantic navigation for AI agents
AI-Manifest: https://contro1.com/api/manifest/contro1
Agent-Manifest: https://contro1.com/api/manifest/contro1

# Navigation Map - Browser automation guide
Navigation-Map: https://contro1.com/api/navigation-map/contro1

# API Documentation
API-Docs: https://contro1.com/api/docs

# ═══════════════════════════════════════════════════════════════════
# Sitemaps
# ═══════════════════════════════════════════════════════════════════

Sitemap: https://contro1.com/sitemap-index.xml
Sitemap: https://contro1.com/sitemap.xml
Sitemap: https://contro1.com/sitemap-api.xml

# ═══════════════════════════════════════════════════════════════════
# AI Agent Guidelines & Context
# ═══════════════════════════════════════════════════════════════════
#
# About Contro1:
# - Platform for making websites AI-agent ready
# - Scans websites and generates llms.txt, robots.txt, manifests
# - Supports browser automation and workflow detection
#
# Recommended paths for AI agents:
# 1. /home - Main landing page with scanner
# 2. /api/scanner/public/registry - Registry of scanned sites
# 3. /api/manifest/{domain} - AI navigation manifests
# 4. /llms.txt - Human-readable AI documentation
#
# API Endpoints (public):
# - POST /api/public/quick-discovery - Quick site analysis
# - GET /api/scanner/public/manifest/{domain} - Get manifest by domain
# - GET /api/manifest/{scanId} - Get manifest by scan ID
# - GET /api/llms-txt/{scanId} - Get llms.txt by scan ID
#
# Authentication:
# - Protected routes require JWT token
# - Public routes available without authentication
# - Register at /register to access full features
#
# Rate Limits:
# - Public API: 10 requests/minute per IP
# - Authenticated API: 100 requests/minute per user
#
# Contact:
# - Email: support@contro1.com
# - Documentation: https://contro1.com/docs
#
# ═══════════════════════════════════════════════════════════════════
# Browser Automation & Testing
# ═══════════════════════════════════════════════════════════════════

# Puppeteer / Playwright / Selenium
# If you're testing browser automation with Contro1:
# - Use /home for UI testing
# - Use /api/public/quick-discovery for API testing
# - Download Puppeteer configs from manifests
# - Check /api/navigation-map/ for detailed selectors

# ═══════════════════════════════════════════════════════════════════
# Statistics
# ═══════════════════════════════════════════════════════════════════
# Public Pages: 1 (home)
# API Endpoints: 15+ public endpoints
# Workflows Supported: Website scanning, Manifest generation, API access
# AI Bots Configured: 9+ major AI agents
# Generated by: Contro1 Semantic Scanner v1.0
# ═══════════════════════════════════════════════════════════════════