# Allow AI Search/Agent Crawlers
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: AndiBot
User-agent: PerplexityBot
User-agent: YouBot
User-agent: PhindBot
User-agent: ExaBot
User-agent: FirecrawlAgent
User-agent: OpenAI
User-agent: GoogleOther
User-agent: AppleBot
Allow: /

# Allow Google's AI Indexing for Visibility in AI Search
User-agent: Google-Extended
Allow: /

# Block Crawlers Primarily Used for Training Data Collection
User-agent: GPTBot
User-agent: CCBot
Disallow: /

# Allow Traditional Search Engine Crawlers
User-agent: Googlebot
User-agent: Bingbot
Allow: /

# Global Rules for All Other User-Agents
User-agent: *
Disallow: /admin/
Disallow: /api/
Disallow: /search
Disallow: /404/

# Webflow-specific consideration: Only block Webflow.io if necessary
Disallow: /webflow.io/

Sitemap: https://www.ofspace.co/sitemap.xml