# Robots.txt for Scrivibe - AI eBook Generator # https://www.scrivibe.com/robots.txt User-agent: * Allow: / # Allow indexing of main pages Allow: /tokens/buy Allow: /register Allow: /login # Disallow user-specific content and admin areas Disallow: /profile Disallow: /my-projects Disallow: /logout Disallow: /book/*/download Disallow: /book/*/download-preview Disallow: /stripe-webhook Disallow: /api/ # Disallow machine-to-machine API endpoints (not human-readable pages) Disallow: /mcp Disallow: /.well-known/mcp.json # Disallow temporary and session files Disallow: /static/temp/ Disallow: /instance/ # Allow CSS, JS, and images for proper rendering Allow: /static/css/ Allow: /static/js/ Allow: /static/img/ Allow: /static/fonts/ # Sitemap location Sitemap: https://www.scrivibe.com/sitemap.xml # Crawl delay to be respectful to server resources Crawl-delay: 1 # Specific instructions for major search engines User-agent: Googlebot Allow: / Crawl-delay: 1 User-agent: Bingbot Allow: / Crawl-delay: 1 User-agent: Slurp Allow: / Crawl-delay: 2 # AI training and indexing crawlers — explicitly welcome User-agent: GPTBot Allow: / User-agent: ChatGPT-User Allow: / User-agent: ClaudeBot Allow: / User-agent: anthropic-ai Allow: / User-agent: PerplexityBot Allow: / User-agent: Applebot-Extended Allow: / User-agent: cohere-ai Allow: / User-agent: Diffbot Allow: / User-agent: Omgilibot Allow: / # Block abusive bots User-agent: SemrushBot Disallow: / User-agent: AhrefsBot Disallow: / User-agent: MJ12bot Disallow: /