# Mockly - robots.txt
# Last updated: 2026-10-29
# Allow search engines and AI crawlers on public content.
# Training / AI use explicitly opted in via Content-Signal and Content-Usage.

User-agent: *
Allow: /
Disallow: /api/
Disallow: /dashboard
Disallow: /admin
Disallow: /embed/
Disallow: /checkout/
Disallow: /lp/
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Content-Usage: search=y, ai-use=y, train-ai=y

# Explicitly named crawlers (public pages). A crawler that matches a named
# group ignores the * group, so the private-path Disallows are repeated here.
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ChatGPT Agent
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: Claude-Web
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Googlebot
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: GoogleOther-Image
User-agent: GoogleOther-Video
User-agent: Google-Agent
User-agent: Google-CloudVertexBot
User-agent: Google-NotebookLM
User-agent: Google-Gemini-CLI
User-agent: Gemini-Deep-Research
User-agent: CloudVertexBot
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: Meta-ExternalAgent
User-agent: meta-externalagent
User-agent: Meta-ExternalFetcher
User-agent: meta-externalfetcher
User-agent: FacebookBot
User-agent: facebookexternalhit
User-agent: Amazonbot
User-agent: Amzn-SearchBot
User-agent: Amzn-User
User-agent: MistralAI-User
User-agent: MistralAI-Index
User-agent: MistralAI-Training
User-agent: Bytespider
User-agent: CCBot
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
User-agent: YouBot
User-agent: Diffbot
User-agent: Diffbot-User
User-agent: DuckAssistBot
User-agent: DuckDuckBot
User-agent: AI2Bot
User-agent: Ai2Bot-Dolma
User-agent: ImagesiftBot
User-agent: DeepSeekBot
User-agent: KimiBot
User-agent: Kimi-SearchBot
User-agent: Kimi-User
User-agent: Kimi-Agent
User-agent: ExaBot
User-agent: ExaSearchBot
User-agent: FirecrawlAgent
User-agent: Bingbot
User-agent: Slurp
User-agent: YandexBot
User-agent: Baiduspider
User-agent: Bravebot
User-agent: AzureAI-SearchBot
Allow: /
Disallow: /api/
Disallow: /dashboard
Disallow: /admin
Disallow: /embed/
Disallow: /checkout/
Disallow: /lp/

Sitemap: https://app.mocklyenglish.com/sitemap.xml

# Structured summaries for LLM / answer-engine consumption (llmstxt.org)
LLM-Content: https://app.mocklyenglish.com/llms.txt
LLM-Full-Content: https://app.mocklyenglish.com/llms-full.txt
AI-Content: https://app.mocklyenglish.com/ai.txt
# Fact sheet: https://app.mocklyenglish.com/ai-fact-sheet.md
# Full catalog: https://app.mocklyenglish.com/catalog.md
# Endpoint index: https://app.mocklyenglish.com/api-catalog.json
# Also at https://app.mocklyenglish.com/.well-known/llms.txt
# Developer docs: https://app.mocklyenglish.com/developers
# Every page as markdown: add .md to any page URL (the homepage is /index.md)
# MCP server: https://app.mocklyenglish.com/mcp