User-agent: *
Allow: /
Sitemap: https://ierhub.com/sitemap.xml

# Explicitly allow AI crawlers. A crawler that matches its own group ignores
# the User-agent: * group, so each group repeats the blocks it needs.
User-agent: GPTBot
Allow: /
Allow: /api/mcp
Allow: /api/badge/
Disallow: /api/
Disallow: /checkout-success
Disallow: /tools/license-lookup?
Disallow: /tools/license-lookup/es?

User-agent: ClaudeBot
Allow: /
Allow: /api/mcp
Allow: /api/badge/
Disallow: /api/
Disallow: /checkout-success
Disallow: /tools/license-lookup?
Disallow: /tools/license-lookup/es?

User-agent: PerplexityBot
Allow: /
Allow: /api/mcp
Allow: /api/badge/
Disallow: /api/
Disallow: /checkout-success
Disallow: /tools/license-lookup?
Disallow: /tools/license-lookup/es?

User-agent: Google-Extended
Allow: /
Allow: /api/mcp
Allow: /api/badge/
Disallow: /api/
Disallow: /checkout-success
Disallow: /tools/license-lookup?
Disallow: /tools/license-lookup/es?

User-agent: Bytespider
Allow: /
Allow: /api/mcp
Allow: /api/badge/
Disallow: /api/
Disallow: /checkout-success
Disallow: /tools/license-lookup?
Disallow: /tools/license-lookup/es?

# OpenAI ChatGPT search + ads crawlers (ads.openai.com landing-page requirement)
User-agent: OAI-SearchBot
Allow: /
Allow: /api/mcp
Allow: /api/badge/
Disallow: /api/
Disallow: /checkout-success
Disallow: /tools/license-lookup?
Disallow: /tools/license-lookup/es?

User-agent: OAI-AdsBot
Allow: /
Allow: /api/mcp
Allow: /api/badge/
Disallow: /api/
Disallow: /checkout-success
Disallow: /tools/license-lookup?
Disallow: /tools/license-lookup/es?

# Block internal routes
User-agent: *
Allow: /api/mcp
Allow: /api/badge/
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /checkout-success
Disallow: /tools/license-lookup?
Disallow: /tools/license-lookup/es?