User-agent: *
Allow: /blog/
Allow: /home
Disallow: /api/
Disallow: /dashboard/
Disallow: /admin/
Disallow: /settings/
Disallow: /onboarding/
Disallow: /login/
Disallow: /cursor-login/

# AI Crawlers — explicitly welcome on public pages
# Per-bot blocks replace the * rules, so disallows are repeated for safety.

# ── OpenAI ──────────────────────────────────────────────────────────
User-agent: GPTBot
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /admin/
Disallow: /settings/
Disallow: /onboarding/
Disallow: /login/
Disallow: /cursor-login/

User-agent: OAI-SearchBot
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /admin/
Disallow: /settings/
Disallow: /onboarding/
Disallow: /login/
Disallow: /cursor-login/

User-agent: ChatGPT-User
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /admin/
Disallow: /settings/
Disallow: /onboarding/
Disallow: /login/
Disallow: /cursor-login/

# ── Anthropic ───────────────────────────────────────────────────────
User-agent: Claude-Web
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /admin/
Disallow: /settings/
Disallow: /onboarding/
Disallow: /login/
Disallow: /cursor-login/

User-agent: ClaudeBot
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /admin/
Disallow: /settings/
Disallow: /onboarding/
Disallow: /login/
Disallow: /cursor-login/

User-agent: anthropic-ai
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /admin/
Disallow: /settings/
Disallow: /onboarding/
Disallow: /login/
Disallow: /cursor-login/

# ── Perplexity ──────────────────────────────────────────────────────
User-agent: PerplexityBot
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /admin/
Disallow: /settings/
Disallow: /onboarding/
Disallow: /login/
Disallow: /cursor-login/

# ── Google Gemini grounding (separate signal from Googlebot) ────────
User-agent: Google-Extended
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /admin/
Disallow: /settings/
Disallow: /onboarding/
Disallow: /login/
Disallow: /cursor-login/

# ── Apple Intelligence ──────────────────────────────────────────────
User-agent: Applebot-Extended
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /admin/
Disallow: /settings/
Disallow: /onboarding/
Disallow: /login/
Disallow: /cursor-login/

# ── Amazon (Alexa / Rufus) ──────────────────────────────────────────
User-agent: Amazonbot
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /admin/
Disallow: /settings/
Disallow: /onboarding/
Disallow: /login/
Disallow: /cursor-login/

# ── Common Crawl (indirect LLM grounding) ───────────────────────────
User-agent: CCBot
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /admin/
Disallow: /settings/
Disallow: /onboarding/
Disallow: /login/
Disallow: /cursor-login/

# ── Meta AI ─────────────────────────────────────────────────────────
User-agent: Meta-ExternalAgent
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /admin/
Disallow: /settings/
Disallow: /onboarding/
Disallow: /login/
Disallow: /cursor-login/

Sitemap: https://www.joinnextdev.com/sitemap.xml
Sitemap: https://www.joinnextdev.com/blog/sitemap.xml