User-agent: *
Allow: /
# /portal is Clerk-gated and every portal page already sets its own
# noindex,nofollow meta tag (see PortalMeta.tsx), so this is defence-in-depth
# only - it stops a crawler fetching the JS bundle for a route it could
# never usefully index anyway.
Disallow: /portal

# AI / answer-engine crawlers — explicitly allowed. Functionally redundant
# with the wildcard rule above (compliant crawlers already fall through to
# it), but named here deliberately: Google-Extended specifically is an
# AI-training opt-in/out token (not a crawler) rather than an indexing bot,
# and naming these removes any ambiguity that access was a deliberate choice.
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: CCBot
Allow: /

# Sitemap
Sitemap: https://www.techskills.org/sitemap.xml