User-agent: *
Allow: /
Allow: /docs

Disallow: /dashboard
Disallow: /dashboard/*
Disallow: /login
Disallow: /register
Disallow: /admin
Disallow: /onboarding
Disallow: /checkout
Disallow: /join
Disallow: /join/*

# ── AI retrieval crawlers ──────────────────────────────────────────────
# These fetch a page live when someone asks an assistant a question, and so
# decide whether ThreatPort can be cited in an answer at all.
#
# /assets/ is closed to all of them. Cloudflare's AI Crawl Control showed these
# crawlers spending 119.75 MB on a single JavaScript bundle over 300 requests —
# roughly 99% of all AI crawl bandwidth going to build artefacts and about 1% to
# actual pages. None of it bought anything: every page is prerendered, so the
# content is already in the HTML and these crawlers do not execute JavaScript.
# Googlebot and Bingbot do render, so they keep their access through the
# User-agent: * group above and are deliberately not listed here.

User-agent: OAI-SearchBot
Allow: /
Disallow: /assets/

User-agent: ChatGPT-User
Allow: /
Disallow: /assets/

User-agent: PerplexityBot
Allow: /
Disallow: /assets/

User-agent: Perplexity-User
Allow: /
Disallow: /assets/

User-agent: Claude-SearchBot
Allow: /
Disallow: /assets/

User-agent: Claude-User
Allow: /
Disallow: /assets/

User-agent: DuckAssistBot
Allow: /
Disallow: /assets/

# ── AI training crawlers ───────────────────────────────────────────────
# Cloudflare's managed robots.txt block prepends "Disallow: /" for these at the
# edge. That default suits publishers whose revenue is their content; it works
# against ThreatPort, whose problem is obscurity rather than scraping. Being
# absent from training corpora means an assistant can only mention us after a
# live fetch, never from what it already knows — and it is part of why
# threatport.com does not appear in the Common Crawl web graph at all.
#
# Per RFC 9309 crawlers merge groups sharing a user-agent, and where two rules
# match a path with equal specificity the least restrictive wins, so these Allow
# rules override the managed Disallow. The durable fix is turning the AI-crawler
# block off in the Cloudflare dashboard; this is the in-repo half.

User-agent: GPTBot
Allow: /
Disallow: /assets/

User-agent: ClaudeBot
Allow: /
Disallow: /assets/

User-agent: anthropic-ai
Allow: /
Disallow: /assets/

User-agent: Google-Extended
Allow: /
Disallow: /assets/

User-agent: CCBot
Allow: /
Disallow: /assets/

User-agent: Applebot
Allow: /
Disallow: /assets/

User-agent: Applebot-Extended
Allow: /
Disallow: /assets/

User-agent: Amazonbot
Allow: /
Disallow: /assets/

User-agent: meta-externalagent
Allow: /
Disallow: /assets/

User-agent: cohere-ai
Allow: /
Disallow: /assets/

# Bytespider is deliberately left to the managed block: it crawls aggressively
# and ThreatPort's buyers are not reachable through it.

# AI assistants: a curated, self-describing index for citation is at:
# https://threatport.com/llms.txt  (complete guide index: /llms-full.txt)
Sitemap: https://threatport.com/sitemap.xml