# dfklabs.com — robots.txt
#
# This file is a request, not a fence. It has no technical force: a crawler
# that ignores it still gets the bytes, because anything a browser can read a
# script can read. Treat what follows as a statement of intent that
# well-behaved agents honour, and nothing more.
#
# The split below is deliberate. Being found and cited is the point of
# publishing. Being absorbed into a training corpus is a different act, and
# consent to one is not consent to the other.

# ── Search and citation: welcome ─────────────────────────────────────────────
# These fetch a page to index it or to answer a question with a link back.

User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Applebot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: PerplexityBot
Allow: /

# ── Model training: not granted ──────────────────────────────────────────────
# These collect text to train on. Publishing under copyright is not a licence
# to do that, and the terms at /terms say so in words as well as here.

User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: omgili
Disallow: /

User-agent: Timpibot
Disallow: /

# ── Everything else ──────────────────────────────────────────────────────────
# Open by default. An agent not named above is assumed to be reading, not
# harvesting; if that changes, it gets named. /admin is the private board and
# /api its endpoints — they are noindex and password-gated, and this line only
# keeps them out of an index, it does not keep anyone out.

User-agent: *
Disallow: /admin
Disallow: /api/
Allow: /

Sitemap: https://dfklabs.com/sitemap.xml