# Non-Tech CEO — nontechceo.co.uk
#
# The position, set by Robert Henry 2026-08-04, and it is deliberately NOT
# "block all AI". Two different kinds of crawler wear the same label:
#
#   SEARCH AND CITATION crawlers fetch a page to answer somebody's question and
#   link back. That is how this site gets found, and the Journal was written
#   answer-first specifically to be cited by them. They stay welcome.
#
#   TRAINING crawlers take the text to train a model. Nothing comes back — no
#   link, no reader, no attribution. Those are refused.
#
# Refusing a crawler in this file only works for crawlers that read it. The
# machine-readable rights reservation is at /.well-known/tdmrep.json, and
# Cloudflare's own AI-bot blocking is switched on at the network layer for the
# ones that ignore both.

# ── Search and AI answer engines: welcome ───────────────────────────────────
User-agent: Googlebot
User-agent: Bingbot
User-agent: DuckDuckBot
User-agent: Applebot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: PerplexityBot
User-agent: Perplexity-User
Allow: /

# ── Model-training and dataset crawlers: refused ────────────────────────────
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: CCBot
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: Meta-ExternalAgent
User-agent: FacebookBot
User-agent: Amazonbot
User-agent: Bytespider
User-agent: TikTokSpider
User-agent: cohere-ai
User-agent: Diffbot
User-agent: Omgilibot
User-agent: Timpibot
User-agent: Webzio-Extended
User-agent: ImagesiftBot
User-agent: img2dataset
User-agent: AI2Bot
User-agent: Scrapy
Disallow: /

# ── Everyone else ───────────────────────────────────────────────────────────
User-agent: *
Allow: /

Sitemap: https://nontechceo.co.uk/sitemap.xml