# Search crawlers welcome. Auth-protected /api/ paths aren't indexed because crawlers
# can't authenticate; no need to Disallow them explicitly.
#
# NOTE ON AI/ML USE: crawling, search indexing, and citation in AI answers are permitted.
# Using any image, photo, or content from this site to train, fine-tune, or build AI/ML
# models, datasets, classifiers, fingerprints, or AI-detection / anti-AI systems is NOT
# permitted. Text & Data Mining rights are reserved per the W3C TDM Reservation Protocol
# (/.well-known/tdmrep.json) and the Terms of Service §28 (https://cmein.com/terms).
#
# Those two rules pull in opposite directions only if "AI" is treated as one thing. The
# split below is the operative version of the policy: agents that crawl to build training
# corpora are named and blocked; agents that crawl to answer a question and cite a source
# are named and allowed, because that citation is traffic. A blanket block costs the
# traffic without adding protection the named blocks and the TDM reservation don't give.
# See also /llms.txt.

User-agent: *
Allow: /

# ---- Training / dataset collection: not permitted ----
User-agent: GPTBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Applebot-Extended
Disallow: /

# Meta-ExternalAgent is the training crawler, and it stays blocked — but it is also the agent
# behind the Reels API's file_url fetch, so blocking it wholesale meant Facebook refused to
# collect our own scheduled clips: "Unable to fetch media from URL, got status code: 403
# Restricted by robots.txt". /blog-images/ is where those clips are published from and holds
# nothing but media we are deliberately handing to Meta to post. Everything else — the articles,
# the photos, the app — stays off limits.
User-agent: Meta-ExternalAgent
Allow: /blog-images/
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

# ---- Search and cited answers: permitted ----
# Listed explicitly so the intent survives any future tightening of the wildcard group.
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

Sitemap: https://cmein.com/sitemap.xml