# Answer engines are welcome. Training crawlers are not.
# Policy in prose: https://ph-therapeutics.com/terms
# Machine-readable reservation: https://ph-therapeutics.com/.well-known/tdmrep.json
# Site summary for agents: https://ph-therapeutics.com/llms.txt

User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Disallow: /admin
Disallow: /api/
Disallow: /mcp
Disallow: /oauth/
Disallow: /search
Disallow: /offline

# Retrieval and citation: allowed, in full.
# ChatGPT search index
User-agent: OAI-SearchBot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Disallow: /admin
Disallow: /api/
Disallow: /mcp
Disallow: /oauth/
Disallow: /search
Disallow: /offline

# ChatGPT fetching a page a user asked about
User-agent: ChatGPT-User
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Disallow: /admin
Disallow: /api/
Disallow: /mcp
Disallow: /oauth/
Disallow: /search
Disallow: /offline

# Claude search index
User-agent: Claude-SearchBot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Disallow: /admin
Disallow: /api/
Disallow: /mcp
Disallow: /oauth/
Disallow: /search
Disallow: /offline

# Claude fetching a page a user asked about
User-agent: Claude-User
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Disallow: /admin
Disallow: /api/
Disallow: /mcp
Disallow: /oauth/
Disallow: /search
Disallow: /offline

# Perplexity search index
User-agent: PerplexityBot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Disallow: /admin
Disallow: /api/
Disallow: /mcp
Disallow: /oauth/
Disallow: /search
Disallow: /offline

# Perplexity fetching a page a user asked about
User-agent: Perplexity-User
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Disallow: /admin
Disallow: /api/
Disallow: /mcp
Disallow: /oauth/
Disallow: /search
Disallow: /offline

# DuckDuckGo assist answers
User-agent: DuckAssistBot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Disallow: /admin
Disallow: /api/
Disallow: /mcp
Disallow: /oauth/
Disallow: /search
Disallow: /offline

# Siri and Spotlight. The training token is Applebot-Extended, refused below
User-agent: Applebot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Disallow: /admin
Disallow: /api/
Disallow: /mcp
Disallow: /oauth/
Disallow: /search
Disallow: /offline

# Bing, and the Copilot answers built on it
User-agent: Bingbot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Disallow: /admin
Disallow: /api/
Disallow: /mcp
Disallow: /oauth/
Disallow: /search
Disallow: /offline

# Google Search, including the AI Overviews built on its index
User-agent: Googlebot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Disallow: /admin
Disallow: /api/
Disallow: /mcp
Disallow: /oauth/
Disallow: /search
Disallow: /offline

# Model training: refused. See the reservation linked above.
# OpenAI model training
User-agent: GPTBot
Content-Signal: search=no, ai-input=no, ai-train=no
Disallow: /

# Anthropic model training
User-agent: ClaudeBot
Content-Signal: search=no, ai-input=no, ai-train=no
Disallow: /

# Anthropic, earlier token
User-agent: anthropic-ai
Content-Signal: search=no, ai-input=no, ai-train=no
Disallow: /

# Anthropic, earlier token
User-agent: Claude-Web
Content-Signal: search=no, ai-input=no, ai-train=no
Disallow: /

# Gemini and Vertex training. Google Search is unaffected
User-agent: Google-Extended
Content-Signal: search=no, ai-input=no, ai-train=no
Disallow: /

# Apple Intelligence training
User-agent: Applebot-Extended
Content-Signal: search=no, ai-input=no, ai-train=no
Disallow: /

# Meta model training
User-agent: Meta-ExternalAgent
Content-Signal: search=no, ai-input=no, ai-train=no
Disallow: /

# Meta, lowercase token seen in the wild
User-agent: meta-externalagent
Content-Signal: search=no, ai-input=no, ai-train=no
Disallow: /

# Meta language model training
User-agent: FacebookBot
Content-Signal: search=no, ai-input=no, ai-train=no
Disallow: /

# Common Crawl — the upstream corpus for most open training sets
User-agent: CCBot
Content-Signal: search=no, ai-input=no, ai-train=no
Disallow: /

# ByteDance model training
User-agent: Bytespider
Content-Signal: search=no, ai-input=no, ai-train=no
Disallow: /

# Huawei model training
User-agent: PanguBot
Content-Signal: search=no, ai-input=no, ai-train=no
Disallow: /

# Allen Institute training corpora
User-agent: AI2Bot
Content-Signal: search=no, ai-input=no, ai-train=no
Disallow: /

# Cohere model training
User-agent: cohere-ai
Content-Signal: search=no, ai-input=no, ai-train=no
Disallow: /

# Cohere training corpus
User-agent: cohere-training-data-crawler
Content-Signal: search=no, ai-input=no, ai-train=no
Disallow: /

# Commercial knowledge-graph and training data resale
User-agent: Diffbot
Content-Signal: search=no, ai-input=no, ai-train=no
Disallow: /

# Webz.io, resells crawled text as training data
User-agent: Omgilibot
Content-Signal: search=no, ai-input=no, ai-train=no
Disallow: /

# Webz.io training corpus
User-agent: Webzio-Extended
Content-Signal: search=no, ai-input=no, ai-train=no
Disallow: /

# Image training corpus
User-agent: ImagesiftBot
Content-Signal: search=no, ai-input=no, ai-train=no
Disallow: /

# Timpi training corpus
User-agent: Timpibot
Content-Signal: search=no, ai-input=no, ai-train=no
Disallow: /

# Kangaroo training corpus
User-agent: Kangaroo Bot
Content-Signal: search=no, ai-input=no, ai-train=no
Disallow: /

# Default library user-agent; overwhelmingly bulk extraction
User-agent: Scrapy
Content-Signal: search=no, ai-input=no, ai-train=no
Disallow: /

Sitemap: https://ph-therapeutics.com/sitemap.xml