# robots.txt for teachersrealestate.com
# Updated 2026-09-03 — restored the intentional May 9, 2026 policy (the
# live copy had been overwritten by the generic retiree-tools placeholder),
# then split the AI-bot section in two: crawlers that hoover content into
# model TRAINING data stay blocked, crawlers that fetch a page live to
# answer a specific user's question (ChatGPT Search, Claude, Perplexity)
# are now allowed, so the site can still be found and cited when someone
# asks an AI assistant something like "best real estate company for
# teachers in Fresno" — see Alex/Claude chat, 2026-09-03, for the reasoning.
# Note: robots.txt is a voluntary signal, not an enforcement mechanism —
# reputable AI companies say they honor it, but it doesn't stop anything
# already scraped historically, and it isn't a guarantee of being cited,
# just removal of an active block.
#
# DEPLOY AT: https://teachersrealestate.com/robots.txt  (NOT /schools/robots.txt)
# This file MUST be served from the site root for crawlers to find it.

# ---- Standard search engines: full access (SEO + feeds AI Overviews/Copilot) ----

User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Slurp
Allow: /

User-agent: Applebot
Allow: /


# ---- AI answer engines: allowed, so the site can be found/cited live ----
# These fetch a specific page to answer a specific user's question — this
# is the "found in AI search" lever, separate from model training below.

# OpenAI: OAI-SearchBot powers ChatGPT Search results; ChatGPT-User is the
# on-demand fetch when a user has ChatGPT open/read a specific link.
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Anthropic: Claude-SearchBot builds Claude's search index; Claude-User is
# the on-demand fetch when answering a live user question.
User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

# Perplexity: PerplexityBot is explicitly for surfacing/citing sites in
# Perplexity answers (Perplexity's own docs say it is not used for model
# training), Perplexity-User is the on-demand fetch equivalent.
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /


# ---- AI training crawlers: blocked ----
# These build future model training datasets, not live answers — blocking
# them doesn't cost you AI-search visibility, it just keeps your content
# out of the training pile.

# OpenAI / ChatGPT model training
User-agent: GPTBot
Disallow: /

# Anthropic / Claude model training
User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

# Common Crawl (used by many AI training pipelines)
User-agent: CCBot
Disallow: /

# Google's AI training opt-out (Gemini/Bard model training). Googlebot
# above still indexes for Search and AI Overviews — this is separate.
User-agent: Google-Extended
Disallow: /

# Apple's AI training opt-out. Applebot above still indexes for
# Spotlight/Siri Search — this is separate, for Apple Intelligence training.
User-agent: Applebot-Extended
Disallow: /

# Meta / Facebook AI training
User-agent: FacebookBot
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: Meta-ExternalFetcher
Disallow: /

# ByteDance / TikTok
User-agent: Bytespider
Disallow: /

# Cohere
User-agent: cohere-ai
Disallow: /

User-agent: cohere-training-data-crawler
Disallow: /

# Amazon Alexa / Bedrock training
User-agent: Amazonbot
Disallow: /

# Misc AI training scrapers seen in the wild
User-agent: ImagesiftBot
Disallow: /

User-agent: omgili
Disallow: /

User-agent: omgilibot
Disallow: /

User-agent: PetalBot
Disallow: /


# ---- SEO/data-scraping tools (not AI answer engines — no search-visibility cost) ----

User-agent: DataForSeoBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: YouBot
Disallow: /


# ---- Default for unnamed crawlers: allow normal indexing, no archive caching ----

User-agent: *
Allow: /
Disallow: /admin/
Disallow: /private/
Disallow: /*?*ref=
Crawl-delay: 5


# ---- Sitemap location ----

Sitemap: https://teachersrealestate.com/sitemap.xml