# ═══════════════════════════════════════════════════════════════════════
# Westcove Development — crawler policy
# ═══════════════════════════════════════════════════════════════════════
# STRATEGY: stay fully discoverable in search AND in AI answers, but keep
# the site out of AI TRAINING corpora, and off the commercial scrapers
# that mine competitor data.
#
#   ALLOWED  — search engines (we need to be found) and AI *answer* bots
#              that cite sources and send real referral traffic.
#   BLOCKED  — crawlers that harvest content to TRAIN models, plus
#              commercial SEO / data-mining scrapers.
#
# LIMITS — read before trusting this file:
#   robots.txt is a REQUEST (RFC 9309), not a wall. Well-behaved crawlers
#   honour it; studies through 2026 find a large share of AI crawlers
#   ignore it outright. Real enforcement has to happen at the CDN/edge
#   (Cloudflare AI-bot blocking / rate limiting). Treat this file as
#   (a) protection against the compliant majority, and (b) the documented
#   statement of intent that any later takedown claim rests on.
#
# NOTE — dev/study pages are deliberately NOT listed here. Enumerating
# them would advertise their paths to exactly the scrapers we care about;
# they carry <meta name="robots" content="noindex"> instead.
#
# On domain migration the Sitemap line below must change — tracked in
# docs/context/DOMAIN-MIGRATION-TODO.md.
# ═══════════════════════════════════════════════════════════════════════


# ─── Search engines — explicitly welcome ──────────────────────────────
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Slurp
Allow: /

User-agent: Applebot
Allow: /


# ─── AI answer engines — allowed: they cite and send visitors ─────────
# These fetch a page to answer a live user's question, which is discovery,
# not training. Flip any of these to `Disallow: /` if you would rather not
# surface in ChatGPT / Perplexity answers at all.
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /


# ─── AI TRAINING crawlers — blocked ───────────────────────────────────
# Google-Extended governs Gemini TRAINING only. Blocking it does not
# affect Google Search indexing or ranking — Googlebot above still runs.
User-agent: GPTBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: AI2Bot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: Omgili
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: Timpibot
Disallow: /

User-agent: YouBot
Disallow: /


# ─── Commercial SEO / data scrapers — blocked ─────────────────────────
# These mine site structure and content for competitor-intelligence
# products. No upside for a private developer.
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: dotbot
Disallow: /

User-agent: rogerbot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: PetalBot
Disallow: /


# ─── Everyone else — open ─────────────────────────────────────────────
# Default-open on purpose: this is a marketing site, and unknown-but-
# legitimate search / link-preview agents should still reach it. Abuse
# is handled at the edge, not here.
User-agent: *
Allow: /

Sitemap: https://www.westcovedev.com/sitemap.xml