# Estate Shutter FL - robots.txt
# Updated: 2026-09-18
#
# WHY THIS FILE LOOKS THE WAY IT DOES
# A robots.txt "group" is one or more consecutive User-agent lines followed by
# that group's rules. A crawler obeys exactly ONE group: the most specific one
# that names it. It does NOT fall back to, or inherit from, the `*` group.
#
# Before 2026-09-18 each AI crawler had its own group containing a single
# `Allow: /` line, so none of them inherited any of the Disallow rules below —
# every one of them was free to crawl /admin, /api/ and the ?lng= duplicate
# URLs. The fix is to list all of those agents as consecutive User-agent lines
# above ONE shared rule block, so the rules are written once and every named
# agent actually gets them. They stay fully allowed on all real content.
#
# If you add a Disallow to the `*` group, add it to the AI group too.

User-agent: *
# Block admin and development areas (not WordPress — React SPA)
Disallow: /admin
Disallow: /admin/
Disallow: /dashboard
Disallow: /dev/
Disallow: /test/
Disallow: /staging/
Disallow: /private/

# Block API endpoints from indexing
Disallow: /api/
Disallow: /_api/

# Block low-value and duplicate content
Disallow: /search/
Disallow: /filter/
Disallow: /404/
Disallow: /error/

# Block tracking and duplicate URL parameters
Disallow: /*?utm_*
Disallow: /*?fbclid=*
Disallow: /*?gclid=*
Disallow: /*?dclid=*
Disallow: /*?msclkid=*
Disallow: /*?ref=*
Disallow: /*?source=*
Disallow: /*?campaign=*
Disallow: /*?tracking_id=*
Disallow: /*?sort=*
Disallow: /*?order=*
Disallow: /*?page=*
Disallow: /*?filter=*
# i18n is client-side only: ?lng= serves the same page (duplicate URL)
Disallow: /*?lng=
Disallow: /*&lng=

# ─────────────────────────────────────────────────────────────────────────────
# AI crawlers — explicitly welcome on all content, and now subject to the same
# Disallow rules as everyone else. One group, many agents (consecutive
# User-agent lines share the rule block that follows them).
# ─────────────────────────────────────────────────────────────────────────────
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: CCBot
User-agent: Claude-Web
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: PerplexityBot
User-agent: Applebot-Extended
User-agent: FacebookBot
User-agent: meta-externalagent
User-agent: cohere-ai
User-agent: YouBot
User-agent: Bytespider
User-agent: Diffbot
Allow: /
Disallow: /admin
Disallow: /admin/
Disallow: /dashboard
Disallow: /dev/
Disallow: /test/
Disallow: /staging/
Disallow: /private/
Disallow: /api/
Disallow: /_api/
Disallow: /search/
Disallow: /filter/
Disallow: /404/
Disallow: /error/
Disallow: /*?utm_*
Disallow: /*?fbclid=*
Disallow: /*?gclid=*
Disallow: /*?dclid=*
Disallow: /*?msclkid=*
Disallow: /*?ref=*
Disallow: /*?source=*
Disallow: /*?campaign=*
Disallow: /*?tracking_id=*
Disallow: /*?sort=*
Disallow: /*?order=*
Disallow: /*?page=*
Disallow: /*?filter=*
Disallow: /*?lng=
Disallow: /*&lng=

# Sitemap
Sitemap: https://www.estateshutterfl.com/sitemap-index.xml