# ------------------------------------------------------------------
# ROBOTS.TXT - Dog Friendly Destinations
# ------------------------------------------------------------------

User-Agent: *
# Content Signals: Allow search/grounding, opt-out of training
Content-signal: search=yes,ai-train=no

# WordPress Defaults
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /cdn-cgi/

# Crawl hygiene - block junk URL patterns (added 6 May 2026)
# Geo-search parameter URLs (75 crawled-not-indexed)
Disallow: /*?place=
Disallow: /*?lat=
Disallow: /*?lng=

# UTM parameter duplicates (12 crawled-not-indexed)
Disallow: /*?utm_source=

# RSS feed endpoints (32 crawled-not-indexed)
Disallow: /*/feed/

# WooCommerce cart actions (4 crawled-not-indexed)
Disallow: /*add-to-cart*

# Bing tracking redirects
Disallow: /ck/

# Crawl hygiene - batch 2 (added 26 Jun 2026)
# /1000 junk URLs (175 crawled-not-indexed)
Disallow: /*/1000
Disallow: /*//1000

# Google Ads click tracking
Disallow: /*?gclid=

# Deep pagination on archives (56 crawled-not-indexed)
Disallow: /features/*/page/
Disallow: /stayfilters/*/page/
Disallow: /eatfilters/*/page/
Disallow: /type/*/page/
Disallow: /people/*/page/
Disallow: /people/

# Other referrer params
Disallow: /*?ref=
Disallow: /*?offset=

# Aggressive Bots to Block (Keep these blocked)
User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

# ------------------------------------------------------------------
# OPTIONAL: AI Bots (Currently ALLOWED for visibility)
# If you prefer to BLOCK them again, uncomment the lines below.
# ------------------------------------------------------------------
# User-agent: GPTBot
# Disallow: /
#
# User-agent: Google-Extended
# Disallow: /
#
# User-agent: ClaudeBot
# Disallow: /
#
# User-agent: Applebot-Extended
# Disallow: /

Sitemap: https://dogfriendlydestinations.com/sitemaps.xml