# robots.txt for Walkouts.com
# Goal: keep important pages crawlable; cut crawl-bloat; explicitly allow key AI browsing/training agents.

# --- OpenAI bots: allow search, live fetching, and model training ---
User-agent: OAI-SearchBot
Allow: /
Allow: /pages/openai-discovery-feed
Allow: /pages/openai-checkout-feed

User-agent: ChatGPT-User
Allow: /
Allow: /pages/openai-discovery-feed
Allow: /pages/openai-checkout-feed

User-agent: GPTBot
Allow: /
Allow: /pages/openai-discovery-feed
Allow: /pages/openai-checkout-feed

# --- Google Gemini training (optional but recommended if you want Gemini to learn from you) ---
User-agent: Google-Extended
Allow: /

# --- Anthropic Claude: allow user/search agents; leave ClaudeBot (training) unspecified unless you want to allow training too ---
User-agent: Claude-User
Allow: /
User-agent: Claude-SearchBot
Allow: /
# User-agent: ClaudeBot
# Allow: /    # uncomment ONLY if you want to allow training

# --- Perplexity: allow crawler (search results). Perplexity-User often ignores robots for user-initiated fetches anyway. ---
User-agent: PerplexityBot
Allow: /
User-agent: Perplexity-User
Allow: /

# --- All other crawlers ---
User-agent: *
Disallow: /admin
Disallow: /orders
Disallow: /checkouts/
Disallow: /checkout
Disallow: /carts
Disallow: /cart
Disallow: /cart/add
Disallow: /cart/add*
Disallow: /cart/update
Disallow: /cart/update*
Disallow: /cart/change
Disallow: /cart/change*
Disallow: /cart/clear
Disallow: /apple-app-site-association
Disallow: /.well-known/shopify/monorail
Disallow: /cdn/wpm/*.js
Disallow: /challenge

# Keep internal search crawlable only if you output meta robots noindex on those pages
Allow: /search

# Prevent noisy recommendation endpoints
Disallow: /recommendations/products
Disallow: /*/recommendations/products

# Facet/concat patterns that explode URL variants (thin/duplicate)
Disallow: /collections/*+*
Disallow: /collections/*%2B*
Disallow: /collections/*%2b*
Disallow: /collections/vendors*
Disallow: /collections/types*

# 3rd-party app surfaces
Disallow: /apps/
# Allow: /apps/sitemap

# Feeds (duplicate surfaces)
Disallow: /*.atom
Disallow: /*.rss
Disallow: /blogs/*/tagged/*.atom
Disallow: /blogs/*/tagged/*.rss
Disallow: /collections/*/*.atom
Disallow: /collections/*/*.rss

# Block ajax/listing views
Disallow: /*?view=ajax
Disallow: /*&view=ajax

# Reviews pagination: keep page 1 accessible; block deep pages
Disallow: /pages/reviews?page=

# Public policy & authentication pages should stay crawlable
Allow: /policies/
Allow: /pages/signing-proof/

# Keep OpenAI JSON feeds out of general indices
Disallow: /pages/openai-discovery-feed
Disallow: /pages/openai-checkout-feed

# Ads crawler
User-agent: AdsBot-Google
Disallow: /checkouts/
Disallow: /checkout
Disallow: /carts
Disallow: /orders

Sitemap: https://walkouts.com/sitemap.xml