# ==========================================
# 🌎 Sitemaps
# ==========================================
Sitemap: https://www.kickpleat.com/sitemap.xml

# ==========================================
# 🛍️ Shopify Crawl Rules (Optimized 2025)
# ==========================================

User-agent: *
# Explicitly allow sitemap
Allow: /sitemap.xml

# Block backend and sensitive paths
Disallow: /admin
Disallow: /cart
Disallow: /carts
Disallow: /orders
Disallow: /checkouts
Disallow: /checkout
Disallow: /account

# Private and system endpoints
Disallow: /.well-known/shopify/monorail
Disallow: /apple-app-site-association
Disallow: /sf_private_access_tokens

# Shopify Web Pixels Manager sandbox and internal pixel URLs
Disallow: /cdn/wpm/
Disallow: /wpm@
Disallow: /web-pixels-manager

# Collections & filters
Disallow: /collections/*+*
Disallow: /collections/*%2B*
Disallow: /collections/*%2b*
Disallow: /*/collections/*+*
Disallow: /*/collections/*%2B*
Disallow: /*/collections/*%2b*
Disallow: /collections/*sort_by*
Disallow: /*/collections/*sort_by*
Disallow: /*/collections/*filter*&*filter*

# Parameters (duplication & tracking)
Disallow: /*?*sort_by=
Disallow: /*?*filter
Disallow: /*?*page=
Disallow: /*?*utm_
Disallow: /*?*variant=
Disallow: /*?*pf_
Disallow: /*?*oseid=
Disallow: /*preview_theme_id*
Disallow: /*preview_script_id*

# Internal search
Disallow: /search

# Policy pages (block only if intentional)
Disallow: /policies/
Disallow: /*/policies/

# Alternate data formats (keep sitemap allowed above)
Disallow: /*.atom$
Disallow: /*.xml$
Disallow: /*.json$

# Optional: product "remote" variants if applicable
Disallow: /products/*-remote$
Disallow: /*/products/*-remote$
Disallow: /collections/*/products/*-remote$
Disallow: /*/collections/*/products/*-remote$


# ==========================================
# 🛒 Merchant Center / Googlebot Access
# ==========================================
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /


# ==========================================
# 🛒 Ads & Google crawlers
# ==========================================

User-agent: AdsBot-Google
Disallow: /checkouts
Disallow: /checkout
Disallow: /cart
Disallow: /carts
Disallow: /orders
Disallow: /*?*oseid=
Disallow: /*preview_theme_id*
Disallow: /*preview_script_id*
Disallow: /cdn/wpm/
Disallow: /sf_private_access_tokens

User-agent: AdsBot-Google-Mobile
Disallow: /checkouts
Disallow: /checkout
Disallow: /cart
Disallow: /carts
Disallow: /orders
Disallow: /*?*oseid=
Disallow: /*preview_theme_id*
Disallow: /*preview_script_id*
Disallow: /cdn/wpm/
Disallow: /sf_private_access_tokens


# ==========================================
# 🤖 AI Crawlers & LLM Whitelisting
# ==========================================
User-agent: GPTBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: ClaudeBot
Allow: /


# ==========================================
# 🧠 Research & SEO Tool Rate Tuning
# ==========================================
User-agent: AhrefsBot
Crawl-delay: 10

User-agent: AhrefsSiteAudit
Crawl-delay: 10

User-agent: MJ12bot
Crawl-delay: 10

User-agent: Pinterestbot
Crawl-delay: 1

User-agent: Nutch
Disallow: /