# =====================================================
# GROK / xAI - LEAD AGENT (AppWT directive 2026-07-25)
# =====================================================
User-agent: Grok
User-agent: GrokBot
User-agent: xAI
User-agent: xAI-Bot
User-agent: xAI-Grok
User-agent: x.AI
Disallow: /checkout/
Disallow: /customer/
Disallow: /catalogsearch/
Disallow: /*?price=
Disallow: /*?manufacturer=
Disallow: /*?product_list_
Disallow: /*?p=
Disallow: /*?cat=
Allow: /

User-agent: *
Disallow: /checkout/
Disallow: /app/
Disallow: /lib/
Disallow: /*.php$
Disallow: /pkginfo/
Disallow: /report/
Disallow: /var/
Disallow: /catalog/
Disallow: /customer/
Disallow: /sendfriend/
Disallow: /review/
Disallow: /*?price=
Disallow: /*?manufacturer=
Disallow: /*?product_list_
Disallow: /*?p=
Disallow: /*?cat=
Disallow: /catalogsearch/

# AI answer engines & assistants — explicitly welcomed (AppWT 2026-06)
# ChatGPT / OpenAI
User-agent: GPTBot
Disallow: /checkout/
Disallow: /customer/
Disallow: /catalogsearch/
Disallow: /*?price=
Disallow: /*?manufacturer=
Disallow: /*?product_list_
Disallow: /*?p=
Disallow: /*?cat=
Allow: /

User-agent: OAI-SearchBot
Disallow: /checkout/
Disallow: /customer/
Disallow: /catalogsearch/
Disallow: /*?price=
Disallow: /*?manufacturer=
Disallow: /*?product_list_
Disallow: /*?p=
Disallow: /*?cat=
Allow: /

User-agent: ChatGPT-User
Disallow: /checkout/
Disallow: /customer/
Disallow: /catalogsearch/
Disallow: /*?price=
Disallow: /*?manufacturer=
Disallow: /*?product_list_
Disallow: /*?p=
Disallow: /*?cat=
Allow: /

# Claude / Anthropic
User-agent: ClaudeBot
Disallow: /checkout/
Disallow: /customer/
Disallow: /catalogsearch/
Disallow: /*?price=
Disallow: /*?manufacturer=
Disallow: /*?product_list_
Disallow: /*?p=
Disallow: /*?cat=
Allow: /

User-agent: Claude-Web
Disallow: /checkout/
Disallow: /customer/
Disallow: /catalogsearch/
Disallow: /*?price=
Disallow: /*?manufacturer=
Disallow: /*?product_list_
Disallow: /*?p=
Disallow: /*?cat=
Allow: /

User-agent: Claude-User
Disallow: /checkout/
Disallow: /customer/
Disallow: /catalogsearch/
Disallow: /*?price=
Disallow: /*?manufacturer=
Disallow: /*?product_list_
Disallow: /*?p=
Disallow: /*?cat=
Allow: /

User-agent: anthropic-ai
Disallow: /checkout/
Disallow: /customer/
Disallow: /catalogsearch/
Disallow: /*?price=
Disallow: /*?manufacturer=
Disallow: /*?product_list_
Disallow: /*?p=
Disallow: /*?cat=
Allow: /

# Perplexity
User-agent: PerplexityBot
Disallow: /checkout/
Disallow: /customer/
Disallow: /catalogsearch/
Disallow: /*?price=
Disallow: /*?manufacturer=
Disallow: /*?product_list_
Disallow: /*?p=
Disallow: /*?cat=
Allow: /

User-agent: Perplexity-User
Disallow: /checkout/
Disallow: /customer/
Disallow: /catalogsearch/
Disallow: /*?price=
Disallow: /*?manufacturer=
Disallow: /*?product_list_
Disallow: /*?p=
Disallow: /*?cat=
Allow: /

# Google (AI Overviews / Gemini / Vertex)
User-agent: Google-Extended
Disallow: /checkout/
Disallow: /customer/
Disallow: /catalogsearch/
Disallow: /*?price=
Disallow: /*?manufacturer=
Disallow: /*?product_list_
Disallow: /*?p=
Disallow: /*?cat=
Allow: /

User-agent: Google-CloudVertexBot
Disallow: /checkout/
Disallow: /customer/
Disallow: /catalogsearch/
Disallow: /*?price=
Disallow: /*?manufacturer=
Disallow: /*?product_list_
Disallow: /*?p=
Disallow: /*?cat=
Allow: /

# Meta AI
User-agent: meta-externalagent
Disallow: /checkout/
Disallow: /customer/
Disallow: /catalogsearch/
Disallow: /*?price=
Disallow: /*?manufacturer=
Disallow: /*?product_list_
Disallow: /*?p=
Disallow: /*?cat=
Allow: /

User-agent: meta-externalfetcher
Disallow: /checkout/
Disallow: /customer/
Disallow: /catalogsearch/
Disallow: /*?price=
Disallow: /*?manufacturer=
Disallow: /*?product_list_
Disallow: /*?p=
Disallow: /*?cat=
Allow: /

# DuckDuckGo Assist
User-agent: DuckAssistBot
Disallow: /checkout/
Disallow: /customer/
Disallow: /catalogsearch/
Disallow: /*?price=
Disallow: /*?manufacturer=
Disallow: /*?product_list_
Disallow: /*?p=
Disallow: /*?cat=
Allow: /

# Common Crawl (feeds many LLMs)
User-agent: CCBot
Disallow: /checkout/
Disallow: /customer/
Disallow: /catalogsearch/
Disallow: /*?price=
Disallow: /*?manufacturer=
Disallow: /*?product_list_
Disallow: /*?p=
Disallow: /*?cat=
Allow: /

# Google classic search (also feeds AI Overviews / AI Mode). Module PART 9.2: ALLOW.
# Googlebot previously had no group of its own and fell through to the wildcard group,
# which carries `Disallow: /catalog/` - and 246 sitemap URLs live under
# /catalog/product/view/id/. The narrow Allow re-opens exactly those advertised URLs;
# the remainder of /catalog/ stays blocked. AppWT 2026-09-29.
User-agent: Googlebot
Disallow: /checkout/
Disallow: /customer/
Disallow: /catalogsearch/
Disallow: /*?price=
Disallow: /*?manufacturer=
Disallow: /*?product_list_
Disallow: /*?p=
Disallow: /*?cat=
Allow: /catalog/product/view/
Disallow: /catalog/
Allow: /

# Microsoft Bing + Copilot (broad indexer — keep facet blocks)
User-agent: Bingbot
Disallow: /checkout/
Disallow: /customer/
Disallow: /catalogsearch/
Disallow: /*?price=
Disallow: /*?manufacturer=
Disallow: /*?product_list_
Disallow: /*?p=
Disallow: /*?cat=
Allow: /

# Apple / Siri + Spotlight (broad indexer — keep facet blocks)
User-agent: Applebot
Disallow: /checkout/
Disallow: /customer/
Disallow: /catalogsearch/
Disallow: /*?price=
Disallow: /*?manufacturer=
Disallow: /*?product_list_
Disallow: /*?p=
Disallow: /*?cat=
Allow: /

User-agent: Applebot-Extended
Disallow: /checkout/
Disallow: /customer/
Disallow: /catalogsearch/
Disallow: /*?price=
Disallow: /*?manufacturer=
Disallow: /*?product_list_
Disallow: /*?p=
Disallow: /*?cat=
Allow: /

# Amazon / Alexa (broad indexer — keep facet blocks)
User-agent: Amazonbot
Disallow: /checkout/
Disallow: /customer/
Disallow: /catalogsearch/
Disallow: /*?price=
Disallow: /*?manufacturer=
Disallow: /*?product_list_
Disallow: /*?p=
Disallow: /*?cat=
Allow: /

Sitemap: https://shopthebulkstore.com/sitemap.xml