# Good Nature Vibe — robots.txt
# https://goodnaturevibe.com
# Source of truth: filesystem file (pub/robots.txt). Magento admin generator is bypassed.
# Last updated: 2026-10-26 (MCP server + server card added to AI discovery pointers)
# Earlier: 2026-10-09 (facet/filter URLs ?size[]= / price= excluded for all partial groups — AGT-P1)
# Previous: 2026-09-05 (owner-approved full public AI access; transactional/private paths remain excluded)
#
# =============================================================================
#  CONTENT SIGNALS POLICY  (canonical preamble — DO NOT remove)
# =============================================================================
# As a condition of accessing this website, you agree to abide by the
# following content signals:
#  (a) If a content-signal = yes, you may collect content for the
#      corresponding use.
#  (b) If a content-signal = no, you may not collect content for the
#      corresponding use.
#  (c) If the website operator does not include a content signal for a
#      corresponding use, the website operator neither grants nor
#      restricts permission via content signal with respect to the
#      corresponding use.
#
# The content signals and their meanings are:
#  search:   building a search index and providing search results (e.g.,
#            returning hyperlinks and short excerpts from your website's
#            contents). Search does not include providing AI-generated
#            search summaries.
#  ai-input: inputting content into one or more AI models (e.g., retrieval
#            augmented generation, grounding, or other real-time taking of
#            content for generative AI search answers).
#  ai-train: training or fine-tuning AI models.
#
# ANY RESTRICTIONS EXPRESSED VIA CONTENT SIGNALS ARE EXPRESS RESERVATIONS
# OF RIGHTS UNDER ARTICLE 4 OF THE EUROPEAN UNION DIRECTIVE 2019/790 ON
# COPYRIGHT AND RELATED RIGHTS IN THE DIGITAL SINGLE MARKET.
#
# Spec: https://contentsignals.org/
# Background: https://blog.cloudflare.com/content-signals-policy/
# IETF draft: https://datatracker.ietf.org/doc/draft-romm-aipref-contentsignals/
#
# =============================================================================
#  POLICY SUMMARY (plain English)
# =============================================================================
#  Search indexing             : ALLOWED  (Googlebot, Bingbot, Applebot, …)
#  AI on-demand / citation     : ALLOWED  (Claude-User, ChatGPT-User, Perplexity-User,
#                                          Gemini-Deep-Research, MistralAI-User,
#                                          DuckAssistBot, Kagi-fetcher, …)
#  AI search-index             : ALLOWED  (Claude-SearchBot, OAI-SearchBot,
#                                          PerplexityBot, meta-webindexer, …)
#  AI model training           : ALLOWED  (GPTBot, ClaudeBot, Google-Extended,
#                                          Applebot-Extended, Meta-ExternalAgent,
#                                          Bytespider, CCBot, DeepSeekBot, …)
#
# =============================================================================
#  AI DISCOVERY POINTERS  (informative — bots that don't follow Link: headers
#  can find the structured resources from this list)
# =============================================================================
#  Site overview (markdown)     : /llms.txt
#  Agent-skills index           : /.well-known/agent-skills/index.json
#  Agentic Commerce Protocol    : /.well-known/acp.json
#  MCP server (agent commerce) : https://mcp.goodnaturevibe.com/mcp
#  MCP server card             : /.well-known/mcp/server-card.json
#  Product catalog (NDJSON)     : /llms-catalog.ljson
#                                 /llms-catalog-{nl,en,fr,es,de}.ljson
#  Content feed (NDJSON)        : /llms-content.ljson
#                                 /llms-content-{nl,en,fr,es,de}.ljson
#
#  Same pointers are also served as Link: headers on the homepage:
#    Link: </llms.txt>;                              rel="describedby"
#    Link: </.well-known/agent-skills/index.json>;   rel="service-meta"
#    Link: </.well-known/acp.json>;                  rel="service-desc"
#    Link: </.well-known/mcp/server-card.json>;      rel="service-desc"
#
# =============================================================================
#  MAINTENANCE NOTES — DO NOT REMOVE Content-Signal directives.
#  - Google Search Console + Bing Webmaster Tools may flag Content-Signal as
#    an unknown directive. That is EXPECTED behaviour, not an error.
#  - Per RFC 9309, per-User-agent groups do NOT inherit from User-agent: *.
#    Every group below carries its own full rule set on purpose.
#  - UA matching is case-insensitive (RFC 9309). Canonical capitalization used
#    for readability only.
#  - Bot taxonomy moves fast; refresh against canonical sources every quarter:
#    https://knownagents.com/agents and https://github.com/ai-robots-txt/ai.robots.txt
# =============================================================================


# -----------------------------------------------------------------------------
#  DEFAULT GROUP — search engines + any unspecified UA.
#  Bots without an explicit group below also inherit these signals.
# -----------------------------------------------------------------------------
User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes

# Block tracking parameters (duplicate URLs hurt crawl budget + indexing)
Disallow: /*?srsltid=
Disallow: /*&srsltid=
Disallow: /*?utm_
Disallow: /*&utm_
Disallow: /*?fbclid=
Disallow: /*&fbclid=
Disallow: /*?gclid=
Disallow: /*&gclid=

# Block filter/sort parameter duplicates
Disallow: /*?price=
Disallow: /*?color=
Disallow: /*?size=
Disallow: /*?material=
Disallow: /*?manufacturer=
Disallow: /*?rating=
Disallow: /*?stock=
Disallow: /*?in-stock=
Disallow: /*?cat=
Disallow: /*?p=
Disallow: /*?product_list_order=
Disallow: /*?product_list_dir=
Disallow: /*?product_list_mode=
Disallow: /*?product_list_limit=
# Facet filters in array/any-position form (AGT-P1, 2026-10-09)
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Internal, checkout, customer, and API paths — not useful for any crawler
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql

# Magento storeview-switcher URLs — 302 redirects, waste crawl budget
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch


# =============================================================================
#  SEARCH ENGINES — explicit Content-Signal groups.
#  These search crawlers are permitted for indexing and AI-supported results.
#  The owner also permits model training, so all groups carry ai-train=yes.
#  Vendors with a
#  separate AI-training token (Google-Extended, Applebot-Extended) are allowed
#  in the AI-TRAINING section further down.
# =============================================================================

User-agent: Googlebot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */paypal/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?srsltid=
Disallow: /*&srsltid=
Disallow: /*?utm_
Disallow: /*&utm_
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

User-agent: Googlebot-Image
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: Googlebot-News
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: Bingbot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */paypal/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?srsltid=
Disallow: /*&srsltid=
Disallow: /*?utm_
Disallow: /*&utm_
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

User-agent: Applebot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */rest/
Disallow: */graphql
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

User-agent: DuckDuckBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: YandexBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: Bravebot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /


# =============================================================================
#  AI ON-DEMAND / CITATION-TIME ASSISTANTS
#  These bots fetch a URL when a user asks an AI assistant to read it.
#  They may quote / cite us in answers. Explicit per-bot groups so the
#  Content-Signal is unambiguous per RFC 9309.
# =============================================================================

# Anthropic — on-demand fetch when a user asks Claude.
User-agent: Claude-User
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Anthropic — search-index crawler for Claude's web search.
User-agent: Claude-SearchBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# OpenAI — on-demand fetch when a user asks ChatGPT to look something up.
# Note: per OpenAI's December 2025 update, ChatGPT-User may bypass robots.txt
# for user-initiated URLs (analogous to meta-externalfetcher). The Content-Signal
# remains as an intent declaration.
User-agent: ChatGPT-User
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# OpenAI — search-index crawler for ChatGPT search.
User-agent: OAI-SearchBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# OpenAI — multi-step browser AI agent (Operator / ChatGPT Agent).
User-agent: ChatGPT-Agent
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */rest/
Disallow: */graphql
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Perplexity — search-index crawler.
User-agent: PerplexityBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */rest/
Disallow: */graphql
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Perplexity — on-demand fetch when a user asks Perplexity.
User-agent: Perplexity-User
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */rest/
Disallow: */graphql
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Google — Gemini Deep Research assistant.
User-agent: Gemini-Deep-Research
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */rest/
Disallow: */graphql
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Google — NotebookLM URL fetcher (research assistant).
User-agent: Google-NotebookLM
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */rest/
Disallow: */graphql
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Google — Firebase AI app support (citation-time fetcher).
User-agent: Google-Firebase
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */rest/
Disallow: */graphql
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Mistral AI — Le Chat user-prompted fetcher.
User-agent: MistralAI-User
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */rest/
Disallow: */graphql
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Meta — AI assistant link fetcher (NOT training; that is Meta-ExternalAgent).
# Note: meta-externalfetcher may bypass robots.txt for user-initiated URLs.
User-agent: meta-externalfetcher
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */rest/
Disallow: */graphql
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Meta — AI search index crawler (separate from Meta-ExternalAgent training).
User-agent: meta-webindexer
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
# 2026-09-05 — preserve clean AI-search crawling while stopping the measured
# store-switch redirect loop and combinatorial facet/query crawl.
Disallow: */stores/
Disallow: /*?*
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */rest/
Disallow: */graphql
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# DuckDuckGo — DuckAssist on-demand AI answer fetcher (NOT training).
User-agent: DuckAssistBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */rest/
Disallow: */graphql
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Kagi — Kagi Assistant fetcher.
User-agent: Kagi-fetcher
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */rest/
Disallow: */graphql
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Amazon — Alexa user-initiated content fetcher (NOT Amazonbot training).
User-agent: Amzn-User
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */rest/
Disallow: */graphql
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# You.com — Youbot is dual-purpose; on-demand variant allowed for citations.
# (You.com's YouBot is explicitly allowed below.)
# No distinct UA published; YouBot covers both — see training section.

# Andi — generative AI search summaries.
User-agent: Andibot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

# Direqt — Anomura AI search indexing.
User-agent: Anomura
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

# Microsoft Azure AI Search.
User-agent: AzureAI-SearchBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

# Cloudflare AutoRAG — CF-managed AI search support.
User-agent: Cloudflare-AutoRAG
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /


# =============================================================================
#  AI MODEL TRAINING crawlers — ALLOW public content + Content-Signal ai-train=yes.
#  Owner-approved access; only shared transactional/private paths remain excluded.
# =============================================================================

# OpenAI — model training crawler.
User-agent: GPTBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Anthropic — model training crawler (current name, since 2026-02 rename).
User-agent: ClaudeBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Anthropic — legacy training crawler name (kept for backwards compat).
User-agent: anthropic-ai
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Anthropic — legacy crawler name.
User-agent: claude-web
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Google — AI training opt-out token (separate from Googlebot).
User-agent: Google-Extended
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Google — internal R&D / data scraping crawlers.
User-agent: GoogleOther
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

User-agent: GoogleOther-Image
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

User-agent: GoogleOther-Video
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Google — Vertex AI custom model building (on-request training).
User-agent: Google-CloudVertexBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Apple — AI training opt-out token (separate from Applebot).
User-agent: Applebot-Extended
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Meta — model training and product improvement.
User-agent: Meta-ExternalAgent
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

User-agent: meta-externalagent
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Meta — legacy training crawler name.
User-agent: FacebookBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Amazon — Alexa service training.
User-agent: Amazonbot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Amazon — Bedrock custom application training.
User-agent: bedrockbot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Amazon — Kendra natural-language search data collection.
User-agent: amazon-kendra
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Cohere — model training (legacy UA name).
User-agent: cohere-ai
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Cohere — current training-data acquisition crawler.
User-agent: cohere-training-data-crawler
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# DeepSeek — model training.
User-agent: DeepSeekBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# ByteDance — LLM training (used by Doubao / TikTok AI).
User-agent: Bytespider
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# ByteDance — TikTok-specific training crawler.
User-agent: TikTokSpider
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Zhipu AI (China) — ChatGLM model training.
User-agent: ChatGLM-Spider
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Huawei — Pangu multimodal model training.
User-agent: PanguBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Huawei — Petal Search assistant + AI search services; public content allowed.
User-agent: PetalBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Allen Institute for AI — research / training crawlers.
User-agent: AI2Bot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

User-agent: Ai2Bot-Dolma
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# LAION — open ML dataset (does NOT respect robots.txt per its own docs).
User-agent: LAIONDownloader
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Firecrawl — LLM training scraper.
User-agent: FirecrawlAgent
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Bright Data — customer-support LLM training crawler.
User-agent: Brightbot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

User-agent: Brightbot 1.0
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Panscient — business-data / ML analysis scraper.
User-agent: Panscient
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Klaviyo — AI personalization training crawler.
User-agent: KlaviyoAIBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Semrush — ContentShake AI tool support crawlers.
User-agent: SemrushBot-OCOB
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

User-agent: SemrushBot-SWA
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# You.com — search and LLM data retrieval (dual-purpose).
User-agent: YouBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# NICT (Japan) — research crawler used in training corpora.
User-agent: ICC-Crawler
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Decentralized / niche LLM training networks.
User-agent: Kangaroo Bot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

User-agent: Timpibot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

User-agent: VelenPublicWebCrawler
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Linguee — translation AI training (does NOT respect robots.txt per its docs).
User-agent: Linguee Bot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# ISS — cyber-risk ML model training (does NOT respect robots.txt per its docs).
User-agent: ISSCyberRiskCrawler
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# img2dataset — image dataset collection for LLMs.
User-agent: img2dataset
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Generic scraper framework default UA.
User-agent: Scrapy
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=


# =============================================================================
#  COMMON CRAWL + DATASET CRAWLERS — ALLOW public content.
#  Common Crawl is used by many datasets; owner explicitly permits ai-train=yes
#  while transactional/private Magento paths remain excluded.
# =============================================================================

User-agent: CCBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

User-agent: Diffbot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

User-agent: Omgilibot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

User-agent: omgili
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

User-agent: ImagesiftBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=


# -----------------------------------------------------------------------------
#  QUARTERLY BOT REFRESH — 2026-07-09 (canonical: ai.robots.txt + knownagents.com)
#  Policy: maximise customer capture — allow AI search / citation / agentic-commerce
#  bots, including training/dataset crawlers; protect only transactional/private URLs.
# -----------------------------------------------------------------------------

#  --- ALLOW: AI search / citation / agentic-commerce (discovery = customers) ---
# Exa — AI search engine index (drives citation traffic).
User-agent: ExaBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Tavily — AI search API used by assistants for live answers.
User-agent: TavilyBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Phind — developer AI search engine.
User-agent: PhindBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Linkup — AI search / grounding provider.
User-agent: LinkupBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Liner — AI answer engine with source citations.
User-agent: LinerBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# iAsk — AI answer/search engine.
User-agent: iAskBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Poggio — citation crawler for AI answers.
User-agent: Poggio-Citations
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Moonshot Kimi — on-demand fetch when a user asks Kimi.
User-agent: Kimi-User
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Amazon — AI search index (product discovery surface).
User-agent: Amzn-SearchBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Amazon — 'Buy for Me' agentic shopping assistant (customer capture).
User-agent: AmazonBuyForMe
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# OpenAI Operator — browser AI agent shopping/acting for a user.
User-agent: Operator
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Amazon Nova Act — agentic browsing/shopping on a user's behalf.
User-agent: NovaAct
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Manus — on-demand fetch when a user asks the Manus agent.
User-agent: Manus-User
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Google — agentic assistant fetch on a user's behalf.
User-agent: Google-Agent
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Google Project Mariner — agentic browsing for a user.
User-agent: GoogleAgent-Mariner
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Google — URL-context fetch for Gemini grounding.
User-agent: GoogleAgent-URLContext
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

#  --- ALLOW: training / scraping / dataset crawlers on public content ---
# QuillBot — paraphrasing/training data crawler.
User-agent: QuillBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Awario — social-listening scraper.
User-agent: Awario
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Apify — generic scraping platform.
User-agent: ApifyBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Apify — website content crawler (scraping platform).
User-agent: ApifyWebsiteContentCrawler
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Crawl4AI — RAG/training scraper framework.
User-agent: Crawl4AI
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Cotoyogi — AI training crawler.
User-agent: Cotoyogi
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Alibaba Tongyi — model training crawler.
User-agent: TongyiBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Baidu Ernie/Yiyan — model training crawler.
User-agent: YiyanBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Allen AI — deep-research evaluation crawler.
User-agent: AI2Bot-DeepResearchEval
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# SB Intuitions — model training crawler.
User-agent: SBIntuitionsBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Generic AI scraper bot.
User-agent: MyCentralAIScraperBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Webz.io — dataset/training crawler (extended token).
User-agent: Webzio-Extended
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# Kunato — AI training crawler.
User-agent: KunatoCrawler
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */customer/
Disallow: */checkout/
Disallow: */cart/
Disallow: */wishlist/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */paypal/
Disallow: */cvb/
Disallow: */rest/
Disallow: */graphql
Disallow: /*stores/store/redirect
Disallow: /*stores/store/switch
Disallow: /*?*size%5B
Disallow: /*?*size[
Disallow: /*?*size=
Disallow: /*?*price=

# =============================================================================
#  Sitemaps (file-level directives — apply to all UA groups)
# =============================================================================
Sitemap: https://goodnaturevibe.com/sitemap_index.xml
Sitemap: https://goodnaturevibe.com/media/sitemap/sitemap_nl.xml
Sitemap: https://goodnaturevibe.com/media/sitemap/sitemap_en.xml
Sitemap: https://goodnaturevibe.com/media/sitemap/sitemap_fr.xml
Sitemap: https://goodnaturevibe.com/media/sitemap/sitemap_es.xml
Sitemap: https://goodnaturevibe.com/media/sitemap/sitemap_de.xml
Sitemap: https://goodnaturevibe.com/media/sitemap/hreflang_home.xml