User-agent: * Allow: / Allow: /products/account Allow: /products/orders Allow: /products/checkout Allow: /*/products/account Allow: /*/products/orders Allow: /*/products/checkout Allow: /collections/account Allow: /collections/orders Allow: /collections/checkout Allow: /*/collections/account Allow: /*/collections/orders Allow: /*/collections/checkout Allow: /pages/checkout Allow: /*/pages/checkout Allow: /blogs/*account Allow: /blogs/*orders Allow: /blogs/*checkout Allow: /*/blogs/*account Allow: /*/blogs/*orders Allow: /*/blogs/*checkout Disallow: /admin Disallow: /cart/ Disallow: /*/cart/ Disallow: /checkout Disallow: /*/checkout Disallow: /checkouts/ Disallow: /*/checkouts/ Disallow: /orders Disallow: /*/orders Allow: /account/login Allow: /*/account/login Disallow: /account Disallow: /*/account Disallow: /76861079588 Disallow: /cdn/wpm/*.js Disallow: /services Disallow: /sf_* Disallow: /.well-known/shopify/monorail Disallow: /cart.js Disallow: /*/cart.js Disallow: /recommendations/products Disallow: /*/recommendations/products Disallow: /collections/*sort_by* Disallow: /*/collections/*sort_by* Disallow: /collections/*+* Disallow: /collections/*%2B* Disallow: /collections/*%2b* Disallow: /*/collections/*+* Disallow: /*/collections/*%2B* Disallow: /*/collections/*%2b* Disallow: /collections/*filter*&*filter* Disallow: /*/collections/*filter*&*filter* Disallow: /blogs/*+* Disallow: /blogs/*%2B* Disallow: /blogs/*%2b* Disallow: /*/blogs/*+* Disallow: /*/blogs/*%2B* Disallow: /*/blogs/*%2b* Disallow: /*?*ls=*&ls=* Disallow: /*?*ls%3*ls%3* Disallow: /*?*oseid=* Disallow: /*?*preview_theme_id=* Disallow: /*?*preview_script_id=* Disallow: /web-pixels@* Disallow: /wpm@* Sitemap: https://beyondbohemian.com/sitemap.xml User-agent: adsbot-google Allow: /products/ Allow: /*/products/ Allow: /collections/ Allow: /*/collections/ Allow: /pages/ Allow: /*/pages/ Allow: /blogs/ Allow: /*/blogs/ Allow: /pages/checkout Allow: /*/pages/checkout Allow: /blogs/*checkout Allow: /*/blogs/*checkout Disallow: /checkout Disallow: /*/checkout Disallow: /checkouts/ Disallow: /*/checkouts/ Disallow: /orders Disallow: /*/orders Disallow: /services Disallow: /sf_* Disallow: /76861079588 Disallow: /cdn/wpm/*.js # ============================================================================= # AI Search & Retrieval Bots - ALLOWED # These bots power real-time AI answers in ChatGPT, Claude, Perplexity, # Mistral, and DuckDuckGo. Allowing them means our products and guides can # appear when people ask AI tools for crystal recommendations. # ============================================================================= User-agent: ChatGPT-User Allow: / Disallow: /admin Disallow: /cart Disallow: /carts Disallow: /orders Disallow: /checkout Disallow: /checkouts/ Disallow: /account Disallow: /search Disallow: /collections/*sort_by* Disallow: /*/collections/*sort_by* Disallow: /collections/*+* Disallow: /collections/*%2B* Disallow: /collections/*%2b* Disallow: /*/collections/*+* Disallow: /*/collections/*%2B* Disallow: /*/collections/*%2b* Disallow: */collections/*filter*&*filter* Disallow: /blogs/*+* Disallow: /*?*oseid=* Disallow: /*preview_theme_id* Disallow: /*preview_script_id* Disallow: /web-pixels@* Disallow: /wpm@* User-agent: OAI-SearchBot Allow: / Disallow: /admin Disallow: /cart Disallow: /carts Disallow: /orders Disallow: /checkout Disallow: /checkouts/ Disallow: /account Disallow: /search Disallow: /collections/*sort_by* Disallow: /*/collections/*sort_by* Disallow: /collections/*+* Disallow: /collections/*%2B* Disallow: /collections/*%2b* Disallow: /*/collections/*+* Disallow: /*/collections/*%2B* Disallow: /*/collections/*%2b* Disallow: */collections/*filter*&*filter* Disallow: /blogs/*+* Disallow: /*?*oseid=* Disallow: /*preview_theme_id* Disallow: /*preview_script_id* Disallow: /web-pixels@* Disallow: /wpm@* User-agent: Claude-SearchBot Allow: / Disallow: /admin Disallow: /cart Disallow: /carts Disallow: /orders Disallow: /checkout Disallow: /checkouts/ Disallow: /account Disallow: /search Disallow: /collections/*sort_by* Disallow: /*/collections/*sort_by* Disallow: /collections/*+* Disallow: /collections/*%2B* Disallow: /collections/*%2b* Disallow: /*/collections/*+* Disallow: /*/collections/*%2B* Disallow: /*/collections/*%2b* Disallow: */collections/*filter*&*filter* Disallow: /blogs/*+* Disallow: /*?*oseid=* Disallow: /*preview_theme_id* Disallow: /*preview_script_id* Disallow: /web-pixels@* Disallow: /wpm@* User-agent: Claude-User Allow: / Disallow: /admin Disallow: /cart Disallow: /carts Disallow: /orders Disallow: /checkout Disallow: /checkouts/ Disallow: /account Disallow: /search Disallow: /collections/*sort_by* Disallow: /*/collections/*sort_by* Disallow: /collections/*+* Disallow: /collections/*%2B* Disallow: /collections/*%2b* Disallow: /*/collections/*+* Disallow: /*/collections/*%2B* Disallow: /*/collections/*%2b* Disallow: */collections/*filter*&*filter* Disallow: /blogs/*+* Disallow: /*?*oseid=* Disallow: /*preview_theme_id* Disallow: /*preview_script_id* Disallow: /web-pixels@* Disallow: /wpm@* User-agent: PerplexityBot Allow: / Disallow: /admin Disallow: /cart Disallow: /carts Disallow: /orders Disallow: /checkout Disallow: /checkouts/ Disallow: /account Disallow: /search Disallow: /collections/*sort_by* Disallow: /*/collections/*sort_by* Disallow: /collections/*+* Disallow: /collections/*%2B* Disallow: /collections/*%2b* Disallow: /*/collections/*+* Disallow: /*/collections/*%2B* Disallow: /*/collections/*%2b* Disallow: */collections/*filter*&*filter* Disallow: /blogs/*+* Disallow: /*?*oseid=* Disallow: /*preview_theme_id* Disallow: /*preview_script_id* Disallow: /web-pixels@* Disallow: /wpm@* User-agent: Perplexity-User Allow: / Disallow: /admin Disallow: /cart Disallow: /carts Disallow: /orders Disallow: /checkout Disallow: /checkouts/ Disallow: /account Disallow: /search Disallow: /collections/*sort_by* Disallow: /*/collections/*sort_by* Disallow: /collections/*+* Disallow: /collections/*%2B* Disallow: /collections/*%2b* Disallow: /*/collections/*+* Disallow: /*/collections/*%2B* Disallow: /*/collections/*%2b* Disallow: */collections/*filter*&*filter* Disallow: /blogs/*+* Disallow: /*?*oseid=* Disallow: /*preview_theme_id* Disallow: /*preview_script_id* Disallow: /web-pixels@* Disallow: /wpm@* User-agent: MistralAI-User Allow: / Disallow: /admin Disallow: /cart Disallow: /carts Disallow: /orders Disallow: /checkout Disallow: /checkouts/ Disallow: /account Disallow: /search Disallow: /collections/*sort_by* Disallow: /*/collections/*sort_by* Disallow: /collections/*+* Disallow: /collections/*%2B* Disallow: /collections/*%2b* Disallow: /*/collections/*+* Disallow: /*/collections/*%2B* Disallow: /*/collections/*%2b* Disallow: */collections/*filter*&*filter* Disallow: /blogs/*+* Disallow: /*?*oseid=* Disallow: /*preview_theme_id* Disallow: /*preview_script_id* Disallow: /web-pixels@* Disallow: /wpm@* User-agent: DuckAssistBot Allow: / Disallow: /admin Disallow: /cart Disallow: /carts Disallow: /orders Disallow: /checkout Disallow: /checkouts/ Disallow: /account Disallow: /search Disallow: /collections/*sort_by* Disallow: /*/collections/*sort_by* Disallow: /collections/*+* Disallow: /collections/*%2B* Disallow: /collections/*%2b* Disallow: /*/collections/*+* Disallow: /*/collections/*%2B* Disallow: /*/collections/*%2b* Disallow: */collections/*filter*&*filter* Disallow: /blogs/*+* Disallow: /*?*oseid=* Disallow: /*preview_theme_id* Disallow: /*preview_script_id* Disallow: /web-pixels@* Disallow: /wpm@* # Opened 2026-08-17 (Paul's decision, after audit). Google-Extended gates AI # Overviews and Gemini grounding: with it blocked the site could rank in Google # yet be absent from the AI answer above the results. GPTBot adds baseline model # knowledge of the brand (the citation path was already open via OAI-SearchBot # and ChatGPT-User below). Applebot-Extended covers Apple Intelligence and Siri. # The trade is deliberate: guide content becomes eligible for AI answers and may # also be used in training. Our moat is the sourcing relationships and named # origins, not the prose, and those do not transfer with copied text. # CCBot was opened the same day and reverted, see the note in the blocked list. User-agent: Google-Extended Allow: / Disallow: /admin Disallow: /cart Disallow: /carts Disallow: /orders Disallow: /checkout Disallow: /checkouts/ Disallow: /account Disallow: /search Disallow: /collections/*sort_by* Disallow: /*/collections/*sort_by* Disallow: /collections/*+* Disallow: /collections/*%2B* Disallow: /collections/*%2b* Disallow: /*/collections/*+* Disallow: /*/collections/*%2B* Disallow: /*/collections/*%2b* Disallow: */collections/*filter*&*filter* Disallow: /blogs/*+* Disallow: /*?*oseid=* Disallow: /*preview_theme_id* Disallow: /*preview_script_id* Disallow: /web-pixels@* Disallow: /wpm@* User-agent: GPTBot Allow: / Disallow: /admin Disallow: /cart Disallow: /carts Disallow: /orders Disallow: /checkout Disallow: /checkouts/ Disallow: /account Disallow: /search Disallow: /collections/*sort_by* Disallow: /*/collections/*sort_by* Disallow: /collections/*+* Disallow: /collections/*%2B* Disallow: /collections/*%2b* Disallow: /*/collections/*+* Disallow: /*/collections/*%2B* Disallow: /*/collections/*%2b* Disallow: */collections/*filter*&*filter* Disallow: /blogs/*+* Disallow: /*?*oseid=* Disallow: /*preview_theme_id* Disallow: /*preview_script_id* Disallow: /web-pixels@* Disallow: /wpm@* User-agent: Applebot-Extended Allow: / Disallow: /admin Disallow: /cart Disallow: /carts Disallow: /orders Disallow: /checkout Disallow: /checkouts/ Disallow: /account Disallow: /search Disallow: /collections/*sort_by* Disallow: /*/collections/*sort_by* Disallow: /collections/*+* Disallow: /collections/*%2B* Disallow: /collections/*%2b* Disallow: /*/collections/*+* Disallow: /*/collections/*%2B* Disallow: /*/collections/*%2b* Disallow: */collections/*filter*&*filter* Disallow: /blogs/*+* Disallow: /*?*oseid=* Disallow: /*preview_theme_id* Disallow: /*preview_script_id* Disallow: /web-pixels@* Disallow: /wpm@* # ============================================================================= # AI Training Bots - BLOCKED # Scrapers with no retrieval or citation benefit to us. The four agents that # do drive AI answers (Google-Extended, GPTBot, CCBot, Applebot-Extended) # were moved to the allowed list above on 2026-08-17. # ============================================================================= # Reverted to blocked 2026-08-17, same day it was opened. Common Crawl is a publicly # downloadable dataset: it does not cite us, does not send traffic, and appears in no # answer anywhere. Every engine that DOES cite (Google, OpenAI, Anthropic, Perplexity, # Apple) runs its own crawler and all of those are allowed above, so CCBot adds no # reach we do not already have. What it does add is the cheapest possible route for a # competitor to bulk-acquire ~495 guide pages. Checked before reverting: beyondbohemian.com # returned 0 captured URLs across CC-MAIN-2024-33, 2025-08 and 2025-30, so nothing had # leaked and the revert costs nothing. User-agent: CCBot Disallow: / User-agent: anthropic-ai Disallow: / User-agent: Bytespider Disallow: / User-agent: Meta-ExternalAgent Disallow: / User-agent: AI2Bot Disallow: / User-agent: Timpibot Disallow: / User-agent: Diffbot Disallow: / User-agent: FacebookBot Disallow: / User-agent: omgili Disallow: / User-agent: Amazonbot Disallow: / User-agent: cohere-ai Disallow: / # ============================================================================= # LLMs.txt Reference # ============================================================================= # LLMs-Txt: https://beyondbohemian.com/pages/llms-txt