# STOTTS. — we welcome search engines and AI answer engines.
# Curated content map for LLMs: /llms.txt

# One group, many names. Under RFC 9309 a crawler obeys only the MOST SPECIFIC
# group that names it, and ignores every other group in the file — so giving
# each AI crawler its own "Allow: /" block silently exempted all of them from
# the Disallow rules below. Stacked User-agent lines share one rule body, so
# the permissions and the exclusions can never drift apart again.
User-agent: *
# OpenAI — ChatGPT search, training, and live browsing on a person's behalf
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
# Perplexity
User-agent: PerplexityBot
User-agent: Perplexity-User
# Anthropic. ClaudeBot alone does not cover the live-fetch agents: a request
# made because a person asked Claude a question comes from Claude-User, and
# the search index uses Claude-SearchBot.
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: Claude-Web
User-agent: anthropic-ai
# Google Gemini / Vertex AI. Google-CloudVertexBot is the grounding fetch and
# is separate from Google-Extended.
User-agent: Google-Extended
User-agent: Google-CloudVertexBot
# Apple. Applebot-Extended governs training only — without Applebot itself the
# pages are not indexed for Siri and Spotlight.
User-agent: Applebot
User-agent: Applebot-Extended
# Common Crawl (the dataset behind many models) and Bing (powers ChatGPT and
# Copilot search)
User-agent: CCBot
User-agent: Bingbot
# Meta AI
User-agent: meta-externalagent
User-agent: FacebookBot
# Amazon — Alexa and Rufus
User-agent: Amazonbot
# Mistral, You.com, and the crawlers behind several smaller answer engines
User-agent: MistralAI-User
User-agent: YouBot
User-agent: Diffbot
User-agent: Timpibot
# ByteDance (Doubao). Named explicitly because hosting platforms block it by
# default and we do not want to.
User-agent: Bytespider
Allow: /
# Internal engine tooling. These are not built for production, but the rule is
# here so anything that ever appears under these paths stays out of every crawl.
# /eyewear/visual-life/results is deliberately NOT listed: it carries an
# X-Robots-Tag noindex, and a crawler that is disallowed from a page can never
# read that header.
Disallow: /eyewear/visual-life/rules
Disallow: /eyewear/visual-life/language-card
Disallow: /eyewear/visual-life/worksheet
Disallow: /eyewear/visual-life/dispensed

Sitemap: https://www.stottsopticians.co.uk/sitemap.xml

# A curated map of this site for language models, in the llmstxt.org format.
# Not a standard robots directive; placed here because it is where crawlers
# already look and costs nothing if ignored.
# LLM-Content: https://www.stottsopticians.co.uk/llms.txt
# LLM-Content: https://www.stottsopticians.co.uk/llms-full.txt