===== amazon =====
# CONDENSED RECORD, not a full copy. Read in full in Chrome on 12 Nov 2026, about 17:50 UK (page text of https://www.amazon.co.uk/robots.txt).
# The wildcard group is summarised; every other group in the file is listed in file order.
User-agent: *
Disallow: /gp/cart
# (one real rule shown as an example; the rest are summarised in the next line)
# About 100 path rules (e.g. Disallow: /gp/cart, Disallow: /wishlist/, Allow: /wishlist/universal). None is "/", "/*" or "*".

User-agent: EtaoSpider
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: GoogleAgent-Mariner
Disallow: /

User-agent: GoogleAgent-Shopping
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Scrapy
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: Devin
Disallow: /

User-agent: omgili
Disallow: /

User-agent: AI2Bot
Disallow: /

User-agent: Gemini-Deep-Research
Disallow: /

User-agent: PanguBot
Disallow: /

User-agent: MistralAI-User
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: Sidetrade indexer bot
Disallow: /

User-agent: Google-NotebookLM
Disallow: /

User-agent: DuckAssistBot
Disallow: /

User-agent: Copilot
Disallow: /

User-agent: Ai2Bot-Dolma
Disallow: /

User-agent: Google-CloudVertexBot
Disallow: /

User-agent: meta-externalfetcher
Disallow: /

User-agent: iaskspider
Disallow: /

User-agent: wpbot
Disallow: /

User-agent: aiHitBot
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: FriendlyCrawler
Disallow: /

User-agent: img2dataset
Disallow: /

User-agent: VelenPublicWebCrawler
Disallow: /

User-agent: YouBot
Disallow: /

User-agent: Brightbot
Disallow: /

User-agent: Claude-User
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Claude-SearchBot
Disallow: /

User-agent: Perplexity-User
Disallow: /

User-agent: ISSCyberRiskCrawler
Disallow: /

User-agent: CopilotNative
Disallow: /

User-agent: CopilotSapphire
Disallow: /

User-agent: imgproxy
Disallow: /

User-agent: SemrushBot-SWA
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: OAI-SearchBot
Disallow: /

User-agent: Timpibot
Disallow: /

User-agent: Thinkbot
Disallow: /

User-agent: AddSearchBot
Disallow: /

User-agent: AI2Bot-DeepResearchEval
Disallow: /

User-agent: Andibot
Disallow: /

User-agent: Anomura
Disallow: /

User-agent: atlassian-bot
Disallow: /

User-agent: BuddyBot
Disallow: /

User-agent: Channel3Bot
Disallow: /

User-agent: ChatGLM-Spider
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: Cloudflare-AutoRAG
Disallow: /

User-agent: CloudflareBrowserRenderingCrawler
Disallow: /

User-agent: cohere-training-data-crawler
Disallow: /

User-agent: Cotoyogi
Disallow: /

User-agent: Crawl4AI
Disallow: /

User-agent: Datenbank Crawler
Disallow: /

User-agent: DeepSeekBot
Disallow: /

User-agent: Echobot Bot
Disallow: /

User-agent: EchoboxBot
Disallow: /

User-agent: factset_spyderbot
Disallow: /

User-agent: Google-Firebase
Disallow: /

User-agent: GoogleOther
Disallow: /

User-agent: Grok-DeepSearch
Disallow: /

User-agent: GrokBot
Disallow: /

User-agent: iAskBot
Disallow: /

User-agent: IbouBot
Disallow: /

User-agent: ICC-Crawler
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: imageSpider
Disallow: /

User-agent: Kangaroo Bot
Disallow: /

User-agent: KlaviyoAIBot
Disallow: /

User-agent: KunatoCrawler
Disallow: /

User-agent: laion-huggingface-processor
Disallow: /

User-agent: Linguee Bot
Disallow: /

User-agent: LinkupBot
Disallow: /

User-agent: Manus-User
Disallow: /

User-agent: meta-webindexer
Disallow: /

User-agent: netEstate Imprint Crawler
Disallow: /

User-agent: Panscient
Disallow: /

User-agent: panscient.com
Disallow: /

User-agent: PhindBot
Disallow: /

User-agent: Poggio-Citations
Disallow: /

User-agent: QualifiedBot
Disallow: /

User-agent: SBIntuitionsBot
Disallow: /

User-agent: ShapBot
Disallow: /

User-agent: TavilyBot
Disallow: /

User-agent: TerraCotta
Disallow: /

User-agent: TwinAgent
Disallow: /

User-agent: webzio-extended
Disallow: /

User-agent: WRTNBot
Disallow: /

User-agent: xAI-Grok
Disallow: /

User-agent: YaK
Disallow: /

User-agent: ZanistaBot
Disallow: /


===== ebay =====
# CONDENSED RECORD, not a full copy. Rules as returned by a web fetch tool on 12 Nov 2026. Checked in Chrome the same day, about 18:10 UK: 20,258 characters, 928 lines, 8 groups, 20 User-agent lines, same 9 blocked and 13 named.
# File opens with a comment block headed "# Robots & Agent Policy" ("# Checkouts are strictly for human users.").
User-agent: *
Disallow: /act/
# (one real rule shown as an example; the rest are summarised in the next line)
# 205 rules as counted by the fetch tool (197 Disallow, 8 Allow), all path or parameter patterns. None is "/", "/*" or "*".
# Other groups: Googlebot, bingbot, AdsBot-Google, AdIdxBot, Twitterbot (not tracked).

# LLM / AI bot directives
User-agent: Bytespider
User-agent: CCBot
User-agent: ChatGLM-Spider
User-agent: ClaudeBot
User-agent: PerplexityBot
User-agent: anthropic-ai
User-agent: AmazonBot
User-agent: meta-externalagent
User-agent: GPTBot
User-agent: Applebot-Extended
Disallow: /
Allow: /help
Allow: /ebaylive/seller
Allow: /sellercentre

User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: Claude-SearchBot
User-agent: Claude-User
Disallow: /act/
# (one real rule shown as an example; the rest are summarised in the next line)
# 198 rules as counted by the fetch tool (193 Disallow, 5 Allow), path or parameter patterns close to the wildcard group's. None is "/", "/*" or "*".

===== temu =====
# CONDENSED RECORD, not a full copy. Rules as returned by a web fetch tool on 12 Nov 2026. Checked in Chrome the same day: 8,239 characters, 275 lines, 5 groups, 33 User-agent lines, same 1 blocked and 9 named.
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Bingbot
User-agent: Google-Extended
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: PerplexityBot
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
DisAllow: /shopping_cart.html
# (one real rule shown as an example; the rest are summarised in the next line)
# 103 rules as counted by the fetch tool, all specific pages or parameter patterns (e.g. DisAllow: /*oauth_code=, DisAllow: /bgt_order_checkout.html). None is "/", "/*" or "*".

# Second group: AdsBot-Google, AdIdxBot, Applebot, Pinterestbot, Twitterbot, FacebookBot. Third: facebookexternalhit. Not tracked.

User-agent: *
Allow: /.well-known/

User-agent: Slurp
User-agent: Y!J-BRU/VSIDX
User-agent: Y!J-BRW
User-agent: Y!J-ASR
User-agent: Y!J-MMP/dscv
User-agent: Y!J-WSC
User-agent: Y!J-DLC
User-agent: YJ-BRU/VSIDX
User-agent: YJ-BRW
User-agent: YJ-ASR
User-agent: YJ-MMP/dscv
User-agent: YJ-WSC
User-agent: YJ-DLC
User-agent: Bytespider
DisAllow: /*

===== apple =====
# Rules as returned by a web fetch tool on 12 Nov 2026 (comments and Sitemap lines not kept). Checked in Chrome the same day: 1,018 characters, the same six User-agent lines.
User-agent: *
Disallow: /*shop/browse/overlay/*
Disallow: /*shop/iphone/payments/overlay/*
Disallow: /cn/*/aow/*
Disallow: /tmall*
Allow: /ac/globalnav/2.0/*/images/ac-globalnav/globalnav/search/*

User-agent: Baiduspider
Disallow: /*

User-agent: HaoSouSpider
Disallow: /*

User-agent: Sogou web spider
Disallow: /*

User-agent: Sogou inst spider
Disallow: /*

User-agent: Sogou spider2
Disallow: /*

===== tesco =====
# Rules as returned by a web fetch tool on 12 Nov 2026 (Sitemap lines not kept). Checked in Chrome the same day: 1,383 characters, 2 User-agent lines (* and meta-externalagent), 24 Disallow lines.
# Differs from the 10 Nov 2026 record, which counted 1 User-agent line and 26 Disallow rules in Chrome.
User-agent: *
Disallow: /shop/*-*/promotions/*?*=*
Disallow: /shop/*-*/clothing/promotions/*?*=*
Disallow: *currentModal*
Disallow: *active-tab*
Disallow: *include-children*
Disallow: *new&new*
Disallow: /account/login/*-*?
Disallow: /shop/*-*/login?*
Disallow: */*-*/manage-cookie-preferences*?
Disallow: /account/register/*-*?
Disallow: /account/register/*-*?newReg
Disallow: /shop/*-*/*&offset=*
Disallow: */login/*-*?from=*
Disallow: */*-*/login?from=*
Disallow: /help/*?dvi=*
Disallow: /shop/*-*/products*?icid
Disallow: /shop/*-*/slots*?icid
Disallow: /shop/*-*/clothing/products*?icid
Disallow: *?queueittoken*
Disallow: */userinteraction*
Disallow: */nrbeacon/*
Disallow: /account/password/*-*/reset-password
Allow: /account/password/*-*/forgotten-password$
Disallow: /account/password/*-*/forgotten-password

User-agent: meta-externalagent
Disallow: /account/

===== argos =====
# Rules as returned by a web fetch tool on 12 Nov 2026. Checked in Chrome the same day: 491 characters, the same four User-agent lines, 14 Disallow lines.
# /robots.txt file for https://www.argos.co.uk
User-agent: *
Disallow: /wishlist
Disallow: /list/*

User-agent: Googlebot
Disallow: /wishlist
Disallow: /list/*
Disallow: /checkout/
Disallow: /confirmation/

User-agent: Googlebot-image
Disallow: /wishlist
Disallow: /list/*
Disallow: /checkout/
Disallow: /confirmation/
Allow: /.well-known/

User-agent: Bingbot
Disallow: /wishlist
Disallow: /list/*
Disallow: /checkout/
Disallow: /confirmation/

===== asda =====
# Full copy as read in Chrome on 12 Nov 2026, about 18:15 UK.
# The web fetch tool returned a different file on 10 and 12 Nov 2026 (header dated 28th Oct 2024, 11 rules including /clothing* and /groceries*). Neither version names a crawler or has a whole-site rule.
# Last Updated Date : 21st Aug 2025
# This is an autogenerated file from asda-sitemap-generator
Sitemap: https://www.asda.com/sitemap-index.xml
User-agent: *
Disallow: /emails/*
Disallow: /account*
Disallow: /registration*
Disallow: */search/*
Disallow: /terms*
Disallow: /privacy*
Disallow: /*onlineopinionV5
Disallow: /asda-estore*
Disallow: /reset-password*

===== lidl =====
# Full copy as read in Chrome on 12 Nov 2026. Same rules as the 10 Nov 2026 record.
User-agent: *
Disallow: /q/search?id=*
Disallow: /cc.js*
Disallow: /1*
Disallow: /2*
Disallow: /3*
Disallow: /4*
Disallow: /5*
Disallow: /6*
Disallow: /7*
Disallow: /8*
Disallow: /9*
Disallow: /cdn/assets/cwv/

Sitemap: https://www.lidl.co.uk/static/sitemap.xml

===== vinted =====
# CONDENSED RECORD, not a full copy. Rules as returned by a web fetch tool on 12 Nov 2026. Checked in Chrome the same day: 3,360 characters, 98 lines, 3 groups, same 5 blocked and 5 named, same two Content-Signal lines.
# File opens with comment blocks on AI use (search indexing allowed; training, fine-tuning and dataset creation not) and on automated agents (not permitted to create accounts, add to cart or buy).
User-agent: *
Disallow: /checkout
Disallow: /util
Disallow: /apipie/
Disallow: /admin_alert/new
Disallow: /member/
Disallow: /inbox
Allow: /
# 17 rules as counted by the fetch tool (6 Disallow, 11 Allow); the other 10 Allow rules are /member/ profile paths.
# Content-Signal: ai-train=no, search=yes, ai-input=yes
# Content-Usage: train-ai=n

User-agent: grapeshot
Disallow: /items

User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Google-Extended
User-agent: Amazonbot
User-agent: CCBot
Disallow: /
# Content-Signal: ai-train=no, search=yes, ai-input=no
# Content-Usage: train-ai=n

===== shein =====
# Full copy as read in Chrome on 12 Nov 2026. The group has no rules.
User-agent: *

Sitemap: https://www.shein.co.uk/sitemap-index.xml