scrape_date,brand,domain,bot,bot_class,blocked,named_in_file,collection,record,source
2026-11-12,Amazon,www.amazon.co.uk,GPTBot,training,yes,yes,chrome,condensed record,https://www.amazon.co.uk/robots.txt
2026-11-12,Amazon,www.amazon.co.uk,OAI-SearchBot,retrieval,yes,yes,chrome,condensed record,https://www.amazon.co.uk/robots.txt
2026-11-12,Amazon,www.amazon.co.uk,ChatGPT-User,retrieval,yes,yes,chrome,condensed record,https://www.amazon.co.uk/robots.txt
2026-11-12,Amazon,www.amazon.co.uk,ClaudeBot,training,yes,yes,chrome,condensed record,https://www.amazon.co.uk/robots.txt
2026-11-12,Amazon,www.amazon.co.uk,Claude-SearchBot,retrieval,yes,yes,chrome,condensed record,https://www.amazon.co.uk/robots.txt
2026-11-12,Amazon,www.amazon.co.uk,Claude-User,retrieval,yes,yes,chrome,condensed record,https://www.amazon.co.uk/robots.txt
2026-11-12,Amazon,www.amazon.co.uk,anthropic-ai,training,no,no,chrome,condensed record,https://www.amazon.co.uk/robots.txt
2026-11-12,Amazon,www.amazon.co.uk,Google-Extended,training,yes,yes,chrome,condensed record,https://www.amazon.co.uk/robots.txt
2026-11-12,Amazon,www.amazon.co.uk,PerplexityBot,retrieval,yes,yes,chrome,condensed record,https://www.amazon.co.uk/robots.txt
2026-11-12,Amazon,www.amazon.co.uk,Perplexity-User,retrieval,yes,yes,chrome,condensed record,https://www.amazon.co.uk/robots.txt
2026-11-12,Amazon,www.amazon.co.uk,CCBot,training,yes,yes,chrome,condensed record,https://www.amazon.co.uk/robots.txt
2026-11-12,Amazon,www.amazon.co.uk,Applebot-Extended,training,no,no,chrome,condensed record,https://www.amazon.co.uk/robots.txt
2026-11-12,Amazon,www.amazon.co.uk,Bytespider,training,yes,yes,chrome,condensed record,https://www.amazon.co.uk/robots.txt
2026-11-12,Amazon,www.amazon.co.uk,Meta-ExternalAgent,training,yes,yes,chrome,condensed record,https://www.amazon.co.uk/robots.txt
2026-11-12,Amazon,www.amazon.co.uk,Amazonbot,training,no,no,chrome,condensed record,https://www.amazon.co.uk/robots.txt
2026-11-12,eBay,www.ebay.co.uk,GPTBot,training,yes,yes,"fetch tool, then chrome",condensed record,https://www.ebay.co.uk/robots.txt
2026-11-12,eBay,www.ebay.co.uk,OAI-SearchBot,retrieval,no,yes,"fetch tool, then chrome",condensed record,https://www.ebay.co.uk/robots.txt
2026-11-12,eBay,www.ebay.co.uk,ChatGPT-User,retrieval,no,yes,"fetch tool, then chrome",condensed record,https://www.ebay.co.uk/robots.txt
2026-11-12,eBay,www.ebay.co.uk,ClaudeBot,training,yes,yes,"fetch tool, then chrome",condensed record,https://www.ebay.co.uk/robots.txt
2026-11-12,eBay,www.ebay.co.uk,Claude-SearchBot,retrieval,no,yes,"fetch tool, then chrome",condensed record,https://www.ebay.co.uk/robots.txt
2026-11-12,eBay,www.ebay.co.uk,Claude-User,retrieval,no,yes,"fetch tool, then chrome",condensed record,https://www.ebay.co.uk/robots.txt
2026-11-12,eBay,www.ebay.co.uk,anthropic-ai,training,yes,yes,"fetch tool, then chrome",condensed record,https://www.ebay.co.uk/robots.txt
2026-11-12,eBay,www.ebay.co.uk,Google-Extended,training,no,no,"fetch tool, then chrome",condensed record,https://www.ebay.co.uk/robots.txt
2026-11-12,eBay,www.ebay.co.uk,PerplexityBot,retrieval,yes,yes,"fetch tool, then chrome",condensed record,https://www.ebay.co.uk/robots.txt
2026-11-12,eBay,www.ebay.co.uk,Perplexity-User,retrieval,no,no,"fetch tool, then chrome",condensed record,https://www.ebay.co.uk/robots.txt
2026-11-12,eBay,www.ebay.co.uk,CCBot,training,yes,yes,"fetch tool, then chrome",condensed record,https://www.ebay.co.uk/robots.txt
2026-11-12,eBay,www.ebay.co.uk,Applebot-Extended,training,yes,yes,"fetch tool, then chrome",condensed record,https://www.ebay.co.uk/robots.txt
2026-11-12,eBay,www.ebay.co.uk,Bytespider,training,yes,yes,"fetch tool, then chrome",condensed record,https://www.ebay.co.uk/robots.txt
2026-11-12,eBay,www.ebay.co.uk,Meta-ExternalAgent,training,yes,yes,"fetch tool, then chrome",condensed record,https://www.ebay.co.uk/robots.txt
2026-11-12,eBay,www.ebay.co.uk,Amazonbot,training,yes,yes,"fetch tool, then chrome",condensed record,https://www.ebay.co.uk/robots.txt
2026-11-12,Temu,www.temu.com,GPTBot,training,no,yes,"fetch tool, then chrome",condensed record,https://www.temu.com/robots.txt
2026-11-12,Temu,www.temu.com,OAI-SearchBot,retrieval,no,yes,"fetch tool, then chrome",condensed record,https://www.temu.com/robots.txt
2026-11-12,Temu,www.temu.com,ChatGPT-User,retrieval,no,yes,"fetch tool, then chrome",condensed record,https://www.temu.com/robots.txt
2026-11-12,Temu,www.temu.com,ClaudeBot,training,no,yes,"fetch tool, then chrome",condensed record,https://www.temu.com/robots.txt
2026-11-12,Temu,www.temu.com,Claude-SearchBot,retrieval,no,yes,"fetch tool, then chrome",condensed record,https://www.temu.com/robots.txt
2026-11-12,Temu,www.temu.com,Claude-User,retrieval,no,yes,"fetch tool, then chrome",condensed record,https://www.temu.com/robots.txt
2026-11-12,Temu,www.temu.com,anthropic-ai,training,no,no,"fetch tool, then chrome",condensed record,https://www.temu.com/robots.txt
2026-11-12,Temu,www.temu.com,Google-Extended,training,no,yes,"fetch tool, then chrome",condensed record,https://www.temu.com/robots.txt
2026-11-12,Temu,www.temu.com,PerplexityBot,retrieval,no,yes,"fetch tool, then chrome",condensed record,https://www.temu.com/robots.txt
2026-11-12,Temu,www.temu.com,Perplexity-User,retrieval,no,no,"fetch tool, then chrome",condensed record,https://www.temu.com/robots.txt
2026-11-12,Temu,www.temu.com,CCBot,training,no,no,"fetch tool, then chrome",condensed record,https://www.temu.com/robots.txt
2026-11-12,Temu,www.temu.com,Applebot-Extended,training,no,no,"fetch tool, then chrome",condensed record,https://www.temu.com/robots.txt
2026-11-12,Temu,www.temu.com,Bytespider,training,yes,yes,"fetch tool, then chrome",condensed record,https://www.temu.com/robots.txt
2026-11-12,Temu,www.temu.com,Meta-ExternalAgent,training,no,no,"fetch tool, then chrome",condensed record,https://www.temu.com/robots.txt
2026-11-12,Temu,www.temu.com,Amazonbot,training,no,no,"fetch tool, then chrome",condensed record,https://www.temu.com/robots.txt
2026-11-12,Apple,www.apple.com,GPTBot,training,no,no,"fetch tool, then chrome",rules as returned,https://www.apple.com/robots.txt
2026-11-12,Apple,www.apple.com,OAI-SearchBot,retrieval,no,no,"fetch tool, then chrome",rules as returned,https://www.apple.com/robots.txt
2026-11-12,Apple,www.apple.com,ChatGPT-User,retrieval,no,no,"fetch tool, then chrome",rules as returned,https://www.apple.com/robots.txt
2026-11-12,Apple,www.apple.com,ClaudeBot,training,no,no,"fetch tool, then chrome",rules as returned,https://www.apple.com/robots.txt
2026-11-12,Apple,www.apple.com,Claude-SearchBot,retrieval,no,no,"fetch tool, then chrome",rules as returned,https://www.apple.com/robots.txt
2026-11-12,Apple,www.apple.com,Claude-User,retrieval,no,no,"fetch tool, then chrome",rules as returned,https://www.apple.com/robots.txt
2026-11-12,Apple,www.apple.com,anthropic-ai,training,no,no,"fetch tool, then chrome",rules as returned,https://www.apple.com/robots.txt
2026-11-12,Apple,www.apple.com,Google-Extended,training,no,no,"fetch tool, then chrome",rules as returned,https://www.apple.com/robots.txt
2026-11-12,Apple,www.apple.com,PerplexityBot,retrieval,no,no,"fetch tool, then chrome",rules as returned,https://www.apple.com/robots.txt
2026-11-12,Apple,www.apple.com,Perplexity-User,retrieval,no,no,"fetch tool, then chrome",rules as returned,https://www.apple.com/robots.txt
2026-11-12,Apple,www.apple.com,CCBot,training,no,no,"fetch tool, then chrome",rules as returned,https://www.apple.com/robots.txt
2026-11-12,Apple,www.apple.com,Applebot-Extended,training,no,no,"fetch tool, then chrome",rules as returned,https://www.apple.com/robots.txt
2026-11-12,Apple,www.apple.com,Bytespider,training,no,no,"fetch tool, then chrome",rules as returned,https://www.apple.com/robots.txt
2026-11-12,Apple,www.apple.com,Meta-ExternalAgent,training,no,no,"fetch tool, then chrome",rules as returned,https://www.apple.com/robots.txt
2026-11-12,Apple,www.apple.com,Amazonbot,training,no,no,"fetch tool, then chrome",rules as returned,https://www.apple.com/robots.txt
2026-11-12,Tesco,www.tesco.com,GPTBot,training,no,no,"fetch tool, then chrome",rules as returned,https://www.tesco.com/robots.txt
2026-11-12,Tesco,www.tesco.com,OAI-SearchBot,retrieval,no,no,"fetch tool, then chrome",rules as returned,https://www.tesco.com/robots.txt
2026-11-12,Tesco,www.tesco.com,ChatGPT-User,retrieval,no,no,"fetch tool, then chrome",rules as returned,https://www.tesco.com/robots.txt
2026-11-12,Tesco,www.tesco.com,ClaudeBot,training,no,no,"fetch tool, then chrome",rules as returned,https://www.tesco.com/robots.txt
2026-11-12,Tesco,www.tesco.com,Claude-SearchBot,retrieval,no,no,"fetch tool, then chrome",rules as returned,https://www.tesco.com/robots.txt
2026-11-12,Tesco,www.tesco.com,Claude-User,retrieval,no,no,"fetch tool, then chrome",rules as returned,https://www.tesco.com/robots.txt
2026-11-12,Tesco,www.tesco.com,anthropic-ai,training,no,no,"fetch tool, then chrome",rules as returned,https://www.tesco.com/robots.txt
2026-11-12,Tesco,www.tesco.com,Google-Extended,training,no,no,"fetch tool, then chrome",rules as returned,https://www.tesco.com/robots.txt
2026-11-12,Tesco,www.tesco.com,PerplexityBot,retrieval,no,no,"fetch tool, then chrome",rules as returned,https://www.tesco.com/robots.txt
2026-11-12,Tesco,www.tesco.com,Perplexity-User,retrieval,no,no,"fetch tool, then chrome",rules as returned,https://www.tesco.com/robots.txt
2026-11-12,Tesco,www.tesco.com,CCBot,training,no,no,"fetch tool, then chrome",rules as returned,https://www.tesco.com/robots.txt
2026-11-12,Tesco,www.tesco.com,Applebot-Extended,training,no,no,"fetch tool, then chrome",rules as returned,https://www.tesco.com/robots.txt
2026-11-12,Tesco,www.tesco.com,Bytespider,training,no,no,"fetch tool, then chrome",rules as returned,https://www.tesco.com/robots.txt
2026-11-12,Tesco,www.tesco.com,Meta-ExternalAgent,training,no,yes,"fetch tool, then chrome",rules as returned,https://www.tesco.com/robots.txt
2026-11-12,Tesco,www.tesco.com,Amazonbot,training,no,no,"fetch tool, then chrome",rules as returned,https://www.tesco.com/robots.txt
2026-11-12,Argos,www.argos.co.uk,GPTBot,training,no,no,"fetch tool, then chrome",rules as returned,https://www.argos.co.uk/robots.txt
2026-11-12,Argos,www.argos.co.uk,OAI-SearchBot,retrieval,no,no,"fetch tool, then chrome",rules as returned,https://www.argos.co.uk/robots.txt
2026-11-12,Argos,www.argos.co.uk,ChatGPT-User,retrieval,no,no,"fetch tool, then chrome",rules as returned,https://www.argos.co.uk/robots.txt
2026-11-12,Argos,www.argos.co.uk,ClaudeBot,training,no,no,"fetch tool, then chrome",rules as returned,https://www.argos.co.uk/robots.txt
2026-11-12,Argos,www.argos.co.uk,Claude-SearchBot,retrieval,no,no,"fetch tool, then chrome",rules as returned,https://www.argos.co.uk/robots.txt
2026-11-12,Argos,www.argos.co.uk,Claude-User,retrieval,no,no,"fetch tool, then chrome",rules as returned,https://www.argos.co.uk/robots.txt
2026-11-12,Argos,www.argos.co.uk,anthropic-ai,training,no,no,"fetch tool, then chrome",rules as returned,https://www.argos.co.uk/robots.txt
2026-11-12,Argos,www.argos.co.uk,Google-Extended,training,no,no,"fetch tool, then chrome",rules as returned,https://www.argos.co.uk/robots.txt
2026-11-12,Argos,www.argos.co.uk,PerplexityBot,retrieval,no,no,"fetch tool, then chrome",rules as returned,https://www.argos.co.uk/robots.txt
2026-11-12,Argos,www.argos.co.uk,Perplexity-User,retrieval,no,no,"fetch tool, then chrome",rules as returned,https://www.argos.co.uk/robots.txt
2026-11-12,Argos,www.argos.co.uk,CCBot,training,no,no,"fetch tool, then chrome",rules as returned,https://www.argos.co.uk/robots.txt
2026-11-12,Argos,www.argos.co.uk,Applebot-Extended,training,no,no,"fetch tool, then chrome",rules as returned,https://www.argos.co.uk/robots.txt
2026-11-12,Argos,www.argos.co.uk,Bytespider,training,no,no,"fetch tool, then chrome",rules as returned,https://www.argos.co.uk/robots.txt
2026-11-12,Argos,www.argos.co.uk,Meta-ExternalAgent,training,no,no,"fetch tool, then chrome",rules as returned,https://www.argos.co.uk/robots.txt
2026-11-12,Argos,www.argos.co.uk,Amazonbot,training,no,no,"fetch tool, then chrome",rules as returned,https://www.argos.co.uk/robots.txt
2026-11-12,Asda,www.asda.com,GPTBot,training,no,no,"fetch tool, then chrome",full copy,https://www.asda.com/robots.txt
2026-11-12,Asda,www.asda.com,OAI-SearchBot,retrieval,no,no,"fetch tool, then chrome",full copy,https://www.asda.com/robots.txt
2026-11-12,Asda,www.asda.com,ChatGPT-User,retrieval,no,no,"fetch tool, then chrome",full copy,https://www.asda.com/robots.txt
2026-11-12,Asda,www.asda.com,ClaudeBot,training,no,no,"fetch tool, then chrome",full copy,https://www.asda.com/robots.txt
2026-11-12,Asda,www.asda.com,Claude-SearchBot,retrieval,no,no,"fetch tool, then chrome",full copy,https://www.asda.com/robots.txt
2026-11-12,Asda,www.asda.com,Claude-User,retrieval,no,no,"fetch tool, then chrome",full copy,https://www.asda.com/robots.txt
2026-11-12,Asda,www.asda.com,anthropic-ai,training,no,no,"fetch tool, then chrome",full copy,https://www.asda.com/robots.txt
2026-11-12,Asda,www.asda.com,Google-Extended,training,no,no,"fetch tool, then chrome",full copy,https://www.asda.com/robots.txt
2026-11-12,Asda,www.asda.com,PerplexityBot,retrieval,no,no,"fetch tool, then chrome",full copy,https://www.asda.com/robots.txt
2026-11-12,Asda,www.asda.com,Perplexity-User,retrieval,no,no,"fetch tool, then chrome",full copy,https://www.asda.com/robots.txt
2026-11-12,Asda,www.asda.com,CCBot,training,no,no,"fetch tool, then chrome",full copy,https://www.asda.com/robots.txt
2026-11-12,Asda,www.asda.com,Applebot-Extended,training,no,no,"fetch tool, then chrome",full copy,https://www.asda.com/robots.txt
2026-11-12,Asda,www.asda.com,Bytespider,training,no,no,"fetch tool, then chrome",full copy,https://www.asda.com/robots.txt
2026-11-12,Asda,www.asda.com,Meta-ExternalAgent,training,no,no,"fetch tool, then chrome",full copy,https://www.asda.com/robots.txt
2026-11-12,Asda,www.asda.com,Amazonbot,training,no,no,"fetch tool, then chrome",full copy,https://www.asda.com/robots.txt
2026-11-12,Lidl,www.lidl.co.uk,GPTBot,training,no,no,"fetch tool, then chrome",full copy,https://www.lidl.co.uk/robots.txt
2026-11-12,Lidl,www.lidl.co.uk,OAI-SearchBot,retrieval,no,no,"fetch tool, then chrome",full copy,https://www.lidl.co.uk/robots.txt
2026-11-12,Lidl,www.lidl.co.uk,ChatGPT-User,retrieval,no,no,"fetch tool, then chrome",full copy,https://www.lidl.co.uk/robots.txt
2026-11-12,Lidl,www.lidl.co.uk,ClaudeBot,training,no,no,"fetch tool, then chrome",full copy,https://www.lidl.co.uk/robots.txt
2026-11-12,Lidl,www.lidl.co.uk,Claude-SearchBot,retrieval,no,no,"fetch tool, then chrome",full copy,https://www.lidl.co.uk/robots.txt
2026-11-12,Lidl,www.lidl.co.uk,Claude-User,retrieval,no,no,"fetch tool, then chrome",full copy,https://www.lidl.co.uk/robots.txt
2026-11-12,Lidl,www.lidl.co.uk,anthropic-ai,training,no,no,"fetch tool, then chrome",full copy,https://www.lidl.co.uk/robots.txt
2026-11-12,Lidl,www.lidl.co.uk,Google-Extended,training,no,no,"fetch tool, then chrome",full copy,https://www.lidl.co.uk/robots.txt
2026-11-12,Lidl,www.lidl.co.uk,PerplexityBot,retrieval,no,no,"fetch tool, then chrome",full copy,https://www.lidl.co.uk/robots.txt
2026-11-12,Lidl,www.lidl.co.uk,Perplexity-User,retrieval,no,no,"fetch tool, then chrome",full copy,https://www.lidl.co.uk/robots.txt
2026-11-12,Lidl,www.lidl.co.uk,CCBot,training,no,no,"fetch tool, then chrome",full copy,https://www.lidl.co.uk/robots.txt
2026-11-12,Lidl,www.lidl.co.uk,Applebot-Extended,training,no,no,"fetch tool, then chrome",full copy,https://www.lidl.co.uk/robots.txt
2026-11-12,Lidl,www.lidl.co.uk,Bytespider,training,no,no,"fetch tool, then chrome",full copy,https://www.lidl.co.uk/robots.txt
2026-11-12,Lidl,www.lidl.co.uk,Meta-ExternalAgent,training,no,no,"fetch tool, then chrome",full copy,https://www.lidl.co.uk/robots.txt
2026-11-12,Lidl,www.lidl.co.uk,Amazonbot,training,no,no,"fetch tool, then chrome",full copy,https://www.lidl.co.uk/robots.txt
2026-11-12,Vinted,www.vinted.co.uk,GPTBot,training,yes,yes,"fetch tool, then chrome",condensed record,https://www.vinted.co.uk/robots.txt
2026-11-12,Vinted,www.vinted.co.uk,OAI-SearchBot,retrieval,no,no,"fetch tool, then chrome",condensed record,https://www.vinted.co.uk/robots.txt
2026-11-12,Vinted,www.vinted.co.uk,ChatGPT-User,retrieval,no,no,"fetch tool, then chrome",condensed record,https://www.vinted.co.uk/robots.txt
2026-11-12,Vinted,www.vinted.co.uk,ClaudeBot,training,yes,yes,"fetch tool, then chrome",condensed record,https://www.vinted.co.uk/robots.txt
2026-11-12,Vinted,www.vinted.co.uk,Claude-SearchBot,retrieval,no,no,"fetch tool, then chrome",condensed record,https://www.vinted.co.uk/robots.txt
2026-11-12,Vinted,www.vinted.co.uk,Claude-User,retrieval,no,no,"fetch tool, then chrome",condensed record,https://www.vinted.co.uk/robots.txt
2026-11-12,Vinted,www.vinted.co.uk,anthropic-ai,training,no,no,"fetch tool, then chrome",condensed record,https://www.vinted.co.uk/robots.txt
2026-11-12,Vinted,www.vinted.co.uk,Google-Extended,training,yes,yes,"fetch tool, then chrome",condensed record,https://www.vinted.co.uk/robots.txt
2026-11-12,Vinted,www.vinted.co.uk,PerplexityBot,retrieval,no,no,"fetch tool, then chrome",condensed record,https://www.vinted.co.uk/robots.txt
2026-11-12,Vinted,www.vinted.co.uk,Perplexity-User,retrieval,no,no,"fetch tool, then chrome",condensed record,https://www.vinted.co.uk/robots.txt
2026-11-12,Vinted,www.vinted.co.uk,CCBot,training,yes,yes,"fetch tool, then chrome",condensed record,https://www.vinted.co.uk/robots.txt
2026-11-12,Vinted,www.vinted.co.uk,Applebot-Extended,training,no,no,"fetch tool, then chrome",condensed record,https://www.vinted.co.uk/robots.txt
2026-11-12,Vinted,www.vinted.co.uk,Bytespider,training,no,no,"fetch tool, then chrome",condensed record,https://www.vinted.co.uk/robots.txt
2026-11-12,Vinted,www.vinted.co.uk,Meta-ExternalAgent,training,no,no,"fetch tool, then chrome",condensed record,https://www.vinted.co.uk/robots.txt
2026-11-12,Vinted,www.vinted.co.uk,Amazonbot,training,yes,yes,"fetch tool, then chrome",condensed record,https://www.vinted.co.uk/robots.txt
2026-11-12,Shein,www.shein.co.uk,GPTBot,training,no,no,"fetch tool, then chrome",full copy,https://www.shein.co.uk/robots.txt
2026-11-12,Shein,www.shein.co.uk,OAI-SearchBot,retrieval,no,no,"fetch tool, then chrome",full copy,https://www.shein.co.uk/robots.txt
2026-11-12,Shein,www.shein.co.uk,ChatGPT-User,retrieval,no,no,"fetch tool, then chrome",full copy,https://www.shein.co.uk/robots.txt
2026-11-12,Shein,www.shein.co.uk,ClaudeBot,training,no,no,"fetch tool, then chrome",full copy,https://www.shein.co.uk/robots.txt
2026-11-12,Shein,www.shein.co.uk,Claude-SearchBot,retrieval,no,no,"fetch tool, then chrome",full copy,https://www.shein.co.uk/robots.txt
2026-11-12,Shein,www.shein.co.uk,Claude-User,retrieval,no,no,"fetch tool, then chrome",full copy,https://www.shein.co.uk/robots.txt
2026-11-12,Shein,www.shein.co.uk,anthropic-ai,training,no,no,"fetch tool, then chrome",full copy,https://www.shein.co.uk/robots.txt
2026-11-12,Shein,www.shein.co.uk,Google-Extended,training,no,no,"fetch tool, then chrome",full copy,https://www.shein.co.uk/robots.txt
2026-11-12,Shein,www.shein.co.uk,PerplexityBot,retrieval,no,no,"fetch tool, then chrome",full copy,https://www.shein.co.uk/robots.txt
2026-11-12,Shein,www.shein.co.uk,Perplexity-User,retrieval,no,no,"fetch tool, then chrome",full copy,https://www.shein.co.uk/robots.txt
2026-11-12,Shein,www.shein.co.uk,CCBot,training,no,no,"fetch tool, then chrome",full copy,https://www.shein.co.uk/robots.txt
2026-11-12,Shein,www.shein.co.uk,Applebot-Extended,training,no,no,"fetch tool, then chrome",full copy,https://www.shein.co.uk/robots.txt
2026-11-12,Shein,www.shein.co.uk,Bytespider,training,no,no,"fetch tool, then chrome",full copy,https://www.shein.co.uk/robots.txt
2026-11-12,Shein,www.shein.co.uk,Meta-ExternalAgent,training,no,no,"fetch tool, then chrome",full copy,https://www.shein.co.uk/robots.txt
2026-11-12,Shein,www.shein.co.uk,Amazonbot,training,no,no,"fetch tool, then chrome",full copy,https://www.shein.co.uk/robots.txt