check_date,company,host,site_type,bot,bot_operator,bot_class,blocked,named_in_file,own_company_bot,collection,sha256_of_copy,source
2026-11-06,OpenAI,openai.com,corporate,GPTBot,OpenAI,training,no,no,yes,chrome,c53100563b9d289efeebb6f826a20c566578e1337e3096e298eb9da5ecc2e498,https://openai.com/robots.txt
2026-11-06,OpenAI,openai.com,corporate,OAI-SearchBot,OpenAI,retrieval,no,no,yes,chrome,c53100563b9d289efeebb6f826a20c566578e1337e3096e298eb9da5ecc2e498,https://openai.com/robots.txt
2026-11-06,OpenAI,openai.com,corporate,ChatGPT-User,OpenAI,retrieval,no,no,yes,chrome,c53100563b9d289efeebb6f826a20c566578e1337e3096e298eb9da5ecc2e498,https://openai.com/robots.txt
2026-11-06,OpenAI,openai.com,corporate,ClaudeBot,Anthropic,training,no,no,no,chrome,c53100563b9d289efeebb6f826a20c566578e1337e3096e298eb9da5ecc2e498,https://openai.com/robots.txt
2026-11-06,OpenAI,openai.com,corporate,Claude-SearchBot,Anthropic,retrieval,no,no,no,chrome,c53100563b9d289efeebb6f826a20c566578e1337e3096e298eb9da5ecc2e498,https://openai.com/robots.txt
2026-11-06,OpenAI,openai.com,corporate,Claude-User,Anthropic,retrieval,no,no,no,chrome,c53100563b9d289efeebb6f826a20c566578e1337e3096e298eb9da5ecc2e498,https://openai.com/robots.txt
2026-11-06,OpenAI,openai.com,corporate,anthropic-ai,Anthropic,training,no,no,no,chrome,c53100563b9d289efeebb6f826a20c566578e1337e3096e298eb9da5ecc2e498,https://openai.com/robots.txt
2026-11-06,OpenAI,openai.com,corporate,Google-Extended,Google,training,no,no,no,chrome,c53100563b9d289efeebb6f826a20c566578e1337e3096e298eb9da5ecc2e498,https://openai.com/robots.txt
2026-11-06,OpenAI,openai.com,corporate,PerplexityBot,Perplexity,retrieval,no,no,no,chrome,c53100563b9d289efeebb6f826a20c566578e1337e3096e298eb9da5ecc2e498,https://openai.com/robots.txt
2026-11-06,OpenAI,openai.com,corporate,Perplexity-User,Perplexity,retrieval,no,no,no,chrome,c53100563b9d289efeebb6f826a20c566578e1337e3096e298eb9da5ecc2e498,https://openai.com/robots.txt
2026-11-06,OpenAI,openai.com,corporate,CCBot,Common Crawl,training,no,no,no,chrome,c53100563b9d289efeebb6f826a20c566578e1337e3096e298eb9da5ecc2e498,https://openai.com/robots.txt
2026-11-06,OpenAI,openai.com,corporate,Applebot-Extended,Apple,training,no,no,no,chrome,c53100563b9d289efeebb6f826a20c566578e1337e3096e298eb9da5ecc2e498,https://openai.com/robots.txt
2026-11-06,OpenAI,openai.com,corporate,Bytespider,ByteDance,training,no,no,no,chrome,c53100563b9d289efeebb6f826a20c566578e1337e3096e298eb9da5ecc2e498,https://openai.com/robots.txt
2026-11-06,OpenAI,openai.com,corporate,Meta-ExternalAgent,Meta,training,no,no,no,chrome,c53100563b9d289efeebb6f826a20c566578e1337e3096e298eb9da5ecc2e498,https://openai.com/robots.txt
2026-11-06,OpenAI,openai.com,corporate,Amazonbot,Amazon,training,no,no,no,chrome,c53100563b9d289efeebb6f826a20c566578e1337e3096e298eb9da5ecc2e498,https://openai.com/robots.txt
2026-11-06,OpenAI,chatgpt.com,product,GPTBot,OpenAI,training,yes,no,yes,chrome,6abd8db88c9da1d617588b69bfc00c637bcb0d1e4a7b71a9d5f26e3c3ea78df2,https://chatgpt.com/robots.txt
2026-11-06,OpenAI,chatgpt.com,product,OAI-SearchBot,OpenAI,retrieval,yes,no,yes,chrome,6abd8db88c9da1d617588b69bfc00c637bcb0d1e4a7b71a9d5f26e3c3ea78df2,https://chatgpt.com/robots.txt
2026-11-06,OpenAI,chatgpt.com,product,ChatGPT-User,OpenAI,retrieval,yes,no,yes,chrome,6abd8db88c9da1d617588b69bfc00c637bcb0d1e4a7b71a9d5f26e3c3ea78df2,https://chatgpt.com/robots.txt
2026-11-06,OpenAI,chatgpt.com,product,ClaudeBot,Anthropic,training,yes,no,no,chrome,6abd8db88c9da1d617588b69bfc00c637bcb0d1e4a7b71a9d5f26e3c3ea78df2,https://chatgpt.com/robots.txt
2026-11-06,OpenAI,chatgpt.com,product,Claude-SearchBot,Anthropic,retrieval,yes,no,no,chrome,6abd8db88c9da1d617588b69bfc00c637bcb0d1e4a7b71a9d5f26e3c3ea78df2,https://chatgpt.com/robots.txt
2026-11-06,OpenAI,chatgpt.com,product,Claude-User,Anthropic,retrieval,yes,no,no,chrome,6abd8db88c9da1d617588b69bfc00c637bcb0d1e4a7b71a9d5f26e3c3ea78df2,https://chatgpt.com/robots.txt
2026-11-06,OpenAI,chatgpt.com,product,anthropic-ai,Anthropic,training,yes,yes,no,chrome,6abd8db88c9da1d617588b69bfc00c637bcb0d1e4a7b71a9d5f26e3c3ea78df2,https://chatgpt.com/robots.txt
2026-11-06,OpenAI,chatgpt.com,product,Google-Extended,Google,training,yes,yes,no,chrome,6abd8db88c9da1d617588b69bfc00c637bcb0d1e4a7b71a9d5f26e3c3ea78df2,https://chatgpt.com/robots.txt
2026-11-06,OpenAI,chatgpt.com,product,PerplexityBot,Perplexity,retrieval,yes,yes,no,chrome,6abd8db88c9da1d617588b69bfc00c637bcb0d1e4a7b71a9d5f26e3c3ea78df2,https://chatgpt.com/robots.txt
2026-11-06,OpenAI,chatgpt.com,product,Perplexity-User,Perplexity,retrieval,yes,no,no,chrome,6abd8db88c9da1d617588b69bfc00c637bcb0d1e4a7b71a9d5f26e3c3ea78df2,https://chatgpt.com/robots.txt
2026-11-06,OpenAI,chatgpt.com,product,CCBot,Common Crawl,training,yes,yes,no,chrome,6abd8db88c9da1d617588b69bfc00c637bcb0d1e4a7b71a9d5f26e3c3ea78df2,https://chatgpt.com/robots.txt
2026-11-06,OpenAI,chatgpt.com,product,Applebot-Extended,Apple,training,yes,no,no,chrome,6abd8db88c9da1d617588b69bfc00c637bcb0d1e4a7b71a9d5f26e3c3ea78df2,https://chatgpt.com/robots.txt
2026-11-06,OpenAI,chatgpt.com,product,Bytespider,ByteDance,training,yes,yes,no,chrome,6abd8db88c9da1d617588b69bfc00c637bcb0d1e4a7b71a9d5f26e3c3ea78df2,https://chatgpt.com/robots.txt
2026-11-06,OpenAI,chatgpt.com,product,Meta-ExternalAgent,Meta,training,yes,no,no,chrome,6abd8db88c9da1d617588b69bfc00c637bcb0d1e4a7b71a9d5f26e3c3ea78df2,https://chatgpt.com/robots.txt
2026-11-06,OpenAI,chatgpt.com,product,Amazonbot,Amazon,training,yes,no,no,chrome,6abd8db88c9da1d617588b69bfc00c637bcb0d1e4a7b71a9d5f26e3c3ea78df2,https://chatgpt.com/robots.txt
2026-11-06,Anthropic,www.anthropic.com,corporate,GPTBot,OpenAI,training,no,no,no,webfetch,b8bb03567cf862a9024bd464bd4bd7afaf054b1029de79727b2ba94711480ec7,https://www.anthropic.com/robots.txt
2026-11-06,Anthropic,www.anthropic.com,corporate,OAI-SearchBot,OpenAI,retrieval,no,no,no,webfetch,b8bb03567cf862a9024bd464bd4bd7afaf054b1029de79727b2ba94711480ec7,https://www.anthropic.com/robots.txt
2026-11-06,Anthropic,www.anthropic.com,corporate,ChatGPT-User,OpenAI,retrieval,no,no,no,webfetch,b8bb03567cf862a9024bd464bd4bd7afaf054b1029de79727b2ba94711480ec7,https://www.anthropic.com/robots.txt
2026-11-06,Anthropic,www.anthropic.com,corporate,ClaudeBot,Anthropic,training,no,no,yes,webfetch,b8bb03567cf862a9024bd464bd4bd7afaf054b1029de79727b2ba94711480ec7,https://www.anthropic.com/robots.txt
2026-11-06,Anthropic,www.anthropic.com,corporate,Claude-SearchBot,Anthropic,retrieval,no,no,yes,webfetch,b8bb03567cf862a9024bd464bd4bd7afaf054b1029de79727b2ba94711480ec7,https://www.anthropic.com/robots.txt
2026-11-06,Anthropic,www.anthropic.com,corporate,Claude-User,Anthropic,retrieval,no,no,yes,webfetch,b8bb03567cf862a9024bd464bd4bd7afaf054b1029de79727b2ba94711480ec7,https://www.anthropic.com/robots.txt
2026-11-06,Anthropic,www.anthropic.com,corporate,anthropic-ai,Anthropic,training,no,no,yes,webfetch,b8bb03567cf862a9024bd464bd4bd7afaf054b1029de79727b2ba94711480ec7,https://www.anthropic.com/robots.txt
2026-11-06,Anthropic,www.anthropic.com,corporate,Google-Extended,Google,training,no,no,no,webfetch,b8bb03567cf862a9024bd464bd4bd7afaf054b1029de79727b2ba94711480ec7,https://www.anthropic.com/robots.txt
2026-11-06,Anthropic,www.anthropic.com,corporate,PerplexityBot,Perplexity,retrieval,no,no,no,webfetch,b8bb03567cf862a9024bd464bd4bd7afaf054b1029de79727b2ba94711480ec7,https://www.anthropic.com/robots.txt
2026-11-06,Anthropic,www.anthropic.com,corporate,Perplexity-User,Perplexity,retrieval,no,no,no,webfetch,b8bb03567cf862a9024bd464bd4bd7afaf054b1029de79727b2ba94711480ec7,https://www.anthropic.com/robots.txt
2026-11-06,Anthropic,www.anthropic.com,corporate,CCBot,Common Crawl,training,no,no,no,webfetch,b8bb03567cf862a9024bd464bd4bd7afaf054b1029de79727b2ba94711480ec7,https://www.anthropic.com/robots.txt
2026-11-06,Anthropic,www.anthropic.com,corporate,Applebot-Extended,Apple,training,no,no,no,webfetch,b8bb03567cf862a9024bd464bd4bd7afaf054b1029de79727b2ba94711480ec7,https://www.anthropic.com/robots.txt
2026-11-06,Anthropic,www.anthropic.com,corporate,Bytespider,ByteDance,training,no,no,no,webfetch,b8bb03567cf862a9024bd464bd4bd7afaf054b1029de79727b2ba94711480ec7,https://www.anthropic.com/robots.txt
2026-11-06,Anthropic,www.anthropic.com,corporate,Meta-ExternalAgent,Meta,training,no,no,no,webfetch,b8bb03567cf862a9024bd464bd4bd7afaf054b1029de79727b2ba94711480ec7,https://www.anthropic.com/robots.txt
2026-11-06,Anthropic,www.anthropic.com,corporate,Amazonbot,Amazon,training,no,no,no,webfetch,b8bb03567cf862a9024bd464bd4bd7afaf054b1029de79727b2ba94711480ec7,https://www.anthropic.com/robots.txt
2026-11-06,Anthropic,claude.ai,product,GPTBot,OpenAI,training,yes,yes,no,curl,75ccbccac0e02fbc11965050f3152c1ef7435fff085f0eee27592a82059b8902,https://claude.ai/robots.txt
2026-11-06,Anthropic,claude.ai,product,OAI-SearchBot,OpenAI,retrieval,yes,yes,no,curl,75ccbccac0e02fbc11965050f3152c1ef7435fff085f0eee27592a82059b8902,https://claude.ai/robots.txt
2026-11-06,Anthropic,claude.ai,product,ChatGPT-User,OpenAI,retrieval,yes,yes,no,curl,75ccbccac0e02fbc11965050f3152c1ef7435fff085f0eee27592a82059b8902,https://claude.ai/robots.txt
2026-11-06,Anthropic,claude.ai,product,ClaudeBot,Anthropic,training,no,no,yes,curl,75ccbccac0e02fbc11965050f3152c1ef7435fff085f0eee27592a82059b8902,https://claude.ai/robots.txt
2026-11-06,Anthropic,claude.ai,product,Claude-SearchBot,Anthropic,retrieval,no,no,yes,curl,75ccbccac0e02fbc11965050f3152c1ef7435fff085f0eee27592a82059b8902,https://claude.ai/robots.txt
2026-11-06,Anthropic,claude.ai,product,Claude-User,Anthropic,retrieval,no,no,yes,curl,75ccbccac0e02fbc11965050f3152c1ef7435fff085f0eee27592a82059b8902,https://claude.ai/robots.txt
2026-11-06,Anthropic,claude.ai,product,anthropic-ai,Anthropic,training,no,no,yes,curl,75ccbccac0e02fbc11965050f3152c1ef7435fff085f0eee27592a82059b8902,https://claude.ai/robots.txt
2026-11-06,Anthropic,claude.ai,product,Google-Extended,Google,training,yes,yes,no,curl,75ccbccac0e02fbc11965050f3152c1ef7435fff085f0eee27592a82059b8902,https://claude.ai/robots.txt
2026-11-06,Anthropic,claude.ai,product,PerplexityBot,Perplexity,retrieval,no,no,no,curl,75ccbccac0e02fbc11965050f3152c1ef7435fff085f0eee27592a82059b8902,https://claude.ai/robots.txt
2026-11-06,Anthropic,claude.ai,product,Perplexity-User,Perplexity,retrieval,no,no,no,curl,75ccbccac0e02fbc11965050f3152c1ef7435fff085f0eee27592a82059b8902,https://claude.ai/robots.txt
2026-11-06,Anthropic,claude.ai,product,CCBot,Common Crawl,training,no,no,no,curl,75ccbccac0e02fbc11965050f3152c1ef7435fff085f0eee27592a82059b8902,https://claude.ai/robots.txt
2026-11-06,Anthropic,claude.ai,product,Applebot-Extended,Apple,training,no,no,no,curl,75ccbccac0e02fbc11965050f3152c1ef7435fff085f0eee27592a82059b8902,https://claude.ai/robots.txt
2026-11-06,Anthropic,claude.ai,product,Bytespider,ByteDance,training,no,no,no,curl,75ccbccac0e02fbc11965050f3152c1ef7435fff085f0eee27592a82059b8902,https://claude.ai/robots.txt
2026-11-06,Anthropic,claude.ai,product,Meta-ExternalAgent,Meta,training,no,no,no,curl,75ccbccac0e02fbc11965050f3152c1ef7435fff085f0eee27592a82059b8902,https://claude.ai/robots.txt
2026-11-06,Anthropic,claude.ai,product,Amazonbot,Amazon,training,no,no,no,curl,75ccbccac0e02fbc11965050f3152c1ef7435fff085f0eee27592a82059b8902,https://claude.ai/robots.txt
2026-11-06,Google,gemini.google.com,product,GPTBot,OpenAI,training,no,no,no,webfetch,6424e2fde98e589b787def8916d42c766d866a1db9dcc0503e49d6e204ea1f71,https://gemini.google.com/robots.txt
2026-11-06,Google,gemini.google.com,product,OAI-SearchBot,OpenAI,retrieval,no,no,no,webfetch,6424e2fde98e589b787def8916d42c766d866a1db9dcc0503e49d6e204ea1f71,https://gemini.google.com/robots.txt
2026-11-06,Google,gemini.google.com,product,ChatGPT-User,OpenAI,retrieval,no,no,no,webfetch,6424e2fde98e589b787def8916d42c766d866a1db9dcc0503e49d6e204ea1f71,https://gemini.google.com/robots.txt
2026-11-06,Google,gemini.google.com,product,ClaudeBot,Anthropic,training,no,no,no,webfetch,6424e2fde98e589b787def8916d42c766d866a1db9dcc0503e49d6e204ea1f71,https://gemini.google.com/robots.txt
2026-11-06,Google,gemini.google.com,product,Claude-SearchBot,Anthropic,retrieval,no,no,no,webfetch,6424e2fde98e589b787def8916d42c766d866a1db9dcc0503e49d6e204ea1f71,https://gemini.google.com/robots.txt
2026-11-06,Google,gemini.google.com,product,Claude-User,Anthropic,retrieval,no,no,no,webfetch,6424e2fde98e589b787def8916d42c766d866a1db9dcc0503e49d6e204ea1f71,https://gemini.google.com/robots.txt
2026-11-06,Google,gemini.google.com,product,anthropic-ai,Anthropic,training,no,no,no,webfetch,6424e2fde98e589b787def8916d42c766d866a1db9dcc0503e49d6e204ea1f71,https://gemini.google.com/robots.txt
2026-11-06,Google,gemini.google.com,product,Google-Extended,Google,training,no,no,yes,webfetch,6424e2fde98e589b787def8916d42c766d866a1db9dcc0503e49d6e204ea1f71,https://gemini.google.com/robots.txt
2026-11-06,Google,gemini.google.com,product,PerplexityBot,Perplexity,retrieval,no,no,no,webfetch,6424e2fde98e589b787def8916d42c766d866a1db9dcc0503e49d6e204ea1f71,https://gemini.google.com/robots.txt
2026-11-06,Google,gemini.google.com,product,Perplexity-User,Perplexity,retrieval,no,no,no,webfetch,6424e2fde98e589b787def8916d42c766d866a1db9dcc0503e49d6e204ea1f71,https://gemini.google.com/robots.txt
2026-11-06,Google,gemini.google.com,product,CCBot,Common Crawl,training,no,no,no,webfetch,6424e2fde98e589b787def8916d42c766d866a1db9dcc0503e49d6e204ea1f71,https://gemini.google.com/robots.txt
2026-11-06,Google,gemini.google.com,product,Applebot-Extended,Apple,training,no,no,no,webfetch,6424e2fde98e589b787def8916d42c766d866a1db9dcc0503e49d6e204ea1f71,https://gemini.google.com/robots.txt
2026-11-06,Google,gemini.google.com,product,Bytespider,ByteDance,training,no,no,no,webfetch,6424e2fde98e589b787def8916d42c766d866a1db9dcc0503e49d6e204ea1f71,https://gemini.google.com/robots.txt
2026-11-06,Google,gemini.google.com,product,Meta-ExternalAgent,Meta,training,no,no,no,webfetch,6424e2fde98e589b787def8916d42c766d866a1db9dcc0503e49d6e204ea1f71,https://gemini.google.com/robots.txt
2026-11-06,Google,gemini.google.com,product,Amazonbot,Amazon,training,no,no,no,webfetch,6424e2fde98e589b787def8916d42c766d866a1db9dcc0503e49d6e204ea1f71,https://gemini.google.com/robots.txt
2026-11-06,Perplexity,www.perplexity.ai,product,GPTBot,OpenAI,training,no,no,no,chrome,33f5967c54d4934e2250adb72f47d9187855f36d5816c074cb550ee3e7116256,https://www.perplexity.ai/robots.txt
2026-11-06,Perplexity,www.perplexity.ai,product,OAI-SearchBot,OpenAI,retrieval,no,no,no,chrome,33f5967c54d4934e2250adb72f47d9187855f36d5816c074cb550ee3e7116256,https://www.perplexity.ai/robots.txt
2026-11-06,Perplexity,www.perplexity.ai,product,ChatGPT-User,OpenAI,retrieval,no,no,no,chrome,33f5967c54d4934e2250adb72f47d9187855f36d5816c074cb550ee3e7116256,https://www.perplexity.ai/robots.txt
2026-11-06,Perplexity,www.perplexity.ai,product,ClaudeBot,Anthropic,training,no,no,no,chrome,33f5967c54d4934e2250adb72f47d9187855f36d5816c074cb550ee3e7116256,https://www.perplexity.ai/robots.txt
2026-11-06,Perplexity,www.perplexity.ai,product,Claude-SearchBot,Anthropic,retrieval,no,no,no,chrome,33f5967c54d4934e2250adb72f47d9187855f36d5816c074cb550ee3e7116256,https://www.perplexity.ai/robots.txt
2026-11-06,Perplexity,www.perplexity.ai,product,Claude-User,Anthropic,retrieval,no,no,no,chrome,33f5967c54d4934e2250adb72f47d9187855f36d5816c074cb550ee3e7116256,https://www.perplexity.ai/robots.txt
2026-11-06,Perplexity,www.perplexity.ai,product,anthropic-ai,Anthropic,training,no,no,no,chrome,33f5967c54d4934e2250adb72f47d9187855f36d5816c074cb550ee3e7116256,https://www.perplexity.ai/robots.txt
2026-11-06,Perplexity,www.perplexity.ai,product,Google-Extended,Google,training,no,no,no,chrome,33f5967c54d4934e2250adb72f47d9187855f36d5816c074cb550ee3e7116256,https://www.perplexity.ai/robots.txt
2026-11-06,Perplexity,www.perplexity.ai,product,PerplexityBot,Perplexity,retrieval,no,no,yes,chrome,33f5967c54d4934e2250adb72f47d9187855f36d5816c074cb550ee3e7116256,https://www.perplexity.ai/robots.txt
2026-11-06,Perplexity,www.perplexity.ai,product,Perplexity-User,Perplexity,retrieval,no,no,yes,chrome,33f5967c54d4934e2250adb72f47d9187855f36d5816c074cb550ee3e7116256,https://www.perplexity.ai/robots.txt
2026-11-06,Perplexity,www.perplexity.ai,product,CCBot,Common Crawl,training,no,no,no,chrome,33f5967c54d4934e2250adb72f47d9187855f36d5816c074cb550ee3e7116256,https://www.perplexity.ai/robots.txt
2026-11-06,Perplexity,www.perplexity.ai,product,Applebot-Extended,Apple,training,no,no,no,chrome,33f5967c54d4934e2250adb72f47d9187855f36d5816c074cb550ee3e7116256,https://www.perplexity.ai/robots.txt
2026-11-06,Perplexity,www.perplexity.ai,product,Bytespider,ByteDance,training,no,no,no,chrome,33f5967c54d4934e2250adb72f47d9187855f36d5816c074cb550ee3e7116256,https://www.perplexity.ai/robots.txt
2026-11-06,Perplexity,www.perplexity.ai,product,Meta-ExternalAgent,Meta,training,no,no,no,chrome,33f5967c54d4934e2250adb72f47d9187855f36d5816c074cb550ee3e7116256,https://www.perplexity.ai/robots.txt
2026-11-06,Perplexity,www.perplexity.ai,product,Amazonbot,Amazon,training,no,no,no,chrome,33f5967c54d4934e2250adb72f47d9187855f36d5816c074cb550ee3e7116256,https://www.perplexity.ai/robots.txt
2026-11-06,Common Crawl,commoncrawl.org,corporate,GPTBot,OpenAI,training,no,no,no,webfetch,56f568d5d0caa2e717efd02bd2eb762ac354a7aabbe8c7fc05aa5711fe9c836a,https://commoncrawl.org/robots.txt
2026-11-06,Common Crawl,commoncrawl.org,corporate,OAI-SearchBot,OpenAI,retrieval,no,no,no,webfetch,56f568d5d0caa2e717efd02bd2eb762ac354a7aabbe8c7fc05aa5711fe9c836a,https://commoncrawl.org/robots.txt
2026-11-06,Common Crawl,commoncrawl.org,corporate,ChatGPT-User,OpenAI,retrieval,no,no,no,webfetch,56f568d5d0caa2e717efd02bd2eb762ac354a7aabbe8c7fc05aa5711fe9c836a,https://commoncrawl.org/robots.txt
2026-11-06,Common Crawl,commoncrawl.org,corporate,ClaudeBot,Anthropic,training,no,no,no,webfetch,56f568d5d0caa2e717efd02bd2eb762ac354a7aabbe8c7fc05aa5711fe9c836a,https://commoncrawl.org/robots.txt
2026-11-06,Common Crawl,commoncrawl.org,corporate,Claude-SearchBot,Anthropic,retrieval,no,no,no,webfetch,56f568d5d0caa2e717efd02bd2eb762ac354a7aabbe8c7fc05aa5711fe9c836a,https://commoncrawl.org/robots.txt
2026-11-06,Common Crawl,commoncrawl.org,corporate,Claude-User,Anthropic,retrieval,no,no,no,webfetch,56f568d5d0caa2e717efd02bd2eb762ac354a7aabbe8c7fc05aa5711fe9c836a,https://commoncrawl.org/robots.txt
2026-11-06,Common Crawl,commoncrawl.org,corporate,anthropic-ai,Anthropic,training,no,no,no,webfetch,56f568d5d0caa2e717efd02bd2eb762ac354a7aabbe8c7fc05aa5711fe9c836a,https://commoncrawl.org/robots.txt
2026-11-06,Common Crawl,commoncrawl.org,corporate,Google-Extended,Google,training,no,no,no,webfetch,56f568d5d0caa2e717efd02bd2eb762ac354a7aabbe8c7fc05aa5711fe9c836a,https://commoncrawl.org/robots.txt
2026-11-06,Common Crawl,commoncrawl.org,corporate,PerplexityBot,Perplexity,retrieval,no,no,no,webfetch,56f568d5d0caa2e717efd02bd2eb762ac354a7aabbe8c7fc05aa5711fe9c836a,https://commoncrawl.org/robots.txt
2026-11-06,Common Crawl,commoncrawl.org,corporate,Perplexity-User,Perplexity,retrieval,no,no,no,webfetch,56f568d5d0caa2e717efd02bd2eb762ac354a7aabbe8c7fc05aa5711fe9c836a,https://commoncrawl.org/robots.txt
2026-11-06,Common Crawl,commoncrawl.org,corporate,CCBot,Common Crawl,training,no,no,yes,webfetch,56f568d5d0caa2e717efd02bd2eb762ac354a7aabbe8c7fc05aa5711fe9c836a,https://commoncrawl.org/robots.txt
2026-11-06,Common Crawl,commoncrawl.org,corporate,Applebot-Extended,Apple,training,no,no,no,webfetch,56f568d5d0caa2e717efd02bd2eb762ac354a7aabbe8c7fc05aa5711fe9c836a,https://commoncrawl.org/robots.txt
2026-11-06,Common Crawl,commoncrawl.org,corporate,Bytespider,ByteDance,training,no,no,no,webfetch,56f568d5d0caa2e717efd02bd2eb762ac354a7aabbe8c7fc05aa5711fe9c836a,https://commoncrawl.org/robots.txt
2026-11-06,Common Crawl,commoncrawl.org,corporate,Meta-ExternalAgent,Meta,training,no,no,no,webfetch,56f568d5d0caa2e717efd02bd2eb762ac354a7aabbe8c7fc05aa5711fe9c836a,https://commoncrawl.org/robots.txt
2026-11-06,Common Crawl,commoncrawl.org,corporate,Amazonbot,Amazon,training,no,no,no,webfetch,56f568d5d0caa2e717efd02bd2eb762ac354a7aabbe8c7fc05aa5711fe9c836a,https://commoncrawl.org/robots.txt
2026-11-06,Apple,www.apple.com,corporate,GPTBot,OpenAI,training,no,no,no,chrome,3115f45c98cfa19341978dcab5dfbc0fd3e0ff4eb3ab1fe8192c3fbf7227dee7,https://www.apple.com/robots.txt
2026-11-06,Apple,www.apple.com,corporate,OAI-SearchBot,OpenAI,retrieval,no,no,no,chrome,3115f45c98cfa19341978dcab5dfbc0fd3e0ff4eb3ab1fe8192c3fbf7227dee7,https://www.apple.com/robots.txt
2026-11-06,Apple,www.apple.com,corporate,ChatGPT-User,OpenAI,retrieval,no,no,no,chrome,3115f45c98cfa19341978dcab5dfbc0fd3e0ff4eb3ab1fe8192c3fbf7227dee7,https://www.apple.com/robots.txt
2026-11-06,Apple,www.apple.com,corporate,ClaudeBot,Anthropic,training,no,no,no,chrome,3115f45c98cfa19341978dcab5dfbc0fd3e0ff4eb3ab1fe8192c3fbf7227dee7,https://www.apple.com/robots.txt
2026-11-06,Apple,www.apple.com,corporate,Claude-SearchBot,Anthropic,retrieval,no,no,no,chrome,3115f45c98cfa19341978dcab5dfbc0fd3e0ff4eb3ab1fe8192c3fbf7227dee7,https://www.apple.com/robots.txt
2026-11-06,Apple,www.apple.com,corporate,Claude-User,Anthropic,retrieval,no,no,no,chrome,3115f45c98cfa19341978dcab5dfbc0fd3e0ff4eb3ab1fe8192c3fbf7227dee7,https://www.apple.com/robots.txt
2026-11-06,Apple,www.apple.com,corporate,anthropic-ai,Anthropic,training,no,no,no,chrome,3115f45c98cfa19341978dcab5dfbc0fd3e0ff4eb3ab1fe8192c3fbf7227dee7,https://www.apple.com/robots.txt
2026-11-06,Apple,www.apple.com,corporate,Google-Extended,Google,training,no,no,no,chrome,3115f45c98cfa19341978dcab5dfbc0fd3e0ff4eb3ab1fe8192c3fbf7227dee7,https://www.apple.com/robots.txt
2026-11-06,Apple,www.apple.com,corporate,PerplexityBot,Perplexity,retrieval,no,no,no,chrome,3115f45c98cfa19341978dcab5dfbc0fd3e0ff4eb3ab1fe8192c3fbf7227dee7,https://www.apple.com/robots.txt
2026-11-06,Apple,www.apple.com,corporate,Perplexity-User,Perplexity,retrieval,no,no,no,chrome,3115f45c98cfa19341978dcab5dfbc0fd3e0ff4eb3ab1fe8192c3fbf7227dee7,https://www.apple.com/robots.txt
2026-11-06,Apple,www.apple.com,corporate,CCBot,Common Crawl,training,no,no,no,chrome,3115f45c98cfa19341978dcab5dfbc0fd3e0ff4eb3ab1fe8192c3fbf7227dee7,https://www.apple.com/robots.txt
2026-11-06,Apple,www.apple.com,corporate,Applebot-Extended,Apple,training,no,no,yes,chrome,3115f45c98cfa19341978dcab5dfbc0fd3e0ff4eb3ab1fe8192c3fbf7227dee7,https://www.apple.com/robots.txt
2026-11-06,Apple,www.apple.com,corporate,Bytespider,ByteDance,training,no,no,no,chrome,3115f45c98cfa19341978dcab5dfbc0fd3e0ff4eb3ab1fe8192c3fbf7227dee7,https://www.apple.com/robots.txt
2026-11-06,Apple,www.apple.com,corporate,Meta-ExternalAgent,Meta,training,no,no,no,chrome,3115f45c98cfa19341978dcab5dfbc0fd3e0ff4eb3ab1fe8192c3fbf7227dee7,https://www.apple.com/robots.txt
2026-11-06,Apple,www.apple.com,corporate,Amazonbot,Amazon,training,no,no,no,chrome,3115f45c98cfa19341978dcab5dfbc0fd3e0ff4eb3ab1fe8192c3fbf7227dee7,https://www.apple.com/robots.txt
2026-11-06,ByteDance,www.bytedance.com,corporate,GPTBot,OpenAI,training,no,no,no,webfetch,7391667bbb4fae2e686ee472216c54d28a7c4894c95ca17e0bdf91efb93197e9,https://www.bytedance.com/robots.txt
2026-11-06,ByteDance,www.bytedance.com,corporate,OAI-SearchBot,OpenAI,retrieval,no,no,no,webfetch,7391667bbb4fae2e686ee472216c54d28a7c4894c95ca17e0bdf91efb93197e9,https://www.bytedance.com/robots.txt
2026-11-06,ByteDance,www.bytedance.com,corporate,ChatGPT-User,OpenAI,retrieval,no,no,no,webfetch,7391667bbb4fae2e686ee472216c54d28a7c4894c95ca17e0bdf91efb93197e9,https://www.bytedance.com/robots.txt
2026-11-06,ByteDance,www.bytedance.com,corporate,ClaudeBot,Anthropic,training,no,no,no,webfetch,7391667bbb4fae2e686ee472216c54d28a7c4894c95ca17e0bdf91efb93197e9,https://www.bytedance.com/robots.txt
2026-11-06,ByteDance,www.bytedance.com,corporate,Claude-SearchBot,Anthropic,retrieval,no,no,no,webfetch,7391667bbb4fae2e686ee472216c54d28a7c4894c95ca17e0bdf91efb93197e9,https://www.bytedance.com/robots.txt
2026-11-06,ByteDance,www.bytedance.com,corporate,Claude-User,Anthropic,retrieval,no,no,no,webfetch,7391667bbb4fae2e686ee472216c54d28a7c4894c95ca17e0bdf91efb93197e9,https://www.bytedance.com/robots.txt
2026-11-06,ByteDance,www.bytedance.com,corporate,anthropic-ai,Anthropic,training,no,no,no,webfetch,7391667bbb4fae2e686ee472216c54d28a7c4894c95ca17e0bdf91efb93197e9,https://www.bytedance.com/robots.txt
2026-11-06,ByteDance,www.bytedance.com,corporate,Google-Extended,Google,training,no,no,no,webfetch,7391667bbb4fae2e686ee472216c54d28a7c4894c95ca17e0bdf91efb93197e9,https://www.bytedance.com/robots.txt
2026-11-06,ByteDance,www.bytedance.com,corporate,PerplexityBot,Perplexity,retrieval,no,no,no,webfetch,7391667bbb4fae2e686ee472216c54d28a7c4894c95ca17e0bdf91efb93197e9,https://www.bytedance.com/robots.txt
2026-11-06,ByteDance,www.bytedance.com,corporate,Perplexity-User,Perplexity,retrieval,no,no,no,webfetch,7391667bbb4fae2e686ee472216c54d28a7c4894c95ca17e0bdf91efb93197e9,https://www.bytedance.com/robots.txt
2026-11-06,ByteDance,www.bytedance.com,corporate,CCBot,Common Crawl,training,no,no,no,webfetch,7391667bbb4fae2e686ee472216c54d28a7c4894c95ca17e0bdf91efb93197e9,https://www.bytedance.com/robots.txt
2026-11-06,ByteDance,www.bytedance.com,corporate,Applebot-Extended,Apple,training,no,no,no,webfetch,7391667bbb4fae2e686ee472216c54d28a7c4894c95ca17e0bdf91efb93197e9,https://www.bytedance.com/robots.txt
2026-11-06,ByteDance,www.bytedance.com,corporate,Bytespider,ByteDance,training,no,no,yes,webfetch,7391667bbb4fae2e686ee472216c54d28a7c4894c95ca17e0bdf91efb93197e9,https://www.bytedance.com/robots.txt
2026-11-06,ByteDance,www.bytedance.com,corporate,Meta-ExternalAgent,Meta,training,no,no,no,webfetch,7391667bbb4fae2e686ee472216c54d28a7c4894c95ca17e0bdf91efb93197e9,https://www.bytedance.com/robots.txt
2026-11-06,ByteDance,www.bytedance.com,corporate,Amazonbot,Amazon,training,no,no,no,webfetch,7391667bbb4fae2e686ee472216c54d28a7c4894c95ca17e0bdf91efb93197e9,https://www.bytedance.com/robots.txt
2026-11-06,Meta,www.meta.com,corporate,GPTBot,OpenAI,training,no,no,no,webfetch,67b4b83725499e81d6fcb57a8c43a1b969ef5d9825b161383d3a0ac94ee6dc17,https://www.meta.com/robots.txt
2026-11-06,Meta,www.meta.com,corporate,OAI-SearchBot,OpenAI,retrieval,no,no,no,webfetch,67b4b83725499e81d6fcb57a8c43a1b969ef5d9825b161383d3a0ac94ee6dc17,https://www.meta.com/robots.txt
2026-11-06,Meta,www.meta.com,corporate,ChatGPT-User,OpenAI,retrieval,no,no,no,webfetch,67b4b83725499e81d6fcb57a8c43a1b969ef5d9825b161383d3a0ac94ee6dc17,https://www.meta.com/robots.txt
2026-11-06,Meta,www.meta.com,corporate,ClaudeBot,Anthropic,training,no,no,no,webfetch,67b4b83725499e81d6fcb57a8c43a1b969ef5d9825b161383d3a0ac94ee6dc17,https://www.meta.com/robots.txt
2026-11-06,Meta,www.meta.com,corporate,Claude-SearchBot,Anthropic,retrieval,no,no,no,webfetch,67b4b83725499e81d6fcb57a8c43a1b969ef5d9825b161383d3a0ac94ee6dc17,https://www.meta.com/robots.txt
2026-11-06,Meta,www.meta.com,corporate,Claude-User,Anthropic,retrieval,no,no,no,webfetch,67b4b83725499e81d6fcb57a8c43a1b969ef5d9825b161383d3a0ac94ee6dc17,https://www.meta.com/robots.txt
2026-11-06,Meta,www.meta.com,corporate,anthropic-ai,Anthropic,training,no,no,no,webfetch,67b4b83725499e81d6fcb57a8c43a1b969ef5d9825b161383d3a0ac94ee6dc17,https://www.meta.com/robots.txt
2026-11-06,Meta,www.meta.com,corporate,Google-Extended,Google,training,no,no,no,webfetch,67b4b83725499e81d6fcb57a8c43a1b969ef5d9825b161383d3a0ac94ee6dc17,https://www.meta.com/robots.txt
2026-11-06,Meta,www.meta.com,corporate,PerplexityBot,Perplexity,retrieval,no,no,no,webfetch,67b4b83725499e81d6fcb57a8c43a1b969ef5d9825b161383d3a0ac94ee6dc17,https://www.meta.com/robots.txt
2026-11-06,Meta,www.meta.com,corporate,Perplexity-User,Perplexity,retrieval,no,no,no,webfetch,67b4b83725499e81d6fcb57a8c43a1b969ef5d9825b161383d3a0ac94ee6dc17,https://www.meta.com/robots.txt
2026-11-06,Meta,www.meta.com,corporate,CCBot,Common Crawl,training,no,no,no,webfetch,67b4b83725499e81d6fcb57a8c43a1b969ef5d9825b161383d3a0ac94ee6dc17,https://www.meta.com/robots.txt
2026-11-06,Meta,www.meta.com,corporate,Applebot-Extended,Apple,training,no,no,no,webfetch,67b4b83725499e81d6fcb57a8c43a1b969ef5d9825b161383d3a0ac94ee6dc17,https://www.meta.com/robots.txt
2026-11-06,Meta,www.meta.com,corporate,Bytespider,ByteDance,training,no,no,no,webfetch,67b4b83725499e81d6fcb57a8c43a1b969ef5d9825b161383d3a0ac94ee6dc17,https://www.meta.com/robots.txt
2026-11-06,Meta,www.meta.com,corporate,Meta-ExternalAgent,Meta,training,no,no,yes,webfetch,67b4b83725499e81d6fcb57a8c43a1b969ef5d9825b161383d3a0ac94ee6dc17,https://www.meta.com/robots.txt
2026-11-06,Meta,www.meta.com,corporate,Amazonbot,Amazon,training,no,no,no,webfetch,67b4b83725499e81d6fcb57a8c43a1b969ef5d9825b161383d3a0ac94ee6dc17,https://www.meta.com/robots.txt
2026-11-06,Meta,www.meta.ai,product,GPTBot,OpenAI,training,not observed,not observed,no,not_observed,,https://www.meta.ai/robots.txt
2026-11-06,Meta,www.meta.ai,product,OAI-SearchBot,OpenAI,retrieval,not observed,not observed,no,not_observed,,https://www.meta.ai/robots.txt
2026-11-06,Meta,www.meta.ai,product,ChatGPT-User,OpenAI,retrieval,not observed,not observed,no,not_observed,,https://www.meta.ai/robots.txt
2026-11-06,Meta,www.meta.ai,product,ClaudeBot,Anthropic,training,not observed,not observed,no,not_observed,,https://www.meta.ai/robots.txt
2026-11-06,Meta,www.meta.ai,product,Claude-SearchBot,Anthropic,retrieval,not observed,not observed,no,not_observed,,https://www.meta.ai/robots.txt
2026-11-06,Meta,www.meta.ai,product,Claude-User,Anthropic,retrieval,not observed,not observed,no,not_observed,,https://www.meta.ai/robots.txt
2026-11-06,Meta,www.meta.ai,product,anthropic-ai,Anthropic,training,not observed,not observed,no,not_observed,,https://www.meta.ai/robots.txt
2026-11-06,Meta,www.meta.ai,product,Google-Extended,Google,training,not observed,not observed,no,not_observed,,https://www.meta.ai/robots.txt
2026-11-06,Meta,www.meta.ai,product,PerplexityBot,Perplexity,retrieval,not observed,not observed,no,not_observed,,https://www.meta.ai/robots.txt
2026-11-06,Meta,www.meta.ai,product,Perplexity-User,Perplexity,retrieval,not observed,not observed,no,not_observed,,https://www.meta.ai/robots.txt
2026-11-06,Meta,www.meta.ai,product,CCBot,Common Crawl,training,not observed,not observed,no,not_observed,,https://www.meta.ai/robots.txt
2026-11-06,Meta,www.meta.ai,product,Applebot-Extended,Apple,training,not observed,not observed,no,not_observed,,https://www.meta.ai/robots.txt
2026-11-06,Meta,www.meta.ai,product,Bytespider,ByteDance,training,not observed,not observed,no,not_observed,,https://www.meta.ai/robots.txt
2026-11-06,Meta,www.meta.ai,product,Meta-ExternalAgent,Meta,training,not observed,not observed,yes,not_observed,,https://www.meta.ai/robots.txt
2026-11-06,Meta,www.meta.ai,product,Amazonbot,Amazon,training,not observed,not observed,no,not_observed,,https://www.meta.ai/robots.txt
2026-11-06,Amazon,www.amazon.com,corporate,GPTBot,OpenAI,training,not observed,not observed,no,not_observed,,https://www.amazon.com/robots.txt
2026-11-06,Amazon,www.amazon.com,corporate,OAI-SearchBot,OpenAI,retrieval,not observed,not observed,no,not_observed,,https://www.amazon.com/robots.txt
2026-11-06,Amazon,www.amazon.com,corporate,ChatGPT-User,OpenAI,retrieval,not observed,not observed,no,not_observed,,https://www.amazon.com/robots.txt
2026-11-06,Amazon,www.amazon.com,corporate,ClaudeBot,Anthropic,training,not observed,not observed,no,not_observed,,https://www.amazon.com/robots.txt
2026-11-06,Amazon,www.amazon.com,corporate,Claude-SearchBot,Anthropic,retrieval,not observed,not observed,no,not_observed,,https://www.amazon.com/robots.txt
2026-11-06,Amazon,www.amazon.com,corporate,Claude-User,Anthropic,retrieval,not observed,not observed,no,not_observed,,https://www.amazon.com/robots.txt
2026-11-06,Amazon,www.amazon.com,corporate,anthropic-ai,Anthropic,training,not observed,not observed,no,not_observed,,https://www.amazon.com/robots.txt
2026-11-06,Amazon,www.amazon.com,corporate,Google-Extended,Google,training,not observed,not observed,no,not_observed,,https://www.amazon.com/robots.txt
2026-11-06,Amazon,www.amazon.com,corporate,PerplexityBot,Perplexity,retrieval,not observed,not observed,no,not_observed,,https://www.amazon.com/robots.txt
2026-11-06,Amazon,www.amazon.com,corporate,Perplexity-User,Perplexity,retrieval,not observed,not observed,no,not_observed,,https://www.amazon.com/robots.txt
2026-11-06,Amazon,www.amazon.com,corporate,CCBot,Common Crawl,training,not observed,not observed,no,not_observed,,https://www.amazon.com/robots.txt
2026-11-06,Amazon,www.amazon.com,corporate,Applebot-Extended,Apple,training,not observed,not observed,no,not_observed,,https://www.amazon.com/robots.txt
2026-11-06,Amazon,www.amazon.com,corporate,Bytespider,ByteDance,training,not observed,not observed,no,not_observed,,https://www.amazon.com/robots.txt
2026-11-06,Amazon,www.amazon.com,corporate,Meta-ExternalAgent,Meta,training,not observed,not observed,no,not_observed,,https://www.amazon.com/robots.txt
2026-11-06,Amazon,www.amazon.com,corporate,Amazonbot,Amazon,training,not observed,not observed,yes,not_observed,,https://www.amazon.com/robots.txt
2026-11-06,xAI,x.ai,corporate,GPTBot,OpenAI,training,no,no,no,webfetch,069241f196e6caa2dfe1098fca611483b3be0b5445b18cd5fdd99df0818528be,https://x.ai/robots.txt
2026-11-06,xAI,x.ai,corporate,OAI-SearchBot,OpenAI,retrieval,no,no,no,webfetch,069241f196e6caa2dfe1098fca611483b3be0b5445b18cd5fdd99df0818528be,https://x.ai/robots.txt
2026-11-06,xAI,x.ai,corporate,ChatGPT-User,OpenAI,retrieval,no,no,no,webfetch,069241f196e6caa2dfe1098fca611483b3be0b5445b18cd5fdd99df0818528be,https://x.ai/robots.txt
2026-11-06,xAI,x.ai,corporate,ClaudeBot,Anthropic,training,no,no,no,webfetch,069241f196e6caa2dfe1098fca611483b3be0b5445b18cd5fdd99df0818528be,https://x.ai/robots.txt
2026-11-06,xAI,x.ai,corporate,Claude-SearchBot,Anthropic,retrieval,no,no,no,webfetch,069241f196e6caa2dfe1098fca611483b3be0b5445b18cd5fdd99df0818528be,https://x.ai/robots.txt
2026-11-06,xAI,x.ai,corporate,Claude-User,Anthropic,retrieval,no,no,no,webfetch,069241f196e6caa2dfe1098fca611483b3be0b5445b18cd5fdd99df0818528be,https://x.ai/robots.txt
2026-11-06,xAI,x.ai,corporate,anthropic-ai,Anthropic,training,no,no,no,webfetch,069241f196e6caa2dfe1098fca611483b3be0b5445b18cd5fdd99df0818528be,https://x.ai/robots.txt
2026-11-06,xAI,x.ai,corporate,Google-Extended,Google,training,no,no,no,webfetch,069241f196e6caa2dfe1098fca611483b3be0b5445b18cd5fdd99df0818528be,https://x.ai/robots.txt
2026-11-06,xAI,x.ai,corporate,PerplexityBot,Perplexity,retrieval,no,no,no,webfetch,069241f196e6caa2dfe1098fca611483b3be0b5445b18cd5fdd99df0818528be,https://x.ai/robots.txt
2026-11-06,xAI,x.ai,corporate,Perplexity-User,Perplexity,retrieval,no,no,no,webfetch,069241f196e6caa2dfe1098fca611483b3be0b5445b18cd5fdd99df0818528be,https://x.ai/robots.txt
2026-11-06,xAI,x.ai,corporate,CCBot,Common Crawl,training,no,no,no,webfetch,069241f196e6caa2dfe1098fca611483b3be0b5445b18cd5fdd99df0818528be,https://x.ai/robots.txt
2026-11-06,xAI,x.ai,corporate,Applebot-Extended,Apple,training,no,no,no,webfetch,069241f196e6caa2dfe1098fca611483b3be0b5445b18cd5fdd99df0818528be,https://x.ai/robots.txt
2026-11-06,xAI,x.ai,corporate,Bytespider,ByteDance,training,no,no,no,webfetch,069241f196e6caa2dfe1098fca611483b3be0b5445b18cd5fdd99df0818528be,https://x.ai/robots.txt
2026-11-06,xAI,x.ai,corporate,Meta-ExternalAgent,Meta,training,no,no,no,webfetch,069241f196e6caa2dfe1098fca611483b3be0b5445b18cd5fdd99df0818528be,https://x.ai/robots.txt
2026-11-06,xAI,x.ai,corporate,Amazonbot,Amazon,training,no,no,no,webfetch,069241f196e6caa2dfe1098fca611483b3be0b5445b18cd5fdd99df0818528be,https://x.ai/robots.txt
2026-11-06,xAI,grok.com,product,GPTBot,OpenAI,training,yes,yes,no,webfetch,3f9434e6d97d4363a78dee2f53b72399ae0568addb3154c8fca9c579566aceed,https://grok.com/robots.txt
2026-11-06,xAI,grok.com,product,OAI-SearchBot,OpenAI,retrieval,no,no,no,webfetch,3f9434e6d97d4363a78dee2f53b72399ae0568addb3154c8fca9c579566aceed,https://grok.com/robots.txt
2026-11-06,xAI,grok.com,product,ChatGPT-User,OpenAI,retrieval,yes,yes,no,webfetch,3f9434e6d97d4363a78dee2f53b72399ae0568addb3154c8fca9c579566aceed,https://grok.com/robots.txt
2026-11-06,xAI,grok.com,product,ClaudeBot,Anthropic,training,yes,yes,no,webfetch,3f9434e6d97d4363a78dee2f53b72399ae0568addb3154c8fca9c579566aceed,https://grok.com/robots.txt
2026-11-06,xAI,grok.com,product,Claude-SearchBot,Anthropic,retrieval,no,no,no,webfetch,3f9434e6d97d4363a78dee2f53b72399ae0568addb3154c8fca9c579566aceed,https://grok.com/robots.txt
2026-11-06,xAI,grok.com,product,Claude-User,Anthropic,retrieval,no,no,no,webfetch,3f9434e6d97d4363a78dee2f53b72399ae0568addb3154c8fca9c579566aceed,https://grok.com/robots.txt
2026-11-06,xAI,grok.com,product,anthropic-ai,Anthropic,training,no,no,no,webfetch,3f9434e6d97d4363a78dee2f53b72399ae0568addb3154c8fca9c579566aceed,https://grok.com/robots.txt
2026-11-06,xAI,grok.com,product,Google-Extended,Google,training,yes,yes,no,webfetch,3f9434e6d97d4363a78dee2f53b72399ae0568addb3154c8fca9c579566aceed,https://grok.com/robots.txt
2026-11-06,xAI,grok.com,product,PerplexityBot,Perplexity,retrieval,yes,yes,no,webfetch,3f9434e6d97d4363a78dee2f53b72399ae0568addb3154c8fca9c579566aceed,https://grok.com/robots.txt
2026-11-06,xAI,grok.com,product,Perplexity-User,Perplexity,retrieval,no,no,no,webfetch,3f9434e6d97d4363a78dee2f53b72399ae0568addb3154c8fca9c579566aceed,https://grok.com/robots.txt
2026-11-06,xAI,grok.com,product,CCBot,Common Crawl,training,no,no,no,webfetch,3f9434e6d97d4363a78dee2f53b72399ae0568addb3154c8fca9c579566aceed,https://grok.com/robots.txt
2026-11-06,xAI,grok.com,product,Applebot-Extended,Apple,training,yes,yes,no,webfetch,3f9434e6d97d4363a78dee2f53b72399ae0568addb3154c8fca9c579566aceed,https://grok.com/robots.txt
2026-11-06,xAI,grok.com,product,Bytespider,ByteDance,training,no,no,no,webfetch,3f9434e6d97d4363a78dee2f53b72399ae0568addb3154c8fca9c579566aceed,https://grok.com/robots.txt
2026-11-06,xAI,grok.com,product,Meta-ExternalAgent,Meta,training,no,no,no,webfetch,3f9434e6d97d4363a78dee2f53b72399ae0568addb3154c8fca9c579566aceed,https://grok.com/robots.txt
2026-11-06,xAI,grok.com,product,Amazonbot,Amazon,training,no,no,no,webfetch,3f9434e6d97d4363a78dee2f53b72399ae0568addb3154c8fca9c579566aceed,https://grok.com/robots.txt
2026-11-06,Mistral,mistral.ai,corporate,GPTBot,OpenAI,training,no,no,no,webfetch,e5543d29282e692d17eb88ba69bbaf6076d6707902fd907e240d10e9312e7bfe,https://mistral.ai/robots.txt
2026-11-06,Mistral,mistral.ai,corporate,OAI-SearchBot,OpenAI,retrieval,no,no,no,webfetch,e5543d29282e692d17eb88ba69bbaf6076d6707902fd907e240d10e9312e7bfe,https://mistral.ai/robots.txt
2026-11-06,Mistral,mistral.ai,corporate,ChatGPT-User,OpenAI,retrieval,no,no,no,webfetch,e5543d29282e692d17eb88ba69bbaf6076d6707902fd907e240d10e9312e7bfe,https://mistral.ai/robots.txt
2026-11-06,Mistral,mistral.ai,corporate,ClaudeBot,Anthropic,training,no,no,no,webfetch,e5543d29282e692d17eb88ba69bbaf6076d6707902fd907e240d10e9312e7bfe,https://mistral.ai/robots.txt
2026-11-06,Mistral,mistral.ai,corporate,Claude-SearchBot,Anthropic,retrieval,no,no,no,webfetch,e5543d29282e692d17eb88ba69bbaf6076d6707902fd907e240d10e9312e7bfe,https://mistral.ai/robots.txt
2026-11-06,Mistral,mistral.ai,corporate,Claude-User,Anthropic,retrieval,no,no,no,webfetch,e5543d29282e692d17eb88ba69bbaf6076d6707902fd907e240d10e9312e7bfe,https://mistral.ai/robots.txt
2026-11-06,Mistral,mistral.ai,corporate,anthropic-ai,Anthropic,training,no,no,no,webfetch,e5543d29282e692d17eb88ba69bbaf6076d6707902fd907e240d10e9312e7bfe,https://mistral.ai/robots.txt
2026-11-06,Mistral,mistral.ai,corporate,Google-Extended,Google,training,no,no,no,webfetch,e5543d29282e692d17eb88ba69bbaf6076d6707902fd907e240d10e9312e7bfe,https://mistral.ai/robots.txt
2026-11-06,Mistral,mistral.ai,corporate,PerplexityBot,Perplexity,retrieval,no,no,no,webfetch,e5543d29282e692d17eb88ba69bbaf6076d6707902fd907e240d10e9312e7bfe,https://mistral.ai/robots.txt
2026-11-06,Mistral,mistral.ai,corporate,Perplexity-User,Perplexity,retrieval,no,no,no,webfetch,e5543d29282e692d17eb88ba69bbaf6076d6707902fd907e240d10e9312e7bfe,https://mistral.ai/robots.txt
2026-11-06,Mistral,mistral.ai,corporate,CCBot,Common Crawl,training,no,no,no,webfetch,e5543d29282e692d17eb88ba69bbaf6076d6707902fd907e240d10e9312e7bfe,https://mistral.ai/robots.txt
2026-11-06,Mistral,mistral.ai,corporate,Applebot-Extended,Apple,training,no,no,no,webfetch,e5543d29282e692d17eb88ba69bbaf6076d6707902fd907e240d10e9312e7bfe,https://mistral.ai/robots.txt
2026-11-06,Mistral,mistral.ai,corporate,Bytespider,ByteDance,training,no,no,no,webfetch,e5543d29282e692d17eb88ba69bbaf6076d6707902fd907e240d10e9312e7bfe,https://mistral.ai/robots.txt
2026-11-06,Mistral,mistral.ai,corporate,Meta-ExternalAgent,Meta,training,no,no,no,webfetch,e5543d29282e692d17eb88ba69bbaf6076d6707902fd907e240d10e9312e7bfe,https://mistral.ai/robots.txt
2026-11-06,Mistral,mistral.ai,corporate,Amazonbot,Amazon,training,no,no,no,webfetch,e5543d29282e692d17eb88ba69bbaf6076d6707902fd907e240d10e9312e7bfe,https://mistral.ai/robots.txt
2026-11-06,Hugging Face,huggingface.co,corporate,GPTBot,OpenAI,training,no,no,no,webfetch,045c12bc2b5f1820752a5c149d2c96030ecfb8f32c42dd034cead715be277193,https://huggingface.co/robots.txt
2026-11-06,Hugging Face,huggingface.co,corporate,OAI-SearchBot,OpenAI,retrieval,no,no,no,webfetch,045c12bc2b5f1820752a5c149d2c96030ecfb8f32c42dd034cead715be277193,https://huggingface.co/robots.txt
2026-11-06,Hugging Face,huggingface.co,corporate,ChatGPT-User,OpenAI,retrieval,no,no,no,webfetch,045c12bc2b5f1820752a5c149d2c96030ecfb8f32c42dd034cead715be277193,https://huggingface.co/robots.txt
2026-11-06,Hugging Face,huggingface.co,corporate,ClaudeBot,Anthropic,training,no,no,no,webfetch,045c12bc2b5f1820752a5c149d2c96030ecfb8f32c42dd034cead715be277193,https://huggingface.co/robots.txt
2026-11-06,Hugging Face,huggingface.co,corporate,Claude-SearchBot,Anthropic,retrieval,no,no,no,webfetch,045c12bc2b5f1820752a5c149d2c96030ecfb8f32c42dd034cead715be277193,https://huggingface.co/robots.txt
2026-11-06,Hugging Face,huggingface.co,corporate,Claude-User,Anthropic,retrieval,no,no,no,webfetch,045c12bc2b5f1820752a5c149d2c96030ecfb8f32c42dd034cead715be277193,https://huggingface.co/robots.txt
2026-11-06,Hugging Face,huggingface.co,corporate,anthropic-ai,Anthropic,training,no,no,no,webfetch,045c12bc2b5f1820752a5c149d2c96030ecfb8f32c42dd034cead715be277193,https://huggingface.co/robots.txt
2026-11-06,Hugging Face,huggingface.co,corporate,Google-Extended,Google,training,no,no,no,webfetch,045c12bc2b5f1820752a5c149d2c96030ecfb8f32c42dd034cead715be277193,https://huggingface.co/robots.txt
2026-11-06,Hugging Face,huggingface.co,corporate,PerplexityBot,Perplexity,retrieval,no,no,no,webfetch,045c12bc2b5f1820752a5c149d2c96030ecfb8f32c42dd034cead715be277193,https://huggingface.co/robots.txt
2026-11-06,Hugging Face,huggingface.co,corporate,Perplexity-User,Perplexity,retrieval,no,no,no,webfetch,045c12bc2b5f1820752a5c149d2c96030ecfb8f32c42dd034cead715be277193,https://huggingface.co/robots.txt
2026-11-06,Hugging Face,huggingface.co,corporate,CCBot,Common Crawl,training,no,no,no,webfetch,045c12bc2b5f1820752a5c149d2c96030ecfb8f32c42dd034cead715be277193,https://huggingface.co/robots.txt
2026-11-06,Hugging Face,huggingface.co,corporate,Applebot-Extended,Apple,training,no,no,no,webfetch,045c12bc2b5f1820752a5c149d2c96030ecfb8f32c42dd034cead715be277193,https://huggingface.co/robots.txt
2026-11-06,Hugging Face,huggingface.co,corporate,Bytespider,ByteDance,training,no,no,no,webfetch,045c12bc2b5f1820752a5c149d2c96030ecfb8f32c42dd034cead715be277193,https://huggingface.co/robots.txt
2026-11-06,Hugging Face,huggingface.co,corporate,Meta-ExternalAgent,Meta,training,no,no,no,webfetch,045c12bc2b5f1820752a5c149d2c96030ecfb8f32c42dd034cead715be277193,https://huggingface.co/robots.txt
2026-11-06,Hugging Face,huggingface.co,corporate,Amazonbot,Amazon,training,no,no,no,webfetch,045c12bc2b5f1820752a5c149d2c96030ecfb8f32c42dd034cead715be277193,https://huggingface.co/robots.txt
2026-11-06,DeepSeek,www.deepseek.com,corporate,GPTBot,OpenAI,training,no,no,no,webfetch,df717a9594a99ad8e5cf5e547cbe3839611e975fc61d142c3eb507c4592f0ae7,https://www.deepseek.com/robots.txt
2026-11-06,DeepSeek,www.deepseek.com,corporate,OAI-SearchBot,OpenAI,retrieval,no,no,no,webfetch,df717a9594a99ad8e5cf5e547cbe3839611e975fc61d142c3eb507c4592f0ae7,https://www.deepseek.com/robots.txt
2026-11-06,DeepSeek,www.deepseek.com,corporate,ChatGPT-User,OpenAI,retrieval,no,no,no,webfetch,df717a9594a99ad8e5cf5e547cbe3839611e975fc61d142c3eb507c4592f0ae7,https://www.deepseek.com/robots.txt
2026-11-06,DeepSeek,www.deepseek.com,corporate,ClaudeBot,Anthropic,training,no,no,no,webfetch,df717a9594a99ad8e5cf5e547cbe3839611e975fc61d142c3eb507c4592f0ae7,https://www.deepseek.com/robots.txt
2026-11-06,DeepSeek,www.deepseek.com,corporate,Claude-SearchBot,Anthropic,retrieval,no,no,no,webfetch,df717a9594a99ad8e5cf5e547cbe3839611e975fc61d142c3eb507c4592f0ae7,https://www.deepseek.com/robots.txt
2026-11-06,DeepSeek,www.deepseek.com,corporate,Claude-User,Anthropic,retrieval,no,no,no,webfetch,df717a9594a99ad8e5cf5e547cbe3839611e975fc61d142c3eb507c4592f0ae7,https://www.deepseek.com/robots.txt
2026-11-06,DeepSeek,www.deepseek.com,corporate,anthropic-ai,Anthropic,training,no,no,no,webfetch,df717a9594a99ad8e5cf5e547cbe3839611e975fc61d142c3eb507c4592f0ae7,https://www.deepseek.com/robots.txt
2026-11-06,DeepSeek,www.deepseek.com,corporate,Google-Extended,Google,training,no,no,no,webfetch,df717a9594a99ad8e5cf5e547cbe3839611e975fc61d142c3eb507c4592f0ae7,https://www.deepseek.com/robots.txt
2026-11-06,DeepSeek,www.deepseek.com,corporate,PerplexityBot,Perplexity,retrieval,no,no,no,webfetch,df717a9594a99ad8e5cf5e547cbe3839611e975fc61d142c3eb507c4592f0ae7,https://www.deepseek.com/robots.txt
2026-11-06,DeepSeek,www.deepseek.com,corporate,Perplexity-User,Perplexity,retrieval,no,no,no,webfetch,df717a9594a99ad8e5cf5e547cbe3839611e975fc61d142c3eb507c4592f0ae7,https://www.deepseek.com/robots.txt
2026-11-06,DeepSeek,www.deepseek.com,corporate,CCBot,Common Crawl,training,no,no,no,webfetch,df717a9594a99ad8e5cf5e547cbe3839611e975fc61d142c3eb507c4592f0ae7,https://www.deepseek.com/robots.txt
2026-11-06,DeepSeek,www.deepseek.com,corporate,Applebot-Extended,Apple,training,no,no,no,webfetch,df717a9594a99ad8e5cf5e547cbe3839611e975fc61d142c3eb507c4592f0ae7,https://www.deepseek.com/robots.txt
2026-11-06,DeepSeek,www.deepseek.com,corporate,Bytespider,ByteDance,training,no,no,no,webfetch,df717a9594a99ad8e5cf5e547cbe3839611e975fc61d142c3eb507c4592f0ae7,https://www.deepseek.com/robots.txt
2026-11-06,DeepSeek,www.deepseek.com,corporate,Meta-ExternalAgent,Meta,training,no,no,no,webfetch,df717a9594a99ad8e5cf5e547cbe3839611e975fc61d142c3eb507c4592f0ae7,https://www.deepseek.com/robots.txt
2026-11-06,DeepSeek,www.deepseek.com,corporate,Amazonbot,Amazon,training,no,no,no,webfetch,df717a9594a99ad8e5cf5e547cbe3839611e975fc61d142c3eb507c4592f0ae7,https://www.deepseek.com/robots.txt
2026-11-06,Cohere,cohere.com,corporate,GPTBot,OpenAI,training,no,no,no,webfetch,35ca3084fe515fe54fafdf6a44f6dba7d71a9618cc74c35620bed890dae68e79,https://cohere.com/robots.txt
2026-11-06,Cohere,cohere.com,corporate,OAI-SearchBot,OpenAI,retrieval,no,no,no,webfetch,35ca3084fe515fe54fafdf6a44f6dba7d71a9618cc74c35620bed890dae68e79,https://cohere.com/robots.txt
2026-11-06,Cohere,cohere.com,corporate,ChatGPT-User,OpenAI,retrieval,no,no,no,webfetch,35ca3084fe515fe54fafdf6a44f6dba7d71a9618cc74c35620bed890dae68e79,https://cohere.com/robots.txt
2026-11-06,Cohere,cohere.com,corporate,ClaudeBot,Anthropic,training,no,no,no,webfetch,35ca3084fe515fe54fafdf6a44f6dba7d71a9618cc74c35620bed890dae68e79,https://cohere.com/robots.txt
2026-11-06,Cohere,cohere.com,corporate,Claude-SearchBot,Anthropic,retrieval,no,no,no,webfetch,35ca3084fe515fe54fafdf6a44f6dba7d71a9618cc74c35620bed890dae68e79,https://cohere.com/robots.txt
2026-11-06,Cohere,cohere.com,corporate,Claude-User,Anthropic,retrieval,no,no,no,webfetch,35ca3084fe515fe54fafdf6a44f6dba7d71a9618cc74c35620bed890dae68e79,https://cohere.com/robots.txt
2026-11-06,Cohere,cohere.com,corporate,anthropic-ai,Anthropic,training,no,no,no,webfetch,35ca3084fe515fe54fafdf6a44f6dba7d71a9618cc74c35620bed890dae68e79,https://cohere.com/robots.txt
2026-11-06,Cohere,cohere.com,corporate,Google-Extended,Google,training,no,no,no,webfetch,35ca3084fe515fe54fafdf6a44f6dba7d71a9618cc74c35620bed890dae68e79,https://cohere.com/robots.txt
2026-11-06,Cohere,cohere.com,corporate,PerplexityBot,Perplexity,retrieval,no,no,no,webfetch,35ca3084fe515fe54fafdf6a44f6dba7d71a9618cc74c35620bed890dae68e79,https://cohere.com/robots.txt
2026-11-06,Cohere,cohere.com,corporate,Perplexity-User,Perplexity,retrieval,no,no,no,webfetch,35ca3084fe515fe54fafdf6a44f6dba7d71a9618cc74c35620bed890dae68e79,https://cohere.com/robots.txt
2026-11-06,Cohere,cohere.com,corporate,CCBot,Common Crawl,training,no,no,no,webfetch,35ca3084fe515fe54fafdf6a44f6dba7d71a9618cc74c35620bed890dae68e79,https://cohere.com/robots.txt
2026-11-06,Cohere,cohere.com,corporate,Applebot-Extended,Apple,training,no,no,no,webfetch,35ca3084fe515fe54fafdf6a44f6dba7d71a9618cc74c35620bed890dae68e79,https://cohere.com/robots.txt
2026-11-06,Cohere,cohere.com,corporate,Bytespider,ByteDance,training,no,no,no,webfetch,35ca3084fe515fe54fafdf6a44f6dba7d71a9618cc74c35620bed890dae68e79,https://cohere.com/robots.txt
2026-11-06,Cohere,cohere.com,corporate,Meta-ExternalAgent,Meta,training,no,no,no,webfetch,35ca3084fe515fe54fafdf6a44f6dba7d71a9618cc74c35620bed890dae68e79,https://cohere.com/robots.txt
2026-11-06,Cohere,cohere.com,corporate,Amazonbot,Amazon,training,no,no,no,webfetch,35ca3084fe515fe54fafdf6a44f6dba7d71a9618cc74c35620bed890dae68e79,https://cohere.com/robots.txt
2026-11-06,Microsoft,copilot.microsoft.com,product,GPTBot,OpenAI,training,no,no,no,webfetch,2c2037c30f51350413083c9a88b1177b8281c7adf296048e1561d7c31e4c8a77,https://copilot.microsoft.com/robots.txt
2026-11-06,Microsoft,copilot.microsoft.com,product,OAI-SearchBot,OpenAI,retrieval,no,no,no,webfetch,2c2037c30f51350413083c9a88b1177b8281c7adf296048e1561d7c31e4c8a77,https://copilot.microsoft.com/robots.txt
2026-11-06,Microsoft,copilot.microsoft.com,product,ChatGPT-User,OpenAI,retrieval,no,no,no,webfetch,2c2037c30f51350413083c9a88b1177b8281c7adf296048e1561d7c31e4c8a77,https://copilot.microsoft.com/robots.txt
2026-11-06,Microsoft,copilot.microsoft.com,product,ClaudeBot,Anthropic,training,no,no,no,webfetch,2c2037c30f51350413083c9a88b1177b8281c7adf296048e1561d7c31e4c8a77,https://copilot.microsoft.com/robots.txt
2026-11-06,Microsoft,copilot.microsoft.com,product,Claude-SearchBot,Anthropic,retrieval,no,no,no,webfetch,2c2037c30f51350413083c9a88b1177b8281c7adf296048e1561d7c31e4c8a77,https://copilot.microsoft.com/robots.txt
2026-11-06,Microsoft,copilot.microsoft.com,product,Claude-User,Anthropic,retrieval,no,no,no,webfetch,2c2037c30f51350413083c9a88b1177b8281c7adf296048e1561d7c31e4c8a77,https://copilot.microsoft.com/robots.txt
2026-11-06,Microsoft,copilot.microsoft.com,product,anthropic-ai,Anthropic,training,no,no,no,webfetch,2c2037c30f51350413083c9a88b1177b8281c7adf296048e1561d7c31e4c8a77,https://copilot.microsoft.com/robots.txt
2026-11-06,Microsoft,copilot.microsoft.com,product,Google-Extended,Google,training,no,no,no,webfetch,2c2037c30f51350413083c9a88b1177b8281c7adf296048e1561d7c31e4c8a77,https://copilot.microsoft.com/robots.txt
2026-11-06,Microsoft,copilot.microsoft.com,product,PerplexityBot,Perplexity,retrieval,no,no,no,webfetch,2c2037c30f51350413083c9a88b1177b8281c7adf296048e1561d7c31e4c8a77,https://copilot.microsoft.com/robots.txt
2026-11-06,Microsoft,copilot.microsoft.com,product,Perplexity-User,Perplexity,retrieval,no,no,no,webfetch,2c2037c30f51350413083c9a88b1177b8281c7adf296048e1561d7c31e4c8a77,https://copilot.microsoft.com/robots.txt
2026-11-06,Microsoft,copilot.microsoft.com,product,CCBot,Common Crawl,training,no,no,no,webfetch,2c2037c30f51350413083c9a88b1177b8281c7adf296048e1561d7c31e4c8a77,https://copilot.microsoft.com/robots.txt
2026-11-06,Microsoft,copilot.microsoft.com,product,Applebot-Extended,Apple,training,no,no,no,webfetch,2c2037c30f51350413083c9a88b1177b8281c7adf296048e1561d7c31e4c8a77,https://copilot.microsoft.com/robots.txt
2026-11-06,Microsoft,copilot.microsoft.com,product,Bytespider,ByteDance,training,no,no,no,webfetch,2c2037c30f51350413083c9a88b1177b8281c7adf296048e1561d7c31e4c8a77,https://copilot.microsoft.com/robots.txt
2026-11-06,Microsoft,copilot.microsoft.com,product,Meta-ExternalAgent,Meta,training,no,no,no,webfetch,2c2037c30f51350413083c9a88b1177b8281c7adf296048e1561d7c31e4c8a77,https://copilot.microsoft.com/robots.txt
2026-11-06,Microsoft,copilot.microsoft.com,product,Amazonbot,Amazon,training,no,no,no,webfetch,2c2037c30f51350413083c9a88b1177b8281c7adf296048e1561d7c31e4c8a77,https://copilot.microsoft.com/robots.txt