AI companies' robots.txt files, as read on 6 November 2026, 15:12 to 15:25 UK.
Collected for https://jacobwhitmore.com/ai-companies-robots-txt/
Files read through a web fetch tool are text as that tool returned it (lower confidence); see the collection column in the CSV.

========================================================================
https://chatgpt.com/robots.txt
collection: Chrome, SHA-256 checked in page
sha256 of this copy: 6abd8db88c9da1d617588b69bfc00c637bcb0d1e4a7b71a9d5f26e3c3ea78df2
========================================================================
# https://www.robotstxt.org/robotstxt.html

User-agent: CCBot
Disallow: /

User-agent: img2dataset
Disallow: /

User-agent: Google-Extended
Disallow: /

# Prevent shared pages and GPTs from being used as Google Ads destinations
User-agent: AdsBot-Google
User-agent: AdsBot-Google-Mobile
Disallow: /s/
Disallow: /g/

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: Omgili
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: magpie-crawler
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: Perplexity‑User
Disallow: /

# General rules for all other bots
User-agent: *
# Allow Googlebot to render the indexable homepage; production CSS and JavaScript live here.
Allow: /unauth-mweb/assets/

# Place allows first to avoid bots skipping after Disallow: /
# Allow exactly the homepage
Allow: /$
# Allow the homepage with any query parameters
Allow: /?*
Allow: /api/share/og/
Allow: /g/
Allow: /s/
Allow: /gg/v/
Allow: /m/
Allow: /share/
Allow: /maps$
Allow: /maps/
Allow: /*/maps$
Allow: /*/maps/
Allow: /canvas/shared/
Allow: /*/images
Allow: /images
Allow: /*/library
Allow: /library
Allow: /favicon.ico
Allow: /assets/favicon
Allow: /cdn/assets/favicon
Allow: /cdn/assets/
Allow: /auth/
Allow: /gpts$
Allow: /codex
Allow: /*/codex
Allow: /search$
Allow: /backend-anon/
Allow: /public-api/
Allow: /sitemap.xml
Allow: /marketing-sitemap.xml
Allow: /writing-tools-sitemap.xml
Allow: /acquisition-landing-pages-sitemap.xml
Allow: /100chats
Allow: /api/public_content/
Allow: /backend-api/public_content/
Allow: /?ref=dotcom
# Static Landing Pages
Allow: /overview
Allow: /*/overview
Allow: /features
Allow: /*/features
Allow: /health$
Allow: /health/$
Allow: /*/health$
Allow: /*/health/$
Allow: /voice$
Allow: /voice/$
Allow: /*/voice$
Allow: /*/voice/$
Allow: /apps
Allow: /*/apps
Allow: /finances
Allow: /*/finances
Allow: /shopping
Allow: /*/shopping
Allow: /use-cases
Allow: /*/use-cases
Allow: /learn
Allow: /*/learn
Allow: /business
Allow: /*/business
Allow: /pricing
Allow: /*/pricing
Allow: /download
Allow: /*/download
Allow: /students
Allow: /students/2026
Allow: /*/college-students
Allow: /college-students
Allow: /contact-sales
Allow: /*/contact-sales
Allow: /100chats-project
Allow: /*/100chats-project
Allow: /merchants
Allow: /*/merchants
Allow: /parent-resources
Allow: /*/parent-resources
Allow: /plans
Allow: /*/plans
Allow: /plugins
Allow: /translate
Allow: /*/translate
Allow: /writing
Allow: /*/writing
Allow: /ask/
Allow: /futures
Allow: /*/futures
Allow: /football
Allow: /*/football
Allow: /remote
Allow: /*/remote
Allow: /work
Allow: /*/work
Allow: /dots
Allow: /*/dots
Allow: /sites
Allow: /*/sites
Allow: /space
Allow: /*/space
Allow: /import-to-chatgpt
Allow: /*/import-to-chatgpt
# Exact locale specific homepages
Allow: /am/$
Allow: /ar/$
Allow: /bg-BG/$
Allow: /bn-BD/$
Allow: /bs-BA/$
Allow: /ca-ES/$
Allow: /cs-CZ/$
Allow: /da-DK/$
Allow: /de-DE/$
Allow: /el-GR/$
Allow: /es-ES/$
Allow: /es-419/$
Allow: /et-EE/$
Allow: /fa-IR/$
Allow: /fi-FI/$
Allow: /fr-FR/$
Allow: /fr-CA/$
Allow: /gu-IN/$
Allow: /he-IL/$
Allow: /hi-IN/$
Allow: /hr-HR/$
Allow: /hu-HU/$
Allow: /hy-AM/$
Allow: /id-ID/$
Allow: /is-IS/$
Allow: /it-IT/$
Allow: /ja-JP/$
Allow: /ka-GE/$
Allow: /kk/$
Allow: /kn-IN/$
Allow: /ko-KR/$
Allow: /lt/$
Allow: /lv-LV/$
Allow: /mk-MK/$
Allow: /ml/$
Allow: /mn/$
Allow: /mr-IN/$
Allow: /ms-MY/$
Allow: /my-MM/$
Allow: /nb-NO/$
Allow: /nl-NL/$
Allow: /pa/$
Allow: /pl-PL/$
Allow: /pt-BR/$
Allow: /pt-PT/$
Allow: /ro-RO/$
Allow: /ru-RU/$
Allow: /sk-SK/$
Allow: /sl-SI/$
Allow: /so-SO/$
Allow: /sq-AL/$
Allow: /sr-RS/$
Allow: /sv-SE/$
Allow: /sw-TZ/$
Allow: /ta-IN/$
Allow: /te-IN/$
Allow: /th-TH/$
Allow: /tl/$
Allow: /tr-TR/$
Allow: /uk-UA/$
Allow: /ur/$
Allow: /vi-VN/$
Allow: /zh-CN/$
Allow: /zh-TW/$
Allow: /zh-HK/$

# Now block everything else
Disallow: /

# Specific disallows (redundant for some bots, but still useful for those that respect precedence)
Disallow: /auth/logout
Disallow: /auth/login?*
Disallow: /backend-anon/sentinel/*
Disallow: /backend-anon/conversation$
Disallow: /account-link/*

Sitemap: https://chatgpt.com/sitemap.xml
Sitemap: https://chatgpt.com/marketing-sitemap.xml
Sitemap: https://chatgpt.com/writing-tools-sitemap.xml
Sitemap: https://chatgpt.com/acquisition-landing-pages-sitemap.xml

========================================================================
https://claude.ai/robots.txt
collection: curl, raw bytes
sha256 of this copy: 75ccbccac0e02fbc11965050f3152c1ef7435fff085f0eee27592a82059b8902
========================================================================
User-Agent: ia_archiver
Allow: /$
Allow: /login
Disallow: /

User-Agent: GPTBot
User-Agent: OAI-SearchBot
User-Agent: ChatGPT-User
User-Agent: Google-Extended
Disallow: /

User-Agent: *
Allow: /api/desktop/
Disallow: /api/*
Disallow: /lti/*

Sitemap: https://claude.ai/sitemap.xml

========================================================================
https://cohere.com/robots.txt
collection: web fetch tool (processed text)
sha256 of this copy: 35ca3084fe515fe54fafdf6a44f6dba7d71a9618cc74c35620bed890dae68e79
========================================================================
User-Agent: *
Allow: /
Disallow: /helloworld
Disallow: /project-pursue
Disallow: /labormap
Disallow: /mcp-landscape
Disallow: /mcp-expertise
Disallow: /studio
Disallow: /studio-cohere-labs
Disallow: /studio-mops
Disallow: /studio-blog
Disallow: /studio-campaign
Disallow: /studio-events
Disallow: /studio-legal

Sitemap: https://cohere.com/sitemap.xml

========================================================================
https://commoncrawl.org/robots.txt
collection: web fetch tool (processed text)
sha256 of this copy: 56f568d5d0caa2e717efd02bd2eb762ac354a7aabbe8c7fc05aa5711fe9c836a
========================================================================
# We like well-behaved bots!
# [ASCII-art comment block omitted in this copy; comment lines only]

User-agent: IndeedBot
Disallow: /

User-agent: Indeedshopbot
Disallow: /

User-agent: *
Disallow: /search?*

Sitemap: https://commoncrawl.org/sitemap.xml

========================================================================
https://copilot.microsoft.com/robots.txt
collection: web fetch tool (processed text)
sha256 of this copy: 2c2037c30f51350413083c9a88b1177b8281c7adf296048e1561d7c31e4c8a77
========================================================================
User-agent: *
Allow: /
Disallow: /chats/*

========================================================================
https://gemini.google.com/robots.txt
collection: web fetch tool (processed text)
sha256 of this copy: 6424e2fde98e589b787def8916d42c766d866a1db9dcc0503e49d6e204ea1f71
========================================================================
User-agent: *
Allow: /app/download
Disallow: /app/
Disallow: /chat/

Sitemap: https://gemini.google.com/sitemap.xml

========================================================================
https://grok.com/robots.txt
collection: web fetch tool (processed text)
sha256 of this copy: 3f9434e6d97d4363a78dee2f53b72399ae0568addb3154c8fca9c579566aceed
========================================================================
User-Agent: *
Allow: /
Allow: /chat$
Allow: /chat/$
Allow: /c$
Allow: /c/$
Disallow: /rest
Disallow: /share-links
Disallow: /c/
Disallow: /chat/
Disallow: /project/*
Disallow: /_worker/*

User-Agent: GPTBot
User-Agent: ChatGPT-User
User-Agent: PerplexityBot
User-Agent: ClaudeBot
User-Agent: Google-Extended
User-Agent: Applebot-Extended
Allow: /$
Allow: /chat$
Allow: /imagine
Allow: /plans$
Allow: /plans/business$
Allow: /project$
Allow: /automations$
Allow: /supergrok/imagine$
Disallow: /

Sitemap: https://grok.com/sitemap.xml

========================================================================
https://huggingface.co/robots.txt
collection: web fetch tool (processed text)
sha256 of this copy: 045c12bc2b5f1820752a5c149d2c96030ecfb8f32c42dd034cead715be277193
========================================================================
User-agent: *
Allow: /

Sitemap: https://huggingface.co/sitemap.xml

========================================================================
https://mistral.ai/robots.txt
collection: web fetch tool (processed text)
sha256 of this copy: e5543d29282e692d17eb88ba69bbaf6076d6707902fd907e240d10e9312e7bfe
========================================================================
User-agent: *
Allow: /

Sitemap: https://mistral.ai/sitemap-index.xml

========================================================================
https://openai.com/robots.txt
collection: Chrome, SHA-256 checked in page
sha256 of this copy: c53100563b9d289efeebb6f826a20c566578e1337e3096e298eb9da5ecc2e498
========================================================================
User-agent: *
Allow: /
Disallow: /microsoft-for-startups/

Sitemap: https://openai.com/sitemap.xml

========================================================================
https://www.anthropic.com/robots.txt
collection: web fetch tool (processed text)
sha256 of this copy: b8bb03567cf862a9024bd464bd4bd7afaf054b1029de79727b2ba94711480ec7
========================================================================
User-Agent: *
Allow: /

Sitemap: https://www.anthropic.com/sitemap.xml

========================================================================
https://www.apple.com/robots.txt
collection: Chrome, SHA-256 checked in page
sha256 of this copy: 3115f45c98cfa19341978dcab5dfbc0fd3e0ff4eb3ab1fe8192c3fbf7227dee7
========================================================================
# robots.txt for https://www.apple.com/

User-agent: *
Disallow: /*shop/browse/overlay/*
Disallow: /*shop/iphone/payments/overlay/*
Disallow: /cn/*/aow/*
Disallow: /tmall*
Allow: /ac/globalnav/2.0/*/images/ac-globalnav/globalnav/search/* 	

User-agent: Baiduspider
Disallow: /*
Allow: /cn/*
Allow: /cn-edu/*
Allow: /cn-k12/*

User-agent: HaoSouSpider
Disallow: /*
Allow: /cn/*
Allow: /cn-edu/*
Allow: /cn-k12/*

User-agent: Sogou web spider
Disallow: /*
Allow: /cn/*
Allow: /cn-k12/*

User-agent: Sogou inst spider
Disallow: /*
Allow: /cn/*
Allow: /cn-k12/*

User-agent: Sogou spider2
Disallow: /*
Allow: /cn/*
Allow: /cn-k12/*

#DaumWebMasterTool:6ff40e0f09564a9d4e3bc80e7cbde7c196677a2f5f610eb9c029836774f9a8e9:D9a8IEFAzmC/NWsQ3tB0Wg==

Sitemap: https://www.apple.com/shop/sitemap.xml
Sitemap: https://www.apple.com/autopush/sitemap/sitemap-index.xml
Sitemap: https://www.apple.com/newsroom/sitemap.xml
Sitemap: https://www.apple.com/retail/sitemap/sitemap.xml
Sitemap: https://www.apple.com/today/sitemap-index.xml

========================================================================
https://www.bytedance.com/robots.txt
collection: web fetch tool (processed text)
sha256 of this copy: 7391667bbb4fae2e686ee472216c54d28a7c4894c95ca17e0bdf91efb93197e9
========================================================================
User-agent: *
Allow: /
Sitemap: https://www.bytedance.com/sitemap.xml

========================================================================
https://www.deepseek.com/robots.txt
collection: web fetch tool (processed text)
sha256 of this copy: df717a9594a99ad8e5cf5e547cbe3839611e975fc61d142c3eb507c4592f0ae7
========================================================================
User-Agent: *
Allow: /

Sitemap: https://www.deepseek.com/sitemap.xml

========================================================================
https://www.meta.com/robots.txt
collection: web fetch tool (processed text)
sha256 of this copy: 67b4b83725499e81d6fcb57a8c43a1b969ef5d9825b161383d3a0ac94ee6dc17
========================================================================
# Notice: Collection of data on Facebook through automated means is
# prohibited unless you have express written permission from Facebook
# and may only be conducted for the limited purpose contained in said
# permission.
# See: http://www.facebook.com/apps/site_scraping_tos_terms.php

User-agent: facebookexternalhit
Allow: *

User-agent: meta-externalads
Allow: *

User-agent: *
Disallow: /*cursor=
Disallow: /*fb_comment_id=
Disallow: /ajax/
Disallow: /tealium/
Disallow: /intern/
Disallow: /internal/
Disallow: /login/
Disallow: /oidc/callback/
Disallow: /*.php
Disallow: */search/
Disallow: *help/support/
Disallow: *help_app/
Disallow: */rma/*
Disallow: /a/fl?
Disallow: /bv/upload/
Disallow: /campaign/
Disallow: /*?cjevent=
Disallow: /*&cjevent=
Disallow: /common/referer_frame.php
Disallow: /connect/invite/
Disallow: /device-view/
Disallow: /*/device-view/
Disallow: /meta-employee-store/
Disallow: /order/
Disallow: /return/
Disallow: /s/
Disallow: /survey/
Disallow: /my/
Disallow: /rx/
Disallow: /cart/
Disallow: /bag/

User-agent: PetalBot
Disallow: /

User-agent: Scrapy
Disallow: /

User-agent: uptimerobot
Disallow: /

User-agent: viberbot
Disallow: /

User-agent: YaK
Disallow: /

User-agent: Yandex
Disallow: /

User-agent: Yeti
Disallow: /

Sitemap: https://www.meta.com/sitemap/www_meta_com_sitemap.xml.gz
Sitemap: https://www.meta.com/sitemap/www_meta_com_help_sitemap.xml.gz
Sitemap: https://www.meta.com/sitemap/www_meta_com_blog_sitemap.xml.gz
Sitemap: https://www.meta.com/sitemap/www_meta_com_experiences_sitemap.xml.gz
Sitemap: https://www.meta.com/sitemap/www_meta_com_about_sitemap.xml.gz
Sitemap: https://www.meta.com/sitemap/www_meta_com_blog_news_sitemap.xml.gz
Sitemap: https://www.meta.com/sitemap/www_meta_com_sitemap_zh_tw.xml.gz
Sitemap: https://www.meta.com/sitemap/www_meta_com_sitemap_de_de.xml.gz
Sitemap: https://www.meta.com/sitemap/www_meta_com_sitemap_es_es.xml.gz
Sitemap: https://www.meta.com/sitemap/www_meta_com_sitemap_pl_pl.xml.gz
Sitemap: https://www.meta.com/sitemap/www_meta_com_sitemap_da_dk.xml.gz
Sitemap: https://www.meta.com/sitemap/www_meta_com_sitemap_de_at.xml.gz
Sitemap: https://www.meta.com/sitemap/www_meta_com_sitemap_de_ch.xml.gz
Sitemap: https://www.meta.com/sitemap/www_meta_com_sitemap_en_au.xml.gz
Sitemap: https://www.meta.com/sitemap/www_meta_com_sitemap_en_ca.xml.gz
Sitemap: https://www.meta.com/sitemap/www_meta_com_sitemap_en_ie.xml.gz
Sitemap: https://www.meta.com/sitemap/www_meta_com_sitemap_en_nz.xml.gz
Sitemap: https://www.meta.com/sitemap/www_meta_com_sitemap_en_sg.xml.gz
Sitemap: https://www.meta.com/sitemap/www_meta_com_sitemap_fi_fi.xml.gz
Sitemap: https://www.meta.com/sitemap/www_meta_com_sitemap_fr_be.xml.gz
Sitemap: https://www.meta.com/sitemap/www_meta_com_sitemap_fr_ca.xml.gz
Sitemap: https://www.meta.com/sitemap/www_meta_com_sitemap_fr_ch.xml.gz
Sitemap: https://www.meta.com/sitemap/www_meta_com_sitemap_fr_fr.xml.gz
Sitemap: https://www.meta.com/sitemap/www_meta_com_sitemap_it_ch.xml.gz
Sitemap: https://www.meta.com/sitemap/www_meta_com_sitemap_ko_kr.xml.gz
Sitemap: https://www.meta.com/sitemap/www_meta_com_sitemap_nl_be.xml.gz
Sitemap: https://www.meta.com/sitemap/www_meta_com_sitemap_sv_se.xml.gz
Sitemap: https://www.meta.com/sitemap/www_meta_com_sitemap_it_it.xml.gz
Sitemap: https://www.meta.com/sitemap/www_meta_com_sitemap_en_gb.xml.gz
Sitemap: https://www.meta.com/sitemap/www_meta_com_sitemap_nl_nl.xml.gz
Sitemap: https://www.meta.com/sitemap/www_meta_com_sitemap_ja_jp.xml.gz

========================================================================
https://www.perplexity.ai/robots.txt
collection: Chrome, SHA-256 checked in page
sha256 of this copy: 33f5967c54d4934e2250adb72f47d9187855f36d5816c074cb550ee3e7116256
========================================================================
# https://www.robotstxt.org/robotstxt.html
User-agent: *
Disallow: /*?*q=
Disallow: /search/new
Disallow: /search?*/
Disallow: /marketing/prerelease/
Disallow: /onboarding/
Disallow: /join/
User-agent: Googlebot
Disallow: /*?*q=
Disallow: /search*
Disallow: /search?*/
Disallow: /marketing/prerelease/
Disallow: /onboarding/
Disallow: /join/
Sitemap: https://www.perplexity.ai/sitemap.xml
Sitemap: https://www.perplexity.ai/help-center/sitemap.xml

========================================================================
https://x.ai/robots.txt
collection: web fetch tool (processed text)
sha256 of this copy: 069241f196e6caa2dfe1098fca611483b3be0b5445b18cd5fdd99df0818528be
========================================================================
User-agent: *
Allow: /
Disallow: /tools/

Sitemap: https://x.ai/sitemap.xml

# Content Signals — declare AI/search usage preferences
# See: https://contentsignals.org/
Content-Signal: ai-train=yes, search=yes, ai-input=yes