{
  "name": "Search and AI crawler registry",
  "description": "Search engine and AI crawlers and robots.txt tokens that matter to business websites: the company that runs each, whether it serves search, AI search, user-requested fetching or model training, how to verify a visit (reverse DNS or published IP ranges) and the official documentation link.",
  "url": "https://poseidondssolutions.com/datasets#ai-crawlers",
  "publisher": "Poseidon DS",
  "last_reviewed": "2026-11-17",
  "license": "https://creativecommons.org/licenses/by/4.0/",
  "sources": [
    {
      "title": "OpenAI — Overview of OpenAI crawlers",
      "url": "https://developers.openai.com/api/docs/bots"
    },
    {
      "title": "Anthropic — Does Anthropic crawl data from the web?",
      "url": "https://support.claude.com/en/articles/8896518"
    },
    {
      "title": "Perplexity — Perplexity crawlers",
      "url": "https://docs.perplexity.ai/guides/bots"
    },
    {
      "title": "Apple — About Applebot",
      "url": "https://support.apple.com/en-us/119829"
    },
    {
      "title": "RFC 9309 — Robots Exclusion Protocol",
      "url": "https://www.rfc-editor.org/rfc/rfc9309"
    }
  ],
  "records": 14,
  "data": [
    {
      "token": "Googlebot",
      "operator": "Google",
      "role": "Search engine",
      "role_key": "search",
      "purpose": "Google Search, including AI Overviews and AI Mode",
      "visits_site": true,
      "verification": "Reverse DNS name ends with googlebot.com or google.com",
      "documentation": "https://developers.google.com/search/docs/crawling-indexing/verifying-googlebot"
    },
    {
      "token": "Bingbot",
      "operator": "Microsoft",
      "role": "Search engine",
      "role_key": "search",
      "purpose": "Bing Search and Microsoft Copilot answers",
      "visits_site": true,
      "verification": "Reverse DNS name ends with search.msn.com",
      "documentation": "https://www.bing.com/webmasters/help/how-to-verify-bingbot-3905dc26"
    },
    {
      "token": "Applebot",
      "operator": "Apple",
      "role": "Search engine",
      "role_key": "search",
      "purpose": "Siri, Spotlight and Safari suggestions",
      "visits_site": true,
      "verification": "Reverse DNS name ends with applebot.apple.com",
      "documentation": "https://support.apple.com/en-us/119829"
    },
    {
      "token": "OAI-SearchBot",
      "operator": "OpenAI",
      "role": "AI search",
      "role_key": "ai_search",
      "purpose": "ChatGPT search results and citations",
      "visits_site": true,
      "verification": "Published IP list: https://openai.com/searchbot.json",
      "documentation": "https://developers.openai.com/api/docs/bots"
    },
    {
      "token": "ChatGPT-User",
      "operator": "OpenAI",
      "role": "AI assistant fetch",
      "role_key": "ai_user",
      "purpose": "Pages a ChatGPT user asks it to open",
      "visits_site": true,
      "verification": "Published IP list: https://openai.com/chatgpt-user.json",
      "documentation": "https://developers.openai.com/api/docs/bots"
    },
    {
      "token": "GPTBot",
      "operator": "OpenAI",
      "role": "AI training",
      "role_key": "training",
      "purpose": "Content that may train OpenAI models",
      "visits_site": true,
      "verification": "Published IP list: https://openai.com/gptbot.json",
      "documentation": "https://developers.openai.com/api/docs/bots"
    },
    {
      "token": "Claude-SearchBot",
      "operator": "Anthropic",
      "role": "AI search",
      "role_key": "ai_search",
      "purpose": "Claude search result quality",
      "visits_site": true,
      "verification": "Published IP list: https://claude.com/crawling/bots.json",
      "documentation": "https://support.claude.com/en/articles/8896518"
    },
    {
      "token": "Claude-User",
      "operator": "Anthropic",
      "role": "AI assistant fetch",
      "role_key": "ai_user",
      "purpose": "Pages a Claude user asks it to read",
      "visits_site": true,
      "verification": "Published IP list: https://claude.com/crawling/bots.json",
      "documentation": "https://support.claude.com/en/articles/8896518"
    },
    {
      "token": "ClaudeBot",
      "operator": "Anthropic",
      "role": "AI training",
      "role_key": "training",
      "purpose": "Content that may train Anthropic models",
      "visits_site": true,
      "verification": "Published IP list: https://claude.com/crawling/bots.json",
      "documentation": "https://support.claude.com/en/articles/8896518"
    },
    {
      "token": "PerplexityBot",
      "operator": "Perplexity",
      "role": "AI search",
      "role_key": "ai_search",
      "purpose": "Perplexity search results",
      "visits_site": true,
      "verification": "Published IP list: https://www.perplexity.com/perplexitybot.json",
      "documentation": "https://docs.perplexity.ai/guides/bots"
    },
    {
      "token": "Perplexity-User",
      "operator": "Perplexity",
      "role": "AI assistant fetch",
      "role_key": "ai_user",
      "purpose": "Pages a Perplexity user asks it to open (Perplexity says this fetcher generally does not follow robots.txt)",
      "visits_site": true,
      "verification": "Published IP list: https://www.perplexity.com/perplexity-user.json",
      "documentation": "https://docs.perplexity.ai/guides/bots"
    },
    {
      "token": "Google-Extended",
      "operator": null,
      "role": "AI training (robots.txt token only)",
      "role_key": "training_token",
      "purpose": "Gemini training and grounding uses (does not affect Google Search)",
      "visits_site": false,
      "verification": "Not applicable — a rule, not a visitor",
      "documentation": null
    },
    {
      "token": "Applebot-Extended",
      "operator": null,
      "role": "AI training (robots.txt token only)",
      "role_key": "training_token",
      "purpose": "Apple foundation-model training",
      "visits_site": false,
      "verification": "Not applicable — a rule, not a visitor",
      "documentation": null
    },
    {
      "token": "__usage_ai-training",
      "operator": null,
      "role": "AI training (robots.txt token only)",
      "role_key": "training_token",
      "purpose": "Shared AI-training preference (IETF draft, Oct 2026; providers are phasing in support)",
      "visits_site": false,
      "verification": "Not applicable — a rule, not a visitor",
      "documentation": null
    }
  ]
}