{
  "name": "AI bot reference list",
  "version": "2026-10-18",
  "license": "CC BY 4.0 (https://creativecommons.org/licenses/by/4.0/)",
  "licenseScope": "Our compilation and descriptions. Bot names, user-agent tokens and the linked official pages belong to their operators and are not covered.",
  "attribution": "Crawlerpeek (https://crawlerpeek.com/bots), with a link to https://crawlerpeek.com/data/ai-bots",
  "cite": "Crawlerpeek. (2026). AI bot reference list: operators, purposes, user agents and robots.txt tokens (Version 2026-10-18) [Data set]. https://crawlerpeek.com/data/ai-bots",
  "count": 24,
  "bots": [
    {
      "slug": "gptbot",
      "name": "GPTBot",
      "operator": "OpenAI",
      "purpose": "ai_training",
      "category": "AI training crawler",
      "robotsToken": "GPTBot",
      "userAgentContains": "gptbot",
      "honoursRobotsTxt": "Yes, per OpenAI",
      "officialSource": "https://developers.openai.com/api/docs/bots",
      "ipListUrl": "https://openai.com/gptbot.json",
      "lastChecked": "2026-10-18",
      "page": "https://crawlerpeek.com/bots/gptbot"
    },
    {
      "slug": "oai-searchbot",
      "name": "OAI-SearchBot",
      "operator": "OpenAI",
      "purpose": "ai_search",
      "category": "AI search crawler",
      "robotsToken": "OAI-SearchBot",
      "userAgentContains": "oai-searchbot",
      "honoursRobotsTxt": "Yes, per OpenAI",
      "officialSource": "https://developers.openai.com/api/docs/bots",
      "ipListUrl": "https://openai.com/searchbot.json",
      "lastChecked": "2026-10-18",
      "page": "https://crawlerpeek.com/bots/oai-searchbot"
    },
    {
      "slug": "chatgpt-user",
      "name": "ChatGPT-User",
      "operator": "OpenAI",
      "purpose": "ai_user_request",
      "category": "AI user-triggered fetcher",
      "robotsToken": "ChatGPT-User",
      "userAgentContains": "chatgpt-user",
      "honoursRobotsTxt": "May not apply (user-initiated)",
      "officialSource": "https://developers.openai.com/api/docs/bots",
      "ipListUrl": "https://openai.com/chatgpt-user.json",
      "lastChecked": "2026-10-18",
      "page": "https://crawlerpeek.com/bots/chatgpt-user"
    },
    {
      "slug": "claudebot",
      "name": "ClaudeBot",
      "operator": "Anthropic",
      "purpose": "ai_training",
      "category": "AI training crawler",
      "robotsToken": "ClaudeBot",
      "userAgentContains": "claudebot",
      "honoursRobotsTxt": "Yes, per Anthropic",
      "officialSource": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "ipListUrl": "https://claude.com/crawling/bots.json",
      "lastChecked": "2026-10-18",
      "page": "https://crawlerpeek.com/bots/claudebot"
    },
    {
      "slug": "claude-searchbot",
      "name": "Claude-SearchBot",
      "operator": "Anthropic",
      "purpose": "ai_search",
      "category": "AI search crawler",
      "robotsToken": "Claude-SearchBot",
      "userAgentContains": "claude-searchbot",
      "honoursRobotsTxt": "Yes, per Anthropic",
      "officialSource": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "ipListUrl": "https://claude.com/crawling/bots.json",
      "lastChecked": "2026-10-18",
      "page": "https://crawlerpeek.com/bots/claude-searchbot"
    },
    {
      "slug": "claude-user",
      "name": "Claude-User",
      "operator": "Anthropic",
      "purpose": "ai_user_request",
      "category": "AI user-triggered fetcher",
      "robotsToken": "Claude-User",
      "userAgentContains": "claude-user",
      "honoursRobotsTxt": "Token documented by Anthropic",
      "officialSource": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "ipListUrl": "https://claude.com/crawling/bots.json",
      "lastChecked": "2026-10-18",
      "page": "https://crawlerpeek.com/bots/claude-user"
    },
    {
      "slug": "perplexitybot",
      "name": "PerplexityBot",
      "operator": "Perplexity",
      "purpose": "ai_search",
      "category": "AI search crawler",
      "robotsToken": "PerplexityBot",
      "userAgentContains": "perplexitybot",
      "honoursRobotsTxt": "Yes, per Perplexity",
      "officialSource": "https://docs.perplexity.ai/guides/bots",
      "ipListUrl": "https://www.perplexity.com/perplexitybot.json",
      "lastChecked": "2026-10-18",
      "page": "https://crawlerpeek.com/bots/perplexitybot"
    },
    {
      "slug": "perplexity-user",
      "name": "Perplexity-User",
      "operator": "Perplexity",
      "purpose": "ai_user_request",
      "category": "AI user-triggered fetcher",
      "robotsToken": "Perplexity-User",
      "userAgentContains": "perplexity-user",
      "honoursRobotsTxt": "Generally ignored (user-initiated)",
      "officialSource": "https://docs.perplexity.ai/guides/bots",
      "ipListUrl": "https://www.perplexity.com/perplexity-user.json",
      "lastChecked": "2026-10-18",
      "page": "https://crawlerpeek.com/bots/perplexity-user"
    },
    {
      "slug": "google-extended",
      "name": "Google-Extended",
      "operator": "Google",
      "purpose": "ai_control_token",
      "category": "AI control token",
      "robotsToken": "Google-Extended",
      "userAgentContains": null,
      "honoursRobotsTxt": "Yes (control token)",
      "officialSource": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
      "ipListUrl": null,
      "lastChecked": "2026-10-18",
      "page": "https://crawlerpeek.com/bots/google-extended"
    },
    {
      "slug": "applebot-extended",
      "name": "Applebot-Extended",
      "operator": "Apple",
      "purpose": "ai_control_token",
      "category": "AI control token",
      "robotsToken": "Applebot-Extended",
      "userAgentContains": null,
      "honoursRobotsTxt": "Yes (control token)",
      "officialSource": "https://support.apple.com/en-us/119829",
      "ipListUrl": null,
      "lastChecked": "2026-10-18",
      "page": "https://crawlerpeek.com/bots/applebot-extended"
    },
    {
      "slug": "ccbot",
      "name": "CCBot",
      "operator": "Common Crawl",
      "purpose": "ai_training",
      "category": "AI training crawler",
      "robotsToken": "CCBot",
      "userAgentContains": "ccbot",
      "honoursRobotsTxt": "Yes, per Common Crawl",
      "officialSource": "https://commoncrawl.org/ccbot",
      "ipListUrl": "https://index.commoncrawl.org/ccbot.json",
      "lastChecked": "2026-10-18",
      "page": "https://crawlerpeek.com/bots/ccbot"
    },
    {
      "slug": "meta-externalagent",
      "name": "meta-externalagent",
      "operator": "Meta",
      "purpose": "ai_training",
      "category": "AI training crawler",
      "robotsToken": "meta-externalagent",
      "userAgentContains": "meta-externalagent",
      "honoursRobotsTxt": "Yes, per Meta",
      "officialSource": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers",
      "ipListUrl": null,
      "lastChecked": "2026-10-18",
      "page": "https://crawlerpeek.com/bots/meta-externalagent"
    },
    {
      "slug": "bytespider",
      "name": "Bytespider",
      "operator": "ByteDance",
      "purpose": "ai_training",
      "category": "AI training crawler",
      "robotsToken": "Bytespider",
      "userAgentContains": "bytespider",
      "honoursRobotsTxt": "Reports differ",
      "officialSource": null,
      "ipListUrl": null,
      "lastChecked": "2026-10-18",
      "page": "https://crawlerpeek.com/bots/bytespider"
    },
    {
      "slug": "googlebot",
      "name": "Googlebot",
      "operator": "Google",
      "purpose": "search_engine",
      "category": "Search engine crawler",
      "robotsToken": "Googlebot",
      "userAgentContains": "googlebot",
      "honoursRobotsTxt": "Yes, per Google",
      "officialSource": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
      "ipListUrl": null,
      "lastChecked": "2026-10-18",
      "page": "https://crawlerpeek.com/bots/googlebot"
    },
    {
      "slug": "bingbot",
      "name": "Bingbot",
      "operator": "Microsoft",
      "purpose": "search_engine",
      "category": "Search engine crawler",
      "robotsToken": "Bingbot",
      "userAgentContains": "bingbot",
      "honoursRobotsTxt": "Yes, per Bing",
      "officialSource": "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0",
      "ipListUrl": null,
      "lastChecked": "2026-10-18",
      "page": "https://crawlerpeek.com/bots/bingbot"
    },
    {
      "slug": "applebot",
      "name": "Applebot",
      "operator": "Apple",
      "purpose": "search_engine",
      "category": "Search engine crawler",
      "robotsToken": "Applebot",
      "userAgentContains": "applebot",
      "honoursRobotsTxt": "Yes, per Apple",
      "officialSource": "https://support.apple.com/en-us/119829",
      "ipListUrl": null,
      "lastChecked": "2026-10-18",
      "page": "https://crawlerpeek.com/bots/applebot"
    },
    {
      "slug": "meta-externalfetcher",
      "name": "meta-externalfetcher",
      "operator": "Meta",
      "purpose": "ai_user_request",
      "category": "AI user-triggered fetcher",
      "robotsToken": "meta-externalfetcher",
      "userAgentContains": "meta-externalfetcher",
      "honoursRobotsTxt": "May bypass (user-initiated)",
      "officialSource": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers",
      "ipListUrl": null,
      "lastChecked": "2026-10-18",
      "page": "https://crawlerpeek.com/bots/meta-externalfetcher"
    },
    {
      "slug": "duckassistbot",
      "name": "DuckAssistBot",
      "operator": "DuckDuckGo",
      "purpose": "ai_user_request",
      "category": "AI user-triggered fetcher",
      "robotsToken": "DuckAssistBot",
      "userAgentContains": "duckassistbot",
      "honoursRobotsTxt": "Yes, per DuckDuckGo",
      "officialSource": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot",
      "ipListUrl": "https://duckduckgo.com/duckassistbot.json",
      "lastChecked": "2026-10-18",
      "page": "https://crawlerpeek.com/bots/duckassistbot"
    },
    {
      "slug": "amazonbot",
      "name": "Amazonbot",
      "operator": "Amazon",
      "purpose": "ai_training",
      "category": "AI training crawler",
      "robotsToken": "Amazonbot",
      "userAgentContains": "amazonbot",
      "honoursRobotsTxt": "Yes, per Amazon",
      "officialSource": "https://developer.amazon.com/amazonbot",
      "ipListUrl": "https://developer.amazon.com/amazonbot/ip-addresses/",
      "lastChecked": "2026-10-18",
      "page": "https://crawlerpeek.com/bots/amazonbot"
    },
    {
      "slug": "ahrefsbot",
      "name": "AhrefsBot",
      "operator": "Ahrefs",
      "purpose": "seo_tool",
      "category": "SEO tool crawler",
      "robotsToken": "AhrefsBot",
      "userAgentContains": "ahrefsbot",
      "honoursRobotsTxt": "Yes, per Ahrefs",
      "officialSource": "https://ahrefs.com/robot",
      "ipListUrl": "https://api.ahrefs.com/v3/public/crawler-ip-ranges",
      "lastChecked": "2026-10-18",
      "page": "https://crawlerpeek.com/bots/ahrefsbot"
    },
    {
      "slug": "semrushbot",
      "name": "SemrushBot",
      "operator": "Semrush",
      "purpose": "seo_tool",
      "category": "SEO tool crawler",
      "robotsToken": "SemrushBot",
      "userAgentContains": "semrushbot",
      "honoursRobotsTxt": "Yes, per Semrush",
      "officialSource": "https://www.semrush.com/bot/",
      "ipListUrl": null,
      "lastChecked": "2026-10-18",
      "page": "https://crawlerpeek.com/bots/semrushbot"
    },
    {
      "slug": "mj12bot",
      "name": "MJ12bot",
      "operator": "Majestic",
      "purpose": "seo_tool",
      "category": "SEO tool crawler",
      "robotsToken": "MJ12bot",
      "userAgentContains": "mj12bot",
      "honoursRobotsTxt": "Yes, per Majestic",
      "officialSource": "https://www.mj12bot.com/",
      "ipListUrl": null,
      "lastChecked": "2026-10-18",
      "page": "https://crawlerpeek.com/bots/mj12bot"
    },
    {
      "slug": "facebookexternalhit",
      "name": "facebookexternalhit",
      "operator": "Meta",
      "purpose": "social_preview",
      "category": "Social preview fetcher",
      "robotsToken": "facebookexternalhit",
      "userAgentContains": "facebookexternalhit",
      "honoursRobotsTxt": "May bypass (integrity checks)",
      "officialSource": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers",
      "ipListUrl": null,
      "lastChecked": "2026-10-18",
      "page": "https://crawlerpeek.com/bots/facebookexternalhit"
    },
    {
      "slug": "slackbot",
      "name": "Slackbot",
      "operator": "Slack",
      "purpose": "social_preview",
      "category": "Social preview fetcher",
      "robotsToken": "Slackbot",
      "userAgentContains": "slackbot",
      "honoursRobotsTxt": "No, per Slack",
      "officialSource": "https://api.slack.com/robots",
      "ipListUrl": null,
      "lastChecked": "2026-10-18",
      "page": "https://crawlerpeek.com/bots/slackbot"
    }
  ]
}