{
  "name": "AI training opt-out tracker: robots.txt of the Tranco top 500",
  "version": "2026-10-31",
  "license": "CC BY 4.0 (https://creativecommons.org/licenses/by/4.0/)",
  "licenseScope": "Our measurements and descriptions. Domain names and ranks come from the Tranco list (cite it as well); the robots.txt files belong to their sites and are not covered.",
  "attribution": "Crawlerpeek (https://crawlerpeek.com/data/ai-training-opt-out), with a link to https://crawlerpeek.com/data/ai-training-opt-out",
  "cite": "Crawlerpeek. (2026). AI training opt-out tracker: robots.txt of the Tranco top 500 (Version 2026-10-31) [Data set]. https://crawlerpeek.com/data/ai-training-opt-out",
  "ranking": "Tranco (Le Pochat et al., NDSS 2019), https://tranco-list.eu/; list ID per run below",
  "note": "Shares are of readable robots.txt files (HTTP 200, not empty, not an HTML page). A file counts for a token when a User-agent line names it exactly (case-insensitive); User-agent: * never counts.",
  "runs": [
    {
      "scanDate": "2026-10-31",
      "trancoListId": "746VX",
      "trancoListCreated": "2026-10-30T22:00Z",
      "trancoListUrl": "https://tranco-list.eu/list/746VX/1000000",
      "periodStart": "2026-10-31T19:48Z",
      "periodEnd": "2026-10-31T19:57Z",
      "userAgent": "Mozilla/5.0 (compatible; robots-txt-adoption-check/1.0; research, 2 req/s max)",
      "maxRequestsPerSecond": 2,
      "note": "First measurement, the day after the IETF draft that defines the __usage_ai-training token (30 October 2026). Top 500 rows of the Tranco list 746VX, identical to that day's daily file.",
      "perDomainFile": "https://crawlerpeek.com/data/ai-training-opt-out-2026-10-31.csv",
      "totals": {
        "sample": 500,
        "readable": 227,
        "tokenGroup": 2,
        "anyAiTrainingControl": 68,
        "allThreeVendorTokens": 43,
        "contentSignal": 8,
        "tokenTypo": 0,
        "tokenStringOutsideGroup": 0
      },
      "vendors": [
        {
          "token": "Google-Extended",
          "operator": "Google",
          "named": 56,
          "disallowRootInThatGroup": 33
        },
        {
          "token": "GPTBot",
          "operator": "OpenAI",
          "named": 57,
          "disallowRootInThatGroup": 38
        },
        {
          "token": "ClaudeBot",
          "operator": "Anthropic",
          "named": 56,
          "disallowRootInThatGroup": 38
        }
      ],
      "tokenGroupDomains": [
        {
          "rank": 16,
          "domain": "twitter.com",
          "rules": "Disallow:/"
        },
        {
          "rank": 47,
          "domain": "x.com",
          "rules": "Disallow:/"
        }
      ]
    }
  ]
}