{
  "@context": "https://schema.org",
  "@type": "Observation",
  "id": "openai-crawler-restriction-head-c4",
  "label": "Share of high-value training tokens restricted to OpenAI crawlers",
  "question": "How much of the good web has been closed to AI training?",
  "answer": "25.9% of tokens in the head of the C4 corpus were restricted to OpenAI's crawlers by April 2024, against 13.3% for Anthropic and Common Crawl.",
  "value": 25.9,
  "display": "25.9%",
  "unit": "percent",
  "evidence_type": "analysis",
  "period": "April 2024",
  "period_iso": "2024-04",
  "year_on_year_pct": null,
  "comparison": {
    "label": "Anthropic and Common Crawl",
    "display": "13.3%",
    "value": 13.3
  },
  "geography": "Global",
  "topics": [
    "generative-engines",
    "governance-risk"
  ],
  "sample": "Tokens in Head C4, the 3,950 highest-token domains of the C4 corpus. Audit of 14,000 web domains with longitudinal robots.txt collection",
  "methodology": "Data Provenance Initiative, \"Consent in Crisis\", audit of 14,000 web domains underlying C4, RefinedWeb and Dolma, with longitudinal robots.txt collection from archived snapshots and human annotation. Head C4 is the 3,950 highest-token domains. Data ends April 2024, so treat it as the historical inflection rather than current state. The paper's forward projections are forecasts and are not used here.",
  "confidence": "high",
  "verified_against_primary": true,
  "source": {
    "id": "data-provenance",
    "name": "Data Provenance Initiative",
    "type": "academic",
    "url": "https://arxiv.org/abs/2407.14933",
    "homepage": "https://www.dataprovenance.org/"
  },
  "located_via": null,
  "last_checked": "2026-10-03",
  "page_url": "https://travel.rankwit.ai/data/openai-crawler-restriction-head-c4"
}