{
  "node_id": "ietf-rfc-9309-robots-exclusion-protocol-2022",
  "title": "IETF RFC 9309 - Robots Exclusion Protocol (Standards Track, September 2022)",
  "domain": "AI Governance & Law",
  "version": "1.0.0",
  "last_updated": "2026-06-13",
  "bluf": "IETF RFC 9309 standardises the Robots Exclusion Protocol as an IETF Standards Track document published in September 2022, formalising the de facto robots.txt convention used since 1994. The protocol defines how website operators communicate access rules to automated clients (crawlers, scrapers, AI training data collectors) via a robots.txt file at the site root. RFC 9309 specifies the file format, parsing requirements, and rule precedence for the User-Agent and Disallow/Allow directives. The Protocol is voluntary at the protocol level - it expresses publisher intent but technical enforcement depends on the requesting client honouring the file. In AI governance, RFC 9309 has become the foundation for AI training data opt-out signalling: publishers use robots.txt to express that named AI crawlers (GPTBot, Google-Extended, ClaudeBot, CCBot, anthropic-ai, PerplexityBot, etc.) should not crawl their site for training data collection. Companion mechanisms include the IETF AI Preferences Working Group proposed standard (under development) and the W3C Text and Data Mining Reservation Protocol. The CNIL AI development recommendations (April-June 2024), the EU AI Office GPAI Code of Practice (July 2025) copyright commitment, and the OpenAI, Google, Anthropic, and Common Crawl crawler operator policies all reference robots.txt opt-out honour as the baseline AI training data opt-out mechanism.",
  "paywall": {
    "status": "LOCKED",
    "unlock_cost_usd": "0.01",
    "skyfire_id": "41779894-ece2-4163-9761-b3b1b76e19b0"
  },
  "crosswalks": {
    "_available_keys": [
      "industry_mapping",
      "regulatory_overlay",
      "ai_governance_overlay",
      "operator_practice"
    ],
    "_note": "Full crosswalk values included in vault response"
  },
  "dependencies": [
    "france-cnil-ai-recommendations-2024",
    "eu-ai-office-gpai-code-of-practice-2025"
  ],
  "primary_citations_count": 8
}