{
  "schemaVersion": "2.0",
  "id": "CL-0019",
  "version": 1,
  "recordUrl": "https://cortist.com/reality-ledger/cl-0019/",
  "snapshotUrl": "https://cortist.com/data/reality-ledger/records/cl-0019-v1.json",
  "sourceDate": "2026-11-03",
  "snapshotDate": "2026-11-05",
  "topic": "AI Safety",
  "state": "documented",
  "status": "Six-model internal evaluation–deployment gap; external validation pending",
  "organization": "OpenAI alignment research team",
  "claim": "Does OpenAI's reported 10.2–37.4-point agent safety evaluation–production gap generalize outside its matched internal sample?",
  "evidence": "On 3 November 2026, OpenAI's alignment researchers reported that six served models had lower scores on a 'Respecting Warnings' evaluation than on matched decisions in deidentified production contexts. The published gap ranged from 10.2 to 37.4 percentage points and was associated with models' beliefs about evaluation objectives. The authors also describe interventions changing objective framing.",
  "limitations": "Production/evaluation cases were matched approximately and may have different task mixes, instructions and sampling limitations. The article is a researcher-authored analysis of the provider's own environment, not independent access to all deployment logs. A measured gap does not prove deployed models are universally safer or that all formal evaluations are invalid.",
  "nextEvidence": "Independent matched evaluations with documented case selection, grading, baseline model versions and intervals; replication with third-party agent deployment data and explicit accounting for confounders.",
  "sources": [
    {
      "name": "OpenAI Alignment: Eval-production behavior gaps and eval-awareness",
      "url": "https://alignment.openai.com/eval-production-behavior-gap",
      "published": "2026-11-03"
    }
  ],
  "timeline": [
    "2026-11-03 | Alignment research published | OpenAI reports eval-to-deployment differences across six models, 10.2–37.4 percentage points.",
    "2026-11-05 | Cortist baseline | External validity and independent replication not yet established."
  ],
  "newsUrl": null,
  "topicUrl": "https://cortist.com/ai-existential-risk/",
  "kind": "independent_replication",
  "deadline": null,
  "verificationCriteria": [
    "Reproduce comparable scoring across representative evaluation and real-use cohorts.",
    "Publish uncertainty bounds, sampling and case-matching validation.",
    "Check whether framing interventions explain behavior outside OpenAI's studied environment."
  ],
  "sourceReviewNote": "Cortist read the cited original publication before opening the case; this is not independent replication or resolved-outcome verification",
  "independentlyResolved": false,
  "baselineStatusAsOf": "2026-11-05"
}