{ "schema": "cortist.milestone-source-audit.v1", "id": "cortist-milestone-source-audit-20261027-v1", "publishedDate": "2026-10-27", "asOfDate": "2026-10-27", "scope": "Five previously published 2027 short-horizon research candidates. This is a source-methodology audit and does not amend historic questions or publish probability.", "sourceRegistry": "/data/milestones/registry-v1-20261027.json", "externalReviewerSignoffs": 0, "independentlyCertified": false, "probabilitiesIssued": 0, "lockedForecasts": 0, "resolvedMilestones": 0, "readyToLock": 0, "reviewedCount": 5, "method": "Original primary sources cross-referenced with underlying v1 candidate text. Verified refers only to explicitly source-documented claims, not third-party audit certification of the candidate.", "rules": [ "Never overwrite original candidate question, cutoff or supporting forecast snapshots.", "Benchmarks and legal decisions must be prospectively fixed and independently verified before scoring.", "A keyword match in Evidence Pulse 2.0 never proves a milestone was achieved.", "For event negatives, absence of public evidence is not sufficient unless the complete relevant archive and publication lag are checked.", "Any revised resolution definition or future official probability must be issued as a new, clearly dated version with independent timestamp and conflict-cleared external review." ], "items": [ { "key": "CM-001", "state": "HOLD_PRIVATE_SPLIT_NOT_FROZEN", "priority": "high", "finding": "Scale Labs gives 222/272 private results in its V2 refresh write-up while the overview leaderboard methodology still describes a 276-instance private set. A metric based only on the label SWE-bench Pro V2 private is not reproducible without an exact submission, task-set and version identifier.", "verified": [ "Official V2 public set has 642 tasks (not the appropriate source for the private milestone).", "The V2 refresh article reports 222/272 leading private solved tasks.", "A separate Scale Labs methodology page still describes 276 proprietary private instances." ], "proposedDecision": "Choose a uniquely versioned 272-task or other explicitly archived private V2 instance set, freeze pass@1 and tool/harness constraints, exclude public and held-out splits and require benchmark-owner audited per-task proof. Preserve the original 90% question unchanged until a distinct registered version is authorized.", "nextAction": "Obtain an official immutable private-set identifier, denominator and adjudication archive from Scale Labs and an independent reviewer.", "sources": [ { "type": "benchmark-operator", "title": "Scale Labs: SWE-bench Pro V2 refresh and public/private results", "url": "https://labs.scale.com/blog/swe-bench-pro-v2" }, { "type": "benchmark-operator", "title": "Scale Labs: SWE-bench Pro V2 leaderboard and private-set methodology", "url": "https://labs.scale.com/leaderboard/swe_bench_pro_public_v2" } ], "id": "CM-001", "originalQuestion": "Will an AI coding agent resolve 90% of SWE-bench Pro V2 private tasks by October 2027?", "originalDeadline": "2027-10-27", "originalVersionState": "research-draft", "originalBaseline": "Scale Labs reported 222/272 private tasks resolved for a leading agent in September 2026; near-perfect public scores are not comparable.", "originalPrelockBlocker": "Freeze a permanent private-split identifier, task denominator and official archive before probability issuance.", "topic": "AI Agents", "originalUrl": "/milestones/private-coding-agent-90-percent-2027/", "outcomeState": "unresolved", "definitionLocked": false, "independentReviewComplete": false, "probability": null, "scorable": false }, { "key": "CM-002", "state": "HOLD_REAL_ROBOT_TASK_SPLIT_NOT_FROZEN", "priority": "high", "finding": "INSAIT confirms LADDER contains physical and simulated multi-step manipulation tests and reports especially poor performance on long sequences; the announcement does not itself define an immutable, independently auditable five-object real-robot subset suitable for our binary threshold.", "verified": [ "LADDER was introduced by INSAIT and NVIDIA on 26 October 2026.", "Both simulation and physical-robot evaluations are included; aggregation across modes cannot satisfy our real-robot-only question.", "The operator says specialized models almost never succeed in moving five or more objects consecutively; this is qualitative evidence, not a frozen 50% baseline." ], "proposedDecision": "Specify exact five-object task IDs, identical hardware or rules for variation, number of independent trials, failed-retry policy, human teleoperation exclusions, end-to-end success definition and signed raw run logs. Define whether success is averaged over tasks, episodes or attempts.", "nextAction": "Request versioned task manifest, official evaluator acceptance rules and recorded trial counts from LADDER maintainers.", "sources": [ { "type": "benchmark-developer", "title": "INSAIT: LADDER original launch and research summary", "url": "https://insait.ai/ladder-a-new-benchmark-for-evaluating-ai-on-complex-robot-tasks/" }, { "type": "benchmark-operator", "title": "LADDER official project and leaderboard", "url": "https://ladderrobotics.ai/" } ], "id": "CM-002", "originalQuestion": "Will a robot complete at least 50% of five-object LADDER tasks by October 2027?", "originalDeadline": "2027-10-27", "originalVersionState": "research-draft", "originalBaseline": "INSAIT introduced LADDER on 26 October 2026 and reported persistent difficulty with long sequences.", "originalPrelockBlocker": "Freeze exact five-object subset, trial counts, evaluation protocol and result archival before locking.", "topic": "AI & Robotics", "originalUrl": "/milestones/ladder-five-object-robot-2027/", "outcomeState": "unresolved", "definitionLocked": false, "independentReviewComplete": false, "probability": null, "scorable": false }, { "key": "CM-003", "state": "HOLD_FIRST_EVENT_BASELINE_UNVERIFIED", "priority": "high", "finding": "Article 101 describes European Commission fines against providers of general-purpose AI models, but a press warning, investigation, national fine or preliminary finding is not an adopted Commission fine. Before calling a future event the first, the pre-publication decision baseline must be independently checked.", "verified": [ "Article 101 identifies the Commission as the decision maker and covers GPAI providers.", "Article 101(2) requires communication of preliminary findings and opportunity to be heard before the decision.", "The official GPAI enforcement guidance identifies 2 August 2026 as the start of fine powers." ], "proposedDecision": "Count only a formal Article 101 Commission fine decision adopted within the original specified window; freeze whether decision date or publication date governs, how late-published decisions are handled and what public register constitutes complete evidence. Never assume that no qualifying earlier decision exists without a dated search and an independent legal reviewer.", "nextAction": "Independently search and archive Commission official fine decisions before the candidate cutoff and obtain a legal sign-off on first-event wording.", "sources": [ { "type": "official-law", "title": "EU AI Act Service Desk: Article 101 (Commission GPAI fines)", "url": "https://ai-act-service-desk.ec.europa.eu/en/ai-act/article-101" }, { "type": "official-regulator", "title": "European Commission: GPAI enforcement responsibilities and fines", "url": "https://digital-strategy.ec.europa.eu/en/faqs/general-purpose-ai-models-ai-act-questions-answers" }, { "type": "official-law", "title": "EU AI Act Service Desk: Article 113 (application dates)", "url": "https://ai-act-service-desk.ec.europa.eu/en/ai-act/article-113" } ], "id": "CM-003", "originalQuestion": "Will the European Commission impose an Article 101 GPAI fine by October 2027?", "originalDeadline": "2027-10-27", "originalVersionState": "research-draft", "originalBaseline": "Article 101 allows sanctions against general-purpose AI model providers; an independently frozen first-fine baseline is still required.", "originalPrelockBlocker": "Independently check baseline for earlier decisions and decide how to handle adoption versus publication timing.", "topic": "AI & Governance", "originalUrl": "/milestones/first-eu-gpai-article-101-fine-2027/", "outcomeState": "unresolved", "definitionLocked": false, "independentReviewComplete": false, "probability": null, "scorable": false }, { "key": "CM-004", "state": "HOLD_SERVICE_AVAILABILITY_PROTOCOL_NOT_FROZEN", "priority": "high", "finding": "Waymo and the City of Munich explicitly target public commercial service towards the END of 2027, while this Cortist draft asks for launch by 27 October 2027. The earlier cutoff is intentional and must not be confused with Waymo’s own expected launch window.", "verified": [ "Waymo announced Munich manual mapping and future safety-driver validation on 25 August 2026.", "Waymo’s Germany FAQ says testing vehicles still have validation drivers and commercial rides are not yet offered as described there.", "The city of Munich independently reports a planned public launch towards the end of 2027." ], "proposedDecision": "Require publicly bookable AND paid passenger service with no in-vehicle safety driver inside the Munich municipal area before 27 October 2027 23:59 UTC. Fix what counts as a member of the public, fare receipt, public booking channel, operator confirmation and independent local regulator or city evidence; exclude mapping, invite-only trials and free demonstrations.", "nextAction": "Archive official operator and Munich city statements and require corroborated booking / fare and no-driver evidence before outcome review.", "sources": [ { "type": "operator", "title": "Waymo: Servus München, launch preparations and 2027 target", "url": "https://waymo.com/blog/2026/08/waymo-in-munich/" }, { "type": "operator", "title": "Waymo: Munich operations and commercial availability FAQ", "url": "https://waymo.com/waymo-in-germany/" }, { "type": "city", "title": "City of Munich: Robotaxi project and late-2027 target", "url": "https://stadt.muenchen.de/lhm-ms-wirtschaftsfoerderung-en/en/business-news/waymo-robotaxi-eng.html" } ], "id": "CM-004", "originalQuestion": "Will paid, driverless Waymo rides launch in Munich by October 2027?", "originalDeadline": "2027-10-27", "originalVersionState": "research-draft", "originalBaseline": "Waymo said in August 2026 it was mapping Munich with safety drivers and targeted public service toward the end of 2027.", "originalPrelockBlocker": "Freeze definition of Munich service area and evidence standard for paid publicly available service.", "topic": "AI & Robotics", "originalUrl": "/milestones/waymo-munich-paid-driverless-rides-2027/", "outcomeState": "unresolved", "definitionLocked": false, "independentReviewComplete": false, "probability": null, "scorable": false }, { "key": "CM-005", "state": "HOLD_UNCOMPARABLE_POWER_NORMALIZATION", "priority": "high", "finding": "MLCommons Endpoints measures multi-point Pareto tradeoffs across throughput, latency and concurrency. Its public materials say normalization rules are still being developed, so a single 2x performance-per-watt result is not independently reproducible unless a specific official measured power field and matched operating point are chosen.", "verified": [ "MLPerf Endpoints benchmarks live inference API throughput, concurrency, interactivity and first-token latency.", "MLCommons says power / throughput per watt is an optional normalization dimension and rules are developing.", "Separate MLPerf Inference Power datasets measure full-system wall power; data or scores from different benchmark suites cannot be silently mixed." ], "proposedDecision": "Select the same GPT-OSS-120B version, workload, quality constraints and serving setup; predefine an interactivity / P90 latency operating band and concurrency, same verified power measurement boundary and units, and a dated baseline result. Freeze a reproducible baseline and future audit before claiming any 2x outcome. Consider changing to fully measured MLPerf Inference Power only via a separately dated public research-candidate version.", "nextAction": "Obtain an audited 2026 Endpoints baseline with officially accepted power and latency normalization, or prepare a distinct future redefined milestone. Do not compare incompatible MLPerf versions or self-reported TDP.", "sources": [ { "type": "benchmark-operator", "title": "MLCommons: MLPerf Endpoints official benchmark and normalization policy", "url": "https://mlcommons.org/benchmarks/endpoints/" }, { "type": "benchmark-operator", "title": "MLCommons: MLPerf Inference Datacenter v6.1 and measured power details", "url": "https://mlcommons.org/benchmarks/inference-datacenter/" }, { "type": "official-documentation", "title": "MLCommons Endpoints technical benchmark overview", "url": "https://docs.mlcommons.org/endpoints-submission-docs/understand/what-is-mlperf-endpoints/" } ], "id": "CM-005", "originalQuestion": "Will verified AI inference performance per watt double by October 2027?", "originalDeadline": "2027-10-27", "originalVersionState": "research-draft", "originalBaseline": "MLCommons launched continuously reviewed Endpoints results in 2026; cross-vintage power normalization still requires a stable baseline.", "originalPrelockBlocker": "Critical: freeze 2026 baseline, comparable power normalization, model/tokenizer, quality tier, latency and reviewer-approved rules.", "topic": "AI Chips & Compute", "originalUrl": "/milestones/mlperf-inference-efficiency-double-2027/", "outcomeState": "unresolved", "definitionLocked": false, "independentReviewComplete": false, "probability": null, "scorable": false } ] }