From 00897dc7c126c9733b672d4921cb5c2a90dfcfbf Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 11 Aug 2026 18:16:49 +0000 Subject: [PATCH] fix(runner): stop the eval slice scoring a rejected call as green The live matrix injected `idempotencyKey` into every case's dictated JSON input, but only mutating operations declare that key -- read operations close their schema with `additionalProperties: false`. `search_tasks` therefore rejected the input the prompt told the model to send. Nothing caught it. A rejected call still emits a `tool_call` entry, so the observed-vs-expected call comparison matched, and a rejected read leaves the control plane `unchanged` exactly like a successful one, so the semantic outcome matched by coincidence. The lone signal was an authorization mismatch, and the hard-invariant gate only fires on escalation (denied -> allowed), never on the reverse. se-get-issue-01 scored 1.0 on all five dimensions while its call was failing. Fixed on three levels: - Root cause: derive the retry key from the catalog descriptor so the dictated input always satisfies the operation's own schema. - Harness gate: fail any live case whose typed result came back `ok: false`, so a denial can never pass on matching calls and state alone. - Scorer: a call the case declared allowed but that was denied now fails `semantic_outcome` -- a rejected call cannot have produced its effect. Trajectory stays green, since the model did choose the right tool. Negative control: reintroducing the injection now aborts the matrix with `deniedCalls: ["search_tasks"]` instead of reporting a silent pass. Live matrix re-run 16/16 with zero authorization mismatches; the fault matrix is unchanged (8 green / 8 red, mean 0.691), confirming the scorer change only catches genuine denials. Co-Authored-By: Claude Opus 5 (1M context) --- .../docs/capability-eval-slice.md | 14 +- .../capability/fault-eval-slice-report.json | 3 +- .../capability/fault-eval-slice-report.md | 2 +- .../capability/live-codex-matrix.json | 120 +++++++++--------- .../evidence/capability/live-codex-matrix.md | 2 +- .../capability/live-eval-slice-report.json | 120 +++++++++--------- .../capability/live-eval-slice-report.md | 2 +- .../src/conformance/capability-eval-suite.ts | 30 ++++- .../src/eval/eval-scoring.test.ts | 39 ++++++ .../paperclip-runner/src/eval/eval-scoring.ts | 19 ++- 10 files changed, 217 insertions(+), 134 deletions(-) diff --git a/packages/paperclip-runner/docs/capability-eval-slice.md b/packages/paperclip-runner/docs/capability-eval-slice.md index 032962b9e0a..3627999ba5c 100644 --- a/packages/paperclip-runner/docs/capability-eval-slice.md +++ b/packages/paperclip-runner/docs/capability-eval-slice.md @@ -36,7 +36,10 @@ separately scored 0..1 value per dimension: that should have been denied. A gate failure forces the overall score to 0 regardless of the other dimensions. - **`semantic_outcome`** — the resulting control-plane state matches expectation - (mutated vs unchanged). + (mutated vs unchanged) *and* a call the case declared allowed was not rejected. + Both halves are required: a rejected read leaves the control plane `unchanged` + exactly like a successful one, so the state comparison alone would score a + failed call as a pass by coincidence. - **`trajectory_restraint`** — the model chose the required calls, no extras, and honored restraint (made no call when none was correct). - **`trace_completeness`** — the run emitted a complete causal trace (run, @@ -66,6 +69,15 @@ dimensions. Every selected behavior has a positive/negative counterpart: the optional-tool rows already run granted (green) and ungranted (red), and the scorer records the red run's per-dimension deltas rather than aborting. +The matrix prompt dictates the exact JSON input for the operation under test, so +that input must satisfy the operation's own schema. Only mutating operations +declare `idempotencyKey`; read operations close their schema with +`additionalProperties: false`. Injecting a retry key unconditionally therefore +made the model dictate an input the tool had to reject — the call was recorded, +state stayed `unchanged` as expected, and the case scored a false green. The +matrix now derives the key from the catalog descriptor and fails any case whose +typed result came back `ok: false`, so a rejected call can never read as green. + ## Behavior and fault execution `runEvalBehaviorFaultMatrix` executes one green and one red case for each diff --git a/packages/paperclip-runner/knowledge/evidence/capability/fault-eval-slice-report.json b/packages/paperclip-runner/knowledge/evidence/capability/fault-eval-slice-report.json index 9e56e6bd3bb..d977c982252 100644 --- a/packages/paperclip-runner/knowledge/evidence/capability/fault-eval-slice-report.json +++ b/packages/paperclip-runner/knowledge/evidence/capability/fault-eval-slice-report.json @@ -417,7 +417,8 @@ "gate": false, "weight": 0.35, "reasons": [ - "control-plane state unchanged, expected mutated" + "control-plane state unchanged, expected mutated", + "operation denied but the case declared it allowed" ] }, "trajectory_restraint": { diff --git a/packages/paperclip-runner/knowledge/evidence/capability/fault-eval-slice-report.md b/packages/paperclip-runner/knowledge/evidence/capability/fault-eval-slice-report.md index ab250de4817..c63473169db 100644 --- a/packages/paperclip-runner/knowledge/evidence/capability/fault-eval-slice-report.md +++ b/packages/paperclip-runner/knowledge/evidence/capability/fault-eval-slice-report.md @@ -2,7 +2,7 @@ type: Evidence Report title: Runner Eval Fault Slice Report description: Eight green/red behaviors with deterministic injected-fault receipts. -generated: { by: process:capability-live-eval, at: 2026-08-11T17:38:30.000Z } +generated: { by: process:capability-live-eval, at: 2026-08-11T18:14:30.145Z } status: stable --- diff --git a/packages/paperclip-runner/knowledge/evidence/capability/live-codex-matrix.json b/packages/paperclip-runner/knowledge/evidence/capability/live-codex-matrix.json index 2add2d79646..588ad5a70a1 100644 --- a/packages/paperclip-runner/knowledge/evidence/capability/live-codex-matrix.json +++ b/packages/paperclip-runner/knowledge/evidence/capability/live-codex-matrix.json @@ -29,10 +29,10 @@ "trace": { "runId": "run-capability-live-ap-approval-deny-01", "sessionId": "session-capability-live-ap-approval-deny-01", - "turnId": "019ff1e7-5e9e-7353-abc9-734f85ed535b", - "itemId": "evidence-000022", + "turnId": "019ff208-7d1d-74f3-8bb2-1f2489e336d5", + "itemId": "evidence-000015", "receiptIds": [ - "evidence-000023" + "evidence-000016" ], "terminalPresent": true }, @@ -41,7 +41,7 @@ "observed": "allowed" }, "efficiency": { - "latencyMs": 4056, + "latencyMs": 5762, "attempts": 1 }, "budget": { @@ -73,10 +73,10 @@ "trace": { "runId": "run-capability-live-ar-no-done-without-upload-01", "sessionId": "session-capability-live-ar-no-done-without-upload-01", - "turnId": "019ff1e7-6ffb-7123-9f0a-37ca2f894aa2", - "itemId": "evidence-000022", + "turnId": "019ff208-94a3-7be2-a152-6558026f2c81", + "itemId": "evidence-000015", "receiptIds": [ - "evidence-000023" + "evidence-000016" ], "terminalPresent": true }, @@ -85,7 +85,7 @@ "observed": "allowed" }, "efficiency": { - "latencyMs": 3568, + "latencyMs": 6153, "attempts": 1 }, "budget": { @@ -117,10 +117,10 @@ "trace": { "runId": "run-capability-live-bl-cancelled-not-resolved-01", "sessionId": "session-capability-live-bl-cancelled-not-resolved-01", - "turnId": "019ff1e7-7f46-7292-b3ab-5b5d6b99e09f", - "itemId": "evidence-000022", + "turnId": "019ff208-ada0-7402-8f7c-9170569cd3c8", + "itemId": "evidence-000015", "receiptIds": [ - "evidence-000023" + "evidence-000016" ], "terminalPresent": true }, @@ -129,7 +129,7 @@ "observed": "allowed" }, "efficiency": { - "latencyMs": 4079, + "latencyMs": 3412, "attempts": 1 }, "budget": { @@ -161,10 +161,10 @@ "trace": { "runId": "run-capability-live-cm-mention-structured-01", "sessionId": "session-capability-live-cm-mention-structured-01", - "turnId": "019ff1e7-909c-70e2-b9b7-8e53cd6dac67", - "itemId": "evidence-000021", + "turnId": "019ff208-bbe7-7cb0-b299-9fe09ccaf31a", + "itemId": "evidence-000015", "receiptIds": [ - "evidence-000023" + "evidence-000016" ], "terminalPresent": true }, @@ -173,7 +173,7 @@ "observed": "allowed" }, "efficiency": { - "latencyMs": 3621, + "latencyMs": 3525, "attempts": 1 }, "budget": { @@ -201,8 +201,8 @@ "trace": { "runId": "run-capability-live-co-409-stop-01", "sessionId": "session-capability-live-co-409-stop-01", - "turnId": "019ff1e7-a051-7760-992c-39e9c9f2b09c", - "itemId": "transcript-000023", + "turnId": "019ff208-caa9-7573-9c98-cdfda85e0d6a", + "itemId": "transcript-000016", "receiptIds": [], "terminalPresent": true }, @@ -211,7 +211,7 @@ "observed": "absent" }, "efficiency": { - "latencyMs": 1864, + "latencyMs": 4763, "attempts": 1 }, "budget": { @@ -243,10 +243,10 @@ "trace": { "runId": "run-capability-live-dp-base-revision-01", "sessionId": "session-capability-live-dp-base-revision-01", - "turnId": "019ff1e7-a905-7551-b66a-08d1fd0832fd", - "itemId": "evidence-000022", + "turnId": "019ff208-de3c-7572-8b4c-622fa11b671b", + "itemId": "evidence-000015", "receiptIds": [ - "evidence-000023" + "evidence-000016" ], "terminalPresent": true }, @@ -255,7 +255,7 @@ "observed": "allowed" }, "efficiency": { - "latencyMs": 3685, + "latencyMs": 3258, "attempts": 1 }, "budget": { @@ -283,8 +283,8 @@ "trace": { "runId": "run-capability-live-er-blocked-dedup-01", "sessionId": "session-capability-live-er-blocked-dedup-01", - "turnId": "019ff1e7-b8d6-7eb2-9d4d-e98b659fdfc0", - "itemId": "transcript-000023", + "turnId": "019ff208-ebe0-74c3-8ec0-5d771d020a9c", + "itemId": "transcript-000016", "receiptIds": [], "terminalPresent": true }, @@ -293,7 +293,7 @@ "observed": "absent" }, "efficiency": { - "latencyMs": 2033, + "latencyMs": 4953, "attempts": 1 }, "budget": { @@ -321,8 +321,8 @@ "trace": { "runId": "run-capability-live-hb-context-01", "sessionId": "session-capability-live-hb-context-01", - "turnId": "019ff1e7-c239-7462-8d23-dda298d0fa58", - "itemId": "transcript-000023", + "turnId": "019ff209-0043-7423-9957-be6301c6e951", + "itemId": "transcript-000016", "receiptIds": [], "terminalPresent": true }, @@ -331,7 +331,7 @@ "observed": "absent" }, "efficiency": { - "latencyMs": 1658, + "latencyMs": 1660, "attempts": 1 }, "budget": { @@ -363,10 +363,10 @@ "trace": { "runId": "run-capability-live-ix-checkbox-01", "sessionId": "session-capability-live-ix-checkbox-01", - "turnId": "019ff1e7-ca3a-7ad2-9d67-35ee17cb6042", - "itemId": "evidence-000021", + "turnId": "019ff209-0839-7db2-9a98-bd9296ef4db5", + "itemId": "evidence-000015", "receiptIds": [ - "evidence-000023" + "evidence-000016" ], "terminalPresent": true }, @@ -375,7 +375,7 @@ "observed": "allowed" }, "efficiency": { - "latencyMs": 9631, + "latencyMs": 8222, "attempts": 1 }, "budget": { @@ -407,10 +407,10 @@ "trace": { "runId": "run-capability-live-mh-blocked-handoff-01", "sessionId": "session-capability-live-mh-blocked-handoff-01", - "turnId": "019ff1e7-f14e-7962-91e0-60f4b2c1ee5e", - "itemId": "evidence-000022", + "turnId": "019ff209-2953-7062-a513-6e7e28b324af", + "itemId": "evidence-000015", "receiptIds": [ - "evidence-000023" + "evidence-000016" ], "terminalPresent": true }, @@ -419,7 +419,7 @@ "observed": "allowed" }, "efficiency": { - "latencyMs": 3474, + "latencyMs": 3811, "attempts": 1 }, "budget": { @@ -451,10 +451,10 @@ "trace": { "runId": "run-capability-live-rf-api-404-report-01", "sessionId": "session-capability-live-rf-api-404-report-01", - "turnId": "019ff1e8-0054-7102-8629-5a629712bc29", - "itemId": "evidence-000023", + "turnId": "019ff209-393b-79c2-bb53-6be66cf76b9a", + "itemId": "evidence-000015", "receiptIds": [ - "evidence-000025" + "evidence-000016" ], "terminalPresent": true }, @@ -463,7 +463,7 @@ "observed": "allowed" }, "efficiency": { - "latencyMs": 5691, + "latencyMs": 2986, "attempts": 1 }, "budget": { @@ -491,8 +491,8 @@ "trace": { "runId": "run-capability-live-rs-dependency-blocked-wake-01", "sessionId": "session-capability-live-rs-dependency-blocked-wake-01", - "turnId": "019ff1e8-17fc-7600-aa46-84429760dcdd", - "itemId": "transcript-000023", + "turnId": "019ff209-45de-7522-a2b0-1d92a284d0e0", + "itemId": "transcript-000016", "receiptIds": [], "terminalPresent": true }, @@ -501,7 +501,7 @@ "observed": "absent" }, "efficiency": { - "latencyMs": 1908, + "latencyMs": 1895, "attempts": 1 }, "budget": { @@ -533,19 +533,19 @@ "trace": { "runId": "run-capability-live-se-get-issue-01", "sessionId": "session-capability-live-se-get-issue-01", - "turnId": "019ff1e8-20ea-73e2-9cc6-3e2d8f6fc948", - "itemId": "evidence-000024", + "turnId": "019ff209-51ed-70b2-95ee-bdadb2065065", + "itemId": "evidence-000014", "receiptIds": [ - "evidence-000025" + "evidence-000016" ], "terminalPresent": true }, "authorization": { "expected": "allowed", - "observed": "denied" + "observed": "allowed" }, "efficiency": { - "latencyMs": 4504, + "latencyMs": 3067, "attempts": 1 }, "budget": { @@ -577,10 +577,10 @@ "trace": { "runId": "run-capability-live-st-backlog-park-01", "sessionId": "session-capability-live-st-backlog-park-01", - "turnId": "019ff1e8-33f6-7f93-88b9-83ad36eac2bb", - "itemId": "evidence-000022", + "turnId": "019ff209-6083-7312-bed2-2d89633f7a43", + "itemId": "evidence-000015", "receiptIds": [ - "evidence-000023" + "evidence-000016" ], "terminalPresent": true }, @@ -589,7 +589,7 @@ "observed": "allowed" }, "efficiency": { - "latencyMs": 4513, + "latencyMs": 3508, "attempts": 1 }, "budget": { @@ -621,10 +621,10 @@ "trace": { "runId": "run-capability-live-su-crossteam-billing-01", "sessionId": "session-capability-live-su-crossteam-billing-01", - "turnId": "019ff1e8-470b-7550-a6b6-8c0c842eb948", - "itemId": "evidence-000022", + "turnId": "019ff209-6f30-7003-8833-a84e62737f6b", + "itemId": "evidence-000015", "receiptIds": [ - "evidence-000023" + "evidence-000016" ], "terminalPresent": true }, @@ -633,7 +633,7 @@ "observed": "allowed" }, "efficiency": { - "latencyMs": 6353, + "latencyMs": 4524, "attempts": 1 }, "budget": { @@ -661,8 +661,8 @@ "trace": { "runId": "run-capability-live-wk-ask-mode-01", "sessionId": "session-capability-live-wk-ask-mode-01", - "turnId": "019ff1e8-617d-7ba3-8c12-84e6c5d011d0", - "itemId": "transcript-000023", + "turnId": "019ff209-81ce-71a3-973c-5afea691f385", + "itemId": "transcript-000016", "receiptIds": [], "terminalPresent": true }, @@ -671,7 +671,7 @@ "observed": "absent" }, "efficiency": { - "latencyMs": 1731, + "latencyMs": 1498, "attempts": 1 }, "budget": { diff --git a/packages/paperclip-runner/knowledge/evidence/capability/live-codex-matrix.md b/packages/paperclip-runner/knowledge/evidence/capability/live-codex-matrix.md index bb9e681c356..c65b9cdfb23 100644 --- a/packages/paperclip-runner/knowledge/evidence/capability/live-codex-matrix.md +++ b/packages/paperclip-runner/knowledge/evidence/capability/live-codex-matrix.md @@ -2,7 +2,7 @@ type: Evidence Report title: Capability Live Codex Matrix description: Real Codex semantic-tool choices and resulting mock control-plane state. -generated: { by: process:capability-live-eval, at: 2026-08-11T17:38:30.000Z } +generated: { by: process:capability-live-eval, at: 2026-08-11T18:14:30.145Z } status: stable --- diff --git a/packages/paperclip-runner/knowledge/evidence/capability/live-eval-slice-report.json b/packages/paperclip-runner/knowledge/evidence/capability/live-eval-slice-report.json index a13b3a43d3b..d8fa313c217 100644 --- a/packages/paperclip-runner/knowledge/evidence/capability/live-eval-slice-report.json +++ b/packages/paperclip-runner/knowledge/evidence/capability/live-eval-slice-report.json @@ -68,15 +68,15 @@ "trace": { "runId": "run-capability-live-ap-approval-deny-01", "sessionId": "session-capability-live-ap-approval-deny-01", - "turnId": "019ff1e7-5e9e-7353-abc9-734f85ed535b", - "itemId": "evidence-000022", + "turnId": "019ff208-7d1d-74f3-8bb2-1f2489e336d5", + "itemId": "evidence-000015", "receiptIds": [ - "evidence-000023" + "evidence-000016" ], "terminalPresent": true }, "efficiency": { - "latencyMs": 4056, + "latencyMs": 5762, "attempts": 1 }, "budget": { @@ -162,15 +162,15 @@ "trace": { "runId": "run-capability-live-ar-no-done-without-upload-01", "sessionId": "session-capability-live-ar-no-done-without-upload-01", - "turnId": "019ff1e7-6ffb-7123-9f0a-37ca2f894aa2", - "itemId": "evidence-000022", + "turnId": "019ff208-94a3-7be2-a152-6558026f2c81", + "itemId": "evidence-000015", "receiptIds": [ - "evidence-000023" + "evidence-000016" ], "terminalPresent": true }, "efficiency": { - "latencyMs": 3568, + "latencyMs": 6153, "attempts": 1 }, "budget": { @@ -256,15 +256,15 @@ "trace": { "runId": "run-capability-live-bl-cancelled-not-resolved-01", "sessionId": "session-capability-live-bl-cancelled-not-resolved-01", - "turnId": "019ff1e7-7f46-7292-b3ab-5b5d6b99e09f", - "itemId": "evidence-000022", + "turnId": "019ff208-ada0-7402-8f7c-9170569cd3c8", + "itemId": "evidence-000015", "receiptIds": [ - "evidence-000023" + "evidence-000016" ], "terminalPresent": true }, "efficiency": { - "latencyMs": 4079, + "latencyMs": 3412, "attempts": 1 }, "budget": { @@ -350,15 +350,15 @@ "trace": { "runId": "run-capability-live-cm-mention-structured-01", "sessionId": "session-capability-live-cm-mention-structured-01", - "turnId": "019ff1e7-909c-70e2-b9b7-8e53cd6dac67", - "itemId": "evidence-000021", + "turnId": "019ff208-bbe7-7cb0-b299-9fe09ccaf31a", + "itemId": "evidence-000015", "receiptIds": [ - "evidence-000023" + "evidence-000016" ], "terminalPresent": true }, "efficiency": { - "latencyMs": 3621, + "latencyMs": 3525, "attempts": 1 }, "budget": { @@ -440,13 +440,13 @@ "trace": { "runId": "run-capability-live-co-409-stop-01", "sessionId": "session-capability-live-co-409-stop-01", - "turnId": "019ff1e7-a051-7760-992c-39e9c9f2b09c", - "itemId": "transcript-000023", + "turnId": "019ff208-caa9-7573-9c98-cdfda85e0d6a", + "itemId": "transcript-000016", "receiptIds": [], "terminalPresent": true }, "efficiency": { - "latencyMs": 1864, + "latencyMs": 4763, "attempts": 1 }, "budget": { @@ -534,15 +534,15 @@ "trace": { "runId": "run-capability-live-dp-base-revision-01", "sessionId": "session-capability-live-dp-base-revision-01", - "turnId": "019ff1e7-a905-7551-b66a-08d1fd0832fd", - "itemId": "evidence-000022", + "turnId": "019ff208-de3c-7572-8b4c-622fa11b671b", + "itemId": "evidence-000015", "receiptIds": [ - "evidence-000023" + "evidence-000016" ], "terminalPresent": true }, "efficiency": { - "latencyMs": 3685, + "latencyMs": 3258, "attempts": 1 }, "budget": { @@ -624,13 +624,13 @@ "trace": { "runId": "run-capability-live-er-blocked-dedup-01", "sessionId": "session-capability-live-er-blocked-dedup-01", - "turnId": "019ff1e7-b8d6-7eb2-9d4d-e98b659fdfc0", - "itemId": "transcript-000023", + "turnId": "019ff208-ebe0-74c3-8ec0-5d771d020a9c", + "itemId": "transcript-000016", "receiptIds": [], "terminalPresent": true }, "efficiency": { - "latencyMs": 2033, + "latencyMs": 4953, "attempts": 1 }, "budget": { @@ -714,13 +714,13 @@ "trace": { "runId": "run-capability-live-hb-context-01", "sessionId": "session-capability-live-hb-context-01", - "turnId": "019ff1e7-c239-7462-8d23-dda298d0fa58", - "itemId": "transcript-000023", + "turnId": "019ff209-0043-7423-9957-be6301c6e951", + "itemId": "transcript-000016", "receiptIds": [], "terminalPresent": true }, "efficiency": { - "latencyMs": 1658, + "latencyMs": 1660, "attempts": 1 }, "budget": { @@ -808,15 +808,15 @@ "trace": { "runId": "run-capability-live-ix-checkbox-01", "sessionId": "session-capability-live-ix-checkbox-01", - "turnId": "019ff1e7-ca3a-7ad2-9d67-35ee17cb6042", - "itemId": "evidence-000021", + "turnId": "019ff209-0839-7db2-9a98-bd9296ef4db5", + "itemId": "evidence-000015", "receiptIds": [ - "evidence-000023" + "evidence-000016" ], "terminalPresent": true }, "efficiency": { - "latencyMs": 9631, + "latencyMs": 8222, "attempts": 1 }, "budget": { @@ -902,15 +902,15 @@ "trace": { "runId": "run-capability-live-mh-blocked-handoff-01", "sessionId": "session-capability-live-mh-blocked-handoff-01", - "turnId": "019ff1e7-f14e-7962-91e0-60f4b2c1ee5e", - "itemId": "evidence-000022", + "turnId": "019ff209-2953-7062-a513-6e7e28b324af", + "itemId": "evidence-000015", "receiptIds": [ - "evidence-000023" + "evidence-000016" ], "terminalPresent": true }, "efficiency": { - "latencyMs": 3474, + "latencyMs": 3811, "attempts": 1 }, "budget": { @@ -996,15 +996,15 @@ "trace": { "runId": "run-capability-live-rf-api-404-report-01", "sessionId": "session-capability-live-rf-api-404-report-01", - "turnId": "019ff1e8-0054-7102-8629-5a629712bc29", - "itemId": "evidence-000023", + "turnId": "019ff209-393b-79c2-bb53-6be66cf76b9a", + "itemId": "evidence-000015", "receiptIds": [ - "evidence-000025" + "evidence-000016" ], "terminalPresent": true }, "efficiency": { - "latencyMs": 5691, + "latencyMs": 2986, "attempts": 1 }, "budget": { @@ -1086,13 +1086,13 @@ "trace": { "runId": "run-capability-live-rs-dependency-blocked-wake-01", "sessionId": "session-capability-live-rs-dependency-blocked-wake-01", - "turnId": "019ff1e8-17fc-7600-aa46-84429760dcdd", - "itemId": "transcript-000023", + "turnId": "019ff209-45de-7522-a2b0-1d92a284d0e0", + "itemId": "transcript-000016", "receiptIds": [], "terminalPresent": true }, "efficiency": { - "latencyMs": 1908, + "latencyMs": 1895, "attempts": 1 }, "budget": { @@ -1175,20 +1175,20 @@ }, "authorization": { "expected": "allowed", - "observed": "denied" + "observed": "allowed" }, "trace": { "runId": "run-capability-live-se-get-issue-01", "sessionId": "session-capability-live-se-get-issue-01", - "turnId": "019ff1e8-20ea-73e2-9cc6-3e2d8f6fc948", - "itemId": "evidence-000024", + "turnId": "019ff209-51ed-70b2-95ee-bdadb2065065", + "itemId": "evidence-000014", "receiptIds": [ - "evidence-000025" + "evidence-000016" ], "terminalPresent": true }, "efficiency": { - "latencyMs": 4504, + "latencyMs": 3067, "attempts": 1 }, "budget": { @@ -1274,15 +1274,15 @@ "trace": { "runId": "run-capability-live-st-backlog-park-01", "sessionId": "session-capability-live-st-backlog-park-01", - "turnId": "019ff1e8-33f6-7f93-88b9-83ad36eac2bb", - "itemId": "evidence-000022", + "turnId": "019ff209-6083-7312-bed2-2d89633f7a43", + "itemId": "evidence-000015", "receiptIds": [ - "evidence-000023" + "evidence-000016" ], "terminalPresent": true }, "efficiency": { - "latencyMs": 4513, + "latencyMs": 3508, "attempts": 1 }, "budget": { @@ -1368,15 +1368,15 @@ "trace": { "runId": "run-capability-live-su-crossteam-billing-01", "sessionId": "session-capability-live-su-crossteam-billing-01", - "turnId": "019ff1e8-470b-7550-a6b6-8c0c842eb948", - "itemId": "evidence-000022", + "turnId": "019ff209-6f30-7003-8833-a84e62737f6b", + "itemId": "evidence-000015", "receiptIds": [ - "evidence-000023" + "evidence-000016" ], "terminalPresent": true }, "efficiency": { - "latencyMs": 6353, + "latencyMs": 4524, "attempts": 1 }, "budget": { @@ -1458,13 +1458,13 @@ "trace": { "runId": "run-capability-live-wk-ask-mode-01", "sessionId": "session-capability-live-wk-ask-mode-01", - "turnId": "019ff1e8-617d-7ba3-8c12-84e6c5d011d0", - "itemId": "transcript-000023", + "turnId": "019ff209-81ce-71a3-973c-5afea691f385", + "itemId": "transcript-000016", "receiptIds": [], "terminalPresent": true }, "efficiency": { - "latencyMs": 1731, + "latencyMs": 1498, "attempts": 1 }, "budget": { diff --git a/packages/paperclip-runner/knowledge/evidence/capability/live-eval-slice-report.md b/packages/paperclip-runner/knowledge/evidence/capability/live-eval-slice-report.md index 448afaba40e..5d146b9d2b8 100644 --- a/packages/paperclip-runner/knowledge/evidence/capability/live-eval-slice-report.md +++ b/packages/paperclip-runner/knowledge/evidence/capability/live-eval-slice-report.md @@ -2,7 +2,7 @@ type: Evidence Report title: Live Runner Eval Slice Report description: Scored real-Codex capability matrix bound to its secret-free eval bundle. -generated: { by: process:capability-live-eval, at: 2026-08-11T17:38:30.000Z } +generated: { by: process:capability-live-eval, at: 2026-08-11T18:14:30.145Z } status: stable --- diff --git a/packages/paperclip-runner/src/conformance/capability-eval-suite.ts b/packages/paperclip-runner/src/conformance/capability-eval-suite.ts index 2e1d6f7276f..7fb39fea7f5 100644 --- a/packages/paperclip-runner/src/conformance/capability-eval-suite.ts +++ b/packages/paperclip-runner/src/conformance/capability-eval-suite.ts @@ -6,6 +6,7 @@ import type { CapabilityFixtureSeed, CapabilityJsonValue } from "../mock-core/ca import { CapabilitySemanticToolRuntime } from "../tools/capability-semantic-tool-runtime.js"; import { CapabilityCodexToolBinding, CapabilityFakeAgentToolBinding } from "../tools/capability-tool-bindings.js"; import { capabilityFixtureRunCapabilities } from "../scenarios/fixture-run-capabilities.js"; +import { capabilitySemanticToolDescriptor } from "../semantic-tools/catalog.js"; import { CapabilityLiveSessionService } from "../live/live-session.js"; const GENERATED_HEADER = "# GENERATED FILE — DO NOT EDIT. Run pnpm generate:capability-inventory.\n"; @@ -323,7 +324,12 @@ export async function runCapabilityLiveCodexMatrix( const expectedCalls = row.primaryDisposition === "control_plane_owned" ? [] : [plan.operationId]; const liveInput = { ...(typeof plan.input === "object" && plan.input !== null && !Array.isArray(plan.input) ? plan.input : {}), - idempotencyKey: `capability-live:${row.id}`, + // Only mutating operations declare a retry key. Read operations close + // their schema with `additionalProperties: false`, so injecting one + // unconditionally makes the model dictate an input the tool must reject. + ...(operationAcceptsIdempotencyKey(plan.operationId) + ? { idempotencyKey: `capability-live:${row.id}` } + : {}), }; const prompt = expectedCalls.length === 0 ? "Do not call any tools. Reply with exactly: control-plane-owned." @@ -337,10 +343,18 @@ export async function runCapabilityLiveCodexMatrix( .map((entry) => String(entry.data.operationId)); const expectedState = operationMutatesState(plan.operationId) ? "mutated" : "unchanged"; const observedState = snapshot.mockState === initialMockState ? "unchanged" : "mutated"; + // A rejected call still emits a `tool_call` entry, and a rejected read + // leaves state `unchanged` exactly like a successful one. Comparing calls + // and state alone therefore reports such a case as green, so the denial + // itself has to be part of the gate. + const deniedCalls = snapshot.evidence + .filter((entry) => entry.kind === "tool_result" && isDeniedToolResult(entry.data.result)) + .map((entry) => String(entry.data.operationId)); if ( turn.status !== "completed" || JSON.stringify(observedCalls) !== JSON.stringify(expectedCalls) || - observedState !== expectedState + observedState !== expectedState || + deniedCalls.length > 0 ) { throw new Error(JSON.stringify({ caseId: row.id, @@ -349,6 +363,7 @@ export async function runCapabilityLiveCodexMatrix( observedCalls, expectedState, observedState, + deniedCalls, toolResults: snapshot.evidence.filter((entry) => entry.kind === "tool_result").map((entry) => entry.data.result), })); } @@ -420,6 +435,17 @@ function operationMutatesState(operationId: string): boolean { return !["checkout_task", "search_tasks", "list_agents"].includes(operationId); } +/** True when the operation's own schema declares the retry key. */ +function operationAcceptsIdempotencyKey(operationId: string): boolean { + const properties = capabilitySemanticToolDescriptor(operationId)?.inputSchema?.properties; + return typeof properties === "object" && properties !== null && "idempotencyKey" in properties; +} + +/** A typed semantic result the dispatcher rejected rather than applied. */ +function isDeniedToolResult(result: unknown): boolean { + return typeof result === "object" && result !== null && (result as { ok?: unknown }).ok === false; +} + async function toolSurfaceParity(sample: Array<{ caseId: string; group: string; semanticOperation: string }>) { const { runtime } = await runtimeFor([ "discovery:tasks:read", diff --git a/packages/paperclip-runner/src/eval/eval-scoring.test.ts b/packages/paperclip-runner/src/eval/eval-scoring.test.ts index 9b35cabcc2f..a7caa6a13a0 100644 --- a/packages/paperclip-runner/src/eval/eval-scoring.test.ts +++ b/packages/paperclip-runner/src/eval/eval-scoring.test.ts @@ -61,6 +61,45 @@ describe("scoreEval — green vertical slice", () => { }); }); +describe("scoreEval — a denied call cannot score a passing outcome", () => { + // Regression: a rejected read operation leaves the control plane `unchanged` + // exactly like a successful one, so comparing final state alone scored the + // case 1.0 across every dimension. This masked a real live-matrix failure + // where `search_tasks` was rejected as `input_invalid`. + it("fails semantic outcome when a declared-allowed read is denied", () => { + const card = scoreEval( + greenObservation({ + caseId: "se-get-issue-01", + expectedCalls: ["search_tasks"], + observedCalls: ["search_tasks"], + finalState: { expected: "unchanged", observed: "unchanged" }, + authorization: { expected: "allowed", observed: "denied" }, + }), + OPTIONS, + ); + expect(card.dimensions.semantic_outcome.score).toBe(0); + expect(card.dimensions.semantic_outcome.reasons.join(" ")).toContain("denied"); + expect(card.overall.passed).toBe(false); + // The model still chose the right tool, so trajectory stays green — the + // failure is the operation's effect, not the model's selection. + expect(card.dimensions.trajectory_restraint.score).toBe(1); + }); + + it("keeps a correctly denied case green when denial is what the case declared", () => { + const card = scoreEval( + greenObservation({ + expectedCalls: [], + observedCalls: [], + finalState: { expected: "unchanged", observed: "unchanged" }, + authorization: { expected: "denied", observed: "denied" }, + }), + OPTIONS, + ); + expect(card.dimensions.semantic_outcome.score).toBe(1); + expect(card.overall.passed).toBe(true); + }); +}); + describe("scoreEval — the hard-invariant gate", () => { it("forces overall 0 when a forbidden call is invoked, independent of outcome", () => { const card = scoreEval( diff --git a/packages/paperclip-runner/src/eval/eval-scoring.ts b/packages/paperclip-runner/src/eval/eval-scoring.ts index 0a79fa95cdf..4a09a8379dc 100644 --- a/packages/paperclip-runner/src/eval/eval-scoring.ts +++ b/packages/paperclip-runner/src/eval/eval-scoring.ts @@ -168,13 +168,18 @@ function scoreHardInvariants(obs: EvalObservation): { score: number; reasons: st } function scoreSemanticOutcome(obs: EvalObservation): { score: number; reasons: string[] } { - const match = obs.finalState.observed === obs.finalState.expected; - return { - score: match ? 1 : 0, - reasons: match - ? [] - : [`control-plane state ${obs.finalState.observed}, expected ${obs.finalState.expected}`], - }; + const reasons: string[] = []; + if (obs.finalState.observed !== obs.finalState.expected) { + reasons.push(`control-plane state ${obs.finalState.observed}, expected ${obs.finalState.expected}`); + } + // A rejected call cannot have produced its semantic effect. Without this the + // state comparison alone scores a denied read operation as a pass, because a + // rejected read leaves the control plane `unchanged` exactly like a + // successful one — the outcome would match by coincidence, not by effect. + if (obs.authorization.expected === "allowed" && obs.authorization.observed !== "allowed") { + reasons.push(`operation ${obs.authorization.observed} but the case declared it allowed`); + } + return { score: reasons.length === 0 ? 1 : 0, reasons }; } function scoreTrajectoryRestraint(obs: EvalObservation): { score: number; reasons: string[] } {