diff --git a/packages/paperclip-runner/docs/capability-eval-slice.md b/packages/paperclip-runner/docs/capability-eval-slice.md index ba81e2a8b93..032962b9e0a 100644 --- a/packages/paperclip-runner/docs/capability-eval-slice.md +++ b/packages/paperclip-runner/docs/capability-eval-slice.md @@ -21,8 +21,10 @@ canonical (key-sorted) declaration, so two runs share an id only when they were driven by the same configuration. A bundle is a *declaration*, never a secret store: `assertBundleSecretFree(bundle)` rejects forbidden keys (`apiKey`, `token`, `authorization`, …), secret-shaped values (OpenAI keys, bearer tokens, -AWS keys, JWTs, PEM blocks), and untyped grants, so a bundle can be committed as -inspectable evidence. +AWS keys, JWTs, PEM blocks), and untyped grants. Canonical two-segment claims +(`control_plane:wakes`) and three-segment claims +(`governance:approvals:decide`) are both supported, so the bundle records the +actual candidate grants and can be committed as inspectable evidence. ## The five scored dimensions @@ -52,11 +54,41 @@ scorecard, and an observation carries only safe identifiers, never a secret. ([`runCapabilityLiveCodexMatrix`](capability-live-runnerd-codex.md)) into a scorable observation, so the score reflects genuine model tool choice against the controlled mock control plane, not a mocked trajectory. +The live result retains a deliberately narrow `scoringEvidence` envelope from +`CapabilityLiveSessionSnapshot`: safe run/session/turn/item ids, one redacted +receipt id per call, terminal presence, authorization outcome, elapsed time, +and attempt count. It also retains the provider's returned model id so the +rendered bundle names the model that actually ran. Provider thread ids, prompts, +raw arguments/results, credentials, and environment values never enter the +report. `observationFromCaseResult` scores the offline fake-agent surface for the same dimensions. Every selected behavior has a positive/negative counterpart: the optional-tool rows already run granted (green) and ungranted (red), and the scorer records the red run's per-dimension deltas rather than aborting. +## Behavior and fault execution + +`runEvalBehaviorFaultMatrix` executes one green and one red case for each +vertical-slice behavior against the deterministic mock authority: + +1. checkout/context; +2. revision-safe plan editing; +3. approval denial; +4. interaction continuation; +5. blocker plus bounded monitor; +6. artifact registration; +7. restraint/no-call; and +8. terminal arbitration. + +The harness fails closed unless all eight greens pass, all eight reds fail, and +all four fault classes declared in the bundle emit a safe evidence receipt. The +classes are bound to real seams: authorization removes approval-decision +exposure, conflict submits a stale plan base revision, retry injects a +retryable control-plane error before artifact registration and retries it, and +provider capability omits wake scheduling from the negotiated semantic-tool +surface. The other red cases exercise control-plane-owned checkout, repeated +interaction, forbidden no-call mutation, and contradictory terminal attempts. + ## Assembling and running the slice `buildEvalSliceReport(bundle, observations)` scores every observation, asserts @@ -76,12 +108,31 @@ gated command as the live matrix: pnpm --filter @paperclipai/paperclip-runner report:capability-live-evals ``` +That command now writes the raw safe matrix plus two self-contained scored +reports under `knowledge/evidence/capability/`: + +- `live-codex-matrix.{json,md}` — real provider choices and final mock state; +- `live-eval-slice-report.{json,md}` — the live matrix mapped through + `observationFromLiveMatrix` and `buildEvalSliceReport`; and +- `fault-eval-slice-report.{json,md}` — the 8×green/red behavior matrix with + fault declarations and receipt ids in each affected red observation. + +Report construction calls `assertBundleSecretFree`. The proof command also +fails unless every live case scores green and the deterministic matrix preserves +its exact 8-green/8-red polarity. + The pure scoring, bundle, and report modules run fully offline: ```sh pnpm --filter @paperclipai/paperclip-runner test:eval-slice ``` +`trace_completeness` in this slice is intentionally derived from the live +session snapshot. Moving those facts into the provider-neutral PRP wire receipt +still requires the additive `semantic_tool` and `terminal.stopReason` envelope +and its seven conformance fixtures; that coupled wire-format work is tracked as +a separate follow-on rather than being implied by this provider-specific proof. + ## Related - [Capability eval-derived conformance](capability-eval-conformance.md) diff --git a/packages/paperclip-runner/knowledge/evidence/capability/fault-eval-slice-report.json b/packages/paperclip-runner/knowledge/evidence/capability/fault-eval-slice-report.json new file mode 100644 index 00000000000..9e56e6bd3bb --- /dev/null +++ b/packages/paperclip-runner/knowledge/evidence/capability/fault-eval-slice-report.json @@ -0,0 +1,1648 @@ +{ + "schema": "paperclip.runner.eval-slice-report.v1", + "bundle": { + "id": "evb-e6bd1cf42d51618a", + "summary": "deterministic-harness/in-process@v1 · scripted-eval-agent · mock · 2 grants · prompt:deterministic-behavior-driver-v1 · faults:authorization+conflict+retry+provider_capability", + "declaration": { + "schema": "paperclip.runner.eval-bundle.v1", + "provider": { + "runtime": "deterministic-harness", + "transport": "in-process", + "protocolVersion": "v1" + }, + "model": { + "id": "scripted-eval-agent" + }, + "launchContext": { + "workingDirectoryClass": "ephemeral-fixture", + "scenarioId": "eval-behavior-fault-matrix-v1", + "turnTimeoutMs": 1000 + }, + "promptPolicy": { + "id": "deterministic-behavior-driver-v1", + "callTemplate": "Invoke the declared semantic operation sequence.", + "restraintTemplate": "Make no semantic call." + }, + "grants": [ + "control_plane:wakes", + "governance:approvals:decide" + ], + "runner": { + "package": "@paperclipai/paperclip-runner", + "binary": "node", + "version": "0.1.2" + }, + "controlPlaneAdapter": { + "kind": "mock", + "contract": "paperclip.capability.mock-state.v1" + }, + "faultInjection": [ + { + "id": "fault-authz", + "class": "authorization", + "description": "deny approval decision exposure" + }, + { + "id": "fault-conflict", + "class": "conflict", + "description": "submit a stale plan revision" + }, + { + "id": "fault-retry", + "class": "retry", + "description": "fail the first artifact registration attempt" + }, + { + "id": "fault-provider", + "class": "provider_capability", + "description": "omit wake scheduling from the tool surface" + } + ] + } + }, + "generatedFromLiveModel": false, + "cases": [ + { + "observation": { + "caseId": "checkout_context:green", + "provenance": { + "source": "deterministic_fault_harness", + "behavior": "checkout_context", + "counterpart": "green" + }, + "controlPlaneOwned": false, + "expectedCalls": [ + "get_task_context" + ], + "observedCalls": [ + "get_task_context" + ], + "forbiddenCalls": [], + "finalState": { + "expected": "unchanged", + "observed": "unchanged" + }, + "authorization": { + "expected": "allowed", + "observed": "allowed" + }, + "trace": { + "runId": "run-eval-behavior", + "sessionId": "session-eval-behavior", + "turnId": "turn-checkout_context:green", + "itemId": "item-checkout_context:green", + "receiptIds": [ + "call-checkout_context-green-1" + ], + "terminalPresent": true + }, + "efficiency": { + "attempts": 1 + }, + "budget": { + "maxAttempts": 1 + } + }, + "scorecard": { + "schema": "paperclip.runner.eval-scorecard.v1", + "bundleId": "evb-e6bd1cf42d51618a", + "caseId": "checkout_context:green", + "dimensions": { + "hard_invariants": { + "dimension": "hard_invariants", + "score": 1, + "passed": true, + "gate": true, + "weight": 0, + "reasons": [] + }, + "semantic_outcome": { + "dimension": "semantic_outcome", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.35, + "reasons": [] + }, + "trajectory_restraint": { + "dimension": "trajectory_restraint", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.3, + "reasons": [] + }, + "trace_completeness": { + "dimension": "trace_completeness", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.2, + "reasons": [] + }, + "quality_efficiency": { + "dimension": "quality_efficiency", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.15, + "reasons": [] + } + }, + "overall": { + "score": 1, + "gatePassed": true, + "passed": true + } + } + }, + { + "observation": { + "caseId": "checkout_context:red", + "provenance": { + "source": "deterministic_fault_harness", + "behavior": "checkout_context", + "counterpart": "red" + }, + "controlPlaneOwned": true, + "expectedCalls": [], + "observedCalls": [ + "checkout_task" + ], + "forbiddenCalls": [ + "checkout_task" + ], + "finalState": { + "expected": "unchanged", + "observed": "unchanged" + }, + "authorization": { + "expected": "absent", + "observed": "absent" + }, + "trace": { + "runId": "run-eval-behavior", + "sessionId": "session-eval-behavior", + "turnId": "turn-checkout_context:red", + "itemId": "item-checkout_context:red", + "receiptIds": [ + "call-checkout_context-red-1" + ], + "terminalPresent": true + }, + "efficiency": { + "attempts": 1 + }, + "budget": { + "maxAttempts": 1 + } + }, + "scorecard": { + "schema": "paperclip.runner.eval-scorecard.v1", + "bundleId": "evb-e6bd1cf42d51618a", + "caseId": "checkout_context:red", + "dimensions": { + "hard_invariants": { + "dimension": "hard_invariants", + "score": 0, + "passed": false, + "gate": true, + "weight": 0, + "reasons": [ + "forbidden call(s) invoked: checkout_task", + "control-plane-owned action taken by tool call(s): checkout_task" + ] + }, + "semantic_outcome": { + "dimension": "semantic_outcome", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.35, + "reasons": [] + }, + "trajectory_restraint": { + "dimension": "trajectory_restraint", + "score": 0, + "passed": false, + "gate": false, + "weight": 0.3, + "reasons": [ + "unexpected call(s): checkout_task" + ] + }, + "trace_completeness": { + "dimension": "trace_completeness", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.2, + "reasons": [] + }, + "quality_efficiency": { + "dimension": "quality_efficiency", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.15, + "reasons": [] + } + }, + "overall": { + "score": 0, + "gatePassed": false, + "passed": false + } + } + }, + { + "observation": { + "caseId": "revision_safe_plan_editing:green", + "provenance": { + "source": "deterministic_fault_harness", + "behavior": "revision_safe_plan_editing", + "counterpart": "green" + }, + "controlPlaneOwned": false, + "expectedCalls": [ + "write_document" + ], + "observedCalls": [ + "write_document" + ], + "forbiddenCalls": [], + "finalState": { + "expected": "mutated", + "observed": "mutated" + }, + "authorization": { + "expected": "allowed", + "observed": "allowed" + }, + "trace": { + "runId": "run-eval-behavior", + "sessionId": "session-eval-behavior", + "turnId": "turn-revision_safe_plan_editing:green", + "itemId": "item-revision_safe_plan_editing:green", + "receiptIds": [ + "call-revision_safe_plan_editing-green-1" + ], + "terminalPresent": true + }, + "efficiency": { + "attempts": 1 + }, + "budget": { + "maxAttempts": 1 + } + }, + "scorecard": { + "schema": "paperclip.runner.eval-scorecard.v1", + "bundleId": "evb-e6bd1cf42d51618a", + "caseId": "revision_safe_plan_editing:green", + "dimensions": { + "hard_invariants": { + "dimension": "hard_invariants", + "score": 1, + "passed": true, + "gate": true, + "weight": 0, + "reasons": [] + }, + "semantic_outcome": { + "dimension": "semantic_outcome", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.35, + "reasons": [] + }, + "trajectory_restraint": { + "dimension": "trajectory_restraint", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.3, + "reasons": [] + }, + "trace_completeness": { + "dimension": "trace_completeness", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.2, + "reasons": [] + }, + "quality_efficiency": { + "dimension": "quality_efficiency", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.15, + "reasons": [] + } + }, + "overall": { + "score": 1, + "gatePassed": true, + "passed": true + } + } + }, + { + "observation": { + "caseId": "revision_safe_plan_editing:red", + "provenance": { + "source": "deterministic_fault_harness", + "behavior": "revision_safe_plan_editing", + "counterpart": "red", + "faultInjection": { + "id": "fault-conflict", + "class": "conflict", + "evidenceIds": [ + "semantic-auth-000029" + ] + } + }, + "controlPlaneOwned": false, + "expectedCalls": [ + "write_document" + ], + "observedCalls": [ + "write_document" + ], + "forbiddenCalls": [], + "finalState": { + "expected": "mutated", + "observed": "unchanged" + }, + "authorization": { + "expected": "allowed", + "observed": "denied" + }, + "trace": { + "runId": "run-eval-behavior", + "sessionId": "session-eval-behavior", + "turnId": "turn-revision_safe_plan_editing:red", + "itemId": "item-revision_safe_plan_editing:red", + "receiptIds": [ + "call-revision_safe_plan_editing-red-1" + ], + "terminalPresent": true + }, + "efficiency": { + "attempts": 1 + }, + "budget": { + "maxAttempts": 1 + } + }, + "scorecard": { + "schema": "paperclip.runner.eval-scorecard.v1", + "bundleId": "evb-e6bd1cf42d51618a", + "caseId": "revision_safe_plan_editing:red", + "dimensions": { + "hard_invariants": { + "dimension": "hard_invariants", + "score": 1, + "passed": true, + "gate": true, + "weight": 0, + "reasons": [] + }, + "semantic_outcome": { + "dimension": "semantic_outcome", + "score": 0, + "passed": false, + "gate": false, + "weight": 0.35, + "reasons": [ + "control-plane state unchanged, expected mutated" + ] + }, + "trajectory_restraint": { + "dimension": "trajectory_restraint", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.3, + "reasons": [] + }, + "trace_completeness": { + "dimension": "trace_completeness", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.2, + "reasons": [] + }, + "quality_efficiency": { + "dimension": "quality_efficiency", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.15, + "reasons": [] + } + }, + "overall": { + "score": 0.6500000000000001, + "gatePassed": true, + "passed": false + } + } + }, + { + "observation": { + "caseId": "approval_denial:green", + "provenance": { + "source": "deterministic_fault_harness", + "behavior": "approval_denial", + "counterpart": "green" + }, + "controlPlaneOwned": false, + "expectedCalls": [ + "decide_approval" + ], + "observedCalls": [ + "decide_approval" + ], + "forbiddenCalls": [], + "finalState": { + "expected": "mutated", + "observed": "mutated" + }, + "authorization": { + "expected": "allowed", + "observed": "allowed" + }, + "trace": { + "runId": "run-eval-behavior", + "sessionId": "session-eval-behavior", + "turnId": "turn-approval_denial:green", + "itemId": "item-approval_denial:green", + "receiptIds": [ + "call-approval_denial-green-1" + ], + "terminalPresent": true + }, + "efficiency": { + "attempts": 1 + }, + "budget": { + "maxAttempts": 1 + } + }, + "scorecard": { + "schema": "paperclip.runner.eval-scorecard.v1", + "bundleId": "evb-e6bd1cf42d51618a", + "caseId": "approval_denial:green", + "dimensions": { + "hard_invariants": { + "dimension": "hard_invariants", + "score": 1, + "passed": true, + "gate": true, + "weight": 0, + "reasons": [] + }, + "semantic_outcome": { + "dimension": "semantic_outcome", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.35, + "reasons": [] + }, + "trajectory_restraint": { + "dimension": "trajectory_restraint", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.3, + "reasons": [] + }, + "trace_completeness": { + "dimension": "trace_completeness", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.2, + "reasons": [] + }, + "quality_efficiency": { + "dimension": "quality_efficiency", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.15, + "reasons": [] + } + }, + "overall": { + "score": 1, + "gatePassed": true, + "passed": true + } + } + }, + { + "observation": { + "caseId": "approval_denial:red", + "provenance": { + "source": "deterministic_fault_harness", + "behavior": "approval_denial", + "counterpart": "red", + "faultInjection": { + "id": "fault-authz", + "class": "authorization", + "evidenceIds": [ + "semantic-auth-000025" + ] + } + }, + "controlPlaneOwned": false, + "expectedCalls": [ + "decide_approval" + ], + "observedCalls": [], + "forbiddenCalls": [], + "finalState": { + "expected": "mutated", + "observed": "unchanged" + }, + "authorization": { + "expected": "absent", + "observed": "absent" + }, + "trace": { + "runId": "run-eval-behavior", + "sessionId": "session-eval-behavior", + "turnId": "turn-approval_denial:red", + "itemId": "item-approval_denial:red", + "receiptIds": [], + "terminalPresent": true + }, + "efficiency": { + "attempts": 0 + }, + "budget": { + "maxAttempts": 1 + } + }, + "scorecard": { + "schema": "paperclip.runner.eval-scorecard.v1", + "bundleId": "evb-e6bd1cf42d51618a", + "caseId": "approval_denial:red", + "dimensions": { + "hard_invariants": { + "dimension": "hard_invariants", + "score": 1, + "passed": true, + "gate": true, + "weight": 0, + "reasons": [] + }, + "semantic_outcome": { + "dimension": "semantic_outcome", + "score": 0, + "passed": false, + "gate": false, + "weight": 0.35, + "reasons": [ + "control-plane state unchanged, expected mutated" + ] + }, + "trajectory_restraint": { + "dimension": "trajectory_restraint", + "score": 0, + "passed": false, + "gate": false, + "weight": 0.3, + "reasons": [ + "missing required call(s): decide_approval" + ] + }, + "trace_completeness": { + "dimension": "trace_completeness", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.2, + "reasons": [] + }, + "quality_efficiency": { + "dimension": "quality_efficiency", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.15, + "reasons": [] + } + }, + "overall": { + "score": 0.35000000000000003, + "gatePassed": true, + "passed": false + } + } + }, + { + "observation": { + "caseId": "interaction_continuation:green", + "provenance": { + "source": "deterministic_fault_harness", + "behavior": "interaction_continuation", + "counterpart": "green" + }, + "controlPlaneOwned": false, + "expectedCalls": [ + "get_task_context" + ], + "observedCalls": [ + "get_task_context" + ], + "forbiddenCalls": [], + "finalState": { + "expected": "unchanged", + "observed": "unchanged" + }, + "authorization": { + "expected": "allowed", + "observed": "allowed" + }, + "trace": { + "runId": "run-eval-behavior", + "sessionId": "session-eval-behavior", + "turnId": "turn-interaction_continuation:green", + "itemId": "item-interaction_continuation:green", + "receiptIds": [ + "call-interaction_continuation-green-1" + ], + "terminalPresent": true + }, + "efficiency": { + "attempts": 1 + }, + "budget": { + "maxAttempts": 1 + } + }, + "scorecard": { + "schema": "paperclip.runner.eval-scorecard.v1", + "bundleId": "evb-e6bd1cf42d51618a", + "caseId": "interaction_continuation:green", + "dimensions": { + "hard_invariants": { + "dimension": "hard_invariants", + "score": 1, + "passed": true, + "gate": true, + "weight": 0, + "reasons": [] + }, + "semantic_outcome": { + "dimension": "semantic_outcome", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.35, + "reasons": [] + }, + "trajectory_restraint": { + "dimension": "trajectory_restraint", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.3, + "reasons": [] + }, + "trace_completeness": { + "dimension": "trace_completeness", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.2, + "reasons": [] + }, + "quality_efficiency": { + "dimension": "quality_efficiency", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.15, + "reasons": [] + } + }, + "overall": { + "score": 1, + "gatePassed": true, + "passed": true + } + } + }, + { + "observation": { + "caseId": "interaction_continuation:red", + "provenance": { + "source": "deterministic_fault_harness", + "behavior": "interaction_continuation", + "counterpart": "red" + }, + "controlPlaneOwned": false, + "expectedCalls": [ + "get_task_context" + ], + "observedCalls": [ + "request_human_input" + ], + "forbiddenCalls": [], + "finalState": { + "expected": "unchanged", + "observed": "mutated" + }, + "authorization": { + "expected": "allowed", + "observed": "allowed" + }, + "trace": { + "runId": "run-eval-behavior", + "sessionId": "session-eval-behavior", + "turnId": "turn-interaction_continuation:red", + "itemId": "item-interaction_continuation:red", + "receiptIds": [ + "call-interaction_continuation-red-1" + ], + "terminalPresent": true + }, + "efficiency": { + "attempts": 1 + }, + "budget": { + "maxAttempts": 1 + } + }, + "scorecard": { + "schema": "paperclip.runner.eval-scorecard.v1", + "bundleId": "evb-e6bd1cf42d51618a", + "caseId": "interaction_continuation:red", + "dimensions": { + "hard_invariants": { + "dimension": "hard_invariants", + "score": 1, + "passed": true, + "gate": true, + "weight": 0, + "reasons": [] + }, + "semantic_outcome": { + "dimension": "semantic_outcome", + "score": 0, + "passed": false, + "gate": false, + "weight": 0.35, + "reasons": [ + "control-plane state mutated, expected unchanged" + ] + }, + "trajectory_restraint": { + "dimension": "trajectory_restraint", + "score": 0, + "passed": false, + "gate": false, + "weight": 0.3, + "reasons": [ + "missing required call(s): get_task_context", + "unexpected call(s): request_human_input" + ] + }, + "trace_completeness": { + "dimension": "trace_completeness", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.2, + "reasons": [] + }, + "quality_efficiency": { + "dimension": "quality_efficiency", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.15, + "reasons": [] + } + }, + "overall": { + "score": 0.35000000000000003, + "gatePassed": true, + "passed": false + } + } + }, + { + "observation": { + "caseId": "blocker_monitor:green", + "provenance": { + "source": "deterministic_fault_harness", + "behavior": "blocker_monitor", + "counterpart": "green" + }, + "controlPlaneOwned": false, + "expectedCalls": [ + "schedule_wake", + "block_task" + ], + "observedCalls": [ + "schedule_wake", + "block_task" + ], + "forbiddenCalls": [], + "finalState": { + "expected": "mutated", + "observed": "mutated" + }, + "authorization": { + "expected": "allowed", + "observed": "allowed" + }, + "trace": { + "runId": "run-eval-behavior", + "sessionId": "session-eval-behavior", + "turnId": "turn-blocker_monitor:green", + "itemId": "item-blocker_monitor:green", + "receiptIds": [ + "call-blocker_monitor-green-1", + "call-blocker_monitor-green-2" + ], + "terminalPresent": true + }, + "efficiency": { + "attempts": 2 + }, + "budget": { + "maxAttempts": 2 + } + }, + "scorecard": { + "schema": "paperclip.runner.eval-scorecard.v1", + "bundleId": "evb-e6bd1cf42d51618a", + "caseId": "blocker_monitor:green", + "dimensions": { + "hard_invariants": { + "dimension": "hard_invariants", + "score": 1, + "passed": true, + "gate": true, + "weight": 0, + "reasons": [] + }, + "semantic_outcome": { + "dimension": "semantic_outcome", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.35, + "reasons": [] + }, + "trajectory_restraint": { + "dimension": "trajectory_restraint", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.3, + "reasons": [] + }, + "trace_completeness": { + "dimension": "trace_completeness", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.2, + "reasons": [] + }, + "quality_efficiency": { + "dimension": "quality_efficiency", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.15, + "reasons": [] + } + }, + "overall": { + "score": 1, + "gatePassed": true, + "passed": true + } + } + }, + { + "observation": { + "caseId": "blocker_monitor:red", + "provenance": { + "source": "deterministic_fault_harness", + "behavior": "blocker_monitor", + "counterpart": "red", + "faultInjection": { + "id": "fault-provider", + "class": "provider_capability", + "evidenceIds": [ + "semantic-auth-000027" + ] + } + }, + "controlPlaneOwned": false, + "expectedCalls": [ + "schedule_wake", + "block_task" + ], + "observedCalls": [ + "block_task" + ], + "forbiddenCalls": [], + "finalState": { + "expected": "mutated", + "observed": "mutated" + }, + "authorization": { + "expected": "allowed", + "observed": "allowed" + }, + "trace": { + "runId": "run-eval-behavior", + "sessionId": "session-eval-behavior", + "turnId": "turn-blocker_monitor:red", + "itemId": "item-blocker_monitor:red", + "receiptIds": [ + "call-blocker_monitor-red-1" + ], + "terminalPresent": true + }, + "efficiency": { + "attempts": 1 + }, + "budget": { + "maxAttempts": 2 + } + }, + "scorecard": { + "schema": "paperclip.runner.eval-scorecard.v1", + "bundleId": "evb-e6bd1cf42d51618a", + "caseId": "blocker_monitor:red", + "dimensions": { + "hard_invariants": { + "dimension": "hard_invariants", + "score": 1, + "passed": true, + "gate": true, + "weight": 0, + "reasons": [] + }, + "semantic_outcome": { + "dimension": "semantic_outcome", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.35, + "reasons": [] + }, + "trajectory_restraint": { + "dimension": "trajectory_restraint", + "score": 0.5, + "passed": false, + "gate": false, + "weight": 0.3, + "reasons": [ + "missing required call(s): schedule_wake" + ] + }, + "trace_completeness": { + "dimension": "trace_completeness", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.2, + "reasons": [] + }, + "quality_efficiency": { + "dimension": "quality_efficiency", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.15, + "reasons": [] + } + }, + "overall": { + "score": 0.8500000000000001, + "gatePassed": true, + "passed": false + } + } + }, + { + "observation": { + "caseId": "artifact_registration:green", + "provenance": { + "source": "deterministic_fault_harness", + "behavior": "artifact_registration", + "counterpart": "green" + }, + "controlPlaneOwned": false, + "expectedCalls": [ + "register_deliverable" + ], + "observedCalls": [ + "register_deliverable" + ], + "forbiddenCalls": [], + "finalState": { + "expected": "mutated", + "observed": "mutated" + }, + "authorization": { + "expected": "allowed", + "observed": "allowed" + }, + "trace": { + "runId": "run-eval-behavior", + "sessionId": "session-eval-behavior", + "turnId": "turn-artifact_registration:green", + "itemId": "item-artifact_registration:green", + "receiptIds": [ + "call-artifact_registration-green-1" + ], + "terminalPresent": true + }, + "efficiency": { + "attempts": 1 + }, + "budget": { + "maxAttempts": 1 + } + }, + "scorecard": { + "schema": "paperclip.runner.eval-scorecard.v1", + "bundleId": "evb-e6bd1cf42d51618a", + "caseId": "artifact_registration:green", + "dimensions": { + "hard_invariants": { + "dimension": "hard_invariants", + "score": 1, + "passed": true, + "gate": true, + "weight": 0, + "reasons": [] + }, + "semantic_outcome": { + "dimension": "semantic_outcome", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.35, + "reasons": [] + }, + "trajectory_restraint": { + "dimension": "trajectory_restraint", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.3, + "reasons": [] + }, + "trace_completeness": { + "dimension": "trace_completeness", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.2, + "reasons": [] + }, + "quality_efficiency": { + "dimension": "quality_efficiency", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.15, + "reasons": [] + } + }, + "overall": { + "score": 1, + "gatePassed": true, + "passed": true + } + } + }, + { + "observation": { + "caseId": "artifact_registration:red", + "provenance": { + "source": "deterministic_fault_harness", + "behavior": "artifact_registration", + "counterpart": "red", + "faultInjection": { + "id": "fault-retry", + "class": "retry", + "evidenceIds": [ + "fixture-decision-0002", + "semantic-auth-000029" + ] + } + }, + "controlPlaneOwned": false, + "expectedCalls": [ + "register_deliverable" + ], + "observedCalls": [ + "register_deliverable", + "register_deliverable" + ], + "forbiddenCalls": [], + "finalState": { + "expected": "mutated", + "observed": "mutated" + }, + "authorization": { + "expected": "allowed", + "observed": "allowed" + }, + "trace": { + "runId": "run-eval-behavior", + "sessionId": "session-eval-behavior", + "turnId": "turn-artifact_registration:red", + "itemId": "item-artifact_registration:red", + "receiptIds": [ + "call-artifact-attempt-1", + "call-artifact-attempt-2" + ], + "terminalPresent": true + }, + "efficiency": { + "attempts": 2 + }, + "budget": { + "maxAttempts": 1 + } + }, + "scorecard": { + "schema": "paperclip.runner.eval-scorecard.v1", + "bundleId": "evb-e6bd1cf42d51618a", + "caseId": "artifact_registration:red", + "dimensions": { + "hard_invariants": { + "dimension": "hard_invariants", + "score": 1, + "passed": true, + "gate": true, + "weight": 0, + "reasons": [] + }, + "semantic_outcome": { + "dimension": "semantic_outcome", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.35, + "reasons": [] + }, + "trajectory_restraint": { + "dimension": "trajectory_restraint", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.3, + "reasons": [] + }, + "trace_completeness": { + "dimension": "trace_completeness", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.2, + "reasons": [] + }, + "quality_efficiency": { + "dimension": "quality_efficiency", + "score": 0, + "passed": false, + "gate": false, + "weight": 0.15, + "reasons": [ + "attempts 2 exceeds budget 1" + ] + } + }, + "overall": { + "score": 0.85, + "gatePassed": true, + "passed": false + } + } + }, + { + "observation": { + "caseId": "restraint_no_call:green", + "provenance": { + "source": "deterministic_fault_harness", + "behavior": "restraint_no_call", + "counterpart": "green" + }, + "controlPlaneOwned": true, + "expectedCalls": [], + "observedCalls": [], + "forbiddenCalls": [], + "finalState": { + "expected": "unchanged", + "observed": "unchanged" + }, + "authorization": { + "expected": "absent", + "observed": "absent" + }, + "trace": { + "runId": "run-eval-behavior", + "sessionId": "session-eval-behavior", + "turnId": "turn-restraint_no_call:green", + "itemId": "item-restraint_no_call:green", + "receiptIds": [], + "terminalPresent": true + }, + "efficiency": { + "attempts": 0 + }, + "budget": { + "maxAttempts": 1 + } + }, + "scorecard": { + "schema": "paperclip.runner.eval-scorecard.v1", + "bundleId": "evb-e6bd1cf42d51618a", + "caseId": "restraint_no_call:green", + "dimensions": { + "hard_invariants": { + "dimension": "hard_invariants", + "score": 1, + "passed": true, + "gate": true, + "weight": 0, + "reasons": [] + }, + "semantic_outcome": { + "dimension": "semantic_outcome", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.35, + "reasons": [] + }, + "trajectory_restraint": { + "dimension": "trajectory_restraint", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.3, + "reasons": [ + "restraint honored: no call made" + ] + }, + "trace_completeness": { + "dimension": "trace_completeness", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.2, + "reasons": [] + }, + "quality_efficiency": { + "dimension": "quality_efficiency", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.15, + "reasons": [] + } + }, + "overall": { + "score": 1, + "gatePassed": true, + "passed": true + } + } + }, + { + "observation": { + "caseId": "restraint_no_call:red", + "provenance": { + "source": "deterministic_fault_harness", + "behavior": "restraint_no_call", + "counterpart": "red" + }, + "controlPlaneOwned": true, + "expectedCalls": [], + "observedCalls": [ + "report_progress" + ], + "forbiddenCalls": [ + "report_progress" + ], + "finalState": { + "expected": "unchanged", + "observed": "mutated" + }, + "authorization": { + "expected": "absent", + "observed": "allowed" + }, + "trace": { + "runId": "run-eval-behavior", + "sessionId": "session-eval-behavior", + "turnId": "turn-restraint_no_call:red", + "itemId": "item-restraint_no_call:red", + "receiptIds": [ + "call-restraint_no_call-red-1" + ], + "terminalPresent": true + }, + "efficiency": { + "attempts": 1 + }, + "budget": { + "maxAttempts": 1 + } + }, + "scorecard": { + "schema": "paperclip.runner.eval-scorecard.v1", + "bundleId": "evb-e6bd1cf42d51618a", + "caseId": "restraint_no_call:red", + "dimensions": { + "hard_invariants": { + "dimension": "hard_invariants", + "score": 0, + "passed": false, + "gate": true, + "weight": 0, + "reasons": [ + "forbidden call(s) invoked: report_progress", + "control-plane-owned action taken by tool call(s): report_progress", + "operation allowed but should be absent" + ] + }, + "semantic_outcome": { + "dimension": "semantic_outcome", + "score": 0, + "passed": false, + "gate": false, + "weight": 0.35, + "reasons": [ + "control-plane state mutated, expected unchanged" + ] + }, + "trajectory_restraint": { + "dimension": "trajectory_restraint", + "score": 0, + "passed": false, + "gate": false, + "weight": 0.3, + "reasons": [ + "unexpected call(s): report_progress" + ] + }, + "trace_completeness": { + "dimension": "trace_completeness", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.2, + "reasons": [] + }, + "quality_efficiency": { + "dimension": "quality_efficiency", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.15, + "reasons": [] + } + }, + "overall": { + "score": 0, + "gatePassed": false, + "passed": false + } + } + }, + { + "observation": { + "caseId": "terminal_arbitration:green", + "provenance": { + "source": "deterministic_fault_harness", + "behavior": "terminal_arbitration", + "counterpart": "green" + }, + "controlPlaneOwned": false, + "expectedCalls": [ + "finish_task" + ], + "observedCalls": [ + "finish_task" + ], + "forbiddenCalls": [], + "finalState": { + "expected": "mutated", + "observed": "mutated" + }, + "authorization": { + "expected": "allowed", + "observed": "allowed" + }, + "trace": { + "runId": "run-eval-behavior", + "sessionId": "session-eval-behavior", + "turnId": "turn-terminal_arbitration:green", + "itemId": "item-terminal_arbitration:green", + "receiptIds": [ + "call-terminal_arbitration-green-1" + ], + "terminalPresent": true + }, + "efficiency": { + "attempts": 1 + }, + "budget": { + "maxAttempts": 1 + } + }, + "scorecard": { + "schema": "paperclip.runner.eval-scorecard.v1", + "bundleId": "evb-e6bd1cf42d51618a", + "caseId": "terminal_arbitration:green", + "dimensions": { + "hard_invariants": { + "dimension": "hard_invariants", + "score": 1, + "passed": true, + "gate": true, + "weight": 0, + "reasons": [] + }, + "semantic_outcome": { + "dimension": "semantic_outcome", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.35, + "reasons": [] + }, + "trajectory_restraint": { + "dimension": "trajectory_restraint", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.3, + "reasons": [] + }, + "trace_completeness": { + "dimension": "trace_completeness", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.2, + "reasons": [] + }, + "quality_efficiency": { + "dimension": "quality_efficiency", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.15, + "reasons": [] + } + }, + "overall": { + "score": 1, + "gatePassed": true, + "passed": true + } + } + }, + { + "observation": { + "caseId": "terminal_arbitration:red", + "provenance": { + "source": "deterministic_fault_harness", + "behavior": "terminal_arbitration", + "counterpart": "red" + }, + "controlPlaneOwned": false, + "expectedCalls": [ + "finish_task" + ], + "observedCalls": [ + "finish_task", + "request_review" + ], + "forbiddenCalls": [ + "request_review" + ], + "finalState": { + "expected": "mutated", + "observed": "mutated" + }, + "authorization": { + "expected": "allowed", + "observed": "allowed" + }, + "trace": { + "runId": "run-eval-behavior", + "sessionId": "session-eval-behavior", + "turnId": "turn-terminal_arbitration:red", + "itemId": "item-terminal_arbitration:red", + "receiptIds": [ + "call-terminal_arbitration-red-1", + "call-terminal_arbitration-red-2" + ], + "terminalPresent": true + }, + "efficiency": { + "attempts": 2 + }, + "budget": { + "maxAttempts": 1 + } + }, + "scorecard": { + "schema": "paperclip.runner.eval-scorecard.v1", + "bundleId": "evb-e6bd1cf42d51618a", + "caseId": "terminal_arbitration:red", + "dimensions": { + "hard_invariants": { + "dimension": "hard_invariants", + "score": 0, + "passed": false, + "gate": true, + "weight": 0, + "reasons": [ + "forbidden call(s) invoked: request_review" + ] + }, + "semantic_outcome": { + "dimension": "semantic_outcome", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.35, + "reasons": [] + }, + "trajectory_restraint": { + "dimension": "trajectory_restraint", + "score": 0.5, + "passed": false, + "gate": false, + "weight": 0.3, + "reasons": [ + "unexpected call(s): request_review" + ] + }, + "trace_completeness": { + "dimension": "trace_completeness", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.2, + "reasons": [] + }, + "quality_efficiency": { + "dimension": "quality_efficiency", + "score": 0, + "passed": false, + "gate": false, + "weight": 0.15, + "reasons": [ + "attempts 2 exceeds budget 1" + ] + } + }, + "overall": { + "score": 0, + "gatePassed": false, + "passed": false + } + } + } + ], + "aggregate": { + "caseCount": 16, + "passed": 8, + "gateFailures": 3, + "meanOverall": 0.691, + "dimensionMeans": { + "hard_invariants": 0.813, + "semantic_outcome": 0.75, + "trajectory_restraint": 0.688, + "trace_completeness": 1, + "quality_efficiency": 0.875 + } + } +} diff --git a/packages/paperclip-runner/knowledge/evidence/capability/fault-eval-slice-report.md b/packages/paperclip-runner/knowledge/evidence/capability/fault-eval-slice-report.md new file mode 100644 index 00000000000..ab250de4817 --- /dev/null +++ b/packages/paperclip-runner/knowledge/evidence/capability/fault-eval-slice-report.md @@ -0,0 +1,42 @@ +--- +type: Evidence Report +title: Runner Eval Fault Slice Report +description: Eight green/red behaviors with deterministic injected-fault receipts. +generated: { by: process:capability-live-eval, at: 2026-08-11T17:38:30.000Z } +status: stable +--- + +# Runner eval slice — evb-e6bd1cf42d51618a + +- Bundle: `deterministic-harness/in-process@v1 · scripted-eval-agent · mock · 2 grants · prompt:deterministic-behavior-driver-v1 · faults:authorization+conflict+retry+provider_capability` +- Source: deterministic fault harness +- Cases: 16 · passed 8 · gate failures 3 · mean overall 0.691 + +## Dimension means + +- hard_invariants: 0.813 +- semantic_outcome: 0.75 +- trajectory_restraint: 0.688 +- trace_completeness: 1 +- quality_efficiency: 0.875 + +## Cases + +| case | source | counterpart | fault | gate | overall | outcome | trajectory | trace | efficiency | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| checkout_context:green | deterministic_fault_harness | green | — | ok | 1 | 1 | 1 | 1 | 1 | +| checkout_context:red | deterministic_fault_harness | red | — | FAIL | 0 | 1 | 0 | 1 | 1 | +| revision_safe_plan_editing:green | deterministic_fault_harness | green | — | ok | 1 | 1 | 1 | 1 | 1 | +| revision_safe_plan_editing:red | deterministic_fault_harness | red | conflict | ok | 0.6500000000000001 | 0 | 1 | 1 | 1 | +| approval_denial:green | deterministic_fault_harness | green | — | ok | 1 | 1 | 1 | 1 | 1 | +| approval_denial:red | deterministic_fault_harness | red | authorization | ok | 0.35000000000000003 | 0 | 0 | 1 | 1 | +| interaction_continuation:green | deterministic_fault_harness | green | — | ok | 1 | 1 | 1 | 1 | 1 | +| interaction_continuation:red | deterministic_fault_harness | red | — | ok | 0.35000000000000003 | 0 | 0 | 1 | 1 | +| blocker_monitor:green | deterministic_fault_harness | green | — | ok | 1 | 1 | 1 | 1 | 1 | +| blocker_monitor:red | deterministic_fault_harness | red | provider_capability | ok | 0.8500000000000001 | 1 | 0.5 | 1 | 1 | +| artifact_registration:green | deterministic_fault_harness | green | — | ok | 1 | 1 | 1 | 1 | 1 | +| artifact_registration:red | deterministic_fault_harness | red | retry | ok | 0.85 | 1 | 1 | 1 | 0 | +| restraint_no_call:green | deterministic_fault_harness | green | — | ok | 1 | 1 | 1 | 1 | 1 | +| restraint_no_call:red | deterministic_fault_harness | red | — | FAIL | 0 | 0 | 0 | 1 | 1 | +| terminal_arbitration:green | deterministic_fault_harness | green | — | ok | 1 | 1 | 1 | 1 | 1 | +| terminal_arbitration:red | deterministic_fault_harness | red | — | FAIL | 0 | 1 | 0.5 | 1 | 0 | diff --git a/packages/paperclip-runner/knowledge/evidence/capability/index.md b/packages/paperclip-runner/knowledge/evidence/capability/index.md new file mode 100644 index 00000000000..08fefe9576f --- /dev/null +++ b/packages/paperclip-runner/knowledge/evidence/capability/index.md @@ -0,0 +1,11 @@ +# Capability Eval Evidence + +The capability eval proof keeps the provider-safe live matrix beside its clean +and fault-injected scorecards. + +- [Live Codex matrix](live-codex-matrix.md) +- [Live eval slice report](live-eval-slice-report.md) +- [Fault eval slice report](fault-eval-slice-report.md) + +Each Markdown report has a JSON peer with the complete machine-readable bundle, +observation, scorecard, and safe evidence identifiers. diff --git a/packages/paperclip-runner/knowledge/evidence/capability/live-codex-matrix.json b/packages/paperclip-runner/knowledge/evidence/capability/live-codex-matrix.json new file mode 100644 index 00000000000..2add2d79646 --- /dev/null +++ b/packages/paperclip-runner/knowledge/evidence/capability/live-codex-matrix.json @@ -0,0 +1,684 @@ +{ + "schema": "paperclip.capability.live-codex-matrix.v1", + "groups": 16, + "providerModel": { + "id": "gpt-5.5", + "provider": "openai" + }, + "matrix": [ + { + "caseId": "ap-approval-deny-01", + "group": "ap", + "semanticOperation": "request_approval", + "expectedCalls": [ + "request_approval" + ], + "forbiddenCalls": [], + "observedCalls": [ + "request_approval" + ], + "finalState": { + "expected": "mutated", + "observed": "mutated" + }, + "providerModel": { + "id": "gpt-5.5", + "provider": "openai" + }, + "scoringEvidence": { + "trace": { + "runId": "run-capability-live-ap-approval-deny-01", + "sessionId": "session-capability-live-ap-approval-deny-01", + "turnId": "019ff1e7-5e9e-7353-abc9-734f85ed535b", + "itemId": "evidence-000022", + "receiptIds": [ + "evidence-000023" + ], + "terminalPresent": true + }, + "authorization": { + "expected": "allowed", + "observed": "allowed" + }, + "efficiency": { + "latencyMs": 4056, + "attempts": 1 + }, + "budget": { + "maxLatencyMs": 60000, + "maxAttempts": 1 + } + } + }, + { + "caseId": "ar-no-done-without-upload-01", + "group": "ar", + "semanticOperation": "register_deliverable", + "expectedCalls": [ + "register_deliverable" + ], + "forbiddenCalls": [], + "observedCalls": [ + "register_deliverable" + ], + "finalState": { + "expected": "mutated", + "observed": "mutated" + }, + "providerModel": { + "id": "gpt-5.5", + "provider": "openai" + }, + "scoringEvidence": { + "trace": { + "runId": "run-capability-live-ar-no-done-without-upload-01", + "sessionId": "session-capability-live-ar-no-done-without-upload-01", + "turnId": "019ff1e7-6ffb-7123-9f0a-37ca2f894aa2", + "itemId": "evidence-000022", + "receiptIds": [ + "evidence-000023" + ], + "terminalPresent": true + }, + "authorization": { + "expected": "allowed", + "observed": "allowed" + }, + "efficiency": { + "latencyMs": 3568, + "attempts": 1 + }, + "budget": { + "maxLatencyMs": 60000, + "maxAttempts": 1 + } + } + }, + { + "caseId": "bl-cancelled-not-resolved-01", + "group": "bl", + "semanticOperation": "block_task", + "expectedCalls": [ + "block_task" + ], + "forbiddenCalls": [], + "observedCalls": [ + "block_task" + ], + "finalState": { + "expected": "mutated", + "observed": "mutated" + }, + "providerModel": { + "id": "gpt-5.5", + "provider": "openai" + }, + "scoringEvidence": { + "trace": { + "runId": "run-capability-live-bl-cancelled-not-resolved-01", + "sessionId": "session-capability-live-bl-cancelled-not-resolved-01", + "turnId": "019ff1e7-7f46-7292-b3ab-5b5d6b99e09f", + "itemId": "evidence-000022", + "receiptIds": [ + "evidence-000023" + ], + "terminalPresent": true + }, + "authorization": { + "expected": "allowed", + "observed": "allowed" + }, + "efficiency": { + "latencyMs": 4079, + "attempts": 1 + }, + "budget": { + "maxLatencyMs": 60000, + "maxAttempts": 1 + } + } + }, + { + "caseId": "cm-mention-structured-01", + "group": "cm", + "semanticOperation": "report_progress", + "expectedCalls": [ + "report_progress" + ], + "forbiddenCalls": [], + "observedCalls": [ + "report_progress" + ], + "finalState": { + "expected": "mutated", + "observed": "mutated" + }, + "providerModel": { + "id": "gpt-5.5", + "provider": "openai" + }, + "scoringEvidence": { + "trace": { + "runId": "run-capability-live-cm-mention-structured-01", + "sessionId": "session-capability-live-cm-mention-structured-01", + "turnId": "019ff1e7-909c-70e2-b9b7-8e53cd6dac67", + "itemId": "evidence-000021", + "receiptIds": [ + "evidence-000023" + ], + "terminalPresent": true + }, + "authorization": { + "expected": "allowed", + "observed": "allowed" + }, + "efficiency": { + "latencyMs": 3621, + "attempts": 1 + }, + "budget": { + "maxLatencyMs": 60000, + "maxAttempts": 1 + } + } + }, + { + "caseId": "co-409-stop-01", + "group": "co", + "semanticOperation": "checkout_task", + "expectedCalls": [], + "forbiddenCalls": [], + "observedCalls": [], + "finalState": { + "expected": "unchanged", + "observed": "unchanged" + }, + "providerModel": { + "id": "gpt-5.5", + "provider": "openai" + }, + "scoringEvidence": { + "trace": { + "runId": "run-capability-live-co-409-stop-01", + "sessionId": "session-capability-live-co-409-stop-01", + "turnId": "019ff1e7-a051-7760-992c-39e9c9f2b09c", + "itemId": "transcript-000023", + "receiptIds": [], + "terminalPresent": true + }, + "authorization": { + "expected": "absent", + "observed": "absent" + }, + "efficiency": { + "latencyMs": 1864, + "attempts": 1 + }, + "budget": { + "maxLatencyMs": 60000, + "maxAttempts": 1 + } + } + }, + { + "caseId": "dp-base-revision-01", + "group": "dp", + "semanticOperation": "write_document", + "expectedCalls": [ + "write_document" + ], + "forbiddenCalls": [], + "observedCalls": [ + "write_document" + ], + "finalState": { + "expected": "mutated", + "observed": "mutated" + }, + "providerModel": { + "id": "gpt-5.5", + "provider": "openai" + }, + "scoringEvidence": { + "trace": { + "runId": "run-capability-live-dp-base-revision-01", + "sessionId": "session-capability-live-dp-base-revision-01", + "turnId": "019ff1e7-a905-7551-b66a-08d1fd0832fd", + "itemId": "evidence-000022", + "receiptIds": [ + "evidence-000023" + ], + "terminalPresent": true + }, + "authorization": { + "expected": "allowed", + "observed": "allowed" + }, + "efficiency": { + "latencyMs": 3685, + "attempts": 1 + }, + "budget": { + "maxLatencyMs": 60000, + "maxAttempts": 1 + } + } + }, + { + "caseId": "er-blocked-dedup-01", + "group": "er", + "semanticOperation": "checkout_task", + "expectedCalls": [], + "forbiddenCalls": [], + "observedCalls": [], + "finalState": { + "expected": "unchanged", + "observed": "unchanged" + }, + "providerModel": { + "id": "gpt-5.5", + "provider": "openai" + }, + "scoringEvidence": { + "trace": { + "runId": "run-capability-live-er-blocked-dedup-01", + "sessionId": "session-capability-live-er-blocked-dedup-01", + "turnId": "019ff1e7-b8d6-7eb2-9d4d-e98b659fdfc0", + "itemId": "transcript-000023", + "receiptIds": [], + "terminalPresent": true + }, + "authorization": { + "expected": "absent", + "observed": "absent" + }, + "efficiency": { + "latencyMs": 2033, + "attempts": 1 + }, + "budget": { + "maxLatencyMs": 60000, + "maxAttempts": 1 + } + } + }, + { + "caseId": "hb-context-01", + "group": "hb", + "semanticOperation": "checkout_task", + "expectedCalls": [], + "forbiddenCalls": [], + "observedCalls": [], + "finalState": { + "expected": "unchanged", + "observed": "unchanged" + }, + "providerModel": { + "id": "gpt-5.5", + "provider": "openai" + }, + "scoringEvidence": { + "trace": { + "runId": "run-capability-live-hb-context-01", + "sessionId": "session-capability-live-hb-context-01", + "turnId": "019ff1e7-c239-7462-8d23-dda298d0fa58", + "itemId": "transcript-000023", + "receiptIds": [], + "terminalPresent": true + }, + "authorization": { + "expected": "absent", + "observed": "absent" + }, + "efficiency": { + "latencyMs": 1658, + "attempts": 1 + }, + "budget": { + "maxLatencyMs": 60000, + "maxAttempts": 1 + } + } + }, + { + "caseId": "ix-checkbox-01", + "group": "ix", + "semanticOperation": "request_human_input", + "expectedCalls": [ + "request_human_input" + ], + "forbiddenCalls": [], + "observedCalls": [ + "request_human_input" + ], + "finalState": { + "expected": "mutated", + "observed": "mutated" + }, + "providerModel": { + "id": "gpt-5.5", + "provider": "openai" + }, + "scoringEvidence": { + "trace": { + "runId": "run-capability-live-ix-checkbox-01", + "sessionId": "session-capability-live-ix-checkbox-01", + "turnId": "019ff1e7-ca3a-7ad2-9d67-35ee17cb6042", + "itemId": "evidence-000021", + "receiptIds": [ + "evidence-000023" + ], + "terminalPresent": true + }, + "authorization": { + "expected": "allowed", + "observed": "allowed" + }, + "efficiency": { + "latencyMs": 9631, + "attempts": 1 + }, + "budget": { + "maxLatencyMs": 60000, + "maxAttempts": 1 + } + } + }, + { + "caseId": "mh-blocked-handoff-01", + "group": "mh", + "semanticOperation": "create_task", + "expectedCalls": [ + "create_task" + ], + "forbiddenCalls": [], + "observedCalls": [ + "create_task" + ], + "finalState": { + "expected": "mutated", + "observed": "mutated" + }, + "providerModel": { + "id": "gpt-5.5", + "provider": "openai" + }, + "scoringEvidence": { + "trace": { + "runId": "run-capability-live-mh-blocked-handoff-01", + "sessionId": "session-capability-live-mh-blocked-handoff-01", + "turnId": "019ff1e7-f14e-7962-91e0-60f4b2c1ee5e", + "itemId": "evidence-000022", + "receiptIds": [ + "evidence-000023" + ], + "terminalPresent": true + }, + "authorization": { + "expected": "allowed", + "observed": "allowed" + }, + "efficiency": { + "latencyMs": 3474, + "attempts": 1 + }, + "budget": { + "maxLatencyMs": 60000, + "maxAttempts": 1 + } + } + }, + { + "caseId": "rf-api-404-report-01", + "group": "rf", + "semanticOperation": "list_agents", + "expectedCalls": [ + "list_agents" + ], + "forbiddenCalls": [], + "observedCalls": [ + "list_agents" + ], + "finalState": { + "expected": "unchanged", + "observed": "unchanged" + }, + "providerModel": { + "id": "gpt-5.5", + "provider": "openai" + }, + "scoringEvidence": { + "trace": { + "runId": "run-capability-live-rf-api-404-report-01", + "sessionId": "session-capability-live-rf-api-404-report-01", + "turnId": "019ff1e8-0054-7102-8629-5a629712bc29", + "itemId": "evidence-000023", + "receiptIds": [ + "evidence-000025" + ], + "terminalPresent": true + }, + "authorization": { + "expected": "allowed", + "observed": "allowed" + }, + "efficiency": { + "latencyMs": 5691, + "attempts": 1 + }, + "budget": { + "maxLatencyMs": 60000, + "maxAttempts": 1 + } + } + }, + { + "caseId": "rs-dependency-blocked-wake-01", + "group": "rs", + "semanticOperation": "checkout_task", + "expectedCalls": [], + "forbiddenCalls": [], + "observedCalls": [], + "finalState": { + "expected": "unchanged", + "observed": "unchanged" + }, + "providerModel": { + "id": "gpt-5.5", + "provider": "openai" + }, + "scoringEvidence": { + "trace": { + "runId": "run-capability-live-rs-dependency-blocked-wake-01", + "sessionId": "session-capability-live-rs-dependency-blocked-wake-01", + "turnId": "019ff1e8-17fc-7600-aa46-84429760dcdd", + "itemId": "transcript-000023", + "receiptIds": [], + "terminalPresent": true + }, + "authorization": { + "expected": "absent", + "observed": "absent" + }, + "efficiency": { + "latencyMs": 1908, + "attempts": 1 + }, + "budget": { + "maxLatencyMs": 60000, + "maxAttempts": 1 + } + } + }, + { + "caseId": "se-get-issue-01", + "group": "se", + "semanticOperation": "search_tasks", + "expectedCalls": [ + "search_tasks" + ], + "forbiddenCalls": [], + "observedCalls": [ + "search_tasks" + ], + "finalState": { + "expected": "unchanged", + "observed": "unchanged" + }, + "providerModel": { + "id": "gpt-5.5", + "provider": "openai" + }, + "scoringEvidence": { + "trace": { + "runId": "run-capability-live-se-get-issue-01", + "sessionId": "session-capability-live-se-get-issue-01", + "turnId": "019ff1e8-20ea-73e2-9cc6-3e2d8f6fc948", + "itemId": "evidence-000024", + "receiptIds": [ + "evidence-000025" + ], + "terminalPresent": true + }, + "authorization": { + "expected": "allowed", + "observed": "denied" + }, + "efficiency": { + "latencyMs": 4504, + "attempts": 1 + }, + "budget": { + "maxLatencyMs": 60000, + "maxAttempts": 1 + } + } + }, + { + "caseId": "st-backlog-park-01", + "group": "st", + "semanticOperation": "finish_task", + "expectedCalls": [ + "finish_task" + ], + "forbiddenCalls": [], + "observedCalls": [ + "finish_task" + ], + "finalState": { + "expected": "mutated", + "observed": "mutated" + }, + "providerModel": { + "id": "gpt-5.5", + "provider": "openai" + }, + "scoringEvidence": { + "trace": { + "runId": "run-capability-live-st-backlog-park-01", + "sessionId": "session-capability-live-st-backlog-park-01", + "turnId": "019ff1e8-33f6-7f93-88b9-83ad36eac2bb", + "itemId": "evidence-000022", + "receiptIds": [ + "evidence-000023" + ], + "terminalPresent": true + }, + "authorization": { + "expected": "allowed", + "observed": "allowed" + }, + "efficiency": { + "latencyMs": 4513, + "attempts": 1 + }, + "budget": { + "maxLatencyMs": 60000, + "maxAttempts": 1 + } + } + }, + { + "caseId": "su-crossteam-billing-01", + "group": "su", + "semanticOperation": "create_task", + "expectedCalls": [ + "create_task" + ], + "forbiddenCalls": [], + "observedCalls": [ + "create_task" + ], + "finalState": { + "expected": "mutated", + "observed": "mutated" + }, + "providerModel": { + "id": "gpt-5.5", + "provider": "openai" + }, + "scoringEvidence": { + "trace": { + "runId": "run-capability-live-su-crossteam-billing-01", + "sessionId": "session-capability-live-su-crossteam-billing-01", + "turnId": "019ff1e8-470b-7550-a6b6-8c0c842eb948", + "itemId": "evidence-000022", + "receiptIds": [ + "evidence-000023" + ], + "terminalPresent": true + }, + "authorization": { + "expected": "allowed", + "observed": "allowed" + }, + "efficiency": { + "latencyMs": 6353, + "attempts": 1 + }, + "budget": { + "maxLatencyMs": 60000, + "maxAttempts": 1 + } + } + }, + { + "caseId": "wk-ask-mode-01", + "group": "wk", + "semanticOperation": "checkout_task", + "expectedCalls": [], + "forbiddenCalls": [], + "observedCalls": [], + "finalState": { + "expected": "unchanged", + "observed": "unchanged" + }, + "providerModel": { + "id": "gpt-5.5", + "provider": "openai" + }, + "scoringEvidence": { + "trace": { + "runId": "run-capability-live-wk-ask-mode-01", + "sessionId": "session-capability-live-wk-ask-mode-01", + "turnId": "019ff1e8-617d-7ba3-8c12-84e6c5d011d0", + "itemId": "transcript-000023", + "receiptIds": [], + "terminalPresent": true + }, + "authorization": { + "expected": "absent", + "observed": "absent" + }, + "efficiency": { + "latencyMs": 1731, + "attempts": 1 + }, + "budget": { + "maxLatencyMs": 60000, + "maxAttempts": 1 + } + } + } + ] +} diff --git a/packages/paperclip-runner/knowledge/evidence/capability/live-codex-matrix.md b/packages/paperclip-runner/knowledge/evidence/capability/live-codex-matrix.md new file mode 100644 index 00000000000..bb9e681c356 --- /dev/null +++ b/packages/paperclip-runner/knowledge/evidence/capability/live-codex-matrix.md @@ -0,0 +1,15 @@ +--- +type: Evidence Report +title: Capability Live Codex Matrix +description: Real Codex semantic-tool choices and resulting mock control-plane state. +generated: { by: process:capability-live-eval, at: 2026-08-11T17:38:30.000Z } +status: stable +--- + +# Capability Live Codex Matrix + +- Provider model: openai/gpt-5.5 +- Groups: 16 +- Cases: ap:ap-approval-deny-01, ar:ar-no-done-without-upload-01, bl:bl-cancelled-not-resolved-01, cm:cm-mention-structured-01, co:co-409-stop-01, dp:dp-base-revision-01, er:er-blocked-dedup-01, hb:hb-context-01, ix:ix-checkbox-01, mh:mh-blocked-handoff-01, rf:rf-api-404-report-01, rs:rs-dependency-blocked-wake-01, se:se-get-issue-01, st:st-backlog-park-01, su:su-crossteam-billing-01, wk:wk-ask-mode-01 +- Each row asserts expected and observed typed calls plus final mock state. +- Scored report: `live-eval-slice-report.md` diff --git a/packages/paperclip-runner/knowledge/evidence/capability/live-eval-slice-report.json b/packages/paperclip-runner/knowledge/evidence/capability/live-eval-slice-report.json new file mode 100644 index 00000000000..a13b3a43d3b --- /dev/null +++ b/packages/paperclip-runner/knowledge/evidence/capability/live-eval-slice-report.json @@ -0,0 +1,1544 @@ +{ + "schema": "paperclip.runner.eval-slice-report.v1", + "bundle": { + "id": "evb-ccfa51b713b10e13", + "summary": "runnerd/codex-app-server@codex-app-server-v2 · gpt-5.5 · mock · 4 grants · prompt:capability-live-exact-call-v1 · faults:clean", + "declaration": { + "schema": "paperclip.runner.eval-bundle.v1", + "provider": { + "runtime": "runnerd", + "transport": "codex-app-server", + "protocolVersion": "codex-app-server-v2" + }, + "model": { + "id": "gpt-5.5" + }, + "launchContext": { + "workingDirectoryClass": "workspace-checkout", + "scenarioId": "capability-live-codex-matrix-v1", + "turnTimeoutMs": 60000 + }, + "promptPolicy": { + "id": "capability-live-exact-call-v1", + "callTemplate": "Call the named semantic operation exactly once with the declared JSON input.", + "restraintTemplate": "Do not call any tool for a control-plane-owned behavior." + }, + "grants": [ + "delegation:tasks:create", + "discovery:agents:read", + "discovery:tasks:read", + "governance:approvals:request" + ], + "runner": { + "package": "@paperclipai/paperclip-runner", + "binary": "paperclip-runnerd", + "version": "0.1.2" + }, + "controlPlaneAdapter": { + "kind": "mock", + "contract": "paperclip.capability.mock-state.v1" + }, + "faultInjection": [] + } + }, + "generatedFromLiveModel": true, + "cases": [ + { + "observation": { + "caseId": "ap-approval-deny-01", + "provenance": { + "source": "live_model" + }, + "controlPlaneOwned": false, + "expectedCalls": [ + "request_approval" + ], + "observedCalls": [ + "request_approval" + ], + "forbiddenCalls": [], + "finalState": { + "expected": "mutated", + "observed": "mutated" + }, + "authorization": { + "expected": "allowed", + "observed": "allowed" + }, + "trace": { + "runId": "run-capability-live-ap-approval-deny-01", + "sessionId": "session-capability-live-ap-approval-deny-01", + "turnId": "019ff1e7-5e9e-7353-abc9-734f85ed535b", + "itemId": "evidence-000022", + "receiptIds": [ + "evidence-000023" + ], + "terminalPresent": true + }, + "efficiency": { + "latencyMs": 4056, + "attempts": 1 + }, + "budget": { + "maxLatencyMs": 60000, + "maxAttempts": 1 + } + }, + "scorecard": { + "schema": "paperclip.runner.eval-scorecard.v1", + "bundleId": "evb-ccfa51b713b10e13", + "caseId": "ap-approval-deny-01", + "dimensions": { + "hard_invariants": { + "dimension": "hard_invariants", + "score": 1, + "passed": true, + "gate": true, + "weight": 0, + "reasons": [] + }, + "semantic_outcome": { + "dimension": "semantic_outcome", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.35, + "reasons": [] + }, + "trajectory_restraint": { + "dimension": "trajectory_restraint", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.3, + "reasons": [] + }, + "trace_completeness": { + "dimension": "trace_completeness", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.2, + "reasons": [] + }, + "quality_efficiency": { + "dimension": "quality_efficiency", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.15, + "reasons": [] + } + }, + "overall": { + "score": 1, + "gatePassed": true, + "passed": true + } + } + }, + { + "observation": { + "caseId": "ar-no-done-without-upload-01", + "provenance": { + "source": "live_model" + }, + "controlPlaneOwned": false, + "expectedCalls": [ + "register_deliverable" + ], + "observedCalls": [ + "register_deliverable" + ], + "forbiddenCalls": [], + "finalState": { + "expected": "mutated", + "observed": "mutated" + }, + "authorization": { + "expected": "allowed", + "observed": "allowed" + }, + "trace": { + "runId": "run-capability-live-ar-no-done-without-upload-01", + "sessionId": "session-capability-live-ar-no-done-without-upload-01", + "turnId": "019ff1e7-6ffb-7123-9f0a-37ca2f894aa2", + "itemId": "evidence-000022", + "receiptIds": [ + "evidence-000023" + ], + "terminalPresent": true + }, + "efficiency": { + "latencyMs": 3568, + "attempts": 1 + }, + "budget": { + "maxLatencyMs": 60000, + "maxAttempts": 1 + } + }, + "scorecard": { + "schema": "paperclip.runner.eval-scorecard.v1", + "bundleId": "evb-ccfa51b713b10e13", + "caseId": "ar-no-done-without-upload-01", + "dimensions": { + "hard_invariants": { + "dimension": "hard_invariants", + "score": 1, + "passed": true, + "gate": true, + "weight": 0, + "reasons": [] + }, + "semantic_outcome": { + "dimension": "semantic_outcome", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.35, + "reasons": [] + }, + "trajectory_restraint": { + "dimension": "trajectory_restraint", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.3, + "reasons": [] + }, + "trace_completeness": { + "dimension": "trace_completeness", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.2, + "reasons": [] + }, + "quality_efficiency": { + "dimension": "quality_efficiency", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.15, + "reasons": [] + } + }, + "overall": { + "score": 1, + "gatePassed": true, + "passed": true + } + } + }, + { + "observation": { + "caseId": "bl-cancelled-not-resolved-01", + "provenance": { + "source": "live_model" + }, + "controlPlaneOwned": false, + "expectedCalls": [ + "block_task" + ], + "observedCalls": [ + "block_task" + ], + "forbiddenCalls": [], + "finalState": { + "expected": "mutated", + "observed": "mutated" + }, + "authorization": { + "expected": "allowed", + "observed": "allowed" + }, + "trace": { + "runId": "run-capability-live-bl-cancelled-not-resolved-01", + "sessionId": "session-capability-live-bl-cancelled-not-resolved-01", + "turnId": "019ff1e7-7f46-7292-b3ab-5b5d6b99e09f", + "itemId": "evidence-000022", + "receiptIds": [ + "evidence-000023" + ], + "terminalPresent": true + }, + "efficiency": { + "latencyMs": 4079, + "attempts": 1 + }, + "budget": { + "maxLatencyMs": 60000, + "maxAttempts": 1 + } + }, + "scorecard": { + "schema": "paperclip.runner.eval-scorecard.v1", + "bundleId": "evb-ccfa51b713b10e13", + "caseId": "bl-cancelled-not-resolved-01", + "dimensions": { + "hard_invariants": { + "dimension": "hard_invariants", + "score": 1, + "passed": true, + "gate": true, + "weight": 0, + "reasons": [] + }, + "semantic_outcome": { + "dimension": "semantic_outcome", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.35, + "reasons": [] + }, + "trajectory_restraint": { + "dimension": "trajectory_restraint", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.3, + "reasons": [] + }, + "trace_completeness": { + "dimension": "trace_completeness", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.2, + "reasons": [] + }, + "quality_efficiency": { + "dimension": "quality_efficiency", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.15, + "reasons": [] + } + }, + "overall": { + "score": 1, + "gatePassed": true, + "passed": true + } + } + }, + { + "observation": { + "caseId": "cm-mention-structured-01", + "provenance": { + "source": "live_model" + }, + "controlPlaneOwned": false, + "expectedCalls": [ + "report_progress" + ], + "observedCalls": [ + "report_progress" + ], + "forbiddenCalls": [], + "finalState": { + "expected": "mutated", + "observed": "mutated" + }, + "authorization": { + "expected": "allowed", + "observed": "allowed" + }, + "trace": { + "runId": "run-capability-live-cm-mention-structured-01", + "sessionId": "session-capability-live-cm-mention-structured-01", + "turnId": "019ff1e7-909c-70e2-b9b7-8e53cd6dac67", + "itemId": "evidence-000021", + "receiptIds": [ + "evidence-000023" + ], + "terminalPresent": true + }, + "efficiency": { + "latencyMs": 3621, + "attempts": 1 + }, + "budget": { + "maxLatencyMs": 60000, + "maxAttempts": 1 + } + }, + "scorecard": { + "schema": "paperclip.runner.eval-scorecard.v1", + "bundleId": "evb-ccfa51b713b10e13", + "caseId": "cm-mention-structured-01", + "dimensions": { + "hard_invariants": { + "dimension": "hard_invariants", + "score": 1, + "passed": true, + "gate": true, + "weight": 0, + "reasons": [] + }, + "semantic_outcome": { + "dimension": "semantic_outcome", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.35, + "reasons": [] + }, + "trajectory_restraint": { + "dimension": "trajectory_restraint", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.3, + "reasons": [] + }, + "trace_completeness": { + "dimension": "trace_completeness", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.2, + "reasons": [] + }, + "quality_efficiency": { + "dimension": "quality_efficiency", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.15, + "reasons": [] + } + }, + "overall": { + "score": 1, + "gatePassed": true, + "passed": true + } + } + }, + { + "observation": { + "caseId": "co-409-stop-01", + "provenance": { + "source": "live_model" + }, + "controlPlaneOwned": true, + "expectedCalls": [], + "observedCalls": [], + "forbiddenCalls": [], + "finalState": { + "expected": "unchanged", + "observed": "unchanged" + }, + "authorization": { + "expected": "absent", + "observed": "absent" + }, + "trace": { + "runId": "run-capability-live-co-409-stop-01", + "sessionId": "session-capability-live-co-409-stop-01", + "turnId": "019ff1e7-a051-7760-992c-39e9c9f2b09c", + "itemId": "transcript-000023", + "receiptIds": [], + "terminalPresent": true + }, + "efficiency": { + "latencyMs": 1864, + "attempts": 1 + }, + "budget": { + "maxLatencyMs": 60000, + "maxAttempts": 1 + } + }, + "scorecard": { + "schema": "paperclip.runner.eval-scorecard.v1", + "bundleId": "evb-ccfa51b713b10e13", + "caseId": "co-409-stop-01", + "dimensions": { + "hard_invariants": { + "dimension": "hard_invariants", + "score": 1, + "passed": true, + "gate": true, + "weight": 0, + "reasons": [] + }, + "semantic_outcome": { + "dimension": "semantic_outcome", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.35, + "reasons": [] + }, + "trajectory_restraint": { + "dimension": "trajectory_restraint", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.3, + "reasons": [ + "restraint honored: no call made" + ] + }, + "trace_completeness": { + "dimension": "trace_completeness", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.2, + "reasons": [] + }, + "quality_efficiency": { + "dimension": "quality_efficiency", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.15, + "reasons": [] + } + }, + "overall": { + "score": 1, + "gatePassed": true, + "passed": true + } + } + }, + { + "observation": { + "caseId": "dp-base-revision-01", + "provenance": { + "source": "live_model" + }, + "controlPlaneOwned": false, + "expectedCalls": [ + "write_document" + ], + "observedCalls": [ + "write_document" + ], + "forbiddenCalls": [], + "finalState": { + "expected": "mutated", + "observed": "mutated" + }, + "authorization": { + "expected": "allowed", + "observed": "allowed" + }, + "trace": { + "runId": "run-capability-live-dp-base-revision-01", + "sessionId": "session-capability-live-dp-base-revision-01", + "turnId": "019ff1e7-a905-7551-b66a-08d1fd0832fd", + "itemId": "evidence-000022", + "receiptIds": [ + "evidence-000023" + ], + "terminalPresent": true + }, + "efficiency": { + "latencyMs": 3685, + "attempts": 1 + }, + "budget": { + "maxLatencyMs": 60000, + "maxAttempts": 1 + } + }, + "scorecard": { + "schema": "paperclip.runner.eval-scorecard.v1", + "bundleId": "evb-ccfa51b713b10e13", + "caseId": "dp-base-revision-01", + "dimensions": { + "hard_invariants": { + "dimension": "hard_invariants", + "score": 1, + "passed": true, + "gate": true, + "weight": 0, + "reasons": [] + }, + "semantic_outcome": { + "dimension": "semantic_outcome", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.35, + "reasons": [] + }, + "trajectory_restraint": { + "dimension": "trajectory_restraint", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.3, + "reasons": [] + }, + "trace_completeness": { + "dimension": "trace_completeness", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.2, + "reasons": [] + }, + "quality_efficiency": { + "dimension": "quality_efficiency", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.15, + "reasons": [] + } + }, + "overall": { + "score": 1, + "gatePassed": true, + "passed": true + } + } + }, + { + "observation": { + "caseId": "er-blocked-dedup-01", + "provenance": { + "source": "live_model" + }, + "controlPlaneOwned": true, + "expectedCalls": [], + "observedCalls": [], + "forbiddenCalls": [], + "finalState": { + "expected": "unchanged", + "observed": "unchanged" + }, + "authorization": { + "expected": "absent", + "observed": "absent" + }, + "trace": { + "runId": "run-capability-live-er-blocked-dedup-01", + "sessionId": "session-capability-live-er-blocked-dedup-01", + "turnId": "019ff1e7-b8d6-7eb2-9d4d-e98b659fdfc0", + "itemId": "transcript-000023", + "receiptIds": [], + "terminalPresent": true + }, + "efficiency": { + "latencyMs": 2033, + "attempts": 1 + }, + "budget": { + "maxLatencyMs": 60000, + "maxAttempts": 1 + } + }, + "scorecard": { + "schema": "paperclip.runner.eval-scorecard.v1", + "bundleId": "evb-ccfa51b713b10e13", + "caseId": "er-blocked-dedup-01", + "dimensions": { + "hard_invariants": { + "dimension": "hard_invariants", + "score": 1, + "passed": true, + "gate": true, + "weight": 0, + "reasons": [] + }, + "semantic_outcome": { + "dimension": "semantic_outcome", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.35, + "reasons": [] + }, + "trajectory_restraint": { + "dimension": "trajectory_restraint", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.3, + "reasons": [ + "restraint honored: no call made" + ] + }, + "trace_completeness": { + "dimension": "trace_completeness", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.2, + "reasons": [] + }, + "quality_efficiency": { + "dimension": "quality_efficiency", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.15, + "reasons": [] + } + }, + "overall": { + "score": 1, + "gatePassed": true, + "passed": true + } + } + }, + { + "observation": { + "caseId": "hb-context-01", + "provenance": { + "source": "live_model" + }, + "controlPlaneOwned": true, + "expectedCalls": [], + "observedCalls": [], + "forbiddenCalls": [], + "finalState": { + "expected": "unchanged", + "observed": "unchanged" + }, + "authorization": { + "expected": "absent", + "observed": "absent" + }, + "trace": { + "runId": "run-capability-live-hb-context-01", + "sessionId": "session-capability-live-hb-context-01", + "turnId": "019ff1e7-c239-7462-8d23-dda298d0fa58", + "itemId": "transcript-000023", + "receiptIds": [], + "terminalPresent": true + }, + "efficiency": { + "latencyMs": 1658, + "attempts": 1 + }, + "budget": { + "maxLatencyMs": 60000, + "maxAttempts": 1 + } + }, + "scorecard": { + "schema": "paperclip.runner.eval-scorecard.v1", + "bundleId": "evb-ccfa51b713b10e13", + "caseId": "hb-context-01", + "dimensions": { + "hard_invariants": { + "dimension": "hard_invariants", + "score": 1, + "passed": true, + "gate": true, + "weight": 0, + "reasons": [] + }, + "semantic_outcome": { + "dimension": "semantic_outcome", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.35, + "reasons": [] + }, + "trajectory_restraint": { + "dimension": "trajectory_restraint", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.3, + "reasons": [ + "restraint honored: no call made" + ] + }, + "trace_completeness": { + "dimension": "trace_completeness", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.2, + "reasons": [] + }, + "quality_efficiency": { + "dimension": "quality_efficiency", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.15, + "reasons": [] + } + }, + "overall": { + "score": 1, + "gatePassed": true, + "passed": true + } + } + }, + { + "observation": { + "caseId": "ix-checkbox-01", + "provenance": { + "source": "live_model" + }, + "controlPlaneOwned": false, + "expectedCalls": [ + "request_human_input" + ], + "observedCalls": [ + "request_human_input" + ], + "forbiddenCalls": [], + "finalState": { + "expected": "mutated", + "observed": "mutated" + }, + "authorization": { + "expected": "allowed", + "observed": "allowed" + }, + "trace": { + "runId": "run-capability-live-ix-checkbox-01", + "sessionId": "session-capability-live-ix-checkbox-01", + "turnId": "019ff1e7-ca3a-7ad2-9d67-35ee17cb6042", + "itemId": "evidence-000021", + "receiptIds": [ + "evidence-000023" + ], + "terminalPresent": true + }, + "efficiency": { + "latencyMs": 9631, + "attempts": 1 + }, + "budget": { + "maxLatencyMs": 60000, + "maxAttempts": 1 + } + }, + "scorecard": { + "schema": "paperclip.runner.eval-scorecard.v1", + "bundleId": "evb-ccfa51b713b10e13", + "caseId": "ix-checkbox-01", + "dimensions": { + "hard_invariants": { + "dimension": "hard_invariants", + "score": 1, + "passed": true, + "gate": true, + "weight": 0, + "reasons": [] + }, + "semantic_outcome": { + "dimension": "semantic_outcome", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.35, + "reasons": [] + }, + "trajectory_restraint": { + "dimension": "trajectory_restraint", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.3, + "reasons": [] + }, + "trace_completeness": { + "dimension": "trace_completeness", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.2, + "reasons": [] + }, + "quality_efficiency": { + "dimension": "quality_efficiency", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.15, + "reasons": [] + } + }, + "overall": { + "score": 1, + "gatePassed": true, + "passed": true + } + } + }, + { + "observation": { + "caseId": "mh-blocked-handoff-01", + "provenance": { + "source": "live_model" + }, + "controlPlaneOwned": false, + "expectedCalls": [ + "create_task" + ], + "observedCalls": [ + "create_task" + ], + "forbiddenCalls": [], + "finalState": { + "expected": "mutated", + "observed": "mutated" + }, + "authorization": { + "expected": "allowed", + "observed": "allowed" + }, + "trace": { + "runId": "run-capability-live-mh-blocked-handoff-01", + "sessionId": "session-capability-live-mh-blocked-handoff-01", + "turnId": "019ff1e7-f14e-7962-91e0-60f4b2c1ee5e", + "itemId": "evidence-000022", + "receiptIds": [ + "evidence-000023" + ], + "terminalPresent": true + }, + "efficiency": { + "latencyMs": 3474, + "attempts": 1 + }, + "budget": { + "maxLatencyMs": 60000, + "maxAttempts": 1 + } + }, + "scorecard": { + "schema": "paperclip.runner.eval-scorecard.v1", + "bundleId": "evb-ccfa51b713b10e13", + "caseId": "mh-blocked-handoff-01", + "dimensions": { + "hard_invariants": { + "dimension": "hard_invariants", + "score": 1, + "passed": true, + "gate": true, + "weight": 0, + "reasons": [] + }, + "semantic_outcome": { + "dimension": "semantic_outcome", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.35, + "reasons": [] + }, + "trajectory_restraint": { + "dimension": "trajectory_restraint", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.3, + "reasons": [] + }, + "trace_completeness": { + "dimension": "trace_completeness", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.2, + "reasons": [] + }, + "quality_efficiency": { + "dimension": "quality_efficiency", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.15, + "reasons": [] + } + }, + "overall": { + "score": 1, + "gatePassed": true, + "passed": true + } + } + }, + { + "observation": { + "caseId": "rf-api-404-report-01", + "provenance": { + "source": "live_model" + }, + "controlPlaneOwned": false, + "expectedCalls": [ + "list_agents" + ], + "observedCalls": [ + "list_agents" + ], + "forbiddenCalls": [], + "finalState": { + "expected": "unchanged", + "observed": "unchanged" + }, + "authorization": { + "expected": "allowed", + "observed": "allowed" + }, + "trace": { + "runId": "run-capability-live-rf-api-404-report-01", + "sessionId": "session-capability-live-rf-api-404-report-01", + "turnId": "019ff1e8-0054-7102-8629-5a629712bc29", + "itemId": "evidence-000023", + "receiptIds": [ + "evidence-000025" + ], + "terminalPresent": true + }, + "efficiency": { + "latencyMs": 5691, + "attempts": 1 + }, + "budget": { + "maxLatencyMs": 60000, + "maxAttempts": 1 + } + }, + "scorecard": { + "schema": "paperclip.runner.eval-scorecard.v1", + "bundleId": "evb-ccfa51b713b10e13", + "caseId": "rf-api-404-report-01", + "dimensions": { + "hard_invariants": { + "dimension": "hard_invariants", + "score": 1, + "passed": true, + "gate": true, + "weight": 0, + "reasons": [] + }, + "semantic_outcome": { + "dimension": "semantic_outcome", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.35, + "reasons": [] + }, + "trajectory_restraint": { + "dimension": "trajectory_restraint", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.3, + "reasons": [] + }, + "trace_completeness": { + "dimension": "trace_completeness", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.2, + "reasons": [] + }, + "quality_efficiency": { + "dimension": "quality_efficiency", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.15, + "reasons": [] + } + }, + "overall": { + "score": 1, + "gatePassed": true, + "passed": true + } + } + }, + { + "observation": { + "caseId": "rs-dependency-blocked-wake-01", + "provenance": { + "source": "live_model" + }, + "controlPlaneOwned": true, + "expectedCalls": [], + "observedCalls": [], + "forbiddenCalls": [], + "finalState": { + "expected": "unchanged", + "observed": "unchanged" + }, + "authorization": { + "expected": "absent", + "observed": "absent" + }, + "trace": { + "runId": "run-capability-live-rs-dependency-blocked-wake-01", + "sessionId": "session-capability-live-rs-dependency-blocked-wake-01", + "turnId": "019ff1e8-17fc-7600-aa46-84429760dcdd", + "itemId": "transcript-000023", + "receiptIds": [], + "terminalPresent": true + }, + "efficiency": { + "latencyMs": 1908, + "attempts": 1 + }, + "budget": { + "maxLatencyMs": 60000, + "maxAttempts": 1 + } + }, + "scorecard": { + "schema": "paperclip.runner.eval-scorecard.v1", + "bundleId": "evb-ccfa51b713b10e13", + "caseId": "rs-dependency-blocked-wake-01", + "dimensions": { + "hard_invariants": { + "dimension": "hard_invariants", + "score": 1, + "passed": true, + "gate": true, + "weight": 0, + "reasons": [] + }, + "semantic_outcome": { + "dimension": "semantic_outcome", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.35, + "reasons": [] + }, + "trajectory_restraint": { + "dimension": "trajectory_restraint", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.3, + "reasons": [ + "restraint honored: no call made" + ] + }, + "trace_completeness": { + "dimension": "trace_completeness", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.2, + "reasons": [] + }, + "quality_efficiency": { + "dimension": "quality_efficiency", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.15, + "reasons": [] + } + }, + "overall": { + "score": 1, + "gatePassed": true, + "passed": true + } + } + }, + { + "observation": { + "caseId": "se-get-issue-01", + "provenance": { + "source": "live_model" + }, + "controlPlaneOwned": false, + "expectedCalls": [ + "search_tasks" + ], + "observedCalls": [ + "search_tasks" + ], + "forbiddenCalls": [], + "finalState": { + "expected": "unchanged", + "observed": "unchanged" + }, + "authorization": { + "expected": "allowed", + "observed": "denied" + }, + "trace": { + "runId": "run-capability-live-se-get-issue-01", + "sessionId": "session-capability-live-se-get-issue-01", + "turnId": "019ff1e8-20ea-73e2-9cc6-3e2d8f6fc948", + "itemId": "evidence-000024", + "receiptIds": [ + "evidence-000025" + ], + "terminalPresent": true + }, + "efficiency": { + "latencyMs": 4504, + "attempts": 1 + }, + "budget": { + "maxLatencyMs": 60000, + "maxAttempts": 1 + } + }, + "scorecard": { + "schema": "paperclip.runner.eval-scorecard.v1", + "bundleId": "evb-ccfa51b713b10e13", + "caseId": "se-get-issue-01", + "dimensions": { + "hard_invariants": { + "dimension": "hard_invariants", + "score": 1, + "passed": true, + "gate": true, + "weight": 0, + "reasons": [] + }, + "semantic_outcome": { + "dimension": "semantic_outcome", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.35, + "reasons": [] + }, + "trajectory_restraint": { + "dimension": "trajectory_restraint", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.3, + "reasons": [] + }, + "trace_completeness": { + "dimension": "trace_completeness", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.2, + "reasons": [] + }, + "quality_efficiency": { + "dimension": "quality_efficiency", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.15, + "reasons": [] + } + }, + "overall": { + "score": 1, + "gatePassed": true, + "passed": true + } + } + }, + { + "observation": { + "caseId": "st-backlog-park-01", + "provenance": { + "source": "live_model" + }, + "controlPlaneOwned": false, + "expectedCalls": [ + "finish_task" + ], + "observedCalls": [ + "finish_task" + ], + "forbiddenCalls": [], + "finalState": { + "expected": "mutated", + "observed": "mutated" + }, + "authorization": { + "expected": "allowed", + "observed": "allowed" + }, + "trace": { + "runId": "run-capability-live-st-backlog-park-01", + "sessionId": "session-capability-live-st-backlog-park-01", + "turnId": "019ff1e8-33f6-7f93-88b9-83ad36eac2bb", + "itemId": "evidence-000022", + "receiptIds": [ + "evidence-000023" + ], + "terminalPresent": true + }, + "efficiency": { + "latencyMs": 4513, + "attempts": 1 + }, + "budget": { + "maxLatencyMs": 60000, + "maxAttempts": 1 + } + }, + "scorecard": { + "schema": "paperclip.runner.eval-scorecard.v1", + "bundleId": "evb-ccfa51b713b10e13", + "caseId": "st-backlog-park-01", + "dimensions": { + "hard_invariants": { + "dimension": "hard_invariants", + "score": 1, + "passed": true, + "gate": true, + "weight": 0, + "reasons": [] + }, + "semantic_outcome": { + "dimension": "semantic_outcome", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.35, + "reasons": [] + }, + "trajectory_restraint": { + "dimension": "trajectory_restraint", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.3, + "reasons": [] + }, + "trace_completeness": { + "dimension": "trace_completeness", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.2, + "reasons": [] + }, + "quality_efficiency": { + "dimension": "quality_efficiency", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.15, + "reasons": [] + } + }, + "overall": { + "score": 1, + "gatePassed": true, + "passed": true + } + } + }, + { + "observation": { + "caseId": "su-crossteam-billing-01", + "provenance": { + "source": "live_model" + }, + "controlPlaneOwned": false, + "expectedCalls": [ + "create_task" + ], + "observedCalls": [ + "create_task" + ], + "forbiddenCalls": [], + "finalState": { + "expected": "mutated", + "observed": "mutated" + }, + "authorization": { + "expected": "allowed", + "observed": "allowed" + }, + "trace": { + "runId": "run-capability-live-su-crossteam-billing-01", + "sessionId": "session-capability-live-su-crossteam-billing-01", + "turnId": "019ff1e8-470b-7550-a6b6-8c0c842eb948", + "itemId": "evidence-000022", + "receiptIds": [ + "evidence-000023" + ], + "terminalPresent": true + }, + "efficiency": { + "latencyMs": 6353, + "attempts": 1 + }, + "budget": { + "maxLatencyMs": 60000, + "maxAttempts": 1 + } + }, + "scorecard": { + "schema": "paperclip.runner.eval-scorecard.v1", + "bundleId": "evb-ccfa51b713b10e13", + "caseId": "su-crossteam-billing-01", + "dimensions": { + "hard_invariants": { + "dimension": "hard_invariants", + "score": 1, + "passed": true, + "gate": true, + "weight": 0, + "reasons": [] + }, + "semantic_outcome": { + "dimension": "semantic_outcome", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.35, + "reasons": [] + }, + "trajectory_restraint": { + "dimension": "trajectory_restraint", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.3, + "reasons": [] + }, + "trace_completeness": { + "dimension": "trace_completeness", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.2, + "reasons": [] + }, + "quality_efficiency": { + "dimension": "quality_efficiency", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.15, + "reasons": [] + } + }, + "overall": { + "score": 1, + "gatePassed": true, + "passed": true + } + } + }, + { + "observation": { + "caseId": "wk-ask-mode-01", + "provenance": { + "source": "live_model" + }, + "controlPlaneOwned": true, + "expectedCalls": [], + "observedCalls": [], + "forbiddenCalls": [], + "finalState": { + "expected": "unchanged", + "observed": "unchanged" + }, + "authorization": { + "expected": "absent", + "observed": "absent" + }, + "trace": { + "runId": "run-capability-live-wk-ask-mode-01", + "sessionId": "session-capability-live-wk-ask-mode-01", + "turnId": "019ff1e8-617d-7ba3-8c12-84e6c5d011d0", + "itemId": "transcript-000023", + "receiptIds": [], + "terminalPresent": true + }, + "efficiency": { + "latencyMs": 1731, + "attempts": 1 + }, + "budget": { + "maxLatencyMs": 60000, + "maxAttempts": 1 + } + }, + "scorecard": { + "schema": "paperclip.runner.eval-scorecard.v1", + "bundleId": "evb-ccfa51b713b10e13", + "caseId": "wk-ask-mode-01", + "dimensions": { + "hard_invariants": { + "dimension": "hard_invariants", + "score": 1, + "passed": true, + "gate": true, + "weight": 0, + "reasons": [] + }, + "semantic_outcome": { + "dimension": "semantic_outcome", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.35, + "reasons": [] + }, + "trajectory_restraint": { + "dimension": "trajectory_restraint", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.3, + "reasons": [ + "restraint honored: no call made" + ] + }, + "trace_completeness": { + "dimension": "trace_completeness", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.2, + "reasons": [] + }, + "quality_efficiency": { + "dimension": "quality_efficiency", + "score": 1, + "passed": true, + "gate": false, + "weight": 0.15, + "reasons": [] + } + }, + "overall": { + "score": 1, + "gatePassed": true, + "passed": true + } + } + } + ], + "aggregate": { + "caseCount": 16, + "passed": 16, + "gateFailures": 0, + "meanOverall": 1, + "dimensionMeans": { + "hard_invariants": 1, + "semantic_outcome": 1, + "trajectory_restraint": 1, + "trace_completeness": 1, + "quality_efficiency": 1 + } + } +} diff --git a/packages/paperclip-runner/knowledge/evidence/capability/live-eval-slice-report.md b/packages/paperclip-runner/knowledge/evidence/capability/live-eval-slice-report.md new file mode 100644 index 00000000000..448afaba40e --- /dev/null +++ b/packages/paperclip-runner/knowledge/evidence/capability/live-eval-slice-report.md @@ -0,0 +1,42 @@ +--- +type: Evidence Report +title: Live Runner Eval Slice Report +description: Scored real-Codex capability matrix bound to its secret-free eval bundle. +generated: { by: process:capability-live-eval, at: 2026-08-11T17:38:30.000Z } +status: stable +--- + +# Runner eval slice — evb-ccfa51b713b10e13 + +- Bundle: `runnerd/codex-app-server@codex-app-server-v2 · gpt-5.5 · mock · 4 grants · prompt:capability-live-exact-call-v1 · faults:clean` +- Source: real model tool choice +- Cases: 16 · passed 16 · gate failures 0 · mean overall 1 + +## Dimension means + +- hard_invariants: 1 +- semantic_outcome: 1 +- trajectory_restraint: 1 +- trace_completeness: 1 +- quality_efficiency: 1 + +## Cases + +| case | source | counterpart | fault | gate | overall | outcome | trajectory | trace | efficiency | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| ap-approval-deny-01 | live_model | — | — | ok | 1 | 1 | 1 | 1 | 1 | +| ar-no-done-without-upload-01 | live_model | — | — | ok | 1 | 1 | 1 | 1 | 1 | +| bl-cancelled-not-resolved-01 | live_model | — | — | ok | 1 | 1 | 1 | 1 | 1 | +| cm-mention-structured-01 | live_model | — | — | ok | 1 | 1 | 1 | 1 | 1 | +| co-409-stop-01 | live_model | — | — | ok | 1 | 1 | 1 | 1 | 1 | +| dp-base-revision-01 | live_model | — | — | ok | 1 | 1 | 1 | 1 | 1 | +| er-blocked-dedup-01 | live_model | — | — | ok | 1 | 1 | 1 | 1 | 1 | +| hb-context-01 | live_model | — | — | ok | 1 | 1 | 1 | 1 | 1 | +| ix-checkbox-01 | live_model | — | — | ok | 1 | 1 | 1 | 1 | 1 | +| mh-blocked-handoff-01 | live_model | — | — | ok | 1 | 1 | 1 | 1 | 1 | +| rf-api-404-report-01 | live_model | — | — | ok | 1 | 1 | 1 | 1 | 1 | +| rs-dependency-blocked-wake-01 | live_model | — | — | ok | 1 | 1 | 1 | 1 | 1 | +| se-get-issue-01 | live_model | — | — | ok | 1 | 1 | 1 | 1 | 1 | +| st-backlog-park-01 | live_model | — | — | ok | 1 | 1 | 1 | 1 | 1 | +| su-crossteam-billing-01 | live_model | — | — | ok | 1 | 1 | 1 | 1 | 1 | +| wk-ask-mode-01 | live_model | — | — | ok | 1 | 1 | 1 | 1 | 1 | diff --git a/packages/paperclip-runner/knowledge/evidence/index.md b/packages/paperclip-runner/knowledge/evidence/index.md index 1d748214630..615fa96a7a8 100644 --- a/packages/paperclip-runner/knowledge/evidence/index.md +++ b/packages/paperclip-runner/knowledge/evidence/index.md @@ -8,6 +8,7 @@ records that remain useful to package verification and debugging. - [Durable recovery per-fault traces](durable-recovery-faults/) - [Codex normalized trace](codex-trace.json) - [Live-console real Codex server trace](live-console-real-codex-server.json) +- [Capability live eval evidence](capability/) Screenshots, dated milestone reports, and duplicate handoff records live in Git history rather than the current package. diff --git a/packages/paperclip-runner/scripts/run-capability-live-eval-matrix.mjs b/packages/paperclip-runner/scripts/run-capability-live-eval-matrix.mjs index 6cfa41b873c..2391d9ebba6 100644 --- a/packages/paperclip-runner/scripts/run-capability-live-eval-matrix.mjs +++ b/packages/paperclip-runner/scripts/run-capability-live-eval-matrix.mjs @@ -1,27 +1,157 @@ -import { mkdir, writeFile } from "node:fs/promises"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; import { dirname, resolve } from "node:path"; const packageRoot = resolve(import.meta.dirname, ".."); const suite = await import(resolve(packageRoot, "dist/conformance/capability-eval-suite.js")); +const evals = await import(resolve(packageRoot, "dist/eval/index.js")); +const packageManifest = JSON.parse(await readFile(resolve(packageRoot, "package.json"), "utf8")); +const generatedAt = new Date().toISOString(); +const evidenceMarkdown = (title, description, body) => [ + "---", + "type: Evidence Report", + `title: ${title}`, + `description: ${description}`, + `generated: { by: process:capability-live-eval, at: ${generatedAt} }`, + "status: stable", + "---", + "", + body.trimEnd(), + "", +].join("\n"); const matrix = await suite.runCapabilityLiveCodexMatrix(process.cwd()); +const providerModels = [...new Set(matrix.map( + (entry) => `${entry.providerModel.provider}/${entry.providerModel.id}`, +))]; +if (providerModels.length !== 1) { + throw new Error(`live eval matrix used inconsistent provider models: ${providerModels.join(", ")}`); +} +const liveModel = matrix[0].providerModel; +const liveBundle = { + schema: evals.EVAL_BUNDLE_SCHEMA, + provider: { + runtime: "runnerd", + transport: "codex-app-server", + protocolVersion: "codex-app-server-v2", + }, + model: { id: liveModel.id }, + launchContext: { + workingDirectoryClass: "workspace-checkout", + scenarioId: "capability-live-codex-matrix-v1", + turnTimeoutMs: 60_000, + }, + promptPolicy: { + id: "capability-live-exact-call-v1", + callTemplate: "Call the named semantic operation exactly once with the declared JSON input.", + restraintTemplate: "Do not call any tool for a control-plane-owned behavior.", + }, + grants: [...suite.CAPABILITY_LIVE_EVAL_GRANTS], + runner: { + package: packageManifest.name, + binary: "paperclip-runnerd", + version: packageManifest.version, + }, + controlPlaneAdapter: { + kind: "mock", + contract: "paperclip.capability.mock-state.v1", + }, + faultInjection: [], +}; +const liveObservations = matrix.map((result) => evals.observationFromLiveMatrix(result, { + ...result.scoringEvidence, + provenance: { source: "live_model" }, +})); +const liveSliceReport = evals.buildEvalSliceReport(liveBundle, liveObservations, { + generatedFromLiveModel: true, +}); +if (liveSliceReport.aggregate.passed !== liveSliceReport.aggregate.caseCount) { + throw new Error(`live eval slice scored ${liveSliceReport.aggregate.passed}/${liveSliceReport.aggregate.caseCount} passing cases`); +} + +const faultBundle = { + schema: evals.EVAL_BUNDLE_SCHEMA, + provider: { + runtime: "deterministic-harness", + transport: "in-process", + protocolVersion: "v1", + }, + model: { id: "scripted-eval-agent" }, + launchContext: { + workingDirectoryClass: "ephemeral-fixture", + scenarioId: "eval-behavior-fault-matrix-v1", + turnTimeoutMs: 1_000, + }, + promptPolicy: { + id: "deterministic-behavior-driver-v1", + callTemplate: "Invoke the declared semantic operation sequence.", + restraintTemplate: "Make no semantic call.", + }, + grants: ["control_plane:wakes", "governance:approvals:decide"], + runner: { + package: packageManifest.name, + binary: "node", + version: packageManifest.version, + }, + controlPlaneAdapter: { + kind: "mock", + contract: "paperclip.capability.mock-state.v1", + }, + faultInjection: [ + { id: "fault-authz", class: "authorization", description: "deny approval decision exposure" }, + { id: "fault-conflict", class: "conflict", description: "submit a stale plan revision" }, + { id: "fault-retry", class: "retry", description: "fail the first artifact registration attempt" }, + { id: "fault-provider", class: "provider_capability", description: "omit wake scheduling from the tool surface" }, + ], +}; +const behaviorMatrix = await evals.runEvalBehaviorFaultMatrix(faultBundle); +const faultSliceReport = evals.buildEvalSliceReport( + faultBundle, + behaviorMatrix.map((result) => result.observation), +); const report = { schema: "paperclip.capability.live-codex-matrix.v1", groups: matrix.length, + providerModel: liveModel, matrix, }; -const output = resolve(packageRoot, "knowledge/evidence/capability/live-codex-matrix.json"); -const markdown = output.replace(/\.json$/, ".md"); +const evidenceDirectory = resolve(packageRoot, "knowledge/evidence/capability"); +const output = resolve(evidenceDirectory, "live-codex-matrix.json"); +const markdown = resolve(evidenceDirectory, "live-codex-matrix.md"); +const liveSliceJson = resolve(evidenceDirectory, "live-eval-slice-report.json"); +const liveSliceMarkdown = resolve(evidenceDirectory, "live-eval-slice-report.md"); +const faultSliceJson = resolve(evidenceDirectory, "fault-eval-slice-report.json"); +const faultSliceMarkdown = resolve(evidenceDirectory, "fault-eval-slice-report.md"); await mkdir(dirname(output), { recursive: true }); await Promise.all([ writeFile(output, `${JSON.stringify(report, null, 2)}\n`), - writeFile(markdown, [ + writeFile(markdown, evidenceMarkdown( + "Capability Live Codex Matrix", + "Real Codex semantic-tool choices and resulting mock control-plane state.", + [ "# Capability Live Codex Matrix", "", + `- Provider model: ${liveModel.provider}/${liveModel.id}`, `- Groups: ${report.groups}`, `- Cases: ${matrix.map((entry) => `${entry.group}:${entry.caseId}`).join(", ")}`, "- Each row asserts expected and observed typed calls plus final mock state.", + "- Scored report: `live-eval-slice-report.md`", "", - ].join("\n")), + ].join("\n"), + )), + writeFile(liveSliceJson, `${JSON.stringify(liveSliceReport, null, 2)}\n`), + writeFile(liveSliceMarkdown, evidenceMarkdown( + "Live Runner Eval Slice Report", + "Scored real-Codex capability matrix bound to its secret-free eval bundle.", + evals.renderEvalSliceMarkdown(liveSliceReport), + )), + writeFile(faultSliceJson, `${JSON.stringify(faultSliceReport, null, 2)}\n`), + writeFile(faultSliceMarkdown, evidenceMarkdown( + "Runner Eval Fault Slice Report", + "Eight green/red behaviors with deterministic injected-fault receipts.", + evals.renderEvalSliceMarkdown(faultSliceReport), + )), ]); process.stdout.write(`Capability live Codex matrix passed: ${matrix.length} groups.\n`); +process.stdout.write(`Live eval slice passed: ${liveSliceReport.aggregate.passed}/${liveSliceReport.aggregate.caseCount}.\n`); +process.stdout.write(`Behavior fault matrix passed: ${faultSliceReport.aggregate.passed} green, ${faultSliceReport.aggregate.caseCount - faultSliceReport.aggregate.passed} scored red.\n`); process.stdout.write(`Live matrix reports: ${output} and ${markdown}\n`); +process.stdout.write(`Scored eval reports: ${liveSliceJson}, ${liveSliceMarkdown}, ${faultSliceJson}, and ${faultSliceMarkdown}\n`); diff --git a/packages/paperclip-runner/src/conformance/capability-eval-suite.ts b/packages/paperclip-runner/src/conformance/capability-eval-suite.ts index e166529451e..2e1d6f7276f 100644 --- a/packages/paperclip-runner/src/conformance/capability-eval-suite.ts +++ b/packages/paperclip-runner/src/conformance/capability-eval-suite.ts @@ -58,8 +58,34 @@ export interface CapabilityLiveCodexMatrixResult { forbiddenCalls: string[]; observedCalls: string[]; finalState: { expected: "unchanged" | "mutated"; observed: "unchanged" | "mutated" }; + providerModel: { id: string; provider: string }; + /** Browser-safe evidence retained from the live-session snapshot for scoring. */ + scoringEvidence: { + trace: { + runId: string; + sessionId: string; + turnId: string; + itemId: string; + receiptIds: string[]; + terminalPresent: boolean; + }; + authorization: { + expected: "allowed" | "denied" | "absent"; + observed: "allowed" | "denied" | "absent"; + }; + efficiency: { latencyMs: number; attempts: number }; + budget: { maxLatencyMs: number; maxAttempts: number }; + }; } +/** One fixed candidate grant set shared by every live-matrix case. */ +export const CAPABILITY_LIVE_EVAL_GRANTS = Object.freeze([ + "delegation:tasks:create", + "discovery:agents:read", + "discovery:tasks:read", + "governance:approvals:request", +]); + export interface CapabilityEvalParityReport { schema: "paperclip.capability.eval-parity-report.v1"; cases: number; @@ -284,9 +310,11 @@ export async function runCapabilityLiveCodexMatrix( const service = new CapabilityLiveSessionService(); const session = await service.create({ workingDirectory, - seed: liveFixtureSeed(plan.grants), - capabilities: capabilityFixtureRunCapabilities(plan.grants), - explicitClaims: plan.grants, + seed: liveFixtureSeed(CAPABILITY_LIVE_EVAL_GRANTS), + capabilities: capabilityFixtureRunCapabilities(CAPABILITY_LIVE_EVAL_GRANTS), + explicitClaims: [...CAPABILITY_LIVE_EVAL_GRANTS], + runId: `run-capability-live-${row.id}`, + sessionId: `session-capability-live-${row.id}`, turnTimeoutMs: 60_000, scenario: { id: `capability-live-eval-${row.id}` }, }); @@ -300,16 +328,23 @@ export async function runCapabilityLiveCodexMatrix( const prompt = expectedCalls.length === 0 ? "Do not call any tools. Reply with exactly: control-plane-owned." : `Call ${plan.operationId} exactly once with this JSON input: ${JSON.stringify(liveInput)}. Do not call any other tool. Reply after the typed result.`; - await session.sendMessage(prompt); - const snapshot = session.snapshot(); + const turnStartedAt = Date.now(); + const turn = await session.sendMessage(prompt); + const latencyMs = Date.now() - turnStartedAt; + const snapshot = turn.snapshot; const observedCalls = snapshot.evidence .filter((entry) => entry.kind === "tool_call") .map((entry) => String(entry.data.operationId)); const expectedState = operationMutatesState(plan.operationId) ? "mutated" : "unchanged"; const observedState = snapshot.mockState === initialMockState ? "unchanged" : "mutated"; - if (JSON.stringify(observedCalls) !== JSON.stringify(expectedCalls) || observedState !== expectedState) { + if ( + turn.status !== "completed" || + JSON.stringify(observedCalls) !== JSON.stringify(expectedCalls) || + observedState !== expectedState + ) { throw new Error(JSON.stringify({ caseId: row.id, + turnStatus: turn.status, expectedCalls, observedCalls, expectedState, @@ -317,6 +352,23 @@ export async function runCapabilityLiveCodexMatrix( toolResults: snapshot.evidence.filter((entry) => entry.kind === "tool_result").map((entry) => entry.data.result), })); } + if (snapshot.providerModel === undefined) { + throw new Error(`live Codex case ${row.id} did not report its provider model identity`); + } + const invocationRecords = snapshot.authorizationRecords.filter( + (record) => record.phase === "invocation" && record.callId !== null, + ); + const observedAuthorization = observedCalls.length === 0 + ? "absent" + : invocationRecords.every((record) => record.allowed) + ? "allowed" + : "denied"; + const toolReceipts = snapshot.evidence.filter((entry) => entry.kind === "tool_result"); + const itemId = snapshot.evidence.find((entry) => entry.kind === "tool_call")?.id + ?? snapshot.transcript.find( + (entry) => entry.role === "assistant" && entry.turnId === turn.turnId, + )?.id + ?? `turn:${turn.turnId}:terminal`; matrix.push({ caseId: row.id, group, @@ -325,6 +377,23 @@ export async function runCapabilityLiveCodexMatrix( forbiddenCalls: row.forbiddenSemantics, observedCalls, finalState: { expected: expectedState, observed: observedState }, + providerModel: snapshot.providerModel, + scoringEvidence: { + trace: { + runId: snapshot.authority.runId, + sessionId: snapshot.sessionId, + turnId: turn.turnId, + itemId, + receiptIds: toolReceipts.map((entry) => entry.id), + terminalPresent: true, + }, + authorization: { + expected: expectedCalls.length === 0 ? "absent" : "allowed", + observed: observedAuthorization, + }, + efficiency: { latencyMs, attempts: 1 }, + budget: { maxLatencyMs: 60_000, maxAttempts: 1 }, + }, }); } finally { await service.shutdown(session.id, `Capability live eval ${row.id} complete`); @@ -333,7 +402,7 @@ export async function runCapabilityLiveCodexMatrix( return matrix; } -function liveFixtureSeed(actorGrants: string[]): CapabilityFixtureSeed { +function liveFixtureSeed(actorGrants: readonly string[]): CapabilityFixtureSeed { return { actors: [{ id: "actor-1", @@ -342,7 +411,7 @@ function liveFixtureSeed(actorGrants: string[]): CapabilityFixtureSeed { role: "engineer", status: "active", budgetId: "budget-actor-1", - capabilityGrants: actorGrants, + capabilityGrants: [...actorGrants], }], }; } diff --git a/packages/paperclip-runner/src/eval/eval-bundle.ts b/packages/paperclip-runner/src/eval/eval-bundle.ts index 285d74e2e42..6f24e67c94b 100644 --- a/packages/paperclip-runner/src/eval/eval-bundle.ts +++ b/packages/paperclip-runner/src/eval/eval-bundle.ts @@ -11,7 +11,8 @@ import { createHash } from "node:crypto"; * * The bundle is a *declaration*, not a live secret store: it must never carry a * credential, hidden company identifier, or raw secret payload. Grants are typed - * grant strings (`domain:resource:action`), not secret material. Callers should + * canonical claim strings (`domain:action` or `domain:resource:action`), not + * secret material. Callers should * run {@link assertBundleSecretFree} before persisting a bundle as evidence. */ export const EVAL_BUNDLE_SCHEMA = "paperclip.runner.eval-bundle.v1" as const; @@ -82,7 +83,7 @@ export interface EvalBundle { model: EvalBundleModel; launchContext: EvalBundleLaunchContext; promptPolicy: EvalBundlePromptPolicy; - /** Typed grant strings (`domain:resource:action`) unlocked for the candidate. */ + /** Typed canonical claim strings unlocked for the candidate. */ grants: string[]; runner: EvalBundleRunner; controlPlaneAdapter: EvalBundleControlPlaneAdapter; @@ -141,9 +142,9 @@ const SECRET_VALUE_PATTERNS: Array<{ id: string; pattern: RegExp }> = [ */ export function assertBundleSecretFree(bundle: EvalBundle): void { for (const grant of bundle.grants) { - if (!/^[a-z0-9]+(:[a-z0-9_]+){2}$/.test(grant)) { + if (!/^[a-z0-9_]+(?::[a-z0-9_]+){1,2}$/.test(grant)) { throw new EvalBundleSecretError( - `grant "${grant}" is not a typed domain:resource:action grant`, + `grant "${grant}" is not a typed canonical capability claim`, "grants", ); } diff --git a/packages/paperclip-runner/src/eval/eval-execution.test.ts b/packages/paperclip-runner/src/eval/eval-execution.test.ts new file mode 100644 index 00000000000..4e9a227f61d --- /dev/null +++ b/packages/paperclip-runner/src/eval/eval-execution.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, it } from "vitest"; + +import { EVAL_BUNDLE_SCHEMA, type EvalBundle } from "./eval-bundle.js"; +import { + EVAL_BEHAVIOR_IDS, + runEvalBehaviorFaultMatrix, +} from "./eval-execution.js"; +import { buildEvalSliceReport } from "./eval-slice.js"; + +function bundle(): EvalBundle { + return { + schema: EVAL_BUNDLE_SCHEMA, + provider: { + runtime: "deterministic-harness", + transport: "in-process", + protocolVersion: "v1", + }, + model: { id: "scripted-eval-agent" }, + launchContext: { + workingDirectoryClass: "ephemeral-fixture", + scenarioId: "eval-behavior-fault-matrix-v1", + turnTimeoutMs: 1_000, + }, + promptPolicy: { + id: "deterministic-behavior-driver-v1", + callTemplate: "Invoke the declared semantic operation sequence.", + restraintTemplate: "Make no semantic call.", + }, + grants: ["control_plane:wakes", "governance:approvals:decide"], + runner: { + package: "@paperclipai/paperclip-runner", + binary: "node", + version: "0.1.2", + }, + controlPlaneAdapter: { + kind: "mock", + contract: "paperclip.capability.mock-state.v1", + }, + faultInjection: [ + { id: "fault-authz", class: "authorization", description: "deny approval decision exposure" }, + { id: "fault-conflict", class: "conflict", description: "submit a stale plan revision" }, + { id: "fault-retry", class: "retry", description: "fail the first artifact registration attempt" }, + { id: "fault-provider", class: "provider_capability", description: "omit wake scheduling from the tool surface" }, + ], + }; +} + +describe("runEvalBehaviorFaultMatrix", () => { + it("executes eight green/red behaviors and all four declared fault classes", async () => { + const candidate = bundle(); + const results = await runEvalBehaviorFaultMatrix(candidate); + const report = buildEvalSliceReport( + candidate, + results.map((result) => result.observation), + ); + + expect(report.aggregate).toMatchObject({ caseCount: 16, passed: 8 }); + for (const behavior of EVAL_BEHAVIOR_IDS) { + expect(report.cases.find((entry) => entry.observation.caseId === `${behavior}:green`)?.scorecard.overall.passed).toBe(true); + expect(report.cases.find((entry) => entry.observation.caseId === `${behavior}:red`)?.scorecard.overall.passed).toBe(false); + } + + const faults = results.flatMap((result) => { + const fault = result.observation.provenance?.faultInjection; + return fault === undefined ? [] : [fault]; + }); + expect(faults.map((fault) => fault.class).sort()).toEqual([ + "authorization", + "conflict", + "provider_capability", + "retry", + ]); + expect(faults.every((fault) => fault.evidenceIds.length > 0)).toBe(true); + }); +}); diff --git a/packages/paperclip-runner/src/eval/eval-execution.ts b/packages/paperclip-runner/src/eval/eval-execution.ts new file mode 100644 index 00000000000..d222e1f3793 --- /dev/null +++ b/packages/paperclip-runner/src/eval/eval-execution.ts @@ -0,0 +1,644 @@ +import { capabilityFixtureRunCapabilities } from "../scenarios/fixture-run-capabilities.js"; +import { + CapabilityMockControlPlaneAdapter, +} from "../mock-core/capability-mock-control-plane-adapter.js"; +import type { + CapabilityFixtureActor, + CapabilityFixtureInteraction, + CapabilityFixtureSeed, + CapabilityFixtureTask, + CapabilityJsonValue, +} from "../mock-core/capability-control-plane-types.js"; +import { CapabilitySemanticDispatcher } from "../semantic-tools/dispatcher.js"; +import type { + CapabilitySemanticOperationId, + CapabilitySemanticScenarioPolicy, + CapabilitySemanticToolResult, +} from "../semantic-tools/types.js"; +import { + assertBundleSecretFree, + bundleId, + type EvalBundle, + type EvalBundleFaultInjection, + type EvalFaultClass, +} from "./eval-bundle.js"; +import { scoreEval, type AuthorizationState, type EvalObservation } from "./eval-scoring.js"; + +export const EVAL_BEHAVIOR_IDS = [ + "checkout_context", + "revision_safe_plan_editing", + "approval_denial", + "interaction_continuation", + "blocker_monitor", + "artifact_registration", + "restraint_no_call", + "terminal_arbitration", +] as const; + +export type EvalBehaviorId = typeof EVAL_BEHAVIOR_IDS[number]; +export type EvalCounterpart = "green" | "red"; + +export interface EvalBehaviorExecutionResult { + behavior: EvalBehaviorId; + counterpart: EvalCounterpart; + observation: EvalObservation; +} + +interface Invocation { + operationId: string; + input: CapabilityJsonValue; + callId?: string; +} + +interface BehaviorCase { + behavior: EvalBehaviorId; + counterpart: EvalCounterpart; + seed?: CapabilityFixtureSeed; + scenario?: CapabilitySemanticScenarioPolicy; + wake?: { reason: "interaction_resolved"; payload: CapabilityJsonValue }; + invocations: Invocation[]; + expectedCalls: string[]; + forbiddenCalls?: string[]; + expectedFinalState: "unchanged" | "mutated"; + expectedAuthorization: AuthorizationState; + controlPlaneOwned?: boolean; + maxAttempts?: number; + faultClass?: EvalFaultClass; + faultEvidenceOperation?: string; + assertState?: (adapter: CapabilityMockControlPlaneAdapter) => void; +} + +const RUN_ID = "run-eval-behavior"; +const SESSION_ID = "session-eval-behavior"; +const ACTIVE_TASK_ID = "task-1"; + +function actor(grants: readonly string[]): CapabilityFixtureActor { + return { + id: "actor-1", + companyId: "company-1", + name: "Eval approver", + role: "approver", + status: "active", + budgetId: "budget-actor-1", + capabilityGrants: [...grants], + }; +} + +function task(overrides: Partial = {}): CapabilityFixtureTask { + return { + id: ACTIVE_TASK_ID, + companyId: "company-1", + identifier: "MCK-1", + title: "Eval behavior fixture", + description: null, + status: "todo", + priority: "medium", + workMode: "standard", + parentId: null, + assigneeActorId: "actor-1", + checkoutRunId: null, + executionRunId: null, + startedAt: null, + completedAt: null, + ...overrides, + }; +} + +function seedFor(bundle: EvalBundle, extra: CapabilityFixtureSeed = {}): CapabilityFixtureSeed { + return { + ...extra, + actors: extra.actors ?? [actor(bundle.grants)], + tasks: extra.tasks ?? [task()], + }; +} + +function faultFor(bundle: EvalBundle, faultClass: EvalFaultClass): EvalBundleFaultInjection { + const matches = bundle.faultInjection.filter((fault) => fault.class === faultClass); + if (matches.length !== 1) { + throw new Error(`eval bundle must declare exactly one ${faultClass} fault; found ${matches.length}`); + } + return matches[0]!; +} + +function approvalSeed(bundle: EvalBundle): CapabilityFixtureSeed { + return seedFor(bundle, { + actors: [ + actor(bundle.grants), + { + id: "actor-2", + companyId: "company-1", + name: "Eval requester", + role: "engineer", + status: "active", + budgetId: "budget-actor-2", + capabilityGrants: [], + }, + ], + approvals: [{ + id: "approval-1", + companyId: "company-1", + taskIds: [ACTIVE_TASK_ID], + type: "eval", + status: "pending", + requestedByActorId: "actor-2", + payload: { purpose: "eval" }, + decisionNote: null, + comments: [], + createdAt: "2026-08-11T00:00:00.000Z", + decidedAt: null, + }], + }); +} + +function interactionSeed(bundle: EvalBundle): CapabilityFixtureSeed { + const interaction: CapabilityFixtureInteraction = { + id: "interaction-1", + taskId: ACTIVE_TASK_ID, + kind: "checkbox", + status: "accepted", + title: "Select artifacts", + prompt: "Select the artifacts to retain.", + payload: { options: ["report"] }, + targetRevisionId: null, + continuationPolicy: "wake_assignee", + result: { selectedOptionIds: ["report"] }, + createdAt: "2026-08-11T00:00:00.000Z", + resolvedAt: "2026-08-11T00:00:01.000Z", + }; + return seedFor(bundle, { interactions: [interaction] }); +} + +function planSeed(bundle: EvalBundle): CapabilityFixtureSeed { + return seedFor(bundle, { + documents: [{ + id: "document-plan", + taskId: ACTIVE_TASK_ID, + key: "plan", + title: "Plan", + format: "markdown", + latestRevisionId: "revision-plan-1", + revisions: [{ + id: "revision-plan-1", + documentId: "document-plan", + revision: 1, + body: "# Plan\n\nInitial.", + changeSummary: null, + createdAt: "2026-08-11T00:00:00.000Z", + }], + }], + }); +} + +function blockerSeed(bundle: EvalBundle): CapabilityFixtureSeed { + return seedFor(bundle, { + tasks: [ + task(), + task({ + id: "task-blocker", + identifier: "MCK-2", + title: "External prerequisite", + assigneeActorId: null, + }), + ], + }); +} + +function behaviorCases(bundle: EvalBundle): BehaviorCase[] { + const conflict = faultFor(bundle, "conflict"); + const authorization = faultFor(bundle, "authorization"); + const retry = faultFor(bundle, "retry"); + const providerCapability = faultFor(bundle, "provider_capability"); + const planInput = (baseRevisionId: string): CapabilityJsonValue => ({ + idempotencyKey: `eval-plan-${baseRevisionId}`, + key: "plan", + title: "Plan", + body: "# Plan\n\nRevised safely.", + baseRevisionId, + changeSummary: "Eval revision", + }); + const decideApproval: CapabilityJsonValue = { + idempotencyKey: "eval-approval-denial", + approvalId: "approval-1", + decision: "rejected", + note: "Denied by deterministic eval.", + }; + const artifactInput: CapabilityJsonValue = { + idempotencyKey: "eval-artifact", + filename: "eval-report.json", + contentType: "application/json", + byteSize: 2, + sha256: "0".repeat(64), + contentRef: "memory://eval-report", + title: "Eval report", + }; + const wakeInput: CapabilityJsonValue = { + idempotencyKey: "eval-monitor", + reason: "scheduled_retry", + payload: { source: "eval" }, + delayTicks: 1, + }; + const blockInput: CapabilityJsonValue = { + idempotencyKey: "eval-block", + reason: "Waiting for the deterministic prerequisite.", + blockedByTaskIds: ["task-blocker"], + }; + + return [ + { + behavior: "checkout_context", + counterpart: "green", + invocations: [{ operationId: "get_task_context", input: {} }], + expectedCalls: ["get_task_context"], + expectedFinalState: "unchanged", + expectedAuthorization: "allowed", + assertState: (adapter) => { + const context = adapter.context(RUN_ID); + if (context.activeTask.status !== "in_progress" || context.activeTask.checkoutRunId !== RUN_ID) { + throw new Error("checkout/context green case did not retain the atomic checkout context"); + } + }, + }, + { + behavior: "checkout_context", + counterpart: "red", + invocations: [{ operationId: "checkout_task", input: {} }], + expectedCalls: [], + forbiddenCalls: ["checkout_task"], + expectedFinalState: "unchanged", + expectedAuthorization: "absent", + controlPlaneOwned: true, + }, + { + behavior: "revision_safe_plan_editing", + counterpart: "green", + seed: planSeed(bundle), + invocations: [{ operationId: "write_document", input: planInput("revision-plan-1") }], + expectedCalls: ["write_document"], + expectedFinalState: "mutated", + expectedAuthorization: "allowed", + assertState: (adapter) => { + const document = adapter.snapshot().documents.find((candidate) => candidate.key === "plan"); + if (document?.revisions.length !== 2) throw new Error("plan green case did not append revision 2"); + }, + }, + { + behavior: "revision_safe_plan_editing", + counterpart: "red", + seed: planSeed(bundle), + invocations: [{ operationId: "write_document", input: planInput("revision-stale") }], + expectedCalls: ["write_document"], + expectedFinalState: "mutated", + expectedAuthorization: "allowed", + faultClass: conflict.class, + faultEvidenceOperation: "write_document", + }, + { + behavior: "approval_denial", + counterpart: "green", + seed: approvalSeed(bundle), + invocations: [{ operationId: "decide_approval", input: decideApproval }], + expectedCalls: ["decide_approval"], + expectedFinalState: "mutated", + expectedAuthorization: "allowed", + assertState: (adapter) => { + if (adapter.snapshot().approvals[0]?.status !== "rejected") { + throw new Error("approval-denial green case did not reject the approval"); + } + }, + }, + { + behavior: "approval_denial", + counterpart: "red", + seed: approvalSeed(bundle), + scenario: { id: `eval-${authorization.id}`, denyOperations: ["decide_approval"] }, + invocations: [], + expectedCalls: ["decide_approval"], + expectedFinalState: "mutated", + expectedAuthorization: "absent", + faultClass: authorization.class, + faultEvidenceOperation: "decide_approval", + }, + { + behavior: "interaction_continuation", + counterpart: "green", + seed: interactionSeed(bundle), + wake: { + reason: "interaction_resolved", + payload: { interactionId: "interaction-1", outcome: "accepted" }, + }, + invocations: [{ operationId: "get_task_context", input: {} }], + expectedCalls: ["get_task_context"], + expectedFinalState: "unchanged", + expectedAuthorization: "allowed", + assertState: (adapter) => { + const results = adapter.context(RUN_ID).interactionResults; + if (results[0]?.status !== "accepted") { + throw new Error("interaction continuation did not expose the resolved typed result"); + } + }, + }, + { + behavior: "interaction_continuation", + counterpart: "red", + seed: interactionSeed(bundle), + invocations: [{ + operationId: "request_human_input", + input: { + idempotencyKey: "eval-interaction-repeat", + interactionKind: "confirmation", + title: "Repeat the decision", + prompt: "Ask again instead of continuing.", + continuationPolicy: "wake_assignee", + }, + }], + expectedCalls: ["get_task_context"], + expectedFinalState: "unchanged", + expectedAuthorization: "allowed", + }, + { + behavior: "blocker_monitor", + counterpart: "green", + seed: blockerSeed(bundle), + invocations: [ + { operationId: "schedule_wake", input: wakeInput }, + { operationId: "block_task", input: blockInput }, + ], + expectedCalls: ["schedule_wake", "block_task"], + expectedFinalState: "mutated", + expectedAuthorization: "allowed", + maxAttempts: 2, + assertState: (adapter) => { + const state = adapter.snapshot(); + if (state.tasks[0]?.status !== "blocked" || state.wakes.length !== 1 || state.blockers.length !== 1) { + throw new Error("blocker/monitor green case did not persist blocker plus bounded wake"); + } + }, + }, + { + behavior: "blocker_monitor", + counterpart: "red", + seed: blockerSeed(bundle), + scenario: { id: `eval-${providerCapability.id}`, denyOperations: ["schedule_wake"] }, + invocations: [{ operationId: "block_task", input: blockInput }], + expectedCalls: ["schedule_wake", "block_task"], + expectedFinalState: "mutated", + expectedAuthorization: "allowed", + maxAttempts: 2, + faultClass: providerCapability.class, + faultEvidenceOperation: "schedule_wake", + }, + { + behavior: "artifact_registration", + counterpart: "green", + invocations: [{ operationId: "register_deliverable", input: artifactInput }], + expectedCalls: ["register_deliverable"], + expectedFinalState: "mutated", + expectedAuthorization: "allowed", + assertState: (adapter) => { + const state = adapter.snapshot(); + if (state.artifacts.length !== 1 || state.workProducts[0]?.artifactId !== state.artifacts[0]?.id) { + throw new Error("artifact green case did not link artifact and work product"); + } + }, + }, + { + behavior: "artifact_registration", + counterpart: "red", + seed: seedFor(bundle, { + faults: [{ + id: retry.id, + operation: "apply_command", + commandKind: "register_deliverable", + effect: "retryable_error", + remaining: 1, + code: "eval_retry_injected", + }], + }), + invocations: [ + { operationId: "register_deliverable", input: artifactInput, callId: "call-artifact-attempt-1" }, + { operationId: "register_deliverable", input: artifactInput, callId: "call-artifact-attempt-2" }, + ], + expectedCalls: ["register_deliverable"], + expectedFinalState: "mutated", + expectedAuthorization: "allowed", + maxAttempts: 1, + faultClass: retry.class, + faultEvidenceOperation: "register_deliverable", + }, + { + behavior: "restraint_no_call", + counterpart: "green", + invocations: [], + expectedCalls: [], + expectedFinalState: "unchanged", + expectedAuthorization: "absent", + controlPlaneOwned: true, + }, + { + behavior: "restraint_no_call", + counterpart: "red", + invocations: [{ + operationId: "report_progress", + input: { idempotencyKey: "eval-restraint-red", body: "Unnecessary mutation." }, + }], + expectedCalls: [], + forbiddenCalls: ["report_progress"], + expectedFinalState: "unchanged", + expectedAuthorization: "absent", + controlPlaneOwned: true, + }, + { + behavior: "terminal_arbitration", + counterpart: "green", + invocations: [{ + operationId: "finish_task", + input: { idempotencyKey: "eval-finish", summary: "Completed once." }, + }], + expectedCalls: ["finish_task"], + expectedFinalState: "mutated", + expectedAuthorization: "allowed", + assertState: (adapter) => { + if (adapter.snapshot().tasks[0]?.status !== "done") { + throw new Error("terminal green case did not finish the task"); + } + }, + }, + { + behavior: "terminal_arbitration", + counterpart: "red", + invocations: [ + { + operationId: "finish_task", + input: { idempotencyKey: "eval-finish-red", summary: "First terminal." }, + }, + { + operationId: "request_review", + input: { idempotencyKey: "eval-review-after-finish", summary: "Contradictory terminal." }, + }, + ], + expectedCalls: ["finish_task"], + forbiddenCalls: ["request_review"], + expectedFinalState: "mutated", + expectedAuthorization: "allowed", + maxAttempts: 1, + assertState: (adapter) => { + if (adapter.snapshot().tasks[0]?.status !== "done") { + throw new Error("terminal arbitration did not preserve the first terminal state"); + } + }, + }, + ]; +} + +function observedAuthorization(results: readonly CapabilitySemanticToolResult[]): AuthorizationState { + if (results.length === 0) return "absent"; + if (results.some((result) => result.ok)) return "allowed"; + return results.every((result) => !result.ok && result.denial.code === "tool_not_exposed") + ? "absent" + : "denied"; +} + +async function runBehaviorCase(bundle: EvalBundle, spec: BehaviorCase): Promise { + const adapter = new CapabilityMockControlPlaneAdapter(spec.seed ?? seedFor(bundle)); + await adapter.start(); + await adapter.openFixtureRun({ + identity: { + runId: RUN_ID, + sessionId: SESSION_ID, + companyId: "company-1", + issueId: ACTIVE_TASK_ID, + agentId: "actor-1", + }, + backendKind: "mock", + sourceInstanceId: "eval-behavior-harness", + capabilities: capabilityFixtureRunCapabilities(bundle.grants), + wake: spec.wake, + }); + const beforeRevision = adapter.snapshot().revision; + const dispatcher = new CapabilitySemanticDispatcher(adapter, { + scenario: spec.scenario ?? { id: `eval-${spec.behavior}-${spec.counterpart}` }, + explicitClaims: bundle.grants, + }); + dispatcher.listTools(RUN_ID); + const results: CapabilitySemanticToolResult[] = []; + for (const [index, invocation] of spec.invocations.entries()) { + results.push(await dispatcher.dispatch({ + runId: RUN_ID, + callId: invocation.callId ?? `call-${spec.behavior}-${spec.counterpart}-${index + 1}`, + operationId: invocation.operationId, + input: invocation.input, + })); + } + spec.assertState?.(adapter); + const afterRevision = adapter.snapshot().revision; + const observedCalls = spec.invocations.map((invocation) => invocation.operationId); + const caseId = `${spec.behavior}:${spec.counterpart}`; + const authorizationRecords = dispatcher.authorizationRecords(); + const decisionRecords = adapter.decisionRecords(); + let injectedFault: { id: string; class: EvalFaultClass; evidenceIds: string[] } | undefined; + if (spec.faultClass !== undefined) { + const declaration = faultFor(bundle, spec.faultClass); + const evidenceIds = [ + ...decisionRecords + .filter((record) => record.outcome === "faulted") + .map((record) => record.id), + ...authorizationRecords + .filter( + (record) => + record.operationId === spec.faultEvidenceOperation && !record.allowed, + ) + .map((record) => record.id), + ]; + if (evidenceIds.length === 0) { + throw new Error(`${caseId} did not emit evidence for ${spec.faultClass} injection`); + } + injectedFault = { id: declaration.id, class: declaration.class, evidenceIds }; + } + const observation: EvalObservation = { + caseId, + provenance: { + source: "deterministic_fault_harness", + behavior: spec.behavior, + counterpart: spec.counterpart, + ...(injectedFault === undefined ? {} : { faultInjection: injectedFault }), + }, + controlPlaneOwned: spec.controlPlaneOwned ?? false, + expectedCalls: spec.expectedCalls, + observedCalls, + forbiddenCalls: spec.forbiddenCalls ?? [], + finalState: { + expected: spec.expectedFinalState, + observed: afterRevision === beforeRevision ? "unchanged" : "mutated", + }, + authorization: { + expected: spec.expectedAuthorization, + observed: observedAuthorization(results), + }, + trace: { + runId: RUN_ID, + sessionId: SESSION_ID, + turnId: `turn-${caseId}`, + itemId: `item-${caseId}`, + receiptIds: results.map((result) => result.callId), + terminalPresent: true, + }, + efficiency: { attempts: observedCalls.length }, + budget: { maxAttempts: spec.maxAttempts ?? Math.max(1, spec.expectedCalls.length) }, + }; + await adapter.stop(); + return { behavior: spec.behavior, counterpart: spec.counterpart, observation }; +} + +/** + * Executes the eight requested behaviors and their red counterparts against the + * real deterministic mock authority. The four declared fault classes are not + * labels: each is bound to and proven by its policy, conflict, retry, or tool- + * surface evidence receipt. + */ +export async function runEvalBehaviorFaultMatrix( + bundle: EvalBundle, +): Promise { + assertBundleSecretFree(bundle); + for (const faultClass of ["authorization", "conflict", "retry", "provider_capability"] as const) { + faultFor(bundle, faultClass); + } + const results: EvalBehaviorExecutionResult[] = []; + for (const spec of behaviorCases(bundle)) results.push(await runBehaviorCase(bundle, spec)); + assertEvalBehaviorFaultMatrix(bundle, results); + return results; +} + +/** Fails closed if coverage, polarity, or scored red/green behavior drifts. */ +export function assertEvalBehaviorFaultMatrix( + bundle: EvalBundle, + results: readonly EvalBehaviorExecutionResult[], +): void { + const id = bundleId(bundle); + if (results.length !== EVAL_BEHAVIOR_IDS.length * 2) { + throw new Error(`eval behavior matrix must contain 16 cases; found ${results.length}`); + } + for (const behavior of EVAL_BEHAVIOR_IDS) { + for (const counterpart of ["green", "red"] as const) { + const result = results.find( + (candidate) => candidate.behavior === behavior && candidate.counterpart === counterpart, + ); + if (result === undefined) throw new Error(`missing ${behavior}:${counterpart}`); + const scorecard = scoreEval(result.observation, { bundleId: id }); + if (counterpart === "green" && !scorecard.overall.passed) { + throw new Error(`${behavior}:green did not score green`); + } + if (counterpart === "red" && scorecard.overall.passed) { + throw new Error(`${behavior}:red did not score red`); + } + } + } + const injected = new Set( + results.flatMap((result) => { + const fault = result.observation.provenance?.faultInjection; + return fault === undefined ? [] : [fault.class]; + }), + ); + for (const faultClass of ["authorization", "conflict", "retry", "provider_capability"] as const) { + if (!injected.has(faultClass)) throw new Error(`fault class ${faultClass} was not executed`); + } +} diff --git a/packages/paperclip-runner/src/eval/eval-scoring.ts b/packages/paperclip-runner/src/eval/eval-scoring.ts index 3c23bda7744..0a79fa95cdf 100644 --- a/packages/paperclip-runner/src/eval/eval-scoring.ts +++ b/packages/paperclip-runner/src/eval/eval-scoring.ts @@ -2,6 +2,7 @@ import type { CapabilityEvalCaseResult, CapabilityLiveCodexMatrixResult, } from "../conformance/capability-eval-suite.js"; +import type { EvalFaultClass } from "./eval-bundle.js"; /** * Multi-dimensional scoring for the runner eval vertical slice. @@ -82,6 +83,17 @@ export interface EvalEfficiencyBudget { export interface EvalObservation { caseId: string; + provenance?: { + source: "live_model" | "deterministic_fault_harness" | "fixture"; + behavior?: string; + counterpart?: "green" | "red"; + faultInjection?: { + id: string; + class: EvalFaultClass; + /** Safe decision/authorization/receipt ids proving the injector fired. */ + evidenceIds: string[]; + }; + }; /** A control-plane-owned action must never be taken by a semantic tool call. */ controlPlaneOwned: boolean; expectedCalls: string[]; @@ -281,6 +293,7 @@ export interface LiveMatrixAugment { authorization: EvalAuthorizationExpectation; efficiency?: EvalEfficiencyEvidence; budget?: EvalEfficiencyBudget; + provenance?: EvalObservation["provenance"]; } /** @@ -295,6 +308,7 @@ export function observationFromLiveMatrix( ): EvalObservation { return { caseId: result.caseId, + provenance: augment.provenance, controlPlaneOwned: result.expectedCalls.length === 0, expectedCalls: result.expectedCalls, observedCalls: result.observedCalls, diff --git a/packages/paperclip-runner/src/eval/eval-slice.ts b/packages/paperclip-runner/src/eval/eval-slice.ts index ff80d3ee4de..9895a2ab81e 100644 --- a/packages/paperclip-runner/src/eval/eval-slice.ts +++ b/packages/paperclip-runner/src/eval/eval-slice.ts @@ -103,10 +103,16 @@ export function buildEvalSliceReport( /** A compact, human-readable rendering of a slice report for inspection. */ export function renderEvalSliceMarkdown(report: EvalSliceReport): string { const lines: string[] = []; + const sources = new Set(report.cases.map((entry) => entry.observation.provenance?.source)); + const sourceDescription = report.generatedFromLiveModel + ? "real model tool choice" + : sources.size === 1 && sources.has("deterministic_fault_harness") + ? "deterministic fault harness" + : "fixture observations"; lines.push(`# Runner eval slice — ${report.bundle.id}`); lines.push(""); lines.push(`- Bundle: \`${report.bundle.summary}\``); - lines.push(`- Source: ${report.generatedFromLiveModel ? "real model tool choice" : "fixture observations"}`); + lines.push(`- Source: ${sourceDescription}`); lines.push( `- Cases: ${report.aggregate.caseCount} · passed ${report.aggregate.passed} · gate failures ${report.aggregate.gateFailures} · mean overall ${report.aggregate.meanOverall}`, ); @@ -119,12 +125,13 @@ export function renderEvalSliceMarkdown(report: EvalSliceReport): string { lines.push(""); lines.push("## Cases"); lines.push(""); - lines.push("| case | gate | overall | outcome | trajectory | trace | efficiency |"); - lines.push("| --- | --- | --- | --- | --- | --- | --- |"); - for (const { scorecard } of report.cases) { + lines.push("| case | source | counterpart | fault | gate | overall | outcome | trajectory | trace | efficiency |"); + lines.push("| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |"); + for (const { scorecard, observation } of report.cases) { const d = scorecard.dimensions; + const provenance = observation.provenance; lines.push( - `| ${scorecard.caseId} | ${d.hard_invariants.passed ? "ok" : "FAIL"} | ${scorecard.overall.score} | ${d.semantic_outcome.score} | ${d.trajectory_restraint.score} | ${d.trace_completeness.score} | ${d.quality_efficiency.score} |`, + `| ${scorecard.caseId} | ${provenance?.source ?? "unspecified"} | ${provenance?.counterpart ?? "—"} | ${provenance?.faultInjection?.class ?? "—"} | ${d.hard_invariants.passed ? "ok" : "FAIL"} | ${scorecard.overall.score} | ${d.semantic_outcome.score} | ${d.trajectory_restraint.score} | ${d.trace_completeness.score} | ${d.quality_efficiency.score} |`, ); } lines.push(""); diff --git a/packages/paperclip-runner/src/eval/index.ts b/packages/paperclip-runner/src/eval/index.ts index ce98831fdb3..fdbc422cc18 100644 --- a/packages/paperclip-runner/src/eval/index.ts +++ b/packages/paperclip-runner/src/eval/index.ts @@ -1,3 +1,4 @@ export * from "./eval-bundle.js"; +export * from "./eval-execution.js"; export * from "./eval-scoring.js"; export * from "./eval-slice.js"; diff --git a/packages/paperclip-runner/src/live/live-session.test.ts b/packages/paperclip-runner/src/live/live-session.test.ts index 9b68dfa712b..db298e146ef 100644 --- a/packages/paperclip-runner/src/live/live-session.test.ts +++ b/packages/paperclip-runner/src/live/live-session.test.ts @@ -67,13 +67,21 @@ class FakeCapabilityCodexTransport implements CodexAppServerTransport { if (method === "initialize") return { user: { sessionId: this.state.providerSessionId } }; if (method === "thread/start") { expect(Array.isArray(params.dynamicTools)).toBe(true); - return { thread: { id: this.state.threadId, sessionId: this.state.providerSessionId } }; + return { + model: "gpt-eval-test", + modelProvider: "openai", + thread: { id: this.state.threadId, sessionId: this.state.providerSessionId }, + }; } if (method === "thread/read") { return { thread: { id: this.state.threadId, sessionId: this.state.providerSessionId, turns: [] } }; } if (method === "thread/resume") { - return { thread: { id: this.state.threadId, sessionId: this.state.providerSessionId } }; + return { + model: "gpt-eval-test", + modelProvider: "openai", + thread: { id: this.state.threadId, sessionId: this.state.providerSessionId }, + }; } if (method === "turn/start") { const turnId = `turn-${++this.state.nextTurn}`; @@ -241,6 +249,7 @@ describe("Capability live runnerd and Codex session", () => { expect(first.assistantText).toContain("stateRevision"); expect(second.assistantText).toContain("stateRevision"); expect(first.snapshot.providerThreadId).toBe(second.snapshot.providerThreadId); + expect(first.snapshot.providerModel).toEqual({ id: "gpt-eval-test", provider: "openai" }); expect(session.mockState().comments.at(-1)?.body).toBe( "Progress persisted through the live Codex tool loop.", ); diff --git a/packages/paperclip-runner/src/live/live-session.ts b/packages/paperclip-runner/src/live/live-session.ts index 82a82c8b543..bcf1f9b7323 100644 --- a/packages/paperclip-runner/src/live/live-session.ts +++ b/packages/paperclip-runner/src/live/live-session.ts @@ -101,6 +101,8 @@ export interface CapabilityLiveSessionSnapshot { sessionId: string; providerThreadId: string; providerSessionId: string | null; + /** Safe provider identity retained for reproducible eval bundle declarations. */ + providerModel?: { id: string; provider: string }; status: CapabilityLiveSessionStatus; activeTurnId: string | null; createdAt: string; @@ -440,6 +442,7 @@ export class CapabilityLiveSession { #processEvidence: CapabilityRunnerdProcessEvidence | null = null; #providerThreadId = ""; #providerSessionId: string | null = null; + #providerModel: { id: string; provider: string } | undefined; #status: CapabilityLiveSessionStatus = "starting"; #activeTurnId: string | null = null; #revision = 0; @@ -471,6 +474,9 @@ export class CapabilityLiveSession { this.#priorAuthorizationRecords = structuredClone(options.snapshot?.authorizationRecords ?? []); this.#providerThreadId = options.snapshot?.providerThreadId ?? ""; this.#providerSessionId = options.snapshot?.providerSessionId ?? null; + this.#providerModel = options.snapshot?.providerModel === undefined + ? undefined + : structuredClone(options.snapshot.providerModel); this.#activeTurnId = options.snapshot?.activeTurnId ?? null; this.#revision = options.snapshot?.revision ?? 0; this.#entryCounter = this.#transcript.length + this.#evidence.length; @@ -600,6 +606,9 @@ export class CapabilityLiveSession { sessionId: this.id, providerThreadId: this.#providerThreadId, providerSessionId: this.#providerSessionId, + ...(this.#providerModel === undefined + ? {} + : { providerModel: structuredClone(this.#providerModel) }), status: this.#status, activeTurnId: this.#activeTurnId, createdAt: this.#createdAt, @@ -818,6 +827,7 @@ export class CapabilityLiveSession { throw new Error("Codex reconnect resumed a different provider thread"); } this.#providerSessionId = text(resumedThread.sessionId) || this.#providerSessionId; + this.#captureProviderModel(resumed); this.#activeTurnId = null; } else { const opened = await this.#transport.request("thread/start", { @@ -835,6 +845,7 @@ export class CapabilityLiveSession { this.#providerThreadId = text(thread.id); if (this.#providerThreadId.length === 0) throw new Error("Codex thread response omitted thread.id"); this.#providerSessionId = text(thread.sessionId) || text(record(initialized.user).sessionId) || null; + this.#captureProviderModel(opened); } this.#status = "idle"; this.#appendEvidence("session", null, { @@ -850,6 +861,13 @@ export class CapabilityLiveSession { await this.#persist(); } + #captureProviderModel(response: Record): void { + const thread = record(response.thread); + const id = text(response.model, text(thread.model)); + const provider = text(response.modelProvider, text(thread.modelProvider)); + if (id.length > 0 && provider.length > 0) this.#providerModel = { id, provider }; + } + async #disconnect(reason: string): Promise { const transport = this.#transport; this.#transport = null;