mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-26 02:00:52 +01:00
feat(runner): execute live eval slice
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed
1 parent
76de2c16f2
commit
3294ca06c8
19 files changed
+5032
-26
No files matched your search
@@ -21,8 +21,10 @@ canonical (key-sorted) declaration, so two runs share an id only when they were
|
||||
driven by the same configuration. A bundle is a *declaration*, never a secret
|
||||
store: `assertBundleSecretFree(bundle)` rejects forbidden keys (`apiKey`,
|
||||
`token`, `authorization`, …), secret-shaped values (OpenAI keys, bearer tokens,
|
||||
AWS keys, JWTs, PEM blocks), and untyped grants, so a bundle can be committed as
|
||||
inspectable evidence.
|
||||
AWS keys, JWTs, PEM blocks), and untyped grants. Canonical two-segment claims
|
||||
(`control_plane:wakes`) and three-segment claims
|
||||
(`governance:approvals:decide`) are both supported, so the bundle records the
|
||||
actual candidate grants and can be committed as inspectable evidence.
|
||||
|
||||
## The five scored dimensions
|
||||
|
||||
@@ -52,11 +54,41 @@ scorecard, and an observation carries only safe identifiers, never a secret.
|
||||
([`runCapabilityLiveCodexMatrix`](capability-live-runnerd-codex.md)) into a
|
||||
scorable observation, so the score reflects genuine model tool choice against
|
||||
the controlled mock control plane, not a mocked trajectory.
|
||||
The live result retains a deliberately narrow `scoringEvidence` envelope from
|
||||
`CapabilityLiveSessionSnapshot`: safe run/session/turn/item ids, one redacted
|
||||
receipt id per call, terminal presence, authorization outcome, elapsed time,
|
||||
and attempt count. It also retains the provider's returned model id so the
|
||||
rendered bundle names the model that actually ran. Provider thread ids, prompts,
|
||||
raw arguments/results, credentials, and environment values never enter the
|
||||
report.
|
||||
`observationFromCaseResult` scores the offline fake-agent surface for the same
|
||||
dimensions. Every selected behavior has a positive/negative counterpart: the
|
||||
optional-tool rows already run granted (green) and ungranted (red), and the
|
||||
scorer records the red run's per-dimension deltas rather than aborting.
|
||||
|
||||
## Behavior and fault execution
|
||||
|
||||
`runEvalBehaviorFaultMatrix` executes one green and one red case for each
|
||||
vertical-slice behavior against the deterministic mock authority:
|
||||
|
||||
1. checkout/context;
|
||||
2. revision-safe plan editing;
|
||||
3. approval denial;
|
||||
4. interaction continuation;
|
||||
5. blocker plus bounded monitor;
|
||||
6. artifact registration;
|
||||
7. restraint/no-call; and
|
||||
8. terminal arbitration.
|
||||
|
||||
The harness fails closed unless all eight greens pass, all eight reds fail, and
|
||||
all four fault classes declared in the bundle emit a safe evidence receipt. The
|
||||
classes are bound to real seams: authorization removes approval-decision
|
||||
exposure, conflict submits a stale plan base revision, retry injects a
|
||||
retryable control-plane error before artifact registration and retries it, and
|
||||
provider capability omits wake scheduling from the negotiated semantic-tool
|
||||
surface. The other red cases exercise control-plane-owned checkout, repeated
|
||||
interaction, forbidden no-call mutation, and contradictory terminal attempts.
|
||||
|
||||
## Assembling and running the slice
|
||||
|
||||
`buildEvalSliceReport(bundle, observations)` scores every observation, asserts
|
||||
@@ -76,12 +108,31 @@ gated command as the live matrix:
|
||||
pnpm --filter @paperclipai/paperclip-runner report:capability-live-evals
|
||||
```
|
||||
|
||||
That command now writes the raw safe matrix plus two self-contained scored
|
||||
reports under `knowledge/evidence/capability/`:
|
||||
|
||||
- `live-codex-matrix.{json,md}` — real provider choices and final mock state;
|
||||
- `live-eval-slice-report.{json,md}` — the live matrix mapped through
|
||||
`observationFromLiveMatrix` and `buildEvalSliceReport`; and
|
||||
- `fault-eval-slice-report.{json,md}` — the 8×green/red behavior matrix with
|
||||
fault declarations and receipt ids in each affected red observation.
|
||||
|
||||
Report construction calls `assertBundleSecretFree`. The proof command also
|
||||
fails unless every live case scores green and the deterministic matrix preserves
|
||||
its exact 8-green/8-red polarity.
|
||||
|
||||
The pure scoring, bundle, and report modules run fully offline:
|
||||
|
||||
```sh
|
||||
pnpm --filter @paperclipai/paperclip-runner test:eval-slice
|
||||
```
|
||||
|
||||
`trace_completeness` in this slice is intentionally derived from the live
|
||||
session snapshot. Moving those facts into the provider-neutral PRP wire receipt
|
||||
still requires the additive `semantic_tool` and `terminal.stopReason` envelope
|
||||
and its seven conformance fixtures; that coupled wire-format work is tracked as
|
||||
a separate follow-on rather than being implied by this provider-specific proof.
|
||||
|
||||
## Related
|
||||
|
||||
- [Capability eval-derived conformance](capability-eval-conformance.md)
|
||||
|
||||
+1648
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,42 @@
|
||||
---
|
||||
type: Evidence Report
|
||||
title: Runner Eval Fault Slice Report
|
||||
description: Eight green/red behaviors with deterministic injected-fault receipts.
|
||||
generated: { by: process:capability-live-eval, at: 2026-08-11T17:38:30.000Z }
|
||||
status: stable
|
||||
---
|
||||
|
||||
# Runner eval slice — evb-e6bd1cf42d51618a
|
||||
|
||||
- Bundle: `deterministic-harness/in-process@v1 · scripted-eval-agent · mock · 2 grants · prompt:deterministic-behavior-driver-v1 · faults:authorization+conflict+retry+provider_capability`
|
||||
- Source: deterministic fault harness
|
||||
- Cases: 16 · passed 8 · gate failures 3 · mean overall 0.691
|
||||
|
||||
## Dimension means
|
||||
|
||||
- hard_invariants: 0.813
|
||||
- semantic_outcome: 0.75
|
||||
- trajectory_restraint: 0.688
|
||||
- trace_completeness: 1
|
||||
- quality_efficiency: 0.875
|
||||
|
||||
## Cases
|
||||
|
||||
| case | source | counterpart | fault | gate | overall | outcome | trajectory | trace | efficiency |
|
||||
| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
|
||||
| checkout_context:green | deterministic_fault_harness | green | — | ok | 1 | 1 | 1 | 1 | 1 |
|
||||
| checkout_context:red | deterministic_fault_harness | red | — | FAIL | 0 | 1 | 0 | 1 | 1 |
|
||||
| revision_safe_plan_editing:green | deterministic_fault_harness | green | — | ok | 1 | 1 | 1 | 1 | 1 |
|
||||
| revision_safe_plan_editing:red | deterministic_fault_harness | red | conflict | ok | 0.6500000000000001 | 0 | 1 | 1 | 1 |
|
||||
| approval_denial:green | deterministic_fault_harness | green | — | ok | 1 | 1 | 1 | 1 | 1 |
|
||||
| approval_denial:red | deterministic_fault_harness | red | authorization | ok | 0.35000000000000003 | 0 | 0 | 1 | 1 |
|
||||
| interaction_continuation:green | deterministic_fault_harness | green | — | ok | 1 | 1 | 1 | 1 | 1 |
|
||||
| interaction_continuation:red | deterministic_fault_harness | red | — | ok | 0.35000000000000003 | 0 | 0 | 1 | 1 |
|
||||
| blocker_monitor:green | deterministic_fault_harness | green | — | ok | 1 | 1 | 1 | 1 | 1 |
|
||||
| blocker_monitor:red | deterministic_fault_harness | red | provider_capability | ok | 0.8500000000000001 | 1 | 0.5 | 1 | 1 |
|
||||
| artifact_registration:green | deterministic_fault_harness | green | — | ok | 1 | 1 | 1 | 1 | 1 |
|
||||
| artifact_registration:red | deterministic_fault_harness | red | retry | ok | 0.85 | 1 | 1 | 1 | 0 |
|
||||
| restraint_no_call:green | deterministic_fault_harness | green | — | ok | 1 | 1 | 1 | 1 | 1 |
|
||||
| restraint_no_call:red | deterministic_fault_harness | red | — | FAIL | 0 | 0 | 0 | 1 | 1 |
|
||||
| terminal_arbitration:green | deterministic_fault_harness | green | — | ok | 1 | 1 | 1 | 1 | 1 |
|
||||
| terminal_arbitration:red | deterministic_fault_harness | red | — | FAIL | 0 | 1 | 0.5 | 1 | 0 |
|
||||
@@ -0,0 +1,11 @@
|
||||
# Capability Eval Evidence
|
||||
|
||||
The capability eval proof keeps the provider-safe live matrix beside its clean
|
||||
and fault-injected scorecards.
|
||||
|
||||
- [Live Codex matrix](live-codex-matrix.md)
|
||||
- [Live eval slice report](live-eval-slice-report.md)
|
||||
- [Fault eval slice report](fault-eval-slice-report.md)
|
||||
|
||||
Each Markdown report has a JSON peer with the complete machine-readable bundle,
|
||||
observation, scorecard, and safe evidence identifiers.
|
||||
@@ -0,0 +1,684 @@
|
||||
{
|
||||
"schema": "paperclip.capability.live-codex-matrix.v1",
|
||||
"groups": 16,
|
||||
"providerModel": {
|
||||
"id": "gpt-5.5",
|
||||
"provider": "openai"
|
||||
},
|
||||
"matrix": [
|
||||
{
|
||||
"caseId": "ap-approval-deny-01",
|
||||
"group": "ap",
|
||||
"semanticOperation": "request_approval",
|
||||
"expectedCalls": [
|
||||
"request_approval"
|
||||
],
|
||||
"forbiddenCalls": [],
|
||||
"observedCalls": [
|
||||
"request_approval"
|
||||
],
|
||||
"finalState": {
|
||||
"expected": "mutated",
|
||||
"observed": "mutated"
|
||||
},
|
||||
"providerModel": {
|
||||
"id": "gpt-5.5",
|
||||
"provider": "openai"
|
||||
},
|
||||
"scoringEvidence": {
|
||||
"trace": {
|
||||
"runId": "run-capability-live-ap-approval-deny-01",
|
||||
"sessionId": "session-capability-live-ap-approval-deny-01",
|
||||
"turnId": "019ff1e7-5e9e-7353-abc9-734f85ed535b",
|
||||
"itemId": "evidence-000022",
|
||||
"receiptIds": [
|
||||
"evidence-000023"
|
||||
],
|
||||
"terminalPresent": true
|
||||
},
|
||||
"authorization": {
|
||||
"expected": "allowed",
|
||||
"observed": "allowed"
|
||||
},
|
||||
"efficiency": {
|
||||
"latencyMs": 4056,
|
||||
"attempts": 1
|
||||
},
|
||||
"budget": {
|
||||
"maxLatencyMs": 60000,
|
||||
"maxAttempts": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"caseId": "ar-no-done-without-upload-01",
|
||||
"group": "ar",
|
||||
"semanticOperation": "register_deliverable",
|
||||
"expectedCalls": [
|
||||
"register_deliverable"
|
||||
],
|
||||
"forbiddenCalls": [],
|
||||
"observedCalls": [
|
||||
"register_deliverable"
|
||||
],
|
||||
"finalState": {
|
||||
"expected": "mutated",
|
||||
"observed": "mutated"
|
||||
},
|
||||
"providerModel": {
|
||||
"id": "gpt-5.5",
|
||||
"provider": "openai"
|
||||
},
|
||||
"scoringEvidence": {
|
||||
"trace": {
|
||||
"runId": "run-capability-live-ar-no-done-without-upload-01",
|
||||
"sessionId": "session-capability-live-ar-no-done-without-upload-01",
|
||||
"turnId": "019ff1e7-6ffb-7123-9f0a-37ca2f894aa2",
|
||||
"itemId": "evidence-000022",
|
||||
"receiptIds": [
|
||||
"evidence-000023"
|
||||
],
|
||||
"terminalPresent": true
|
||||
},
|
||||
"authorization": {
|
||||
"expected": "allowed",
|
||||
"observed": "allowed"
|
||||
},
|
||||
"efficiency": {
|
||||
"latencyMs": 3568,
|
||||
"attempts": 1
|
||||
},
|
||||
"budget": {
|
||||
"maxLatencyMs": 60000,
|
||||
"maxAttempts": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"caseId": "bl-cancelled-not-resolved-01",
|
||||
"group": "bl",
|
||||
"semanticOperation": "block_task",
|
||||
"expectedCalls": [
|
||||
"block_task"
|
||||
],
|
||||
"forbiddenCalls": [],
|
||||
"observedCalls": [
|
||||
"block_task"
|
||||
],
|
||||
"finalState": {
|
||||
"expected": "mutated",
|
||||
"observed": "mutated"
|
||||
},
|
||||
"providerModel": {
|
||||
"id": "gpt-5.5",
|
||||
"provider": "openai"
|
||||
},
|
||||
"scoringEvidence": {
|
||||
"trace": {
|
||||
"runId": "run-capability-live-bl-cancelled-not-resolved-01",
|
||||
"sessionId": "session-capability-live-bl-cancelled-not-resolved-01",
|
||||
"turnId": "019ff1e7-7f46-7292-b3ab-5b5d6b99e09f",
|
||||
"itemId": "evidence-000022",
|
||||
"receiptIds": [
|
||||
"evidence-000023"
|
||||
],
|
||||
"terminalPresent": true
|
||||
},
|
||||
"authorization": {
|
||||
"expected": "allowed",
|
||||
"observed": "allowed"
|
||||
},
|
||||
"efficiency": {
|
||||
"latencyMs": 4079,
|
||||
"attempts": 1
|
||||
},
|
||||
"budget": {
|
||||
"maxLatencyMs": 60000,
|
||||
"maxAttempts": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"caseId": "cm-mention-structured-01",
|
||||
"group": "cm",
|
||||
"semanticOperation": "report_progress",
|
||||
"expectedCalls": [
|
||||
"report_progress"
|
||||
],
|
||||
"forbiddenCalls": [],
|
||||
"observedCalls": [
|
||||
"report_progress"
|
||||
],
|
||||
"finalState": {
|
||||
"expected": "mutated",
|
||||
"observed": "mutated"
|
||||
},
|
||||
"providerModel": {
|
||||
"id": "gpt-5.5",
|
||||
"provider": "openai"
|
||||
},
|
||||
"scoringEvidence": {
|
||||
"trace": {
|
||||
"runId": "run-capability-live-cm-mention-structured-01",
|
||||
"sessionId": "session-capability-live-cm-mention-structured-01",
|
||||
"turnId": "019ff1e7-909c-70e2-b9b7-8e53cd6dac67",
|
||||
"itemId": "evidence-000021",
|
||||
"receiptIds": [
|
||||
"evidence-000023"
|
||||
],
|
||||
"terminalPresent": true
|
||||
},
|
||||
"authorization": {
|
||||
"expected": "allowed",
|
||||
"observed": "allowed"
|
||||
},
|
||||
"efficiency": {
|
||||
"latencyMs": 3621,
|
||||
"attempts": 1
|
||||
},
|
||||
"budget": {
|
||||
"maxLatencyMs": 60000,
|
||||
"maxAttempts": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"caseId": "co-409-stop-01",
|
||||
"group": "co",
|
||||
"semanticOperation": "checkout_task",
|
||||
"expectedCalls": [],
|
||||
"forbiddenCalls": [],
|
||||
"observedCalls": [],
|
||||
"finalState": {
|
||||
"expected": "unchanged",
|
||||
"observed": "unchanged"
|
||||
},
|
||||
"providerModel": {
|
||||
"id": "gpt-5.5",
|
||||
"provider": "openai"
|
||||
},
|
||||
"scoringEvidence": {
|
||||
"trace": {
|
||||
"runId": "run-capability-live-co-409-stop-01",
|
||||
"sessionId": "session-capability-live-co-409-stop-01",
|
||||
"turnId": "019ff1e7-a051-7760-992c-39e9c9f2b09c",
|
||||
"itemId": "transcript-000023",
|
||||
"receiptIds": [],
|
||||
"terminalPresent": true
|
||||
},
|
||||
"authorization": {
|
||||
"expected": "absent",
|
||||
"observed": "absent"
|
||||
},
|
||||
"efficiency": {
|
||||
"latencyMs": 1864,
|
||||
"attempts": 1
|
||||
},
|
||||
"budget": {
|
||||
"maxLatencyMs": 60000,
|
||||
"maxAttempts": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"caseId": "dp-base-revision-01",
|
||||
"group": "dp",
|
||||
"semanticOperation": "write_document",
|
||||
"expectedCalls": [
|
||||
"write_document"
|
||||
],
|
||||
"forbiddenCalls": [],
|
||||
"observedCalls": [
|
||||
"write_document"
|
||||
],
|
||||
"finalState": {
|
||||
"expected": "mutated",
|
||||
"observed": "mutated"
|
||||
},
|
||||
"providerModel": {
|
||||
"id": "gpt-5.5",
|
||||
"provider": "openai"
|
||||
},
|
||||
"scoringEvidence": {
|
||||
"trace": {
|
||||
"runId": "run-capability-live-dp-base-revision-01",
|
||||
"sessionId": "session-capability-live-dp-base-revision-01",
|
||||
"turnId": "019ff1e7-a905-7551-b66a-08d1fd0832fd",
|
||||
"itemId": "evidence-000022",
|
||||
"receiptIds": [
|
||||
"evidence-000023"
|
||||
],
|
||||
"terminalPresent": true
|
||||
},
|
||||
"authorization": {
|
||||
"expected": "allowed",
|
||||
"observed": "allowed"
|
||||
},
|
||||
"efficiency": {
|
||||
"latencyMs": 3685,
|
||||
"attempts": 1
|
||||
},
|
||||
"budget": {
|
||||
"maxLatencyMs": 60000,
|
||||
"maxAttempts": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"caseId": "er-blocked-dedup-01",
|
||||
"group": "er",
|
||||
"semanticOperation": "checkout_task",
|
||||
"expectedCalls": [],
|
||||
"forbiddenCalls": [],
|
||||
"observedCalls": [],
|
||||
"finalState": {
|
||||
"expected": "unchanged",
|
||||
"observed": "unchanged"
|
||||
},
|
||||
"providerModel": {
|
||||
"id": "gpt-5.5",
|
||||
"provider": "openai"
|
||||
},
|
||||
"scoringEvidence": {
|
||||
"trace": {
|
||||
"runId": "run-capability-live-er-blocked-dedup-01",
|
||||
"sessionId": "session-capability-live-er-blocked-dedup-01",
|
||||
"turnId": "019ff1e7-b8d6-7eb2-9d4d-e98b659fdfc0",
|
||||
"itemId": "transcript-000023",
|
||||
"receiptIds": [],
|
||||
"terminalPresent": true
|
||||
},
|
||||
"authorization": {
|
||||
"expected": "absent",
|
||||
"observed": "absent"
|
||||
},
|
||||
"efficiency": {
|
||||
"latencyMs": 2033,
|
||||
"attempts": 1
|
||||
},
|
||||
"budget": {
|
||||
"maxLatencyMs": 60000,
|
||||
"maxAttempts": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"caseId": "hb-context-01",
|
||||
"group": "hb",
|
||||
"semanticOperation": "checkout_task",
|
||||
"expectedCalls": [],
|
||||
"forbiddenCalls": [],
|
||||
"observedCalls": [],
|
||||
"finalState": {
|
||||
"expected": "unchanged",
|
||||
"observed": "unchanged"
|
||||
},
|
||||
"providerModel": {
|
||||
"id": "gpt-5.5",
|
||||
"provider": "openai"
|
||||
},
|
||||
"scoringEvidence": {
|
||||
"trace": {
|
||||
"runId": "run-capability-live-hb-context-01",
|
||||
"sessionId": "session-capability-live-hb-context-01",
|
||||
"turnId": "019ff1e7-c239-7462-8d23-dda298d0fa58",
|
||||
"itemId": "transcript-000023",
|
||||
"receiptIds": [],
|
||||
"terminalPresent": true
|
||||
},
|
||||
"authorization": {
|
||||
"expected": "absent",
|
||||
"observed": "absent"
|
||||
},
|
||||
"efficiency": {
|
||||
"latencyMs": 1658,
|
||||
"attempts": 1
|
||||
},
|
||||
"budget": {
|
||||
"maxLatencyMs": 60000,
|
||||
"maxAttempts": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"caseId": "ix-checkbox-01",
|
||||
"group": "ix",
|
||||
"semanticOperation": "request_human_input",
|
||||
"expectedCalls": [
|
||||
"request_human_input"
|
||||
],
|
||||
"forbiddenCalls": [],
|
||||
"observedCalls": [
|
||||
"request_human_input"
|
||||
],
|
||||
"finalState": {
|
||||
"expected": "mutated",
|
||||
"observed": "mutated"
|
||||
},
|
||||
"providerModel": {
|
||||
"id": "gpt-5.5",
|
||||
"provider": "openai"
|
||||
},
|
||||
"scoringEvidence": {
|
||||
"trace": {
|
||||
"runId": "run-capability-live-ix-checkbox-01",
|
||||
"sessionId": "session-capability-live-ix-checkbox-01",
|
||||
"turnId": "019ff1e7-ca3a-7ad2-9d67-35ee17cb6042",
|
||||
"itemId": "evidence-000021",
|
||||
"receiptIds": [
|
||||
"evidence-000023"
|
||||
],
|
||||
"terminalPresent": true
|
||||
},
|
||||
"authorization": {
|
||||
"expected": "allowed",
|
||||
"observed": "allowed"
|
||||
},
|
||||
"efficiency": {
|
||||
"latencyMs": 9631,
|
||||
"attempts": 1
|
||||
},
|
||||
"budget": {
|
||||
"maxLatencyMs": 60000,
|
||||
"maxAttempts": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"caseId": "mh-blocked-handoff-01",
|
||||
"group": "mh",
|
||||
"semanticOperation": "create_task",
|
||||
"expectedCalls": [
|
||||
"create_task"
|
||||
],
|
||||
"forbiddenCalls": [],
|
||||
"observedCalls": [
|
||||
"create_task"
|
||||
],
|
||||
"finalState": {
|
||||
"expected": "mutated",
|
||||
"observed": "mutated"
|
||||
},
|
||||
"providerModel": {
|
||||
"id": "gpt-5.5",
|
||||
"provider": "openai"
|
||||
},
|
||||
"scoringEvidence": {
|
||||
"trace": {
|
||||
"runId": "run-capability-live-mh-blocked-handoff-01",
|
||||
"sessionId": "session-capability-live-mh-blocked-handoff-01",
|
||||
"turnId": "019ff1e7-f14e-7962-91e0-60f4b2c1ee5e",
|
||||
"itemId": "evidence-000022",
|
||||
"receiptIds": [
|
||||
"evidence-000023"
|
||||
],
|
||||
"terminalPresent": true
|
||||
},
|
||||
"authorization": {
|
||||
"expected": "allowed",
|
||||
"observed": "allowed"
|
||||
},
|
||||
"efficiency": {
|
||||
"latencyMs": 3474,
|
||||
"attempts": 1
|
||||
},
|
||||
"budget": {
|
||||
"maxLatencyMs": 60000,
|
||||
"maxAttempts": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"caseId": "rf-api-404-report-01",
|
||||
"group": "rf",
|
||||
"semanticOperation": "list_agents",
|
||||
"expectedCalls": [
|
||||
"list_agents"
|
||||
],
|
||||
"forbiddenCalls": [],
|
||||
"observedCalls": [
|
||||
"list_agents"
|
||||
],
|
||||
"finalState": {
|
||||
"expected": "unchanged",
|
||||
"observed": "unchanged"
|
||||
},
|
||||
"providerModel": {
|
||||
"id": "gpt-5.5",
|
||||
"provider": "openai"
|
||||
},
|
||||
"scoringEvidence": {
|
||||
"trace": {
|
||||
"runId": "run-capability-live-rf-api-404-report-01",
|
||||
"sessionId": "session-capability-live-rf-api-404-report-01",
|
||||
"turnId": "019ff1e8-0054-7102-8629-5a629712bc29",
|
||||
"itemId": "evidence-000023",
|
||||
"receiptIds": [
|
||||
"evidence-000025"
|
||||
],
|
||||
"terminalPresent": true
|
||||
},
|
||||
"authorization": {
|
||||
"expected": "allowed",
|
||||
"observed": "allowed"
|
||||
},
|
||||
"efficiency": {
|
||||
"latencyMs": 5691,
|
||||
"attempts": 1
|
||||
},
|
||||
"budget": {
|
||||
"maxLatencyMs": 60000,
|
||||
"maxAttempts": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"caseId": "rs-dependency-blocked-wake-01",
|
||||
"group": "rs",
|
||||
"semanticOperation": "checkout_task",
|
||||
"expectedCalls": [],
|
||||
"forbiddenCalls": [],
|
||||
"observedCalls": [],
|
||||
"finalState": {
|
||||
"expected": "unchanged",
|
||||
"observed": "unchanged"
|
||||
},
|
||||
"providerModel": {
|
||||
"id": "gpt-5.5",
|
||||
"provider": "openai"
|
||||
},
|
||||
"scoringEvidence": {
|
||||
"trace": {
|
||||
"runId": "run-capability-live-rs-dependency-blocked-wake-01",
|
||||
"sessionId": "session-capability-live-rs-dependency-blocked-wake-01",
|
||||
"turnId": "019ff1e8-17fc-7600-aa46-84429760dcdd",
|
||||
"itemId": "transcript-000023",
|
||||
"receiptIds": [],
|
||||
"terminalPresent": true
|
||||
},
|
||||
"authorization": {
|
||||
"expected": "absent",
|
||||
"observed": "absent"
|
||||
},
|
||||
"efficiency": {
|
||||
"latencyMs": 1908,
|
||||
"attempts": 1
|
||||
},
|
||||
"budget": {
|
||||
"maxLatencyMs": 60000,
|
||||
"maxAttempts": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"caseId": "se-get-issue-01",
|
||||
"group": "se",
|
||||
"semanticOperation": "search_tasks",
|
||||
"expectedCalls": [
|
||||
"search_tasks"
|
||||
],
|
||||
"forbiddenCalls": [],
|
||||
"observedCalls": [
|
||||
"search_tasks"
|
||||
],
|
||||
"finalState": {
|
||||
"expected": "unchanged",
|
||||
"observed": "unchanged"
|
||||
},
|
||||
"providerModel": {
|
||||
"id": "gpt-5.5",
|
||||
"provider": "openai"
|
||||
},
|
||||
"scoringEvidence": {
|
||||
"trace": {
|
||||
"runId": "run-capability-live-se-get-issue-01",
|
||||
"sessionId": "session-capability-live-se-get-issue-01",
|
||||
"turnId": "019ff1e8-20ea-73e2-9cc6-3e2d8f6fc948",
|
||||
"itemId": "evidence-000024",
|
||||
"receiptIds": [
|
||||
"evidence-000025"
|
||||
],
|
||||
"terminalPresent": true
|
||||
},
|
||||
"authorization": {
|
||||
"expected": "allowed",
|
||||
"observed": "denied"
|
||||
},
|
||||
"efficiency": {
|
||||
"latencyMs": 4504,
|
||||
"attempts": 1
|
||||
},
|
||||
"budget": {
|
||||
"maxLatencyMs": 60000,
|
||||
"maxAttempts": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"caseId": "st-backlog-park-01",
|
||||
"group": "st",
|
||||
"semanticOperation": "finish_task",
|
||||
"expectedCalls": [
|
||||
"finish_task"
|
||||
],
|
||||
"forbiddenCalls": [],
|
||||
"observedCalls": [
|
||||
"finish_task"
|
||||
],
|
||||
"finalState": {
|
||||
"expected": "mutated",
|
||||
"observed": "mutated"
|
||||
},
|
||||
"providerModel": {
|
||||
"id": "gpt-5.5",
|
||||
"provider": "openai"
|
||||
},
|
||||
"scoringEvidence": {
|
||||
"trace": {
|
||||
"runId": "run-capability-live-st-backlog-park-01",
|
||||
"sessionId": "session-capability-live-st-backlog-park-01",
|
||||
"turnId": "019ff1e8-33f6-7f93-88b9-83ad36eac2bb",
|
||||
"itemId": "evidence-000022",
|
||||
"receiptIds": [
|
||||
"evidence-000023"
|
||||
],
|
||||
"terminalPresent": true
|
||||
},
|
||||
"authorization": {
|
||||
"expected": "allowed",
|
||||
"observed": "allowed"
|
||||
},
|
||||
"efficiency": {
|
||||
"latencyMs": 4513,
|
||||
"attempts": 1
|
||||
},
|
||||
"budget": {
|
||||
"maxLatencyMs": 60000,
|
||||
"maxAttempts": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"caseId": "su-crossteam-billing-01",
|
||||
"group": "su",
|
||||
"semanticOperation": "create_task",
|
||||
"expectedCalls": [
|
||||
"create_task"
|
||||
],
|
||||
"forbiddenCalls": [],
|
||||
"observedCalls": [
|
||||
"create_task"
|
||||
],
|
||||
"finalState": {
|
||||
"expected": "mutated",
|
||||
"observed": "mutated"
|
||||
},
|
||||
"providerModel": {
|
||||
"id": "gpt-5.5",
|
||||
"provider": "openai"
|
||||
},
|
||||
"scoringEvidence": {
|
||||
"trace": {
|
||||
"runId": "run-capability-live-su-crossteam-billing-01",
|
||||
"sessionId": "session-capability-live-su-crossteam-billing-01",
|
||||
"turnId": "019ff1e8-470b-7550-a6b6-8c0c842eb948",
|
||||
"itemId": "evidence-000022",
|
||||
"receiptIds": [
|
||||
"evidence-000023"
|
||||
],
|
||||
"terminalPresent": true
|
||||
},
|
||||
"authorization": {
|
||||
"expected": "allowed",
|
||||
"observed": "allowed"
|
||||
},
|
||||
"efficiency": {
|
||||
"latencyMs": 6353,
|
||||
"attempts": 1
|
||||
},
|
||||
"budget": {
|
||||
"maxLatencyMs": 60000,
|
||||
"maxAttempts": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"caseId": "wk-ask-mode-01",
|
||||
"group": "wk",
|
||||
"semanticOperation": "checkout_task",
|
||||
"expectedCalls": [],
|
||||
"forbiddenCalls": [],
|
||||
"observedCalls": [],
|
||||
"finalState": {
|
||||
"expected": "unchanged",
|
||||
"observed": "unchanged"
|
||||
},
|
||||
"providerModel": {
|
||||
"id": "gpt-5.5",
|
||||
"provider": "openai"
|
||||
},
|
||||
"scoringEvidence": {
|
||||
"trace": {
|
||||
"runId": "run-capability-live-wk-ask-mode-01",
|
||||
"sessionId": "session-capability-live-wk-ask-mode-01",
|
||||
"turnId": "019ff1e8-617d-7ba3-8c12-84e6c5d011d0",
|
||||
"itemId": "transcript-000023",
|
||||
"receiptIds": [],
|
||||
"terminalPresent": true
|
||||
},
|
||||
"authorization": {
|
||||
"expected": "absent",
|
||||
"observed": "absent"
|
||||
},
|
||||
"efficiency": {
|
||||
"latencyMs": 1731,
|
||||
"attempts": 1
|
||||
},
|
||||
"budget": {
|
||||
"maxLatencyMs": 60000,
|
||||
"maxAttempts": 1
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
---
|
||||
type: Evidence Report
|
||||
title: Capability Live Codex Matrix
|
||||
description: Real Codex semantic-tool choices and resulting mock control-plane state.
|
||||
generated: { by: process:capability-live-eval, at: 2026-08-11T17:38:30.000Z }
|
||||
status: stable
|
||||
---
|
||||
|
||||
# Capability Live Codex Matrix
|
||||
|
||||
- Provider model: openai/gpt-5.5
|
||||
- Groups: 16
|
||||
- Cases: ap:ap-approval-deny-01, ar:ar-no-done-without-upload-01, bl:bl-cancelled-not-resolved-01, cm:cm-mention-structured-01, co:co-409-stop-01, dp:dp-base-revision-01, er:er-blocked-dedup-01, hb:hb-context-01, ix:ix-checkbox-01, mh:mh-blocked-handoff-01, rf:rf-api-404-report-01, rs:rs-dependency-blocked-wake-01, se:se-get-issue-01, st:st-backlog-park-01, su:su-crossteam-billing-01, wk:wk-ask-mode-01
|
||||
- Each row asserts expected and observed typed calls plus final mock state.
|
||||
- Scored report: `live-eval-slice-report.md`
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,42 @@
|
||||
---
|
||||
type: Evidence Report
|
||||
title: Live Runner Eval Slice Report
|
||||
description: Scored real-Codex capability matrix bound to its secret-free eval bundle.
|
||||
generated: { by: process:capability-live-eval, at: 2026-08-11T17:38:30.000Z }
|
||||
status: stable
|
||||
---
|
||||
|
||||
# Runner eval slice — evb-ccfa51b713b10e13
|
||||
|
||||
- Bundle: `runnerd/codex-app-server@codex-app-server-v2 · gpt-5.5 · mock · 4 grants · prompt:capability-live-exact-call-v1 · faults:clean`
|
||||
- Source: real model tool choice
|
||||
- Cases: 16 · passed 16 · gate failures 0 · mean overall 1
|
||||
|
||||
## Dimension means
|
||||
|
||||
- hard_invariants: 1
|
||||
- semantic_outcome: 1
|
||||
- trajectory_restraint: 1
|
||||
- trace_completeness: 1
|
||||
- quality_efficiency: 1
|
||||
|
||||
## Cases
|
||||
|
||||
| case | source | counterpart | fault | gate | overall | outcome | trajectory | trace | efficiency |
|
||||
| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
|
||||
| ap-approval-deny-01 | live_model | — | — | ok | 1 | 1 | 1 | 1 | 1 |
|
||||
| ar-no-done-without-upload-01 | live_model | — | — | ok | 1 | 1 | 1 | 1 | 1 |
|
||||
| bl-cancelled-not-resolved-01 | live_model | — | — | ok | 1 | 1 | 1 | 1 | 1 |
|
||||
| cm-mention-structured-01 | live_model | — | — | ok | 1 | 1 | 1 | 1 | 1 |
|
||||
| co-409-stop-01 | live_model | — | — | ok | 1 | 1 | 1 | 1 | 1 |
|
||||
| dp-base-revision-01 | live_model | — | — | ok | 1 | 1 | 1 | 1 | 1 |
|
||||
| er-blocked-dedup-01 | live_model | — | — | ok | 1 | 1 | 1 | 1 | 1 |
|
||||
| hb-context-01 | live_model | — | — | ok | 1 | 1 | 1 | 1 | 1 |
|
||||
| ix-checkbox-01 | live_model | — | — | ok | 1 | 1 | 1 | 1 | 1 |
|
||||
| mh-blocked-handoff-01 | live_model | — | — | ok | 1 | 1 | 1 | 1 | 1 |
|
||||
| rf-api-404-report-01 | live_model | — | — | ok | 1 | 1 | 1 | 1 | 1 |
|
||||
| rs-dependency-blocked-wake-01 | live_model | — | — | ok | 1 | 1 | 1 | 1 | 1 |
|
||||
| se-get-issue-01 | live_model | — | — | ok | 1 | 1 | 1 | 1 | 1 |
|
||||
| st-backlog-park-01 | live_model | — | — | ok | 1 | 1 | 1 | 1 | 1 |
|
||||
| su-crossteam-billing-01 | live_model | — | — | ok | 1 | 1 | 1 | 1 | 1 |
|
||||
| wk-ask-mode-01 | live_model | — | — | ok | 1 | 1 | 1 | 1 | 1 |
|
||||
@@ -8,6 +8,7 @@ records that remain useful to package verification and debugging.
|
||||
- [Durable recovery per-fault traces](durable-recovery-faults/)
|
||||
- [Codex normalized trace](codex-trace.json)
|
||||
- [Live-console real Codex server trace](live-console-real-codex-server.json)
|
||||
- [Capability live eval evidence](capability/)
|
||||
|
||||
Screenshots, dated milestone reports, and duplicate handoff records live in Git
|
||||
history rather than the current package.
|
||||
@@ -1,27 +1,157 @@
|
||||
import { mkdir, writeFile } from "node:fs/promises";
|
||||
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
||||
import { dirname, resolve } from "node:path";
|
||||
|
||||
const packageRoot = resolve(import.meta.dirname, "..");
|
||||
const suite = await import(resolve(packageRoot, "dist/conformance/capability-eval-suite.js"));
|
||||
const evals = await import(resolve(packageRoot, "dist/eval/index.js"));
|
||||
const packageManifest = JSON.parse(await readFile(resolve(packageRoot, "package.json"), "utf8"));
|
||||
const generatedAt = new Date().toISOString();
|
||||
const evidenceMarkdown = (title, description, body) => [
|
||||
"---",
|
||||
"type: Evidence Report",
|
||||
`title: ${title}`,
|
||||
`description: ${description}`,
|
||||
`generated: { by: process:capability-live-eval, at: ${generatedAt} }`,
|
||||
"status: stable",
|
||||
"---",
|
||||
"",
|
||||
body.trimEnd(),
|
||||
"",
|
||||
].join("\n");
|
||||
const matrix = await suite.runCapabilityLiveCodexMatrix(process.cwd());
|
||||
const providerModels = [...new Set(matrix.map(
|
||||
(entry) => `${entry.providerModel.provider}/${entry.providerModel.id}`,
|
||||
))];
|
||||
if (providerModels.length !== 1) {
|
||||
throw new Error(`live eval matrix used inconsistent provider models: ${providerModels.join(", ")}`);
|
||||
}
|
||||
const liveModel = matrix[0].providerModel;
|
||||
const liveBundle = {
|
||||
schema: evals.EVAL_BUNDLE_SCHEMA,
|
||||
provider: {
|
||||
runtime: "runnerd",
|
||||
transport: "codex-app-server",
|
||||
protocolVersion: "codex-app-server-v2",
|
||||
},
|
||||
model: { id: liveModel.id },
|
||||
launchContext: {
|
||||
workingDirectoryClass: "workspace-checkout",
|
||||
scenarioId: "capability-live-codex-matrix-v1",
|
||||
turnTimeoutMs: 60_000,
|
||||
},
|
||||
promptPolicy: {
|
||||
id: "capability-live-exact-call-v1",
|
||||
callTemplate: "Call the named semantic operation exactly once with the declared JSON input.",
|
||||
restraintTemplate: "Do not call any tool for a control-plane-owned behavior.",
|
||||
},
|
||||
grants: [...suite.CAPABILITY_LIVE_EVAL_GRANTS],
|
||||
runner: {
|
||||
package: packageManifest.name,
|
||||
binary: "paperclip-runnerd",
|
||||
version: packageManifest.version,
|
||||
},
|
||||
controlPlaneAdapter: {
|
||||
kind: "mock",
|
||||
contract: "paperclip.capability.mock-state.v1",
|
||||
},
|
||||
faultInjection: [],
|
||||
};
|
||||
const liveObservations = matrix.map((result) => evals.observationFromLiveMatrix(result, {
|
||||
...result.scoringEvidence,
|
||||
provenance: { source: "live_model" },
|
||||
}));
|
||||
const liveSliceReport = evals.buildEvalSliceReport(liveBundle, liveObservations, {
|
||||
generatedFromLiveModel: true,
|
||||
});
|
||||
if (liveSliceReport.aggregate.passed !== liveSliceReport.aggregate.caseCount) {
|
||||
throw new Error(`live eval slice scored ${liveSliceReport.aggregate.passed}/${liveSliceReport.aggregate.caseCount} passing cases`);
|
||||
}
|
||||
|
||||
const faultBundle = {
|
||||
schema: evals.EVAL_BUNDLE_SCHEMA,
|
||||
provider: {
|
||||
runtime: "deterministic-harness",
|
||||
transport: "in-process",
|
||||
protocolVersion: "v1",
|
||||
},
|
||||
model: { id: "scripted-eval-agent" },
|
||||
launchContext: {
|
||||
workingDirectoryClass: "ephemeral-fixture",
|
||||
scenarioId: "eval-behavior-fault-matrix-v1",
|
||||
turnTimeoutMs: 1_000,
|
||||
},
|
||||
promptPolicy: {
|
||||
id: "deterministic-behavior-driver-v1",
|
||||
callTemplate: "Invoke the declared semantic operation sequence.",
|
||||
restraintTemplate: "Make no semantic call.",
|
||||
},
|
||||
grants: ["control_plane:wakes", "governance:approvals:decide"],
|
||||
runner: {
|
||||
package: packageManifest.name,
|
||||
binary: "node",
|
||||
version: packageManifest.version,
|
||||
},
|
||||
controlPlaneAdapter: {
|
||||
kind: "mock",
|
||||
contract: "paperclip.capability.mock-state.v1",
|
||||
},
|
||||
faultInjection: [
|
||||
{ id: "fault-authz", class: "authorization", description: "deny approval decision exposure" },
|
||||
{ id: "fault-conflict", class: "conflict", description: "submit a stale plan revision" },
|
||||
{ id: "fault-retry", class: "retry", description: "fail the first artifact registration attempt" },
|
||||
{ id: "fault-provider", class: "provider_capability", description: "omit wake scheduling from the tool surface" },
|
||||
],
|
||||
};
|
||||
const behaviorMatrix = await evals.runEvalBehaviorFaultMatrix(faultBundle);
|
||||
const faultSliceReport = evals.buildEvalSliceReport(
|
||||
faultBundle,
|
||||
behaviorMatrix.map((result) => result.observation),
|
||||
);
|
||||
const report = {
|
||||
schema: "paperclip.capability.live-codex-matrix.v1",
|
||||
groups: matrix.length,
|
||||
providerModel: liveModel,
|
||||
matrix,
|
||||
};
|
||||
const output = resolve(packageRoot, "knowledge/evidence/capability/live-codex-matrix.json");
|
||||
const markdown = output.replace(/\.json$/, ".md");
|
||||
const evidenceDirectory = resolve(packageRoot, "knowledge/evidence/capability");
|
||||
const output = resolve(evidenceDirectory, "live-codex-matrix.json");
|
||||
const markdown = resolve(evidenceDirectory, "live-codex-matrix.md");
|
||||
const liveSliceJson = resolve(evidenceDirectory, "live-eval-slice-report.json");
|
||||
const liveSliceMarkdown = resolve(evidenceDirectory, "live-eval-slice-report.md");
|
||||
const faultSliceJson = resolve(evidenceDirectory, "fault-eval-slice-report.json");
|
||||
const faultSliceMarkdown = resolve(evidenceDirectory, "fault-eval-slice-report.md");
|
||||
await mkdir(dirname(output), { recursive: true });
|
||||
await Promise.all([
|
||||
writeFile(output, `${JSON.stringify(report, null, 2)}\n`),
|
||||
writeFile(markdown, [
|
||||
writeFile(markdown, evidenceMarkdown(
|
||||
"Capability Live Codex Matrix",
|
||||
"Real Codex semantic-tool choices and resulting mock control-plane state.",
|
||||
[
|
||||
"# Capability Live Codex Matrix",
|
||||
"",
|
||||
`- Provider model: ${liveModel.provider}/${liveModel.id}`,
|
||||
`- Groups: ${report.groups}`,
|
||||
`- Cases: ${matrix.map((entry) => `${entry.group}:${entry.caseId}`).join(", ")}`,
|
||||
"- Each row asserts expected and observed typed calls plus final mock state.",
|
||||
"- Scored report: `live-eval-slice-report.md`",
|
||||
"",
|
||||
].join("\n")),
|
||||
].join("\n"),
|
||||
)),
|
||||
writeFile(liveSliceJson, `${JSON.stringify(liveSliceReport, null, 2)}\n`),
|
||||
writeFile(liveSliceMarkdown, evidenceMarkdown(
|
||||
"Live Runner Eval Slice Report",
|
||||
"Scored real-Codex capability matrix bound to its secret-free eval bundle.",
|
||||
evals.renderEvalSliceMarkdown(liveSliceReport),
|
||||
)),
|
||||
writeFile(faultSliceJson, `${JSON.stringify(faultSliceReport, null, 2)}\n`),
|
||||
writeFile(faultSliceMarkdown, evidenceMarkdown(
|
||||
"Runner Eval Fault Slice Report",
|
||||
"Eight green/red behaviors with deterministic injected-fault receipts.",
|
||||
evals.renderEvalSliceMarkdown(faultSliceReport),
|
||||
)),
|
||||
]);
|
||||
process.stdout.write(`Capability live Codex matrix passed: ${matrix.length} groups.\n`);
|
||||
process.stdout.write(`Live eval slice passed: ${liveSliceReport.aggregate.passed}/${liveSliceReport.aggregate.caseCount}.\n`);
|
||||
process.stdout.write(`Behavior fault matrix passed: ${faultSliceReport.aggregate.passed} green, ${faultSliceReport.aggregate.caseCount - faultSliceReport.aggregate.passed} scored red.\n`);
|
||||
process.stdout.write(`Live matrix reports: ${output} and ${markdown}\n`);
|
||||
process.stdout.write(`Scored eval reports: ${liveSliceJson}, ${liveSliceMarkdown}, ${faultSliceJson}, and ${faultSliceMarkdown}\n`);
|
||||
@@ -58,8 +58,34 @@ export interface CapabilityLiveCodexMatrixResult {
|
||||
forbiddenCalls: string[];
|
||||
observedCalls: string[];
|
||||
finalState: { expected: "unchanged" | "mutated"; observed: "unchanged" | "mutated" };
|
||||
providerModel: { id: string; provider: string };
|
||||
/** Browser-safe evidence retained from the live-session snapshot for scoring. */
|
||||
scoringEvidence: {
|
||||
trace: {
|
||||
runId: string;
|
||||
sessionId: string;
|
||||
turnId: string;
|
||||
itemId: string;
|
||||
receiptIds: string[];
|
||||
terminalPresent: boolean;
|
||||
};
|
||||
authorization: {
|
||||
expected: "allowed" | "denied" | "absent";
|
||||
observed: "allowed" | "denied" | "absent";
|
||||
};
|
||||
efficiency: { latencyMs: number; attempts: number };
|
||||
budget: { maxLatencyMs: number; maxAttempts: number };
|
||||
};
|
||||
}
|
||||
|
||||
/** One fixed candidate grant set shared by every live-matrix case. */
|
||||
export const CAPABILITY_LIVE_EVAL_GRANTS = Object.freeze([
|
||||
"delegation:tasks:create",
|
||||
"discovery:agents:read",
|
||||
"discovery:tasks:read",
|
||||
"governance:approvals:request",
|
||||
]);
|
||||
|
||||
export interface CapabilityEvalParityReport {
|
||||
schema: "paperclip.capability.eval-parity-report.v1";
|
||||
cases: number;
|
||||
@@ -284,9 +310,11 @@ export async function runCapabilityLiveCodexMatrix(
|
||||
const service = new CapabilityLiveSessionService();
|
||||
const session = await service.create({
|
||||
workingDirectory,
|
||||
seed: liveFixtureSeed(plan.grants),
|
||||
capabilities: capabilityFixtureRunCapabilities(plan.grants),
|
||||
explicitClaims: plan.grants,
|
||||
seed: liveFixtureSeed(CAPABILITY_LIVE_EVAL_GRANTS),
|
||||
capabilities: capabilityFixtureRunCapabilities(CAPABILITY_LIVE_EVAL_GRANTS),
|
||||
explicitClaims: [...CAPABILITY_LIVE_EVAL_GRANTS],
|
||||
runId: `run-capability-live-${row.id}`,
|
||||
sessionId: `session-capability-live-${row.id}`,
|
||||
turnTimeoutMs: 60_000,
|
||||
scenario: { id: `capability-live-eval-${row.id}` },
|
||||
});
|
||||
@@ -300,16 +328,23 @@ export async function runCapabilityLiveCodexMatrix(
|
||||
const prompt = expectedCalls.length === 0
|
||||
? "Do not call any tools. Reply with exactly: control-plane-owned."
|
||||
: `Call ${plan.operationId} exactly once with this JSON input: ${JSON.stringify(liveInput)}. Do not call any other tool. Reply after the typed result.`;
|
||||
await session.sendMessage(prompt);
|
||||
const snapshot = session.snapshot();
|
||||
const turnStartedAt = Date.now();
|
||||
const turn = await session.sendMessage(prompt);
|
||||
const latencyMs = Date.now() - turnStartedAt;
|
||||
const snapshot = turn.snapshot;
|
||||
const observedCalls = snapshot.evidence
|
||||
.filter((entry) => entry.kind === "tool_call")
|
||||
.map((entry) => String(entry.data.operationId));
|
||||
const expectedState = operationMutatesState(plan.operationId) ? "mutated" : "unchanged";
|
||||
const observedState = snapshot.mockState === initialMockState ? "unchanged" : "mutated";
|
||||
if (JSON.stringify(observedCalls) !== JSON.stringify(expectedCalls) || observedState !== expectedState) {
|
||||
if (
|
||||
turn.status !== "completed" ||
|
||||
JSON.stringify(observedCalls) !== JSON.stringify(expectedCalls) ||
|
||||
observedState !== expectedState
|
||||
) {
|
||||
throw new Error(JSON.stringify({
|
||||
caseId: row.id,
|
||||
turnStatus: turn.status,
|
||||
expectedCalls,
|
||||
observedCalls,
|
||||
expectedState,
|
||||
@@ -317,6 +352,23 @@ export async function runCapabilityLiveCodexMatrix(
|
||||
toolResults: snapshot.evidence.filter((entry) => entry.kind === "tool_result").map((entry) => entry.data.result),
|
||||
}));
|
||||
}
|
||||
if (snapshot.providerModel === undefined) {
|
||||
throw new Error(`live Codex case ${row.id} did not report its provider model identity`);
|
||||
}
|
||||
const invocationRecords = snapshot.authorizationRecords.filter(
|
||||
(record) => record.phase === "invocation" && record.callId !== null,
|
||||
);
|
||||
const observedAuthorization = observedCalls.length === 0
|
||||
? "absent"
|
||||
: invocationRecords.every((record) => record.allowed)
|
||||
? "allowed"
|
||||
: "denied";
|
||||
const toolReceipts = snapshot.evidence.filter((entry) => entry.kind === "tool_result");
|
||||
const itemId = snapshot.evidence.find((entry) => entry.kind === "tool_call")?.id
|
||||
?? snapshot.transcript.find(
|
||||
(entry) => entry.role === "assistant" && entry.turnId === turn.turnId,
|
||||
)?.id
|
||||
?? `turn:${turn.turnId}:terminal`;
|
||||
matrix.push({
|
||||
caseId: row.id,
|
||||
group,
|
||||
@@ -325,6 +377,23 @@ export async function runCapabilityLiveCodexMatrix(
|
||||
forbiddenCalls: row.forbiddenSemantics,
|
||||
observedCalls,
|
||||
finalState: { expected: expectedState, observed: observedState },
|
||||
providerModel: snapshot.providerModel,
|
||||
scoringEvidence: {
|
||||
trace: {
|
||||
runId: snapshot.authority.runId,
|
||||
sessionId: snapshot.sessionId,
|
||||
turnId: turn.turnId,
|
||||
itemId,
|
||||
receiptIds: toolReceipts.map((entry) => entry.id),
|
||||
terminalPresent: true,
|
||||
},
|
||||
authorization: {
|
||||
expected: expectedCalls.length === 0 ? "absent" : "allowed",
|
||||
observed: observedAuthorization,
|
||||
},
|
||||
efficiency: { latencyMs, attempts: 1 },
|
||||
budget: { maxLatencyMs: 60_000, maxAttempts: 1 },
|
||||
},
|
||||
});
|
||||
} finally {
|
||||
await service.shutdown(session.id, `Capability live eval ${row.id} complete`);
|
||||
@@ -333,7 +402,7 @@ export async function runCapabilityLiveCodexMatrix(
|
||||
return matrix;
|
||||
}
|
||||
|
||||
function liveFixtureSeed(actorGrants: string[]): CapabilityFixtureSeed {
|
||||
function liveFixtureSeed(actorGrants: readonly string[]): CapabilityFixtureSeed {
|
||||
return {
|
||||
actors: [{
|
||||
id: "actor-1",
|
||||
@@ -342,7 +411,7 @@ function liveFixtureSeed(actorGrants: string[]): CapabilityFixtureSeed {
|
||||
role: "engineer",
|
||||
status: "active",
|
||||
budgetId: "budget-actor-1",
|
||||
capabilityGrants: actorGrants,
|
||||
capabilityGrants: [...actorGrants],
|
||||
}],
|
||||
};
|
||||
}
|
||||
|
||||
@@ -11,7 +11,8 @@ import { createHash } from "node:crypto";
|
||||
*
|
||||
* The bundle is a *declaration*, not a live secret store: it must never carry a
|
||||
* credential, hidden company identifier, or raw secret payload. Grants are typed
|
||||
* grant strings (`domain:resource:action`), not secret material. Callers should
|
||||
* canonical claim strings (`domain:action` or `domain:resource:action`), not
|
||||
* secret material. Callers should
|
||||
* run {@link assertBundleSecretFree} before persisting a bundle as evidence.
|
||||
*/
|
||||
export const EVAL_BUNDLE_SCHEMA = "paperclip.runner.eval-bundle.v1" as const;
|
||||
@@ -82,7 +83,7 @@ export interface EvalBundle {
|
||||
model: EvalBundleModel;
|
||||
launchContext: EvalBundleLaunchContext;
|
||||
promptPolicy: EvalBundlePromptPolicy;
|
||||
/** Typed grant strings (`domain:resource:action`) unlocked for the candidate. */
|
||||
/** Typed canonical claim strings unlocked for the candidate. */
|
||||
grants: string[];
|
||||
runner: EvalBundleRunner;
|
||||
controlPlaneAdapter: EvalBundleControlPlaneAdapter;
|
||||
@@ -141,9 +142,9 @@ const SECRET_VALUE_PATTERNS: Array<{ id: string; pattern: RegExp }> = [
|
||||
*/
|
||||
export function assertBundleSecretFree(bundle: EvalBundle): void {
|
||||
for (const grant of bundle.grants) {
|
||||
if (!/^[a-z0-9]+(:[a-z0-9_]+){2}$/.test(grant)) {
|
||||
if (!/^[a-z0-9_]+(?::[a-z0-9_]+){1,2}$/.test(grant)) {
|
||||
throw new EvalBundleSecretError(
|
||||
`grant "${grant}" is not a typed domain:resource:action grant`,
|
||||
`grant "${grant}" is not a typed canonical capability claim`,
|
||||
"grants",
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { EVAL_BUNDLE_SCHEMA, type EvalBundle } from "./eval-bundle.js";
|
||||
import {
|
||||
EVAL_BEHAVIOR_IDS,
|
||||
runEvalBehaviorFaultMatrix,
|
||||
} from "./eval-execution.js";
|
||||
import { buildEvalSliceReport } from "./eval-slice.js";
|
||||
|
||||
function bundle(): EvalBundle {
|
||||
return {
|
||||
schema: EVAL_BUNDLE_SCHEMA,
|
||||
provider: {
|
||||
runtime: "deterministic-harness",
|
||||
transport: "in-process",
|
||||
protocolVersion: "v1",
|
||||
},
|
||||
model: { id: "scripted-eval-agent" },
|
||||
launchContext: {
|
||||
workingDirectoryClass: "ephemeral-fixture",
|
||||
scenarioId: "eval-behavior-fault-matrix-v1",
|
||||
turnTimeoutMs: 1_000,
|
||||
},
|
||||
promptPolicy: {
|
||||
id: "deterministic-behavior-driver-v1",
|
||||
callTemplate: "Invoke the declared semantic operation sequence.",
|
||||
restraintTemplate: "Make no semantic call.",
|
||||
},
|
||||
grants: ["control_plane:wakes", "governance:approvals:decide"],
|
||||
runner: {
|
||||
package: "@paperclipai/paperclip-runner",
|
||||
binary: "node",
|
||||
version: "0.1.2",
|
||||
},
|
||||
controlPlaneAdapter: {
|
||||
kind: "mock",
|
||||
contract: "paperclip.capability.mock-state.v1",
|
||||
},
|
||||
faultInjection: [
|
||||
{ id: "fault-authz", class: "authorization", description: "deny approval decision exposure" },
|
||||
{ id: "fault-conflict", class: "conflict", description: "submit a stale plan revision" },
|
||||
{ id: "fault-retry", class: "retry", description: "fail the first artifact registration attempt" },
|
||||
{ id: "fault-provider", class: "provider_capability", description: "omit wake scheduling from the tool surface" },
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
describe("runEvalBehaviorFaultMatrix", () => {
|
||||
it("executes eight green/red behaviors and all four declared fault classes", async () => {
|
||||
const candidate = bundle();
|
||||
const results = await runEvalBehaviorFaultMatrix(candidate);
|
||||
const report = buildEvalSliceReport(
|
||||
candidate,
|
||||
results.map((result) => result.observation),
|
||||
);
|
||||
|
||||
expect(report.aggregate).toMatchObject({ caseCount: 16, passed: 8 });
|
||||
for (const behavior of EVAL_BEHAVIOR_IDS) {
|
||||
expect(report.cases.find((entry) => entry.observation.caseId === `${behavior}:green`)?.scorecard.overall.passed).toBe(true);
|
||||
expect(report.cases.find((entry) => entry.observation.caseId === `${behavior}:red`)?.scorecard.overall.passed).toBe(false);
|
||||
}
|
||||
|
||||
const faults = results.flatMap((result) => {
|
||||
const fault = result.observation.provenance?.faultInjection;
|
||||
return fault === undefined ? [] : [fault];
|
||||
});
|
||||
expect(faults.map((fault) => fault.class).sort()).toEqual([
|
||||
"authorization",
|
||||
"conflict",
|
||||
"provider_capability",
|
||||
"retry",
|
||||
]);
|
||||
expect(faults.every((fault) => fault.evidenceIds.length > 0)).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,644 @@
|
||||
import { capabilityFixtureRunCapabilities } from "../scenarios/fixture-run-capabilities.js";
|
||||
import {
|
||||
CapabilityMockControlPlaneAdapter,
|
||||
} from "../mock-core/capability-mock-control-plane-adapter.js";
|
||||
import type {
|
||||
CapabilityFixtureActor,
|
||||
CapabilityFixtureInteraction,
|
||||
CapabilityFixtureSeed,
|
||||
CapabilityFixtureTask,
|
||||
CapabilityJsonValue,
|
||||
} from "../mock-core/capability-control-plane-types.js";
|
||||
import { CapabilitySemanticDispatcher } from "../semantic-tools/dispatcher.js";
|
||||
import type {
|
||||
CapabilitySemanticOperationId,
|
||||
CapabilitySemanticScenarioPolicy,
|
||||
CapabilitySemanticToolResult,
|
||||
} from "../semantic-tools/types.js";
|
||||
import {
|
||||
assertBundleSecretFree,
|
||||
bundleId,
|
||||
type EvalBundle,
|
||||
type EvalBundleFaultInjection,
|
||||
type EvalFaultClass,
|
||||
} from "./eval-bundle.js";
|
||||
import { scoreEval, type AuthorizationState, type EvalObservation } from "./eval-scoring.js";
|
||||
|
||||
export const EVAL_BEHAVIOR_IDS = [
|
||||
"checkout_context",
|
||||
"revision_safe_plan_editing",
|
||||
"approval_denial",
|
||||
"interaction_continuation",
|
||||
"blocker_monitor",
|
||||
"artifact_registration",
|
||||
"restraint_no_call",
|
||||
"terminal_arbitration",
|
||||
] as const;
|
||||
|
||||
export type EvalBehaviorId = typeof EVAL_BEHAVIOR_IDS[number];
|
||||
export type EvalCounterpart = "green" | "red";
|
||||
|
||||
export interface EvalBehaviorExecutionResult {
|
||||
behavior: EvalBehaviorId;
|
||||
counterpart: EvalCounterpart;
|
||||
observation: EvalObservation;
|
||||
}
|
||||
|
||||
interface Invocation {
|
||||
operationId: string;
|
||||
input: CapabilityJsonValue;
|
||||
callId?: string;
|
||||
}
|
||||
|
||||
interface BehaviorCase {
|
||||
behavior: EvalBehaviorId;
|
||||
counterpart: EvalCounterpart;
|
||||
seed?: CapabilityFixtureSeed;
|
||||
scenario?: CapabilitySemanticScenarioPolicy;
|
||||
wake?: { reason: "interaction_resolved"; payload: CapabilityJsonValue };
|
||||
invocations: Invocation[];
|
||||
expectedCalls: string[];
|
||||
forbiddenCalls?: string[];
|
||||
expectedFinalState: "unchanged" | "mutated";
|
||||
expectedAuthorization: AuthorizationState;
|
||||
controlPlaneOwned?: boolean;
|
||||
maxAttempts?: number;
|
||||
faultClass?: EvalFaultClass;
|
||||
faultEvidenceOperation?: string;
|
||||
assertState?: (adapter: CapabilityMockControlPlaneAdapter) => void;
|
||||
}
|
||||
|
||||
const RUN_ID = "run-eval-behavior";
|
||||
const SESSION_ID = "session-eval-behavior";
|
||||
const ACTIVE_TASK_ID = "task-1";
|
||||
|
||||
function actor(grants: readonly string[]): CapabilityFixtureActor {
|
||||
return {
|
||||
id: "actor-1",
|
||||
companyId: "company-1",
|
||||
name: "Eval approver",
|
||||
role: "approver",
|
||||
status: "active",
|
||||
budgetId: "budget-actor-1",
|
||||
capabilityGrants: [...grants],
|
||||
};
|
||||
}
|
||||
|
||||
function task(overrides: Partial<CapabilityFixtureTask> = {}): CapabilityFixtureTask {
|
||||
return {
|
||||
id: ACTIVE_TASK_ID,
|
||||
companyId: "company-1",
|
||||
identifier: "MCK-1",
|
||||
title: "Eval behavior fixture",
|
||||
description: null,
|
||||
status: "todo",
|
||||
priority: "medium",
|
||||
workMode: "standard",
|
||||
parentId: null,
|
||||
assigneeActorId: "actor-1",
|
||||
checkoutRunId: null,
|
||||
executionRunId: null,
|
||||
startedAt: null,
|
||||
completedAt: null,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function seedFor(bundle: EvalBundle, extra: CapabilityFixtureSeed = {}): CapabilityFixtureSeed {
|
||||
return {
|
||||
...extra,
|
||||
actors: extra.actors ?? [actor(bundle.grants)],
|
||||
tasks: extra.tasks ?? [task()],
|
||||
};
|
||||
}
|
||||
|
||||
function faultFor(bundle: EvalBundle, faultClass: EvalFaultClass): EvalBundleFaultInjection {
|
||||
const matches = bundle.faultInjection.filter((fault) => fault.class === faultClass);
|
||||
if (matches.length !== 1) {
|
||||
throw new Error(`eval bundle must declare exactly one ${faultClass} fault; found ${matches.length}`);
|
||||
}
|
||||
return matches[0]!;
|
||||
}
|
||||
|
||||
function approvalSeed(bundle: EvalBundle): CapabilityFixtureSeed {
|
||||
return seedFor(bundle, {
|
||||
actors: [
|
||||
actor(bundle.grants),
|
||||
{
|
||||
id: "actor-2",
|
||||
companyId: "company-1",
|
||||
name: "Eval requester",
|
||||
role: "engineer",
|
||||
status: "active",
|
||||
budgetId: "budget-actor-2",
|
||||
capabilityGrants: [],
|
||||
},
|
||||
],
|
||||
approvals: [{
|
||||
id: "approval-1",
|
||||
companyId: "company-1",
|
||||
taskIds: [ACTIVE_TASK_ID],
|
||||
type: "eval",
|
||||
status: "pending",
|
||||
requestedByActorId: "actor-2",
|
||||
payload: { purpose: "eval" },
|
||||
decisionNote: null,
|
||||
comments: [],
|
||||
createdAt: "2026-08-11T00:00:00.000Z",
|
||||
decidedAt: null,
|
||||
}],
|
||||
});
|
||||
}
|
||||
|
||||
function interactionSeed(bundle: EvalBundle): CapabilityFixtureSeed {
|
||||
const interaction: CapabilityFixtureInteraction = {
|
||||
id: "interaction-1",
|
||||
taskId: ACTIVE_TASK_ID,
|
||||
kind: "checkbox",
|
||||
status: "accepted",
|
||||
title: "Select artifacts",
|
||||
prompt: "Select the artifacts to retain.",
|
||||
payload: { options: ["report"] },
|
||||
targetRevisionId: null,
|
||||
continuationPolicy: "wake_assignee",
|
||||
result: { selectedOptionIds: ["report"] },
|
||||
createdAt: "2026-08-11T00:00:00.000Z",
|
||||
resolvedAt: "2026-08-11T00:00:01.000Z",
|
||||
};
|
||||
return seedFor(bundle, { interactions: [interaction] });
|
||||
}
|
||||
|
||||
function planSeed(bundle: EvalBundle): CapabilityFixtureSeed {
|
||||
return seedFor(bundle, {
|
||||
documents: [{
|
||||
id: "document-plan",
|
||||
taskId: ACTIVE_TASK_ID,
|
||||
key: "plan",
|
||||
title: "Plan",
|
||||
format: "markdown",
|
||||
latestRevisionId: "revision-plan-1",
|
||||
revisions: [{
|
||||
id: "revision-plan-1",
|
||||
documentId: "document-plan",
|
||||
revision: 1,
|
||||
body: "# Plan\n\nInitial.",
|
||||
changeSummary: null,
|
||||
createdAt: "2026-08-11T00:00:00.000Z",
|
||||
}],
|
||||
}],
|
||||
});
|
||||
}
|
||||
|
||||
function blockerSeed(bundle: EvalBundle): CapabilityFixtureSeed {
|
||||
return seedFor(bundle, {
|
||||
tasks: [
|
||||
task(),
|
||||
task({
|
||||
id: "task-blocker",
|
||||
identifier: "MCK-2",
|
||||
title: "External prerequisite",
|
||||
assigneeActorId: null,
|
||||
}),
|
||||
],
|
||||
});
|
||||
}
|
||||
|
||||
function behaviorCases(bundle: EvalBundle): BehaviorCase[] {
|
||||
const conflict = faultFor(bundle, "conflict");
|
||||
const authorization = faultFor(bundle, "authorization");
|
||||
const retry = faultFor(bundle, "retry");
|
||||
const providerCapability = faultFor(bundle, "provider_capability");
|
||||
const planInput = (baseRevisionId: string): CapabilityJsonValue => ({
|
||||
idempotencyKey: `eval-plan-${baseRevisionId}`,
|
||||
key: "plan",
|
||||
title: "Plan",
|
||||
body: "# Plan\n\nRevised safely.",
|
||||
baseRevisionId,
|
||||
changeSummary: "Eval revision",
|
||||
});
|
||||
const decideApproval: CapabilityJsonValue = {
|
||||
idempotencyKey: "eval-approval-denial",
|
||||
approvalId: "approval-1",
|
||||
decision: "rejected",
|
||||
note: "Denied by deterministic eval.",
|
||||
};
|
||||
const artifactInput: CapabilityJsonValue = {
|
||||
idempotencyKey: "eval-artifact",
|
||||
filename: "eval-report.json",
|
||||
contentType: "application/json",
|
||||
byteSize: 2,
|
||||
sha256: "0".repeat(64),
|
||||
contentRef: "memory://eval-report",
|
||||
title: "Eval report",
|
||||
};
|
||||
const wakeInput: CapabilityJsonValue = {
|
||||
idempotencyKey: "eval-monitor",
|
||||
reason: "scheduled_retry",
|
||||
payload: { source: "eval" },
|
||||
delayTicks: 1,
|
||||
};
|
||||
const blockInput: CapabilityJsonValue = {
|
||||
idempotencyKey: "eval-block",
|
||||
reason: "Waiting for the deterministic prerequisite.",
|
||||
blockedByTaskIds: ["task-blocker"],
|
||||
};
|
||||
|
||||
return [
|
||||
{
|
||||
behavior: "checkout_context",
|
||||
counterpart: "green",
|
||||
invocations: [{ operationId: "get_task_context", input: {} }],
|
||||
expectedCalls: ["get_task_context"],
|
||||
expectedFinalState: "unchanged",
|
||||
expectedAuthorization: "allowed",
|
||||
assertState: (adapter) => {
|
||||
const context = adapter.context(RUN_ID);
|
||||
if (context.activeTask.status !== "in_progress" || context.activeTask.checkoutRunId !== RUN_ID) {
|
||||
throw new Error("checkout/context green case did not retain the atomic checkout context");
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
behavior: "checkout_context",
|
||||
counterpart: "red",
|
||||
invocations: [{ operationId: "checkout_task", input: {} }],
|
||||
expectedCalls: [],
|
||||
forbiddenCalls: ["checkout_task"],
|
||||
expectedFinalState: "unchanged",
|
||||
expectedAuthorization: "absent",
|
||||
controlPlaneOwned: true,
|
||||
},
|
||||
{
|
||||
behavior: "revision_safe_plan_editing",
|
||||
counterpart: "green",
|
||||
seed: planSeed(bundle),
|
||||
invocations: [{ operationId: "write_document", input: planInput("revision-plan-1") }],
|
||||
expectedCalls: ["write_document"],
|
||||
expectedFinalState: "mutated",
|
||||
expectedAuthorization: "allowed",
|
||||
assertState: (adapter) => {
|
||||
const document = adapter.snapshot().documents.find((candidate) => candidate.key === "plan");
|
||||
if (document?.revisions.length !== 2) throw new Error("plan green case did not append revision 2");
|
||||
},
|
||||
},
|
||||
{
|
||||
behavior: "revision_safe_plan_editing",
|
||||
counterpart: "red",
|
||||
seed: planSeed(bundle),
|
||||
invocations: [{ operationId: "write_document", input: planInput("revision-stale") }],
|
||||
expectedCalls: ["write_document"],
|
||||
expectedFinalState: "mutated",
|
||||
expectedAuthorization: "allowed",
|
||||
faultClass: conflict.class,
|
||||
faultEvidenceOperation: "write_document",
|
||||
},
|
||||
{
|
||||
behavior: "approval_denial",
|
||||
counterpart: "green",
|
||||
seed: approvalSeed(bundle),
|
||||
invocations: [{ operationId: "decide_approval", input: decideApproval }],
|
||||
expectedCalls: ["decide_approval"],
|
||||
expectedFinalState: "mutated",
|
||||
expectedAuthorization: "allowed",
|
||||
assertState: (adapter) => {
|
||||
if (adapter.snapshot().approvals[0]?.status !== "rejected") {
|
||||
throw new Error("approval-denial green case did not reject the approval");
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
behavior: "approval_denial",
|
||||
counterpart: "red",
|
||||
seed: approvalSeed(bundle),
|
||||
scenario: { id: `eval-${authorization.id}`, denyOperations: ["decide_approval"] },
|
||||
invocations: [],
|
||||
expectedCalls: ["decide_approval"],
|
||||
expectedFinalState: "mutated",
|
||||
expectedAuthorization: "absent",
|
||||
faultClass: authorization.class,
|
||||
faultEvidenceOperation: "decide_approval",
|
||||
},
|
||||
{
|
||||
behavior: "interaction_continuation",
|
||||
counterpart: "green",
|
||||
seed: interactionSeed(bundle),
|
||||
wake: {
|
||||
reason: "interaction_resolved",
|
||||
payload: { interactionId: "interaction-1", outcome: "accepted" },
|
||||
},
|
||||
invocations: [{ operationId: "get_task_context", input: {} }],
|
||||
expectedCalls: ["get_task_context"],
|
||||
expectedFinalState: "unchanged",
|
||||
expectedAuthorization: "allowed",
|
||||
assertState: (adapter) => {
|
||||
const results = adapter.context(RUN_ID).interactionResults;
|
||||
if (results[0]?.status !== "accepted") {
|
||||
throw new Error("interaction continuation did not expose the resolved typed result");
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
behavior: "interaction_continuation",
|
||||
counterpart: "red",
|
||||
seed: interactionSeed(bundle),
|
||||
invocations: [{
|
||||
operationId: "request_human_input",
|
||||
input: {
|
||||
idempotencyKey: "eval-interaction-repeat",
|
||||
interactionKind: "confirmation",
|
||||
title: "Repeat the decision",
|
||||
prompt: "Ask again instead of continuing.",
|
||||
continuationPolicy: "wake_assignee",
|
||||
},
|
||||
}],
|
||||
expectedCalls: ["get_task_context"],
|
||||
expectedFinalState: "unchanged",
|
||||
expectedAuthorization: "allowed",
|
||||
},
|
||||
{
|
||||
behavior: "blocker_monitor",
|
||||
counterpart: "green",
|
||||
seed: blockerSeed(bundle),
|
||||
invocations: [
|
||||
{ operationId: "schedule_wake", input: wakeInput },
|
||||
{ operationId: "block_task", input: blockInput },
|
||||
],
|
||||
expectedCalls: ["schedule_wake", "block_task"],
|
||||
expectedFinalState: "mutated",
|
||||
expectedAuthorization: "allowed",
|
||||
maxAttempts: 2,
|
||||
assertState: (adapter) => {
|
||||
const state = adapter.snapshot();
|
||||
if (state.tasks[0]?.status !== "blocked" || state.wakes.length !== 1 || state.blockers.length !== 1) {
|
||||
throw new Error("blocker/monitor green case did not persist blocker plus bounded wake");
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
behavior: "blocker_monitor",
|
||||
counterpart: "red",
|
||||
seed: blockerSeed(bundle),
|
||||
scenario: { id: `eval-${providerCapability.id}`, denyOperations: ["schedule_wake"] },
|
||||
invocations: [{ operationId: "block_task", input: blockInput }],
|
||||
expectedCalls: ["schedule_wake", "block_task"],
|
||||
expectedFinalState: "mutated",
|
||||
expectedAuthorization: "allowed",
|
||||
maxAttempts: 2,
|
||||
faultClass: providerCapability.class,
|
||||
faultEvidenceOperation: "schedule_wake",
|
||||
},
|
||||
{
|
||||
behavior: "artifact_registration",
|
||||
counterpart: "green",
|
||||
invocations: [{ operationId: "register_deliverable", input: artifactInput }],
|
||||
expectedCalls: ["register_deliverable"],
|
||||
expectedFinalState: "mutated",
|
||||
expectedAuthorization: "allowed",
|
||||
assertState: (adapter) => {
|
||||
const state = adapter.snapshot();
|
||||
if (state.artifacts.length !== 1 || state.workProducts[0]?.artifactId !== state.artifacts[0]?.id) {
|
||||
throw new Error("artifact green case did not link artifact and work product");
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
behavior: "artifact_registration",
|
||||
counterpart: "red",
|
||||
seed: seedFor(bundle, {
|
||||
faults: [{
|
||||
id: retry.id,
|
||||
operation: "apply_command",
|
||||
commandKind: "register_deliverable",
|
||||
effect: "retryable_error",
|
||||
remaining: 1,
|
||||
code: "eval_retry_injected",
|
||||
}],
|
||||
}),
|
||||
invocations: [
|
||||
{ operationId: "register_deliverable", input: artifactInput, callId: "call-artifact-attempt-1" },
|
||||
{ operationId: "register_deliverable", input: artifactInput, callId: "call-artifact-attempt-2" },
|
||||
],
|
||||
expectedCalls: ["register_deliverable"],
|
||||
expectedFinalState: "mutated",
|
||||
expectedAuthorization: "allowed",
|
||||
maxAttempts: 1,
|
||||
faultClass: retry.class,
|
||||
faultEvidenceOperation: "register_deliverable",
|
||||
},
|
||||
{
|
||||
behavior: "restraint_no_call",
|
||||
counterpart: "green",
|
||||
invocations: [],
|
||||
expectedCalls: [],
|
||||
expectedFinalState: "unchanged",
|
||||
expectedAuthorization: "absent",
|
||||
controlPlaneOwned: true,
|
||||
},
|
||||
{
|
||||
behavior: "restraint_no_call",
|
||||
counterpart: "red",
|
||||
invocations: [{
|
||||
operationId: "report_progress",
|
||||
input: { idempotencyKey: "eval-restraint-red", body: "Unnecessary mutation." },
|
||||
}],
|
||||
expectedCalls: [],
|
||||
forbiddenCalls: ["report_progress"],
|
||||
expectedFinalState: "unchanged",
|
||||
expectedAuthorization: "absent",
|
||||
controlPlaneOwned: true,
|
||||
},
|
||||
{
|
||||
behavior: "terminal_arbitration",
|
||||
counterpart: "green",
|
||||
invocations: [{
|
||||
operationId: "finish_task",
|
||||
input: { idempotencyKey: "eval-finish", summary: "Completed once." },
|
||||
}],
|
||||
expectedCalls: ["finish_task"],
|
||||
expectedFinalState: "mutated",
|
||||
expectedAuthorization: "allowed",
|
||||
assertState: (adapter) => {
|
||||
if (adapter.snapshot().tasks[0]?.status !== "done") {
|
||||
throw new Error("terminal green case did not finish the task");
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
behavior: "terminal_arbitration",
|
||||
counterpart: "red",
|
||||
invocations: [
|
||||
{
|
||||
operationId: "finish_task",
|
||||
input: { idempotencyKey: "eval-finish-red", summary: "First terminal." },
|
||||
},
|
||||
{
|
||||
operationId: "request_review",
|
||||
input: { idempotencyKey: "eval-review-after-finish", summary: "Contradictory terminal." },
|
||||
},
|
||||
],
|
||||
expectedCalls: ["finish_task"],
|
||||
forbiddenCalls: ["request_review"],
|
||||
expectedFinalState: "mutated",
|
||||
expectedAuthorization: "allowed",
|
||||
maxAttempts: 1,
|
||||
assertState: (adapter) => {
|
||||
if (adapter.snapshot().tasks[0]?.status !== "done") {
|
||||
throw new Error("terminal arbitration did not preserve the first terminal state");
|
||||
}
|
||||
},
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
function observedAuthorization(results: readonly CapabilitySemanticToolResult[]): AuthorizationState {
|
||||
if (results.length === 0) return "absent";
|
||||
if (results.some((result) => result.ok)) return "allowed";
|
||||
return results.every((result) => !result.ok && result.denial.code === "tool_not_exposed")
|
||||
? "absent"
|
||||
: "denied";
|
||||
}
|
||||
|
||||
async function runBehaviorCase(bundle: EvalBundle, spec: BehaviorCase): Promise<EvalBehaviorExecutionResult> {
|
||||
const adapter = new CapabilityMockControlPlaneAdapter(spec.seed ?? seedFor(bundle));
|
||||
await adapter.start();
|
||||
await adapter.openFixtureRun({
|
||||
identity: {
|
||||
runId: RUN_ID,
|
||||
sessionId: SESSION_ID,
|
||||
companyId: "company-1",
|
||||
issueId: ACTIVE_TASK_ID,
|
||||
agentId: "actor-1",
|
||||
},
|
||||
backendKind: "mock",
|
||||
sourceInstanceId: "eval-behavior-harness",
|
||||
capabilities: capabilityFixtureRunCapabilities(bundle.grants),
|
||||
wake: spec.wake,
|
||||
});
|
||||
const beforeRevision = adapter.snapshot().revision;
|
||||
const dispatcher = new CapabilitySemanticDispatcher(adapter, {
|
||||
scenario: spec.scenario ?? { id: `eval-${spec.behavior}-${spec.counterpart}` },
|
||||
explicitClaims: bundle.grants,
|
||||
});
|
||||
dispatcher.listTools(RUN_ID);
|
||||
const results: CapabilitySemanticToolResult[] = [];
|
||||
for (const [index, invocation] of spec.invocations.entries()) {
|
||||
results.push(await dispatcher.dispatch({
|
||||
runId: RUN_ID,
|
||||
callId: invocation.callId ?? `call-${spec.behavior}-${spec.counterpart}-${index + 1}`,
|
||||
operationId: invocation.operationId,
|
||||
input: invocation.input,
|
||||
}));
|
||||
}
|
||||
spec.assertState?.(adapter);
|
||||
const afterRevision = adapter.snapshot().revision;
|
||||
const observedCalls = spec.invocations.map((invocation) => invocation.operationId);
|
||||
const caseId = `${spec.behavior}:${spec.counterpart}`;
|
||||
const authorizationRecords = dispatcher.authorizationRecords();
|
||||
const decisionRecords = adapter.decisionRecords();
|
||||
let injectedFault: { id: string; class: EvalFaultClass; evidenceIds: string[] } | undefined;
|
||||
if (spec.faultClass !== undefined) {
|
||||
const declaration = faultFor(bundle, spec.faultClass);
|
||||
const evidenceIds = [
|
||||
...decisionRecords
|
||||
.filter((record) => record.outcome === "faulted")
|
||||
.map((record) => record.id),
|
||||
...authorizationRecords
|
||||
.filter(
|
||||
(record) =>
|
||||
record.operationId === spec.faultEvidenceOperation && !record.allowed,
|
||||
)
|
||||
.map((record) => record.id),
|
||||
];
|
||||
if (evidenceIds.length === 0) {
|
||||
throw new Error(`${caseId} did not emit evidence for ${spec.faultClass} injection`);
|
||||
}
|
||||
injectedFault = { id: declaration.id, class: declaration.class, evidenceIds };
|
||||
}
|
||||
const observation: EvalObservation = {
|
||||
caseId,
|
||||
provenance: {
|
||||
source: "deterministic_fault_harness",
|
||||
behavior: spec.behavior,
|
||||
counterpart: spec.counterpart,
|
||||
...(injectedFault === undefined ? {} : { faultInjection: injectedFault }),
|
||||
},
|
||||
controlPlaneOwned: spec.controlPlaneOwned ?? false,
|
||||
expectedCalls: spec.expectedCalls,
|
||||
observedCalls,
|
||||
forbiddenCalls: spec.forbiddenCalls ?? [],
|
||||
finalState: {
|
||||
expected: spec.expectedFinalState,
|
||||
observed: afterRevision === beforeRevision ? "unchanged" : "mutated",
|
||||
},
|
||||
authorization: {
|
||||
expected: spec.expectedAuthorization,
|
||||
observed: observedAuthorization(results),
|
||||
},
|
||||
trace: {
|
||||
runId: RUN_ID,
|
||||
sessionId: SESSION_ID,
|
||||
turnId: `turn-${caseId}`,
|
||||
itemId: `item-${caseId}`,
|
||||
receiptIds: results.map((result) => result.callId),
|
||||
terminalPresent: true,
|
||||
},
|
||||
efficiency: { attempts: observedCalls.length },
|
||||
budget: { maxAttempts: spec.maxAttempts ?? Math.max(1, spec.expectedCalls.length) },
|
||||
};
|
||||
await adapter.stop();
|
||||
return { behavior: spec.behavior, counterpart: spec.counterpart, observation };
|
||||
}
|
||||
|
||||
/**
|
||||
* Executes the eight requested behaviors and their red counterparts against the
|
||||
* real deterministic mock authority. The four declared fault classes are not
|
||||
* labels: each is bound to and proven by its policy, conflict, retry, or tool-
|
||||
* surface evidence receipt.
|
||||
*/
|
||||
export async function runEvalBehaviorFaultMatrix(
|
||||
bundle: EvalBundle,
|
||||
): Promise<EvalBehaviorExecutionResult[]> {
|
||||
assertBundleSecretFree(bundle);
|
||||
for (const faultClass of ["authorization", "conflict", "retry", "provider_capability"] as const) {
|
||||
faultFor(bundle, faultClass);
|
||||
}
|
||||
const results: EvalBehaviorExecutionResult[] = [];
|
||||
for (const spec of behaviorCases(bundle)) results.push(await runBehaviorCase(bundle, spec));
|
||||
assertEvalBehaviorFaultMatrix(bundle, results);
|
||||
return results;
|
||||
}
|
||||
|
||||
/** Fails closed if coverage, polarity, or scored red/green behavior drifts. */
|
||||
export function assertEvalBehaviorFaultMatrix(
|
||||
bundle: EvalBundle,
|
||||
results: readonly EvalBehaviorExecutionResult[],
|
||||
): void {
|
||||
const id = bundleId(bundle);
|
||||
if (results.length !== EVAL_BEHAVIOR_IDS.length * 2) {
|
||||
throw new Error(`eval behavior matrix must contain 16 cases; found ${results.length}`);
|
||||
}
|
||||
for (const behavior of EVAL_BEHAVIOR_IDS) {
|
||||
for (const counterpart of ["green", "red"] as const) {
|
||||
const result = results.find(
|
||||
(candidate) => candidate.behavior === behavior && candidate.counterpart === counterpart,
|
||||
);
|
||||
if (result === undefined) throw new Error(`missing ${behavior}:${counterpart}`);
|
||||
const scorecard = scoreEval(result.observation, { bundleId: id });
|
||||
if (counterpart === "green" && !scorecard.overall.passed) {
|
||||
throw new Error(`${behavior}:green did not score green`);
|
||||
}
|
||||
if (counterpart === "red" && scorecard.overall.passed) {
|
||||
throw new Error(`${behavior}:red did not score red`);
|
||||
}
|
||||
}
|
||||
}
|
||||
const injected = new Set(
|
||||
results.flatMap((result) => {
|
||||
const fault = result.observation.provenance?.faultInjection;
|
||||
return fault === undefined ? [] : [fault.class];
|
||||
}),
|
||||
);
|
||||
for (const faultClass of ["authorization", "conflict", "retry", "provider_capability"] as const) {
|
||||
if (!injected.has(faultClass)) throw new Error(`fault class ${faultClass} was not executed`);
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ import type {
|
||||
CapabilityEvalCaseResult,
|
||||
CapabilityLiveCodexMatrixResult,
|
||||
} from "../conformance/capability-eval-suite.js";
|
||||
import type { EvalFaultClass } from "./eval-bundle.js";
|
||||
|
||||
/**
|
||||
* Multi-dimensional scoring for the runner eval vertical slice.
|
||||
@@ -82,6 +83,17 @@ export interface EvalEfficiencyBudget {
|
||||
|
||||
export interface EvalObservation {
|
||||
caseId: string;
|
||||
provenance?: {
|
||||
source: "live_model" | "deterministic_fault_harness" | "fixture";
|
||||
behavior?: string;
|
||||
counterpart?: "green" | "red";
|
||||
faultInjection?: {
|
||||
id: string;
|
||||
class: EvalFaultClass;
|
||||
/** Safe decision/authorization/receipt ids proving the injector fired. */
|
||||
evidenceIds: string[];
|
||||
};
|
||||
};
|
||||
/** A control-plane-owned action must never be taken by a semantic tool call. */
|
||||
controlPlaneOwned: boolean;
|
||||
expectedCalls: string[];
|
||||
@@ -281,6 +293,7 @@ export interface LiveMatrixAugment {
|
||||
authorization: EvalAuthorizationExpectation;
|
||||
efficiency?: EvalEfficiencyEvidence;
|
||||
budget?: EvalEfficiencyBudget;
|
||||
provenance?: EvalObservation["provenance"];
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -295,6 +308,7 @@ export function observationFromLiveMatrix(
|
||||
): EvalObservation {
|
||||
return {
|
||||
caseId: result.caseId,
|
||||
provenance: augment.provenance,
|
||||
controlPlaneOwned: result.expectedCalls.length === 0,
|
||||
expectedCalls: result.expectedCalls,
|
||||
observedCalls: result.observedCalls,
|
||||
|
||||
@@ -103,10 +103,16 @@ export function buildEvalSliceReport(
|
||||
/** A compact, human-readable rendering of a slice report for inspection. */
|
||||
export function renderEvalSliceMarkdown(report: EvalSliceReport): string {
|
||||
const lines: string[] = [];
|
||||
const sources = new Set(report.cases.map((entry) => entry.observation.provenance?.source));
|
||||
const sourceDescription = report.generatedFromLiveModel
|
||||
? "real model tool choice"
|
||||
: sources.size === 1 && sources.has("deterministic_fault_harness")
|
||||
? "deterministic fault harness"
|
||||
: "fixture observations";
|
||||
lines.push(`# Runner eval slice — ${report.bundle.id}`);
|
||||
lines.push("");
|
||||
lines.push(`- Bundle: \`${report.bundle.summary}\``);
|
||||
lines.push(`- Source: ${report.generatedFromLiveModel ? "real model tool choice" : "fixture observations"}`);
|
||||
lines.push(`- Source: ${sourceDescription}`);
|
||||
lines.push(
|
||||
`- Cases: ${report.aggregate.caseCount} · passed ${report.aggregate.passed} · gate failures ${report.aggregate.gateFailures} · mean overall ${report.aggregate.meanOverall}`,
|
||||
);
|
||||
@@ -119,12 +125,13 @@ export function renderEvalSliceMarkdown(report: EvalSliceReport): string {
|
||||
lines.push("");
|
||||
lines.push("## Cases");
|
||||
lines.push("");
|
||||
lines.push("| case | gate | overall | outcome | trajectory | trace | efficiency |");
|
||||
lines.push("| --- | --- | --- | --- | --- | --- | --- |");
|
||||
for (const { scorecard } of report.cases) {
|
||||
lines.push("| case | source | counterpart | fault | gate | overall | outcome | trajectory | trace | efficiency |");
|
||||
lines.push("| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |");
|
||||
for (const { scorecard, observation } of report.cases) {
|
||||
const d = scorecard.dimensions;
|
||||
const provenance = observation.provenance;
|
||||
lines.push(
|
||||
`| ${scorecard.caseId} | ${d.hard_invariants.passed ? "ok" : "FAIL"} | ${scorecard.overall.score} | ${d.semantic_outcome.score} | ${d.trajectory_restraint.score} | ${d.trace_completeness.score} | ${d.quality_efficiency.score} |`,
|
||||
`| ${scorecard.caseId} | ${provenance?.source ?? "unspecified"} | ${provenance?.counterpart ?? "—"} | ${provenance?.faultInjection?.class ?? "—"} | ${d.hard_invariants.passed ? "ok" : "FAIL"} | ${scorecard.overall.score} | ${d.semantic_outcome.score} | ${d.trajectory_restraint.score} | ${d.trace_completeness.score} | ${d.quality_efficiency.score} |`,
|
||||
);
|
||||
}
|
||||
lines.push("");
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
export * from "./eval-bundle.js";
|
||||
export * from "./eval-execution.js";
|
||||
export * from "./eval-scoring.js";
|
||||
export * from "./eval-slice.js";
|
||||
@@ -67,13 +67,21 @@ class FakeCapabilityCodexTransport implements CodexAppServerTransport {
|
||||
if (method === "initialize") return { user: { sessionId: this.state.providerSessionId } };
|
||||
if (method === "thread/start") {
|
||||
expect(Array.isArray(params.dynamicTools)).toBe(true);
|
||||
return { thread: { id: this.state.threadId, sessionId: this.state.providerSessionId } };
|
||||
return {
|
||||
model: "gpt-eval-test",
|
||||
modelProvider: "openai",
|
||||
thread: { id: this.state.threadId, sessionId: this.state.providerSessionId },
|
||||
};
|
||||
}
|
||||
if (method === "thread/read") {
|
||||
return { thread: { id: this.state.threadId, sessionId: this.state.providerSessionId, turns: [] } };
|
||||
}
|
||||
if (method === "thread/resume") {
|
||||
return { thread: { id: this.state.threadId, sessionId: this.state.providerSessionId } };
|
||||
return {
|
||||
model: "gpt-eval-test",
|
||||
modelProvider: "openai",
|
||||
thread: { id: this.state.threadId, sessionId: this.state.providerSessionId },
|
||||
};
|
||||
}
|
||||
if (method === "turn/start") {
|
||||
const turnId = `turn-${++this.state.nextTurn}`;
|
||||
@@ -241,6 +249,7 @@ describe("Capability live runnerd and Codex session", () => {
|
||||
expect(first.assistantText).toContain("stateRevision");
|
||||
expect(second.assistantText).toContain("stateRevision");
|
||||
expect(first.snapshot.providerThreadId).toBe(second.snapshot.providerThreadId);
|
||||
expect(first.snapshot.providerModel).toEqual({ id: "gpt-eval-test", provider: "openai" });
|
||||
expect(session.mockState().comments.at(-1)?.body).toBe(
|
||||
"Progress persisted through the live Codex tool loop.",
|
||||
);
|
||||
|
||||
@@ -101,6 +101,8 @@ export interface CapabilityLiveSessionSnapshot {
|
||||
sessionId: string;
|
||||
providerThreadId: string;
|
||||
providerSessionId: string | null;
|
||||
/** Safe provider identity retained for reproducible eval bundle declarations. */
|
||||
providerModel?: { id: string; provider: string };
|
||||
status: CapabilityLiveSessionStatus;
|
||||
activeTurnId: string | null;
|
||||
createdAt: string;
|
||||
@@ -440,6 +442,7 @@ export class CapabilityLiveSession {
|
||||
#processEvidence: CapabilityRunnerdProcessEvidence | null = null;
|
||||
#providerThreadId = "";
|
||||
#providerSessionId: string | null = null;
|
||||
#providerModel: { id: string; provider: string } | undefined;
|
||||
#status: CapabilityLiveSessionStatus = "starting";
|
||||
#activeTurnId: string | null = null;
|
||||
#revision = 0;
|
||||
@@ -471,6 +474,9 @@ export class CapabilityLiveSession {
|
||||
this.#priorAuthorizationRecords = structuredClone(options.snapshot?.authorizationRecords ?? []);
|
||||
this.#providerThreadId = options.snapshot?.providerThreadId ?? "";
|
||||
this.#providerSessionId = options.snapshot?.providerSessionId ?? null;
|
||||
this.#providerModel = options.snapshot?.providerModel === undefined
|
||||
? undefined
|
||||
: structuredClone(options.snapshot.providerModel);
|
||||
this.#activeTurnId = options.snapshot?.activeTurnId ?? null;
|
||||
this.#revision = options.snapshot?.revision ?? 0;
|
||||
this.#entryCounter = this.#transcript.length + this.#evidence.length;
|
||||
@@ -600,6 +606,9 @@ export class CapabilityLiveSession {
|
||||
sessionId: this.id,
|
||||
providerThreadId: this.#providerThreadId,
|
||||
providerSessionId: this.#providerSessionId,
|
||||
...(this.#providerModel === undefined
|
||||
? {}
|
||||
: { providerModel: structuredClone(this.#providerModel) }),
|
||||
status: this.#status,
|
||||
activeTurnId: this.#activeTurnId,
|
||||
createdAt: this.#createdAt,
|
||||
@@ -818,6 +827,7 @@ export class CapabilityLiveSession {
|
||||
throw new Error("Codex reconnect resumed a different provider thread");
|
||||
}
|
||||
this.#providerSessionId = text(resumedThread.sessionId) || this.#providerSessionId;
|
||||
this.#captureProviderModel(resumed);
|
||||
this.#activeTurnId = null;
|
||||
} else {
|
||||
const opened = await this.#transport.request("thread/start", {
|
||||
@@ -835,6 +845,7 @@ export class CapabilityLiveSession {
|
||||
this.#providerThreadId = text(thread.id);
|
||||
if (this.#providerThreadId.length === 0) throw new Error("Codex thread response omitted thread.id");
|
||||
this.#providerSessionId = text(thread.sessionId) || text(record(initialized.user).sessionId) || null;
|
||||
this.#captureProviderModel(opened);
|
||||
}
|
||||
this.#status = "idle";
|
||||
this.#appendEvidence("session", null, {
|
||||
@@ -850,6 +861,13 @@ export class CapabilityLiveSession {
|
||||
await this.#persist();
|
||||
}
|
||||
|
||||
#captureProviderModel(response: Record<string, unknown>): void {
|
||||
const thread = record(response.thread);
|
||||
const id = text(response.model, text(thread.model));
|
||||
const provider = text(response.modelProvider, text(thread.modelProvider));
|
||||
if (id.length > 0 && provider.length > 0) this.#providerModel = { id, provider };
|
||||
}
|
||||
|
||||
async #disconnect(reason: string): Promise<void> {
|
||||
const transport = this.#transport;
|
||||
this.#transport = null;
|
||||
|
||||
Reference in new issue
Block a user