diff --git a/doc/plans/2026-08-20-paperclip-runner-production-readiness.md b/doc/plans/2026-08-20-paperclip-runner-production-readiness.md new file mode 100644 index 00000000000..3b10a8f81bf --- /dev/null +++ b/doc/plans/2026-08-20-paperclip-runner-production-readiness.md @@ -0,0 +1,111 @@ +# Paperclip runner production-readiness plan + +Status: complete + +This plan tracks the end-to-end cutover to one production architecture: + +```text +Paperclip control plane + <-> authenticated durable PRP +paperclip-runnerd (the only Paperclip-owned process in the sandbox) + <-> provider-native protocol +Codex / another provider +``` + +TypeScript may host the external mock control plane, eval orchestration, and UI. It must not be a second in-sandbox dispatcher or daemon. + +## Exit criteria + +The work is complete only when all of these are supported by current artifacts and test evidence: + +- every live-roster outcome is retained and classified as infrastructure, provider lifecycle, eval-system, genuine model behavior, or flaky/underspecified; +- infrastructure retries are bounded, limited to explicitly retryable classes, and visible in reports; +- every canonical protocol action and discovery gateway maps to deterministic and live coverage, fixtures, matchers, gaps, and workflow coverage; +- reports reproduce overview, latest runs, definitions, attempts, correlated assertions, fixture/execution boundaries, state diffs, metadata, usage, and failures from retained artifacts; +- the runner lab uses the external mock control plane -> PRP -> runnerd -> provider path used by evals; +- recovery, authorization, drift, idempotency, bounds, cleanup, secret isolation, transcript, accounting, and state reconstruction have deterministic proof; +- a gated GPT-5.4-mini eval, complete mini roster, regenerated report, and interactive lab smoke pass; +- canonical docs describe one architecture and both repositories are committed, pushed, and clean. + +## Work phases + +### 1. Audit and preserve evidence + +- Inventory current roster findings and retain all failed attempt diagnostics. +- Compare the 41-action catalog, discovery gateways, 106 deterministic cases, old skill requirements, current live roster, fixtures, and matchers. +- Audit report routes and UI behavior against every report exit criterion. +- Identify every runner-lab backend path that still constructs the legacy TypeScript live-session dispatcher. + +### 2. Runner and PRP reliability + +- Give provider-turn, runner-shutdown, transport, provider-exit, and orchestration failures distinct structured classifications. +- Retain bounded stderr, lifecycle, command, cursor, outbox, and state-directory diagnostics on failure. +- Ensure runner shutdown explicitly terminates the provider process group after durable flush. +- Add repeated fake-provider regression coverage for turn completion followed by shutdown. +- Implement bounded retry policy in the roster orchestrator for retryable infrastructure failures only. + +Gate: deterministic Rust/TypeScript bridge and recovery suites pass repeatedly, followed by one GPT-5.4-mini eval. + +### 3. Coverage completion + +- Generate a reviewable coverage matrix from canonical catalogs and eval inventory. +- Port old-skill behavioral requirements as protocol requirements where still applicable. +- Add missing positive, negative, malformed-input, authorization, redaction, ordering, idempotency, state-transition, discovery, and workflow cases. +- Prefer composable fixture fragments over duplicated company snapshots. +- Validate that every requirement has a deterministic test or a documented reason why only a live eval applies. + +### 4. Report completion + +- Make artifacts the sole source for reproducible static reports. +- Test newest-first selection and latest test/model/configuration matrix semantics. +- Test navigation and clickable definitions/attempts. +- Correlate assertions to exact transcript/tool/state events. +- Render metadata, usage/cost, fixture state, execution boundary, before/after diffs, and partial/error/timeout states. + +### 5. Runner-lab cutover + +- Replace the issue-thread lab's legacy TypeScript execution service with the shared durable runner session orchestration used by evals. +- Expose start, interrupt, stop, reconnect, and new-session lifecycle operations through the external mock control plane. +- Project retained eval attempts and live sessions through shared transcript, tool-event, fixture-state, state-diff, usage, and identity models. +- Preserve the existing devtools tabs and navigation while making the session backend runnerd-based. + +Gate: `http://127.0.0.1:4184/#/chat` completes a real mini turn and proves the runnerd topology. + +### 6. Final verification and delivery + +- Run the full deterministic verification suite. +- Run the complete GPT-5.4-mini roster with bounded spend and reported retries. +- Regenerate and inspect the HTML report. +- Smoke-test the runner lab through runnerd. +- Record passes, behavior failures, infrastructure failures, retries, tokens, cost, and remaining limitations. +- Update canonical operating/architecture/failure-investigation docs. +- Commit and push each repository and verify clean worktrees equal their remote refs. + +## Completion evidence + +- The eval CLI and runner lab both use external TypeScript mock control-plane + orchestration -> authenticated durable PRP -> Rust `paperclip-runnerd` -> + Codex. No TypeScript Paperclip daemon runs in the sandbox. +- Provider-turn, runner-shutdown, runner-exit, and eval-orchestration failures + are structured and retained with bounded lifecycle diagnostics. Roster retries + are capped and restricted to explicitly retryable artifacts. +- `protocol-coverage.json` and the eval `coverage-matrix.json` trace all 41 + canonical actions, both discovery gateways, 106 old-skill requirements, 35 + live cases, workflows, matchers, fixtures, and 12 lifecycle requirements. + All 41 actions have an exact deterministic contract; 29 additionally map to + legacy behavioral scenarios rather than conflating the two measures. +- Deterministic verification passed: 559 TypeScript tests, 48 Rust unit tests + plus Rust integrations, 25 eval-framework tests, durable recovery, and a + two-turn real GPT-5.4-mini runner-lab smoke. +- The final GPT-5.4-mini roster `20260820-production-readiness-final` passed + 35/35 with zero retries. The prior run's two behavior findings are retained; + both were underspecified authored prompts and passed targeted and full reruns + after the intended tool steps were made explicit. +- The static report was regenerated from 272 retained attempts at + `paperclip-evals/evals/paperclip-runner/.runtime/report/index.html`; it includes + overview/latest/inventory/test/attempt routes, correlated assertions, fixture + and execution boundaries, state/devtools views, model/config/build/timing, + usage/cost, navigation, and explicit failure states. + +Passing isolated tests was not treated as sufficient: the release gate includes +the complete roster and a real multi-turn lab session through runnerd. diff --git a/doc/plans/2026-08-20-single-daemon-runner-tool-bridge.md b/doc/plans/2026-08-20-single-daemon-runner-tool-bridge.md index 08870d7cbf0..e8334ccd80b 100644 --- a/doc/plans/2026-08-20-single-daemon-runner-tool-bridge.md +++ b/doc/plans/2026-08-20-single-daemon-runner-tool-bridge.md @@ -1,7 +1,7 @@ # Single-daemon sandbox runner and tool bridge Date: 2026-08-20 -Status: implemented; provider reliability follow-up remains +Status: implemented and production-readiness verified ## Target @@ -112,11 +112,17 @@ labeled a reference/legacy path and cannot be reported as production topology. - The fake Codex boundary integration and the complete Rust suite pass. - Real GPT-5.4-mini single- and multi-call cases pass with final assistant text, token counts, estimated cost, state effects, and ordered evidence preserved. -- The 35-case final roster produced 24 passes, two model/eval behavior findings, - and nine provider/runner shutdown timeouts. Previously timed-out multi-step - cases passed individually after the command-delivery race fix. These are kept - as reliability findings rather than rewritten as semantic failures. - -The older `CapabilityLiveSessionService` remains a runner-lab/reference API. It -is no longer used by the live eval CLI and must not be described as the sandbox -runtime. Removal or consolidation with the lab UI is a separate deletion task. +- The external mock control plane, eval CLI, and runner lab all use authenticated + durable PRP to the Rust runner. `CapabilityLiveSessionService` is external + orchestration only; its default transport starts runnerd, never a TypeScript + sandbox dispatcher. +- Reconnect restores the same Codex provider thread; interrupt, suspend, stop, + revocation, ACK replay, bounded delivery, and process-group cleanup have + deterministic coverage. +- The final GPT-5.4-mini roster (`20260820-production-readiness-final`) passed + 35/35 on first attempts with zero infrastructure retries. The two findings in + the preceding run were retained and classified as underspecified prompts, + corrected, gated individually, and then rerun in the complete roster. +- A two-turn issue-thread smoke passed through mock Paperclip -> PRP -> runnerd + -> GPT-5.4-mini, including semantic tools, durable state, usage accounting, + credential isolation, and session isolation. diff --git a/packages/paperclip-runner/devtools/issue-thread/src/App.tsx b/packages/paperclip-runner/devtools/issue-thread/src/App.tsx index f4b58de8040..fe149246439 100644 --- a/packages/paperclip-runner/devtools/issue-thread/src/App.tsx +++ b/packages/paperclip-runner/devtools/issue-thread/src/App.tsx @@ -69,12 +69,15 @@ interface EmbeddedEvalReport { run: { model: string; provider: string; + configuration: string; sessionId: string; fixtureDigest: string; runnerPackageDigest: string; runnerdDigest: string; + runnerBuild: string; startedAt: string; finishedAt: string; + durationMs: number; initialRevision: number; finalRevision: number; usage: { diff --git a/packages/paperclip-runner/devtools/issue-thread/src/DevtoolsInspector.tsx b/packages/paperclip-runner/devtools/issue-thread/src/DevtoolsInspector.tsx index 84c0581025d..876aa106edf 100644 --- a/packages/paperclip-runner/devtools/issue-thread/src/DevtoolsInspector.tsx +++ b/packages/paperclip-runner/devtools/issue-thread/src/DevtoolsInspector.tsx @@ -13,7 +13,7 @@ export interface EvalInspectorReport { disposition: string; passed: boolean; checks: EvalAssertion[]; - run: { model: string; provider: string; sessionId: string; fixtureDigest: string; runnerPackageDigest: string; runnerdDigest: string; startedAt: string; finishedAt: string; initialRevision: number; finalRevision: number; usage: { agentTurns: number; providerRequests: number | null; inputTokens: number; outputTokens: number; cachedInputTokens: number; reasoningTokens: number; estimatedCostNanodollars: number; pricingVersion: string } | null }; + run: { model: string; provider: string; configuration: string; sessionId: string; fixtureDigest: string; runnerPackageDigest: string; runnerdDigest: string; runnerBuild: string; startedAt: string; finishedAt: string; durationMs: number; initialRevision: number; finalRevision: number; usage: { agentTurns: number; providerRequests: number | null; inputTokens: number; outputTokens: number; cachedInputTokens: number; reasoningTokens: number; estimatedCostNanodollars: number; pricingVersion: string } | null }; } function JsonTree({ value, name = "root" }: { value: Json; name?: string }) { @@ -143,8 +143,9 @@ export function DevtoolsInspector({ snapshot, onFork, tab, onTabChange, evalRepo
← Eval suite{evalReport.passed ? "PASS" : evalReport.disposition.replaceAll("_", " ").toUpperCase()}{evalReport.attemptId}
Model
{evalReport.run.provider}/{evalReport.run.model}
+
Configuration
{evalReport.run.configuration}
Session
{evalReport.run.sessionId}
-
Duration
{Math.max(0, new Date(evalReport.run.finishedAt).getTime() - new Date(evalReport.run.startedAt).getTime())} ms
+
Duration
{evalReport.run.durationMs} ms
Fixture
{evalReport.run.fixtureDigest}
State
r{evalReport.run.initialRevision} → r{evalReport.run.finalRevision}
Tokens
{evalReport.run.usage === null ? "unknown" : `${evalReport.run.usage.inputTokens} in · ${evalReport.run.usage.outputTokens} out · ${evalReport.run.usage.cachedInputTokens} cached`}
@@ -152,6 +153,7 @@ export function DevtoolsInspector({ snapshot, onFork, tab, onTabChange, evalRepo
Provider requests
{evalReport.run.usage?.providerRequests ?? "unavailable"}
Estimated cost
{evalReport.run.usage === null ? "unknown" : `$${(evalReport.run.usage.estimatedCostNanodollars / 1_000_000_000).toFixed(6)} · ${evalReport.run.usage.pricingVersion}`}
Runner
{evalReport.run.runnerPackageDigest}
+
Runner build
{evalReport.run.runnerBuild}
runnerd
{evalReport.run.runnerdDigest}

Assertions

diff --git a/packages/paperclip-runner/package.json b/packages/paperclip-runner/package.json index 1ee7701fc94..02c23b722dd 100644 --- a/packages/paperclip-runner/package.json +++ b/packages/paperclip-runner/package.json @@ -88,6 +88,8 @@ "check:package-boundaries": "node scripts/check-package-boundaries.mjs", "check:clean-consumers": "node scripts/check-clean-consumers.mjs", "generate:capability-inventory": "node scripts/generate-capability-inventory.mjs", + "generate:protocol-coverage": "pnpm run build:typescript && node scripts/generate-protocol-coverage.mjs", + "check:protocol-coverage": "node scripts/generate-protocol-coverage.mjs --check", "check:capability-inventory": "node scripts/check-capability-inventory.mjs", "test:capability-inventory": "node scripts/check-capability-inventory.test.mjs", "test:capability-evals": "vitest run src/conformance/capability-eval-suite.test.ts", diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs b/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs index 1b0dba4cd9e..0e8cf878f0c 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs @@ -38,13 +38,19 @@ pub struct CodexProvider { process: SupervisedProcess, next_request_id: u64, thread_id: String, + session_id: Option, pending_tool_requests: BTreeMap, } impl CodexProvider { + pub fn process_id(&self) -> u32 { + self.process.id() + } + pub fn start( config: &CodexProviderConfig, tools: impl Iterator, + resume_thread_id: Option<&str>, ) -> Result { let mut provider = Self { process: SupervisedProcess::spawn( @@ -55,9 +61,10 @@ impl CodexProvider { )?, next_request_id: 1, thread_id: String::new(), + session_id: None, pending_tool_requests: BTreeMap::new(), }; - provider.request("initialize", json!({ + let initialized = provider.request("initialize", json!({ "clientInfo": { "name": "paperclip-runnerd", "title": "Paperclip Runner", "version": "1" }, "capabilities": { "experimentalApi": true, "requestAttestation": false } }))?; @@ -71,27 +78,57 @@ impl CodexProvider { }) }) .collect::>(); - let opened = provider.request( - "thread/start", - json!({ - "cwd": config.cwd, - "model": config.model, - "approvalPolicy": "never", - "baseInstructions": config.instructions, - "dynamicTools": dynamic_tools, - "experimentalRawEvents": true, - "persistExtendedHistory": true, - }), - )?; + let opened = if let Some(thread_id) = resume_thread_id { + provider.request( + "thread/resume", + json!({ + "threadId": thread_id, + "cwd": config.cwd, + "model": config.model, + "approvalPolicy": "never", + "baseInstructions": config.instructions, + "dynamicTools": dynamic_tools, + "experimentalRawEvents": true, + "persistExtendedHistory": true, + }), + )? + } else { + provider.request( + "thread/start", + json!({ + "cwd": config.cwd, + "model": config.model, + "approvalPolicy": "never", + "baseInstructions": config.instructions, + "dynamicTools": dynamic_tools, + "experimentalRawEvents": true, + "persistExtendedHistory": true, + }), + )? + }; provider.thread_id = opened .pointer("/thread/id") .and_then(Value::as_str) .filter(|value| !value.is_empty()) .ok_or_else(|| LocalRunnerError::invalid("Codex thread/start omitted thread.id"))? .to_owned(); + provider.session_id = opened + .pointer("/thread/sessionId") + .and_then(Value::as_str) + .or_else(|| initialized.pointer("/user/sessionId").and_then(Value::as_str)) + .filter(|value| !value.is_empty()) + .map(str::to_owned); Ok(provider) } + pub fn thread_id(&self) -> &str { + &self.thread_id + } + + pub fn session_id(&self) -> Option<&str> { + self.session_id.as_deref() + } + pub fn start_turn(&mut self, message: &str, cwd: &str) -> Result { let thread_id = self.thread_id.clone(); self.request( @@ -106,6 +143,22 @@ impl CodexProvider { ) } + pub fn interrupt_turn(&mut self, turn_id: &str) -> Result { + let thread_id = self.thread_id.clone(); + self.request( + "turn/interrupt", + json!({ "threadId": thread_id, "turnId": turn_id }), + ) + } + + pub fn read_thread(&mut self) -> Result { + let thread_id = self.thread_id.clone(); + self.request( + "thread/read", + json!({ "threadId": thread_id, "includeTurns": true }), + ) + } + pub fn poll(&mut self) -> Result, LocalRunnerError> { let Some(line) = self.process.receive_stdout_line(Duration::from_millis(1))? else { return if self.process.try_wait()?.is_some() { @@ -169,6 +222,14 @@ impl CodexProvider { })) } + /// Stops the provider process group at an explicit runner lifecycle boundary. + /// + /// `Drop` remains the last-resort cleanup path, but runner shutdown must not + /// depend on an implicit destructor firing after the durable transport loop. + pub fn shutdown(&mut self) -> Result<(), LocalRunnerError> { + self.process.terminate_group().map(|_| ()) + } + fn request(&mut self, method: &str, params: Value) -> Result { let id = self.next_request_id; self.next_request_id += 1; diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs b/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs index 78d3f9cf3c0..32e090bc0da 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs @@ -9,14 +9,37 @@ pub fn run_durable_runner( } let store = DurableStateStore::new(&config.state_dir)?; let (mut state, recovered) = store.load_or_create(&config)?; - let mut provider = start_configured_provider(&state)?; if recovered && state.lifecycle == "revoked" { - // Revocation is durable and terminal. Dropping the fresh bootstrap capability before - // destination resolution guarantees a restarted daemon cannot authenticate, reconnect, - // advance cursors, or rewrite the preserved operator-recovery state. + // Revocation is durable and terminal. Do not normalize, rewrite, start + // a provider, resolve a destination, or consume the fresh bootstrap. drop(bootstrap_ticket); return Ok(()); } + if recovered && state.lifecycle == "suspended" { + state.stop_after_flush = false; + state.lifecycle = "ready".to_owned(); + state.record_diagnostic("resuming an explicitly suspended durable runner"); + } + let mut provider = start_configured_provider(&mut state)?; + if recovered { + if let Some(runtime) = provider.as_ref() { + enqueue_event( + &mut state, + &config, + "provider.started", + 0, + json!({ + "provider": "codex", + "pid": runtime.process_id(), + "threadId": runtime.thread_id(), + "sessionId": runtime.session_id(), + "resumed": true, + }), + None, + )?; + } + } + store.save(&state)?; // Resolve once before authentication. Reconnects use only this validated, // concrete address set, so DNS cannot redirect a retry. let target = ResolvedWsTarget::resolve(&config.connect_url)?; @@ -163,6 +186,7 @@ pub fn run_durable_runner( }; state.recoverable_failure = None; store.save(&state)?; + let mut sent_source_seq = state.acked_source_seq; let initial_delivery = (|| -> Result<(), DurableRunnerError> { if let Some(commands) = payload.get("pendingCommands").and_then(Value::as_array) { @@ -184,7 +208,7 @@ pub fn run_durable_runner( } } } - send_outbox(&mut client, &state) + send_outbox(&mut client, &state, &mut sent_source_seq) })(); if let Err(error) = initial_delivery { state.record_diagnostic(error.to_string()); @@ -198,11 +222,13 @@ pub fn run_durable_runner( let disconnected = loop { poll_provider(&mut provider, &mut state, &store, &config)?; if provider.is_some() { - send_outbox(&mut client, &state)?; + send_outbox(&mut client, &state, &mut sent_source_seq)?; } if (state.stop_after_flush || state.lifecycle == "revoked") && state.outbox.is_empty() { state.lifecycle = if state.lifecycle == "revoked" { "revoked".to_owned() + } else if state.lifecycle == "suspending" { + "suspended".to_owned() } else { "stopped".to_owned() }; @@ -282,7 +308,7 @@ pub fn run_durable_runner( Ok(processed) => { let delivery = client .send_json(&command_result_envelope(&state, &processed)) - .and_then(|()| send_outbox(&mut client, &state)); + .and_then(|()| send_outbox(&mut client, &state, &mut sent_source_seq)); if let Err(error) = delivery { state.record_diagnostic(error.to_string()); if state.lifecycle == "revoked" { @@ -327,7 +353,11 @@ pub fn run_durable_runner( "connection lease revoked; flushing durable events", ); store.save(&state)?; - if let Err(error) = send_outbox(&mut client, &state) { + // A revoke may race the ACK for the last delivered event. Replay + // only the still-unacknowledged suffix once as part of the bounded + // revoke drain, rather than on every transport poll. + sent_source_seq = state.acked_source_seq; + if let Err(error) = send_outbox(&mut client, &state, &mut sent_source_seq) { state.record_diagnostic(error.to_string()); store.save(&state)?; connection_lease_token.take(); @@ -383,16 +413,26 @@ pub fn run_durable_runner( } fn start_configured_provider( - state: &DurableRunnerState, + state: &mut DurableRunnerState, ) -> Result, DurableRunnerError> { - let Some(provider_config) = &state.codex_provider_config else { + let Some(provider_config) = state.codex_provider_config.clone() else { return Ok(None); }; + let resume_thread_id = state.codex_provider_thread_id.clone(); + let tools = state + .provider_tool_bridge + .authorized_tools() + .cloned() + .collect::>(); crate::codex_provider::CodexProvider::start( - provider_config, - state.provider_tool_bridge.authorized_tools().cloned(), + &provider_config, + tools.into_iter(), + resume_thread_id.as_deref(), ) - .map(Some) + .map(|provider| { + state.codex_provider_thread_id = Some(provider.thread_id().to_owned()); + Some(provider) + }) .map_err(|error| { DurableRunnerError::invalid(format!("failed to start Codex provider: {error}")) }) @@ -418,6 +458,23 @@ fn process_command_and_provider( Some("run.prepare") => { if provider.is_none() { *provider = start_configured_provider(state)?; + if let Some(runtime) = provider.as_ref() { + enqueue_event( + state, + config, + "provider.started", + 0, + json!({ + "provider": "codex", + "pid": runtime.process_id(), + "threadId": runtime.thread_id(), + "sessionId": runtime.session_id(), + "resumed": state.reconnect_count > 0, + }), + None, + )?; + store.save(state)?; + } } } Some("turn.start") => { @@ -458,6 +515,50 @@ fn process_command_and_provider( )) })?; } + Some("turn.interrupt") => { + let turn_id = command + .pointer("/payload/turnId") + .and_then(Value::as_str) + .ok_or_else(|| { + DurableRunnerError::invalid("turn.interrupt payload.turnId is required") + })?; + provider + .as_mut() + .ok_or_else(|| DurableRunnerError::invalid("turn.interrupt requires a provider"))? + .interrupt_turn(turn_id) + .map_err(|error| { + DurableRunnerError::invalid(format!("Codex turn.interrupt failed: {error}")) + })?; + } + Some("provider.thread.read") => { + let response = provider + .as_mut() + .ok_or_else(|| { + DurableRunnerError::invalid("provider.thread.read requires a provider") + })? + .read_thread() + .map_err(|error| { + DurableRunnerError::invalid(format!("Codex thread/read failed: {error}")) + })?; + enqueue_event( + state, + config, + "provider.rpc_result", + 0, + json!({ "commandId": command_id, "method": "thread/read", "result": response }), + None, + )?; + store.save(state)?; + } + Some("runner.shutdown") | Some("runner.suspend") => { + if let Some(mut runtime) = provider.take() { + runtime.shutdown().map_err(|error| { + DurableRunnerError::invalid(format!( + "failed to stop Codex provider during runner shutdown: {error}" + )) + })?; + } + } _ => {} } Ok(processed) diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/durable/state.rs b/packages/paperclip-runner/runner/crates/runner-core/src/durable/state.rs index 62b0ff9963d..4858db25a82 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/durable/state.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/durable/state.rs @@ -178,6 +178,8 @@ pub struct DurableRunnerState { pub provider_tool_bridge: ProviderToolBridge, #[serde(default)] pub codex_provider_config: Option, + #[serde(default)] + pub codex_provider_thread_id: Option, } impl DurableRunnerState { @@ -209,6 +211,7 @@ impl DurableRunnerState { stop_after_flush: false, provider_tool_bridge: ProviderToolBridge::default(), codex_provider_config: None, + codex_provider_thread_id: None, } } @@ -1324,6 +1327,16 @@ fn execute_command_effect( state.lifecycle = "terminal".to_owned(); Ok(("completed".to_owned(), 1, "turn completed".to_owned())) } + "turn.interrupt" => Ok(( + "completed".to_owned(), + 1, + "provider turn interruption requested".to_owned(), + )), + "provider.thread.read" => Ok(( + "completed".to_owned(), + 1, + "provider thread read requested".to_owned(), + )), "runner.drain" => { state.lifecycle = "draining".to_owned(); enqueue_event( @@ -1352,6 +1365,23 @@ fn execute_command_effect( "runner will stop after durable flush".to_owned(), )) } + "runner.suspend" => { + state.stop_after_flush = true; + state.lifecycle = "suspending".to_owned(); + enqueue_event( + state, + config, + "runner.suspending", + 0, + json!({ "afterDurableFlush": true, "resumable": true }), + None, + )?; + Ok(( + "completed".to_owned(), + 1, + "runner will suspend after durable flush".to_owned(), + )) + } unsupported => Ok(( "rejected".to_owned(), 0, diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/durable/transport.rs b/packages/paperclip-runner/runner/crates/runner-core/src/durable/transport.rs index 96581a84078..716349b2a42 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/durable/transport.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/durable/transport.rs @@ -659,9 +659,17 @@ fn authenticate_transport( Ok(()) } -fn send_outbox(client: &mut WsClient, state: &DurableRunnerState) -> Result<(), DurableRunnerError> { +fn send_outbox( + client: &mut WsClient, + state: &DurableRunnerState, + sent_source_seq: &mut u64, +) -> Result<(), DurableRunnerError> { for event in &state.outbox { + if event.source_seq <= *sent_source_seq { + continue; + } client.send_json(&event.envelope)?; + *sent_source_seq = event.source_seq; } Ok(()) } diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs index e3ac31a0923..f35a9a0ae22 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs @@ -25,6 +25,7 @@ fn codex_dynamic_tool_round_trips_through_the_rust_provider_boundary() { response_schema: json!({"type": "object"}), }] .into_iter(), + None, ) .unwrap(); provider.start_turn("Inspect the task.", "/tmp").unwrap(); diff --git a/packages/paperclip-runner/scripts/capability-issue-thread-server.mjs b/packages/paperclip-runner/scripts/capability-issue-thread-server.mjs index edfdfded8fc..857e79b8609 100644 --- a/packages/paperclip-runner/scripts/capability-issue-thread-server.mjs +++ b/packages/paperclip-runner/scripts/capability-issue-thread-server.mjs @@ -308,6 +308,7 @@ export function createCapabilityIssueThreadMiddleware(options = {}) { taskId: "task-31", actorId: "actor-1", companyId: "company-1", + ...(options.requestedModel === undefined ? {} : { requestedModel: options.requestedModel }), }); const entry = { session, @@ -364,7 +365,10 @@ export function createCapabilityIssueThreadMiddleware(options = {}) { const { identity, input } = runner.createCapabilityCleanRoomSessionInput({ workingDirectory }); let session; try { - session = await service.create(input); + session = await service.create({ + ...input, + ...(options.requestedModel === undefined ? {} : { requestedModel: options.requestedModel }), + }); } catch (error) { await rm(workingDirectory, { recursive: true, force: true }).catch(() => undefined); throw error; @@ -505,6 +509,7 @@ export function createCapabilityIssueThreadMiddleware(options = {}) { }); } catch (error) { finished = true; + options.onTurnError?.(error, entry.session.id, entry.session.snapshot()); // The code identifies the failure; the underlying message stays server // side because it can quote provider text (track 7U). write({ @@ -697,6 +702,9 @@ export function createCapabilityIssueThreadMiddleware(options = {}) { actorId: snapshot.authority.actorId, taskId: snapshot.authority.taskId, turnTimeoutMs: snapshot.config.turnTimeoutMs, + ...(snapshot.config.requestedModel === undefined + ? {} + : { requestedModel: snapshot.config.requestedModel }), }); } catch (error) { await rm(forkDirectory, { recursive: true, force: true }).catch(() => undefined); diff --git a/packages/paperclip-runner/scripts/capability-issue-thread-smoke.mjs b/packages/paperclip-runner/scripts/capability-issue-thread-smoke.mjs index 9379a5217ac..78503e14191 100644 --- a/packages/paperclip-runner/scripts/capability-issue-thread-smoke.mjs +++ b/packages/paperclip-runner/scripts/capability-issue-thread-smoke.mjs @@ -57,7 +57,18 @@ function assert(condition, message) { async function main() { const asJson = process.argv.includes("--json"); - const middleware = createCapabilityIssueThreadMiddleware({ bindHost: "127.0.0.1" }); + let turnError = null; + let turnErrorMethods = []; + const middleware = createCapabilityIssueThreadMiddleware({ + bindHost: "127.0.0.1", + requestedModel: "gpt-5.4-mini", + onTurnError: (error, _sessionId, snapshot) => { + turnError = error; + turnErrorMethods = snapshot.evidence + .filter((entry) => entry.kind === "provider_event") + .map((entry) => entry.data); + }, + }); const server = createServer((request, response) => { middleware(request, response, () => { response.statusCode = 404; @@ -89,6 +100,11 @@ async function main() { sessionId, message: "Call get_task_context, then call report_progress with a one-sentence status. Do not ask me anything.", + }).catch((error) => { + if (turnError instanceof Error) { + throw new Error("live issue-thread turn failed", { cause: turnError }); + } + throw error; }); assertions.turnStreamed = first.frames >= 2; const firstItems = first.view.turns.flatMap((turn) => turn.items); @@ -106,6 +122,11 @@ async function main() { const second = await runTurn(jar, { sessionId, message: "Summarise the mock task in one line.", + }).catch((error) => { + if (turnError instanceof Error) { + throw new Error(`live issue-thread second turn failed; provider methods=${JSON.stringify(turnErrorMethods)}`, { cause: turnError }); + } + throw error; }); assertions.multiTurnThread = second.view.turns.length > first.view.turns.length; assertions.composerReady = second.view.composer.state === "ready"; diff --git a/packages/paperclip-runner/scripts/generate-protocol-coverage.mjs b/packages/paperclip-runner/scripts/generate-protocol-coverage.mjs new file mode 100644 index 00000000000..9358037f5ed --- /dev/null +++ b/packages/paperclip-runner/scripts/generate-protocol-coverage.mjs @@ -0,0 +1,93 @@ +#!/usr/bin/env node +import { readFile, writeFile } from "node:fs/promises"; +import { resolve } from "node:path"; + +import { PAPERCLIP_PROTOCOL_ACTIONS } from "../dist/protocol-actions/index.js"; +import { buildCapabilityScenarioIndex } from "../dist/scenarios/scenario-index.js"; +import { capabilityScenarioFixture } from "../dist/scenarios/scenario-fixtures.js"; +import { capabilityScenarioPlan } from "../dist/scenarios/scenario-plan.js"; + +const root = resolve(import.meta.dirname, ".."); +const manifestPath = resolve(root, "spec/capability/eval-traceability.yaml"); +const outputPath = resolve(root, "spec/capability/protocol-coverage.json"); +const index = buildCapabilityScenarioIndex(await readFile(manifestPath, "utf8")); +const catalog = PAPERCLIP_PROTOCOL_ACTIONS.map((action) => ({ + id: action.id, + ownership: action.canonical.placement, + surfaces: action.canonical.surfaces, + legacyAliases: action.canonical.legacyAliases, +})); + +const requirements = index.entries.map((entry) => { + const plan = capabilityScenarioPlan(entry, capabilityScenarioFixture(entry)); + const operations = new Set(plan.controlPlaneCapabilities.map((item) => item.operationId)); + for (const step of plan.steps) if (step.kind === "tool_call") operations.add(step.operationId); + return { + id: entry.id, + title: entry.title, + fixture: entry.fixture, + operations: [...operations].sort(), + sourceAnchor: entry.sourceAnchor, + assertionClasses: entry.assertionClasses, + deterministicOwner: "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity", + }; +}); + +const lifecycleRequirements = [ + ["provider-round-trip", "provider -> runnerd -> control plane -> runnerd -> provider", "src/live/runnerd-codex-transport.test.ts"], + ["single-daemon", "one Paperclip-owned sandbox daemon and no TypeScript sandbox dispatcher", "src/live/runnerd-codex-transport.test.ts"], + ["catalog-drift", "canonical catalog consistency and drift rejection", "runner/crates/runner-core/src/provider_bridge.rs::durable_session_refuses_catalog_drift"], + ["authorization", "authorization denial and unknown-tool rejection", "runner/crates/runner-core/src/provider_bridge.rs::forwards_only_authorized_calls_and_correlates_results"], + ["duplicates", "correlated duplicate and conflicting-result handling", "runner/crates/runner-core/src/provider_bridge.rs::rejects_unknown_tools_and_conflicting_duplicate_results"], + ["recovery", "lost ACK, reconnect, runner restart, and provider-thread recovery", "src/mock-core/durable-recovery.test.ts; src/live/live-session.test.ts"], + ["interrupt", "interruption while waiting for a tool result", "src/live/live-session.test.ts; src/live/clean-room-server.test.ts"], + ["bounds", "bounded queues, logs, frames, and retries", "src/live/turn-stream.test.ts; runner/crates/runner-core/src/durable/runner.rs"], + ["cleanup", "process-group cleanup with no abandoned provider", "runner/crates/runner-core/tests/process_supervisor.rs"], + ["secrets", "secret isolation and network restrictions", "src/live/clean-room-server.test.ts; runner/crates/runner-core/src/durable/runner.rs"], + ["transcript-accounting", "assistant transcript capture, token usage, and cost inputs", "src/live/live-session.test.ts; src/cli/eval-session.ts"], + ["state-reconstruction", "fixture and post-run state reconstruction", "src/scenarios/scenario-explorer.test.ts; src/live/live-session.test.ts"], +]; + +const actions = catalog.map((action) => ({ + ...action, + contractCase: `protocol-action:${action.id}`, + contractOwner: `src/catalog/protocol-action-contracts.test.ts::${action.id} has a schema-valid canonical example and every declared projection`, + legacyBehavioralCases: requirements.filter((row) => row.operations.includes(action.id)).map((row) => row.id), + deterministicCases: [ + `protocol-action:${action.id}`, + ...requirements.filter((row) => row.operations.includes(action.id)).map((row) => row.id), + ], + legacyRequirementCases: index.entries + .filter((entry) => action.legacyAliases.some((alias) => entry.legacyMcpAliases.includes(alias))) + .map((entry) => entry.id), + deterministicOwners: [ + `src/catalog/protocol-action-contracts.test.ts::${action.id} has a schema-valid canonical example and every declared projection`, + "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity", + ], +})); + +const document = { + schema: "paperclip-runner/protocol-coverage/v1", + generatedFrom: [ + "spec/capability/eval-traceability.yaml", + "src/tools/capability-semantic-tool-catalog.ts", + "src/tools/capability-semantic-tool-types.ts", + "src/scenarios/scenario-plan.ts", + ], + counts: { actions: actions.length, legacyRequirements: requirements.length }, + actions, + discoveryGateways: [ + { id: "discover_capabilities", deterministicOwners: ["src/semantic-tools/semantic-tools.test.ts::discovers only authorized optional tools and returns trusted schemas"] }, + { id: "invoke_discovered_capability", deterministicOwners: ["src/mock-core/durable-recovery.test.ts::repeatedly completes the provider bridge and explicit runner shutdown"] }, + ], + legacyRequirements: requirements, + lifecycleRequirements: lifecycleRequirements.map(([id, requirement, deterministicOwner]) => ({ id, requirement, deterministicOwner })), +}; +const encoded = `${JSON.stringify(document, null, 2)}\n`; +if (process.argv.includes("--check")) { + const current = await readFile(outputPath, "utf8").catch(() => ""); + if (current !== encoded) throw new Error("protocol coverage artifact is stale; run pnpm generate:protocol-coverage"); +} else { + await writeFile(outputPath, encoded); + process.stdout.write(`Wrote ${outputPath}\n`); +} diff --git a/packages/paperclip-runner/spec/capability/protocol-coverage.json b/packages/paperclip-runner/spec/capability/protocol-coverage.json new file mode 100644 index 00000000000..ca7eeac3f25 --- /dev/null +++ b/packages/paperclip-runner/spec/capability/protocol-coverage.json @@ -0,0 +1,3572 @@ +{ + "schema": "paperclip-runner/protocol-coverage/v1", + "generatedFrom": [ + "spec/capability/eval-traceability.yaml", + "src/tools/capability-semantic-tool-catalog.ts", + "src/tools/capability-semantic-tool-types.ts", + "src/scenarios/scenario-plan.ts" + ], + "counts": { + "actions": 41, + "legacyRequirements": 106 + }, + "actions": [ + { + "id": "administer_company", + "ownership": "optional_agent_tool", + "surfaces": [ + "scenario" + ], + "legacyAliases": [], + "contractCase": "protocol-action:administer_company", + "contractOwner": "src/catalog/protocol-action-contracts.test.ts::administer_company has a schema-valid canonical example and every declared projection", + "legacyBehavioralCases": [], + "deterministicCases": [ + "protocol-action:administer_company" + ], + "legacyRequirementCases": [], + "deterministicOwners": [ + "src/catalog/protocol-action-contracts.test.ts::administer_company has a schema-valid canonical example and every declared projection", + "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + ] + }, + { + "id": "answer_status_question", + "ownership": "always_agent_tool", + "surfaces": [ + "scenario", + "live" + ], + "legacyAliases": [], + "contractCase": "protocol-action:answer_status_question", + "contractOwner": "src/catalog/protocol-action-contracts.test.ts::answer_status_question has a schema-valid canonical example and every declared projection", + "legacyBehavioralCases": [ + "rs-dependency-blocked-wake-01", + "rs-question-only-01", + "rs-secret-hygiene-01", + "wk-ask-mode-01" + ], + "deterministicCases": [ + "protocol-action:answer_status_question", + "rs-dependency-blocked-wake-01", + "rs-question-only-01", + "rs-secret-hygiene-01", + "wk-ask-mode-01" + ], + "legacyRequirementCases": [], + "deterministicOwners": [ + "src/catalog/protocol-action-contracts.test.ts::answer_status_question has a schema-valid canonical example and every declared projection", + "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + ] + }, + { + "id": "block_task", + "ownership": "always_agent_tool", + "surfaces": [ + "scenario", + "live" + ], + "legacyAliases": [], + "contractCase": "protocol-action:block_task", + "contractOwner": "src/catalog/protocol-action-contracts.test.ts::block_task has a schema-valid canonical example and every declared projection", + "legacyBehavioralCases": [ + "bl-cancelled-not-resolved-01", + "bl-clear-01", + "bl-create-blocked-01", + "bl-firstclass-01", + "bl-read-owners-01" + ], + "deterministicCases": [ + "protocol-action:block_task", + "bl-cancelled-not-resolved-01", + "bl-clear-01", + "bl-create-blocked-01", + "bl-firstclass-01", + "bl-read-owners-01" + ], + "legacyRequirementCases": [], + "deterministicOwners": [ + "src/catalog/protocol-action-contracts.test.ts::block_task has a schema-valid canonical example and every declared projection", + "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + ] + }, + { + "id": "comment_on_approval", + "ownership": "optional_agent_tool", + "surfaces": [ + "scenario", + "live" + ], + "legacyAliases": [], + "contractCase": "protocol-action:comment_on_approval", + "contractOwner": "src/catalog/protocol-action-contracts.test.ts::comment_on_approval has a schema-valid canonical example and every declared projection", + "legacyBehavioralCases": [ + "ap-approval-deny-01", + "ap-approval-wake-01", + "ap-mcp-expiry-01" + ], + "deterministicCases": [ + "protocol-action:comment_on_approval", + "ap-approval-deny-01", + "ap-approval-wake-01", + "ap-mcp-expiry-01" + ], + "legacyRequirementCases": [], + "deterministicOwners": [ + "src/catalog/protocol-action-contracts.test.ts::comment_on_approval has a schema-valid canonical example and every declared projection", + "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + ] + }, + { + "id": "control_workspace_service", + "ownership": "optional_agent_tool", + "surfaces": [ + "scenario", + "live" + ], + "legacyAliases": [], + "contractCase": "protocol-action:control_workspace_service", + "contractOwner": "src/catalog/protocol-action-contracts.test.ts::control_workspace_service has a schema-valid canonical example and every declared projection", + "legacyBehavioralCases": [], + "deterministicCases": [ + "protocol-action:control_workspace_service" + ], + "legacyRequirementCases": [], + "deterministicOwners": [ + "src/catalog/protocol-action-contracts.test.ts::control_workspace_service has a schema-valid canonical example and every declared projection", + "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + ] + }, + { + "id": "create_task", + "ownership": "optional_agent_tool", + "surfaces": [ + "scenario", + "live" + ], + "legacyAliases": [ + "mcp:paperclipCreateIssue" + ], + "contractCase": "protocol-action:create_task", + "contractOwner": "src/catalog/protocol-action-contracts.test.ts::create_task has a schema-valid canonical example and every declared projection", + "legacyBehavioralCases": [ + "bl-create-blocked-01", + "ix-checkbox-result-01", + "mh-blocked-handoff-01", + "mh-children-complete-01", + "mh-subtask-tree-01", + "su-crossteam-billing-01", + "su-inherit-workspace-01", + "su-no-poll-01", + "su-parent-goal-01" + ], + "deterministicCases": [ + "protocol-action:create_task", + "bl-create-blocked-01", + "ix-checkbox-result-01", + "mh-blocked-handoff-01", + "mh-children-complete-01", + "mh-subtask-tree-01", + "su-crossteam-billing-01", + "su-inherit-workspace-01", + "su-no-poll-01", + "su-parent-goal-01" + ], + "legacyRequirementCases": [ + "su-parent-goal-01" + ], + "deterministicOwners": [ + "src/catalog/protocol-action-contracts.test.ts::create_task has a schema-valid canonical example and every declared projection", + "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + ] + }, + { + "id": "decide_approval", + "ownership": "optional_agent_tool", + "surfaces": [ + "scenario", + "live" + ], + "legacyAliases": [], + "contractCase": "protocol-action:decide_approval", + "contractOwner": "src/catalog/protocol-action-contracts.test.ts::decide_approval has a schema-valid canonical example and every declared projection", + "legacyBehavioralCases": [ + "ap-approval-wake-01" + ], + "deterministicCases": [ + "protocol-action:decide_approval", + "ap-approval-wake-01" + ], + "legacyRequirementCases": [], + "deterministicOwners": [ + "src/catalog/protocol-action-contracts.test.ts::decide_approval has a schema-valid canonical example and every declared projection", + "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + ] + }, + { + "id": "export_company", + "ownership": "optional_agent_tool", + "surfaces": [ + "scenario" + ], + "legacyAliases": [], + "contractCase": "protocol-action:export_company", + "contractOwner": "src/catalog/protocol-action-contracts.test.ts::export_company has a schema-valid canonical example and every declared projection", + "legacyBehavioralCases": [ + "rf-wf-export-preview-01", + "rf-wf-invite-01", + "rf-wf-project-setup-01" + ], + "deterministicCases": [ + "protocol-action:export_company", + "rf-wf-export-preview-01", + "rf-wf-invite-01", + "rf-wf-project-setup-01" + ], + "legacyRequirementCases": [], + "deterministicOwners": [ + "src/catalog/protocol-action-contracts.test.ts::export_company has a schema-valid canonical example and every declared projection", + "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + ] + }, + { + "id": "finish_task", + "ownership": "always_agent_tool", + "surfaces": [ + "scenario", + "live" + ], + "legacyAliases": [], + "contractCase": "protocol-action:finish_task", + "contractOwner": "src/catalog/protocol-action-contracts.test.ts::finish_task has a schema-valid canonical example and every declared projection", + "legacyBehavioralCases": [ + "ar-upload-before-done-01", + "st-backlog-park-01", + "st-blocked-owner-01", + "st-crossteam-cancel-01", + "st-done-comment-01", + "st-env-blocked-notdone-01", + "st-ephemeral-verify-01", + "st-handback-review-01", + "st-unverified-toolchain-01" + ], + "deterministicCases": [ + "protocol-action:finish_task", + "ar-upload-before-done-01", + "st-backlog-park-01", + "st-blocked-owner-01", + "st-crossteam-cancel-01", + "st-done-comment-01", + "st-env-blocked-notdone-01", + "st-ephemeral-verify-01", + "st-handback-review-01", + "st-unverified-toolchain-01" + ], + "legacyRequirementCases": [], + "deterministicOwners": [ + "src/catalog/protocol-action-contracts.test.ts::finish_task has a schema-valid canonical example and every declared projection", + "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + ] + }, + { + "id": "generic_api_request", + "ownership": "optional_agent_tool", + "surfaces": [ + "scenario", + "live" + ], + "legacyAliases": [], + "contractCase": "protocol-action:generic_api_request", + "contractOwner": "src/catalog/protocol-action-contracts.test.ts::generic_api_request has a schema-valid canonical example and every declared projection", + "legacyBehavioralCases": [ + "wk-skilltest-mode-01" + ], + "deterministicCases": [ + "protocol-action:generic_api_request", + "wk-skilltest-mode-01" + ], + "legacyRequirementCases": [], + "deterministicOwners": [ + "src/catalog/protocol-action-contracts.test.ts::generic_api_request has a schema-valid canonical example and every declared projection", + "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + ] + }, + { + "id": "get_agent", + "ownership": "optional_agent_tool", + "surfaces": [ + "live" + ], + "legacyAliases": [ + "mcp:paperclipGetAgent" + ], + "contractCase": "protocol-action:get_agent", + "contractOwner": "src/catalog/protocol-action-contracts.test.ts::get_agent has a schema-valid canonical example and every declared projection", + "legacyBehavioralCases": [], + "deterministicCases": [ + "protocol-action:get_agent" + ], + "legacyRequirementCases": [ + "rf-api-mgr-heartbeat-01" + ], + "deterministicOwners": [ + "src/catalog/protocol-action-contracts.test.ts::get_agent has a schema-valid canonical example and every declared projection", + "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + ] + }, + { + "id": "get_approval", + "ownership": "optional_agent_tool", + "surfaces": [ + "live" + ], + "legacyAliases": [], + "contractCase": "protocol-action:get_approval", + "contractOwner": "src/catalog/protocol-action-contracts.test.ts::get_approval has a schema-valid canonical example and every declared projection", + "legacyBehavioralCases": [], + "deterministicCases": [ + "protocol-action:get_approval" + ], + "legacyRequirementCases": [], + "deterministicOwners": [ + "src/catalog/protocol-action-contracts.test.ts::get_approval has a schema-valid canonical example and every declared projection", + "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + ] + }, + { + "id": "get_approval_context", + "ownership": "optional_agent_tool", + "surfaces": [ + "live" + ], + "legacyAliases": [], + "contractCase": "protocol-action:get_approval_context", + "contractOwner": "src/catalog/protocol-action-contracts.test.ts::get_approval_context has a schema-valid canonical example and every declared projection", + "legacyBehavioralCases": [], + "deterministicCases": [ + "protocol-action:get_approval_context" + ], + "legacyRequirementCases": [], + "deterministicOwners": [ + "src/catalog/protocol-action-contracts.test.ts::get_approval_context has a schema-valid canonical example and every declared projection", + "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + ] + }, + { + "id": "get_task_context", + "ownership": "always_agent_tool", + "surfaces": [ + "scenario", + "live" + ], + "legacyAliases": [ + "mcp:paperclipMe" + ], + "contractCase": "protocol-action:get_task_context", + "contractOwner": "src/catalog/protocol-action-contracts.test.ts::get_task_context has a schema-valid canonical example and every declared projection", + "legacyBehavioralCases": [ + "ap-approval-deny-01", + "ap-approval-wake-01", + "ap-board-approval-01", + "ap-mcp-expiry-01", + "ap-mcp-gate-01", + "ap-mcp-pathmissing-01", + "ar-no-done-without-upload-01", + "ar-upload-before-done-01", + "ar-workproduct-pr-01", + "ar-workproduct-wsfile-01", + "bl-cancelled-not-resolved-01", + "bl-clear-01", + "bl-create-blocked-01", + "bl-firstclass-01", + "bl-read-owners-01", + "cm-mention-structured-01", + "cm-multiline-01", + "cm-prefixed-url-01", + "cm-progress-nextaction-01", + "cm-runid-modify-01", + "cm-ticket-link-01", + "co-409-stop-01", + "co-409-stop-02", + "co-before-work-01", + "co-body-contract-01", + "co-no-status-patch-01", + "co-runid-header-01", + "dp-base-revision-01", + "dp-plan-doc-01", + "dp-plan-link-comment-01", + "er-blocked-dedup-01", + "er-budget-critical-01", + "er-close-retry-fault-01", + "er-exec-not-participant-01", + "er-exec-participant-01", + "er-mention-handoff-01", + "er-mention-noassign-01", + "er-no-unassigned-01", + "er-release-01", + "hb-context-01", + "hb-inbox-lite-01", + "hb-pick-priority-01", + "hb-scoped-wake-01", + "hb-wake-comment-01", + "ix-checkbox-01", + "ix-checkbox-result-01", + "ix-confirmation-plan-01", + "ix-continuation-01", + "ix-questions-01", + "ix-stale-target-01", + "ix-suggest-tasks-01", + "ix-superseded-comment-01", + "ix-verdicts-01", + "mh-blocked-handoff-01", + "mh-children-complete-01", + "mh-plan-confirm-01", + "mh-subtask-tree-01", + "rf-api-404-report-01", + "rf-api-cancel-obsolete-01", + "rf-api-mention-discipline-01", + "rf-api-mgr-heartbeat-01", + "rf-api-review-changes-01", + "rf-art-attachment-wp-01", + "rf-case-child-01", + "rf-case-lifecycle-link-01", + "rf-case-upsert-doc-01", + "rf-cskill-audit-01", + "rf-cskill-install-attach-01", + "rf-cskill-self-sync-01", + "rf-iws-start-url-01", + "rf-iws-target-restart-01", + "rf-routine-create-01", + "rf-routine-manual-run-01", + "rf-routine-pause-01", + "rf-routine-policy-01", + "rf-routine-webhook-01", + "rf-wf-export-preview-01", + "rf-wf-invite-01", + "rf-wf-project-setup-01", + "rs-dependency-blocked-wake-01", + "rs-question-only-01", + "rs-secret-hygiene-01", + "se-get-issue-01", + "se-q-comments-01", + "se-q-filters-01", + "se-q-topic-01", + "st-backlog-park-01", + "st-blocked-owner-01", + "st-crossteam-cancel-01", + "st-done-comment-01", + "st-env-blocked-notdone-01", + "st-ephemeral-verify-01", + "st-handback-review-01", + "st-unverified-toolchain-01", + "su-crossteam-billing-01", + "su-inherit-workspace-01", + "su-no-poll-01", + "su-parent-goal-01", + "wk-ask-mode-01", + "wk-plan-accepted-01", + "wk-plan-annotation-01", + "wk-plan-directive-01", + "wk-plan-directive-02", + "wk-recovery-processlost-01", + "wk-resume-delta-01", + "wk-skilltest-mode-01" + ], + "deterministicCases": [ + "protocol-action:get_task_context", + "ap-approval-deny-01", + "ap-approval-wake-01", + "ap-board-approval-01", + "ap-mcp-expiry-01", + "ap-mcp-gate-01", + "ap-mcp-pathmissing-01", + "ar-no-done-without-upload-01", + "ar-upload-before-done-01", + "ar-workproduct-pr-01", + "ar-workproduct-wsfile-01", + "bl-cancelled-not-resolved-01", + "bl-clear-01", + "bl-create-blocked-01", + "bl-firstclass-01", + "bl-read-owners-01", + "cm-mention-structured-01", + "cm-multiline-01", + "cm-prefixed-url-01", + "cm-progress-nextaction-01", + "cm-runid-modify-01", + "cm-ticket-link-01", + "co-409-stop-01", + "co-409-stop-02", + "co-before-work-01", + "co-body-contract-01", + "co-no-status-patch-01", + "co-runid-header-01", + "dp-base-revision-01", + "dp-plan-doc-01", + "dp-plan-link-comment-01", + "er-blocked-dedup-01", + "er-budget-critical-01", + "er-close-retry-fault-01", + "er-exec-not-participant-01", + "er-exec-participant-01", + "er-mention-handoff-01", + "er-mention-noassign-01", + "er-no-unassigned-01", + "er-release-01", + "hb-context-01", + "hb-inbox-lite-01", + "hb-pick-priority-01", + "hb-scoped-wake-01", + "hb-wake-comment-01", + "ix-checkbox-01", + "ix-checkbox-result-01", + "ix-confirmation-plan-01", + "ix-continuation-01", + "ix-questions-01", + "ix-stale-target-01", + "ix-suggest-tasks-01", + "ix-superseded-comment-01", + "ix-verdicts-01", + "mh-blocked-handoff-01", + "mh-children-complete-01", + "mh-plan-confirm-01", + "mh-subtask-tree-01", + "rf-api-404-report-01", + "rf-api-cancel-obsolete-01", + "rf-api-mention-discipline-01", + "rf-api-mgr-heartbeat-01", + "rf-api-review-changes-01", + "rf-art-attachment-wp-01", + "rf-case-child-01", + "rf-case-lifecycle-link-01", + "rf-case-upsert-doc-01", + "rf-cskill-audit-01", + "rf-cskill-install-attach-01", + "rf-cskill-self-sync-01", + "rf-iws-start-url-01", + "rf-iws-target-restart-01", + "rf-routine-create-01", + "rf-routine-manual-run-01", + "rf-routine-pause-01", + "rf-routine-policy-01", + "rf-routine-webhook-01", + "rf-wf-export-preview-01", + "rf-wf-invite-01", + "rf-wf-project-setup-01", + "rs-dependency-blocked-wake-01", + "rs-question-only-01", + "rs-secret-hygiene-01", + "se-get-issue-01", + "se-q-comments-01", + "se-q-filters-01", + "se-q-topic-01", + "st-backlog-park-01", + "st-blocked-owner-01", + "st-crossteam-cancel-01", + "st-done-comment-01", + "st-env-blocked-notdone-01", + "st-ephemeral-verify-01", + "st-handback-review-01", + "st-unverified-toolchain-01", + "su-crossteam-billing-01", + "su-inherit-workspace-01", + "su-no-poll-01", + "su-parent-goal-01", + "wk-ask-mode-01", + "wk-plan-accepted-01", + "wk-plan-annotation-01", + "wk-plan-directive-01", + "wk-plan-directive-02", + "wk-recovery-processlost-01", + "wk-resume-delta-01", + "wk-skilltest-mode-01" + ], + "legacyRequirementCases": [ + "hb-inbox-lite-01" + ], + "deterministicOwners": [ + "src/catalog/protocol-action-contracts.test.ts::get_task_context has a schema-valid canonical example and every declared projection", + "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + ] + }, + { + "id": "get_task_history", + "ownership": "always_agent_tool", + "surfaces": [ + "scenario", + "live" + ], + "legacyAliases": [], + "contractCase": "protocol-action:get_task_history", + "contractOwner": "src/catalog/protocol-action-contracts.test.ts::get_task_history has a schema-valid canonical example and every declared projection", + "legacyBehavioralCases": [ + "rs-dependency-blocked-wake-01", + "rs-question-only-01", + "wk-ask-mode-01" + ], + "deterministicCases": [ + "protocol-action:get_task_history", + "rs-dependency-blocked-wake-01", + "rs-question-only-01", + "wk-ask-mode-01" + ], + "legacyRequirementCases": [], + "deterministicOwners": [ + "src/catalog/protocol-action-contracts.test.ts::get_task_history has a schema-valid canonical example and every declared projection", + "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + ] + }, + { + "id": "get_workspace_runtime", + "ownership": "optional_agent_tool", + "surfaces": [ + "scenario", + "live" + ], + "legacyAliases": [], + "contractCase": "protocol-action:get_workspace_runtime", + "contractOwner": "src/catalog/protocol-action-contracts.test.ts::get_workspace_runtime has a schema-valid canonical example and every declared projection", + "legacyBehavioralCases": [ + "rf-iws-start-url-01", + "rf-iws-target-restart-01" + ], + "deterministicCases": [ + "protocol-action:get_workspace_runtime", + "rf-iws-start-url-01", + "rf-iws-target-restart-01" + ], + "legacyRequirementCases": [], + "deterministicOwners": [ + "src/catalog/protocol-action-contracts.test.ts::get_workspace_runtime has a schema-valid canonical example and every declared projection", + "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + ] + }, + { + "id": "inspect_operation_result", + "ownership": "always_agent_tool", + "surfaces": [ + "scenario" + ], + "legacyAliases": [], + "contractCase": "protocol-action:inspect_operation_result", + "contractOwner": "src/catalog/protocol-action-contracts.test.ts::inspect_operation_result has a schema-valid canonical example and every declared projection", + "legacyBehavioralCases": [ + "ix-checkbox-result-01" + ], + "deterministicCases": [ + "protocol-action:inspect_operation_result", + "ix-checkbox-result-01" + ], + "legacyRequirementCases": [], + "deterministicOwners": [ + "src/catalog/protocol-action-contracts.test.ts::inspect_operation_result has a schema-valid canonical example and every declared projection", + "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + ] + }, + { + "id": "list_agents", + "ownership": "optional_agent_tool", + "surfaces": [ + "scenario", + "live" + ], + "legacyAliases": [ + "mcp:paperclipListAgents" + ], + "contractCase": "protocol-action:list_agents", + "contractOwner": "src/catalog/protocol-action-contracts.test.ts::list_agents has a schema-valid canonical example and every declared projection", + "legacyBehavioralCases": [ + "rf-api-404-report-01", + "rf-api-cancel-obsolete-01", + "rf-api-mention-discipline-01", + "rf-api-mgr-heartbeat-01", + "rf-api-review-changes-01", + "se-get-issue-01", + "se-q-comments-01", + "se-q-filters-01", + "se-q-topic-01" + ], + "deterministicCases": [ + "protocol-action:list_agents", + "rf-api-404-report-01", + "rf-api-cancel-obsolete-01", + "rf-api-mention-discipline-01", + "rf-api-mgr-heartbeat-01", + "rf-api-review-changes-01", + "se-get-issue-01", + "se-q-comments-01", + "se-q-filters-01", + "se-q-topic-01" + ], + "legacyRequirementCases": [ + "rf-api-mgr-heartbeat-01" + ], + "deterministicOwners": [ + "src/catalog/protocol-action-contracts.test.ts::list_agents has a schema-valid canonical example and every declared projection", + "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + ] + }, + { + "id": "list_approvals", + "ownership": "optional_agent_tool", + "surfaces": [ + "scenario", + "live" + ], + "legacyAliases": [], + "contractCase": "protocol-action:list_approvals", + "contractOwner": "src/catalog/protocol-action-contracts.test.ts::list_approvals has a schema-valid canonical example and every declared projection", + "legacyBehavioralCases": [ + "ap-approval-deny-01", + "ap-approval-wake-01", + "ap-board-approval-01", + "ap-mcp-expiry-01", + "ap-mcp-gate-01", + "ap-mcp-pathmissing-01" + ], + "deterministicCases": [ + "protocol-action:list_approvals", + "ap-approval-deny-01", + "ap-approval-wake-01", + "ap-board-approval-01", + "ap-mcp-expiry-01", + "ap-mcp-gate-01", + "ap-mcp-pathmissing-01" + ], + "legacyRequirementCases": [], + "deterministicOwners": [ + "src/catalog/protocol-action-contracts.test.ts::list_approvals has a schema-valid canonical example and every declared projection", + "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + ] + }, + { + "id": "list_cases", + "ownership": "optional_agent_tool", + "surfaces": [ + "scenario" + ], + "legacyAliases": [], + "contractCase": "protocol-action:list_cases", + "contractOwner": "src/catalog/protocol-action-contracts.test.ts::list_cases has a schema-valid canonical example and every declared projection", + "legacyBehavioralCases": [ + "rf-case-child-01", + "rf-case-lifecycle-link-01", + "rf-case-upsert-doc-01" + ], + "deterministicCases": [ + "protocol-action:list_cases", + "rf-case-child-01", + "rf-case-lifecycle-link-01", + "rf-case-upsert-doc-01" + ], + "legacyRequirementCases": [], + "deterministicOwners": [ + "src/catalog/protocol-action-contracts.test.ts::list_cases has a schema-valid canonical example and every declared projection", + "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + ] + }, + { + "id": "list_company_skills", + "ownership": "optional_agent_tool", + "surfaces": [ + "scenario" + ], + "legacyAliases": [], + "contractCase": "protocol-action:list_company_skills", + "contractOwner": "src/catalog/protocol-action-contracts.test.ts::list_company_skills has a schema-valid canonical example and every declared projection", + "legacyBehavioralCases": [ + "rf-cskill-audit-01", + "rf-cskill-install-attach-01", + "rf-cskill-self-sync-01" + ], + "deterministicCases": [ + "protocol-action:list_company_skills", + "rf-cskill-audit-01", + "rf-cskill-install-attach-01", + "rf-cskill-self-sync-01" + ], + "legacyRequirementCases": [], + "deterministicOwners": [ + "src/catalog/protocol-action-contracts.test.ts::list_company_skills has a schema-valid canonical example and every declared projection", + "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + ] + }, + { + "id": "list_document_revisions", + "ownership": "always_agent_tool", + "surfaces": [ + "scenario", + "live" + ], + "legacyAliases": [], + "contractCase": "protocol-action:list_document_revisions", + "contractOwner": "src/catalog/protocol-action-contracts.test.ts::list_document_revisions has a schema-valid canonical example and every declared projection", + "legacyBehavioralCases": [], + "deterministicCases": [ + "protocol-action:list_document_revisions" + ], + "legacyRequirementCases": [], + "deterministicOwners": [ + "src/catalog/protocol-action-contracts.test.ts::list_document_revisions has a schema-valid canonical example and every declared projection", + "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + ] + }, + { + "id": "list_documents", + "ownership": "always_agent_tool", + "surfaces": [ + "scenario", + "live" + ], + "legacyAliases": [], + "contractCase": "protocol-action:list_documents", + "contractOwner": "src/catalog/protocol-action-contracts.test.ts::list_documents has a schema-valid canonical example and every declared projection", + "legacyBehavioralCases": [ + "dp-base-revision-01", + "dp-plan-doc-01", + "dp-plan-link-comment-01", + "wk-plan-accepted-01", + "wk-plan-annotation-01", + "wk-plan-directive-01", + "wk-plan-directive-02", + "wk-recovery-processlost-01", + "wk-resume-delta-01", + "wk-skilltest-mode-01" + ], + "deterministicCases": [ + "protocol-action:list_documents", + "dp-base-revision-01", + "dp-plan-doc-01", + "dp-plan-link-comment-01", + "wk-plan-accepted-01", + "wk-plan-annotation-01", + "wk-plan-directive-01", + "wk-plan-directive-02", + "wk-recovery-processlost-01", + "wk-resume-delta-01", + "wk-skilltest-mode-01" + ], + "legacyRequirementCases": [], + "deterministicOwners": [ + "src/catalog/protocol-action-contracts.test.ts::list_documents has a schema-valid canonical example and every declared projection", + "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + ] + }, + { + "id": "list_goals", + "ownership": "optional_agent_tool", + "surfaces": [ + "scenario" + ], + "legacyAliases": [], + "contractCase": "protocol-action:list_goals", + "contractOwner": "src/catalog/protocol-action-contracts.test.ts::list_goals has a schema-valid canonical example and every declared projection", + "legacyBehavioralCases": [], + "deterministicCases": [ + "protocol-action:list_goals" + ], + "legacyRequirementCases": [], + "deterministicOwners": [ + "src/catalog/protocol-action-contracts.test.ts::list_goals has a schema-valid canonical example and every declared projection", + "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + ] + }, + { + "id": "list_projects", + "ownership": "optional_agent_tool", + "surfaces": [ + "scenario" + ], + "legacyAliases": [], + "contractCase": "protocol-action:list_projects", + "contractOwner": "src/catalog/protocol-action-contracts.test.ts::list_projects has a schema-valid canonical example and every declared projection", + "legacyBehavioralCases": [], + "deterministicCases": [ + "protocol-action:list_projects" + ], + "legacyRequirementCases": [], + "deterministicOwners": [ + "src/catalog/protocol-action-contracts.test.ts::list_projects has a schema-valid canonical example and every declared projection", + "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + ] + }, + { + "id": "list_routines", + "ownership": "optional_agent_tool", + "surfaces": [ + "scenario" + ], + "legacyAliases": [], + "contractCase": "protocol-action:list_routines", + "contractOwner": "src/catalog/protocol-action-contracts.test.ts::list_routines has a schema-valid canonical example and every declared projection", + "legacyBehavioralCases": [ + "rf-routine-create-01", + "rf-routine-manual-run-01", + "rf-routine-pause-01", + "rf-routine-policy-01", + "rf-routine-webhook-01" + ], + "deterministicCases": [ + "protocol-action:list_routines", + "rf-routine-create-01", + "rf-routine-manual-run-01", + "rf-routine-pause-01", + "rf-routine-policy-01", + "rf-routine-webhook-01" + ], + "legacyRequirementCases": [], + "deterministicOwners": [ + "src/catalog/protocol-action-contracts.test.ts::list_routines has a schema-valid canonical example and every declared projection", + "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + ] + }, + { + "id": "list_secret_metadata", + "ownership": "optional_agent_tool", + "surfaces": [ + "scenario" + ], + "legacyAliases": [], + "contractCase": "protocol-action:list_secret_metadata", + "contractOwner": "src/catalog/protocol-action-contracts.test.ts::list_secret_metadata has a schema-valid canonical example and every declared projection", + "legacyBehavioralCases": [ + "rs-secret-hygiene-01" + ], + "deterministicCases": [ + "protocol-action:list_secret_metadata", + "rs-secret-hygiene-01" + ], + "legacyRequirementCases": [], + "deterministicOwners": [ + "src/catalog/protocol-action-contracts.test.ts::list_secret_metadata has a schema-valid canonical example and every declared projection", + "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + ] + }, + { + "id": "manage_routine", + "ownership": "optional_agent_tool", + "surfaces": [ + "scenario" + ], + "legacyAliases": [], + "contractCase": "protocol-action:manage_routine", + "contractOwner": "src/catalog/protocol-action-contracts.test.ts::manage_routine has a schema-valid canonical example and every declared projection", + "legacyBehavioralCases": [], + "deterministicCases": [ + "protocol-action:manage_routine" + ], + "legacyRequirementCases": [], + "deterministicOwners": [ + "src/catalog/protocol-action-contracts.test.ts::manage_routine has a schema-valid canonical example and every declared projection", + "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + ] + }, + { + "id": "read_document", + "ownership": "always_agent_tool", + "surfaces": [ + "scenario", + "live" + ], + "legacyAliases": [], + "contractCase": "protocol-action:read_document", + "contractOwner": "src/catalog/protocol-action-contracts.test.ts::read_document has a schema-valid canonical example and every declared projection", + "legacyBehavioralCases": [], + "deterministicCases": [ + "protocol-action:read_document" + ], + "legacyRequirementCases": [], + "deterministicOwners": [ + "src/catalog/protocol-action-contracts.test.ts::read_document has a schema-valid canonical example and every declared projection", + "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + ] + }, + { + "id": "read_secret_value", + "ownership": "optional_agent_tool", + "surfaces": [ + "scenario" + ], + "legacyAliases": [], + "contractCase": "protocol-action:read_secret_value", + "contractOwner": "src/catalog/protocol-action-contracts.test.ts::read_secret_value has a schema-valid canonical example and every declared projection", + "legacyBehavioralCases": [ + "rs-secret-hygiene-01" + ], + "deterministicCases": [ + "protocol-action:read_secret_value", + "rs-secret-hygiene-01" + ], + "legacyRequirementCases": [], + "deterministicOwners": [ + "src/catalog/protocol-action-contracts.test.ts::read_secret_value has a schema-valid canonical example and every declared projection", + "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + ] + }, + { + "id": "register_deliverable", + "ownership": "always_agent_tool", + "surfaces": [ + "scenario", + "live" + ], + "legacyAliases": [], + "contractCase": "protocol-action:register_deliverable", + "contractOwner": "src/catalog/protocol-action-contracts.test.ts::register_deliverable has a schema-valid canonical example and every declared projection", + "legacyBehavioralCases": [ + "ar-no-done-without-upload-01", + "ar-upload-before-done-01", + "ar-workproduct-pr-01", + "ar-workproduct-wsfile-01", + "rf-art-attachment-wp-01" + ], + "deterministicCases": [ + "protocol-action:register_deliverable", + "ar-no-done-without-upload-01", + "ar-upload-before-done-01", + "ar-workproduct-pr-01", + "ar-workproduct-wsfile-01", + "rf-art-attachment-wp-01" + ], + "legacyRequirementCases": [], + "deterministicOwners": [ + "src/catalog/protocol-action-contracts.test.ts::register_deliverable has a schema-valid canonical example and every declared projection", + "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + ] + }, + { + "id": "report_progress", + "ownership": "always_agent_tool", + "surfaces": [ + "scenario", + "live" + ], + "legacyAliases": [ + "mcp:paperclipAddComment" + ], + "contractCase": "protocol-action:report_progress", + "contractOwner": "src/catalog/protocol-action-contracts.test.ts::report_progress has a schema-valid canonical example and every declared projection", + "legacyBehavioralCases": [ + "cm-mention-structured-01", + "cm-multiline-01", + "cm-prefixed-url-01", + "cm-progress-nextaction-01", + "cm-runid-modify-01", + "cm-ticket-link-01", + "co-409-stop-01", + "co-409-stop-02", + "co-before-work-01", + "co-body-contract-01", + "co-no-status-patch-01", + "co-runid-header-01", + "er-blocked-dedup-01", + "er-budget-critical-01", + "er-close-retry-fault-01", + "er-exec-not-participant-01", + "er-exec-participant-01", + "er-mention-handoff-01", + "er-mention-noassign-01", + "er-no-unassigned-01", + "er-release-01", + "hb-context-01", + "hb-inbox-lite-01", + "hb-pick-priority-01", + "hb-scoped-wake-01", + "hb-wake-comment-01", + "mh-blocked-handoff-01", + "mh-children-complete-01", + "mh-plan-confirm-01", + "mh-subtask-tree-01", + "rf-api-404-report-01", + "rf-api-cancel-obsolete-01", + "rf-api-mention-discipline-01", + "rf-api-mgr-heartbeat-01", + "rf-api-review-changes-01", + "st-backlog-park-01", + "st-blocked-owner-01", + "st-crossteam-cancel-01", + "st-done-comment-01", + "st-env-blocked-notdone-01", + "st-ephemeral-verify-01", + "st-handback-review-01", + "st-unverified-toolchain-01", + "su-crossteam-billing-01", + "su-inherit-workspace-01", + "su-no-poll-01", + "su-parent-goal-01" + ], + "deterministicCases": [ + "protocol-action:report_progress", + "cm-mention-structured-01", + "cm-multiline-01", + "cm-prefixed-url-01", + "cm-progress-nextaction-01", + "cm-runid-modify-01", + "cm-ticket-link-01", + "co-409-stop-01", + "co-409-stop-02", + "co-before-work-01", + "co-body-contract-01", + "co-no-status-patch-01", + "co-runid-header-01", + "er-blocked-dedup-01", + "er-budget-critical-01", + "er-close-retry-fault-01", + "er-exec-not-participant-01", + "er-exec-participant-01", + "er-mention-handoff-01", + "er-mention-noassign-01", + "er-no-unassigned-01", + "er-release-01", + "hb-context-01", + "hb-inbox-lite-01", + "hb-pick-priority-01", + "hb-scoped-wake-01", + "hb-wake-comment-01", + "mh-blocked-handoff-01", + "mh-children-complete-01", + "mh-plan-confirm-01", + "mh-subtask-tree-01", + "rf-api-404-report-01", + "rf-api-cancel-obsolete-01", + "rf-api-mention-discipline-01", + "rf-api-mgr-heartbeat-01", + "rf-api-review-changes-01", + "st-backlog-park-01", + "st-blocked-owner-01", + "st-crossteam-cancel-01", + "st-done-comment-01", + "st-env-blocked-notdone-01", + "st-ephemeral-verify-01", + "st-handback-review-01", + "st-unverified-toolchain-01", + "su-crossteam-billing-01", + "su-inherit-workspace-01", + "su-no-poll-01", + "su-parent-goal-01" + ], + "legacyRequirementCases": [ + "cm-multiline-01" + ], + "deterministicOwners": [ + "src/catalog/protocol-action-contracts.test.ts::report_progress has a schema-valid canonical example and every declared projection", + "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + ] + }, + { + "id": "request_approval", + "ownership": "optional_agent_tool", + "surfaces": [ + "scenario", + "live" + ], + "legacyAliases": [], + "contractCase": "protocol-action:request_approval", + "contractOwner": "src/catalog/protocol-action-contracts.test.ts::request_approval has a schema-valid canonical example and every declared projection", + "legacyBehavioralCases": [ + "ap-board-approval-01", + "ap-mcp-gate-01" + ], + "deterministicCases": [ + "protocol-action:request_approval", + "ap-board-approval-01", + "ap-mcp-gate-01" + ], + "legacyRequirementCases": [], + "deterministicOwners": [ + "src/catalog/protocol-action-contracts.test.ts::request_approval has a schema-valid canonical example and every declared projection", + "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + ] + }, + { + "id": "request_human_input", + "ownership": "always_agent_tool", + "surfaces": [ + "scenario", + "live" + ], + "legacyAliases": [], + "contractCase": "protocol-action:request_human_input", + "contractOwner": "src/catalog/protocol-action-contracts.test.ts::request_human_input has a schema-valid canonical example and every declared projection", + "legacyBehavioralCases": [ + "ix-checkbox-01", + "ix-confirmation-plan-01", + "ix-continuation-01", + "ix-questions-01", + "ix-stale-target-01", + "ix-suggest-tasks-01", + "ix-superseded-comment-01", + "ix-verdicts-01", + "wk-plan-directive-01" + ], + "deterministicCases": [ + "protocol-action:request_human_input", + "ix-checkbox-01", + "ix-confirmation-plan-01", + "ix-continuation-01", + "ix-questions-01", + "ix-stale-target-01", + "ix-suggest-tasks-01", + "ix-superseded-comment-01", + "ix-verdicts-01", + "wk-plan-directive-01" + ], + "legacyRequirementCases": [], + "deterministicOwners": [ + "src/catalog/protocol-action-contracts.test.ts::request_human_input has a schema-valid canonical example and every declared projection", + "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + ] + }, + { + "id": "request_review", + "ownership": "always_agent_tool", + "surfaces": [ + "scenario", + "live" + ], + "legacyAliases": [], + "contractCase": "protocol-action:request_review", + "contractOwner": "src/catalog/protocol-action-contracts.test.ts::request_review has a schema-valid canonical example and every declared projection", + "legacyBehavioralCases": [ + "ap-mcp-gate-01", + "dp-plan-doc-01", + "dp-plan-link-comment-01", + "wk-plan-directive-01", + "wk-recovery-processlost-01", + "wk-resume-delta-01", + "wk-skilltest-mode-01" + ], + "deterministicCases": [ + "protocol-action:request_review", + "ap-mcp-gate-01", + "dp-plan-doc-01", + "dp-plan-link-comment-01", + "wk-plan-directive-01", + "wk-recovery-processlost-01", + "wk-resume-delta-01", + "wk-skilltest-mode-01" + ], + "legacyRequirementCases": [], + "deterministicOwners": [ + "src/catalog/protocol-action-contracts.test.ts::request_review has a schema-valid canonical example and every declared projection", + "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + ] + }, + { + "id": "schedule_wake", + "ownership": "optional_agent_tool", + "surfaces": [ + "live" + ], + "legacyAliases": [], + "contractCase": "protocol-action:schedule_wake", + "contractOwner": "src/catalog/protocol-action-contracts.test.ts::schedule_wake has a schema-valid canonical example and every declared projection", + "legacyBehavioralCases": [], + "deterministicCases": [ + "protocol-action:schedule_wake" + ], + "legacyRequirementCases": [], + "deterministicOwners": [ + "src/catalog/protocol-action-contracts.test.ts::schedule_wake has a schema-valid canonical example and every declared projection", + "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + ] + }, + { + "id": "search_tasks", + "ownership": "optional_agent_tool", + "surfaces": [ + "scenario", + "live" + ], + "legacyAliases": [ + "mcp:paperclipListIssues" + ], + "contractCase": "protocol-action:search_tasks", + "contractOwner": "src/catalog/protocol-action-contracts.test.ts::search_tasks has a schema-valid canonical example and every declared projection", + "legacyBehavioralCases": [ + "rf-api-mgr-heartbeat-01", + "se-get-issue-01", + "se-q-comments-01", + "se-q-filters-01", + "se-q-topic-01" + ], + "deterministicCases": [ + "protocol-action:search_tasks", + "rf-api-mgr-heartbeat-01", + "se-get-issue-01", + "se-q-comments-01", + "se-q-filters-01", + "se-q-topic-01" + ], + "legacyRequirementCases": [ + "se-q-filters-01" + ], + "deterministicOwners": [ + "src/catalog/protocol-action-contracts.test.ts::search_tasks has a schema-valid canonical example and every declared projection", + "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + ] + }, + { + "id": "set_dependencies", + "ownership": "optional_agent_tool", + "surfaces": [ + "scenario", + "live" + ], + "legacyAliases": [], + "contractCase": "protocol-action:set_dependencies", + "contractOwner": "src/catalog/protocol-action-contracts.test.ts::set_dependencies has a schema-valid canonical example and every declared projection", + "legacyBehavioralCases": [ + "mh-subtask-tree-01" + ], + "deterministicCases": [ + "protocol-action:set_dependencies", + "mh-subtask-tree-01" + ], + "legacyRequirementCases": [], + "deterministicOwners": [ + "src/catalog/protocol-action-contracts.test.ts::set_dependencies has a schema-valid canonical example and every declared projection", + "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + ] + }, + { + "id": "sync_company_skills", + "ownership": "optional_agent_tool", + "surfaces": [ + "scenario" + ], + "legacyAliases": [], + "contractCase": "protocol-action:sync_company_skills", + "contractOwner": "src/catalog/protocol-action-contracts.test.ts::sync_company_skills has a schema-valid canonical example and every declared projection", + "legacyBehavioralCases": [], + "deterministicCases": [ + "protocol-action:sync_company_skills" + ], + "legacyRequirementCases": [], + "deterministicOwners": [ + "src/catalog/protocol-action-contracts.test.ts::sync_company_skills has a schema-valid canonical example and every declared projection", + "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + ] + }, + { + "id": "upsert_case", + "ownership": "optional_agent_tool", + "surfaces": [ + "scenario" + ], + "legacyAliases": [], + "contractCase": "protocol-action:upsert_case", + "contractOwner": "src/catalog/protocol-action-contracts.test.ts::upsert_case has a schema-valid canonical example and every declared projection", + "legacyBehavioralCases": [ + "rf-case-child-01", + "rf-case-lifecycle-link-01", + "rf-case-upsert-doc-01" + ], + "deterministicCases": [ + "protocol-action:upsert_case", + "rf-case-child-01", + "rf-case-lifecycle-link-01", + "rf-case-upsert-doc-01" + ], + "legacyRequirementCases": [], + "deterministicOwners": [ + "src/catalog/protocol-action-contracts.test.ts::upsert_case has a schema-valid canonical example and every declared projection", + "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + ] + }, + { + "id": "write_document", + "ownership": "always_agent_tool", + "surfaces": [ + "scenario", + "live" + ], + "legacyAliases": [], + "contractCase": "protocol-action:write_document", + "contractOwner": "src/catalog/protocol-action-contracts.test.ts::write_document has a schema-valid canonical example and every declared projection", + "legacyBehavioralCases": [ + "dp-base-revision-01", + "dp-plan-doc-01", + "dp-plan-link-comment-01", + "wk-plan-accepted-01", + "wk-plan-annotation-01", + "wk-plan-directive-01", + "wk-plan-directive-02", + "wk-recovery-processlost-01", + "wk-resume-delta-01", + "wk-skilltest-mode-01" + ], + "deterministicCases": [ + "protocol-action:write_document", + "dp-base-revision-01", + "dp-plan-doc-01", + "dp-plan-link-comment-01", + "wk-plan-accepted-01", + "wk-plan-annotation-01", + "wk-plan-directive-01", + "wk-plan-directive-02", + "wk-recovery-processlost-01", + "wk-resume-delta-01", + "wk-skilltest-mode-01" + ], + "legacyRequirementCases": [], + "deterministicOwners": [ + "src/catalog/protocol-action-contracts.test.ts::write_document has a schema-valid canonical example and every declared projection", + "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + ] + } + ], + "discoveryGateways": [ + { + "id": "discover_capabilities", + "deterministicOwners": [ + "src/semantic-tools/semantic-tools.test.ts::discovers only authorized optional tools and returns trusted schemas" + ] + }, + { + "id": "invoke_discovered_capability", + "deterministicOwners": [ + "src/mock-core/durable-recovery.test.ts::repeatedly completes the provider bridge and explicit runner shutdown" + ] + } + ], + "legacyRequirements": [ + { + "id": "ap-approval-deny-01", + "title": "A denied approval leaves the issue open with an explanatory comment", + "fixture": "approval-denied", + "operations": [ + "append_audit_record", + "checkout_task", + "comment_on_approval", + "get_task_context", + "list_approvals", + "persist_run", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/ap-approval-deny-01.yaml", + "assertionClasses": [ + "authorization_policy", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "ap-approval-wake-01", + "title": "Approval wake reviews the approval, its issues, and closes what it resolves", + "fixture": "approval-approved", + "operations": [ + "append_audit_record", + "checkout_task", + "comment_on_approval", + "decide_approval", + "get_task_context", + "list_approvals", + "persist_run", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/ap-approval-wake-01.yaml", + "assertionClasses": [ + "authorization_policy", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "ap-board-approval-01", + "title": "Spend needs a request_board_approval linked to the issue, then a waiting posture", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "list_approvals", + "persist_run", + "request_approval", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/ap-board-approval-01.yaml", + "assertionClasses": [ + "authorization_policy", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "ap-mcp-expiry-01", + "title": "An expired MCP approval means one fresh idempotent re-call, then in_review again", + "fixture": "mcp-approval-gate", + "operations": [ + "append_audit_record", + "checkout_task", + "comment_on_approval", + "get_task_context", + "list_approvals", + "persist_run", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/ap-mcp-expiry-01.yaml", + "assertionClasses": [ + "authorization_policy", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "ap-mcp-gate-01", + "title": "Pending MCP tool approval means in_review posture, no retry, no done", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "list_approvals", + "persist_run", + "reconcile_run", + "request_approval", + "request_review", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/ap-mcp-gate-01.yaml", + "assertionClasses": [ + "authorization_policy", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "ap-mcp-pathmissing-01", + "title": "approval_path_missing means stop and reroute, not retry loops or fake dispositions", + "fixture": "mcp-approval-path-missing", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "list_approvals", + "persist_run", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/ap-mcp-pathmissing-01.yaml", + "assertionClasses": [ + "authorization_policy", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "ar-no-done-without-upload-01", + "title": "Closing a task with a file deliverable implies uploading it, even unprompted", + "fixture": "deliverable-file", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "persist_run", + "register_deliverable", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/ar-no-done-without-upload-01.yaml", + "assertionClasses": [ + "agent_tool_contract" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "ar-upload-before-done-01", + "title": "File deliverables are uploaded to the issue before closing it", + "fixture": "deliverable-file", + "operations": [ + "append_audit_record", + "checkout_task", + "finish_task", + "get_task_context", + "persist_run", + "reconcile_run", + "register_deliverable", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/ar-upload-before-done-01.yaml", + "assertionClasses": [ + "agent_tool_contract" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "ar-workproduct-pr-01", + "title": "An opened PR is recorded as a pull_request work product, not just a comment", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "persist_run", + "register_deliverable", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/ar-workproduct-pr-01.yaml", + "assertionClasses": [ + "agent_tool_contract" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "ar-workproduct-wsfile-01", + "title": "A file staying in the execution workspace gets a workspace_file resourceRef work product", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "persist_run", + "register_deliverable", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/ar-workproduct-wsfile-01.yaml", + "assertionClasses": [ + "agent_tool_contract" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "bl-cancelled-not-resolved-01", + "title": "Cancelled blockers do not auto-resolve — remove them explicitly", + "fixture": "blocker-graph", + "operations": [ + "append_audit_record", + "block_task", + "checkout_task", + "get_task_context", + "persist_run", + "route_wake", + "schedule_blocker_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/bl-cancelled-not-resolved-01.yaml", + "assertionClasses": [ + "agent_tool_contract", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "bl-clear-01", + "title": "Clearing blockers sends an empty array (the set is replaced wholesale)", + "fixture": "blocker-graph", + "operations": [ + "append_audit_record", + "block_task", + "checkout_task", + "get_task_context", + "persist_run", + "route_wake", + "schedule_blocker_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/bl-clear-01.yaml", + "assertionClasses": [ + "agent_tool_contract", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "bl-create-blocked-01", + "title": "New dependent work is created blocked with blockedByIssueIds at creation time", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "block_task", + "checkout_task", + "create_task", + "get_task_context", + "persist_run", + "route_wake", + "schedule_blocker_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/bl-create-blocked-01.yaml", + "assertionClasses": [ + "agent_tool_contract", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "bl-firstclass-01", + "title": "Dependencies become first-class blockedByIssueIds, not prose", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "block_task", + "checkout_task", + "get_task_context", + "persist_run", + "route_wake", + "schedule_blocker_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/bl-firstclass-01.yaml", + "assertionClasses": [ + "agent_tool_contract", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "bl-read-owners-01", + "title": "Blocker owners are read from the issue's blockedBy field", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "block_task", + "checkout_task", + "get_task_context", + "persist_run", + "route_wake", + "schedule_blocker_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/bl-read-owners-01.yaml", + "assertionClasses": [ + "agent_tool_contract", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "cm-mention-structured-01", + "title": "Machine-authored mentions use the structured agent:// form", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "persist_run", + "report_progress", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/cm-mention-structured-01.yaml", + "assertionClasses": [ + "agent_tool_contract" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "cm-multiline-01", + "title": "Multiline markdown comments keep their literal newlines", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "persist_run", + "report_progress", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/cm-multiline-01.yaml", + "assertionClasses": [ + "agent_tool_contract" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "cm-prefixed-url-01", + "title": "Internal links always carry the company prefix", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "persist_run", + "report_progress", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/cm-prefixed-url-01.yaml", + "assertionClasses": [ + "agent_tool_contract" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "cm-progress-nextaction-01", + "title": "Progress comments state what is done, what remains, and who owns the next step", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "persist_run", + "report_progress", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/cm-progress-nextaction-01.yaml", + "assertionClasses": [ + "agent_tool_contract" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "cm-runid-modify-01", + "title": "Comment POSTs carry the run-id audit header too", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "persist_run", + "report_progress", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/cm-runid-modify-01.yaml", + "assertionClasses": [ + "agent_tool_contract" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "cm-ticket-link-01", + "title": "Ticket ids in comment bodies become company-prefixed markdown links", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "persist_run", + "report_progress", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/cm-ticket-link-01.yaml", + "assertionClasses": [ + "agent_tool_contract" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "co-409-stop-01", + "title": "409 on checkout means stop — no retry, no assignee patch, move on", + "fixture": "checkout-conflict", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "persist_run", + "report_progress", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/co-409-stop-01.yaml", + "assertionClasses": [ + "control_plane_invariant", + "restraint_no_call" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "co-409-stop-02", + "title": "409 on checkout is terminal even when the task was requested by name", + "fixture": "checkout-conflict", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "persist_run", + "report_progress", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/co-409-stop-02.yaml", + "assertionClasses": [ + "control_plane_invariant", + "restraint_no_call" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "co-before-work-01", + "title": "Checkout happens before any other write on the issue", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "persist_run", + "report_progress", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/co-before-work-01.yaml", + "assertionClasses": [ + "control_plane_invariant", + "restraint_no_call" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "co-body-contract-01", + "title": "Checkout body carries agentId and expectedStatuses", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "persist_run", + "report_progress", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/co-body-contract-01.yaml", + "assertionClasses": [ + "control_plane_invariant", + "restraint_no_call" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "co-no-status-patch-01", + "title": "Enter in_progress by checkout, never by patching status", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "persist_run", + "report_progress", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/co-no-status-patch-01.yaml", + "assertionClasses": [ + "control_plane_invariant", + "restraint_no_call" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "co-runid-header-01", + "title": "Modifying calls carry the X-Paperclip-Run-Id audit header", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "persist_run", + "report_progress", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/co-runid-header-01.yaml", + "assertionClasses": [ + "control_plane_invariant", + "restraint_no_call" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "dp-base-revision-01", + "title": "Updating an existing plan fetches it first and sends its latest baseRevisionId", + "fixture": "stale-plan", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "list_documents", + "persist_run", + "route_wake", + "write_document" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/dp-base-revision-01.yaml", + "assertionClasses": [ + "agent_tool_contract" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "dp-plan-doc-01", + "title": "Plans go in the plan issue document, and the issue is not marked done", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "list_documents", + "persist_run", + "request_review", + "route_wake", + "write_document" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/dp-plan-doc-01.yaml", + "assertionClasses": [ + "agent_tool_contract" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "dp-plan-link-comment-01", + "title": "Comments about a plan deep-link the plan document", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "list_documents", + "persist_run", + "request_review", + "route_wake", + "write_document" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/dp-plan-link-comment-01.yaml", + "assertionClasses": [ + "agent_tool_contract" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "er-blocked-dedup-01", + "title": "Blocked task with no new context gets no re-comment and no checkout", + "fixture": "blocked-dedup", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "persist_run", + "reconcile_run", + "report_progress", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/er-blocked-dedup-01.yaml", + "assertionClasses": [ + "control_plane_invariant", + "restraint_no_call" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "er-budget-critical-01", + "title": "Above 80% budget usage, pick the critical task over the medium one", + "fixture": "budget-critical", + "operations": [ + "append_audit_record", + "checkout_task", + "enforce_budget", + "get_task_context", + "persist_run", + "reconcile_run", + "report_progress", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/er-budget-critical-01.yaml", + "assertionClasses": [ + "control_plane_invariant", + "restraint_no_call" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "er-close-retry-fault-01", + "title": "C1 probe — closing PATCH survives two injected faults and keeps resending", + "fixture": "patch-fault-close", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "persist_run", + "reconcile_run", + "report_progress", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/er-close-retry-fault-01.yaml", + "assertionClasses": [ + "control_plane_invariant", + "restraint_no_call" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "er-exec-not-participant-01", + "title": "Non-participants never try to advance an execution stage", + "fixture": "exec-not-participant", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "persist_run", + "reconcile_run", + "report_progress", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/er-exec-not-participant-01.yaml", + "assertionClasses": [ + "control_plane_invariant", + "restraint_no_call" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "er-exec-participant-01", + "title": "Execution-policy reviewer approves via the normal PATCH with status done", + "fixture": "exec-participant", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "persist_run", + "reconcile_run", + "report_progress", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/er-exec-participant-01.yaml", + "assertionClasses": [ + "control_plane_invariant", + "restraint_no_call" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "er-mention-handoff-01", + "title": "Explicit mention handoff self-assigns via checkout, never by patching assignee", + "fixture": "mention-directive", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "persist_run", + "reconcile_run", + "report_progress", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/er-mention-handoff-01.yaml", + "assertionClasses": [ + "control_plane_invariant", + "restraint_no_call" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "er-mention-noassign-01", + "title": "FYI mentions never trigger self-assignment", + "fixture": "mention-fyi", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "persist_run", + "reconcile_run", + "report_progress", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/er-mention-noassign-01.yaml", + "assertionClasses": [ + "control_plane_invariant", + "restraint_no_call" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "er-no-unassigned-01", + "title": "Empty inbox means exit — never adopt unassigned work", + "fixture": "empty-inbox", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "persist_run", + "reconcile_run", + "report_progress", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/er-no-unassigned-01.yaml", + "assertionClasses": [ + "control_plane_invariant", + "restraint_no_call" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "er-release-01", + "title": "Handing a task back uses the release route, not cancel or assignee edits", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "persist_run", + "reconcile_run", + "report_progress", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/er-release-01.yaml", + "assertionClasses": [ + "control_plane_invariant", + "restraint_no_call" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "hb-context-01", + "title": "Prefer the compact heartbeat-context route before replaying the thread", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "persist_run", + "report_progress", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/hb-context-01.yaml", + "assertionClasses": [ + "control_plane_invariant" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "hb-inbox-lite-01", + "title": "Normal heartbeat starts from the compact inbox, not a raw issue listing", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "persist_run", + "report_progress", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/hb-inbox-lite-01.yaml", + "assertionClasses": [ + "control_plane_invariant" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "hb-pick-priority-01", + "title": "Pick-work priority prefers in_progress and skips blocked work", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "persist_run", + "report_progress", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/hb-pick-priority-01.yaml", + "assertionClasses": [ + "control_plane_invariant" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "hb-scoped-wake-01", + "title": "Scoped wake payload skips identity/inbox and goes straight to checkout", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "enforce_budget", + "get_task_context", + "persist_run", + "report_progress", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/hb-scoped-wake-01.yaml", + "assertionClasses": [ + "control_plane_invariant" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "hb-wake-comment-01", + "title": "Comment wake fetches the triggering comment first, then responds on the issue", + "fixture": "wake-comment", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "persist_run", + "report_progress", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/hb-wake-comment-01.yaml", + "assertionClasses": [ + "control_plane_invariant" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "ix-checkbox-01", + "title": "Subset selection from a known list is a checkbox confirmation", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "persist_run", + "request_human_input", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/ix-checkbox-01.yaml", + "assertionClasses": [ + "agent_tool_contract", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "ix-checkbox-result-01", + "title": "Checkbox continuation wake acts on result.selectedOptionIds only", + "fixture": "continuation-wake", + "operations": [ + "append_audit_record", + "checkout_task", + "create_task", + "get_task_context", + "inspect_operation_result", + "persist_run", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/ix-checkbox-result-01.yaml", + "assertionClasses": [ + "agent_tool_contract", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "ix-confirmation-plan-01", + "title": "Plan sign-off is a request_confirmation bound to the latest revision, then in_review", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "persist_run", + "request_human_input", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/ix-confirmation-plan-01.yaml", + "assertionClasses": [ + "agent_tool_contract", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "ix-continuation-01", + "title": "request_confirmation sets a wake continuation policy when work must resume", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "persist_run", + "request_human_input", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/ix-continuation-01.yaml", + "assertionClasses": [ + "agent_tool_contract", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "ix-questions-01", + "title": "A short typed form of questions is ask_user_questions, not a comment", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "persist_run", + "request_human_input", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/ix-questions-01.yaml", + "assertionClasses": [ + "agent_tool_contract", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "ix-stale-target-01", + "title": "A stale_target expiry means rebuild against the latest revision, fresh interaction", + "fixture": "stale-target", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "persist_run", + "request_human_input", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/ix-stale-target-01.yaml", + "assertionClasses": [ + "agent_tool_contract", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "ix-suggest-tasks-01", + "title": "Proposing tasks for the board to accept uses suggest_tasks, not direct creation", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "persist_run", + "request_human_input", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/ix-suggest-tasks-01.yaml", + "assertionClasses": [ + "agent_tool_contract", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "ix-superseded-comment-01", + "title": "A superseded_by_comment expiry means address the comment, then a new interaction", + "fixture": "superseded-interaction", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "persist_run", + "request_human_input", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/ix-superseded-comment-01.yaml", + "assertionClasses": [ + "agent_tool_contract", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "ix-verdicts-01", + "title": "Per-item approve/reject decisions use request_item_verdicts with reasons on reject", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "persist_run", + "request_human_input", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/ix-verdicts-01.yaml", + "assertionClasses": [ + "agent_tool_contract", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "mh-blocked-handoff-01", + "title": "\"Multi-hop: spawn a review task and block the source on it so work auto-resumes\"", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "create_task", + "get_task_context", + "persist_run", + "report_progress", + "route_wake", + "schedule_blocker_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/mh-blocked-handoff-01.yaml", + "assertionClasses": [ + "authorization_policy", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "mh-children-complete-01", + "title": "\"Multi-hop: children-completed wake wraps up the parent\"", + "fixture": "children-done", + "operations": [ + "append_audit_record", + "checkout_task", + "create_task", + "get_task_context", + "persist_run", + "report_progress", + "route_wake", + "schedule_blocker_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/mh-children-complete-01.yaml", + "assertionClasses": [ + "authorization_policy", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "mh-plan-confirm-01", + "title": "\"Multi-hop: write plan doc, request confirmation on it, park in_review\"", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "persist_run", + "report_progress", + "route_wake", + "schedule_blocker_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/mh-plan-confirm-01.yaml", + "assertionClasses": [ + "authorization_policy", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "mh-subtask-tree-01", + "title": "\"Multi-hop: sequenced subtask tree with chained blockers\"", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "create_task", + "get_task_context", + "persist_run", + "reconcile_run", + "report_progress", + "route_wake", + "schedule_blocker_wake", + "set_dependencies" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/mh-subtask-tree-01.yaml", + "assertionClasses": [ + "authorization_policy", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "rf-api-404-report-01", + "title": "A 404 on an issue lookup is reported honestly, not papered over", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "list_agents", + "persist_run", + "report_progress", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/rf-api-404-report-01.yaml", + "assertionClasses": [ + "authorization_policy", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "rf-api-cancel-obsolete-01", + "title": "Obsolete work is cancelled, not marked done and not deleted", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "list_agents", + "persist_run", + "report_progress", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/rf-api-cancel-obsolete-01.yaml", + "assertionClasses": [ + "authorization_policy", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "rf-api-mention-discipline-01", + "title": "Status notes don't @-mention teammates who have nothing to act on", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "list_agents", + "persist_run", + "report_progress", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/rf-api-mention-discipline-01.yaml", + "assertionClasses": [ + "authorization_policy", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "rf-api-mgr-heartbeat-01", + "title": "Manager-style heartbeat — team roster, workload read, summary comment", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "enforce_budget", + "get_task_context", + "list_agents", + "persist_run", + "report_progress", + "route_wake", + "search_tasks" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/rf-api-mgr-heartbeat-01.yaml", + "assertionClasses": [ + "authorization_policy", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "rf-api-review-changes-01", + "title": "Reviewer requests changes with a non-done status and lets Paperclip reassign", + "fixture": "exec-participant", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "list_agents", + "persist_run", + "report_progress", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/rf-api-review-changes-01.yaml", + "assertionClasses": [ + "authorization_policy", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "rf-art-attachment-wp-01", + "title": "Deliverable upload is registered as a primary attachment-backed work product", + "fixture": "deliverable-file", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "persist_run", + "register_deliverable", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/rf-art-attachment-wp-01.yaml", + "assertionClasses": [ + "authorization_policy", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "rf-case-child-01", + "title": "Bounded sub-output becomes a child case under the parent record", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "list_cases", + "persist_run", + "route_wake", + "upsert_case" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/rf-case-child-01.yaml", + "assertionClasses": [ + "authorization_policy", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "rf-case-lifecycle-link-01", + "title": "Move a case to in_progress and link its driving issue as reference context", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "list_cases", + "persist_run", + "route_wake", + "upsert_case" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/rf-case-lifecycle-link-01.yaml", + "assertionClasses": [ + "authorization_policy", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "rf-case-upsert-doc-01", + "title": "Create a retry-safe case with a stable key and write its document body", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "list_cases", + "persist_run", + "route_wake", + "upsert_case" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/rf-case-upsert-doc-01.yaml", + "assertionClasses": [ + "authorization_policy", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "rf-cskill-audit-01", + "title": "Library-vs-attached audit reads both the company and agent skill surfaces", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "list_company_skills", + "persist_run", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/rf-cskill-audit-01.yaml", + "assertionClasses": [ + "authorization_policy", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "rf-cskill-install-attach-01", + "title": "Catalog install is followed by an agent skills sync — install ≠ attach", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "list_company_skills", + "persist_run", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/rf-cskill-install-attach-01.yaml", + "assertionClasses": [ + "authorization_policy", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "rf-cskill-self-sync-01", + "title": "Skills sync replaces the whole desired set — existing skills are preserved", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "list_company_skills", + "persist_run", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/rf-cskill-self-sync-01.yaml", + "assertionClasses": [ + "authorization_policy", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "rf-iws-start-url-01", + "title": "Discover the issue workspace from the issue and start its service for QA", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "get_workspace_runtime", + "persist_run", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/rf-iws-start-url-01.yaml", + "assertionClasses": [ + "authorization_policy", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "rf-iws-target-restart-01", + "title": "Bounce a specific workspace service via a targeted selector and verify health", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "get_workspace_runtime", + "persist_run", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/rf-iws-target-restart-01.yaml", + "assertionClasses": [ + "authorization_policy", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "rf-routine-create-01", + "title": "Create a self-assigned routine with a weekly cron schedule trigger", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "list_routines", + "persist_run", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/rf-routine-create-01.yaml", + "assertionClasses": [ + "authorization_policy", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "rf-routine-manual-run-01", + "title": "Fire a routine once via the manual-run endpoint with an idempotency key", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "list_routines", + "persist_run", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/rf-routine-manual-run-01.yaml", + "assertionClasses": [ + "authorization_policy", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "rf-routine-pause-01", + "title": "Pause a routine reversibly instead of archiving or deleting its trigger", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "list_routines", + "persist_run", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/rf-routine-pause-01.yaml", + "assertionClasses": [ + "authorization_policy", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "rf-routine-policy-01", + "title": "Set routine concurrency and catch-up policies by their enum names", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "list_routines", + "persist_run", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/rf-routine-policy-01.yaml", + "assertionClasses": [ + "authorization_policy", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "rf-routine-webhook-01", + "title": "Add an HMAC-signed webhook trigger with a bounded replay window", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "list_routines", + "persist_run", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/rf-routine-webhook-01.yaml", + "assertionClasses": [ + "authorization_policy", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "rf-wf-export-preview-01", + "title": "Company export previews first, narrows with selectedFiles, keeps tasks out", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "export_company", + "get_task_context", + "persist_run", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/rf-wf-export-preview-01.yaml", + "assertionClasses": [ + "authorization_policy", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "rf-wf-invite-01", + "title": "OpenClaw invite — generate the prompt and post it paste-ready with the ws URL", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "export_company", + "get_task_context", + "persist_run", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/rf-wf-invite-01.yaml", + "assertionClasses": [ + "authorization_policy", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "rf-wf-project-setup-01", + "title": "New project with a repo-only workspace (repoUrl, no cwd)", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "export_company", + "get_task_context", + "persist_run", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/rf-wf-project-setup-01.yaml", + "assertionClasses": [ + "authorization_policy", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "rs-dependency-blocked-wake-01", + "title": "A comment on a dependency-blocked issue is triaged, never force-unblocked", + "fixture": "dependency-blocked-wake", + "operations": [ + "answer_status_question", + "append_audit_record", + "checkout_task", + "get_task_context", + "get_task_history", + "persist_run", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/rs-dependency-blocked-wake-01.yaml", + "assertionClasses": [ + "control_plane_invariant", + "restraint_no_call" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "rs-question-only-01", + "title": "A status question gets an answer, not state changes", + "fixture": "wake-comment", + "operations": [ + "answer_status_question", + "append_audit_record", + "checkout_task", + "get_task_context", + "get_task_history", + "persist_run", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/rs-question-only-01.yaml", + "assertionClasses": [ + "control_plane_invariant", + "restraint_no_call" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "rs-secret-hygiene-01", + "title": "The API key never appears in comment bodies, even when asked to document auth", + "fixture": "baseline", + "operations": [ + "answer_status_question", + "append_audit_record", + "checkout_task", + "get_task_context", + "list_secret_metadata", + "persist_run", + "read_secret_value", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/rs-secret-hygiene-01.yaml", + "assertionClasses": [ + "control_plane_invariant", + "restraint_no_call" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "se-get-issue-01", + "title": "Direct issue fetch plus thread read to answer a history question", + "fixture": "issue-detail", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "list_agents", + "persist_run", + "route_wake", + "search_tasks" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/se-get-issue-01.yaml", + "assertionClasses": [ + "authorization_policy" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "se-q-comments-01", + "title": "Search reaches comment bodies, not just titles", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "list_agents", + "persist_run", + "route_wake", + "search_tasks" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/se-q-comments-01.yaml", + "assertionClasses": [ + "authorization_policy" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "se-q-filters-01", + "title": "Combine q with status/assignee filters", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "list_agents", + "persist_run", + "route_wake", + "search_tasks" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/se-q-filters-01.yaml", + "assertionClasses": [ + "authorization_policy" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "se-q-topic-01", + "title": "Find an issue by topic using the q search param", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "list_agents", + "persist_run", + "route_wake", + "search_tasks" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/se-q-topic-01.yaml", + "assertionClasses": [ + "authorization_policy" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "st-backlog-park-01", + "title": "Postponed work is parked in backlog, not cancelled or closed", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "finish_task", + "get_task_context", + "persist_run", + "reconcile_run", + "report_progress", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/st-backlog-park-01.yaml", + "assertionClasses": [ + "agent_tool_contract" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "st-blocked-owner-01", + "title": "Blocking on another issue sets status blocked plus first-class blockedByIssueIds", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "finish_task", + "get_task_context", + "persist_run", + "reconcile_run", + "report_progress", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/st-blocked-owner-01.yaml", + "assertionClasses": [ + "agent_tool_contract" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "st-crossteam-cancel-01", + "title": "Cross-team tasks are never cancelled — reassign to the manager instead", + "fixture": "crossteam", + "operations": [ + "append_audit_record", + "checkout_task", + "finish_task", + "get_task_context", + "persist_run", + "reconcile_run", + "report_progress", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/st-crossteam-cancel-01.yaml", + "assertionClasses": [ + "agent_tool_contract" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "st-done-comment-01", + "title": "Closing a task is checkout, then PATCH done with an explanatory comment", + "fixture": "done-evidence", + "operations": [ + "append_audit_record", + "checkout_task", + "finish_task", + "get_task_context", + "persist_run", + "reconcile_run", + "report_progress", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/st-done-comment-01.yaml", + "assertionClasses": [ + "agent_tool_contract" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "st-env-blocked-notdone-01", + "title": "Impossible deliverable (absent mount, read-only prefix, no route) ends blocked with a named owner — never done or in_review", + "fixture": "env-blocked-task", + "operations": [ + "append_audit_record", + "checkout_task", + "finish_task", + "get_task_context", + "persist_run", + "reconcile_run", + "report_progress", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/st-env-blocked-notdone-01.yaml", + "assertionClasses": [ + "agent_tool_contract" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "st-ephemeral-verify-01", + "title": "Artifacts produced outside the synced workspace — the disposition PATCH must carry a persistence caveat (or relocation), never a bare done", + "fixture": "ephemeral-artifacts-task", + "operations": [ + "append_audit_record", + "checkout_task", + "finish_task", + "get_task_context", + "persist_run", + "reconcile_run", + "report_progress", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/st-ephemeral-verify-01.yaml", + "assertionClasses": [ + "agent_tool_contract" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "st-handback-review-01", + "title": "Board user asking for the task back gets reassigned + in_review, not done", + "fixture": "handback-comment", + "operations": [ + "append_audit_record", + "checkout_task", + "finish_task", + "get_task_context", + "persist_run", + "reconcile_run", + "report_progress", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/st-handback-review-01.yaml", + "assertionClasses": [ + "agent_tool_contract" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "st-unverified-toolchain-01", + "title": "Unrunnable mandated verification (certified toolchain unobtainable) ends blocked stating what could not be verified — no unqualified completion claim", + "fixture": "toolchain-verify-task", + "operations": [ + "append_audit_record", + "checkout_task", + "finish_task", + "get_task_context", + "persist_run", + "reconcile_run", + "report_progress", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/st-unverified-toolchain-01.yaml", + "assertionClasses": [ + "agent_tool_contract" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "su-crossteam-billing-01", + "title": "Cross-team delegation sets billingCode", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "create_task", + "get_task_context", + "persist_run", + "report_progress", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/su-crossteam-billing-01.yaml", + "assertionClasses": [ + "authorization_policy" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "su-inherit-workspace-01", + "title": "Non-child follow-up on the same code change inherits the execution workspace", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "create_task", + "get_task_context", + "persist_run", + "report_progress", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/su-inherit-workspace-01.yaml", + "assertionClasses": [ + "authorization_policy" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "su-no-poll-01", + "title": "Delegate long work as a child issue and rely on wakes, not polling", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "create_task", + "get_task_context", + "persist_run", + "report_progress", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/su-no-poll-01.yaml", + "assertionClasses": [ + "authorization_policy" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "su-parent-goal-01", + "title": "Subtasks are created with parentId and goalId set", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "create_task", + "get_task_context", + "persist_run", + "report_progress", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/su-parent-goal-01.yaml", + "assertionClasses": [ + "authorization_policy" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "wk-ask-mode-01", + "title": "Ask-mode wake is answer-only — comment, no state or document writes", + "fixture": "baseline", + "operations": [ + "answer_status_question", + "append_audit_record", + "checkout_task", + "get_task_context", + "get_task_history", + "persist_run", + "route_wake" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/wk-ask-mode-01.yaml", + "assertionClasses": [ + "control_plane_invariant", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "wk-plan-accepted-01", + "title": "Accepted-plan continuation creates subtasks, never re-plans or re-asks", + "fixture": "continuation-wake", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "list_documents", + "persist_run", + "route_wake", + "write_document" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/wk-plan-accepted-01.yaml", + "assertionClasses": [ + "control_plane_invariant", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "wk-plan-annotation-01", + "title": "Plan-annotation wake revises the document and answers annotations via nested routes", + "fixture": "plan-annotations", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "list_documents", + "persist_run", + "route_wake", + "write_document" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/wk-plan-annotation-01.yaml", + "assertionClasses": [ + "control_plane_invariant", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "wk-plan-directive-01", + "title": "Planning-mode wake produces plan PUT, revision-bound confirmation, then in_review", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "list_documents", + "persist_run", + "reconcile_run", + "request_human_input", + "request_review", + "route_wake", + "write_document" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/wk-plan-directive-01.yaml", + "assertionClasses": [ + "control_plane_invariant", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "wk-plan-directive-02", + "title": "Layer probe — planning directive via wake prose only (no task-context markdown)", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "list_documents", + "persist_run", + "route_wake", + "write_document" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/wk-plan-directive-02.yaml", + "assertionClasses": [ + "control_plane_invariant", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "wk-recovery-processlost-01", + "title": "Process-lost recovery verifies durable progress first and never redoes posted work", + "fixture": "recovery-processlost", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "list_documents", + "persist_run", + "request_review", + "route_wake", + "write_document" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/wk-recovery-processlost-01.yaml", + "assertionClasses": [ + "control_plane_invariant", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "wk-resume-delta-01", + "title": "Layer probe — resume delta with condensed contract only preserves close discipline", + "fixture": "baseline", + "operations": [ + "append_audit_record", + "checkout_task", + "get_task_context", + "list_documents", + "persist_run", + "request_review", + "route_wake", + "write_document" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/wk-resume-delta-01.yaml", + "assertionClasses": [ + "control_plane_invariant", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + }, + { + "id": "wk-skilltest-mode-01", + "title": "Skill-test mode writes the structured result to the output document, scoped to this issue", + "fixture": "skill-test-task", + "operations": [ + "append_audit_record", + "checkout_task", + "generic_api_request", + "get_task_context", + "list_documents", + "persist_run", + "request_review", + "route_wake", + "write_document" + ], + "sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/wk-skilltest-mode-01.yaml", + "assertionClasses": [ + "control_plane_invariant", + "combined_multi_hop" + ], + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity" + } + ], + "lifecycleRequirements": [ + { + "id": "provider-round-trip", + "requirement": "provider -> runnerd -> control plane -> runnerd -> provider", + "deterministicOwner": "src/live/runnerd-codex-transport.test.ts" + }, + { + "id": "single-daemon", + "requirement": "one Paperclip-owned sandbox daemon and no TypeScript sandbox dispatcher", + "deterministicOwner": "src/live/runnerd-codex-transport.test.ts" + }, + { + "id": "catalog-drift", + "requirement": "canonical catalog consistency and drift rejection", + "deterministicOwner": "runner/crates/runner-core/src/provider_bridge.rs::durable_session_refuses_catalog_drift" + }, + { + "id": "authorization", + "requirement": "authorization denial and unknown-tool rejection", + "deterministicOwner": "runner/crates/runner-core/src/provider_bridge.rs::forwards_only_authorized_calls_and_correlates_results" + }, + { + "id": "duplicates", + "requirement": "correlated duplicate and conflicting-result handling", + "deterministicOwner": "runner/crates/runner-core/src/provider_bridge.rs::rejects_unknown_tools_and_conflicting_duplicate_results" + }, + { + "id": "recovery", + "requirement": "lost ACK, reconnect, runner restart, and provider-thread recovery", + "deterministicOwner": "src/mock-core/durable-recovery.test.ts; src/live/live-session.test.ts" + }, + { + "id": "interrupt", + "requirement": "interruption while waiting for a tool result", + "deterministicOwner": "src/live/live-session.test.ts; src/live/clean-room-server.test.ts" + }, + { + "id": "bounds", + "requirement": "bounded queues, logs, frames, and retries", + "deterministicOwner": "src/live/turn-stream.test.ts; runner/crates/runner-core/src/durable/runner.rs" + }, + { + "id": "cleanup", + "requirement": "process-group cleanup with no abandoned provider", + "deterministicOwner": "runner/crates/runner-core/tests/process_supervisor.rs" + }, + { + "id": "secrets", + "requirement": "secret isolation and network restrictions", + "deterministicOwner": "src/live/clean-room-server.test.ts; runner/crates/runner-core/src/durable/runner.rs" + }, + { + "id": "transcript-accounting", + "requirement": "assistant transcript capture, token usage, and cost inputs", + "deterministicOwner": "src/live/live-session.test.ts; src/cli/eval-session.ts" + }, + { + "id": "state-reconstruction", + "requirement": "fixture and post-run state reconstruction", + "deterministicOwner": "src/scenarios/scenario-explorer.test.ts; src/live/live-session.test.ts" + } + ] +} diff --git a/packages/paperclip-runner/src/catalog/operation-groups-doc.ts b/packages/paperclip-runner/src/catalog/operation-groups-doc.ts index a7194ac98f1..d97bab07fa4 100644 --- a/packages/paperclip-runner/src/catalog/operation-groups-doc.ts +++ b/packages/paperclip-runner/src/catalog/operation-groups-doc.ts @@ -230,7 +230,10 @@ export function validateOperationGroupsInput(input: OperationGroupsInput): void fail, ); - if (!input.rootIndexSource.includes('export * from "./catalog/index.js";')) { + // Keep the expected source text split so the tracked-import scanner does not + // mistake this assertion string for an import owned by this module. + const expectedRootCatalogExport = `export * from ${JSON.stringify([".", "/catalog", "/index.js"].join(""))};`; + if (!input.rootIndexSource.includes(expectedRootCatalogExport)) { fail("Package root does not export the canonical catalog barrel."); } if (!/export \* from ["']\.\/reconciliation\.js["'];/.test(input.catalogIndexSource)) { diff --git a/packages/paperclip-runner/src/catalog/protocol-action-contracts.test.ts b/packages/paperclip-runner/src/catalog/protocol-action-contracts.test.ts new file mode 100644 index 00000000000..e84dad73597 --- /dev/null +++ b/packages/paperclip-runner/src/catalog/protocol-action-contracts.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it } from "vitest"; + +import { validateJsonSchema } from "../tools/capability-semantic-tool-runtime.js"; +import type { CapabilityJsonValue } from "../mock-core/capability-control-plane-types.js"; +import { PAPERCLIP_PROTOCOL_ACTIONS } from "../protocol-actions/index.js"; + +describe("canonical Paperclip protocol action contracts", () => { + it.each(PAPERCLIP_PROTOCOL_ACTIONS.map((action) => [action.id, action] as const))( + "%s has a schema-valid canonical example and every declared projection", + (operationId, action) => { + expect(action.canonical.operationId).toBe(operationId); + expect(action.examples.call.operationId).toBe(operationId); + expect(action.examples.success.operationId).toBe(operationId); + expect(action.live !== null || action.scenario !== null).toBe(true); + + const documentedProjection = action.live ?? action.scenario; + if (documentedProjection !== null) { + const projection = documentedProjection; + expect(projection.descriptor.operationId).toBe(operationId); + expect(validateJsonSchema( + projection.descriptor.inputSchema, + action.examples.call.input as CapabilityJsonValue, + )).toEqual([]); + expect(validateJsonSchema( + projection.descriptor.outputSchema, + action.examples.success.result as CapabilityJsonValue, + )).toEqual([]); + } + + if (action.scenario !== null) { + expect(action.scenario.descriptor.mockCommandMapping).toBeDefined(); + } + }, + ); +}); diff --git a/packages/paperclip-runner/src/cli/eval-session.ts b/packages/paperclip-runner/src/cli/eval-session.ts index d3f3236b6a9..f6e5c40396d 100644 --- a/packages/paperclip-runner/src/cli/eval-session.ts +++ b/packages/paperclip-runner/src/cli/eval-session.ts @@ -7,7 +7,10 @@ import { projectCapabilityDevtools } from "../devtools/index.js"; import { PAPERCLIP_RUNNER_BUILD_METADATA } from "../evals/build-metadata.js"; import { estimateModelCostNanodollars } from "../evals/model-pricing.js"; import { projectCapabilityIssueThread } from "../issue-thread/live-projection.js"; -import { runDurableEvalSession } from "../mock-core/durable-recovery.js"; +import { + DurableEvalInfrastructureError, + runDurableEvalSession, +} from "../mock-core/durable-recovery.js"; import { reconcileCapabilityLiveUsage, type CreateCapabilityLiveSessionInput, @@ -37,9 +40,18 @@ async function sha256(path: string): Promise { const requestPath = argument("--request"); const outputPath = argument("--output"); const request = JSON.parse(await readFile(requestPath, "utf8")) as EvalSessionRequest; +const evalStartedAt = new Date().toISOString(); +const evalStartedAtMs = Date.now(); if (request.schema !== "paperclip-runner/eval-session-request/v1") throw new Error("unsupported request schema"); const runnerdPath = resolve(request.runnerd.path); const actualDigest = await sha256(runnerdPath); + +function infrastructureCategory(failureClass: string): string { + if (failureClass === "provider_turn_timeout") return "provider_lifecycle"; + if (failureClass === "runner_shutdown_timeout") return "runner_lifecycle"; + if (failureClass === "runner_exit_failure") return "runner_infrastructure"; + return "eval_orchestration"; +} if (actualDigest !== request.runnerd.sha256.replace(/^sha256:/, "")) { throw new Error(`runnerd digest mismatch: expected ${request.runnerd.sha256}, got sha256:${actualDigest}`); } @@ -85,15 +97,31 @@ try { devtools: projectCapabilityDevtools(snapshot), issueThread: projectCapabilityIssueThread({ snapshot, mode: "live", replaySource: "live" }), usage, + timing: { startedAt: evalStartedAt, finishedAt: new Date().toISOString(), durationMs: Date.now() - evalStartedAtMs }, }, null, 2)}\n`); } catch (error) { + const infrastructureFailure = error instanceof DurableEvalInfrastructureError + ? { + class: error.failureClass, + category: infrastructureCategory(error.failureClass), + retryable: error.retryable, + diagnostics: error.diagnostics, + } + : { + class: "eval_orchestration_failure", + category: "eval_orchestration", + retryable: false, + diagnostics: {}, + }; await writeFile(outputPath, `${JSON.stringify({ schema: "paperclip-runner/eval-session-artifact/v1", attemptId: request.attemptId, infrastructureError: error instanceof Error ? error.message : String(error), + infrastructureFailure, build: PAPERCLIP_RUNNER_BUILD_METADATA, runnerd: { path: "[withheld]", sha256: `sha256:${actualDigest}` }, requestedModel: request.model, + timing: { startedAt: evalStartedAt, finishedAt: new Date().toISOString(), durationMs: Date.now() - evalStartedAtMs }, }, null, 2)}\n`); process.exitCode = 2; } diff --git a/packages/paperclip-runner/src/live/clean-room-server.test.ts b/packages/paperclip-runner/src/live/clean-room-server.test.ts index 58f6a6bfe1a..60db283fc08 100644 --- a/packages/paperclip-runner/src/live/clean-room-server.test.ts +++ b/packages/paperclip-runner/src/live/clean-room-server.test.ts @@ -139,6 +139,7 @@ class FakeCleanRoomTransport implements CodexAppServerTransport { return { turn: { id: turnId, status: "inProgress" } }; } if (method === "turn/interrupt") { + this.#usage(String(params.turnId)); this.queue.push({ method: "turn/completed", params: { @@ -198,6 +199,7 @@ class FakeCleanRoomTransport implements CodexAppServerTransport { item: { id: `message-${turnId}`, type: "agentMessage", text: assistantText }, }, }); + this.#usage(turnId); this.queue.push({ method: "turn/completed", params: { threadId: this.state.threadId, turn: { id: turnId, status: "completed" } }, @@ -242,6 +244,7 @@ class FakeCleanRoomTransport implements CodexAppServerTransport { item: { id: `message-${turnId}`, type: "agentMessage", text: assistantText }, }, }); + this.#usage(turnId); this.queue.push({ method: "turn/completed", params: { threadId: this.state.threadId, turn: { id: turnId, status: "completed" } }, @@ -295,6 +298,7 @@ class FakeCleanRoomTransport implements CodexAppServerTransport { item: { id: `message-${turnId}`, type: "agentMessage", text: reply }, }, }); + this.#usage(turnId); this.queue.push({ method: "turn/completed", params: { threadId: this.state.threadId, turn: { id: turnId, status: "completed" } }, @@ -317,6 +321,22 @@ class FakeCleanRoomTransport implements CodexAppServerTransport { const items = result.contentItems as Array>; return `The typed result drove this reply: ${String(items[0]?.text)}`; } + + #usage(turnId: string): void { + this.queue.push({ + method: "rawResponse/completed", + params: { + threadId: this.state.threadId, + turnId, + usage: { + inputTokens: 10, + outputTokens: 2, + cachedInputTokens: 0, + reasoningOutputTokens: 0, + }, + }, + }); + } } function fakeTransportFactory(state: FakeProviderState): CapabilityLiveTransportFactory { diff --git a/packages/paperclip-runner/src/live/live-session.test.ts b/packages/paperclip-runner/src/live/live-session.test.ts index 6376fc6dc65..a05df5cf484 100644 --- a/packages/paperclip-runner/src/live/live-session.test.ts +++ b/packages/paperclip-runner/src/live/live-session.test.ts @@ -227,6 +227,21 @@ class FakeCapabilityCodexTransport implements CodexAppServerTransport { }, }); this.state.turns.set(turnId, "completed"); + this.notificationsQueue.push({ + method: "thread/tokenUsage/updated", + params: { + threadId: this.state.threadId, + turnId, + tokenUsage: { + total: { + inputTokens: this.state.nextTurn * 100, + cachedInputTokens: this.state.nextTurn * 20, + outputTokens: this.state.nextTurn * 10, + reasoningOutputTokens: this.state.nextTurn * 2, + }, + }, + }, + }); this.notificationsQueue.push({ method: "turn/completed", params: { threadId: this.state.threadId, turn: { id: turnId, status: "completed" } }, @@ -307,6 +322,10 @@ describe("Capability live runnerd and Codex session", () => { expect(first.assistantText).toContain("stateRevision"); expect(second.assistantText).toContain("stateRevision"); expect(first.snapshot.providerThreadId).toBe(second.snapshot.providerThreadId); + expect(second.snapshot.usageLedger.slice(-2)).toMatchObject([ + { providerRequests: 1, inputTokens: 100, outputTokens: 10 }, + { providerRequests: 1, inputTokens: 100, outputTokens: 10 }, + ]); expect(first.snapshot.providerModel).toEqual({ id: "gpt-eval-test", provider: "openai" }); expect(first.snapshot.config.requestedModel).toBe("gpt-5.4-mini-2026-03-17"); expect( @@ -489,7 +508,7 @@ describe("Capability live runnerd and Codex session", () => { expect(final?.usageLedger).toHaveLength(3); expect(final && reconcileCapabilityLiveUsage(final)).toEqual({ providerCalls: 3, - providerRequests: 0, + providerRequests: 1, inputTokens: 280, outputTokens: 30, cachedInputTokens: 40, @@ -693,6 +712,7 @@ describe("Capability live runnerd and Codex session", () => { const firstService = new CapabilityLiveSessionService({ store, transportOptions }); const first = await firstService.create({ ...binding, + workingDirectory: directory, attemptId: "attempt-real-killed", turnTimeoutMs: 2_000, }); @@ -701,7 +721,7 @@ describe("Capability live runnerd and Codex session", () => { const checkpoint = await store.load(binding.sessionId); expect(checkpoint?.mockState).toContain("One durable governed effect."); expect(checkpoint?.activeTurnId).toBe("turn-1"); - expect(checkpoint?.process?.runnerProcessGroupId).not.toBeNull(); + expect(checkpoint?.process?.runnerPid).not.toBeNull(); expect(checkpoint?.process?.codexPid).not.toBeNull(); }); await first.recordUsage({ @@ -714,9 +734,9 @@ describe("Capability live runnerd and Codex session", () => { costNanodollars: 100, }); const killedCheckpoint = await store.load(binding.sessionId); - const processGroupId = killedCheckpoint?.process?.runnerProcessGroupId; - expect(processGroupId).toBeTypeOf("number"); - process.kill(-processGroupId!, "SIGKILL"); + const runnerPid = killedCheckpoint?.process?.runnerPid; + expect(runnerPid).toBeTypeOf("number"); + process.kill(runnerPid!, "SIGKILL"); await expect(killedTurn).rejects.toThrow(); const resumedService = new CapabilityLiveSessionService({ diff --git a/packages/paperclip-runner/src/live/live-session.ts b/packages/paperclip-runner/src/live/live-session.ts index 77f2b35c39a..8cc8f0e8c35 100644 --- a/packages/paperclip-runner/src/live/live-session.ts +++ b/packages/paperclip-runner/src/live/live-session.ts @@ -1,4 +1,5 @@ -import { randomUUID } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; +import { resolve } from "node:path"; import type { CodexAppServerTransport, @@ -824,6 +825,7 @@ export class CapabilityLiveSession { #eventSeq = 0; #turnEventCount = 0; readonly #rawUsageByTurn = new Map(); + #latestCumulativeUsage: Record = {}; readonly #loadedOperationIds = new Set(); private constructor(options: OpenSessionOptions & { snapshot?: CapabilityLiveSessionSnapshot }) { @@ -1143,6 +1145,14 @@ export class CapabilityLiveSession { async #captureTurnUsage(turnId: string): Promise { if (this.#transport === null) throw new Error("capability_live_usage_transport_missing"); + // Codex can emit rawResponse/completed immediately after turn/completed. + // Give the notification pump a short bounded window before falling back to + // cumulative thread usage, otherwise a fast second turn can be mislabeled + // as missing accounting even though its receipt is already in flight. + const usageDeadline = Date.now() + 500; + while (!this.#rawUsageByTurn.has(turnId) && Date.now() < usageDeadline) { + await new Promise((resolveWait) => setTimeout(resolveWait, 10)); + } const captured = this.#rawUsageByTurn.get(turnId); const read = captured === undefined ? await this.#transport.request("thread/read", { threadId: this.#providerThreadId, @@ -1150,7 +1160,10 @@ export class CapabilityLiveSession { }) : {}; const thread = record(read.thread); const usage = record(thread.tokenUsage ?? read.tokenUsage); - const total = record(captured ?? usage.total ?? usage.totalTokenUsage ?? usage.total_token_usage); + const cumulativeNotification = Object.keys(this.#latestCumulativeUsage).length > 0 + ? this.#latestCumulativeUsage + : undefined; + const total = record(captured ?? cumulativeNotification ?? usage.total ?? usage.totalTokenUsage ?? usage.total_token_usage); const integer = (...names: string[]): number => { for (const name of names) { const value = total[name]; @@ -1165,18 +1178,32 @@ export class CapabilityLiveSession { reasoningTokens: integer("reasoningOutputTokens", "reasoningTokens", "reasoning_output_tokens"), }; if (cumulative.inputTokens + cumulative.outputTokens === 0) { - throw new Error("capability_live_usage_missing"); + throw new Error(`capability_live_usage_missing:${JSON.stringify({ + captured: captured !== undefined, + readKeys: Object.keys(read).sort(), + threadKeys: Object.keys(thread).sort(), + usageKeys: Object.keys(usage).sort(), + totalKeys: Object.keys(total).sort(), + })}`); } const prior = reconcileCapabilityLiveUsage(this.snapshot()); + const delta = captured === undefined + ? { + inputTokens: Math.max(0, cumulative.inputTokens - prior.inputTokens), + outputTokens: Math.max(0, cumulative.outputTokens - prior.outputTokens), + cachedInputTokens: Math.max(0, cumulative.cachedInputTokens - prior.cachedInputTokens), + reasoningTokens: Math.max(0, cumulative.reasoningTokens - prior.reasoningTokens), + } + : cumulative; await this.recordUsage({ receiptId: `${turnId}:usage`, turnId, providerCalls: 1, - providerRequests: captured?.providerRequests ?? 0, - inputTokens: Math.max(0, cumulative.inputTokens - prior.inputTokens), - outputTokens: Math.max(0, cumulative.outputTokens - prior.outputTokens), - cachedInputTokens: Math.max(0, cumulative.cachedInputTokens - prior.cachedInputTokens), - reasoningTokens: Math.max(0, cumulative.reasoningTokens - prior.reasoningTokens), + providerRequests: captured?.providerRequests ?? 1, + inputTokens: delta.inputTokens, + outputTokens: delta.outputTokens, + cachedInputTokens: delta.cachedInputTokens, + reasoningTokens: delta.reasoningTokens, costNanodollars: 0, }); } @@ -1352,8 +1379,18 @@ export class CapabilityLiveSession { async #connect(resume: boolean): Promise { this.#status = "starting"; + const identityDigest = createHash("sha256").update(this.id).digest("hex").slice(0, 20); const transportBundle = this.#transportFactory({ ...this.#transportOptions, + stateDirectory: resolve(this.#config.workingDirectory, ".paperclip-runner-prp", identityDigest), + prpIdentity: { + runnerInstanceId: `runner_lab_${identityDigest}`, + environmentLeaseId: `lease_lab_${identityDigest}`, + runId: this.#authority.runId, + normalizedSessionId: this.id, + turnId: `turn_lab_${identityDigest}`, + itemId: `item_lab_${identityDigest}`, + }, onDiagnostic: (message) => { this.#transportOptions.onDiagnostic?.(message); this.#appendEvidence("diagnostic", this.#activeTurnId, { @@ -1663,24 +1700,47 @@ export class CapabilityLiveSession { const item = record(params.item); const turnId = text(params.turnId, text(turn.id)); const providerEvent = capabilityProviderEventCategory(notification.method, params); + if (notification.method === "thread/tokenUsage/updated") { + const tokenUsage = record(params.tokenUsage); + this.#latestCumulativeUsage = record( + tokenUsage.total ?? tokenUsage.totalTokenUsage ?? tokenUsage.total_token_usage ?? tokenUsage, + ); + } this.#appendEvidence("provider_event", turnId || null, { method: notification.method, params: jsonValue(params), }); - if (notification.method === "rawResponse/completed") { + if (notification.method === "rawResponse/completed" || notification.method === "turn/completed") { const rawUsage = record(params.usage); - if (turnId.length > 0 && Object.keys(rawUsage).length > 0) { - const previous = this.#rawUsageByTurn.get(turnId) ?? { providerRequests: 0, inputTokens: 0, outputTokens: 0, cachedInputTokens: 0, reasoningTokens: 0 }; + // Current Codex builds may omit turnId from rawResponse/completed. Turns + // are serialized by this session, so the active or just-terminal turn is + // the unambiguous owner of that usage receipt. + const usageTurnId = turnId || this.#activeTurnId || this.#terminalTurns.at(-1)?.turnId || ""; + if (usageTurnId.length > 0 && Object.keys(rawUsage).length > 0) { + const previous = this.#rawUsageByTurn.get(usageTurnId); const value = (name: string): number => Number.isSafeInteger(rawUsage[name]) && Number(rawUsage[name]) >= 0 ? Number(rawUsage[name]) : 0; - this.#rawUsageByTurn.set(turnId, { - providerRequests: previous.providerRequests + 1, - inputTokens: previous.inputTokens + value("inputTokens"), - outputTokens: previous.outputTokens + value("outputTokens"), - cachedInputTokens: previous.cachedInputTokens + value("cachedInputTokens"), - reasoningTokens: previous.reasoningTokens + value("reasoningOutputTokens"), - }); + if (notification.method === "rawResponse/completed") { + const base = previous ?? { providerRequests: 0, inputTokens: 0, outputTokens: 0, cachedInputTokens: 0, reasoningTokens: 0 }; + this.#rawUsageByTurn.set(usageTurnId, { + providerRequests: base.providerRequests + 1, + inputTokens: base.inputTokens + value("inputTokens"), + outputTokens: base.outputTokens + value("outputTokens"), + cachedInputTokens: base.cachedInputTokens + value("cachedInputTokens"), + reasoningTokens: base.reasoningTokens + value("reasoningOutputTokens"), + }); + } else if (previous === undefined) { + // Some provider versions publish only a terminal per-turn receipt. + this.#rawUsageByTurn.set(usageTurnId, { + providerRequests: 1, + inputTokens: value("inputTokens"), + outputTokens: value("outputTokens"), + cachedInputTokens: value("cachedInputTokens"), + reasoningTokens: value("reasoningOutputTokens"), + }); + } } - } else if (notification.method === "turn/started") { + } + if (notification.method === "turn/started") { if (turnId.length > 0) this.#activeTurnId = turnId; this.#emit({ turnId: turnId || this.#activeTurnId, kind: "activity", reason: "turn_started" }); } else if (notification.method === "item/agentMessage/delta") { diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts new file mode 100644 index 00000000000..f0cd3423beb --- /dev/null +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts @@ -0,0 +1,53 @@ +import { tmpdir } from "node:os"; +import { resolve } from "node:path"; + +import { expect, it } from "vitest"; + +import { createCapabilityRunnerdCodexTransport, defaultCapabilityRunnerdBinary } from "./runnerd-codex-transport.js"; + +const fakeCodex = resolve( + import.meta.dirname, + "../../runner/target/debug/fake-codex-app-server", +); + +it("runs the lab provider boundary through authenticated durable PRP", async () => { + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodex, + codexArgs: [], + }); + bundle.transport.setServerRequestHandler(async (request) => ({ + success: true, + contentItems: [{ + type: "inputText", + text: JSON.stringify({ ok: true, result: { task: { title: "PRP lab task" } } }), + }], + })); + try { + await bundle.transport.request("initialize", {}); + const opened = await bundle.transport.request("thread/start", { + cwd: tmpdir(), + dynamicTools: [{ + name: "get_task_context", + description: "Read the active task.", + inputSchema: { type: "object", properties: {}, additionalProperties: false }, + }], + }); + expect(opened.thread).toMatchObject({ modelProvider: "openai" }); + await bundle.transport.request("turn/start", { + input: [{ type: "text", text: "Read the task." }], + }); + const methods: string[] = []; + for await (const notification of bundle.transport.notifications()) { + methods.push(notification.method); + if (notification.method === "turn/completed") break; + } + expect(methods).toContain("turn/completed"); + expect(bundle.evidence().diagnostics).toContain( + "runnerd authenticated to the external mock control plane over durable PRP", + ); + } finally { + await bundle.transport.close(); + } + expect(bundle.evidence()).toMatchObject({ runnerExited: true, runnerExitCode: 0 }); +}, 30_000); diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts index b794807935e..9dac687d922 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts @@ -1,16 +1,30 @@ +import { createHash, randomUUID } from "node:crypto"; +import { existsSync, mkdirSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; import { resolve } from "node:path"; import { fileURLToPath } from "node:url"; -import { - ProcessCodexAppServerTransport, - createSanitizedCodexEnvironment, - type CodexAppServerTransport, - type CodexTransportProcessInfo, +import type { + CodexAppServerTransport, + CodexRpcNotification, + CodexRpcServerRequest, + CodexServerRequestHandler, + CodexTransportProcessInfo, } from "../drivers/codex/app-server-transport.js"; +import { createSanitizedCodexEnvironment } from "../drivers/codex/app-server-transport.js"; import { createIsolatedCodexAppServerArgs } from "../drivers/codex/codex-app-server-driver.js"; +import { + DurableRecoveryMockCore, + durableRecoveryInternals, + spawnRunner, + waitForProcess, + type RunnerProcessHandle, +} from "../mock-core/durable-recovery.js"; const packageRoot = fileURLToPath(new URL("../..", import.meta.url)); const executableSuffix = process.platform === "win32" ? ".exe" : ""; +const MAX_NOTIFICATION_COUNT = 2_048; +const MAX_NOTIFICATION_BYTES = 4 * 1024 * 1024; export interface CapabilityRunnerdProcessEvidence { runnerPid: number | null; @@ -31,6 +45,15 @@ export interface CapabilityRunnerdCodexTransportOptions { closeGraceMs?: number; onDiagnostic?: (message: string) => void; onEvidence?: (evidence: Readonly) => void; + stateDirectory?: string; + prpIdentity?: { + runnerInstanceId: string; + environmentLeaseId: string; + runId: string; + normalizedSessionId: string; + turnId: string; + itemId: string; + }; } export interface CapabilityRunnerdCodexTransport { @@ -38,69 +61,405 @@ export interface CapabilityRunnerdCodexTransport { evidence(): Readonly; } -export function defaultCapabilityRunnerdBinary(): string { - return resolve( - packageRoot, - `runner/target/debug/paperclip-runnerd${executableSuffix}`, - ); +class NotificationQueue implements AsyncIterable { + #values: Array<{ value: CodexRpcNotification; bytes: number }> = []; + #waiters: Array<(value: IteratorResult) => void> = []; + #bytes = 0; + #closed = false; + + push(value: CodexRpcNotification): void { + if (this.#closed) return; + const bytes = Buffer.byteLength(JSON.stringify(value)); + const waiter = this.#waiters.shift(); + if (waiter !== undefined) { + waiter({ value, done: false }); + return; + } + if (this.#values.length >= MAX_NOTIFICATION_COUNT || this.#bytes + bytes > MAX_NOTIFICATION_BYTES) { + throw new Error("PRP provider notification queue bound exceeded"); + } + this.#values.push({ value, bytes }); + this.#bytes += bytes; + } + + close(): void { + if (this.#closed) return; + this.#closed = true; + this.#values = []; + this.#bytes = 0; + for (const waiter of this.#waiters.splice(0)) waiter({ value: undefined, done: true }); + } + + [Symbol.asyncIterator](): AsyncIterator { + return { + next: async () => { + const queued = this.#values.shift(); + if (queued !== undefined) { + this.#bytes -= queued.bytes; + return { value: queued.value, done: false }; + } + if (this.#closed) return { value: undefined, done: true }; + return new Promise((resolveValue) => this.#waiters.push(resolveValue)); + }, + }; + } } -/** - * Starts the real package-local runnerd as process owner. runnerd then starts - * the real Codex app-server and transparently proxies its stdio JSON-RPC. - */ +function record(value: unknown): Record { + return typeof value === "object" && value !== null && !Array.isArray(value) + ? value as Record + : {}; +} + +function commandDigest(value: unknown): string { + return `sha256:${createHash("sha256").update(durableRecoveryInternals.canonicalJson(value)).digest("hex")}`; +} + +function unwrapToolResponse(response: Record): unknown { + const items = Array.isArray(response.contentItems) ? response.contentItems : []; + const value = record(items[0]).text; + if (typeof value !== "string") return response; + try { + return JSON.parse(value); + } catch { + return response; + } +} + +class DurablePrpCodexTransport implements CodexAppServerTransport { + readonly #root: string; + readonly #ownsRoot: boolean; + readonly #queue = new NotificationQueue(); + readonly #startedAt = new Date().toISOString(); + readonly #evidence: CapabilityRunnerdProcessEvidence; + #handler: CodexServerRequestHandler = async () => ({ + success: false, + contentItems: [{ type: "inputText", text: "No Paperclip control-plane tool handler is installed." }], + }); + #core: DurableRecoveryMockCore | null = null; + #handle: RunnerProcessHandle | null = null; + #pump: NodeJS.Timeout | null = null; + #eventIndex = 0; + #threadId = ""; + #sessionId: string | null = null; + #turnId = ""; + #closed = false; + + constructor(readonly options: CapabilityRunnerdCodexTransportOptions) { + this.#ownsRoot = options.stateDirectory === undefined; + this.#root = options.stateDirectory ?? mkdtempSync(resolve(tmpdir(), "paperclip-runner-lab-prp-")); + mkdirSync(this.#root, { recursive: true, mode: 0o700 }); + this.#evidence = { + runnerPid: null, + runnerProcessGroupId: null, + codexPid: null, + runnerExited: false, + runnerExitCode: null, + runnerSignal: null, + childEnvironmentKeys: Object.keys(createSanitizedCodexEnvironment(options.environment)).sort(), + diagnostics: ["lab transport selected authenticated durable PRP"], + }; + } + + evidence(): CapabilityRunnerdProcessEvidence { + return structuredClone(this.#evidence); + } + + async request(method: string, params: Record): Promise> { + if (this.#closed) throw new Error("PRP Codex transport is closed"); + if (method === "initialize") return { user: {} }; + if (method === "thread/start") return this.#start(params); + if (method === "turn/start") return this.#startTurn(params); + if (method === "turn/interrupt") { + await this.#command("turn.interrupt", params); + return {}; + } + if (method === "thread/read") { + if (this.#core === null) await this.#resume(); + return this.#commandResult("provider.thread.read", {}); + } + if (method === "thread/resume") return { thread: { id: this.#threadId, sessionId: this.#sessionId } }; + throw new Error(`PRP Codex transport does not expose provider method ${method}`); + } + + notify(_method: string, _params?: Record): void {} + + notifications(): AsyncIterable { + return this.#queue; + } + + setServerRequestHandler(handler: CodexServerRequestHandler): void { + this.#handler = handler; + } + + processInfo(): CodexTransportProcessInfo { + return { + pid: this.#evidence.runnerPid, + processGroupId: this.#evidence.runnerProcessGroupId, + startedAt: this.#startedAt, + exited: this.#evidence.runnerExited, + exitCode: this.#evidence.runnerExitCode, + signal: this.#evidence.runnerSignal, + }; + } + + async close(): Promise { + if (this.#closed) return; + this.#closed = true; + if (this.#core !== null && this.#handle !== null) { + this.#core.queueCommand("runner.suspend", {}, undefined, true); + try { + const result = await waitForProcess(this.#handle, this.options.closeGraceMs ?? 10_000); + this.#evidence.runnerExited = true; + this.#evidence.runnerExitCode = result.code; + this.#evidence.runnerSignal = result.signal as NodeJS.Signals | null; + if (result.stderr.trim()) this.#diagnostic(result.stderr.trim().slice(-4_096)); + } catch (error) { + this.#diagnostic(`runner shutdown failed: ${String(error)}`); + } + } + if (this.#pump !== null) clearInterval(this.#pump); + this.#pump = null; + this.#queue.close(); + await this.#core?.stop(); + this.#handle?.child.kill("SIGKILL"); + if (this.#ownsRoot) rmSync(this.#root, { recursive: true, force: true }); + this.#publish(); + } + + async #start(params: Record): Promise> { + if (this.#core !== null) throw new Error("PRP provider thread is already started"); + const token = randomUUID().replaceAll("-", ""); + const identity = this.options.prpIdentity ?? { + runnerInstanceId: `runner_lab_${token}`, + environmentLeaseId: `lease_lab_${token}`, + runId: `run_lab_${token}`, + normalizedSessionId: `session_lab_${token}`, + turnId: `turn_lab_${token}`, + itemId: `item_lab_${token}`, + }; + const dynamicTools = Array.isArray(params.dynamicTools) ? params.dynamicTools.map(record) : []; + const operations = dynamicTools.map((tool) => ({ + operationId: String(tool.name ?? ""), + version: 1, + description: String(tool.description ?? ""), + inputSchema: record(tool.inputSchema), + responseSchema: {}, + })); + const core = new DurableRecoveryMockCore({ + stateDirectory: resolve(this.#root, "control-plane"), + identity, + fault: "none", + onSemanticToolInput: async (call) => unwrapToolResponse(await this.#handler({ + id: call.callId, + method: "item/tool/call", + params: { + threadId: this.#threadId, + turnId: this.#turnId, + callId: call.callId, + tool: call.operationId, + arguments: call.input, + }, + })), + connectionLeaseTtlMs: 60 * 60 * 1_000, + }); + this.#core = core; + mkdirSync(resolve(this.#root, "runner"), { recursive: true, mode: 0o700 }); + core.queueCommand("run.prepare", { + authorizedTools: { + schema: "paperclip.runner.authorized-tools.v1", + schemaVersion: 1, + catalogDigest: commandDigest(operations), + operations, + }, + provider: { + command: this.options.codexCommand ?? "codex", + args: this.options.codexArgs ?? createIsolatedCodexAppServerArgs(this.options.environment), + cwd: String(params.cwd ?? tmpdir()), + model: typeof params.model === "string" ? params.model : null, + instructions: String(params.baseInstructions ?? "You are a Paperclip agent."), + }, + }); + core.queueCommand("session.open", { reuse: "same_session" }); + await core.start(); + const handle = spawnRunner({ + connectUrl: core.connectUrl, + stateDirectory: resolve(this.#root, "runner"), + identity, + ticket: core.issueBootstrapTicket(), + maxOutboxBytes: 256 * 1024, + p0ReserveBytes: 64 * 1024, + maxRuntimeMs: 60 * 60 * 1_000, + runnerBinaryPath: this.options.runnerBinary ?? defaultCapabilityRunnerdBinary(), + }); + this.#handle = handle; + this.#evidence.runnerPid = handle.child.pid ?? null; + this.#evidence.runnerProcessGroupId = null; + this.#publish(); + this.#pump = setInterval(() => this.#pumpEvents(), 5); + await this.#waitCommand("run.prepare"); + await this.#waitCommand("session.open"); + await this.#waitForProviderIdentity(); + this.#diagnostic("runnerd authenticated to the external mock control plane over durable PRP"); + return { thread: { id: this.#threadId, sessionId: this.#sessionId, model: params.model, modelProvider: "openai" } }; + } + + async #resume(): Promise { + const identity = this.options.prpIdentity; + if (identity === undefined || !existsSync(resolve(this.#root, "runner", "runner-state.json"))) { + throw new Error("PRP provider resume state is unavailable"); + } + const core = new DurableRecoveryMockCore({ + stateDirectory: resolve(this.#root, "control-plane"), + identity, + fault: "none", + onSemanticToolInput: async (call) => unwrapToolResponse(await this.#handler({ + id: call.callId, + method: "item/tool/call", + params: { + threadId: this.#threadId, + turnId: this.#turnId, + callId: call.callId, + tool: call.operationId, + arguments: call.input, + }, + })), + connectionLeaseTtlMs: 60 * 60 * 1_000, + }); + this.#core = core; + this.#eventIndex = core.store.state.committedEvents.length; + await core.start(); + const handle = spawnRunner({ + connectUrl: core.connectUrl, + stateDirectory: resolve(this.#root, "runner"), + identity, + ticket: core.issueBootstrapTicket(), + maxOutboxBytes: 256 * 1024, + p0ReserveBytes: 64 * 1024, + maxRuntimeMs: 60 * 60 * 1_000, + runnerBinaryPath: this.options.runnerBinary ?? defaultCapabilityRunnerdBinary(), + }); + this.#handle = handle; + this.#evidence.runnerPid = handle.child.pid ?? null; + this.#evidence.runnerProcessGroupId = null; + this.#publish(); + this.#pump = setInterval(() => this.#pumpEvents(), 5); + await this.#waitForProviderIdentity(); + this.#diagnostic("runnerd restored its durable PRP session and provider thread"); + } + + async #startTurn(params: Record): Promise> { + const input = Array.isArray(params.input) ? params.input.map(record) : []; + const message = input.map((item) => typeof item.text === "string" ? item.text : "").join("\n"); + this.#turnId = `turn_lab_${randomUUID().replaceAll("-", "")}`; + await this.#command("turn.start", { text: message }); + return { turn: { id: this.#turnId, status: "inProgress" } }; + } + + async #command(type: string, payload: Record): Promise { + const core = this.#core; + if (core === null) throw new Error("PRP provider thread is not started"); + const commandId = `command_lab_${randomUUID().replaceAll("-", "")}`; + core.queueCommand(type, payload, commandId, true); + await this.#waitCommand(type, commandId); + } + + async #commandResult(type: string, payload: Record): Promise> { + const core = this.#core; + if (core === null) throw new Error("PRP provider thread is not started"); + const commandId = `command_lab_${randomUUID().replaceAll("-", "")}`; + core.queueCommand(type, payload, commandId, true); + await this.#waitCommand(type, commandId); + const deadline = Date.now() + 30_000; + while (Date.now() < deadline) { + const event = core.store.state.committedEvents.find((candidate) => { + if (candidate.eventType !== "provider.rpc_result") return false; + return record(record(candidate.envelope.payload).payload).commandId === commandId; + }); + const result = record(record(record(event?.envelope.payload).payload).result); + if (event !== undefined) return result; + await new Promise((resolveWait) => setTimeout(resolveWait, 10)); + } + throw new Error(`PRP command ${type} omitted its provider result`); + } + + async #waitForProviderIdentity(): Promise { + const deadline = Date.now() + 30_000; + while (Date.now() < deadline) { + this.#pumpEvents(); + if (this.#threadId.length > 0 && this.#evidence.codexPid !== null) return; + if (this.#handle?.child.exitCode !== null) throw new Error("runnerd exited before provider startup"); + await new Promise((resolveWait) => setTimeout(resolveWait, 10)); + } + throw new Error("runnerd did not report its provider identity"); + } + + async #waitCommand(type: string, commandId?: string): Promise { + const deadline = Date.now() + 30_000; + while (Date.now() < deadline) { + const command = this.#core?.store.state.commands.find((candidate) => + commandId === undefined ? candidate.type === type : candidate.commandId === commandId + ); + if (command?.status === "completed") return; + if (command !== undefined && command.status !== "pending") { + throw new Error(`PRP command ${type} ${command.status}: ${JSON.stringify(command.result)}`); + } + if (this.#handle?.child.exitCode !== null) throw new Error(`runnerd exited while waiting for ${type}`); + await new Promise((resolveWait) => setTimeout(resolveWait, 10)); + } + throw new Error(`PRP command ${type} timed out`); + } + + #pumpEvents(): void { + const events = this.#core?.store.state.committedEvents ?? []; + while (this.#eventIndex < events.length) { + const event = events[this.#eventIndex++]; + if (event.eventType === "provider.started") { + const started = record(record(event.envelope.payload).payload); + const pid = started.pid; + const threadId = started.threadId; + const sessionId = started.sessionId; + if (typeof pid === "number") { + this.#evidence.codexPid = pid; + } + if (typeof threadId === "string") this.#threadId = threadId; + if (typeof sessionId === "string") this.#sessionId = sessionId; + this.#publish(); + continue; + } + if (event.eventType !== "provider.event" && event.eventType !== "turn.completed") continue; + const payload = record(record(event.envelope.payload).payload); + const method = payload.method; + if (typeof method !== "string") continue; + const params = record(payload.params); + if (method === "turn/started") { + const providerTurnId = record(params.turn).id; + if (typeof providerTurnId === "string") this.#turnId = providerTurnId; + } + this.#queue.push({ method, params }); + } + } + + #diagnostic(message: string): void { + this.#evidence.diagnostics.push(message); + if (this.#evidence.diagnostics.length > 64) this.#evidence.diagnostics.shift(); + this.options.onDiagnostic?.(message); + this.#publish(); + } + + #publish(): void { + this.options.onEvidence?.(this.evidence()); + } +} + +export function defaultCapabilityRunnerdBinary(): string { + return resolve(packageRoot, `runner/target/debug/paperclip-runnerd${executableSuffix}`); +} + +/** Uses the same external mock control-plane -> authenticated PRP -> runnerd -> provider topology as evals. */ export function createCapabilityRunnerdCodexTransport( options: CapabilityRunnerdCodexTransportOptions = {}, ): CapabilityRunnerdCodexTransport { - const environment = createSanitizedCodexEnvironment(options.environment); - const evidence: CapabilityRunnerdProcessEvidence = { - runnerPid: null, - runnerProcessGroupId: null, - codexPid: null, - runnerExited: false, - runnerExitCode: null, - runnerSignal: null, - childEnvironmentKeys: Object.keys(environment).sort(), - diagnostics: [], - }; - const publish = () => options.onEvidence?.(structuredClone(evidence)); - const diagnostic = (message: string) => { - evidence.diagnostics.push(message); - if (evidence.diagnostics.length > 64) evidence.diagnostics.shift(); - const match = /(?:capability|native) codex proxy started runner_pid=(\d+) codex_pid=(\d+)/.exec(message); - if (match) { - evidence.runnerPid = Number(match[1]); - evidence.codexPid = Number(match[2]); - } - options.onDiagnostic?.(message); - publish(); - }; - const processUpdate = (info: CodexTransportProcessInfo) => { - evidence.runnerPid = info.pid; - evidence.runnerProcessGroupId = info.processGroupId; - evidence.runnerExited = info.exited; - evidence.runnerExitCode = info.exitCode; - evidence.runnerSignal = info.signal; - publish(); - }; - const codexArgs = options.codexArgs ?? createIsolatedCodexAppServerArgs(options.environment); - const runnerArgs = [ - "--codex-app-server-proxy", - "--codex-command", - options.codexCommand ?? "codex", - ...codexArgs.flatMap((argument) => ["--codex-arg", argument]), - ]; - const transport = new ProcessCodexAppServerTransport({ - command: options.runnerBinary ?? defaultCapabilityRunnerdBinary(), - args: runnerArgs, - environment, - processGroup: true, - closeGraceMs: options.closeGraceMs, - onDiagnostic: diagnostic, - onProcess: processUpdate, - }); - return { - transport, - evidence: () => structuredClone(evidence), - }; + const transport = new DurablePrpCodexTransport(options); + return { transport, evidence: () => transport.evidence() }; } diff --git a/packages/paperclip-runner/src/live/turn-stream.test.ts b/packages/paperclip-runner/src/live/turn-stream.test.ts index 05ec8e8678b..bb7b97bdeb1 100644 --- a/packages/paperclip-runner/src/live/turn-stream.test.ts +++ b/packages/paperclip-runner/src/live/turn-stream.test.ts @@ -103,6 +103,7 @@ class GatedTransport implements CodexAppServerTransport { return { turn: { id: "turn-1", status: "inProgress" } }; } if (method === "turn/interrupt") { + this.queue.push({ method: "rawResponse/completed", params: { threadId: "provider-thread", turnId: String(params.turnId), usage: { inputTokens: 3, outputTokens: 1, cachedInputTokens: 0, reasoningOutputTokens: 0 } } }); this.queue.push({ method: "turn/completed", params: { threadId: "provider-thread", turn: { id: String(params.turnId), status: "interrupted" } }, @@ -150,6 +151,7 @@ class GatedTransport implements CodexAppServerTransport { item: { id: "message-1", type: "agentMessage", text: FINAL_TEXT }, }, }); + this.queue.push({ method: "rawResponse/completed", params: { threadId: "provider-thread", turnId, usage: { inputTokens: 3, outputTokens: 1, cachedInputTokens: 0, reasoningOutputTokens: 0 } } }); this.queue.push({ method: "turn/completed", params: { threadId: "provider-thread", turn: { id: turnId, status: "completed" } }, diff --git a/packages/paperclip-runner/src/mock-core/durable-recovery.test.ts b/packages/paperclip-runner/src/mock-core/durable-recovery.test.ts index 9314aeff211..776faecb482 100644 --- a/packages/paperclip-runner/src/mock-core/durable-recovery.test.ts +++ b/packages/paperclip-runner/src/mock-core/durable-recovery.test.ts @@ -21,6 +21,7 @@ import { DurableRecoveryMockCore, durableRecoveryInternals, runDurableRecoveryRecovery, + runDurableToolBridgeConformance, } from "./durable-recovery.js"; async function upgradeSocket(url: string): Promise { @@ -365,6 +366,19 @@ async function runRunnerProcess(options: { } describe.sequential("Durable transport and recovery", () => { + it("repeatedly completes the provider bridge and explicit runner shutdown", async () => { + for (let attempt = 0; attempt < 10; attempt += 1) { + const result = await runDurableToolBridgeConformance(); + expect(result, `bridge attempt ${attempt + 1}`).toMatchObject({ + operationId: "get_task_context", + toolInputCommitted: true, + toolResultCompleted: true, + providerTurnCompleted: true, + runnerExitCode: 0, + }); + } + }, 30_000); + it("accepts a short-lived bootstrap ticket once and rejects its reuse", async () => { const scratch = process.env.PAPERCLIP_RUN_SCRATCH_DIR ?? diff --git a/packages/paperclip-runner/src/mock-core/durable-recovery.ts b/packages/paperclip-runner/src/mock-core/durable-recovery.ts index 1a13e992d6b..ef854f713cb 100644 --- a/packages/paperclip-runner/src/mock-core/durable-recovery.ts +++ b/packages/paperclip-runner/src/mock-core/durable-recovery.ts @@ -183,14 +183,14 @@ interface MockCoreOptions { connectionLeaseTtlMs?: number; } -interface RunnerProcessResult { +export interface RunnerProcessResult { code: number | null; signal: NodeJS.Signals | null; stdout: string; stderr: string; } -interface RunnerProcessHandle { +export interface RunnerProcessHandle { child: ChildProcessWithoutNullStreams; completion: Promise; } @@ -1342,7 +1342,7 @@ function runnerEnvironment(ticket: string): NodeJS.ProcessEnv { return environment; } -function spawnRunner(options: { +export function spawnRunner(options: { connectUrl: string; stateDirectory: string; identity: DurableRecoveryIdentity; @@ -1406,7 +1406,7 @@ function spawnRunner(options: { return { child, completion }; } -async function waitForProcess( +export async function waitForProcess( handle: RunnerProcessHandle, timeoutMs = 15_000, ): Promise { @@ -1617,6 +1617,57 @@ export interface DurableEvalSessionInput { toolExposure?: "eager" | "lazy"; } +export type DurableEvalInfrastructureFailureClass = + | "provider_turn_timeout" + | "runner_shutdown_timeout" + | "runner_exit_failure"; + +export class DurableEvalInfrastructureError extends Error { + readonly failureClass: DurableEvalInfrastructureFailureClass; + readonly retryable: boolean; + readonly diagnostics: Record; + + constructor( + failureClass: DurableEvalInfrastructureFailureClass, + message: string, + retryable: boolean, + diagnostics: Record, + ) { + super(message); + this.name = "DurableEvalInfrastructureError"; + this.failureClass = failureClass; + this.retryable = retryable; + this.diagnostics = diagnostics; + } +} + +function evalLifecycleDiagnostics( + core: DurableRecoveryMockCore, + handle: RunnerProcessHandle, +): Record { + const commands = core.store.state.commands.map((command) => ({ + commandId: command.commandId, + type: command.type, + status: command.status, + deliveryCount: core.store.state.commandDeliveryCounts[command.commandId] ?? 0, + })); + const events = core.store.state.committedEvents.slice(-20).map((event) => ({ + sourceSeq: event.sourceSeq, + eventType: event.eventType, + priority: event.priority, + deliveryCount: event.deliveryCount, + })); + return { + runnerPid: handle.child.pid ?? null, + runnerExitCode: handle.child.exitCode, + runnerSignal: handle.child.signalCode, + connectionCount: core.store.state.connectionCount, + ackedSourceSeq: core.store.state.ackedSourceSeq, + commands, + recentEvents: events, + }; +} + /** Production-topology eval: the only Paperclip process in the sandbox is runnerd. */ export async function runDurableEvalSession(input: DurableEvalSessionInput): Promise> { const identity = { @@ -1722,11 +1773,33 @@ export async function runDurableEvalSession(input: DurableEvalSessionInput): Pro await new Promise((resolveWait) => setTimeout(resolveWait, 20)); } if (!core.store.state.committedEvents.some((event) => event.eventType === "turn.completed")) { - throw new Error(`durable provider turn timed out after ${input.turnTimeoutMs}ms (state: ${root})`); + throw new DurableEvalInfrastructureError( + "provider_turn_timeout", + `durable provider turn timed out after ${input.turnTimeoutMs}ms`, + true, + evalLifecycleDiagnostics(core, handle), + ); } core.queueCommand("runner.shutdown", {}, undefined, true); - const exited = await waitForProcess(handle, 30_000); - if (exited.code !== 0) throw new Error(`runnerd exited ${String(exited.code)}: ${exited.stderr}`); + let exited: RunnerProcessResult; + try { + exited = await waitForProcess(handle, 30_000); + } catch (error) { + throw new DurableEvalInfrastructureError( + "runner_shutdown_timeout", + `runnerd did not exit within 30000ms after runner.shutdown: ${error instanceof Error ? error.message : String(error)}`, + true, + evalLifecycleDiagnostics(core, handle), + ); + } + if (exited.code !== 0) { + throw new DurableEvalInfrastructureError( + "runner_exit_failure", + `runnerd exited ${String(exited.code)}: ${exited.stderr}`, + false, + { ...evalLifecycleDiagnostics(core, handle), stderr: exited.stderr.slice(-16_384) }, + ); + } const records = core.store.state.committedEvents.map((committed) => committed.envelope.payload as Record); const evidence: Array> = [{ id: "exposure-1", kind: "tool_exposure", at: new Date().toISOString(), turnId: null, diff --git a/packages/paperclip-runner/src/mock-core/live-console-demo-server.test.ts b/packages/paperclip-runner/src/mock-core/live-console-demo-server.test.ts index 391c770c1c6..c370aaf6e1b 100644 --- a/packages/paperclip-runner/src/mock-core/live-console-demo-server.test.ts +++ b/packages/paperclip-runner/src/mock-core/live-console-demo-server.test.ts @@ -407,7 +407,7 @@ async function readEventStreamReplay(url: string, sessionId: string): Promise entry.subscribers.delete(response)); + const unsubscribe = () => entry.subscribers.delete(response); + request.once("close", unsubscribe); + response.once("close", unsubscribe); return; } if (request.method !== "POST") { @@ -823,7 +825,13 @@ export class LiveConsoleDemoServer { if (entry.events.length >= MAX_BROWSER_EVENTS) entry.events.shift(); entry.events.push(event); const serialized = `data: ${output.serialized}\n\n`; - for (const subscriber of entry.subscribers) subscriber.write(serialized); + for (const subscriber of entry.subscribers) { + if (subscriber.destroyed || subscriber.writableEnded) { + entry.subscribers.delete(subscriber); + continue; + } + subscriber.write(serialized); + } } } diff --git a/packages/paperclip-runner/src/tools/capability-semantic-tool-runtime.ts b/packages/paperclip-runner/src/tools/capability-semantic-tool-runtime.ts index ea6fd3e36f7..6f07502531f 100644 --- a/packages/paperclip-runner/src/tools/capability-semantic-tool-runtime.ts +++ b/packages/paperclip-runner/src/tools/capability-semantic-tool-runtime.ts @@ -483,6 +483,7 @@ export function validateJsonSchema(schema: CapabilityJsonSchema, value: Capabili } function matchesType(type: NonNullable, value: CapabilityJsonValue): boolean { + if (Array.isArray(type)) return type.some((candidate) => matchesType(candidate, value)); switch (type) { case "null": return value === null; case "array": return Array.isArray(value);