diff --git a/doc/run-log-events.md b/doc/run-log-events.md index 402f3e7ef82..7b0abaa8b50 100644 --- a/doc/run-log-events.md +++ b/doc/run-log-events.md @@ -34,6 +34,47 @@ credential material are never written to the run log. These records remain run-log events. They do not create an OpenTelemetry or Paperclip Telemetry export, and legacy adapters do not use this writer. +## Semantic Settlement Diagnostics and Retry Logs + +Before interrupting a Codex turn, the runner durably records a +`harness.diagnostic` event with code `provider_interrupt_requested`, the stop +reason, provider turn ID, and pending tool call IDs. A harmless identical result +replay records `semantic_tool_result_duplicate` with warning severity, call ID, +operation ID, and result digest. It does not fail the task or create a user +attention request. A true conflict includes the call and operation IDs and both +result digests in its error; full arguments belong to the canonical semantic +input record, not the error message. + +Incomplete close emits the bounded `native_session_settlement_incomplete` +runner diagnostic. Its evidence includes runner suspension, provider drain, +final provider state, pending call/operation/source-event IDs and input digests, +and incomplete result-delivery command IDs and statuses. If execution and +cleanup both fail, execution retains its original error identity and cleanup is +attached as `cleanupError`. + +Instruction writes also commit an `agent.instruction_write_attempted` activity +row and a run-scoped `instructionToolAttempts` entry before permitting the +filesystem effect. They retain the call ID, operation ID, and input digest, not +instruction text. A later transaction rollback cannot erase this attempt proof. +A missing success or definite pre-write failure receipt means the outcome is +unknown, even if the current file contains the requested text. Known validation +and stale-base failures are saved under the attempt's `failure` entry and replay +their original status, message, and details without a new write. Unknown +outcomes remain blocked until reconciled; a +committed receipt with a lost acknowledgement can replay its exact result. + +The NDJSON output log appends across repeated `begin` calls for one run. Each +new handle adds an `attemptId` to its lines. If the local file is absent and a +durable S3 mirror exists, `begin` restores that prefix before appending. A +failed restore must not replace the mirror with an empty or partial attempt. +Publishing a restored prefix uses an atomic create-if-absent operation, so a +concurrent restore cannot overwrite lines another attempt has already appended. +Earlier attempts therefore remain available for incident diagnosis. Existing +records without `attemptId` remain readable. + +These records use the instance run log and its configured storage. They add no +Paperclip Telemetry or OpenTelemetry export. + ## Omitted Unsafe Workspace Export `workspace_export_omitted` is an informational system event in the local run log. diff --git a/packages/paperclip-runner/docs/durable-recovery.md b/packages/paperclip-runner/docs/durable-recovery.md index 857de7b7121..8ee0b22c975 100644 --- a/packages/paperclip-runner/docs/durable-recovery.md +++ b/packages/paperclip-runner/docs/durable-recovery.md @@ -185,6 +185,105 @@ The trace records `logicalEffectCount`. Every completed command has exactly one logical effect. A policy rejection, such as a new turn during drain or storage pressure, has zero effects. +## Semantic operations that outlive a turn + +An execution session includes the provider process, the runner's durable call +receipts and event outbox, and the controller's durable command queue. Ending a +provider turn closes admission of new work. It does not establish whether a +semantic operation already dispatched to the server succeeded or failed. + +The tool bridge retains pending call IDs, operation IDs, and exact arguments +across that boundary. It accepts the authority's late result without delivering +it to an obsolete provider turn. An identical result is an idempotent replay; +a different result for that call remains a conflict. Codex records harmless +result replay as `semantic_tool_result_duplicate` in a diagnostic event, with +no task failure. Conflict errors identify the call, operation, and both result +digests without including the result bodies. + +The controller checkpoints a reusable session only after every admitted tool +has a matching completed result-delivery command and the provider has drained. +A callback rejected before business dispatch can report +`semantic_tool_not_dispatched`. A callback rejected after dispatch has an +unknown outcome; the controller must not invent a failure or execute it again. +After controller restart, an already committed input without a result remains +unsettled. It requires authoritative reconciliation, not an automatic retry. +A different case is a runner crash after the controller saved the result but +before its delivery command completed. Codex/OpenCode explicitly opt in to +reconciling that exact `semantic_tool.result` command through their durable tool +receipt. The full command fingerprint must match. This delivers the saved +answer without executing the business operation. Unsupported providers and all +other indeterminate commands remain non-reexecutable. This change does not +repair legacy journals already containing contradictory or failed receipts. + +`update_agent_instructions` and `restore_agent_instructions` use a durable, +company/run-scoped mutation receipt keyed by call ID. Before attempting a file +write, the authority commits an `instructionToolAttempts` record and an +`agent.instruction_write_attempted` activity row with the call ID, operation ID, +and input digest. This evidence survives a later filesystem or database failure. +Completed calls and retries after authority restart return the exact original receipt. +Changing the arguments under that ID is an idempotency conflict. Receipt replay +still checks current instruction-write authorization. Concurrent callers in the +same process wait for the first handler before checking the receipt again. + +The filesystem and PostgreSQL are not one atomic store. If a write is visible +but its receipt transaction rolls back, the attempt record remains durable and +the result is unknown. An identical call with an attempt but no committed result +raises `paperclip_runner_instruction_outcome_unknown`; it does not write again +or pretend to have succeeded. This also applies to another process reaching a +reserved call before its effect transaction starts. The controller retains an +unsettled operation for authoritative reconciliation. If only the commit +acknowledgement was lost, the committed receipt is replayed normally. +The native tool wrapper propagates `SemanticToolOutcomeUnknownError` instead +of turning it into a completed failed tool response. Ordinary validation, +authorization, and stale-base errors still return normal tool errors. Before +returning a definite pre-write failure, the authority saves its status, message, +and details under the attempt's `failure` receipt. Replays return that original +error without executing again, even if the old CAS base becomes current again. +Failure-receipt storage errors retain an unsettled operation instead of exposing +an unrecorded final answer. + +### Lossless arguments and bounded transport + +Execution arguments do not use the diagnostic preview formatter. Accepted +input remains exact, including Unicode, line endings, and long document tails. +The credential policy rejects input it would have to alter before dispatch. +Diagnostic copies remain redacted and bounded. The semantic input limit is +480 KiB of encoded JSON, leaving room for the encrypted frame's hex expansion +and envelope inside the 1 MiB wire limit. Oversized input fails before a write; +it is never truncated into an apparently successful write. + +A result command retains a SHA-256 `inputDigest` instead of duplicating the +full input alongside the full result. The canonical input event retains the +arguments. Settlement validates the digest and all run, event, call, operation, +and correlation identities. Legacy result commands carrying the full input +remain supported. Large results still have to fit the command transport limit; +exceeding it leaves the outcome unsettled rather than retrying an effect. + +### Deterministic fault coverage + +These tests use explicit barriers, durable reloads, real runner processes with +scripted providers, and a real PostgreSQL database. They do not require a paid +model or rely on an LLM choosing the desired timing. + +| Fault | Required assertion | Regression suite | +| --- | --- | --- | +| Stop before/after server result; restart before late delivery | No invented failure; pending identity survives; late result accepted | Rust `provider_bridge`, `codex_provider` | +| Provider terminal before tool completion | New turn blocked until actual result; safe replay and later reuse | Rust `acpx_provider_turns`, `acpx_provider_state` | +| Crash before/after applying a saved delivery receipt, before command completion | Exact result delivery recovers only for opted-in providers; changed commands and ordinary operations never replay | Rust durable runner tests | +| Lost result acknowledgement; conflicting redelivery | Identical result accepted; changed result rejected with IDs and digests | Rust `codex_provider`, `provider_bridge` | +| Concurrent writes held at filesystem commit; authority restart | One write, one audit record, same receipt; revoked authorization rejected | Server `agent-instruction-tools.integration.test.ts` | +| Failure after file rename or rollback while saving the receipt | Durable attempt and audit evidence survive; no success claim and no second write | Server `agent-instruction-tools.integration.test.ts` | +| Receipt commits but its database acknowledgement is lost | Exact original result replays; one file-update audit and one attempt record | Server `agent-instruction-tools.integration.test.ts` | +| Tool handler throws an unknown-outcome error versus a validation error | Unknown propagates without a completed tool event; validation returns an ordinary failed response | `codex-app-server-driver.test.ts` | +| Invalid arguments, oversized input, or stale base; restart and replay | Original failure replays with zero writes, even after the base becomes valid | Server `agent-instruction-tools.integration.test.ts` | +| Failure receipt cannot be stored | No unrecorded final failure; attempt remains unsettled and never reexecutes | Server `agent-instruction-tools.integration.test.ts` | +| Long Unicode input and result; corrupted input digest | Exact bytes across persistence/wire; no copied input in result; bad proof blocks settlement | Rust durable-state tests; `durable-prp-control-plane.test.ts` | +| Handler finishes during close, after close budget, or persistence fails | Checkpoint marked settled only with completed delivery and drained provider | `runnerd-codex-transport.test.ts` | +| Controller restarts with committed input but no result | No second dispatch and no fabricated outcome | `durable-prp-control-plane.test.ts` | +| Execution and cleanup both fail | Original execution error retained; cleanup error attached | `native-session-runtime.test.ts` | +| Retry in same process or after local log loss | Earlier attempts survive locally and in the durable mirror | Server `run-log-store.test.ts` | +| Two restores overlap and one appends before the other publishes | Complete prefix is published without replacement; both attempts survive | Server `run-log-store.test.ts` | + ## Residual local trust and revocation window The authenticated session removes trust in whichever process wins the configured diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_session.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_session.rs index c92f0449402..a50de43ef68 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_session.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_session.rs @@ -355,6 +355,11 @@ impl AcpxProviderSession { "ACPX provider session already has an active turn", )); } + if self.state.has_pending_tools() { + return Err(LocalRunnerError::invalid( + "ACPX provider session still has unsettled semantic tools", + )); + } let rotate_turn_identity_ledger = self.state.settled_turn_identity_capacity_reached(); let identity_validation = if rotate_turn_identity_ledger { self.state @@ -618,47 +623,38 @@ impl AcpxProviderSession { } } AcpxProviderStateEvent::TurnTerminal { .. } => { - let settlements = match next_bridge.settle_turn("acpx_turn_settled") { - Ok(settlements) => settlements, - Err(error) => { - return Err(self.fail_closed(LocalRunnerError::invalid(format!( + next_bridge + .settle_turn("acpx_turn_settled") + .map_err(|error| { + self.fail_closed(LocalRunnerError::invalid(format!( "ACPX provider tool settlement failed: {error}" - )))); - } - }; - let reserved_settlements = match next_reserved_bridge - .settle_turn("acpx_reserved_terminal_unsettled") - { - Ok(settlements) => settlements, - Err(error) => { - return Err(self.fail_closed(LocalRunnerError::invalid(format!( - "ACPX reserved terminal settlement failed: {error}" - )))); - } - }; - if !reserved_settlements.is_empty() { - return Err(self.fail_closed(LocalRunnerError::invalid( - "ACPX turn terminated before its reserved terminal invocation produced a correlated result", - ))); - } - // `accept_event` clears the candidate reducer's pending - // tools while the bridge clones settle the corresponding - // calls above. Prove both halves reached the same terminal - // state before committing any of them to the reusable - // session. - if next_state.has_pending_tools() - || next_bridge.pending_calls().next().is_some() - || next_reserved_bridge.pending_calls().next().is_some() + ))) + })?; + next_reserved_bridge + .settle_turn("acpx_turn_settled") + .map_err(|error| { + self.fail_closed(LocalRunnerError::invalid(format!( + "ACPX reserved tool settlement failed: {error}" + ))) + })?; + // Both ledgers must retain exactly the same dispatched + // effects. Ending the provider turn cannot determine whether + // a server-side operation committed. + let pending: Vec<_> = next_bridge + .pending_calls() + .chain(next_reserved_bridge.pending_calls()) + .collect(); + if pending.len() != next_state.pending_tool_count() + || pending.iter().any(|call| { + next_state.pending_tool(&call.call_id).is_none_or(|other| { + other.operation_id != call.operation_id || other.input != call.input + }) + }) { return Err(self.fail_closed(LocalRunnerError::invalid( "ACPX terminal settlement left provider tool state inconsistent", ))); } - reconciled_events.extend( - settlements - .into_iter() - .map(AcpxProviderStateEvent::ToolResult), - ); } AcpxProviderStateEvent::PermissionRequest { .. } => { if self.config.agent == "codex" @@ -685,9 +681,11 @@ impl AcpxProviderSession { } pub fn deliver_tool_result(&mut self, result: &ToolResult) -> Result<(), LocalRunnerError> { - let turn_id = self.ensure_active_turn()?.to_owned(); + self.ensure_open()?; let mut next_state = self.state.clone(); - next_state.complete_tool(&result.call_id, &result.operation_id)?; + if next_state.pending_tool(&result.call_id).is_some() { + next_state.complete_tool(&result.call_id, &result.operation_id)?; + } let mut next_bridge = self.tool_bridge.clone(); let mut next_reserved_bridge = self.reserved_tool_bridge.clone(); let bridge = if is_reserved_terminal_operation(&result.operation_id) { @@ -695,9 +693,24 @@ impl AcpxProviderSession { } else { &mut next_bridge }; + let duplicate = bridge.has_completed_call(&result.call_id); + let detached = bridge.turn_closed(); bridge.apply_result(result.clone()).map_err(|error| { - LocalRunnerError::invalid(format!("ACPX tool result is invalid: {error}")) + LocalRunnerError::invalid(format!( + "ACPX tool result for call {} operation {} is invalid: {error}", + result.call_id, result.operation_id + )) })?; + if duplicate { + return Ok(()); + } + if detached { + self.state = next_state; + self.tool_bridge = next_bridge; + self.reserved_tool_bridge = next_reserved_bridge; + return Ok(()); + } + let turn_id = self.ensure_active_turn()?.to_owned(); let resolution = if result.is_error { // The durable result remains authoritative for correlation and // retry bookkeeping, but provider-facing failures expose only a diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_state.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_state.rs index 876cf9d00f1..ceef3283952 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_state.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_state.rs @@ -365,7 +365,8 @@ impl AcpxProviderState { .to_owned(); self.scope.clear_turn(&turn_id)?; let mut events = self.end_pending_runtime_requests(status); - self.clear_pending_requests(); + // Dispatched semantic effects outlive the provider turn. Their + // authority results, not this terminal event, retire them. if status == AcpxTurnStatus::Completed && !self.assistant_text.is_empty() { events.push(AcpxProviderStateEvent::AssistantMessage { turn_id: turn_id.clone(), @@ -646,12 +647,8 @@ impl AcpxProviderState { events } - fn clear_pending_requests(&mut self) { - self.pending_tools.clear(); - self.pending_tool_input_bytes = 0; - self.pending_permissions.clear(); - self.pending_inputs.clear(); - self.pending_runtime_request_bytes = 0; + pub(crate) fn pending_tool_count(&self) -> usize { + self.pending_tools.len() } } diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs b/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs index 0a1d9bdf5cc..9bcb817a83e 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs @@ -251,6 +251,13 @@ pub enum TerminalDeliveryReconciliation { pub trait CommandExecutor { fn execute(&mut self, command: &Command) -> Result; + /// Opt in only when semantic result delivery validates an exact durable + /// call receipt and cannot dispatch the underlying business operation. + /// Ordinary commands and providers without this proof remain indeterminate. + fn can_reconcile_result_delivery(&mut self) -> Result { + Ok(false) + } + /// Advances provider-side event correlation after a durable `run.attach` /// has moved runnerd to the next run-bound authority. The runner validates /// and persists the new authority before invoking this infallible hook. @@ -1724,6 +1731,15 @@ fn process_command( } } match state.begin_command(command)? { + CommandDisposition::Replay(result) + if result.status == "indeterminate" + && command.command_type == "semantic_tool.result" + && executor.can_reconcile_result_delivery()? => + { + // begin_command already checked the complete immutable command + // fingerprint. Only replay receipt delivery, never the operation. + state.resume_result_delivery(command)?; + } CommandDisposition::Replay(result) => { let lifecycle = if result.status == "pending" || state.pending_provider_cleanup.is_some() { @@ -1742,8 +1758,8 @@ fn process_command( CommandDisposition::Execute => {} } // Persist the pending marker before any command effect. If the process dies - // in the effect window, recovery returns an indeterminate result and never - // executes the same logical command twice. + // in the effect window, recovery remains indeterminate unless the provider + // explicitly supports exact, idempotent semantic result delivery above. store.save(state)?; let mut execution = match executor.execute(command) { Ok(execution) => execution, @@ -4485,6 +4501,150 @@ mod tests { } } + #[test] + fn interrupted_result_delivery_reconciles_exact_receipts_without_reexecuting_operations() { + use crate::provider_bridge::{ + authorized_tool_catalog_digest, AuthorizedTool, AuthorizedToolSet, ProviderToolBridge, + ToolResult, + }; + struct ReceiptExecutor { + path: PathBuf, + deliveries: usize, + supported: bool, + } + impl CommandExecutor for ReceiptExecutor { + fn can_reconcile_result_delivery(&mut self) -> Result { + Ok(self.supported) + } + fn execute( + &mut self, + command: &Command, + ) -> Result { + assert_eq!( + command.command_type, "semantic_tool.result", + "must never redispatch an operation" + ); + self.deliveries += 1; + let mut bridge: ProviderToolBridge = + serde_json::from_slice(&fs::read(&self.path).unwrap()).unwrap(); + let result: ToolResult = serde_json::from_value(command.payload.clone()).unwrap(); + bridge + .apply_result(result) + .map_err(|error| DurableRunnerError::invalid(error.to_string()))?; + fs::write(&self.path, serde_json::to_vec(&bridge).unwrap()).unwrap(); + Ok(CommandExecution::result(json!({"status":"delivered"}))) + } + } + for applied in [false, true] { + for supported in [false, true] { + let directory = std::env::temp_dir().join(format!( + "result-delivery-crash-{}-{applied}-{supported}", + std::process::id() + )); + let _ = fs::remove_dir_all(&directory); + let config = config(directory.clone()); + let store = DurableStateStore::new(&directory).unwrap(); + let (mut state, _) = store.load_or_create(&config).unwrap(); + let operations = vec![AuthorizedTool { + operation_id: "write_document".into(), + version: 1, + description: "Fixture write already committed on the server".into(), + input_schema: json!({"type":"object"}), + response_schema: json!({"type":"object"}), + }]; + let mut bridge = ProviderToolBridge::default(); + bridge + .prepare(AuthorizedToolSet { + schema: "paperclip.runner.authorized-tools.v1".into(), + schema_version: 1, + catalog_digest: authorized_tool_catalog_digest(&operations).unwrap(), + operations, + }) + .unwrap(); + bridge + .begin_call( + "call-1".into(), + "write_document".into(), + json!({"content":"exact original"}), + ) + .unwrap(); + bridge.settle_turn("provider_turn_stopped").unwrap(); + let path = directory.join("tool-receipts.json"); + fs::write(&path, serde_json::to_vec(&bridge).unwrap()).unwrap(); + let mut executor = ReceiptExecutor { + path: path.clone(), + deliveries: 0, + supported, + }; + let mut delivery = command("semantic_tool.result"); + delivery.payload = json!({"callId":"call-1", "operationId":"write_document", "isError":false, + "result":{"revisionId":"one-committed-write"}}); + assert_eq!( + state.begin_command(&delivery).unwrap(), + CommandDisposition::Execute + ); + store.save(&state).unwrap(); + if applied { + executor.execute(&delivery).unwrap(); + } + // Crash before receipt application, or after receipt application + // but before the command completion journal commits. + let (mut recovered, _) = store.load_or_create(&config).unwrap(); + let before = executor.deliveries; + let (result, _) = + process_command(&mut recovered, &store, &config, &mut executor, &delivery) + .unwrap(); + assert_eq!( + result.status, + if supported { + "completed" + } else { + "indeterminate" + } + ); + assert_eq!(executor.deliveries, before + usize::from(supported)); + let saved: ProviderToolBridge = + serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); + assert_eq!( + saved.pending_calls().count(), + usize::from(!applied && !supported) + ); + let (mut reopened, _) = store.load_or_create(&config).unwrap(); + assert_eq!( + process_command(&mut reopened, &store, &config, &mut executor, &delivery) + .unwrap() + .0, + result + ); + assert_eq!(executor.deliveries, before + usize::from(supported)); + let mut changed = delivery.clone(); + changed.payload["result"]["revisionId"] = json!("conflicting-write"); + assert!( + process_command(&mut reopened, &store, &config, &mut executor, &changed) + .unwrap_err() + .to_string() + .contains("different command data") + ); + assert_eq!(executor.deliveries, before + usize::from(supported)); + let mut ordinary = command("session.open"); + ordinary.command_id = "ordinary-effect".into(); + ordinary.controller_seq = 2; + reopened.begin_command(&ordinary).unwrap(); + store.save(&reopened).unwrap(); + let (mut after_crash, _) = store.load_or_create(&config).unwrap(); + assert_eq!( + process_command(&mut after_crash, &store, &config, &mut executor, &ordinary) + .unwrap() + .0 + .status, + "indeterminate" + ); + assert_eq!(executor.deliveries, before + usize::from(supported)); + fs::remove_dir_all(directory).unwrap(); + } + } + } + #[test] fn process_death_after_journaling_remains_indeterminate_without_reexecution() { let directory = std::env::temp_dir().join(format!( diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/durable/state.rs b/packages/paperclip-runner/runner/crates/runner-core/src/durable/state.rs index 9d160c5ebe5..2a41066875c 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/durable/state.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/durable/state.rs @@ -775,6 +775,28 @@ impl DurableState { Ok(CommandDisposition::Execute) } + pub(crate) fn resume_result_delivery( + &mut self, + command: &Command, + ) -> Result<(), DurableRunnerError> { + if command.command_type != "semantic_tool.result" + || !matches!(self.begin_command(command)?, CommandDisposition::Replay(ref prior) if prior.status == "indeterminate") + { + return Err(DurableRunnerError::invalid( + "only exact indeterminate result delivery can resume", + )); + } + self.processed_commands + .get_mut(&command.command_id) + .expect("validated retained delivery receipt") + .status = "pending".to_owned(); + self.record_diagnostic(format!( + "reconciling semantic result delivery command {}", + command.command_id + )); + Ok(()) + } + pub fn complete_command( &mut self, command: &Command, @@ -1671,52 +1693,55 @@ pub(crate) fn sanitize_semantic_tool_input( operation_id: &str, input: &Value, ) -> Result { - let mut sanitized = sanitize_value(input); - // Mutation prose is the user's intended work, not a diagnostic. Preserve - // ordinary references to a token in these declared text fields; credential - // syntax and high-confidence secret values are still scrubbed. All other - // fields and operations retain the strict diagnostic policy. + // This value is executed by the authority. Diagnostic preview limits and + // redaction must never turn it into a different, apparently valid write. + // Hex-encoded authenticated frames double JSON bytes. Reserve envelope + // space below the 1 MiB wire cap before any effect can start. + // Enforce the transport envelope before dispatch, and reject content that + // cannot cross its credential policy instead of silently rewriting it. + if serde_json::to_vec(input) + .map_err(|_| DurableRunnerError::invalid("semantic tool input is not serializable"))? + .len() + > 480 * 1024 + { + return Err(DurableRunnerError::invalid( + "semantic tool input exceeds the 480 KiB encrypted transport limit", + )); + } + let mut checked = sanitize_bounded_display_value(input); let prose_fields: &[&str] = match operation_id { "create_task" => &["title", "description", "initialPlan"], "create_project" => &["name", "description"], "write_document" => &["title", "body", "changeSummary"], + "update_agent_instructions" => &["content"], _ => &[], }; - if let Some(sanitized_input) = sanitized.as_object_mut() { + if let Some(object) = checked.as_object_mut() { for field in prose_fields { if let Some(text) = input.get(*field).and_then(Value::as_str) { - sanitized_input.insert( + object.insert( (*field).to_owned(), - // The tool/API schema bounds business content. A diagnostic - // preview limit must never truncate a plan or document. Value::String(redact_sensitive_text_values_with_context(text, true)), ); } } } - if !matches!(operation_id, "paperclip_finish" | "paperclip_block") { - return Ok(sanitized); - } - let Some(summary) = input.get("summary").and_then(Value::as_str) else { - return Ok(sanitized); - }; - if summary.chars().count() > MAX_COMPLETION_SUMMARY_CHARS { + if matches!(operation_id, "paperclip_finish" | "paperclip_block") + && input + .get("summary") + .and_then(Value::as_str) + .is_some_and(|summary| summary.chars().count() > MAX_COMPLETION_SUMMARY_CHARS) + { return Err(DurableRunnerError::invalid( "semantic completion summary exceeds the 12,000 character limit", )); } - let Some(sanitized_input) = sanitized.as_object_mut() else { - return Ok(sanitized); - }; - // Completion summary is the schema-bounded user-facing answer, not an - // untrusted diagnostic snippet. Preserve it in full while applying the - // same credential scrubber used by every durable string. All other fields - // retain the generic 4 KiB diagnostic bound. - sanitized_input.insert( - "summary".to_owned(), - Value::String(redact_sensitive_text_values(summary)), - ); - Ok(sanitized) + if checked != *input { + return Err(DurableRunnerError::invalid( + "semantic tool input contains credential material; refusing to execute altered arguments", + )); + } + Ok(input.clone()) } fn finalize_semantic_tool_input_payload( @@ -2944,7 +2969,7 @@ mod tests { let config = config(PathBuf::from("unused")); let mut state = DurableState::new(&config); let summary = format!( - "token=do-not-persist {} Authorization: Bearer late-provider-secret COMPLETE-DURABLE-SUMMARY", + "Answer: {} COMPLETE-DURABLE-SUMMARY", "A complete paragraph for the user. ".repeat(180) ); let input = json!({"summary": summary}); @@ -2988,11 +3013,11 @@ mod tests { .pointer("/payload/payload/semantic_tool/input") .unwrap(); let transmitted_summary = transmitted["summary"].as_str().unwrap(); - assert!(transmitted_summary.starts_with("token=[REDACTED] ")); + assert!(transmitted_summary.starts_with("Answer: ")); assert!(transmitted_summary.ends_with(" COMPLETE-DURABLE-SUMMARY")); assert!(!transmitted_summary.contains("do-not-persist")); assert!(!transmitted_summary.contains("late-provider-secret")); - assert!(transmitted_summary.contains("Authorization: Bearer [REDACTED]")); + assert_eq!(transmitted, &input); assert!(!transmitted_summary.contains("…[truncated]")); assert_eq!( state.outbox[0] @@ -3013,7 +3038,7 @@ mod tests { let mut changed_tail = payload.clone(); changed_tail["semantic_tool"]["input"]["summary"] = Value::String(format!( - "token=[REDACTED] {} CHANGED-DURABLE-SUMMARY", + "Answer: {} CHANGED-DURABLE-SUMMARY", "A complete paragraph for the user. ".repeat(180) )); assert!(state @@ -3431,16 +3456,31 @@ mod tests { "{}\nAuthorization: Bearer late-credential\nFINAL-ACCEPTANCE-42", "Document content. ".repeat(400) ); - let safe = - sanitize_semantic_tool_input("write_document", &json!({"body": document})).unwrap(); - let body = safe["body"].as_str().unwrap(); - assert!(body.len() > 4096); - assert!(body.ends_with("FINAL-ACCEPTANCE-42")); - assert!(!body.contains("late-credential")); + assert!( + sanitize_semantic_tool_input("write_document", &json!({"body": document})).is_err() + ); } #[test] - fn semantic_prose_does_not_exempt_credential_syntax_or_shapes() { + fn semantic_credentials_are_rejected_instead_of_executing_redacted_arguments() { + for input in [ + json!({"content": "Authorization: Bearer opaque-credential"}), + json!({"content": "literal token=opaque-credential"}), + json!({"content": "ghp_secretvalue12345678901234567890"}), + json!({"token": "opaque-credential"}), + json!({"nested": {"password": "opaque-credential"}}), + ] { + let error = + sanitize_semantic_tool_input("update_agent_instructions", &input).unwrap_err(); + assert!(error + .to_string() + .contains("refusing to execute altered arguments")); + assert!(!error.to_string().contains("opaque-credential")); + } + } + + #[test] + fn semantic_prose_rejects_all_previously_scrubbed_credential_shapes() { for text in [ "auth token opaque-credential", "access token opaque-credential", @@ -3453,56 +3493,57 @@ mod tests { "literal access_token opaque-credential", "literal \"token\" opaque-credential", "literal token \"opaque-credential\"", + "Include the literal token sk-proj-secretvalue123456 in the document.", + "Include the literal token ghp_secretvalue12345678901234567890 in the document.", + "Include the literal token github_pat_secretvalue12345678901234567890 in the document.", + "Include the literal token eyJhbGciOiJIUzI1NiJ9.c2VjcmV0LWNsYWlt.signaturesecret in the document.", ] { - assert!(!redact_text(text).contains("opaque-credential"), "{text}"); - let input = json!({"description": text, "initialPlan": text}); - assert!( - !sanitize_semantic_tool_input("create_task", &input) - .unwrap() - .to_string() - .contains("opaque-credential"), - "{text}" - ); + for (operation, field) in [ + ("create_task", "description"), ("create_task", "initialPlan"), + ("write_document", "body"), ("update_agent_instructions", "content"), + ] { + let input = json!({field: text}); + assert!(sanitize_semantic_tool_input(operation, &input).is_err(), "{operation}: {text}"); + } } - for secret in [ - "sk-proj-secretvalue123456", - "ghp_secretvalue12345678901234567890", - "github_pat_secretvalue12345678901234567890", - "eyJhbGciOiJIUzI1NiJ9.c2VjcmV0LWNsYWlt.signaturesecret", - ] { - let text = format!("Include the literal token {secret} in the document."); - assert!(!redact_text(&text).contains(secret), "{text}"); - assert!( - !sanitize_semantic_tool_input("write_document", &json!({"body": text})) - .unwrap() - .to_string() - .contains(secret) - ); - } - let input = json!({ - "description": "Include the literal token ACCEPTANCE-42. Authorization: Bearer opaque-credential", - "token": "opaque-credential", - }); - let safe = sanitize_semantic_tool_input("create_task", &input).unwrap(); - assert!(safe["description"] - .as_str() - .unwrap() - .contains("ACCEPTANCE-42")); - assert!(!safe.to_string().contains("opaque-credential")); - let diagnostic = json!({"description": "the token opaque-credential"}); - assert!( - !sanitize_semantic_tool_input("get_task_context", &diagnostic) - .unwrap() - .to_string() - .contains("opaque-credential") - ); - assert!(!sanitize_semantic_tool_input( - "create_task", - &json!({"diagnostic": "token opaque-credential"}) + assert!(sanitize_semantic_tool_input( + "get_task_context", + &json!({ + "description": "the token opaque-credential", + }) ) - .unwrap() - .to_string() - .contains("opaque-credential")); + .is_err()); + } + + #[test] + fn instruction_arguments_survive_the_complete_outbox_without_byte_changes() { + let content = format!( + "{}\nFINAL-TAIL-🦀", + "# Instructions — 日本語 🦀\n".repeat(400) + ); + let input = json!({"entryFile": "AGENTS.md", "baseRevisionId": null, "content": content}); + let config = config(PathBuf::from("unused")); + let mut state = DurableState::new(&config); + state.enqueue_executor_event(&config, "instruction-write".to_owned(), "semantic_tool.input".to_owned(), EventPriority::P0, + json!({"semantic_tool": {"schema":"paperclip.prp.semantic_tool.v1", "schemaVersion":1, "phase":"input", "operationId":"update_agent_instructions", "content":{"digest":semantic_value_digest(&input)}, "input": input}})).unwrap(); + let encoded = serde_json::to_vec(&state).unwrap(); + let recovered: DurableState = serde_json::from_slice(&encoded).unwrap(); + let actual = recovered.outbox[0] + .envelope + .pointer("/payload/payload/semantic_tool/input") + .unwrap(); + assert_eq!(actual, &input); + assert_eq!( + actual["content"].as_str().unwrap().as_bytes(), + content.as_bytes() + ); + assert_eq!(semantic_value_digest(actual), semantic_value_digest(&input)); + assert!(sanitize_semantic_tool_input( + "update_agent_instructions", + &json!({"content":"🦀".repeat(200_000)}) + ) + .is_err()); + assert_eq!(state.outbox.len(), 1); } #[test] diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/native_provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/native_provider_backend.rs index 73535dc9758..7bc424c5c89 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/native_provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/native_provider_backend.rs @@ -19,6 +19,14 @@ enum SelectedExecutor { } impl CommandExecutor for SelectedExecutor { + fn can_reconcile_result_delivery(&mut self) -> Result { + match self { + Self::LocalFacade(executor) => executor.can_reconcile_result_delivery(), + Self::Acpx(executor) => executor.can_reconcile_result_delivery(), + Self::Managed(executor) => executor.can_reconcile_result_delivery(), + } + } + fn retained_events(&mut self) -> Result, DurableRunnerError> { match self { Self::LocalFacade(executor) => executor.retained_events(), @@ -172,6 +180,13 @@ impl NativeProviderCommandExecutor { } impl CommandExecutor for NativeProviderCommandExecutor { + fn can_reconcile_result_delivery(&mut self) -> Result { + self.select_recovery()?; + self.selected + .as_mut() + .map_or(Ok(false), CommandExecutor::can_reconcile_result_delivery) + } + fn retained_events(&mut self) -> Result, DurableRunnerError> { self.selected .as_mut() diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs index 90cddbdc285..aea835f7244 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs @@ -3070,30 +3070,31 @@ impl CodexCommandExecutor { "reason": reason, }))); } - let has_pending_tools = self + let mut next_state = self .state - .as_ref() - .is_some_and(|state| state.tool_bridge.pending_calls().next().is_some()); - if has_pending_tools { - let identity = self.event_identity()?; - let mut next_state = self - .state - .clone() - .expect("Codex state remains available during interruption"); - let cancelled = next_state - .tool_bridge - .cancel_pending_calls("provider_turn_stopped") - .map_err(|error| { - DurableRunnerError::invalid(format!( - "failed to cancel pending semantic tools: {error}" - )) - })?; - for result in cancelled { - next_state.push_terminal_event(semantic_result_event(&identity, &result))?; - } - self.persist_state(&next_state)?; - self.state = Some(next_state); - } + .clone() + .expect("Codex state remains available during interruption"); + next_state + .tool_bridge + .cancel_pending_calls("provider_turn_stopped") + .map_err(|error| { + DurableRunnerError::invalid(format!( + "failed to stop semantic tool admission: {error}" + )) + })?; + next_state.push_terminal_event(NormalizedProviderEvent { + event_type: "harness.diagnostic".to_owned(), + priority: EventPriority::P0, + payload: json!({ + "code": "provider_interrupt_requested", + "reason": reason, + "providerTurnId": provider_turn_id, + "pendingCallIds": next_state.tool_bridge.pending_calls().map(|call| call.call_id.clone()).collect::>(), + }), + })?; + // Record cause and pending identities before touching the provider. + self.persist_state(&next_state)?; + self.state = Some(next_state); self.ensure_provider()?.interrupt_turn().map_err(|error| { DurableRunnerError::invalid(format!("Codex turn interrupt failed: {error}")) })?; @@ -3711,6 +3712,13 @@ impl CodexCommandExecutor { operation_id: String, input: Value, ) -> Result<(), DurableRunnerError> { + if let Err(error) = sanitize_semantic_tool_input(&operation_id, &input) { + return self.reject_tool_call( + call_id, + operation_id, + ProviderBridgeError::invalid(error.to_string()), + ); + } let identity = self.event_identity()?; let admission = self .state @@ -3763,6 +3771,7 @@ impl CodexCommandExecutor { .state .clone() .ok_or_else(|| DurableRunnerError::invalid("Codex provider is not prepared"))?; + let delivery_detached = next_state.tool_bridge.turn_closed(); let terminal_tool_input = next_state .tool_bridge .pending_calls() @@ -3772,13 +3781,24 @@ impl CodexCommandExecutor { .tool_bridge .apply_result(result.clone()) .map_err(|error| { - DurableRunnerError::invalid(format!("semantic tool result was rejected: {error}")) + DurableRunnerError::invalid(format!( + "semantic tool result was rejected for call {} operation {}: {error}", + result.call_id, result.operation_id + )) })?; if was_completed { - return Ok(CommandExecution::result(json!({ - "status": "duplicate", - "callId": result.call_id, - }))); + return Ok(CommandExecution { + result: json!({ "status": "duplicate", "callId": result.call_id }), + events: vec![( + "harness.diagnostic".to_owned(), + EventPriority::P0, + json!({ + "code": "semantic_tool_result_duplicate", "severity": "warning", + "callId": result.call_id, "operationId": result.operation_id, + "resultDigest": semantic_value_digest(&result.result), + }), + )], + }); } let terminal_tool_authoritative = terminal_tool_input @@ -3796,6 +3816,12 @@ impl CodexCommandExecutor { next_state.push_event(semantic_result_event(&identity, &result))?; self.persist_state(&next_state)?; self.state = Some(next_state); + if delivery_detached { + return Ok(CommandExecution::result(json!({ + "status": "settled_after_turn", + "callId": result.call_id, + }))); + } let provider = self.ensure_provider()?; if terminal_tool_authoritative { provider @@ -4401,6 +4427,12 @@ impl CodexCommandExecutor { } impl CommandExecutor for CodexCommandExecutor { + fn can_reconcile_result_delivery(&mut self) -> Result { + // ProviderToolBridge persists the exact call/result and rejects changed + // receipts. Delivery cannot execute a server-side semantic operation. + Ok(true) + } + fn execute(&mut self, command: &Command) -> Result { self.startup_command = Some(ProviderStartupCommand { command_id: command.command_id.clone(), @@ -4457,7 +4489,13 @@ impl CommandExecutor for CodexCommandExecutor { "session.goal.get" => self.get_goal(), "session.goal.set" => self.set_goal(&command.payload), "session.goal.clear" => self.clear_goal(&command.payload), - "turn.interrupt" | "run.cancel" => self.interrupt_turn(&command.command_type), + "turn.interrupt" | "run.cancel" => self.interrupt_turn( + command + .payload + .get("reason") + .and_then(Value::as_str) + .unwrap_or(&command.command_type), + ), "turn.stop" => self.stop_turn_for_suspension(&command.command_type), "request.resolve" => self.resolve_request(&command.payload), "semantic_tool.result" => self.deliver_semantic_result(&command.payload), @@ -5474,7 +5512,7 @@ mod tests { } #[test] - fn semantic_input_digest_covers_the_transmitted_redacted_value() { + fn semantic_input_rejects_credential_material_before_dispatch() { let identity = ProviderEventIdentity { runner_instance_id: "runner-1".to_owned(), run_id: "run-1".to_owned(), @@ -5487,17 +5525,14 @@ mod tests { operation_id: "get_task_context".to_owned(), input: json!({"password": "do-not-persist", "safe": true}), }; - let event = semantic_input_event(&identity, &call).unwrap(); - let transmitted = &event.payload["semantic_tool"]["input"]; - assert_eq!(transmitted["password"], "[REDACTED]"); - assert_eq!( - event.payload["semantic_tool"]["content"]["digest"], - semantic_value_digest(transmitted) - ); + assert!(semantic_input_event(&identity, &call) + .unwrap_err() + .to_string() + .contains("refusing to execute altered arguments")); } #[test] - fn semantic_finish_input_preserves_a_complete_long_redacted_summary() { + fn semantic_finish_input_preserves_a_complete_long_summary() { let identity = ProviderEventIdentity { runner_instance_id: "runner-1".to_owned(), run_id: "run-1".to_owned(), @@ -5506,7 +5541,7 @@ mod tests { item_id: "item-1".to_owned(), }; let summary = format!( - "token=do-not-persist {} Authorization: Bearer late-provider-secret COMPLETE-LONG-SUMMARY", + "Answer: {} COMPLETE-LONG-SUMMARY", "A complete paragraph for the user. ".repeat(180) ); assert!(summary.len() > 4_096); @@ -5519,11 +5554,11 @@ mod tests { let event = semantic_input_event(&identity, &call).unwrap(); let transmitted = &event.payload["semantic_tool"]["input"]; let transmitted_summary = transmitted["summary"].as_str().unwrap(); - assert!(transmitted_summary.starts_with("token=[REDACTED] ")); + assert!(transmitted_summary.starts_with("Answer: ")); assert!(transmitted_summary.ends_with(" COMPLETE-LONG-SUMMARY")); assert!(!transmitted_summary.contains("do-not-persist")); assert!(!transmitted_summary.contains("late-provider-secret")); - assert!(transmitted_summary.contains("Authorization: Bearer [REDACTED]")); + assert_eq!(transmitted, &call.input); assert!(!transmitted_summary.contains("…[truncated]")); assert_eq!( event.payload["semantic_tool"]["content"]["digest"], diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/provider_bridge.rs b/packages/paperclip-runner/runner/crates/runner-core/src/provider_bridge.rs index 77307873814..a1007983368 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/provider_bridge.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/provider_bridge.rs @@ -192,6 +192,10 @@ pub struct ProviderToolBridge { catalog_operations: Vec, catalog_digest: Option, pending: BTreeMap, + // A provider stopping is not evidence that a dispatched server operation + // failed. Keep its input and identity pending until the authority replies. + #[serde(default)] + turn_closed: bool, #[serde(deserialize_with = "deserialize_retained_results")] completed: BTreeMap, // Keep authoritative result bodies through the provider's replay window; @@ -227,7 +231,7 @@ pub struct ProviderBridgeError { } impl ProviderBridgeError { - fn invalid(message: impl Into) -> Self { + pub(crate) fn invalid(message: impl Into) -> Self { Self { message: message.into(), safe_provider_message: None, @@ -321,6 +325,7 @@ impl ProviderToolBridge { )); } self.prepare_internal(tool_set, true)?; + self.turn_closed = false; self.completed.clear(); self.settled_results.clear(); self.retained_result_bytes = 0; @@ -577,6 +582,7 @@ impl ProviderToolBridge { // can replay the just-settled turn. Release that bulky data at the // verified turn boundary, but retain every call-ID tombstone and any // recovered legacy filter until the owning provider process is reaped. + self.turn_closed = false; self.settled_results.clear(); self.retained_result_bytes = 0; self.durable_run_receipt_limit_reached = self.replay_history_blocks_admission(); @@ -687,6 +693,11 @@ impl ProviderToolBridge { "provider reused a completed tool call id", )); } + if self.turn_closed { + return Err(ProviderBridgeError::invalid( + "cannot admit a new tool call after the provider turn stopped", + )); + } // A durable receipt ledger may be exactly full or legacy state may // contain only a probabilistic summary of evicted identities. Neither // state can admit more work safely in this provider-process epoch. The @@ -769,18 +780,14 @@ impl ProviderToolBridge { return if existing.result == result { Ok(existing.result.result.clone()) } else { - Err(ProviderBridgeError::invalid( - "conflicting duplicate tool result", - )) + Err(conflicting_tool_result(&existing.result, &result)) }; } if let Some(existing) = self.settled_results.get(&result.call_id) { return if existing.result == result { Ok(existing.result.result.clone()) } else { - Err(ProviderBridgeError::invalid( - "conflicting duplicate settled tool result", - )) + Err(conflicting_tool_result(&existing.result, &result)) }; } if self.has_settled_call_id(&result.call_id) { @@ -848,7 +855,16 @@ impl ProviderToolBridge { )?; self.pending.remove(&result.call_id); self.retained_result_bytes = next_retained_bytes; - self.completed.insert(result.call_id.clone(), completed); + if self.turn_closed { + self.settled_call_ids.extend_recent( + std::iter::once(result.call_id.clone()), + MAX_SETTLED_CALL_IDS, + ); + self.settled_results + .insert(result.call_id.clone(), completed); + } else { + self.completed.insert(result.call_id.clone(), completed); + } Ok(result.result) } @@ -856,41 +872,29 @@ impl ProviderToolBridge { self.pending.values() } + /// Close provider admission/delivery, without inventing an operation outcome. + /// This marker is durable, including when interruption beats result delivery. pub fn cancel_pending_calls( &mut self, code: &str, ) -> Result, ProviderBridgeError> { - let mut next = self.clone(); - let results = next.cancel_pending_calls_internal(code)?; - *self = next; - Ok(results) + self.settle_turn(code) + } + + pub fn turn_closed(&self) -> bool { + self.turn_closed } pub fn settle_turn(&mut self, code: &str) -> Result, ProviderBridgeError> { validate_stable_id(code, "tool cancellation code")?; - let results = self - .pending - .values() - .map(|call| cancelled_tool_result(call, code)) - .collect::>(); - let mut settled_entries = self.completed.clone(); - for (call_id, call) in &self.pending { - settled_entries.insert( - call_id.clone(), - CompletedToolCall { - call: call.clone(), - result: cancelled_tool_result(call, code), - }, - ); + if !self.settled_identity_capacity_allows(0) { + return Err(ProviderBridgeError::active_turn_receipt_limit()); } let next_retained_bytes = - retained_result_bytes(self.settled_results.iter().chain(settled_entries.iter()))?; - ensure_settled_result_capacity(next_retained_bytes, std::iter::empty())?; - - // Byte capacity was reserved at admission. Keep every identity exact - // for the lifetime of this provider-process epoch; prepare_turn - // releases only the bulky result bodies. - let new_identity_count = settled_entries + retained_result_bytes(self.settled_results.iter().chain(self.completed.iter()))?; + ensure_settled_result_capacity(next_retained_bytes, self.pending.values())?; + let new_identity_count = self + .completed .keys() .filter(|call_id| !self.settled_call_ids.contains(call_id)) .count(); @@ -904,28 +908,14 @@ impl ProviderToolBridge { } let evicted = self .settled_call_ids - .extend_recent(settled_entries.keys().cloned(), MAX_SETTLED_CALL_IDS); + .extend_recent(self.completed.keys().cloned(), MAX_SETTLED_CALL_IDS); debug_assert!(evicted.is_empty()); - self.pending.clear(); - self.completed.clear(); - self.settled_results.append(&mut settled_entries); + self.settled_results.append(&mut self.completed); self.retained_result_bytes = next_retained_bytes; - Ok(results) - } - - fn cancel_pending_calls_internal( - &mut self, - code: &str, - ) -> Result, ProviderBridgeError> { - validate_stable_id(code, "tool cancellation code")?; - let pending = self.pending.values().cloned().collect::>(); - let mut results = Vec::with_capacity(pending.len()); - for call in pending { - let result = cancelled_tool_result(&call, code); - self.apply_result(result.clone())?; - results.push(result); - } - Ok(results) + self.turn_closed = true; + // Pending effects survive both normal turn termination and process + // loss. They block the next turn/checkpoint, but accept a late result. + Ok(Vec::new()) } fn retained_value_bytes(&self) -> Result { @@ -1271,19 +1261,18 @@ where deserializer.deserialize_map(RetainedResultsVisitor) } -fn cancelled_tool_result(call: &PendingToolCall, code: &str) -> ToolResult { - ToolResult { - call_id: call.call_id.clone(), - operation_id: call.operation_id.clone(), - result: serde_json::json!({ - "error": { - "code": code, - "message": "The provider turn stopped before this semantic tool completed", - "retryable": false, - }, - }), - is_error: true, - } +fn conflicting_tool_result(existing: &ToolResult, incoming: &ToolResult) -> ProviderBridgeError { + // Correlate both receipts without leaking arguments or result content into + // a user-visible error or diagnostic stream. + let digest = |result: &ToolResult| { + semantic_value_digest(&serde_json::json!({ + "operationId": result.operation_id, "isError": result.is_error, "result": result.result, + })) + }; + ProviderBridgeError::invalid(format!( + "conflicting duplicate tool result: callId={} operationId={} existingDigest={} incomingDigest={}", + incoming.call_id, incoming.operation_id, digest(existing), digest(incoming), + )) } pub fn authorized_tool_catalog_digest( @@ -1886,11 +1875,13 @@ mod tests { let cancelled = pending_at_capacity .settle_turn("semantic_tool_turn_receipt_limit") .unwrap(); - assert_eq!(cancelled.len(), 1); + assert!(cancelled.is_empty()); + assert_eq!(pending_at_capacity.pending_calls().count(), 1); assert_eq!( pending_at_capacity.settled_call_ids.len(), - MAX_SETTLED_CALL_IDS + MAX_SETTLED_CALL_IDS - 1 ); + assert!(pending_at_capacity.prepare_turn().is_err()); bridge .apply_result(ToolResult { diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_event_payload.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_event_payload.rs index a82105de9b5..0dab9435c0e 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_event_payload.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_event_payload.rs @@ -176,7 +176,7 @@ fn rejects_unclassified_and_malformed_runtime_payloads() { #[test] fn decodes_tool_and_permission_requests_after_scope_validation() { let scope = active_scope(); - let tool = decode_acpx_event( + let rejected = decode_acpx_event( &scope, &event( GeneratedAcpxSidecarEventType::RuntimeToolCalled, @@ -187,14 +187,10 @@ fn decodes_tool_and_permission_requests_after_scope_validation() { }), ), ) - .unwrap(); - assert!(matches!( - tool, - AcpxEventPayload::ToolCalled { call_id, operation_id, input, .. } - if call_id == "call-1" - && operation_id == "get_issue" - && input["apiToken"] == "[REDACTED]" - )); + .unwrap_err(); + assert!(rejected + .to_string() + .contains("refusing to execute altered arguments")); let permission = decode_acpx_event( &scope, diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_provider_resolutions.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_provider_resolutions.rs index 89e4cdee123..5a61230715d 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_provider_resolutions.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_provider_resolutions.rs @@ -99,9 +99,9 @@ fn commits_each_resolution_only_after_sidecar_acknowledgement() { .unwrap(); assert!(session.state().pending_tool("call-1").is_none()); assert!(session.state().pending_question_set("input-1").is_none()); - assert!(session + session .deliver_tool_result(&tool_result("issues.read")) - .is_err()); + .expect("an identical result replay must be idempotent"); session.shutdown("test complete").unwrap(); } @@ -200,7 +200,9 @@ fn redacts_failed_tool_payload_without_losing_correlation_or_retry() { failed.operation_id = "issues.read".to_owned(); session.deliver_tool_result(&failed).unwrap(); assert!(session.state().pending_tool("call-1").is_none()); - assert!(session.deliver_tool_result(&failed).is_err()); + session + .deliver_tool_result(&failed) + .expect("the failure receipt also deduplicates"); session.shutdown("test complete").unwrap(); } diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_provider_state.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_provider_state.rs index 072a812d563..f4f4d231919 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_provider_state.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_provider_state.rs @@ -465,7 +465,7 @@ fn accepts_global_process_and_diagnostic_events_without_an_active_turn() { } #[test] -fn terminal_events_clear_pending_requests_and_reject_late_turn_events() { +fn terminal_events_retain_dispatched_tools_and_reject_late_turn_events() { let mut state = AcpxProviderState::new("run-1").unwrap(); state.begin_turn("turn-1").unwrap(); state @@ -484,8 +484,10 @@ fn terminal_events_clear_pending_requests_and_reject_late_turn_events() { json!({"status":"cancelled","error":{"message":"token=secret"}}), )) .unwrap(); + assert!(state.pending_tool("call-1").is_some()); + assert!(state.begin_turn("turn-2").is_err()); + state.complete_tool("call-1", "issues.read").unwrap(); assert!(state.pending_tool("call-1").is_none()); - assert!(state.complete_tool("call-1", "issues.read").is_err()); assert!(state .accept_event(&event( 3, @@ -509,10 +511,8 @@ fn mutation_prose_survives_sidecar_decode_pending_state_and_semantic_projection( "description": "The document must contain the token CHAT8322bda781b81.", "initialPlan": plan, "idempotencyKey": "CHAT8322bda781b81-task", - "apiToken": "actual-credential", }); - let mut expected = input.clone(); - expected["apiToken"] = json!("[REDACTED]"); + let expected = input.clone(); let emitted = state .accept_event(&event( 1, @@ -578,34 +578,27 @@ fn mutation_prose_survives_sidecar_decode_pending_state_and_semantic_projection( false, ), ] { - state - .complete_tool( - "call-1", - state - .pending_tool("call-1") - .unwrap() - .operation_id - .clone() - .as_str(), - ) - .unwrap(); - let emitted = state - .accept_event(&event( - 2, - GeneratedAcpxSidecarEventType::RuntimeToolCalled, - Some("turn-1"), - json!({"callId": "call-1", "operationId": operation, "input": {field: prose}}), - )) - .unwrap(); - let AcpxProviderStateEvent::ToolCall { input, .. } = &emitted[0] else { + let mut candidate = AcpxProviderState::new("run-1").unwrap(); + candidate.begin_turn("turn-1").unwrap(); + let emitted = candidate.accept_event(&event( + 1, + GeneratedAcpxSidecarEventType::RuntimeToolCalled, + Some("turn-1"), + json!({"callId":"call-1", "operationId":operation, "input":{field:prose}}), + )); + if !preserved { + assert!(emitted + .unwrap_err() + .to_string() + .contains("refusing to execute altered arguments")); + assert!(candidate.pending_tool("call-1").is_none()); + continue; + } + let events = emitted.unwrap(); + let AcpxProviderStateEvent::ToolCall { input, .. } = &events[0] else { panic!("expected tool call"); }; - assert_eq!( - input[field] == json!(prose), - preserved, - "{operation}: {prose}" - ); - assert!(!input.to_string().contains("actual-credential")); + assert_eq!(input[field], json!(prose)); } } diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_provider_turns.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_provider_turns.rs index f2897a54158..0f8188c6237 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_provider_turns.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_provider_turns.rs @@ -7,7 +7,7 @@ use paperclip_runner_core::acpx_provider_session::{ use paperclip_runner_core::acpx_provider_state::AcpxProviderStateEvent; use paperclip_runner_core::acpx_sidecar_transport::AcpxSidecarTransportConfig; use paperclip_runner_core::provider_bridge::{ - authorized_tool_catalog_digest, AuthorizedTool, AuthorizedToolSet, + authorized_tool_catalog_digest, AuthorizedTool, AuthorizedToolSet, ToolResult, }; use serde_json::json; @@ -219,45 +219,51 @@ fn admits_only_catalog_authorized_tool_calls() { session.shutdown("test complete").unwrap(); } +fn late_result(call_id: &str) -> ToolResult { + ToolResult { + call_id: call_id.to_owned(), + operation_id: "issues.read".to_owned(), + result: json!({"id":"issue-1"}), + is_error: false, + } +} + #[test] -fn returns_pending_tool_cancellations_before_the_terminal_event() { +fn retains_pending_tools_until_authoritative_late_results_and_deduplicates_retries() { let mut session = AcpxProviderSession::start(&config("turns-tool-terminal")).unwrap(); - session - .start_turn("turn-1", "Please help", &std::env::temp_dir()) - .unwrap(); - let tool = session.poll_event(Duration::from_secs(1)).unwrap().unwrap(); - assert!(matches!(tool[0], AcpxProviderStateEvent::ToolCall { .. })); - - let terminal = session.poll_event(Duration::from_secs(1)).unwrap().unwrap(); - assert!(matches!( - &terminal[0], - AcpxProviderStateEvent::ToolResult(result) - if result.call_id == "call-1" - && result.operation_id == "issues.read" - && result.is_error - && result.result["error"]["code"] == "acpx_turn_settled" - )); - assert!(matches!( - terminal.last().unwrap(), - AcpxProviderStateEvent::TurnTerminal { turn_id, .. } if turn_id == "turn-1" - )); - - assert!(session.state().pending_tool("call-1").is_none()); - session - .start_turn("turn-2", "Please continue", &std::env::temp_dir()) - .expect("terminal settlement must leave the session reusable"); - let next_tool = session.poll_event(Duration::from_secs(1)).unwrap().unwrap(); - assert!(matches!( - &next_tool[0], - AcpxProviderStateEvent::ToolCall { call_id, operation_id, .. } - if call_id == "call-2" && operation_id == "issues.read" - )); - let next_terminal = session.poll_event(Duration::from_secs(1)).unwrap().unwrap(); - assert!(matches!( - next_terminal.last().unwrap(), - AcpxProviderStateEvent::TurnTerminal { turn_id, .. } if turn_id == "turn-2" - )); - assert!(session.state().pending_tool("call-2").is_none()); + for (turn_id, call_id) in [("turn-1", "call-1"), ("turn-2", "call-2")] { + session + .start_turn(turn_id, "Please help", &std::env::temp_dir()) + .unwrap(); + let tool = session.poll_event(Duration::from_secs(1)).unwrap().unwrap(); + assert!( + matches!(&tool[0], AcpxProviderStateEvent::ToolCall { call_id: actual, .. } if actual == call_id) + ); + let terminal = session.poll_event(Duration::from_secs(1)).unwrap().unwrap(); + assert_eq!(terminal.len(), 1, "must not invent a cancellation result"); + assert!( + matches!(&terminal[0], AcpxProviderStateEvent::TurnTerminal { turn_id: actual, .. } if actual == turn_id) + ); + assert!(session.state().pending_tool(call_id).is_some()); + assert!(session + .start_turn("blocked-turn", "Wait for settlement", &std::env::temp_dir()) + .is_err()); + session.deliver_tool_result(&late_result(call_id)).unwrap(); + session + .deliver_tool_result(&late_result(call_id)) + .expect("lost acknowledgement is harmless"); + let mut conflict = late_result(call_id); + conflict.result = json!({"id":"different"}); + let error = session + .deliver_tool_result(&conflict) + .unwrap_err() + .to_string(); + assert!( + error.contains(call_id) && error.contains("issues.read"), + "{error}" + ); + assert!(session.state().pending_tool(call_id).is_none()); + } session.shutdown("test complete").unwrap(); } @@ -276,12 +282,10 @@ fn rotates_settled_tool_receipts_between_reusable_turns() { if call_id == "call-reused" && operation_id == "issues.read" )); let terminal = session.poll_event(Duration::from_secs(1)).unwrap().unwrap(); - assert!(matches!( - &terminal[0], - AcpxProviderStateEvent::ToolResult(result) - if result.call_id == "call-reused" - && result.result["error"]["code"] == "acpx_turn_settled" - )); + assert_eq!(terminal.len(), 1); + session + .deliver_tool_result(&late_result("call-reused")) + .unwrap(); assert!(matches!( terminal.last().unwrap(), AcpxProviderStateEvent::TurnTerminal { turn_id: settled, .. } @@ -332,6 +336,9 @@ fn reaps_the_old_provider_before_admitting_a_late_tool_callback() { .unwrap(); session.poll_event(Duration::from_secs(1)).unwrap().unwrap(); session.poll_event(Duration::from_secs(1)).unwrap().unwrap(); + session + .deliver_tool_result(&late_result("call-reused")) + .unwrap(); session .start_turn("turn-2", "Use a fresh provider", &std::env::temp_dir()) @@ -360,6 +367,9 @@ fn fails_closed_before_reusing_a_settled_turn_identity() { .unwrap(); session.poll_event(Duration::from_secs(1)).unwrap().unwrap(); session.poll_event(Duration::from_secs(1)).unwrap().unwrap(); + session + .deliver_tool_result(&late_result("call-reused")) + .unwrap(); let reused = session .start_turn("turn-1", "Do not alias old events", &std::env::temp_dir()) @@ -578,62 +588,28 @@ fn reserved_completion_waits_for_feedback_and_allows_correction_in_same_turn() { } #[test] -fn correlates_reserved_results_by_raw_digest_without_exposing_sensitive_values() { - let mut session = - AcpxProviderSession::start(&config("turns-sensitive-reserved-result-terminal")).unwrap(); - session - .start_turn("turn-1", "Please help", &std::env::temp_dir()) - .unwrap(); - - assert!(matches!( - &session.poll_event(Duration::from_secs(1)).unwrap().unwrap()[0], - AcpxProviderStateEvent::ToolCall { operation_id, .. } - if operation_id == "paperclip_finish" - )); - let result_events = session.poll_event(Duration::from_secs(1)).unwrap().unwrap(); - assert!(matches!( - &result_events[0], - AcpxProviderStateEvent::SemanticResult(result) - if result.result["summary"] - .as_str() - .is_some_and(|summary| summary.contains("REDACTED") - && !summary.contains("matching-sensitive-value")) - )); - - let terminal = session.poll_event(Duration::from_secs(1)).unwrap().unwrap(); - assert!(matches!( - terminal.last().unwrap(), - AcpxProviderStateEvent::TurnTerminal { turn_id, .. } if turn_id == "turn-1" - )); - session.shutdown("test complete").unwrap(); -} - -#[test] -fn rejects_sensitive_reserved_results_that_only_match_after_redaction() { - let mut session = AcpxProviderSession::start(&config( +fn rejects_sensitive_reserved_input_before_dispatch_without_exposing_values() { + for mode in [ + "turns-sensitive-reserved-result-terminal", "turns-mismatched-sensitive-reserved-result-terminal", - )) - .unwrap(); - session - .start_turn("turn-1", "Please help", &std::env::temp_dir()) - .unwrap(); - assert!(matches!( - &session.poll_event(Duration::from_secs(1)).unwrap().unwrap()[0], - AcpxProviderStateEvent::ToolCall { operation_id, .. } - if operation_id == "paperclip_finish" - )); - - let error = session - .poll_event(Duration::from_secs(1)) - .unwrap_err() - .to_string(); - assert!( - error.contains("does not match its authorized invocation"), - "{error}" - ); - assert!(!error.contains("matching-sensitive-value"), "{error}"); - assert!(!error.contains("different-sensitive-value"), "{error}"); - assert!(session.shutdown("already closed").is_ok()); + ] { + let mut session = AcpxProviderSession::start(&config(mode)).unwrap(); + session + .start_turn("turn-1", "Please help", &std::env::temp_dir()) + .unwrap(); + let error = session + .poll_event(Duration::from_secs(1)) + .unwrap_err() + .to_string(); + assert!( + error.contains("refusing to execute altered arguments"), + "{error}" + ); + assert!(!error.contains("matching-sensitive-value"), "{error}"); + assert!(!error.contains("different-sensitive-value"), "{error}"); + assert!(session.state().pending_tool("call-finish").is_none()); + session.shutdown("already closed").unwrap(); + } } #[test] diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs index 060f5d1959a..dac280fdf3e 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs @@ -3054,6 +3054,103 @@ fn codex_resume_advertises_the_same_authorized_tools() { fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); } +#[test] +fn interrupted_tool_accepts_one_authoritative_result_across_restart_and_lost_ack() { + for restart in [false, true] { + let directory = temporary_directory(if restart { + "interrupt-tool-restart" + } else { + "interrupt-tool-live" + }); + let config = provider_config(&directory, &["--require-dynamic-tool", "--emit-tool-call"]); + let runner_config = durable_config(&directory); + let mut executor = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + executor + .execute(&command( + "prepare", + 1, + "run.prepare", + json!({"provider":config,"authorizedTools":task_context_tool_set()}), + )) + .unwrap(); + executor + .execute(&command("open", 2, "session.open", json!({}))) + .unwrap(); + executor + .execute(&command( + "turn", + 3, + "turn.start", + json!({"text":"Hold the actual operation until after interruption."}), + )) + .unwrap(); + wait_for_executor_event(&mut executor, "semantic_tool.input"); + executor + .execute(&command( + "interrupt", + 4, + "turn.interrupt", + json!({"reason":"test_result_delivery_barrier"}), + )) + .unwrap(); + let saved: Value = + serde_json::from_slice(&fs::read(directory.join("codex-provider-state.json")).unwrap()) + .unwrap(); + assert_eq!(saved["toolBridge"]["pending"].as_object().unwrap().len(), 1); + assert!(saved["pendingEvents"] + .as_array() + .unwrap() + .iter() + .any(|event| event["payload"]["reason"] == "test_result_delivery_barrier")); + if restart { + drop(executor); + executor = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + } + let result = json!({"callId":"semantic-call-1", "operationId":"get_task_context", "result":{"ok":true,"savedHash":"exact-write"}, "isError":false}); + let delivered = executor + .execute(&command( + "result", + 5, + "semantic_tool.result", + result.clone(), + )) + .unwrap(); + assert_eq!(delivered.result["status"], "settled_after_turn"); + // Lose the command ACK, restart, then deliver precisely the same result. + drop(executor); + executor = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + let replay = executor + .execute(&command( + "retry-result", + 6, + "semantic_tool.result", + result.clone(), + )) + .unwrap(); + assert_eq!(replay.result["status"], "duplicate"); + assert!(replay + .events + .iter() + .any(|(kind, _, value)| kind == "harness.diagnostic" + && value["code"] == "semantic_tool_result_duplicate" + && value["callId"] == "semantic-call-1")); + let mut conflict = result; + conflict["result"]["savedHash"] = json!("different-write"); + let error = executor + .execute(&command("conflict", 7, "semantic_tool.result", conflict)) + .unwrap_err(); + assert!(error.to_string().contains("semantic-call-1")); + assert!(error.to_string().contains("get_task_context")); + assert!(error.to_string().contains("conflicting duplicate")); + assert!(error.to_string().contains("existingDigest=sha256:")); + assert!(error.to_string().contains("incomingDigest=sha256:")); + assert!(!error.to_string().contains("different-write")); + assert_eq!(call_count(&directory, "turn/start"), 1); + executor.shutdown().unwrap(); + fs::remove_dir_all(directory).unwrap(); + } +} + #[test] fn durable_backend_routes_a_semantic_tool_result_back_to_codex() { let directory = temporary_directory("durable-dynamic-tool"); @@ -3450,33 +3547,15 @@ fn durable_backend_settles_pending_tools_when_recovery_finds_the_turn_ended() { break; } } - let semantic_result = observed - .iter() - .position(|event| event == "semantic_tool.result") - .expect("recovery settles the pending semantic tool"); - let reconciled = observed - .iter() - .position(|event| event == "session.reconciled") - .expect("recovery emits a reconciliation event"); - let terminal = observed - .iter() - .position(|event| event == "run.terminal") - .expect("offline turn recovery terminates the run"); - assert!(semantic_result < reconciled); - assert!(reconciled < terminal); - assert!(recovered - .execute(&command( - "late-result", - 5, - "semantic_tool.result", - json!({ - "callId": "semantic-call-1", - "operationId": "get_task_context", - "result": {"ok": true}, - "isError": false, - }), - )) - .is_err()); + assert!( + !observed.iter().any(|event| event == "semantic_tool.result"), + "recovery must not invent an effect outcome" + ); + assert!(observed.iter().any(|event| event == "session.reconciled")); + let late = recovered.execute(&command("late-result", 5, "semantic_tool.result", json!({ + "callId":"semantic-call-1", "operationId":"get_task_context", "result":{"ok":true}, "isError":false, + }))).unwrap(); + assert_eq!(late.result["status"], "settled_after_turn"); recovered.shutdown().expect("stop recovered provider"); fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); @@ -3976,28 +4055,12 @@ fn durable_backend_settles_tools_before_a_natural_terminal_event() { break; } } - let semantic_result = observed - .iter() - .position(|event| event == "semantic_tool.result") - .expect("terminal settlement emits a failed semantic result"); - let terminal = observed - .iter() - .position(|event| event == "turn.completed") - .expect("provider terminal event is emitted"); - assert!(semantic_result < terminal); - assert!(executor - .execute(&command( - "late-result", - 4, - "semantic_tool.result", - json!({ - "callId": "semantic-call-1", - "operationId": "get_task_context", - "result": {"ok": true}, - "isError": false, - }), - )) - .is_err()); + assert!(!observed.iter().any(|event| event == "semantic_tool.result")); + assert!(observed.iter().any(|event| event == "turn.completed")); + let late = executor.execute(&command("late-result", 4, "semantic_tool.result", json!({ + "callId":"semantic-call-1", "operationId":"get_task_context", "result":{"ok":true}, "isError":false, + }))).unwrap(); + assert_eq!(late.result["status"], "settled_after_turn"); executor.shutdown().unwrap(); fs::remove_dir_all(directory).unwrap(); @@ -4167,27 +4230,15 @@ fn durable_stop_settles_pending_semantic_tools_without_a_courtesy_interrupt() { .execute(&command("stop", 4, "turn.stop", json!({}))) .unwrap(); assert_eq!(stopped.result["providerExitConfirmed"], true); - let result = wait_for_executor_event(&mut executor, "semantic_tool.result"); - assert_eq!(result.payload["semantic_tool"]["outcome"], "failed"); - assert_eq!( - result.payload["semantic_tool"]["callId"], - input.payload["semantic_tool"]["callId"] - ); - assert_eq!( - result.payload["semantic_tool"]["correlation"], - input.payload["semantic_tool"]["correlation"] - ); - assert!(executor - .execute(&command( - "late-result", - 5, - "semantic_tool.result", - json!({ - "callId": "semantic-call-1", "operationId": "get_task_context", - "result": {"ok": true}, "isError": false, - }) - )) - .is_err()); + assert_eq!(input.payload["semantic_tool"]["callId"], "semantic-call-1"); + assert!(!poll_and_ack(&mut executor) + .unwrap() + .iter() + .any(|event| event.event_type == "semantic_tool.result")); + let late = executor.execute(&command("late-result", 5, "semantic_tool.result", json!({ + "callId":"semantic-call-1", "operationId":"get_task_context", "result":{"ok":true}, "isError":false, + }))).unwrap(); + assert_eq!(late.result["status"], "settled_after_turn"); assert_eq!(call_count(&directory, "turn/interrupt"), 0); assert_eq!(call_count(&directory, "thread/start"), 1); assert_eq!(call_count(&directory, "turn/start"), 1); diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/provider_bridge.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/provider_bridge.rs index 2504f383ed1..61a0a022502 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/provider_bridge.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/provider_bridge.rs @@ -288,26 +288,62 @@ fn recovered_bridge_rejects_tampered_authorization_state() { } #[test] -fn cancellation_completes_pending_calls_and_rejects_late_results() { - let mut bridge = ProviderToolBridge::default(); - bridge.prepare(tools("computed")).unwrap(); - bridge - .begin_call("call-1".into(), "get_task_context".into(), json!({})) - .unwrap(); - let cancelled = bridge - .cancel_pending_calls("provider_turn_stopped") - .unwrap(); - assert_eq!(cancelled.len(), 1); - assert!(cancelled[0].is_error); - assert_eq!(bridge.pending_calls().count(), 0); - assert!(bridge - .apply_result(ToolResult { - call_id: "call-1".into(), - operation_id: "get_task_context".into(), - result: json!({"ok": true}), - is_error: false, - }) - .is_err()); +fn stop_and_restart_keep_inflight_effects_pending_until_authoritative_result() { + for stop_before_result in [false, true] { + for restart in [false, true] { + let mut bridge = ProviderToolBridge::default(); + bridge.prepare(tools("computed")).unwrap(); + bridge + .begin_call("call-1".into(), "get_task_context".into(), json!({})) + .unwrap(); + let result = ToolResult { + call_id: "call-1".into(), + operation_id: "get_task_context".into(), + result: json!({"written": true}), + is_error: false, + }; + if !stop_before_result { + bridge.apply_result(result.clone()).unwrap(); + } + assert!(bridge + .cancel_pending_calls("provider_turn_stopped") + .unwrap() + .is_empty()); + if restart { + bridge = serde_json::from_str(&serde_json::to_string(&bridge).unwrap()).unwrap(); + bridge.attach_existing_run().unwrap(); + } + assert!(bridge + .begin_call("new-call".into(), "get_task_context".into(), json!({})) + .is_err()); + if stop_before_result { + assert_eq!(bridge.pending_calls().count(), 1); + assert!(bridge.prepare_turn().is_err()); + assert!(bridge.attach_run(tools("computed")).is_err()); + } + bridge.apply_result(result.clone()).unwrap(); + bridge + .apply_result(result.clone()) + .expect("identical delivery is idempotent"); + assert_eq!(bridge.pending_calls().count(), 0); + assert_eq!( + bridge + .replay_result("call-1", "get_task_context", &json!({})) + .unwrap(), + Some(result.clone()) + ); + assert!(bridge + .apply_result(ToolResult { + result: json!({"written": false}), + ..result + }) + .is_err()); + bridge.prepare_turn().unwrap(); + bridge + .begin_call("new-call".into(), "get_task_context".into(), json!({})) + .unwrap(); + } + } } #[test] @@ -333,6 +369,7 @@ fn turn_settlement_releases_value_capacity_without_reusing_call_ids() { assert!(bridge .begin_call("call-1".into(), "get_task_context".into(), json!({})) .is_err()); + bridge.prepare_turn().unwrap(); bridge .begin_call("call-2".into(), "get_task_context".into(), json!({})) .expect("a new turn can use a fresh call id after releasing exact values"); @@ -405,8 +442,18 @@ fn turn_settlement_cannot_be_blocked_by_completed_value_pressure() { } let settled = bridge.settle_turn("provider_turn_terminated").unwrap(); - assert_eq!(settled.len(), 1); - assert!(settled[0].is_error); + assert!(settled.is_empty()); + assert_eq!(bridge.pending_calls().count(), 1); + assert!(bridge.prepare_turn().is_err()); + bridge + .apply_result(ToolResult { + call_id: "call-4".into(), + operation_id: "get_task_context".into(), + result: json!({"ok": true}), + is_error: false, + }) + .unwrap(); + bridge.prepare_turn().unwrap(); assert!(bridge .begin_call("call-0".into(), "get_task_context".into(), json!({})) .is_err()); @@ -662,6 +709,7 @@ fn settles_completed_receipts_before_the_next_turn() { .begin_call("call-next".into(), "get_task_context".into(), json!({})) .is_err()); bridge.settle_turn("provider_turn_terminated").unwrap(); + bridge.prepare_turn().unwrap(); assert!(bridge .begin_call("call-next".into(), "get_task_context".into(), json!({})) .is_ok()); @@ -704,9 +752,6 @@ fn settlement_preserves_call_ids_for_the_durable_run() { assert!(bridge .begin_call("call-1".into(), "get_task_context".into(), json!({})) .is_err()); - bridge - .begin_call("call-2".into(), "get_task_context".into(), json!({})) - .unwrap(); let encoded = serde_json::to_string(&bridge).unwrap(); let mut recovered: ProviderToolBridge = serde_json::from_str(&encoded).unwrap(); @@ -759,16 +804,18 @@ fn exact_identity_overflow_saturates_the_durable_run() { .expect("the controlled turn stop retains replay protection"); assert!(recovered.durable_run_receipt_limit_reached()); assert!(recovered.has_completed_call("settled-0")); + assert!(!recovered.has_completed_call("last-call")); + assert_eq!(recovered.pending_calls().count(), 1); + assert!(recovered.prepare_turn().is_err()); + recovered + .apply_result(ToolResult { + call_id: "last-call".into(), + operation_id: "get_task_context".into(), + result: json!({"ok": true}), + is_error: false, + }) + .unwrap(); assert!(recovered.has_completed_call("last-call")); - let stopped_turn_receipt = recovered - .replay_result("last-call", "get_task_context", &json!({})) - .unwrap() - .expect("the call admitted before exhaustion retains an exact terminal receipt"); - assert!(stopped_turn_receipt.is_error); - assert_eq!( - stopped_turn_receipt.result["error"]["code"], - "provider_turn_terminated" - ); assert!(recovered .begin_call("settled-0".into(), "get_task_context".into(), json!({})) .is_err()); @@ -831,12 +878,12 @@ fn settled_result_byte_exhaustion_recovers_after_turn_cleanup() { json!({}), ) .expect_err("settled byte exhaustion must stop the durable run"); - assert!(error.is_active_turn_receipt_limit()); + assert!(error.to_string().contains("provider turn stopped")); let encoded = serde_json::to_string(&bridge).unwrap(); let mut recovered: ProviderToolBridge = serde_json::from_str(&encoded).unwrap(); recovered.attach_existing_run().unwrap(); - assert!(recovered.durable_run_receipt_limit_reached()); + assert!(recovered.turn_closed()); assert_eq!( recovered .replay_result("large-settled-0", "get_task_context", &json!({})) @@ -868,8 +915,16 @@ fn settled_result_byte_exhaustion_recovers_after_turn_cleanup() { .expect("releasing bulky results clears transient byte pressure"); recovered .settle_turn("provider_turn_terminated") - .expect("the admitted next-turn call remains settleable"); - + .expect("closing admission preserves the admitted next-turn call"); + assert!(recovered.attach_run(tools("computed")).is_err()); + recovered + .apply_result(ToolResult { + call_id: "after-turn-boundary".into(), + operation_id: "get_task_context".into(), + result: json!({"ok":true}), + is_error: false, + }) + .unwrap(); recovered.attach_run(tools("computed")).unwrap(); recovered .begin_call( @@ -1003,18 +1058,19 @@ fn recovery_rejects_state_without_room_for_a_pending_result() { } #[test] -fn settles_pending_receipts_with_explicit_terminal_results() { +fn stopped_provider_does_not_prove_pending_effect_failed() { let mut bridge = ProviderToolBridge::default(); bridge.prepare(tools("computed")).unwrap(); bridge .begin_call("call-1".into(), "get_task_context".into(), json!({})) .unwrap(); - - let settled = bridge.settle_turn("provider_turn_terminated").unwrap(); - assert_eq!(settled.len(), 1); - assert_eq!(settled[0].call_id, "call-1"); - assert!(settled[0].is_error); - assert_eq!(bridge.pending_calls().count(), 0); + assert!(bridge + .settle_turn("provider_turn_terminated") + .unwrap() + .is_empty()); + assert_eq!(bridge.pending_calls().count(), 1); + assert!(!bridge.has_completed_call("call-1")); + assert!(bridge.prepare_turn().is_err()); } #[test] diff --git a/packages/paperclip-runner/src/contracts/native-session-backend.ts b/packages/paperclip-runner/src/contracts/native-session-backend.ts index 3fb4f928e50..7a84e9577d2 100644 --- a/packages/paperclip-runner/src/contracts/native-session-backend.ts +++ b/packages/paperclip-runner/src/contracts/native-session-backend.ts @@ -93,11 +93,26 @@ export interface NativeSessionSnapshotOptions { signal: AbortSignal; } +/** A dispatched operation may have taken effect, but has no proven result. */ +export class SemanticToolOutcomeUnknownError extends Error { + readonly code = "semantic_tool_outcome_unknown"; + + constructor(message: string, options?: ErrorOptions) { + super(message, options); + this.name = "SemanticToolOutcomeUnknownError"; + } +} + +export function isSemanticToolOutcomeUnknownError(error: unknown): error is SemanticToolOutcomeUnknownError { + // Match across the server's vendored and source-mode runtime boundaries. + return error instanceof Error && "code" in error && error.code === "semantic_tool_outcome_unknown"; +} + /** The exact close owner has torn down its controller without proving suspension. */ export class NativeSessionCloseUnrecoverableError extends Error { readonly code = "native_session_close_unrecoverable"; - constructor() { + constructor(readonly settlement?: Record) { super( "provider_transport_failed: runner did not durably suspend before checkpoint", ); diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts index 283429f55d6..f1932806069 100644 --- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts +++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts @@ -26,6 +26,7 @@ import { validatePrpEvent } from "../protocol/replay-contract.js"; import { digestPaperclipSemanticContent } from "../semantic-tools/receipts.js"; import { DurablePrpControlPlane, + SemanticToolNotDispatchedError, inspectWarmRunTransition, spawnRunner, type RunnerProcessLaunchSpec, @@ -819,7 +820,9 @@ function sendMaskedJson(socket: Socket, value: unknown): void { } else if (payload.length <= 0xffff) { header.push(0x80 | 126, payload.length >>> 8, payload.length & 0xff); } else { - throw new Error("Test client frame exceeds the supported size."); + const length = Buffer.alloc(8); + length.writeBigUInt64BE(BigInt(payload.length)); + header.push(0x80 | 127, ...length); } const masked = Buffer.from(payload); for (let index = 0; index < masked.length; index += 1) { @@ -2216,7 +2219,74 @@ describe.sequential("DurablePrpControlPlane", () => { } }); - it("recovers one semantic call from the durable event after a coordinator restart", async () => { + it("delivers a large lossless result without copying its large input into the command", async () => { + const root = mkdtempSync(resolve(tmpdir(), "paperclip-large-tool-result-")); + const content = "日本語🦀".repeat(34_000) + "END"; + const args = { content }; + const handler = vi.fn(async () => ({ result: { content } })); + const options = { stateDirectory: root, identity, expectedRunnerVersion, expectedRunnerDigest, onSemanticToolInput: handler }; + const core = new DurablePrpControlPlane(options); + try { + await core.start(); + const client = (await authenticate(core, core.issueBootstrapTicket()))!; + const event = semanticInputEvent(); + const semantic = ((event.payload as any).payload as any).semantic_tool; + semantic.input = args; + semantic.content.digest = digestPaperclipSemanticContent(args); + sendSecure(client, event); + const frames = [await receiveSecure(client), await receiveSecure(client)]; + const wire = frames.find((frame) => frame?.kind === "command")!.payload as any; + expect(wire.payload.result.content).toBe(content); + expect(wire.payload.input).toBeUndefined(); + expect(wire.payload.inputDigest).toBe(digestPaperclipSemanticContent(args).slice("sha256:".length)); + expect(Buffer.byteLength(JSON.stringify(wire))).toBeLessThan(1024 * 1024); + sendSecure(client, { protocol: "paperclip.runner", version: 1, kind: "command_result", payload: { + commandId: wire.commandId, controllerSeq: wire.controllerSeq, commandType: wire.type, + status: "completed", result: { status: "delivered", callId: "call-1" }, + } }); + await vi.waitFor(() => expect(core.store.state.commands[0]?.status).toBe("completed")); + expect(core.semanticToolResultsSettled()).toBe(true); + const savedDigest = core.store.state.commands[0]!.payload.inputDigest; + core.store.state.commands[0]!.payload.inputDigest = `sha256:${"0".repeat(64)}`; + expect(core.semanticToolResultsSettled()).toBe(false); + core.store.state.commands[0]!.payload.inputDigest = savedDigest; + const reopened = new DurablePrpControlPlane(options); + expect(reopened.semanticToolResultsSettled()).toBe(true); + await reopened.stop(); + expect(handler).toHaveBeenCalledTimes(1); + client.socket.destroy(); + } finally { await core.stop(); rmSync(root, { recursive: true, force: true }); } + }); + + it.each(["before_dispatch", "during_effect"] as const)("settles only proven pre-dispatch cancellation (%s)", async (stage) => { + const root = mkdtempSync(resolve(tmpdir(), "paperclip-tool-dispatch-boundary-")); + const handler = vi.fn(async () => { + if (stage === "before_dispatch") throw new SemanticToolNotDispatchedError(); + throw new Error("connection lost after possibly committing a write"); + }); + const core = new DurablePrpControlPlane({ stateDirectory: root, identity, + expectedRunnerVersion, expectedRunnerDigest, onSemanticToolInput: handler }); + try { + await core.start(); + const client = (await authenticate(core, core.issueBootstrapTicket()))!; + sendSecure(client, semanticInputEvent()); + await receiveSecure(client); + await vi.waitFor(() => expect(handler).toHaveBeenCalledTimes(1)); + if (stage === "before_dispatch") { + await vi.waitFor(() => expect(core.store.state.commands).toHaveLength(1)); + expect(core.store.state.commands[0]).toMatchObject({ type: "semantic_tool.result", payload: { + callId: "call-1", isError: true, result: { error: { code: "semantic_tool_not_dispatched" } }, + } }); + } else { + await vi.waitFor(() => expect(core.semanticToolSettlementDiagnostics()).toMatchObject({ persistenceFailed: true })); + expect(core.store.state.commands).toEqual([]); + } + expect(core.semanticToolResultsSettled()).toBe(false); + client.socket.destroy(); + } finally { await core.stop(); rmSync(root, { recursive: true, force: true }); } + }); + + it("retains an uncertain semantic outcome without redispatching after a coordinator restart", async () => { const root = mkdtempSync(resolve(tmpdir(), "paperclip-prp-recovery-")); let firstCalls = 0; const first = new DurablePrpControlPlane({ @@ -2264,22 +2334,13 @@ describe.sequential("DurablePrpControlPlane", () => { await recovered.start(); const client = await authenticate(recovered, leaseToken!); sendSecure(client!, semanticInputEvent()); - const outcomes = [ - await receiveSecure(client!), - await receiveSecure(client!), - ]; - const command = outcomes.find((outcome) => outcome?.kind === "command"); - expect(outcomes.some((outcome) => outcome?.kind === "ack")).toBe(true); - expect(command?.payload).toMatchObject({ - type: "semantic_tool.result", - payload: { - callId: "call-1", - operationId: "get_task_context", - result: { ok: true, operationId: "get_task_context" }, - isError: false, - }, + await expect(receiveSecure(client!)).resolves.toMatchObject({ kind: "ack" }); + expect(recoveredCalls).toBe(0); + expect(recovered.semanticToolResultsSettled()).toBe(false); + expect(recovered.semanticToolSettlementDiagnostics()).toMatchObject({ + pending: [{ callId: "call-1", operationId: "get_task_context" }], deliveries: [], }); - expect(recoveredCalls).toBe(1); + expect(recovered.store.state.commands.filter((entry) => entry.type === "semantic_tool.result")).toEqual([]); const tampered = semanticInputEvent(2); const tamperedEvent = tampered.payload as Record; diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts index b3385626ecb..62f1d6207b4 100644 --- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts +++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts @@ -52,7 +52,8 @@ const coreStateSchema = "paperclip.runner.durable.control-plane-state.v1"; const transitionCoreStateSchema = "paperclip.runner.durable.control-plane-state.warm-transition.v1"; const maxFrameBytes = 1024 * 1024; -const maxCommandBytes = maxFrameBytes - 4 * 1024; +// Secure frames hex-encode ciphertext. Reserve envelope/tag space as well. +const maxCommandBytes = Math.floor((maxFrameBytes - 4 * 1024) / 2); const maxCommands = 500; // A provider can emit several 100-event runner batches before the transport's // polling turn regains the event loop. Match the transport's explicit deferred @@ -224,6 +225,13 @@ interface SecureChannel { sessionId: string; } +/** Only the dispatch boundary may assert that no business operation started. */ +export class SemanticToolNotDispatchedError extends Error { + constructor() { + super("semantic_tool_not_dispatched"); + } +} + export interface DurablePrpControlPlaneOptions { stateDirectory: string; identity: DurableRecoveryIdentity; @@ -586,7 +594,11 @@ function unsettledSemanticInput( command.payload.sourceEventType !== event.eventType || canonicalJson(command.payload.correlation) !== canonicalJson(expectedCorrelation) || - canonicalJson(command.payload.input) !== canonicalJson(semantic.input) + (command.payload.inputDigest === undefined + ? canonicalJson(command.payload.input) !== canonicalJson(semantic.input) + : command.payload.inputDigest !== canonicalDigest(semantic.input) + || (command.payload.input !== undefined + && canonicalJson(command.payload.input) !== canonicalJson(semantic.input))) ); } catch { // Malformed retained evidence cannot establish settled authority, and @@ -1622,6 +1634,35 @@ export class DurablePrpControlPlane { ); } + /** Safe run-log evidence: identities and delivery state, never tool contents. */ + semanticToolSettlementDiagnostics(): Record { + const pending = this.#store.state.committedEvents.filter((event) => + unsettledSemanticInput(event, this.#store.state), + ); + return { + persistenceFailed: this.#semanticResultPersistenceFailed, + pending: pending.map((entry) => { + const event = entry.envelope.payload as Record; + const payload = event.payload as Record; + const tool = payload.semantic_tool as Record; + return { + sourceEventId: entry.sourceEventId, + callId: tool.callId, + operationId: tool.operationId, + inputDigest: canonicalDigest(tool.input), + }; + }), + deliveries: this.#store.state.commands + .filter((command) => command.type === "semantic_tool.result" && command.status !== "completed") + .map((command) => ({ + commandId: command.commandId, + callId: command.payload.callId, + operationId: command.payload.operationId, + status: command.status, + })), + }; + } + /** * Atomically advances a settled reusable runner to a new run authority while * retaining its existing connection lease secret. The runner performs the @@ -3215,13 +3256,21 @@ export class DurablePrpControlPlane { const alreadyQueued = this.#store.state.commands.some( (command) => command.commandId === commandId, ); - if (!alreadyQueued && !this.#pendingSemanticCalls.has(commandId)) { + // A committed input without a result after restart may already have + // changed the outside world. Never redispatch it on event replay. Only + // the original admitted callback may publish its result; missing outcome + // evidence stays pending for authoritative reconciliation. + if (existing === undefined && !alreadyQueued && !this.#pendingSemanticCalls.has(commandId)) { this.#pendingSemanticCalls.add(commandId); const queueResult = (result: unknown, isError: boolean): void => { try { + // Retain the full input once in its canonical event. Copying a + // large write into its result command can exceed the wire bound + // after the write has already committed. Bind it by exact digest. + const { input, ...metadata } = call; this.queueCommand( "semantic_tool.result", - { ...call, result, isError }, + { ...metadata, inputDigest: canonicalDigest(input), result, isError }, commandId, true, ); @@ -3237,9 +3286,20 @@ export class DurablePrpControlPlane { .then((outcome) => queueResult(outcome.result, outcome.isError === true), ) - .catch(() => - queueResult({ code: "semantic_tool_bridge_failed" }, true), - ) + .catch((error: unknown) => { + if (error instanceof SemanticToolNotDispatchedError) { + queueResult({ error: { + code: "semantic_tool_not_dispatched", + retryable: false, + message: "The turn stopped before this operation was dispatched.", + } }, true); + return; + } + // A rejected dispatcher promise does not prove that its effect + // rolled back. Do not fabricate a final failure or retry the write. + this.#semanticResultPersistenceFailed = true; + this.disconnectActiveRunner(); + }) .finally(() => this.#pendingSemanticCalls.delete(commandId)); } } diff --git a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.test.ts b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.test.ts index bc000a79251..3f68152e452 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.test.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it, vi } from "vitest"; import { resolve as resolvePath } from "node:path"; import { fileURLToPath } from "node:url"; +import { SemanticToolOutcomeUnknownError } from "../../contracts/native-session-backend.js"; import { CODEX_BLOCK_RESULT_OUTPUT_SCHEMA, @@ -2958,6 +2959,29 @@ describe("Codex app-server Codex driver", () => { ); }); + it.each(["unknown", "validation"])("preserves the dynamic tool outcome boundary (%s)", async (outcome) => { + const transport = new FakeCodexTransport(); + const error = outcome === "unknown" + ? new SemanticToolOutcomeUnknownError("write may have committed") + : new Error("invalid instruction input"); + const handler = vi.fn(async () => { throw error; }); + const session = await makeDriver([transport], { + dynamicTools: [{ name: "update_agent_instructions", description: "Write instructions.", inputSchema: { type: "object" } }], + dynamicToolHandler: handler, + }).openSession({ runId: "run-uncertain-write", normalizedSessionId: "session-uncertain-write", workingDirectory: TEST_WORKING_DIRECTORY }); + await session.startTurn({ message: { role: "user", text: "Update instructions." } }); + const response = transport.invoke({ id: "rpc-write", method: "item/tool/call", params: { + threadId: "thread-1", turnId: "turn-1", callId: "uncertain-write", tool: "update_agent_instructions", arguments: {}, + } }); + if (outcome === "unknown") await expect(response).rejects.toBe(error); + else await expect(response).resolves.toMatchObject({ success: false }); + transport.push("turn/completed", { threadId: "thread-1", turn: { id: "turn-1", status: "completed", items: [] } }); + const events = await collectUntilTerminal(session.events()); + const completions = events.filter((event) => event.eventType === "item.completed" && event.itemId === "uncertain-write"); + expect(completions).toHaveLength(outcome === "unknown" ? 0 : 1); + expect(handler).toHaveBeenCalledTimes(1); + }); + it("fails closed when an agent message changes a tool-committed result", async () => { const transport = new FakeCodexTransport(); const session = await makeDriver([transport]).openSession({ diff --git a/packages/paperclip-runner/src/drivers/codex/codex-harness-session.ts b/packages/paperclip-runner/src/drivers/codex/codex-harness-session.ts index a7843d516d4..2b22e9bbb2d 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-harness-session.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-harness-session.ts @@ -404,6 +404,7 @@ export class CodexHarnessSession await this.transport.request("turn/interrupt", { threadId: this.opened.threadId, turnId, + ...(reason ? { reason: boundedText(reason) } : {}), }); if (this.activeTurnId !== turnId) { throw new HarnessOperationAlreadyTerminalError("interruption"); diff --git a/packages/paperclip-runner/src/drivers/codex/codex-session-server-requests.ts b/packages/paperclip-runner/src/drivers/codex/codex-session-server-requests.ts index 6e352180721..e23b9f942a1 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-session-server-requests.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-session-server-requests.ts @@ -1,4 +1,5 @@ import { isAcpxCanonicalInputMethod } from "../acpx/profile-extensions.js"; +import { isSemanticToolOutcomeUnknownError } from "../../contracts/native-session-backend.js"; import type { HarnessRuntimeRequest, PaperclipQuestionSet } from "../../contracts/harness-driver.js"; import { CODEX_BLOCK_TOOL_NAME, @@ -147,6 +148,10 @@ async function handleServerRequestBody( ); return dynamicToolResponse(result); } catch (error) { + // Preserve uncertain effects for the durable controller. A normal + // failed tool response would falsely settle this call and permit + // the provider to continue as if the write had not happened. + if (isSemanticToolOutcomeUnknownError(error)) throw error; const message = boundedText( error instanceof Error ? error.message : error, ); diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts index 7ee61f16def..a434a1061e1 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts @@ -737,9 +737,10 @@ it("refuses a reusable close checkpoint when the local provider snapshot is unre } }, 15_000); -it.each(["after_budget", "within_budget", "persistence_failure"] as const)( +it.each(["after_budget", "within_budget", "interrupted_within_budget", "persistence_failure"] as const)( "fences reusable suspension against late semantic completion (%s)", async (mode) => { + const settles = mode === "within_budget" || mode === "interrupted_within_budget"; const stateDirectory = await mkdtemp( join(tmpdir(), "runnerd-late-semantic-close-"), ); @@ -760,23 +761,26 @@ it.each(["after_budget", "within_budget", "persistence_failure"] as const)( // The default 96/48-frame stress burst spends seconds on unrelated // durable text fsyncs before handler entry, consuming this barrier test's // wall-clock budget under the full suite. Stress cases retain defaults. - codexArgs: fakeCodexArgs( + codexArgs: mode === "interrupted_within_budget" + ? fakeCodexArgs(stateDirectory, "--emit-tool-call") + : fakeCodexArgs( stateDirectory, "--split-event-burst", "--split-event-prefix-count", "2", "--split-event-suffix-count", "2", ), stateDirectory, - closeGraceMs: 2_000, + closeGraceMs: 5_000, controlPlaneRegistration: async (authority) => { core = authority; await authority.start(); return { checkpoint, release: () => undefined }; }, }); - bundle.transport.setServerRequestHandler(async () => { + const handler = vi.fn(async () => { entered(); await handlerRelease; return { success: true, contentItems: [] }; }); + bundle.transport.setServerRequestHandler(handler); try { await bundle.transport.request("thread/start", { cwd: tmpdir(), @@ -815,6 +819,10 @@ it.each(["after_budget", "within_budget", "persistence_failure"] as const)( return queue(type, ...args); }); } + if (mode === "interrupted_within_budget") { + await bundle.transport.request("turn/interrupt", { reason: "test-stop-during-server-write" }); + expect(core.semanticToolResultsSettled()).toBe(false); + } const closing = bundle.transport.close().then( () => null, (error: unknown) => error, @@ -825,7 +833,8 @@ it.each(["after_budget", "within_budget", "persistence_failure"] as const)( release(); } const closeFailure = await closing; - if (mode !== "within_budget") { + expect(handler).toHaveBeenCalledTimes(1); + if (!settles) { const artifact = readRunnerdArtifactBinding( defaultCapabilityRunnerdBinary(), ); @@ -863,9 +872,9 @@ it.each(["after_budget", "within_budget", "persistence_failure"] as const)( control.identity.runId, ); expect(late[0].status).toBe( - mode === "within_budget" ? "completed" : "pending", + settles ? "completed" : "pending", ); - if (mode === "within_budget") { + if (settles) { const results = control.committedEvents.filter( (event: { eventType: string }) => event.eventType === "semantic_tool.result", @@ -875,7 +884,7 @@ it.each(["after_budget", "within_budget", "persistence_failure"] as const)( } }); } - if (mode === "within_budget") { + if (settles) { expect(closeFailure).toBeNull(); expect(core.semanticToolResultsSettled()).toBe(true); expect(checkpoint).toHaveBeenCalledWith("settled"); @@ -883,6 +892,10 @@ it.each(["after_budget", "within_budget", "persistence_failure"] as const)( expect(closeFailure).toBeInstanceOf( NativeSessionCloseUnrecoverableError, ); + expect(closeFailure).toHaveProperty("settlement.semanticTools.pending", expect.arrayContaining([ + expect.objectContaining({ callId: expect.any(String), operationId: expect.any(String), + sourceEventId: expect.any(String), inputDigest: expect.stringMatching(/^[a-f0-9]{64}$/) }), + ])); expect(checkpoint).toHaveBeenCalledWith("unsettled"); expect(checkpoint).not.toHaveBeenCalledWith("settled"); } @@ -892,7 +905,7 @@ it.each(["after_budget", "within_budget", "persistence_failure"] as const)( await rm(stateDirectory, { recursive: true, force: true }); } }, - 15_000, + 25_000, ); it("infers a remote provider turn until its own terminal event is durable", () => { diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts index 3947426a073..1a49999f210 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts @@ -48,6 +48,7 @@ import type { } from "../contracts/harness-driver.js"; import { DurablePrpControlPlane, + SemanticToolNotDispatchedError, durableRecoveryInternals, inspectWarmRunTransition, spawnRunner, @@ -4355,7 +4356,18 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { this.#controlPlaneRelease = null; } if (suspensionRequired && !runnerSettled) { - throw new NativeSessionCloseUnrecoverableError(); + const settlement = { + runnerSuspended, + providerDrained, + semanticTools: this.#core?.semanticToolSettlementDiagnostics(), + finalProviderState, + }; + try { + this.options.onDiagnostic?.(`native_session_settlement_incomplete ${JSON.stringify(settlement)}`); + } catch { + // Keep the settlement failure authoritative if its observer fails. + } + throw new NativeSessionCloseUnrecoverableError(settlement); } if (this.#ownsRoot && !adoptedRunner) { rmSync(this.#root, { recursive: true, force: true }); @@ -5497,7 +5509,11 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { NonNullable >[0], ) { - this.#throwIfFailed(); + try { + this.#throwIfFailed(); + } catch { + throw new SemanticToolNotDispatchedError(); + } const core = this.#core; const threadId = this.#threadId; const epoch = this.#turnStartResponseEpoch; @@ -5508,8 +5524,14 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { const accepted = admission === null ? true - : await Promise.race([admission.settled, this.#failureSignal]); - this.#throwIfFailed(); + : await Promise.race([admission.settled, this.#failureSignal]).catch(() => { + throw new SemanticToolNotDispatchedError(); + }); + try { + this.#throwIfFailed(); + } catch { + throw new SemanticToolNotDispatchedError(); + } if ( !accepted || this.#closed || @@ -5522,9 +5544,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { core.store.state.identity.normalizedSessionId || call.correlation.turnId !== core.store.state.identity.turnId ) { - throw new Error( - "PRP semantic tool call no longer belongs to an admitted turn", - ); + throw new SemanticToolNotDispatchedError(); } const outcome = unwrapToolResponse( await this.#handler({ diff --git a/packages/paperclip-runner/src/native-session-runtime.test.ts b/packages/paperclip-runner/src/native-session-runtime.test.ts index 1c02ca05c5b..2be94a11009 100644 --- a/packages/paperclip-runner/src/native-session-runtime.test.ts +++ b/packages/paperclip-runner/src/native-session-runtime.test.ts @@ -3901,7 +3901,7 @@ describe("executeNativeSession recovery", () => { { typed: false, closeFails: true }, { typed: true, closeFails: true, startupRace: true }, ])( - "preserves a permanent integrity failure through required cleanup (%j)", + "preserves the primary execution failure through required cleanup (%j)", async ({ typed, closeFails, startupRace = false }) => { const failure = typed ? new NativeSessionProtocolIntegrityError( @@ -3996,8 +3996,9 @@ describe("executeNativeSession recovery", () => { timeoutMs: 900_000, }; await expect(executeNativeSession(options)).rejects.toBe( - typed ? failure : closeFailure, + failure, ); + if (closeFails) expect(failure).toHaveProperty("cleanupError", closeFailure); expect(close).toHaveBeenCalledOnce(); expect(onSession).toHaveBeenLastCalledWith(null); expect(port.completeRun).not.toHaveBeenCalled(); @@ -4065,6 +4066,8 @@ describe("executeNativeSession recovery", () => { async *events() { throw new Error("cancelled remote transport"); }, startTurn: async () => ({ turnId: "remote-turn" }), result: async () => null, + async snapshot() { return { backendKind: "mock", sessionId: "stop-session", identity: this.identity(), + providerSessionId: "stop-provider", cursor: null, activeTurnId: null, pendingRuntimeRequests: [], lineage: [] }; }, close: vi.fn(async () => { throw failure; }), }; const backend: NativeSessionBackend = { @@ -4080,7 +4083,7 @@ describe("executeNativeSession recovery", () => { const options = { input: scopedInput, backend, controlPlane, runnerInstanceId: "remote-runner", controlPlaneInstanceId: "control", requireSessionCloseBeforeReturn: true, remoteCleanupScope: binding.remoteCleanupScope }; - await expect(executeNativeSession(options)).rejects.toBe(failure); + await expect(executeNativeSession(options)).rejects.toMatchObject({ message: "cancelled remote transport", cleanupError: failure }); expect(completeTerminatedRemoteNativeSessionCleanup({ ...binding, runId: "other-run" })).toBe(true); expect(completeTerminatedRemoteNativeSessionCleanup({ ...binding, companyId: "other-company" })).toBe(true); await expect(executeNativeSession(options)).rejects.toBeInstanceOf(NativeSessionCleanupQuarantinedError); @@ -4091,7 +4094,7 @@ describe("executeNativeSession recovery", () => { const independentBackend = { ...backend, openSession: vi.fn(async () => independentSession) }; await expect(executeNativeSession({ ...options, remoteCleanupScope: "other-sandbox", input: { ...scopedInput, binding: { ...scopedInput.binding, runId: independent.runId } }, - backend: independentBackend })).rejects.toBe(failure); + backend: independentBackend })).rejects.toMatchObject({ message: "cancelled remote transport", cleanupError: failure }); expect(independentBackend.openSession).toHaveBeenCalledOnce(); expect(completeTerminatedRemoteNativeSessionCleanup(binding)).toBe(true); // Same company/run, different sandbox: its quarantine must remain intact. @@ -4099,7 +4102,7 @@ describe("executeNativeSession recovery", () => { backend: independentBackend })).rejects.toBeInstanceOf(NativeSessionCleanupQuarantinedError); expect(independentBackend.openSession).toHaveBeenCalledOnce(); // Reopening is now possible; the old failure/result was never rewritten. - await expect(executeNativeSession(options)).rejects.toBe(failure); + await expect(executeNativeSession(options)).rejects.toMatchObject({ message: "cancelled remote transport", cleanupError: failure }); expect(backend.openSession).toHaveBeenCalledTimes(2); expect(controlPlane.completeRun).not.toHaveBeenCalled(); completeTerminatedRemoteNativeSessionCleanup(binding); @@ -4116,6 +4119,8 @@ describe("executeNativeSession recovery", () => { identity: () => scopedIdentity, capabilities: async () => capabilities, async *events() { throw new Error("local transport stopped"); }, startTurn: async () => ({ turnId: "local-turn" }), result: async () => null, + async snapshot() { return { backendKind: "mock", sessionId: "stop-session", identity: this.identity(), + providerSessionId: "stop-provider", cursor: null, activeTurnId: null, pendingRuntimeRequests: [], lineage: [] }; }, close: vi.fn(async () => { throw failure; }), }; const backend: NativeSessionBackend = { @@ -4129,14 +4134,14 @@ describe("executeNativeSession recovery", () => { }; const options = { input: scopedInput, backend, controlPlane, runnerInstanceId: binding.runnerInstanceId, controlPlaneInstanceId: "control", requireSessionCloseBeforeReturn: true }; - await expect(executeNativeSession(options)).rejects.toBe(failure); + await expect(executeNativeSession(options)).rejects.toMatchObject({ message: "local transport stopped", cleanupError: failure }); completeTerminatedLocalNativeSessionCleanup({ ...binding, companyId: "other-company" }); completeTerminatedLocalNativeSessionCleanup({ ...binding, runId: "other-run" }); expect(completeTerminatedLocalNativeSessionCleanup({ ...binding, runnerInstanceId: "other-runner" })).toBe(false); await expect(executeNativeSession(options)).rejects.toBeInstanceOf(NativeSessionCleanupQuarantinedError); expect(backend.openSession).toHaveBeenCalledOnce(); expect(completeTerminatedLocalNativeSessionCleanup(binding)).toBe(true); - await expect(executeNativeSession(options)).rejects.toBe(failure); + await expect(executeNativeSession(options)).rejects.toMatchObject({ message: "local transport stopped", cleanupError: failure }); expect(backend.openSession).toHaveBeenCalledTimes(2); expect(controlPlane.completeRun).not.toHaveBeenCalled(); completeTerminatedLocalNativeSessionCleanup(binding); diff --git a/packages/paperclip-runner/src/native-session-runtime.ts b/packages/paperclip-runner/src/native-session-runtime.ts index 27b5c204c3a..933352b44d8 100644 --- a/packages/paperclip-runner/src/native-session-runtime.ts +++ b/packages/paperclip-runner/src/native-session-runtime.ts @@ -2123,7 +2123,7 @@ export async function executeNativeSession( let goalCheckpointRequiresSuspension = Boolean( options.sessionGoalControl || options.resumeSessionGoalHeartbeat || persistedSession?.goal, ); - let protocolIntegrityFailure: NativeSessionProtocolIntegrityError | null = null; + let executionFailure: { error: unknown } | null = null; try { // Ownership publication is part of the execution-owned lifetime. If the // callback fails, the finally block below still quarantines and closes the @@ -2730,9 +2730,7 @@ export async function executeNativeSession( executionSucceeded = true; return { ...durableExecutionResult, ...enrichment }; } catch (error) { - if (error instanceof NativeSessionProtocolIntegrityError) { - protocolIntegrityFailure = error; - } + executionFailure = { error }; throw error; } finally { const shouldClose = @@ -2757,7 +2755,20 @@ export async function executeNativeSession( // The exact cleanup owner remains retained/quarantined above. Its // rejection must not turn permanent integrity failure into a // generic retryable transport failure at the control-plane boundary. - throw protocolIntegrityFailure ?? closeError; + if (executionFailure !== null) { + // Keep the original identity/classification (including arbitrary + // thrown values). Cleanup remains inspectable without replacing + // the initiating failure with a generic shutdown timeout. + if (executionFailure.error instanceof Error) { + try { + Object.defineProperty(executionFailure.error, "cleanupError", { + value: closeError, configurable: true, + }); + } catch { /* Frozen errors still retain their original identity. */ } + } + throw executionFailure.error; + } + throw closeError; } await options.onSessionClosed?.(); } diff --git a/server/src/services/native-runtime/agent-instruction-tools.integration.test.ts b/server/src/services/native-runtime/agent-instruction-tools.integration.test.ts index fe79d00aa8e..8bcebc91d3a 100644 --- a/server/src/services/native-runtime/agent-instruction-tools.integration.test.ts +++ b/server/src/services/native-runtime/agent-instruction-tools.integration.test.ts @@ -1,10 +1,10 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import { randomUUID } from "node:crypto"; -import { eq } from "drizzle-orm"; -import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest"; -import { agents, authUsers, companies, companyMemberships, principalPermissionGrants, heartbeatRuns, issues, agentInstructionRevisions, createDb } from "@paperclipai/db"; +import { createHash, randomUUID } from "node:crypto"; +import { eq, sql } from "drizzle-orm"; +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { activityLog, agents, authUsers, companies, companyMemberships, principalPermissionGrants, heartbeatRuns, issues, agentInstructionRevisions, createDb } from "@paperclipai/db"; import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; import { resolveManagedInstructionsRoot } from "../agent-instructions.js"; import { PaperclipRunnerToolAuthority } from "./paperclip-runner-tool-authority.js"; @@ -64,6 +64,189 @@ describe("canonical instruction tools through native authority", () => { await expect(call("update_agent_instructions", { entryFile, content: "stale", baseRevisionId: first.revision.id })).rejects.toMatchObject({ status: 409 }); expect((await call("read_agent_instructions")).content).toBe("changed\r\n"); }); + it("serializes duplicate writes at the filesystem commit and replays the exact receipt after restart", async () => { + const first = await call("read_agent_instructions"); + const content = "\uFEFF# Instructions — 日本語 🦀\r\n".repeat(600) + "\nFINAL TAIL\n"; + const request = { tool: "update_agent_instructions", callId: randomUUID(), arguments: { + targetAgentId, entryFile, content, baseRevisionId: first.revision.id, + } }; + const originalRename = fs.rename.bind(fs); + let reached!: () => void, release!: () => void; + const atCommit = new Promise((resolve) => { reached = resolve; }); + const barrier = new Promise((resolve) => { release = resolve; }); + let writes = 0; + const spy = vi.spyOn(fs, "rename").mockImplementation(async (source, destination) => { + if (destination === path.join(root, entryFile)) { + writes++; + reached(); + await barrier; + } + return originalRename(source, destination); + }); + try { + const pending = authority.execute(request); + await atCommit; + const duplicate = new PaperclipRunnerToolAuthority(db, { companyId, agentId, issueId, runId }).execute(request); + expect(await fs.readFile(path.join(root, entryFile), "utf8")).toBe(original); + release(); + const receipt = await pending as any; + expect(await duplicate).toEqual(receipt); + authority = new PaperclipRunnerToolAuthority(db, { companyId, agentId, issueId, runId }); + expect(await authority.execute(request)).toEqual(receipt); + const bytes = await fs.readFile(path.join(root, entryFile)); + expect(bytes).toEqual(Buffer.from(content)); + expect(receipt.revision.byteLength).toBe(bytes.length); + expect(receipt.revision.contentHash).toBe(createHash("sha256").update(bytes).digest("hex")); + expect(writes).toBe(1); + expect((await db.select().from(activityLog).where(eq(activityLog.runId, runId))).filter((event) => event.action === "agent.files_updated")).toHaveLength(1); + await expect(authority.execute({ ...request, arguments: { ...request.arguments, content: "conflicting replay" } })).rejects.toThrow("idempotency_conflict"); + expect(await fs.readFile(path.join(root, entryFile))).toEqual(bytes); + await expect(call("update_agent_instructions", { entryFile, baseRevisionId: receipt.revision.id, content: "🦀".repeat(300_000) })).rejects.toMatchObject({ status: 422 }); + expect(writes).toBe(1); + await db.delete(principalPermissionGrants).where(eq(principalPermissionGrants.principalId, userId)); + await expect(authority.execute(request)).rejects.toMatchObject({ status: 403 }); + } finally { + release(); + spy.mockRestore(); + } + }); + + it.each(["after_file_rename", "receipt_persistence"])("retains durable attempt evidence and refuses replay after rollback at %s", async (fault) => { + const first = await call("read_agent_instructions"); + const request = { tool: "update_agent_instructions", callId: randomUUID(), arguments: { + targetAgentId, entryFile, content: "written before database rollback\r\n", baseRevisionId: first.revision.id, + } }; + const key = `instruction:${request.callId}`; + if (fault === "receipt_persistence") { + // This is an actual PostgreSQL transaction rollback, after the real + // filesystem rename and attempted audit insert, before receipt commit. + await db.execute(sql.raw(`CREATE FUNCTION reject_instruction_test_receipt() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + IF NEW.id = '${runId}' AND NEW.result_json->'semanticToolReceipts' ? '${key}' THEN + RAISE EXCEPTION 'injected receipt persistence failure'; + END IF; + RETURN NEW; + END $$`)); + await db.execute(sql.raw(`CREATE TRIGGER reject_instruction_test_receipt BEFORE UPDATE ON heartbeat_runs + FOR EACH ROW EXECUTE FUNCTION reject_instruction_test_receipt()`)); + } + const originalRename = fs.rename.bind(fs); + let writes = 0; + const spy = vi.spyOn(fs, "rename").mockImplementation(async (source, destination) => { + await originalRename(source, destination); + if (destination === path.join(root, entryFile)) { + writes++; + if (fault === "after_file_rename") throw new Error("injected failure after rename"); + } + }); + try { + await expect(authority.execute(request)).rejects.toMatchObject({ code: "semantic_tool_outcome_unknown" }); + expect(await fs.readFile(path.join(root, entryFile), "utf8")).toBe(request.arguments.content); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + expect((run!.resultJson as any).instructionToolAttempts[key]).toMatchObject({ + operationId: request.tool, targetAgentId, inputDigest: expect.stringMatching(/^[a-f0-9]{64}$/), + }); + expect((run!.resultJson as any).semanticToolReceipts?.[key]).toBeUndefined(); + const audit = await db.select().from(activityLog).where(eq(activityLog.runId, runId)); + expect(audit.filter((event) => event.action === "agent.instruction_write_attempted")).toHaveLength(1); + expect(audit.filter((event) => event.action === "agent.files_updated")).toHaveLength(0); + authority = new PaperclipRunnerToolAuthority(db, { companyId, agentId, issueId, runId }); + await expect(authority.execute(request)).rejects.toMatchObject({ code: "semantic_tool_outcome_unknown" }); + await expect(authority.execute({ ...request, arguments: { ...request.arguments, content: "different" } })) + .rejects.toThrow("idempotency_conflict"); + expect(writes).toBe(1); + await db.delete(principalPermissionGrants).where(eq(principalPermissionGrants.principalId, userId)); + await expect(authority.execute(request)).rejects.toMatchObject({ status: 403 }); + } finally { + spy.mockRestore(); + if (fault === "receipt_persistence") { + await db.execute(sql.raw("DROP TRIGGER reject_instruction_test_receipt ON heartbeat_runs")); + await db.execute(sql.raw("DROP FUNCTION reject_instruction_test_receipt()")); + } + } + }); + + it("replays a committed receipt after its database commit acknowledgement is lost", async () => { + const first = await call("read_agent_instructions"); + const request = { tool: "update_agent_instructions", callId: randomUUID(), arguments: { + targetAgentId, entryFile, content: "committed but acknowledgement lost\n", baseRevisionId: first.revision.id, + } }; + const transaction = db.transaction.bind(db); + let commits = 0; + const spy = vi.spyOn(db, "transaction").mockImplementation(async (...args) => { + const result = await transaction(...args); + if (++commits === 2) throw new Error("injected lost commit acknowledgement"); + return result; + }); + try { + await expect(authority.execute(request)).rejects.toThrow("injected lost commit acknowledgement"); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + const receipt = (run!.resultJson as any).semanticToolReceipts[`instruction:${request.callId}`].result; + expect(receipt).toMatchObject({ changed: true, content: request.arguments.content }); + authority = new PaperclipRunnerToolAuthority(db, { companyId, agentId, issueId, runId }); + expect(await authority.execute(request)).toEqual(receipt); + const audit = await db.select().from(activityLog).where(eq(activityLog.runId, runId)); + expect(audit.filter((event) => event.action === "agent.instruction_write_attempted")).toHaveLength(1); + expect(audit.filter((event) => event.action === "agent.files_updated")).toHaveLength(1); + expect(await fs.readFile(path.join(root, entryFile), "utf8")).toBe(request.arguments.content); + } finally { spy.mockRestore(); } + }); + + it.each(["validation", "stale_base", "oversized"])("replays a definite %s failure after restart without executing again", async (fault) => { + const first = await call("read_agent_instructions"); + const request = { tool: "update_agent_instructions", callId: randomUUID(), arguments: { + targetAgentId, entryFile, content: fault === "oversized" ? "🦀".repeat(300_000) : "requested update", + baseRevisionId: first.revision.id, ...(fault === "validation" ? { unexpected: true } : {}), + } }; + if (fault === "stale_base") await fs.writeFile(path.join(root, entryFile), "newer canonical bytes"); + const originalRename = fs.rename.bind(fs); + let writes = 0; + const spy = vi.spyOn(fs, "rename").mockImplementation(async (source, destination) => { + if (destination === path.join(root, entryFile)) writes++; + return originalRename(source, destination); + }); + try { + const failure = await authority.execute(request).then(() => null, (error: unknown) => error) as any; + expect(failure).toMatchObject({ status: fault === "validation" ? 400 : fault === "stale_base" ? 409 : 422 }); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + const saved = (run!.resultJson as any).instructionToolAttempts[`instruction:${request.callId}`].failure; + expect(saved).toMatchObject({ status: failure.status, message: failure.message }); + // The stale request would now pass CAS if it were executed again. + await fs.writeFile(path.join(root, entryFile), original); + authority = new PaperclipRunnerToolAuthority(db, { companyId, agentId, issueId, runId }); + await expect(authority.execute(request)).rejects.toMatchObject(saved); + await expect(authority.execute(request)).rejects.toMatchObject(saved); + await expect(authority.execute({ ...request, arguments: { ...request.arguments, content: "different" } })) + .rejects.toThrow("idempotency_conflict"); + expect(writes).toBe(0); + expect(await fs.readFile(path.join(root, entryFile), "utf8")).toBe(original); + const audit = await db.select().from(activityLog).where(eq(activityLog.runId, runId)); + expect(audit.filter((event) => event.action === "agent.instruction_write_attempted")).toHaveLength(1); + expect(audit.filter((event) => event.action === "agent.files_updated")).toHaveLength(0); + } finally { spy.mockRestore(); } + }); + + it("does not claim a final failure when its failure receipt cannot be persisted", async () => { + const first = await call("read_agent_instructions"); + const request = { tool: "update_agent_instructions", callId: randomUUID(), arguments: { + targetAgentId, entryFile, content: "invalid input", baseRevisionId: first.revision.id, unexpected: true, + } }; + const transaction = db.transaction.bind(db); + let attempts = 0; + const spy = vi.spyOn(db, "transaction").mockImplementation(async (...args) => { + if (++attempts === 3) throw new Error("injected failure receipt outage"); + return transaction(...args); + }); + try { + await expect(authority.execute(request)).rejects.toMatchObject({ code: "semantic_tool_outcome_unknown" }); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + expect((run!.resultJson as any).instructionToolAttempts[`instruction:${request.callId}`].failure).toBeUndefined(); + authority = new PaperclipRunnerToolAuthority(db, { companyId, agentId, issueId, runId }); + await expect(authority.execute(request)).rejects.toMatchObject({ code: "semantic_tool_outcome_unknown" }); + expect(await fs.readFile(path.join(root, entryFile), "utf8")).toBe(original); + } finally { spy.mockRestore(); } + }); + it("rejects supplied identity and missing CAS bases before changing content", async () => { const first = await call("read_agent_instructions"); for (const identity of ["companyId", "agentId", "runId", "responsibleUserId", "onBehalfOfUserId", "actor"]) { diff --git a/server/src/services/native-runtime/paperclip-runner-tool-authority.ts b/server/src/services/native-runtime/paperclip-runner-tool-authority.ts index 27fd2f45e65..6155700edfc 100644 --- a/server/src/services/native-runtime/paperclip-runner-tool-authority.ts +++ b/server/src/services/native-runtime/paperclip-runner-tool-authority.ts @@ -1,3 +1,4 @@ +import { authorizeInstructionCommit } from "../agent-instruction-authorization.js"; import { executeAgentInstructionTool } from "./agent-instruction-tools.js"; import { createReadStream } from "node:fs"; import { publicChatTaskUrl } from "../chat-task-url.js"; @@ -26,7 +27,7 @@ import { getStorageService } from "../../storage/index.js"; import type { StorageService } from "../../storage/types.js"; import { assetService } from "../assets.js"; import { workspaceFileResourceService } from "../workspace-file-resources.js"; -import { badRequest, forbidden } from "../../errors.js"; +import { badRequest, forbidden, notFound, HttpError } from "../../errors.js"; import { searchRunnerApi } from "./runner-api-catalog.js"; import { executeRunnerApi, validateRunnerApiCall, RUNNER_API_MAX_BYTES, type RunnerApiFile } from "./runner-api-client.js"; import { acquireRunnerApiResponseSlot, runnerApiCompanyCaptureMaxBytes, RUNNER_API_RESPONSE_MAX_BYTES, RUNNER_API_RESPONSE_RUN_MAX_BYTES, RunnerApiResponseLimitError } from "./runner-api-response-limits.js"; @@ -48,7 +49,7 @@ import { issues, issueThreadInteractions, } from "@paperclipai/db"; -import { CAPABILITY_SEMANTIC_TOOL_CATALOG, runnerCodexDynamicToolsFit } from "../../vendor/paperclip-runner/index.js"; +import { CAPABILITY_SEMANTIC_TOOL_CATALOG, runnerCodexDynamicToolsFit, SemanticToolOutcomeUnknownError } from "../../vendor/paperclip-runner/index.js"; import { agentService } from "../agents.js"; import { approvalService } from "../approvals.js"; import { documentService } from "../documents.js"; @@ -139,6 +140,25 @@ type ToolReceipt = { result: unknown; }; +// Coalesce live callers, but never use this process-local lock as crash proof. +// The database attempt record below is the authority after process loss. +const instructionCallLocks = new WeakMap>>(); +const instructionPreWriteFailureStatuses = new Set([400, 401, 403, 404, 409, 422]); +async function withInstructionCallLock(db: Db, key: string, work: () => Promise): Promise { + let locks = instructionCallLocks.get(db); + if (!locks) instructionCallLocks.set(db, locks = new Map()); + const previous = locks.get(key); + let release!: () => void; + const pending = new Promise((resolve) => { release = resolve; }); + locks.set(key, pending); + await previous; + try { return await work(); } + finally { + release(); + if (locks.get(key) === pending) locks.delete(key); + } +} + function record(value: unknown): Record { return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record @@ -371,12 +391,14 @@ export class PaperclipRunnerToolAuthority { } switch (call.tool) { case "read_agent_instructions": - case "update_agent_instructions": case "get_agent_instruction_history": - case "restore_agent_instructions": return executeAgentInstructionTool({ db: this.db, binding: { companyId: this.binding.companyId, agentId: this.binding.agentId, runId: this.binding.runId, }, tool: call.tool, arguments: call.arguments }); + case "update_agent_instructions": + case "restore_agent_instructions": { + return this.#withInstructionMutationReceipt(call.tool, call.callId, input); + } case "create_skill": { const apiUrl = this.binding.apiUrl ?? process.env.PAPERCLIP_API_URL; const token = createLocalAgentJwt(this.binding.agentId, this.binding.companyId, context.actor.adapterType, this.binding.runId, context.run.responsibleUserId); @@ -1476,6 +1498,109 @@ export class PaperclipRunnerToolAuthority { return null; } + async #withInstructionMutationReceipt( + tool: "update_agent_instructions" | "restore_agent_instructions", + callId: string, + input: Record, + ): Promise { + if (!callId || callId.length > 500) throw badRequest("A bounded runner call id is required"); + const key = `instruction:${callId}`; + const digest = createHash("sha256").update(canonicalJson(input)).digest("hex"); + const targetAgentId = typeof input.targetAgentId === "string" ? input.targetAgentId : this.binding.agentId; + const authorize = async (tx: Db) => { + const [target] = await tx.select({ id: agents.id, companyId: agents.companyId }).from(agents) + .where(and(eq(agents.id, targetAgentId), eq(agents.companyId, this.binding.companyId))); + if (!target) throw notFound("Agent not found"); + return authorizeInstructionCommit(tx, { + type: "agent", source: "agent_jwt", companyId: this.binding.companyId, + agentId: this.binding.agentId, runId: this.binding.runId, + }, target); + }; + return withInstructionCallLock(this.db, `${this.binding.companyId}:${this.binding.runId}:${key}`, async () => { + // A filesystem rename cannot roll back with PostgreSQL. Commit proof of + // the attempt *before* permitting that effect. If the effect transaction + // or its acknowledgement is lost, a missing result is unknown, not a + // license to run the instruction write a second time. + const reserved = await this.db.transaction(async (tx) => { + const context = await this.#lockAuthorizedMutationContext(tx as unknown as Db); + const bound = await authorize(tx as unknown as Db); + const resultJson = record(context.run.resultJson); + if (record(resultJson.semanticToolReceipts)[key] !== undefined) return false; + const attempts = record(resultJson.instructionToolAttempts); + const prior = record(attempts[key]); + if (attempts[key] !== undefined) { + if (prior.operationId !== tool || prior.inputDigest !== digest) { + throw new Error("paperclip_runner_tool_idempotency_conflict"); + } + const failure = record(prior.failure); + if (typeof failure.status === "number" && instructionPreWriteFailureStatuses.has(failure.status) + && typeof failure.message === "string") { + throw new HttpError(failure.status, failure.message, failure.details); + } + throw new SemanticToolOutcomeUnknownError(`paperclip_runner_instruction_outcome_unknown call_id=${callId} operation_id=${tool} input_digest=${digest}`); + } + if (Object.keys(attempts).length >= 512) throw badRequest("Run instruction mutation limit reached"); + attempts[key] = { operationId: tool, inputDigest: digest, targetAgentId }; + await tx.update(heartbeatRuns).set({ resultJson: { ...resultJson, instructionToolAttempts: attempts } }) + .where(eq(heartbeatRuns.id, this.binding.runId)); + await tx.insert(activityLog).values({ + companyId: this.binding.companyId, actorType: "agent", actorId: this.binding.agentId, + agentId: this.binding.agentId, runId: this.binding.runId, + responsibleUserId: bound.type === "agent" ? bound.onBehalfOfUserId : null, + action: "agent.instruction_write_attempted", entityType: "agent", entityId: targetAgentId, + details: { callId, operationId: tool, inputDigest: digest }, + }); + return true; + }); + try { + return await this.#withMutationReceipt(tool, key, input, (tx) => + executeAgentInstructionTool({ db: tx, binding: { + companyId: this.binding.companyId, agentId: this.binding.agentId, runId: this.binding.runId, + }, tool, arguments: input }), { beforeReceiptReplay: async (tx) => { await authorize(tx); } }); + } catch (error) { + // Instruction validation, authorization and CAS errors precede the + // file write. Storage errors and lost commit acknowledgements do not + // prove that a reserved filesystem effect rolled back. + if (!reserved) throw error; + if (error instanceof HttpError && instructionPreWriteFailureStatuses.has(error.status)) { + const failure = { status: error.status, message: error.message, ...(error.details === undefined ? {} : { details: error.details }) }; + try { + // Save the definite failure before exposing it as a final answer. + // This is evidence of the admitted call, not another mutation, so + // retain it even if the run stopped after the effect rolled back. + await this.db.transaction(async (tx) => { + const [run] = await tx.select().from(heartbeatRuns).where(and( + eq(heartbeatRuns.id, this.binding.runId), eq(heartbeatRuns.companyId, this.binding.companyId), + eq(heartbeatRuns.agentId, this.binding.agentId), + )).for("update"); + if (!run) throw new Error("instruction_failure_run_missing"); + const resultJson = record(run.resultJson); + const attempts = record(resultJson.instructionToolAttempts); + const prior = record(attempts[key]); + if (prior.operationId !== tool || prior.inputDigest !== digest + || record(resultJson.semanticToolReceipts)[key] !== undefined) { + throw new Error("instruction_failure_receipt_conflict"); + } + attempts[key] = { ...prior, failure }; + await tx.update(heartbeatRuns).set({ resultJson: { ...resultJson, instructionToolAttempts: attempts } }) + .where(eq(heartbeatRuns.id, this.binding.runId)); + }); + } catch (persistenceError) { + throw new SemanticToolOutcomeUnknownError( + `paperclip_runner_instruction_failure_receipt_unavailable call_id=${callId}`, + { cause: new AggregateError([error, persistenceError], "Definite instruction failure could not be saved") }, + ); + } + throw error; + } + throw new SemanticToolOutcomeUnknownError( + `paperclip_runner_instruction_outcome_unknown call_id=${callId}: ${error instanceof Error ? error.message : String(error)}`, + { cause: error }, + ); + } + }); + } + async #withMutationReceipt( operationId: string, idempotencyKey: string, diff --git a/server/src/services/run-log-store.test.ts b/server/src/services/run-log-store.test.ts index ccadb78c30f..75b7efda358 100644 --- a/server/src/services/run-log-store.test.ts +++ b/server/src/services/run-log-store.test.ts @@ -85,6 +85,96 @@ describe("createDurableRunLogStore", () => { expect(calls.get).toBe(0); // local file present -> no S3 read }); + it.each([false, true])("retains every attempt across retry (pod restart: %s)", async (restart) => { + const { provider, objects } = createMemoryProvider(); + let store = createDurableRunLogStore({ basePath: baseDir, s3: { provider } }); + const attempts: string[] = []; + for (let attempt = 1; attempt <= 3; attempt++) { + if (restart && attempt > 1) { + await fs.rm(baseDir, { recursive: true, force: true }); + store = createDurableRunLogStore({ basePath: baseDir, s3: { provider } }); + } + const handle = await store.begin(begin); + attempts.push(handle.attemptId!); + await store.append(handle, { stream: "stderr", chunk: `attempt-${attempt}`, ts: "same-time", seq: attempt }); + const summary = await store.finalize(handle); + const saved = objects.get(handle.logRef)!; + const lines = saved.toString().trim().split("\n").map((line) => JSON.parse(line)); + expect(lines.map((line) => line.chunk)).toEqual(Array.from({length: attempt}, (_, i) => `attempt-${i + 1}`)); + expect(lines.map((line) => line.attemptId)).toEqual(attempts); + expect(new Set(attempts).size).toBe(attempt); + expect(summary.bytes).toBe(saved.length); + expect(summary.sha256).toBe(createHash("sha256").update(saved).digest("hex")); + } + }); + + it("never replaces concurrent appends when two stores restore the same missing log", async () => { + const { provider, objects } = createMemoryProvider(); + const firstStore = createDurableRunLogStore({ basePath: baseDir, s3: { provider } }); + const secondStore = createDurableRunLogStore({ basePath: baseDir, s3: { provider } }); + const original = await firstStore.begin(begin); + await firstStore.append(original, { stream: "stderr", chunk: "original", ts: "t0" }); + await firstStore.finalize(original); + const prefix = Buffer.from(objects.get(original.logRef)!); + await fs.rm(baseDir, { recursive: true, force: true }); + let reached!: () => void, release!: () => void; + const restoring = new Promise((resolve) => { reached = resolve; }); + const barrier = new Promise((resolve) => { release = resolve; }); + const spy = vi.spyOn(provider, "getObject").mockImplementationOnce(async () => ({ + stream: Readable.from((async function* () { + yield prefix; + reached(); + await barrier; + })()), + contentLength: prefix.length, + })); + const slow = firstStore.begin(begin); + try { + await restoring; + const fast = await secondStore.begin(begin); + await secondStore.append(fast, { stream: "stderr", chunk: "fast attempt", ts: "t1" }); + release(); + const late = await slow; + await firstStore.append(late, { stream: "stderr", chunk: "slow attempt", ts: "t2" }); + await firstStore.finalize(late); + const saved = objects.get(original.logRef)!; + expect(saved).toEqual(await fs.readFile(path.join(baseDir, original.logRef))); + expect(saved.toString().trim().split("\n").map((line) => JSON.parse(line).chunk)) + .toEqual(["original", "fast attempt", "slow attempt"]); + } finally { + release(); + await slow.catch(() => undefined); + spy.mockRestore(); + } + }); + + it.each(["stream_failure", "short_read"])("preserves the durable prefix when restore fails (%s)", async (fault) => { + const { provider, objects, calls } = createMemoryProvider(); + const store = createDurableRunLogStore({ basePath: baseDir, s3: { provider } }); + const handle = await store.begin(begin); + await store.append(handle, { stream: "stderr", chunk: "original failure evidence", ts: "t1" }); + await store.finalize(handle); + const prefix = Buffer.from(objects.get(handle.logRef)!); + await fs.rm(baseDir, { recursive: true, force: true }); + const spy = vi.spyOn(provider, "getObject").mockResolvedValueOnce({ + stream: Readable.from((async function* () { + yield prefix.subarray(0, 10); + if (fault === "stream_failure") throw new Error("storage connection lost"); + })()), + contentLength: prefix.length, + }); + try { + await expect(store.begin(begin)).rejects.toThrow( + fault === "stream_failure" ? "storage connection lost" : "incomplete prefix", + ); + await expect(fs.stat(path.join(baseDir, handle.logRef))).rejects.toMatchObject({ code: "ENOENT" }); + expect(objects.get(handle.logRef)).toEqual(prefix); + expect(calls.put).toBe(1); + const next = await store.begin(begin); + expect(await fs.readFile(path.join(baseDir, next.logRef))).toEqual(prefix); + } finally { spy.mockRestore(); } + }); + it("uploads the complete log to S3 on finalize", async () => { const { provider, objects, calls } = createMemoryProvider(); const store = createDurableRunLogStore({ basePath: baseDir, s3: { provider, keyPrefix: "run-logs" } }); @@ -133,7 +223,7 @@ describe("createDurableRunLogStore", () => { expect(summary.sha256).toBe(createHash("sha256").update(local).digest("hex")); expect(objects.get(handle.logRef)).toEqual(local); expect(local.toString()).toBe(outcome === "fails" ? "" : JSON.stringify({ - ts: "t1", stream: "stderr", chunk: "late diagnostic", + ts: "t1", attemptId: handle.attemptId, stream: "stderr", chunk: "late diagnostic", }) + "\n"); } finally { release(); diff --git a/server/src/services/run-log-store.ts b/server/src/services/run-log-store.ts index 7e7b4597eae..85a51ba1e0b 100644 --- a/server/src/services/run-log-store.ts +++ b/server/src/services/run-log-store.ts @@ -1,6 +1,7 @@ -import { createReadStream, promises as fs } from "node:fs"; +import { createReadStream, createWriteStream, promises as fs } from "node:fs"; import path from "node:path"; -import { createHash } from "node:crypto"; +import { pipeline } from "node:stream/promises"; +import { createHash, randomUUID } from "node:crypto"; import { addAbortSignal } from "node:stream"; import { notFound } from "../errors.js"; import { resolvePaperclipInstanceRoot } from "../home-paths.js"; @@ -12,6 +13,7 @@ export type RunLogStoreType = "local_file"; export interface RunLogHandle { store: RunLogStoreType; logRef: string; + attemptId?: string; } export interface RunLogReadOptions { @@ -295,9 +297,37 @@ export function createDurableRunLogStore(options: DurableRunLogStoreOptions): Ru const relPath = path.join(relDir, `${runId}.ndjson`); await ensureDir(relDir); const absPath = resolveWithin(basePath, relPath); - await fs.writeFile(absPath, "", "utf8"); await retireInflightMirror(relPath); - return { store: "local_file", logRef: relPath }; + // Retries share a run ID. Never truncate their earlier evidence. On a + // new pod, restore the durable prefix before opening another attempt. + const exists = await fs.stat(absPath).then(() => true, (error) => { + if (error.code === "ENOENT") return false; + throw error; + }); + if (!exists && s3) { + const key = s3Key(relPath); + const head = await s3.provider.headObject({ objectKey: key }); + if (head.exists) { + const temporary = `${absPath}.${randomUUID()}.restore`; + try { + const object = await s3.provider.getObject({ objectKey: key }); + await pipeline(object.stream, createWriteStream(temporary, { flags: "wx", mode: 0o600 })); + if (typeof head.contentLength === "number" && (await fs.stat(temporary)).size !== head.contentLength) { + throw new Error("Durable run log restore returned an incomplete prefix"); + } + // Publish the complete prefix without replacing a file another + // attempt has already restored and started appending to. + await fs.link(temporary, absPath).catch((error: NodeJS.ErrnoException) => { + if (error.code !== "EEXIST") throw error; + }); + } finally { + await fs.rm(temporary, { force: true }); + } + } + } + const file = await fs.open(absPath, "a", 0o600); + await file.close(); + return { store: "local_file", logRef: relPath, attemptId: randomUUID() }; }, async append(handle, event) { @@ -305,6 +335,7 @@ export function createDurableRunLogStore(options: DurableRunLogStoreOptions): Ru const absPath = resolveWithin(basePath, handle.logRef); const line = JSON.stringify({ ts: event.ts, + ...(handle.attemptId ? { attemptId: handle.attemptId } : {}), stream: event.stream, chunk: event.chunk, // Monotonic per-run sequence so readers can dedupe and order records diff --git a/server/src/vendor/paperclip-runner/index.ts b/server/src/vendor/paperclip-runner/index.ts index 4308fa67ff7..4142f8fbf4d 100644 --- a/server/src/vendor/paperclip-runner/index.ts +++ b/server/src/vendor/paperclip-runner/index.ts @@ -78,6 +78,7 @@ export const NativeSessionCleanupQuarantinedError = runner.NativeSessionCleanupQuarantinedError; export const NativeSessionProtocolIntegrityError = runner.NativeSessionProtocolIntegrityError; +export const SemanticToolOutcomeUnknownError = runner.SemanticToolOutcomeUnknownError; export const PaperclipSemanticDispatcher = runner.PaperclipSemanticDispatcher; export const CAPABILITY_SEMANTIC_TOOL_CATALOG = runner.CAPABILITY_SEMANTIC_TOOL_CATALOG;