diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index bf481923610..f18aa7128a7 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1258,6 +1258,13 @@ checks coordinator ownership in its terminal update, so a result recorded after the process check is also protected. Terminal task status remains authoritative; exhausted finalization retries do not keep an otherwise orphaned run alive. +A native failure that schedules a same-run resume marks the run `failed` but +keeps the task's execution and checkout locks for that resume. The sweep leaves +those locks in place and clears them only when the task is done or cancelled. +If a scheduled resume finds its execution lock free, restart recovery reclaims +it, as the local resume claim does, unless another run for the task is queued, +running, or scheduled. A lock held by another run still blocks the resume. + Automatic productivity reviews are retired. Run counts, missing comments, and elapsed task time do not create review tasks or impose continuation holds. Bounded continuation, provider recovery, budget limits, explicit blockers, and normal review/approval stages remain in force. Existing productivity-review tasks, comments, assignments, and dependencies remain unchanged and readable; their historical origins still identify them as recovery work for recursion suppression. ### Issue-thread interaction resolution diff --git a/packages/paperclip-runner/docs/durable-recovery.md b/packages/paperclip-runner/docs/durable-recovery.md index 3205943c454..d1e0be1a1af 100644 --- a/packages/paperclip-runner/docs/durable-recovery.md +++ b/packages/paperclip-runner/docs/durable-recovery.md @@ -70,6 +70,35 @@ a scripted harness, removes the controller's launch files, then recovers the same process and provider session and completes another turn. This exercises the real transport; it does not qualify a hosted OpenCode or ACPX deployment. +## Settled ACPX sessions across a Runner upgrade + +Saved ACPX provider state records the digest of the launch profile that wrote +it: the provider-pack authority, the sidecar command and arguments, each launch +artifact's SHA-256, and any candidate admission. A deploy that ships a new +provider pack or sidecar build changes the digest. The state stores only the +digest, so restore cannot tell which part changed. + +Restore re-binds a saved session to the current launch profile, and keeps the +conversation, only when all of these are true: + +- The session is suspended. It has no active turn, pending runtime request, + undelivered provider event, or unconfirmed provider exit. +- The startup has no candidate admission. +- The saved descriptor names this startup's sidecar command and arguments. + +Every other binding stays exact. The saved descriptor must match this build's +qualified agent profile (ACPX version, agent server package and version, and +command digest). Session start verifies each artifact and the restored provider +identity. Candidate grants and run bindings keep their own checks. + +Any other launch-profile mismatch, and a saved descriptor that names a release +this build does not qualify, fails restore with the closed command-result +classification `provider_state_incompatible`. The controller reads the classification from the +authenticated result, not from the message. If the run allows a replacement +after a failed resume, it opens a fresh provider session as before. Otherwise the +run fails with `native_provider_state_incompatible` and is not retried. A board +Retry then continues the task in a fresh provider session in the same workspace. + ## Profile identity Every ACPX provider's `commandDigest` attests only its own upstream release. diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs index 1e445b261d1..bcc1625dc88 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs @@ -26,7 +26,7 @@ use crate::durable::QualifiedLaunchArtifact; use crate::durable::{ create_private_temporary_file, open_private_regular_file, verify_private_directory, AcpxLaunchProfile, Command, CommandExecution, CommandExecutor, DurableRunnerConfig, - DurableRunnerError, EventPriority, PolledEvent, + DurableRunnerError, EventPriority, PolledEvent, PROVIDER_STATE_INCOMPATIBLE, }; use crate::generated_acpx_sidecar_contract::GeneratedAcpxSidecarCommand; use crate::process_supervisor::{VerifiedProcessArgument, VerifiedProcessLaunch}; @@ -279,6 +279,19 @@ fn only_registered_asset_text_changed( } impl AcpxProviderDescriptor { + // Whether this build qualifies the release that the descriptor names. + fn names_qualified_release(&self) -> bool { + acpx_release_profile(&self.agent).is_some_and(|expected| { + self.provider_version == QUALIFIED_ACPX_VERSION + && self.acpx_version == QUALIFIED_ACPX_VERSION + && self.agent_server_package == expected.agent_server_package + && self.agent_server_version == expected.agent_server_version + && self.agent_runtime_package.as_deref() == expected.agent_runtime_package + && self.agent_runtime_version.as_deref() == expected.agent_runtime_version + && self.command_digest == expected.command_digest + }) + } + fn validate_session( &self, context: &AcpxEventProjectionContext, @@ -622,11 +635,21 @@ impl AcpxDurableState { // so its persisted descriptor legitimately carries the prior run ID // until run.attach rotates authority. Fresh command descriptors still // use validate(), which binds them to the current run. - self.descriptor.validate_session(context)?; + // A deploy that changes the agent's qualified release also leaves + // saved state that this build cannot restore. Keep the same check and + // only name its class. + self.descriptor.validate_session(context).map_err(|error| { + if self.descriptor.names_qualified_release() { + error + } else { + error.with_classification(PROVIDER_STATE_INCOMPATIBLE) + } + })?; if self.launch_profile_digest != expected_launch_profile_digest { return Err(DurableRunnerError::invalid( "ACPX durable launch profile digest does not match runner startup", - )); + ) + .with_classification(PROVIDER_STATE_INCOMPATIBLE)); } validate_pending_runtime_requests(&self.pending_runtime_requests)?; let mut ids = HashSet::new(); @@ -960,6 +983,34 @@ impl AcpxCommandExecutor { prior.canonical_digest() } + // A deploy changes the provider-pack authority and the sidecar and Node + // artifact digests, so a saved digest from the previous build never + // matches. The digest is a hash, so restore cannot tell which part + // changed. Re-bind only when no provider work is in flight: + // - The session is suspended. No turn, request, undelivered event, or + // unconfirmed provider exit remains. Nothing from the old provider + // process continues; the next launch only loads the conversation. + // - The startup is ordinary. Candidate admission carries company, model + // and expiry authority in the digest, so candidate sessions stay + // exactly bound. + // - The saved descriptor names this startup's sidecar command and + // arguments. Only the qualified content at those paths changed. + // validate() still binds the descriptor to this build's qualified agent + // profile (ACPX, agent server and command digest), and session start + // still verifies every artifact and the restored provider identity. + fn rebinds_settled_session(&self, state: &AcpxDurableState) -> bool { + self.launch_profile.as_ref().is_some_and(|profile| { + profile.candidate_admission.is_none() + && state.descriptor.sidecar_command == profile.command + && state.descriptor.sidecar_args == profile.args + }) && state.lifecycle == "suspended" + && state.identity.is_some() + && state.active_turn_id.is_none() + && !state.provider_exit_unconfirmed + && state.pending_events.is_empty() + && state.pending_runtime_requests.is_empty() + } + fn restore(&mut self) -> Result<(), DurableRunnerError> { self.ensure_persistence_healthy()?; if self.restore_checked { @@ -1016,10 +1067,15 @@ impl AcpxCommandExecutor { file.read_to_end(&mut bytes).map_err(|error| { DurableRunnerError::invalid(format!("failed to read ACPX provider state: {error}")) })?; - let state: AcpxDurableState = serde_json::from_slice(&bytes).map_err(|error| { + let mut state: AcpxDurableState = serde_json::from_slice(&bytes).map_err(|error| { DurableRunnerError::invalid(format!("ACPX provider state is malformed: {error}")) })?; let launch_profile_digest = self.restore_launch_profile_digest(&state)?; + if state.launch_profile_digest != launch_profile_digest + && self.rebinds_settled_session(&state) + { + state.launch_profile_digest = launch_profile_digest.clone(); + } state.validate(&self.context, &launch_profile_digest)?; self.state = Some(state); self.restore_session_if_needed() @@ -5298,14 +5354,11 @@ mod tests { "grant_run", "injected_candidate", "context_run", - "authority", "command", "args", - "artifact", "missing_profile", - "saved_digest", ] { - let (directory, mut config, payload, before, sentinel, candidate) = + let (directory, mut config, payload, _, sentinel, candidate) = kimi_cold_restore_fixture("kimi-released-cold-reject"); assert!(config .acpx_launch_profile @@ -5316,26 +5369,11 @@ mod tests { match field { "context_run" => config.run_id = "foreign-run".into(), "missing_profile" => config.acpx_launch_profile = None, - "saved_digest" => { - let mut state: AcpxDurableState = serde_json::from_slice(&before).unwrap(); - state.launch_profile_digest = format!("sha256:{}", "0".repeat(64)); - fs::write( - directory.join("acpx-provider-state.json"), - serde_json::to_vec(&state).unwrap(), - ) - .unwrap(); - } _ => { let profile = config.acpx_launch_profile.as_mut().unwrap(); match field { - "authority" => { - profile.authority_digest = format!("sha256:{}", "0".repeat(64)) - } "command" => profile.command = directory.join("foreign-command"), "args" => profile.args.push("foreign-argument".into()), - "artifact" => { - profile.artifacts[0].sha256 = format!("sha256:{}", "0".repeat(64)) - } _ => { let mut grant = candidate; match field { @@ -5382,6 +5420,60 @@ mod tests { } } + #[test] + #[cfg(any( + all(target_os = "macos", target_arch = "aarch64"), + all(target_os = "linux", target_arch = "x86_64") + ))] + fn released_kimi_cold_restore_rebinds_a_settled_session_to_new_launch_content() { + // An ordinary startup re-binds a settled session after a provider-pack + // or artifact change at the same launch paths. The checks above for + // candidate grants, command, arguments and run binding still apply. + for field in ["authority", "artifact", "saved_digest"] { + let (directory, mut config, payload, before, sentinel, _) = + kimi_cold_restore_fixture("kimi-released-cold-rebind"); + let profile = config.acpx_launch_profile.as_mut().unwrap(); + match field { + "authority" => profile.authority_digest = format!("sha256:{}", "0".repeat(64)), + "artifact" => { + let command = profile.command.clone(); + write_artifact(&command, b"#!/bin/sh\nexit 2\n", true); + profile.artifacts[0] = artifact(&command); + } + "saved_digest" => { + let mut state: AcpxDurableState = serde_json::from_slice(&before).unwrap(); + state.launch_profile_digest = format!("sha256:{}", "0".repeat(64)); + fs::write( + directory.join(ACPX_PROVIDER_STATE_FILE), + serde_json::to_vec(&state).unwrap(), + ) + .unwrap(); + } + _ => unreachable!(), + } + config.validate().unwrap(); + let mut recovered = AcpxCommandExecutor::with_runner_config(&directory, &config); + recovered.restore().expect(field); + recovered.attach_run(&payload).expect(field); + let saved: AcpxDurableState = + serde_json::from_slice(&fs::read(recovered.state_path()).unwrap()).unwrap(); + assert_eq!(saved.descriptor.run_id, "run-2", "{field}"); + assert_eq!( + saved.launch_profile_digest, + config + .acpx_launch_profile + .as_ref() + .unwrap() + .canonical_digest() + .unwrap(), + "{field}" + ); + assert!(recovered.session.is_none(), "{field}"); + assert!(!sentinel.exists(), "{field}"); + fs::remove_dir_all(directory).unwrap(); + } + } + #[test] #[cfg(any( all(target_os = "macos", target_arch = "aarch64"), @@ -5498,4 +5590,217 @@ mod tests { assert!(!sentinel.exists()); fs::remove_dir_all(directory).unwrap(); } + + // A settled Claude session saved by one runner build. The next deploy + // ships a new sidecar build and provider-pack authority at the same + // qualified launch paths, and the next run cold-attaches in a new runner. + #[cfg(unix)] + fn settled_session_after_runner_upgrade( + label: &str, + ) -> (PathBuf, DurableRunnerConfig, Value, Vec) { + let directory = temporary_directory(label); + let command = directory.join("node"); + let sidecar = directory.join("acpx-runtime-sidecar.cjs"); + write_artifact(&command, b"#!/bin/sh\nexit 1\n", true); + write_artifact(&sidecar, b"// sidecar build 1\n", false); + let profile = AcpxLaunchProfile { + candidate_admission: None, + authority_digest: format!("sha256:{}", "d".repeat(64)), + command: command.clone(), + args: vec![sidecar.to_string_lossy().into_owned()], + artifacts: vec![artifact(&command), artifact(&sidecar)], + }; + let runtime = directory.join("runtime"); + let workspace = directory.join("workspace"); + fs::create_dir_all(&runtime).unwrap(); + fs::create_dir_all(&workspace).unwrap(); + fs::set_permissions(&runtime, fs::Permissions::from_mode(0o700)).unwrap(); + fs::set_permissions(&workspace, fs::Permissions::from_mode(0o700)).unwrap(); + let mut value = descriptor("claude"); + value["runtimeDirectory"] = json!(runtime); + value["cwd"] = json!(workspace); + value["sidecarCommand"] = json!(command); + value["sidecarArgs"] = json!([sidecar]); + let desc: AcpxProviderDescriptor = serde_json::from_value(value.clone()).unwrap(); + let identity = AcpxProviderSessionIdentity { + kind: "acpx".to_owned(), + normalized_session_id: "session-1".to_owned(), + acpx_record_id: "record-1".to_owned(), + backend_session_id: "backend-1".to_owned(), + agent_session_id: "agent-1".to_owned(), + profile_digest: desc.command_digest.clone(), + workspace_digest: format!("sha256:{}", "a".repeat(64)), + requested_model: desc.model.clone(), + effective_model: desc.model.clone(), + permission_mode: Some(desc.permission_mode), + mode: None, + pi_thinking_level: None, + claude_effort: None, + grok_effort: None, + grok_disable_web_search: None, + provider_lifetime_fence_candidates: [60_001, 60_002, 60_003], + }; + let operations = Vec::new(); + let tools = AuthorizedToolSet { + schema: TOOL_SET_SCHEMA.to_owned(), + schema_version: 1, + catalog_digest: authorized_tool_catalog_digest(&operations).unwrap(), + operations, + }; + let mut state = AcpxDurableState::new(desc, tools, profile.canonical_digest().unwrap()); + state.lifecycle = "suspended".to_owned(); + state.identity = Some(identity); + let config = test_config(&directory, Some(profile)); + let mut executor = AcpxCommandExecutor::with_runner_config(&directory, &config); + executor.state = Some(state); + executor.save_state().unwrap(); + let saved = fs::read(executor.state_path()).unwrap(); + + write_artifact(&sidecar, b"// sidecar build 2\n", false); + let mut next_config = config; + next_config.run_id = "run-2".to_owned(); + next_config.turn_id = "turn-2".to_owned(); + next_config.item_id = "item-2".to_owned(); + let next_profile = next_config.acpx_launch_profile.as_mut().unwrap(); + next_profile.authority_digest = format!("sha256:{}", "e".repeat(64)); + next_profile.artifacts = vec![artifact(&command), artifact(&sidecar)]; + next_config.validate().unwrap(); + let mut provider = value; + provider["runId"] = json!("run-2"); + ( + directory, + next_config, + json!({ "provider": provider }), + saved, + ) + } + + #[cfg(unix)] + #[test] + fn settled_session_rebinds_to_the_upgraded_runner_launch_profile() { + let (directory, config, payload, saved) = + settled_session_after_runner_upgrade("upgrade-rebind"); + let prior: AcpxDurableState = serde_json::from_slice(&saved).unwrap(); + let current_digest = config + .acpx_launch_profile + .as_ref() + .unwrap() + .canonical_digest() + .unwrap(); + assert_ne!(prior.launch_profile_digest, current_digest); + + let mut recovered = AcpxCommandExecutor::with_runner_config(&directory, &config); + recovered + .restore() + .expect("a settled session survives a new qualified sidecar build"); + assert!(recovered.session.is_none()); + assert!(recovered.poll_events().unwrap().is_empty()); + assert_eq!(fs::read(recovered.state_path()).unwrap(), saved); + recovered + .attach_run(&payload) + .expect("the next run cold-attaches the same conversation"); + assert!(recovered.session.is_none()); + + let attached: AcpxDurableState = + serde_json::from_slice(&fs::read(recovered.state_path()).unwrap()).unwrap(); + assert_eq!(attached.launch_profile_digest, current_digest); + assert_eq!(attached.descriptor.run_id, "run-2"); + assert_eq!(attached.lifecycle, "suspended"); + assert_eq!(attached.identity, prior.identity); + assert_eq!( + attached.descriptor.command_digest, + prior.descriptor.command_digest + ); + fs::remove_dir_all(directory).unwrap(); + } + + #[cfg(unix)] + #[test] + fn unrestorable_saved_sessions_keep_strict_bindings_and_name_their_class() { + for field in [ + "active_turn", + "pending_request", + "pending_event", + "session_open", + "exit_unconfirmed", + "no_identity", + "command_moved", + "args_changed", + "agent_server_release", + "acpx_release", + ] { + let (directory, mut config, _, saved) = + settled_session_after_runner_upgrade("upgrade-strict"); + let state_path = directory.join(ACPX_PROVIDER_STATE_FILE); + let mut state: AcpxDurableState = serde_json::from_slice(&saved).unwrap(); + let profile = config.acpx_launch_profile.as_mut().unwrap(); + match field { + "active_turn" => { + state.lifecycle = "turn_active".into(); + state.active_turn_id = Some("turn-1".into()); + } + "pending_request" => { + state.lifecycle = "turn_active".into(); + state.active_turn_id = Some("turn-1".into()); + state + .push(NormalizedProviderEvent { + event_type: "runtime_request.created".into(), + priority: EventPriority::P0, + payload: json!({"request":pending_input_request("input-1")}), + }) + .unwrap(); + state.pending_events.clear(); + } + "pending_event" => state + .push(NormalizedProviderEvent { + event_type: "session.updated".into(), + priority: EventPriority::P1, + payload: json!({}), + }) + .unwrap(), + "session_open" => state.lifecycle = "session_open".into(), + "exit_unconfirmed" => { + state.lifecycle = "closed".into(); + state.provider_exit_unconfirmed = true; + } + "no_identity" => { + state.lifecycle = "prepared".into(); + state.identity = None; + } + "command_moved" => { + let moved = directory.join("relocated-node"); + write_artifact(&moved, b"#!/bin/sh\nexit 1\n", true); + profile.command = moved.clone(); + profile.artifacts[0] = artifact(&moved); + } + "args_changed" => profile.args.push("--changed".into()), + "agent_server_release" => state.descriptor.agent_server_version = "0.0.1".into(), + "acpx_release" => { + state.descriptor.acpx_version = "0.0.1".into(); + state.descriptor.provider_version = "0.0.1".into(); + } + _ => unreachable!(), + } + config.validate().unwrap(); + fs::write(&state_path, serde_json::to_vec(&state).unwrap()).unwrap(); + let retained = fs::read(&state_path).unwrap(); + + let mut recovered = AcpxCommandExecutor::with_runner_config(&directory, &config); + let error = recovered.restore().unwrap_err(); + let expected = if field.ends_with("_release") { + "does not match a qualified immutable profile" + } else { + "launch profile digest does not match runner startup" + }; + assert!(error.to_string().contains(expected), "{field}: {error}"); + assert_eq!( + error.classification(), + Some(crate::durable::PROVIDER_STATE_INCOMPATIBLE), + "{field}" + ); + assert!(recovered.session.is_none(), "{field}"); + assert_eq!(fs::read(&state_path).unwrap(), retained, "{field}"); + fs::remove_dir_all(directory).unwrap(); + } + } } diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/durable/mod.rs b/packages/paperclip-runner/runner/crates/runner-core/src/durable/mod.rs index fc33db5cc3c..4b026797f66 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/durable/mod.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/durable/mod.rs @@ -37,12 +37,17 @@ pub fn redact_diagnostic_text(input: &str) -> String { state::redact_text(input) } +/// Saved provider state was written under a launch or provider profile that +/// this runner cannot adopt. The outcome is deterministic for this runner +/// build: retrying the same restore cannot succeed. +pub const PROVIDER_STATE_INCOMPATIBLE: &str = "provider_state_incompatible"; + #[derive(Clone, Debug, PartialEq, Eq)] -pub struct DurableRunnerError(String, Option); +pub struct DurableRunnerError(String, Option, Option<&'static str>); impl DurableRunnerError { pub fn invalid(message: impl Into) -> Self { - Self(message.into(), None) + Self(message.into(), None, None) } pub fn with_provider_failure(mut self, value: Option) -> Self { @@ -53,6 +58,17 @@ impl DurableRunnerError { pub fn provider_failure(&self) -> Option { self.1.clone() } + + /// Adds a closed failure class that the controller reads from the command + /// result instead of parsing the message. + pub fn with_classification(mut self, classification: &'static str) -> Self { + self.2 = Some(classification); + self + } + + pub fn classification(&self) -> Option<&'static str> { + self.2 + } } impl Display for DurableRunnerError { diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs b/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs index ddfd4034c94..97bf9644745 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs @@ -1930,6 +1930,9 @@ fn process_command( "code": "command_execution_failed", "message": message, }); + if let Some(classification) = error.classification() { + failure["classification"] = json!(classification); + } if let Some(provider_failure) = error.provider_failure() { failure["providerFailure"] = provider_failure; } @@ -2489,6 +2492,17 @@ mod tests { } } + struct IncompatibleStateExecutor; + + impl CommandExecutor for IncompatibleStateExecutor { + fn execute(&mut self, _command: &Command) -> Result { + Err( + DurableRunnerError::invalid("saved provider state cannot be adopted") + .with_classification(crate::durable::PROVIDER_STATE_INCOMPATIBLE), + ) + } + } + impl CommandExecutor for ShutdownFailingExecutor { fn execute(&mut self, _command: &Command) -> Result { Ok(CommandExecution::result(json!({"status": "completed"}))) @@ -4865,6 +4879,46 @@ mod tests { .diagnostics .iter() .all(|diagnostic| !diagnostic.contains("test-secret"))); + assert!(failed.result.get("classification").is_none()); + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn classified_executor_failure_keeps_its_closed_class_in_the_durable_result() { + let directory = std::env::temp_dir().join(format!( + "paperclip-runner-command-classified-failure-{}", + std::process::id() + )); + let _ = fs::remove_dir_all(&directory); + let config = config(directory.clone()); + let store = DurableStateStore::new(&directory).unwrap(); + let (mut state, _) = store.load_or_create(&config).unwrap(); + let command = command("run.attach"); + let (failed, _) = process_command( + &mut state, + &store, + &config, + &mut IncompatibleStateExecutor, + &command, + ) + .unwrap(); + let (mut recovered, _) = store.load_or_create(&config).unwrap(); + let replay = process_command( + &mut recovered, + &store, + &config, + &mut IncompatibleStateExecutor, + &command, + ) + .unwrap() + .0; + assert_eq!(failed.status, "failed"); + assert_eq!(failed.result["code"], "command_execution_failed"); + assert_eq!( + failed.result["classification"], + "provider_state_incompatible" + ); + assert_eq!(failed, replay); fs::remove_dir_all(directory).unwrap(); } diff --git a/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts b/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts index 094362e0fa6..6791372425a 100644 --- a/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts +++ b/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it, vi } from "vitest"; import type { HarnessDriver, HarnessSession, PersistedHarnessSession } from "../contracts/harness-driver.js"; +import { PROVIDER_STATE_INCOMPATIBLE, runnerRecoveryFailureClass } from "../contracts/harness-driver.js"; import type { PrpEvent, PrpStructuredRunResult, PrpTerminalState } from "../protocol/replay-contract.js"; import { NativeSessionProtocolIntegrityError } from "../contracts/native-session-backend.js"; import { HarnessDriverBackend } from "./harness-driver-backend.js"; @@ -323,6 +324,36 @@ describe("HarnessDriverBackend", () => { expect(recoveredDispositionAllowance).toBe(true); }); + it("keeps the runner's closed class of a failed recovery", async () => { + const commandFailure = Object.defineProperty(new Error("PRP command run.attach failed"), "runnerCommandResult", { + value: { status: "failed", commandType: "run.attach", + result: { code: "command_execution_failed", message: "saved launch profile changed", + classification: PROVIDER_STATE_INCOMPATIBLE } }, + enumerable: false, + }); + expect(runnerRecoveryFailureClass(commandFailure)).toBe(PROVIDER_STATE_INCOMPATIBLE); + expect(runnerRecoveryFailureClass(new Error("wrapped", { cause: commandFailure }))).toBe(PROVIDER_STATE_INCOMPATIBLE); + expect(runnerRecoveryFailureClass(new Error(`text only: ${PROVIDER_STATE_INCOMPATIBLE}`))).toBeUndefined(); + const unclassified = Object.defineProperty(new Error("PRP command run.attach failed"), "runnerCommandResult", { + value: { status: "failed", result: { code: "command_execution_failed", classification: "other" } }, + enumerable: false, + }); + expect(runnerRecoveryFailureClass(unclassified)).toBeUndefined(); + + const backend = new HarnessDriverBackend({ + ...driver, + recoverSession: async () => ({ recovered: false, reason: String(commandFailure), + failureClass: runnerRecoveryFailureClass(commandFailure) }), + }); + await expect(backend.recoverSession({ + backendKind: "runner", driverKind: "fake", sessionId: "driver-1", providerSessionId: "provider-1", + providerIdentity, + identity: { runId: "run-2", sessionId: "session-1", companyId: "company-1", issueId: "issue-1", agentId: "agent-1" }, + }, { signal: new AbortController().signal })).resolves.toEqual({ + recovered: false, reason: "Error: PRP command run.attach failed", failureClass: PROVIDER_STATE_INCOMPATIBLE, + }); + }); + it("preserves a consumed disposition allowance in native snapshots", async () => { class ConsumedRecoverySession extends FakeHarnessSession { override async snapshot(): Promise { diff --git a/packages/paperclip-runner/src/backends/harness-driver-backend.ts b/packages/paperclip-runner/src/backends/harness-driver-backend.ts index 1339830d047..56caad7ecab 100644 --- a/packages/paperclip-runner/src/backends/harness-driver-backend.ts +++ b/packages/paperclip-runner/src/backends/harness-driver-backend.ts @@ -162,7 +162,11 @@ export class HarnessDriverBackend implements NativeSessionBackend { recoveryOptions, ); if (!recovered.recovered || recovered.session === undefined) { - return { recovered: false, reason: recovered.reason }; + return { + recovered: false, + reason: recovered.reason, + ...(recovered.failureClass ? { failureClass: recovered.failureClass } : {}), + }; } try { assertProviderSessionIdentity( diff --git a/packages/paperclip-runner/src/contracts/harness-driver.ts b/packages/paperclip-runner/src/contracts/harness-driver.ts index ede129c07b7..adccad8d49b 100644 --- a/packages/paperclip-runner/src/contracts/harness-driver.ts +++ b/packages/paperclip-runner/src/contracts/harness-driver.ts @@ -496,10 +496,29 @@ export interface PersistedHarnessSession { | "allow_replacement_after_resume_failure"; } +/** Runner proof that saved provider state cannot be restored by this runner + * build. Restoring the same state again fails the same way. */ +export const PROVIDER_STATE_INCOMPATIBLE = "provider_state_incompatible"; + export interface HarnessSessionRecoveryResult { recovered: boolean; session?: HarnessSession; reason?: string; + /** Closed runner class of a deterministic recovery failure. */ + failureClass?: typeof PROVIDER_STATE_INCOMPATIBLE; +} + +/** Reads the closed class of an authenticated runner command failure. */ +export function runnerRecoveryFailureClass( + error: unknown, + depth = 0, +): typeof PROVIDER_STATE_INCOMPATIBLE | undefined { + if (!error || typeof error !== "object" || depth > 5) return undefined; + const value = error as Record; + const result = value.runnerCommandResult as Record | undefined; + const detail = result?.result as Record | undefined; + if (detail?.classification === PROVIDER_STATE_INCOMPATIBLE) return PROVIDER_STATE_INCOMPATIBLE; + return runnerRecoveryFailureClass(value.cause, depth + 1); } export interface HarnessSession { diff --git a/packages/paperclip-runner/src/contracts/native-session-backend.ts b/packages/paperclip-runner/src/contracts/native-session-backend.ts index 9761858e790..07b829c0fd5 100644 --- a/packages/paperclip-runner/src/contracts/native-session-backend.ts +++ b/packages/paperclip-runner/src/contracts/native-session-backend.ts @@ -15,6 +15,7 @@ import type { HarnessRuntimeRequestHandoff, HarnessRuntimeRequestResolution, HarnessGoalOperation, + HarnessSessionRecoveryResult, HarnessThreadGoal, HarnessThreadLineageEntry, NativeRuntimeContextCapabilities, @@ -82,6 +83,7 @@ export interface NativeSessionRecoveryResult { recovered: boolean; session?: NativeSession; reason?: string; + failureClass?: HarnessSessionRecoveryResult["failureClass"]; } /** Runner-owned cause: the process was restored, but its active turn was lost. */ @@ -278,6 +280,17 @@ export interface NativeSessionBackend { ): Promise; } +/** Same-session recovery is forbidden here, and the runner proved that the + * saved provider session cannot be restored by its build. Retrying the same + * run cannot succeed. */ +export class NativeProviderStateIncompatibleError extends Error { + readonly code = "native_provider_state_incompatible"; + constructor(reason: string) { + super(`native_session_recovery_failed: ${reason}`); + this.name = "NativeProviderStateIncompatibleError"; + } +} + /** A provider failed terminal is not a missing completion proposal. */ export class NativeProviderTerminalFailure extends Error { readonly code = "native_provider_terminal_failed"; diff --git a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts index 995424acc63..c3ea63d6b03 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts @@ -20,7 +20,7 @@ import type { PersistedHarnessTurnTerminal, } from "../../contracts/harness-driver.js"; import { NativeSessionProtocolIntegrityError, nativeRestartInterruptedTurnId } from "../../contracts/native-session-backend.js"; -import { HarnessReconciliationError } from "../../contracts/harness-driver.js"; +import { HarnessReconciliationError, runnerRecoveryFailureClass } from "../../contracts/harness-driver.js"; import { CODEX_CODEX_PROTOCOL_VERSION, CODEX_SKILLLESS_BASE_INSTRUCTIONS, @@ -677,7 +677,12 @@ export class CodexAppServerDriver implements HarnessDriver { await cancellation.close().catch(() => {}); if (error instanceof NativeSessionProtocolIntegrityError) throw error; if (options.signal.aborted) options.signal.throwIfAborted(); - return { recovered: false, reason: redactCodexDiagnostic(String(error)) }; + const failureClass = runnerRecoveryFailureClass(error); + return { + recovered: false, + reason: redactCodexDiagnostic(String(error)), + ...(failureClass ? { failureClass } : {}), + }; } finally { cancellation.detach(); } diff --git a/packages/paperclip-runner/src/native-session-runtime.test.ts b/packages/paperclip-runner/src/native-session-runtime.test.ts index 11f665119dd..784f95ed6f4 100644 --- a/packages/paperclip-runner/src/native-session-runtime.test.ts +++ b/packages/paperclip-runner/src/native-session-runtime.test.ts @@ -15,10 +15,12 @@ import type { PersistedNativeSession, } from "./contracts/native-session-backend.js"; import { + NativeProviderStateIncompatibleError, NativeSessionCloseUnrecoverableError, NativeSessionCleanupQuarantinedError, NativeSessionProtocolIntegrityError, } from "./contracts/native-session-backend.js"; +import { PROVIDER_STATE_INCOMPATIBLE } from "./contracts/harness-driver.js"; import type { PrpEvent, PrpStructuredRunResult, @@ -5641,6 +5643,40 @@ describe("executeNativeSession recovery", () => { expect(openSession).not.toHaveBeenCalled(); }); + it("types a runner-proven incompatible session when same-session recovery is required", async () => { + for (const providerRecoveryPolicy of ["same_session_only", "allow_replacement_after_governed_wait"] as const) { + const checkpoint: PersistedNativeSession = { + backendKind: "mock", sessionId: "driver-incompatible", identity, + providerSessionId: "provider-incompatible", providerRecoveryPolicy, + cursor: "0", activeTurnId: null, pendingRuntimeRequests: [], lineage: [], + }; + const openRun = vi.fn(async () => undefined); + const openSession = vi.fn(); + const openReplacementSession = vi.fn(); + const backend: NativeSessionBackend = { + descriptor: async () => ({ kind: "mock", name: "incompatible", version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true } }), + openSession, openReplacementSession, + recoverSession: async () => ({ recovered: false, reason: "saved launch profile changed", + failureClass: PROVIDER_STATE_INCOMPATIBLE }), + }; + const port: ControlPlanePort = { + openRun, appendEvent: vi.fn(), completeRun: vi.fn(), + loadSessionCheckpoint: async () => structuredClone(checkpoint), + replayEvents: async () => ({ events: [], highestContiguousSourceSeq: 0 }), + }; + const failure = await executeNativeSession({ + input, backend, controlPlane: port, + runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", + }).catch((error: unknown) => error); + expect(failure, providerRecoveryPolicy).toBeInstanceOf(NativeProviderStateIncompatibleError); + expect((failure as Error).message).toBe("native_session_recovery_failed: saved launch profile changed"); + expect(openRun).not.toHaveBeenCalled(); + expect(openSession).not.toHaveBeenCalled(); + expect(openReplacementSession).not.toHaveBeenCalled(); + } + }); + it("bounds paginated recovery replay with one signal and observes a late rejection", async () => { vi.useFakeTimers(); let rejectStalledReplay = (_error: Error) => {}; diff --git a/packages/paperclip-runner/src/native-session-runtime.ts b/packages/paperclip-runner/src/native-session-runtime.ts index fbbb6a98133..a5f55fdf47a 100644 --- a/packages/paperclip-runner/src/native-session-runtime.ts +++ b/packages/paperclip-runner/src/native-session-runtime.ts @@ -19,8 +19,9 @@ import type { NativeSessionBackend, } from "./contracts/native-session-backend.js"; import type { PersistedNativeSession } from "./contracts/native-session-backend.js"; -import type { HarnessThreadGoal } from "./contracts/harness-driver.js"; +import { PROVIDER_STATE_INCOMPATIBLE, type HarnessThreadGoal } from "./contracts/harness-driver.js"; import { + NativeProviderStateIncompatibleError, NativeProviderTerminalFailure, NativeSessionCloseUnrecoverableError, NativeSessionCleanupQuarantinedError, @@ -2164,6 +2165,9 @@ export async function executeNativeSession( }; if (!recovery.recovered || !recovery.session) { if (!replacementAllowed) { + if (recovery.failureClass === PROVIDER_STATE_INCOMPATIBLE) { + throw new NativeProviderStateIncompatibleError(recovery.reason ?? "unknown"); + } throw new Error( `native_session_recovery_failed: ${recovery.reason ?? "unknown"}`, ); diff --git a/server/src/services/native-runtime/native-restart-recovery.ts b/server/src/services/native-runtime/native-restart-recovery.ts index 6f0cdaf91d8..24afea8b80f 100644 --- a/server/src/services/native-runtime/native-restart-recovery.ts +++ b/server/src/services/native-runtime/native-restart-recovery.ts @@ -40,6 +40,7 @@ import { isNativeRunnerOwnershipHeld } from "./native-runner-ownership.js"; import { nativeRestartInterruptedTurnId, type PersistedNativeSession } from "../../vendor/paperclip-runner/index.js"; import { retireSupersededNativeRetry } from "./native-superseded-retry.js"; import { legacyWorkspaceRecoveryNotMigrated } from "../legacy-workspace-recovery-migration.js"; +import { appendHeartbeatRunEvent } from "../heartbeat-run-events.js"; import { requestNativeStartupCleanup, readNativeStartupContext, promoteNativeStartupCleanup } from "./native-startup-cleanup.js"; export type NativeControllerIdentity = { @@ -583,6 +584,7 @@ export async function claimNativeRestartRecoveries(input: { run: heartbeatRuns, coordinator: nativeRunFinalizations, issueExecutionRunId: issues.executionRunId, + issueCheckoutRunId: issues.checkoutRunId, issueAssigneeAgentId: issues.assigneeAgentId, issueStatus: issues.status, }) @@ -630,7 +632,20 @@ export async function claimNativeRestartRecoveries(input: { if (!input.liveRunExecutions?.has(row.run.id) && await retireSupersededNativeRetry(tx as unknown as Db, { run: row.run, coordinator: row.coordinator, issueExecutionRunId: row.issueExecutionRunId, now, })) return { kind: "blocked", runId: row.run.id, reason: "native_retry_superseded" } as const; - if (row.issueExecutionRunId !== row.run.id && !(stopCleanup && row.issueExecutionRunId === null)) { + // A scheduled same-run resume can find its task lock released while no + // other run holds the task. Like the local resume claim, reclaim the + // free lock unless another run is admitted for this task. + const reclaimsExecution = row.issueExecutionRunId === null + && row.coordinator.phase === "retryable_failure" + && (row.issueCheckoutRunId === null || row.issueCheckoutRunId === row.run.id) + && !(await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, row.run.companyId), ne(heartbeatRuns.id, row.run.id), + inArray(heartbeatRuns.status, ["queued", "running", "scheduled_retry"]), + or(eq(heartbeatRuns.issueId, row.coordinator.issueId), eq(heartbeatRuns.nativeIssueId, row.coordinator.issueId), + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${row.coordinator.issueId}`), + )).limit(1)).length; + if (row.issueExecutionRunId !== row.run.id && !(stopCleanup && row.issueExecutionRunId === null) + && !reclaimsExecution) { return { kind: "blocked", runId: row.run.id, @@ -1165,6 +1180,23 @@ export async function claimNativeRestartRecoveries(input: { updatedAt: now, }) .where(eq(heartbeatRuns.id, row.run.id)); + if (reclaimsExecution) { + const [agent] = await tx.select({ name: agents.name }).from(agents) + .where(eq(agents.id, row.run.agentId)).limit(1); + await tx.update(issues).set({ + executionRunId: row.run.id, + executionAgentNameKey: agent?.name.trim().toLowerCase() || null, + executionLockedAt: now, + updatedAt: now, + }).where(and(eq(issues.id, row.coordinator.issueId), eq(issues.companyId, row.run.companyId), + isNull(issues.executionRunId))); + await appendHeartbeatRunEvent(tx as unknown as Db, { + companyId: row.run.companyId, runId: row.run.id, agentId: row.run.agentId, + eventType: "native.recovery.execution_reclaimed", stream: "system", level: "info", + message: "Reclaimed the original task execution for native session recovery.", + payload: { issueId: row.coordinator.issueId }, + }); + } claimScope = { companyId: row.run.companyId, issueId: row.coordinator.issueId, controller, remote: Boolean(durableAuthority) }; const common = { diff --git a/server/src/services/native-runtime/native-scheduled-resume-lock.test.ts b/server/src/services/native-runtime/native-scheduled-resume-lock.test.ts new file mode 100644 index 00000000000..6ddf261b653 --- /dev/null +++ b/server/src/services/native-runtime/native-scheduled-resume-lock.test.ts @@ -0,0 +1,137 @@ +import { randomUUID } from "node:crypto"; +import { and, eq } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { + agents, applyPendingMigrations, companies, createDb, environmentLeases, environments, heartbeatRunEvents, + heartbeatRuns, issues, nativeRunFinalizations, +} from "@paperclipai/db"; +import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; +import { recoveryService } from "../recovery/service.js"; +import { claimNativeRestartRecoveries } from "./native-restart-recovery.js"; + +const externalUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_TEST_DATABASE_URL"); +const support = externalUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); + +// A remote per-turn run fails while restoring its provider session. The +// coordinator schedules a same-run resume and keeps the task locks. The +// stale-lock sweeper then sees a failed run, and the remote resume must still +// run when its retry delay ends. +(support.supported ? describe : describe.skip)("scheduled remote native resume and the task execution lock", () => { + let db: ReturnType; + let temporary: Awaited> | undefined; + beforeAll(async () => { + if (externalUrl) { + await applyPendingMigrations(externalUrl); + db = createDb(externalUrl); + } else { + temporary = await startEmbeddedPostgresTestDatabase("native-scheduled-resume-lock-"); + db = createDb(temporary.connectionString); + } + }, 30_000); + afterAll(async () => { await db?.$client.end(); await temporary?.cleanup(); }); + + async function seed(options: { lockedBy?: "self" | "none" } = {}) { + const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), runId = randomUUID(); + const sessionId = randomUUID(); + const failedAt = new Date(Date.now() - 5 * 60_000); + await db.insert(companies).values({ id: companyId, name: "Scheduled resume", issuePrefix: companyId.slice(0, 8) }); + await db.insert(agents).values({ id: agentId, companyId, name: "Engineer", adapterType: "paperclip_runner" }); + const lock = options.lockedBy === "none" ? null : runId; + await db.insert(issues).values({ id: issueId, companyId, title: "Continue after approval", status: "in_progress", + assigneeAgentId: agentId }); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, issueId, nativeIssueId: issueId, + runtimeMode: "native", status: "failed", nativePhase: "retryable_failure", nativeSessionId: sessionId, + processPid: 4_242, processStartedAt: failedAt, startedAt: failedAt, finishedAt: new Date(), + errorCode: "native_session_interrupted", error: "native_session_recovery_failed: provider session was not restored", + contextSnapshot: { issueId }, + }); + const [environment] = await db.insert(environments).values({ name: `Scheduled resume ${runId}`, driver: "sandbox" }).returning(); + const [lease] = await db.insert(environmentLeases).values({ companyId, issueId, heartbeatRunId: runId, + environmentId: environment.id, provider: "daytona", providerLeaseId: randomUUID(), status: "active", + leasePolicy: "ephemeral", metadata: { driver: "sandbox", remoteCwd: "/work" }, + }).returning(); + await db.update(heartbeatRuns).set({ runnerProfileJson: { + nativeWorkspaceSync: { + schema: "paperclip.native-workspace-sync/v3", state: "prepared", descriptorSha256: "a".repeat(64), + baselineSha256: "b".repeat(64), finalHostSha256: null, workspaceId: runId, + leaseId: lease.id, providerLeaseId: lease.providerLeaseId, remoteCwd: "/work", resourceDisposition: "destroy", + }, + sessionCheckpoint: { + sessionId: "driver-session", providerSessionId: "provider-session", + identity: { runId, companyId, issueId, agentId, sessionId }, + }, + } }).where(eq(heartbeatRuns.id, runId)); + await db.update(issues).set({ executionRunId: lock, checkoutRunId: lock, executionLockedAt: lock ? failedAt : null }) + .where(eq(issues.id, issueId)); + await db.insert(nativeRunFinalizations).values({ runId, companyId, issueId, phase: "retryable_failure", attempt: 1, + nextAttemptAt: new Date(Date.now() - 60_000), recoveryState: "resuming_session", + failureCode: "native_session_interrupted", + failureDetail: { recoveryMode: "exact_checkpoint_resume", checkpointExists: true, + providerSessionEstablished: true, providerEventsExist: false }, + }); + return { companyId, agentId, issueId, runId }; + } + + const readIssue = async (id: string) => (await db.select().from(issues).where(eq(issues.id, id)))[0]!; + const readCoordinator = async (runId: string) => + (await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)))[0]!; + const sweep = () => recoveryService(db, { enqueueWakeup: vi.fn() }).sweepStaleIssueLocks(); + + it("keeps the task locks while the failed run's coordinator owns a scheduled resume", async () => { + const f = await seed(); + const result = await sweep(); + expect(result.issueIds).not.toContain(f.issueId); + expect(await readIssue(f.issueId)).toMatchObject({ executionRunId: f.runId, checkoutRunId: f.runId }); + }); + + it("keeps the task locks when a resume claim returns the run to running before the clear", async () => { + const f = await seed(); + await db.update(nativeRunFinalizations).set({ phase: "terminal_failure", nextAttemptAt: null }) + .where(eq(nativeRunFinalizations.runId, f.runId)); + const beforeStaleLockClear = vi.fn(async (issueId: string) => { + if (issueId !== f.issueId) return; + await db.update(nativeRunFinalizations).set({ phase: "observed" }).where(eq(nativeRunFinalizations.runId, f.runId)); + await db.update(heartbeatRuns).set({ status: "running", nativePhase: "observed" }).where(eq(heartbeatRuns.id, f.runId)); + }); + const result = await recoveryService(db, { enqueueWakeup: vi.fn(), beforeStaleLockClear }).sweepStaleIssueLocks(); + expect(beforeStaleLockClear).toHaveBeenCalledWith(f.issueId); + expect(result.issueIds).not.toContain(f.issueId); + expect(await readIssue(f.issueId)).toMatchObject({ executionRunId: f.runId, checkoutRunId: f.runId }); + }); + + it("still clears a closed task's locks for a failed run with a scheduled resume", async () => { + const f = await seed(); + await db.update(issues).set({ status: "cancelled" }).where(eq(issues.id, f.issueId)); + expect((await sweep()).issueIds).toContain(f.issueId); + expect(await readIssue(f.issueId)).toMatchObject({ executionRunId: null, checkoutRunId: null }); + }); + + it("claims a due remote resume whose cleared task lock is free and reclaims it", async () => { + const f = await seed({ lockedBy: "none" }); + const dispositions = await claimNativeRestartRecoveries({ db, restartKind: "hard", runIds: [f.runId] }); + expect(dispositions).toEqual([expect.objectContaining({ kind: "reattach_remote_runner", runId: f.runId })]); + expect(await readIssue(f.issueId)).toMatchObject({ executionRunId: f.runId }); + expect(await readCoordinator(f.runId)).toMatchObject({ phase: "observed", attempt: 1, nextAttemptAt: null }); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)))[0]) + .toMatchObject({ status: "running", nativePhase: "observed" }); + const events = await db.select({ eventType: heartbeatRunEvents.eventType }).from(heartbeatRunEvents) + .where(and(eq(heartbeatRunEvents.runId, f.runId), eq(heartbeatRunEvents.eventType, "native.recovery.execution_reclaimed"))); + expect(events).toHaveLength(1); + }); + + it("does not reclaim a task that another run owns or has admitted", async () => { + for (const competitor of ["lock", "queued"] as const) { + const f = await seed({ lockedBy: "none" }); + const otherRunId = randomUUID(); + await db.insert(heartbeatRuns).values({ id: otherRunId, companyId: f.companyId, agentId: f.agentId, + issueId: f.issueId, status: competitor === "lock" ? "running" : "queued", contextSnapshot: { issueId: f.issueId } }); + if (competitor === "lock") await db.update(issues).set({ executionRunId: otherRunId }).where(eq(issues.id, f.issueId)); + expect(await claimNativeRestartRecoveries({ db, restartKind: "hard", runIds: [f.runId] })).toEqual([ + { kind: "blocked", runId: f.runId, reason: "issue_execution_lock_changed" }, + ]); + expect((await readIssue(f.issueId)).executionRunId).toBe(competitor === "lock" ? otherRunId : null); + expect(await readCoordinator(f.runId)).toMatchObject({ phase: "retryable_failure", attempt: 1 }); + } + }); +}); diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 86df326e85f..4e76dd2b69c 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -84,6 +84,7 @@ import * as noLaunchProofModule from "./native-maintenance-no-launch.js"; import { NativeSessionCleanupQuarantinedError, NativeSessionCloseUnrecoverableError, + NativeProviderStateIncompatibleError, NativeProviderTerminalFailure, NativeSessionProtocolIntegrityError, } from "../../vendor/paperclip-runner/index.js"; @@ -11752,6 +11753,21 @@ describe("native session bounded recovery", () => { }, ); + it("stops same-run retries when the runner proves the saved provider session incompatible", () => { + const failure = new NativeProviderStateIncompatibleError( + "Error: PRP command run.attach failed: ACPX durable launch profile digest does not match runner startup", + ); + const code = nativeSessionFailureSourceCode(failure); + expect(code).toBe("native_provider_state_incompatible"); + const disposition = nativeSessionFailureDisposition(1, new Date(), code); + expect(disposition).toEqual({ phase: "terminal_failure", failureCode: code, nextAttemptAt: null }); + expect(nativeSessionRecoveryProjection({ ...disposition, agentId: "agent" })).toMatchObject({ + issueStatus: "blocked", recoveryOwner: { kind: "board" }, + }); + // Only the typed runner class is deterministic; the same text stays retryable. + expect(nativeSessionFailureSourceCode(new Error(failure.message))).toBe("native_session_interrupted"); + }); + it("makes only typed operator-required cleanup quarantine terminal on the first attempt", () => { const code = nativeSessionFailureSourceCode( new NativeSessionCleanupQuarantinedError(), diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 06a4f78b4a2..c152cb32e15 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -140,6 +140,7 @@ import type { PrpStructuredRunResult, } from "../../vendor/paperclip-runner/index.js"; import { + NativeProviderStateIncompatibleError, NativeProviderTerminalFailure, NativeSessionCleanupQuarantinedError, NativeSessionCloseUnrecoverableError, @@ -218,7 +219,7 @@ import { renderNativeRunnerStagedAttachmentPrompt, stageNativeRunnerWakeAttachments, } from "./native-runner-file-handoff.js"; -import { nativeRuntimeContractForProvider, nativeToolContractFingerprintForTarget } from "./native-session-resume.js"; +import { NATIVE_PROVIDER_STATE_INCOMPATIBLE, nativeRuntimeContractForProvider, nativeToolContractFingerprintForTarget } from "./native-session-resume.js"; import { verifyRetainedMaintenanceNoLaunch } from "./native-maintenance-no-launch.js"; import { issueRecoveryActionService } from "../issue-recovery-actions.js"; import { reportRunFailure } from "../run-failure-report.js"; @@ -6989,6 +6990,7 @@ export function nativeSessionFailureDisposition( sourceFailureCode === "runner_remote_provider_artifact_incompatible" || sourceFailureCode === "runner_remote_provider_verification_unavailable" || sourceFailureCode === "native_provider_terminal_failed" || + sourceFailureCode === NATIVE_PROVIDER_STATE_INCOMPATIBLE || sourceFailureCode === "native_current_wake_comments_unread" || sourceFailureCode === "native_current_wake_comments_changed_after_read" || sourceFailureCode === "native_session_cleanup_quarantined" || @@ -7039,6 +7041,7 @@ export function nativeSessionFailureSourceCode( ): | "native_permission_declined" | "native_provider_terminal_failed" + | typeof NATIVE_PROVIDER_STATE_INCOMPATIBLE | "native_provider_approval_required" | "native_provider_usage_limit" | "native_session_cleanup_quarantined" @@ -7065,6 +7068,7 @@ export function nativeSessionFailureSourceCode( | "provider_session_open_rejected" | "native_session_interrupted" { if (error instanceof NativePermissionDeclinedError) return "native_permission_declined"; + if (error instanceof NativeProviderStateIncompatibleError) return NATIVE_PROVIDER_STATE_INCOMPATIBLE; if (error instanceof NativeProviderTerminalFailure) { if (error.providerCode === "approval_required") return "native_provider_approval_required"; // Failed terminals retain their security meaning across the provider facade. @@ -7155,6 +7159,9 @@ export function nativeSessionFailureSourceCode( return "native_session_interrupted"; } +const NATIVE_PROVIDER_STATE_INCOMPATIBLE_ACTION = + "The current runner cannot restore this task's saved provider session, so retrying this run cannot succeed. Retry the task to continue in a fresh provider session with the same workspace. The earlier provider conversation is not carried over."; + const NATIVE_CLEANUP_OPERATOR_RECOVERY_MESSAGE = "Send a new message to continue after Paperclip verifies that the previous provider and its tools have stopped. If cleanup cannot be verified, inspect the run and its environment. Clearing a task session does not resolve this quarantine. Automatic retries are stopped."; @@ -10491,6 +10498,8 @@ async function executePaperclipNativeSessionWithinScope( ? "Provider files could not be verified after two read attempts. Inspect the retained workspace and command transport before retrying the original run." : sourceFailureCode === "native_provider_terminal_failed" ? "The provider session is permanently unusable. Verify stopped execution, completed actions, and task context before starting a linked continuation." + : sourceFailureCode === NATIVE_PROVIDER_STATE_INCOMPATIBLE + ? NATIVE_PROVIDER_STATE_INCOMPATIBLE_ACTION : recoveryEvidence.recoveryMode === "ambiguous_state" ? "Inspect the original provider failure and durable events; state is ambiguous and a replacement provider session is forbidden." : integrityFailure @@ -10683,6 +10692,8 @@ async function executePaperclipNativeSessionWithinScope( ? NATIVE_CLEANUP_OPERATOR_RECOVERY_MESSAGE : sourceFailureCode === "native_provider_terminal_failed" ? "Verify that the failed provider stopped and reconcile its action outcomes. A linked continuation can proceed only after these checks succeed." + : sourceFailureCode === NATIVE_PROVIDER_STATE_INCOMPATIBLE + ? NATIVE_PROVIDER_STATE_INCOMPATIBLE_ACTION : recoveryEvidence.recoveryMode === "ambiguous_state" ? "Inspect the original provider failure and explicitly resolve the ambiguous session state; do not open a replacement provider session." : integrityFailure diff --git a/server/src/services/native-runtime/native-session-resume-projection.test.ts b/server/src/services/native-runtime/native-session-resume-projection.test.ts index 502bcaf68f3..2838347e5bd 100644 --- a/server/src/services/native-runtime/native-session-resume-projection.test.ts +++ b/server/src/services/native-runtime/native-session-resume-projection.test.ts @@ -28,6 +28,7 @@ const supported = Boolean(externalUrl) || (await getEmbeddedPostgresTestSupport( }); const select = () => findNativeSessionResumeRun(db, input); const source = () => run(0, { runnerInstanceId: randomUUID(), processPid: 17, processGroupId: 19, processStartedAt: at(0), + errorCode: "native_session_interrupted", runnerProfileJson: profile, contextSnapshot: { paperclipWake, paperclipIssue: { ...paperclipIssue, unrelated: "Omit" }, prompt: "Omit" } }); const writeProfile = (id: string, value: unknown) => db.execute(sql`update ${heartbeatRuns} set runner_profile_json = ${JSON.stringify(value)}::jsonb where ${heartbeatRuns.id} = ${id}`); diff --git a/server/src/services/native-runtime/native-session-resume.test.ts b/server/src/services/native-runtime/native-session-resume.test.ts index 932260c5640..9765563ee98 100644 --- a/server/src/services/native-runtime/native-session-resume.test.ts +++ b/server/src/services/native-runtime/native-session-resume.test.ts @@ -62,6 +62,7 @@ import { buildNativeExecutionInput } from "./native-execution-input.js"; import { buildNativeExecutionWithCheckpoint, findNativeSessionResumeRun, + NATIVE_PROVIDER_STATE_INCOMPATIBLE, NATIVE_TOOL_CONTRACT_FINGERPRINT, isUnusedLegacyNativeRetryReplacement, nativeToolContractFingerprintForTarget, @@ -278,6 +279,28 @@ describe("capability-gated connection tool refresh", () => { }); }); +describe("a source whose saved provider session the runner proved incompatible", () => { + it("continues the task in a fresh provider session with the same workspace", () => { + const current = execution(currentRunId); + expect(rebindNativeSessionCheckpoint({ previousRun: { ...previousRun(), errorCode: "native_session_interrupted" }, + currentExecution: current })).not.toBeNull(); + const source = { ...previousRun(), errorCode: NATIVE_PROVIDER_STATE_INCOMPATIBLE }; + expect(rebindNativeSessionCheckpoint({ previousRun: source, currentExecution: current })).toBeNull(); + const buildExecution = vi.fn(({ normalizedSessionId: sessionId, resumedSession }) => { + const built = execution(currentRunId); + built.session.normalizedSessionId = sessionId; + built.task.prompt = resumedSession ? "wake delta" : "original goal and complete handoff"; + return built; + }); + const result = buildNativeExecutionWithCheckpoint({ previousRun: source, normalizedSessionId, buildExecution }); + expect(result.checkpoint).toBeNull(); + expect(result.normalizedSessionId).not.toBe(normalizedSessionId); + expect(result.execution.task.prompt).toContain("complete handoff"); + expect(result.execution.binding.executionWorkspaceId).toBe(current.binding.executionWorkspaceId); + expect(result.execution.workspace).toEqual(current.workspace); + }); +}); + it("wires exact-session recovery and guarded selected identity into heartbeat persistence", () => { const source = readFileSync( new URL("../heartbeat/runtime-selection.ts", import.meta.url), @@ -1219,6 +1242,41 @@ const recoveryFakeCodex = resolve( currentExecution: execution(currentRunId), }), ).toBeNull(); + // A run whose saved provider session the runner proved unrestorable + // stays the barrier, and its checkpoint is not resumed. + const unrestorableId = randomUUID(); + const sourceCheckpoint = source.runnerProfileJson.sessionCheckpoint; + await db.insert(heartbeatRuns).values({ + ...unused, + id: unrestorableId, + createdAt: at(8), + errorCode: NATIVE_PROVIDER_STATE_INCOMPATIBLE, + runnerProfileJson: { + ...source.runnerProfileJson, + nativeExecutionInput: execution(unrestorableId), + sessionCheckpoint: { + ...sourceCheckpoint, + identity: { ...sourceCheckpoint.identity, runId: unrestorableId }, + }, + }, + }); + const unrestorable = await findNativeSessionResumeRun(db, selection); + expect(unrestorable).toMatchObject({ + id: unrestorableId, + errorCode: NATIVE_PROVIDER_STATE_INCOMPATIBLE, + }); + expect( + rebindNativeSessionCheckpoint({ + previousRun: { ...unrestorable!, errorCode: null }, + currentExecution: execution(currentRunId), + }), + ).not.toBeNull(); + expect( + rebindNativeSessionCheckpoint({ + previousRun: unrestorable!, + currentExecution: execution(currentRunId), + }), + ).toBeNull(); const fresh = buildNativeExecutionWithCheckpoint({ previousRun: null, diff --git a/server/src/services/native-runtime/native-session-resume.ts b/server/src/services/native-runtime/native-session-resume.ts index 135eb9c27b2..9e4fe58b486 100644 --- a/server/src/services/native-runtime/native-session-resume.ts +++ b/server/src/services/native-runtime/native-session-resume.ts @@ -94,6 +94,10 @@ export function nativeToolContractFingerprintForTarget( export const NATIVE_TOOL_CONTRACT_FINGERPRINT = nativeToolContractFingerprintForTarget("local"); +/** The runner proved that this run's saved provider session cannot be + * restored by the current runner build. Its checkpoint is not resumable. */ +export const NATIVE_PROVIDER_STATE_INCOMPATIBLE = "native_provider_state_incompatible"; + function record(value: unknown): Record { return typeof value === "object" && value !== null && !Array.isArray(value) ? (value as Record) @@ -341,6 +345,7 @@ async function selectNativeSessionResumeBarrier(db: Db, input: NativeSessionResu nativeIssueId: heartbeatRuns.nativeIssueId, runtimeMode: heartbeatRuns.runtimeMode, status: heartbeatRuns.status, + errorCode: heartbeatRuns.errorCode, createdAt: heartbeatRuns.createdAt, processPid: heartbeatRuns.processPid, processGroupId: heartbeatRuns.processGroupId, @@ -596,6 +601,7 @@ export function rebindNativeSessionCheckpoint(input: { companyId: string; agentId: string; nativeSessionId: string | null; + errorCode?: string | null; runnerProfileJson: unknown; }; currentExecution: NativeExecutionInput; @@ -604,6 +610,9 @@ export function rebindNativeSessionCheckpoint(input: { refreshTools?: boolean; }): PersistedNativeSession | null { if (input.refreshTools && input.toolRefreshOnResume !== true) return null; + // The source is still the progress barrier. Its provider session cannot be + // restored, so the next run starts a fresh session in the same workspace. + if (input.previousRun.errorCode === NATIVE_PROVIDER_STATE_INCOMPATIBLE) return null; const previousProfile = record(input.previousRun.runnerProfileJson); if ( previousProfile.nativeToolContractFingerprint !== diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index f935b8a2df9..b380f431ed5 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -975,6 +975,7 @@ export function recoveryService( ) => boolean; liveRunExecutions?: Readonly<{ has(id: string): boolean }>; beforeOrphanedRunTerminalWrite?: (runId: string) => Promise; + beforeStaleLockClear?: (issueId: string) => Promise; }, ) { const issuesSvc = issueService(db); @@ -6477,6 +6478,26 @@ export function recoveryService( if (!status) return true; // missing run row → no real claim return TERMINAL_HEARTBEAT_RUN_STATUSES.has(status); }; + // A native failure that schedules a same-run resume marks its run failed + // but keeps both task locks for that resume. Clearing them lets another + // run take the task first, and then the resume can never run. Only a + // closed task releases them here. + const pendingResumeLock = sql`exists (select 1 from ${nativeRunFinalizations} pending_resume + where pending_resume.company_id = ${issues.companyId} + and pending_resume.run_id in (${issues.checkoutRunId}, ${issues.executionRunId}) + and pending_resume.phase = 'retryable_failure' + and pending_resume.result_id is null)`; + const lockRunLive = sql`exists (select 1 from ${heartbeatRuns} lock_run + where lock_run.id in (${issues.checkoutRunId}, ${issues.executionRunId}) + and lock_run.status not in (${sql.join([...TERMINAL_HEARTBEAT_RUN_STATUSES].map(status => sql`${status}`), sql`, `)}))`; + const pendingResumeRunIds = new Set(referencedRunIds.length === 0 ? [] : (await db + .select({ runId: nativeRunFinalizations.runId }) + .from(nativeRunFinalizations) + .where(and( + inArray(nativeRunFinalizations.runId, referencedRunIds), + eq(nativeRunFinalizations.phase, "retryable_failure"), + isNull(nativeRunFinalizations.resultId), + ))).map(row => row.runId)); for (const issue of candidates) { const originalOwner = issue.executionRunId ? runById.get(issue.executionRunId) : undefined; @@ -6497,7 +6518,13 @@ export function recoveryService( ) { continue; } + const taskClosed = issue.status === "done" || issue.status === "cancelled"; + if (!taskClosed && [issue.checkoutRunId, issue.executionRunId] + .some(runId => runId !== null && pendingResumeRunIds.has(runId))) { + continue; + } + await deps.beforeStaleLockClear?.(issue.id); const updated = await db.transaction(async tx => { await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: issue.companyId, issueId: issue.id }); await notifyNativeStatusReassessment(tx, { companyId: issue.companyId, issueId: issue.id }); @@ -6519,6 +6546,10 @@ export function recoveryService( issue.executionRunId ? eq(issues.executionRunId, issue.executionRunId) : isNull(issues.executionRunId), + // The reads above can be stale here. A resume claim can return a + // referenced run to running, and a failure can schedule a resume. + not(lockRunLive), + taskClosed ? undefined : not(pendingResumeLock), ), ) .returning({ id: issues.id }) diff --git a/server/src/vendor/paperclip-runner/index.ts b/server/src/vendor/paperclip-runner/index.ts index 80b70e5b9f2..51ddeff3ea2 100644 --- a/server/src/vendor/paperclip-runner/index.ts +++ b/server/src/vendor/paperclip-runner/index.ts @@ -182,6 +182,7 @@ export const validatePrpStructuredRunResult = runner.validatePrpStructuredRunResult; export const NativeProviderTerminalFailure = runner.NativeProviderTerminalFailure; +export const NativeProviderStateIncompatibleError = runner.NativeProviderStateIncompatibleError; export const nativeRestartInterruptedTurnId = runner.nativeRestartInterruptedTurnId; export const completeTerminatedRemoteNativeSessionCleanup = runner.completeTerminatedRemoteNativeSessionCleanup;