diff --git a/packages/paperclip-runner/acpx-profiles.json b/packages/paperclip-runner/acpx-profiles.json index eda0ebf68c0..0e5443132cc 100644 --- a/packages/paperclip-runner/acpx-profiles.json +++ b/packages/paperclip-runner/acpx-profiles.json @@ -32,12 +32,12 @@ "requiresProviderPolicy": true }, "copilot": { - "agentProfileVersion": 66, + "agentProfileVersion": 68, "agentServerPackage": "@github/copilot", "agentServerVersion": "1.0.88", "agentRuntimePackage": null, "agentRuntimeVersion": null, - "commandDigest": "sha256:0cddc219af25b741b7138f38faa7514dd5ee3427b1d3168e8a4bfbe5180983e8", + "commandDigest": "sha256:1ce8bb0e00df81fa9eb71cff5063ce769637f5db7b3aa0ada9ade651db089c63", "requiresProviderPolicy": true }, "claude": { diff --git a/packages/paperclip-runner/docs/durable-recovery.md b/packages/paperclip-runner/docs/durable-recovery.md index 3bfcfe49db3..43898274399 100644 --- a/packages/paperclip-runner/docs/durable-recovery.md +++ b/packages/paperclip-runner/docs/durable-recovery.md @@ -57,6 +57,110 @@ PRP v2 command vocabulary. Incoming wire commands still undergo the negotiated protocol validation before execution; ordinary persisted v1 summaries remain compatible. +## Adoption and retired launch artifacts + +A live runner is adopted with its admitted artifact binding and authenticated +process identity. ACPX and OpenCode launch profiles are constructed only when a +new runner will actually be spawned. Adoption does not read the retired +controller's provider-pack executables, sidecar or proxy files. Fresh launches +still require the existing artifact digests and provider-pack authority. + +The component regression starts the production runner and OpenCode proxy against +a scripted harness, removes the controller's launch files, then recovers the +same process and provider session and completes another turn. This exercises the +real transport; it does not qualify a hosted OpenCode or ACPX deployment. + +## Source-bound profile history + +Copilot profile v68 binds the merged hot-upgrade and current master transport. +Published v66 and the earlier hot-upgrade v67 identity remain byte-for-byte +historical records. The distinct development v66 identity is retained under +`copilot-profile-v66-hot-upgrade-candidate-historical-identity.json`; it does not +replace the published v66 fixture. Decoding historical versions does not grant +fresh-launch permission. Installation still requires the exact current profile, +package and digest. These records do not establish hosted harness qualification. + +## External controller checkpoints + +The optional controller persistence callback stores recovery control state, not +an output transcript. Text deltas do not trigger callbacks; usage updates are +coalesced into the next control checkpoint. Authentication, command intents and +outcomes, semantic inputs, lifecycle and result events retain commit barriers. +Priority-zero and unknown event kinds are never treated as disposable output. + +Installing the callback makes it the controller's durable authority. The local +state file is replaced once with a fail-closed marker; ordinary control changes +no longer rewrite or fsync a second local journal. The controller can continue +if that directory disappears. A new controller must restore the committed +external checkpoint before opening the directory; the marker cannot recover +an execution or authorize a new one. This does not change the sandbox runner's +journal or controllers without an external persistence callback. + +Fresh semantic inputs reserve their original live callback before awaiting the +checkpoint barrier. Recovery cannot steal first dispatch when no saved outcome +exists. The reservation is released if persistence or admission fails; business +execution still starts only after durable persistence succeeds. + +Recovery and live semantic-tool dispatch share per-call ownership. Overlapping +reconciliation calls cannot invoke the same callback twice while its outcome is +pending. An already queued authoritative result is replayed without resolving +its business outcome again. Failed or unknown outcomes retain the existing +reconciliation and authority rules. + +Pending completion inputs (`paperclip_finish` and `paperclip_block`) use the +driver's existing completion handler even when its caller supplies no recovery +lookup for ordinary tools. Codex transport waits for both authenticated startup +and handler installation; the external driver waits for session opening. +An ordinary tool with no outcome lookup remains held. Reconciliation does not +turn a missing lookup into permission to repeat that tool. + +Transport readers use a committed view when external persistence is enabled. +Lifecycle and result events, command outcomes and warm-handoff authority become +visible only after the checkpoint callback succeeds. A failed write leaves the +previous view in place until a successful retry. Internal admission still sees +staged intents so overlapping recovery cannot duplicate them. Command results +flush their checkpoint even when no next command is waiting. + +Streamed output continues without checkpoint writes. The view stops before a +pending control event to preserve event order, reuses immutable event records +and unchanged decoded commands, and caches the visible projection between +changes. It neither copies a second transcript nor adds an idle scan. + +A rejected persistence callback closes the current connection and keeps the last +successful revision. A later flush or reconnect retries from a complete checkpoint, +including when the failed response hid a successful commit. Queued writes also +rebuild their delta against the last successful callback. Commit barriers remain +in force, and no retry timer runs when the controller is idle. + +Supporting runners advertise `recoveryCursorVersion: 1` in `auth_hello`. The core +includes that capability and `recoveryAckedSourceSeq` in both authentication +proofs. The runner verifies the cursor equals its persisted acknowledgement. +After authentication and owner admission, a replacement core can reconcile that +cursor with its sparse checkpoint. It only skips already-acknowledged history; +new delivery remains contiguous. Unacknowledged events use the existing outbox. +A peer without this capability cannot attach to a sparse external checkpoint. +New runners can still authenticate to older controllers that use full journals. + +There is no per-output external checkpoint timer or remote-controller cache +fsync. Local-only controllers retain their full file journal. Existing application +callbacks, the bounded runner outbox, and backpressure remain. Losing the +controller before acknowledged display output reaches its application log can +leave a display tail gap; this must not trigger provider or tool reexecution. + +Prepared environments retain at most two opaque configuration IDs, each bound to +its exact execution identity across a warm handoff. They are captured with control +state, never output, and do not duplicate instructions or preparation requirements. +An execution driver can provide `authorizeRunnerAdoption(configurationId, identity)` +to recheck current authority before registering a surviving runner, without +preparing its resources again. The driver owns policy; Runner knows no project or +account rules. Missing bindings and rejected checks fail closed. Without this +optional hook, adoption retains the existing `prepareLaunch` recovery callback +and requires its controller-local preparation cache. Older checkpoints without +a configuration binding cannot use the cache-independent adoption path. +Supplying the hook for adoption without `prepareLaunch` is rejected at construction; +the transport must not silently skip a requested authority check. Drivers that do +not use prepared environments keep their existing adoption contract. + ## Connection and authentication The connection starts in this order: @@ -424,11 +528,49 @@ A `revoke` envelope persists the `revoked` lifecycle. The runner flushes any existing durable events and exits. It does not accept new work or delete unacknowledged facts. +## Process identity + +An external process owner can publish `startedAt` after launch dispatch returns. +`spawnRunner` preserves that live value through its restart wrapper. Once the +owner has a verified process identity, consumers use its birth time rather than +controller construction or attachment time. Replacing the controller must not +change the admitted process identity. + ## Verification The production durable control plane and live-session suites cover reconnect, ACK replay, restart recovery, backpressure, lease expiry, drain, and revoke. +`src/control-plane/runner-recovery-fixture.test.ts` builds the test-only Cargo +example `recovery-fixture`. It links the unchanged production durable runner with +a scripted `CommandExecutor`, whose worker thread performs bounded hashing steps +and retains each resulting output until the normal executor ACK transfers it to +the runner outbox. There is one process, no HTTP counter service, and no production +provider-admission switch. Private fixture receipts report completed steps, queue +depth and process identity; they do not acknowledge or recover protocol events. + +The short experiment starts one turn, removes the controller, then releases real +work. Eight retained events fill the fixture queue and pause work explicitly. +A replacement authenticates and drains through PRP. At step 32 the executor waits +for a controller command, survives another replacement, then completes 64 steps +without repeating the turn. The test runs with ordinary filesystem persistence +and with opaque external checkpoints restored onto empty controller storage. +An independent short-deadline case checks termination with recovery state intact. +These prove the runner mechanism, not Codex, Claude, Daytona or application +authorization. Server adoption and installed-artifact qualification have separate +owners and evidence. + +Run the short experiment from this package with: + +```sh +pnpm exec vitest run src/control-plane/runner-recovery-fixture.test.ts --maxWorkers=1 +``` + +The fixture builds through Cargo during test setup. Production distribution builds +do not include the example. Existing control-plane tests separately inject failed +checkpoint persistence, stale/unauthenticated cursors and output floods, including +the zero-per-text-checkpoint-write gate. + Use `--json` for the complete trace or `--output ` to write it. The CLI and browser show connection counts, safe lease ID and expiry, stable identities, source and ACK cursors, outbox current/peak bytes, command delivery/effect @@ -441,3 +583,5 @@ Regenerate the checked fault matrix and every exact per-fault trace with: ```sh # Recorded evidence generation is deferred from this release. ``` + +Checkpoint decoding also exposes read-only recovery evidence: the admitted runner/session identity, authenticated connection count, and highest recorded source sequence. This projection grants no authority and adds no checkpoint writes. Qualification compares it across controller replacement; a replayed canonical event by itself does not prove that a replacement controller has authenticated a runner. diff --git a/packages/paperclip-runner/runner/crates/runner-core/examples/recovery-fixture.rs b/packages/paperclip-runner/runner/crates/runner-core/examples/recovery-fixture.rs new file mode 100644 index 00000000000..0398a6b2f3f --- /dev/null +++ b/packages/paperclip-runner/runner/crates/runner-core/examples/recovery-fixture.rs @@ -0,0 +1,233 @@ +use std::collections::VecDeque; +use std::fs; +use std::path::PathBuf; +use std::sync::{Arc, Condvar, Mutex}; +use std::thread; +use std::time::Duration; + +use paperclip_runner_core::durable::{ + capture_bootstrap_ticket, run_durable_runner, Command, CommandExecution, CommandExecutor, + DurableRunnerConfig, DurableRunnerError, EventPriority, PolledEvent, +}; +use serde::Deserialize; +use serde_json::json; +use sha2::{Digest, Sha256}; + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct Script { + steps: usize, + wait_after: usize, + retention_limit: usize, + release_file: PathBuf, + receipt_file: PathBuf, +} + +#[derive(Default)] +struct Work { + started: bool, + stopped: bool, + resumed: bool, + completed: usize, + acknowledged: usize, + peak_retained: usize, + events: VecDeque, +} + +type Shared = Arc<(Mutex, Condvar)>; + +fn receipt(script: &Script, work: &Work, phase: &str) { + let bytes = serde_json::to_vec(&json!({ + "pid": std::process::id(), "phase": phase, "completedSteps": work.completed, + "retainedEvents": work.events.len(), "acknowledgedEvents": work.acknowledged, + "peakRetainedEvents": work.peak_retained, + })) + .unwrap(); + let temporary = script.receipt_file.with_extension("pending"); + fs::write(&temporary, bytes).unwrap(); + fs::rename(temporary, &script.receipt_file).unwrap(); +} + +fn workload(script: Script, shared: Shared) { + let (lock, changed) = &*shared; + let mut work = lock.lock().unwrap(); + receipt(&script, &work, "admitted"); + loop { + if work.stopped { + receipt(&script, &work, "stopped"); + return; + } + if !work.started || !script.release_file.exists() { + work = changed + .wait_timeout(work, Duration::from_millis(10)) + .unwrap() + .0; + continue; + } + if work.completed == script.steps { + receipt(&script, &work, "completed"); + work = changed.wait(work).unwrap(); + continue; + } + if work.completed == script.wait_after && !work.resumed { + receipt(&script, &work, "awaiting_controller"); + work = changed.wait(work).unwrap(); + continue; + } + if work.events.len() == script.retention_limit { + receipt(&script, &work, "backpressured"); + work = changed.wait(work).unwrap(); + continue; + } + let step = work.completed + 1; + drop(work); + let mut digest = Sha256::digest(step.to_le_bytes()); + for _ in 0..4096 { + digest = Sha256::digest(digest); + } + work = lock.lock().unwrap(); + work.events.push_back(PolledEvent { + executor_event_id: format!("fixture-step-{step}"), + event_type: "item.delta".to_owned(), + priority: EventPriority::P2, + payload: json!({"text": format!("step={step} sha256={digest:x}\n")}), + }); + work.completed = step; + work.peak_retained = work.peak_retained.max(work.events.len()); + receipt(&script, &work, "computing"); + } +} + +struct ScriptedExecutor { + shared: Shared, + worker: Option>, +} + +impl ScriptedExecutor { + fn new(script: Script) -> Self { + let shared = Arc::new((Mutex::new(Work::default()), Condvar::new())); + let worker_shared = shared.clone(); + Self { + shared, + worker: Some(thread::spawn(move || workload(script, worker_shared))), + } + } +} + +impl CommandExecutor for ScriptedExecutor { + fn execute(&mut self, command: &Command) -> Result { + let mut work = self.shared.0.lock().unwrap(); + match command.command_type.as_str() { + "turn.start" => { + if work.started { + return Err(DurableRunnerError::invalid("fixture turn started twice")); + } + work.started = true; + } + "turn.steer" => work.resumed = true, + "session.snapshot" => {} + "turn.stop" | "session.close" | "runner.shutdown" | "runner.suspend" => { + work.stopped = true + } + _ => return Err(DurableRunnerError::invalid("unsupported fixture command")), + } + self.shared.1.notify_all(); + Ok(CommandExecution::result(json!({ + "completedSteps": work.completed, "retainedEvents": work.events.len(), + "acknowledgedEvents": work.acknowledged, "pid": std::process::id(), + }))) + } + + fn poll_events(&mut self) -> Result, DurableRunnerError> { + self.retained_events() + } + + fn retained_events(&mut self) -> Result, DurableRunnerError> { + Ok(self + .shared + .0 + .lock() + .unwrap() + .events + .iter() + .cloned() + .collect()) + } + + fn acknowledge_events(&mut self, count: usize) -> Result<(), DurableRunnerError> { + let mut work = self.shared.0.lock().unwrap(); + if count > work.events.len() { + return Err(DurableRunnerError::invalid( + "fixture acknowledged an unowned event", + )); + } + work.events.drain(..count); + work.acknowledged += count; + self.shared.1.notify_all(); + Ok(()) + } + + fn shutdown(&mut self) -> Result<(), DurableRunnerError> { + self.shared.0.lock().unwrap().stopped = true; + self.shared.1.notify_all(); + if let Some(worker) = self.worker.take() { + worker + .join() + .map_err(|_| DurableRunnerError::invalid("fixture worker failed"))?; + } + Ok(()) + } +} + +impl Drop for ScriptedExecutor { + fn drop(&mut self) { + let _ = self.shutdown(); + } +} + +fn main() -> Result<(), Box> { + let args: Vec = std::env::args().skip(1).collect(); + let value = |key: &str| -> Result { + args.windows(2) + .find(|pair| pair[0] == key) + .map(|pair| pair[1].clone()) + .ok_or_else(|| DurableRunnerError::invalid(format!("missing {key}"))) + }; + let script: Script = serde_json::from_slice(&fs::read(value("--fixture-script")?)?)?; + if script.steps == 0 + || script.steps > 4096 + || script.wait_after > script.steps + || !(1..=256).contains(&script.retention_limit) + { + return Err("fixture workload exceeds its bounds".into()); + } + let config = DurableRunnerConfig { + connect_url: value("--connect-url")?, + ca_bundle_path: None, + state_dir: value("--state-dir")?.into(), + runner_instance_id: value("--runner-id")?, + environment_lease_id: value("--environment-lease-id")?, + run_id: value("--run-id")?, + normalized_session_id: value("--session-id")?, + turn_id: value("--turn-id")?, + item_id: value("--item-id")?, + runner_version: value("--runner-version")?, + runner_digest: value("--runner-digest")?, + acpx_launch_profile: None, + opencode_launch_profile: None, + max_outbox_bytes: value("--max-outbox-bytes")?.parse()?, + p0_reserve_bytes: value("--p0-reserve-bytes")?.parse()?, + max_frame_bytes: 1024 * 1024, + reconnect_delay: Duration::from_millis(20), + reconnect_grace: Some(Duration::from_millis( + value("--reconnect-grace-ms")?.parse()?, + )), + max_runtime: Duration::ZERO, + }; + run_durable_runner( + config, + capture_bootstrap_ticket()?.ok_or("missing ticket")?, + ScriptedExecutor::new(script), + )?; + Ok(()) +} diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/durable/transport.rs b/packages/paperclip-runner/runner/crates/runner-core/src/durable/transport.rs index ae2be51ddfb..3631e7fc815 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/durable/transport.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/durable/transport.rs @@ -1014,6 +1014,7 @@ impl AuthenticatedTransport { "protocolMin": PROTOCOL_MIN_VERSION, "protocolMax": PROTOCOL_VERSION, "warmTransitionVersion": 1, + "recoveryCursorVersion": 1, "runnerInstanceId": state.runner_instance_id, "environmentLeaseId": state.environment_lease_id, "runId": state.run_id, @@ -1200,6 +1201,10 @@ struct AuthChallenge { warm_transition_version: Option, #[serde(default)] warm_transition_id: Option, + #[serde(default)] + recovery_cursor_version: Option, + #[serde(default)] + recovery_acked_source_seq: Option, } fn validate_challenge( @@ -1275,6 +1280,15 @@ fn validate_challenge( "warm transition capability or receipt was not authenticated", )); } + if (challenge.recovery_cursor_version.is_some() + || challenge.recovery_acked_source_seq.is_some()) + && (challenge.recovery_cursor_version != Some(1) + || challenge.recovery_acked_source_seq != Some(state.acked_source_seq)) + { + return Err(DurableRunnerError::invalid( + "recovery cursor does not match the runner's durable acknowledgement", + )); + } match expected_lease { Some(lease) if challenge.credential_lease_id.as_deref() == Some(lease.lease_id.as_str()) @@ -1319,6 +1333,12 @@ fn challenge_signing_bytes(challenge: &AuthChallenge) -> Vec { if let Some(id) = &challenge.warm_transition_id { body["warmTransitionId"] = json!(id); } + if let Some(version) = challenge.recovery_cursor_version { + body["recoveryCursorVersion"] = json!(version); + } + if let Some(cursor) = challenge.recovery_acked_source_seq { + body["recoveryAckedSourceSeq"] = json!(cursor); + } canonical_json(&body).into_bytes() } @@ -1806,6 +1826,8 @@ mod tests { server_proof: String::new(), warm_transition_version: None, warm_transition_id: None, + recovery_cursor_version: Some(1), + recovery_acked_source_seq: payload["resume"]["ackedSourceSeq"].as_u64(), }; let signing = challenge_signing_bytes(&challenge); challenge.server_proof = hex_encode(&hmac_domain( @@ -1838,6 +1860,8 @@ mod tests { "credentialLeaseId": &challenge.credential_lease_id, "revocationEpoch": challenge.revocation_epoch, "serverProof": &challenge.server_proof, + "recoveryCursorVersion": challenge.recovery_cursor_version, + "recoveryAckedSourceSeq": challenge.recovery_acked_source_seq, }, }), config.max_frame_bytes, @@ -2505,6 +2529,8 @@ mod tests { server_proof: String::new(), warm_transition_version: None, warm_transition_id: None, + recovery_cursor_version: None, + recovery_acked_source_seq: None, }; let signing = challenge_signing_bytes(&challenge); challenge.server_proof = hex_encode(&hmac_domain( diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/generated_acpx_profiles.rs b/packages/paperclip-runner/runner/crates/runner-core/src/generated_acpx_profiles.rs index 4bae7fe1489..5e9a354c50c 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/generated_acpx_profiles.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/generated_acpx_profiles.rs @@ -47,7 +47,7 @@ pub(crate) fn acpx_release_profile(agent: &str) -> Option { agent_runtime_package: None, agent_runtime_version: None, command_digest: - "sha256:0cddc219af25b741b7138f38faa7514dd5ee3427b1d3168e8a4bfbe5180983e8", + "sha256:1ce8bb0e00df81fa9eb71cff5063ce769637f5db7b3aa0ada9ade651db089c63", requires_provider_policy: true, }, "claude" => AcpxReleaseProfile { diff --git a/packages/paperclip-runner/scripts/copilot-profile-identity.mjs b/packages/paperclip-runner/scripts/copilot-profile-identity.mjs index bc511760487..c7d9b1143b7 100644 --- a/packages/paperclip-runner/scripts/copilot-profile-identity.mjs +++ b/packages/paperclip-runner/scripts/copilot-profile-identity.mjs @@ -7,9 +7,9 @@ import { fileURLToPath } from "node:url"; import { resolve, dirname } from "node:path"; const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); -const fixture = resolve(root, "test/fixtures/copilot-profile-v66-identity.json"); +const fixture = resolve(root, "test/fixtures/copilot-profile-v68-identity.json"); const historical = JSON.parse(readFileSync(resolve(root, "test/fixtures/copilot-profile-v12-identity.json"), "utf8")); -const declaration = { ...historical.declaration, agentProfileVersion: 66 }; +const declaration = { ...historical.declaration, agentProfileVersion: 68 }; delete declaration.permissionIdentitySourceSha256; const sources = { displayEventSourceSha256: "src/drivers/acpx/copilot-events.ts", diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts index b2a88f8d86b..f3f8473ddb3 100644 --- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts +++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts @@ -6,6 +6,7 @@ import { } from "node:crypto"; import { chmodSync, + existsSync, mkdtempSync, readFileSync, rmSync, @@ -29,6 +30,8 @@ import { validatePrpEvent } from "../protocol/replay-contract.js"; import { digestPaperclipSemanticContent } from "../semantic-tools/receipts.js"; import { DurablePrpControlPlane, + decodeDurablePrpCheckpoint, + type DurablePrpCheckpoint, durableRecoveryInternals, SemanticToolNotDispatchedError, inspectWarmRunTransition, @@ -49,6 +52,605 @@ const identity: DurableRecoveryIdentity = { const expectedRunnerVersion = "0.3.0"; const expectedRunnerDigest = `sha256:${"a".repeat(64)}`; +it("continues externally persisted commands after losing the local controller directory", async () => { + const root = mkdtempSync(resolve(tmpdir(), "runner-external-store-")); + const options = { stateDirectory: resolve(root, "control-plane"), identity, expectedRunnerVersion, expectedRunnerDigest }; + const core = new DurablePrpControlPlane(options); + let checkpoint!: DurablePrpCheckpoint; + core.setCheckpointPersistence(async saved => { checkpoint = saved; }); + let restored: DurablePrpControlPlane | undefined; + try { + await core.start(); + const client = (await authenticate(core, core.issueBootstrapTicket()))!; + rmSync(options.stateDirectory, { recursive: true }); + core.queueCommand("session.snapshot", {}, "external-only-command", true); + expect(await receiveSecure(client)).toMatchObject({ kind: "command", payload: { commandId: "external-only-command" } }); + await core.flushCheckpoint(); + expect(existsSync(options.stateDirectory)).toBe(false); + client.socket.destroy(); + await core.stop(); + checkpoint.restoreSessionDirectory(root); + restored = new DurablePrpControlPlane(options); + expect(restored.getCommand("external-only-command")).toMatchObject({ type: "session.snapshot" }); + } finally { + await restored?.stop(); + await core.stop(); + rmSync(root, { recursive: true, force: true }); + } +}); + +it("requires the external checkpoint instead of accepting a stale local controller file", async () => { + const root = mkdtempSync(resolve(tmpdir(), "runner-external-store-restart-")); + const options = { stateDirectory: root, identity, expectedRunnerVersion, expectedRunnerDigest }; + const core = new DurablePrpControlPlane(options); + core.setCheckpointPersistence(async () => {}); + try { + core.queueCommand("session.snapshot", {}, "external-command"); + await core.flushCheckpoint(); + await core.stop(); + expect(() => new DurablePrpControlPlane(options)).toThrow(/external checkpoint/i); + } finally { + await core.stop(); + rmSync(root, { recursive: true, force: true }); + } +}); + +it("restores authentication and commands from opaque checkpoint records", async () => { + const root = mkdtempSync(resolve(tmpdir(), "runner-checkpoint-codec-")); + const options = { stateDirectory: resolve(root, "control-plane"), identity, expectedRunnerVersion, expectedRunnerDigest }; + const first = new DurablePrpControlPlane(options); + let checkpoint!: DurablePrpCheckpoint; + first.setCheckpointPersistence(async saved => { checkpoint = saved; }); + let restored: DurablePrpControlPlane | undefined; + try { + const ticket = first.issueBootstrapTicket(); + first.queueCommand("session.snapshot", {}, "maintenance"); + first.queueCommand("turn.start", {}, "execution"); + first.persistPreparedLaunchConfiguration(identity, "retained-configuration"); + await first.flushCheckpoint(); + expect(checkpoint.workCommandIds).toEqual(["execution"]); + await first.stop(); + rmSync(options.stateDirectory, { recursive: true }); + decodeDurablePrpCheckpoint(new Map(checkpoint.records)).restoreSessionDirectory(root); + restored = new DurablePrpControlPlane(options); + expect(restored.preparedLaunchConfigurationId(identity)).toBe("retained-configuration"); + expect(restored.preparedLaunchConfigurationId({ ...identity, runId: "unrelated" })).toBeUndefined(); + expect(() => restored!.persistPreparedLaunchConfiguration(identity, "substituted-configuration")).toThrow("configuration_changed"); + expect(restored.getCommand("execution")).toMatchObject({ type: "turn.start" }); + await restored.start(); + const client = (await authenticate(restored, ticket))!; + expect(client.welcome.kind).toBe("welcome"); + client.socket.destroy(); + } finally { + await restored?.stop(); + await first.stop(); + rmSync(root, { recursive: true, force: true }); + } +}); + +it.each(["missing record", "invalid state", "wrong sequence", "invalid preparation identity", "duplicate preparation", "malformed header", "malformed cursor"])("rejects %s at checkpoint decoding", async fault => { + const root = mkdtempSync(resolve(tmpdir(), "runner-checkpoint-invalid-")); + const core = new DurablePrpControlPlane({ stateDirectory: root, identity, expectedRunnerVersion, expectedRunnerDigest }); + let checkpoint!: DurablePrpCheckpoint; + core.setCheckpointPersistence(async saved => { checkpoint = saved; }); + try { + core.queueCommand("session.snapshot", {}, "saved-command"); + await core.flushCheckpoint(); + const records = new Map(checkpoint.records); + if (fault === "missing record") records.delete("command:1"); + if (fault === "malformed header") records.set("header", "{"); + if (fault === "malformed cursor") records.set("cursor", "{"); + if (fault === "invalid state") { + const cursor = JSON.parse(records.get("cursor")!); + cursor.ackedSourceSeq = -1; + records.set("cursor", JSON.stringify(cursor)); + } + if (fault === "wrong sequence") { + const command = JSON.parse(records.get("command:1")!); + command.controllerSeq = 2; + records.set("command:1", JSON.stringify(command)); + } + if (fault === "invalid preparation identity" || fault === "duplicate preparation") { + const header = JSON.parse(records.get("header")!); + const binding = { identity, configurationId: "retained-configuration" }; + header.state.preparedLaunchConfigurations = fault === "duplicate preparation" ? [binding, binding] + : [{ ...binding, identity: { runId: identity.runId } }]; + records.set("header", JSON.stringify(header)); + } + expect(() => decodeDurablePrpCheckpoint(records)).toThrow(/native_runner_checkpoint_/); + } finally { + await core.stop(); + rmSync(root, { recursive: true, force: true }); + } +}); + +it("restores the previous checkpoint format and upgrades it on the next capture", async () => { + const root = mkdtempSync(resolve(tmpdir(), "runner-checkpoint-v1-")); + const core = new DurablePrpControlPlane({ stateDirectory: root, identity, expectedRunnerVersion, expectedRunnerDigest }); + let checkpoint!: DurablePrpCheckpoint; + core.setCheckpointPersistence(async saved => { checkpoint = saved; }); + try { + core.queueCommand("session.snapshot", {}, "old-command"); + await core.flushCheckpoint(); + const records = new Map(checkpoint.records); + const header = JSON.parse(records.get("header")!); + const cursor = JSON.parse(records.get("cursor")!); + records.delete("cursor"); + records.set("header", JSON.stringify({ ...header, schema: "paperclip.runner.controller-checkpoint.v1", + state: { ...header.state, ackedSourceSeq: cursor.ackedSourceSeq, replayDeliveries: cursor.replayDeliveries }, + commands: ["command:1"], events: [] })); + const upgraded = decodeDurablePrpCheckpoint(records); + expect(upgraded.records.get("command:1")).toContain("old-command"); + expect(JSON.parse(upgraded.records.get("header")!).schema).toBe("paperclip.runner.controller-checkpoint.v2"); + expect(upgraded.reset).toBe(true); + } finally { + await core.stop(); + rmSync(root, { recursive: true, force: true }); + } +}); + +function ordinaryOutput(sourceSeq: number): Record { + const envelope = semanticInputEvent(sourceSeq); + Object.assign(envelope.payload as Record, { + sourceEventId: `output-${sourceSeq}`, eventType: "item.delta", priority: 2, + payload: { text: "streaming output" }, + }); + return envelope; +} + +it("streams output without checkpoint writes and recovers its authenticated cursor on empty disk", async () => { + const root = mkdtempSync(resolve(tmpdir(), "runner-control-checkpoint-")); + const options = { stateDirectory: resolve(root, "control-plane"), identity, expectedRunnerVersion, expectedRunnerDigest }; + const core = new DurablePrpControlPlane(options); + const snapshots: DurablePrpCheckpoint[] = []; + core.setCheckpointPersistence(async saved => { snapshots.push(saved); }); + let restored: DurablePrpControlPlane | undefined; + try { + await core.start(); + const client = (await authenticate(core, core.issueBootstrapTicket()))!; + const before = snapshots.length; + const localState = readFileSync(core.store.path, "utf8"); + for (let seq = 1; seq <= 64; seq++) { + sendSecure(client, ordinaryOutput(seq)); + expect(await receiveSecure(client)).toMatchObject({ kind: "ack", payload: { ackedSourceSeq: seq } }); + } + await core.flushCheckpoint(); + expect(snapshots).toHaveLength(before); + expect(readFileSync(core.store.path, "utf8")).toBe(localState); + const saved = snapshots.at(-1)!; + expect(saved.recoveryEvidence).toEqual({ runnerInstanceId: identity.runnerInstanceId, + normalizedSessionId: identity.normalizedSessionId, authenticatedConnectionCount: 1, highestRecordedSourceSeq: 0 }); + expect(decodeDurablePrpCheckpoint(saved.records).recoveryEvidence).toEqual(saved.recoveryEvidence); + expect(JSON.parse(saved.records.get("cursor")!).ackedSourceSeq).toBe(0); + expect([...saved.records.keys()].filter(key => key.startsWith("event:"))).toEqual([]); + const token = client.leaseToken!; + client.socket.destroy(); + await core.stop(); + await core.drainPendingConnectionProcessing(); + rmSync(options.stateDirectory, { recursive: true }); + decodeDurablePrpCheckpoint(saved.records).restoreSessionDirectory(root); + restored = new DurablePrpControlPlane(options); + restored.setCheckpointPersistence(async saved => { snapshots.push(saved); }); + await restored.start(); + const replacement = (await authenticate(restored, token, identity, expectedRunnerDigest, undefined, false, 1, 64))!; + expect(replacement.welcome.payload).toMatchObject({ ackedSourceSeq: 64 }); + expect(restored.recoveredSourceSeq).toBe(64); + expect(snapshots.at(-1)!.recoveryEvidence).toMatchObject({ authenticatedConnectionCount: 2, highestRecordedSourceSeq: 64 }); + sendSecure(replacement, ordinaryOutput(65)); + expect(await receiveSecure(replacement)).toMatchObject({ kind: "ack", payload: { ackedSourceSeq: 65 } }); + // Commands remain an immediate durable barrier, without copying text. + const count = snapshots.length; + restored.queueCommand("session.snapshot", {}, "after-reconnect"); + await restored.flushCheckpoint(); + expect(snapshots).toHaveLength(count + 1); + expect([...snapshots.at(-1)!.records.values()].join("\n")).not.toContain("streaming output"); + replacement.socket.destroy(); + } finally { + await restored?.stop(); + await core.stop(); + rmSync(root, { recursive: true, force: true }); + } +}); + +it("excludes streamed output from checkpoints even when the live window is full", async () => { + const root = mkdtempSync(resolve(tmpdir(), "runner-checkpoint-window-")); + const core = new DurablePrpControlPlane({ stateDirectory: root, identity, expectedRunnerVersion, expectedRunnerDigest }); + core.store.state.committedEvents = Array.from({ length: 4096 }, (_, index) => ({ + sourceSeq: index + 1, sourceEventId: `output-${index + 1}`, eventType: "item.delta", priority: 2 as const, + envelope: ordinaryOutput(index + 1), deliveryCount: 1, logicalEffectCount: 1 as const, + })); + const anchor = core.store.state.committedEvents[0]!; + anchor.eventType = "turn.started"; + (anchor.envelope.payload as Record).eventType = "turn.started"; + core.store.state.ackedSourceSeq = 4096; + core.store.save(); + const snapshots: DurablePrpCheckpoint[] = []; + core.setCheckpointPersistence(async saved => { snapshots.push(saved); }); + try { + await core.start(); + const client = (await authenticate(core, core.issueBootstrapTicket()))!; + const before = snapshots.length; + sendSecure(client, ordinaryOutput(4097)); + expect(await receiveSecure(client)).toMatchObject({ kind: "ack", payload: { ackedSourceSeq: 4097 } }); + expect(snapshots).toHaveLength(before); + const checkpoint = snapshots.at(-1)!; + expect(core.store.state.committedEvents[0]!.eventType).toBe("turn.started"); + expect(checkpoint.records.size).toBe(3); // authority + cursor + turn anchor, zero output rows. + expect(decodeDurablePrpCheckpoint(checkpoint.records).records).toEqual(checkpoint.records); + client.socket.destroy(); + } finally { + await core.stop(); + await core.drainPendingConnectionProcessing(); + rmSync(root, { recursive: true, force: true }); + } +}); + +it.each(["unsupported", "invalid cursor", "invalid proof"])("does not advance recovery from an unauthenticated cursor: %s", async fault => { + const root = mkdtempSync(resolve(tmpdir(), "runner-recovery-cursor-auth-")); + const core = new DurablePrpControlPlane({ stateDirectory: root, identity, expectedRunnerVersion, expectedRunnerDigest }); + core.setCheckpointPersistence(async () => {}); + try { + await core.start(); + const material = credentialMaterial(core.issueBootstrapTicket()); + const { socket, reader } = await upgradeSocket(core.connectUrl); + const hello = authHello(material.credentialId); + const payload = hello.payload as Record; + payload.resume = { ackedSourceSeq: 50, nextSourceEventSeq: 51 }; + if (fault === "unsupported") delete payload.recoveryCursorVersion; + if (fault === "invalid cursor") payload.resume = { ackedSourceSeq: 50, nextSourceEventSeq: 50 }; + sendMaskedJson(socket, hello); + if (fault === "invalid proof") { + const challenge = (await reader.next())!.payload as Record; + expect(challenge.recoveryAckedSourceSeq).toBe(50); + expect(core.store.state.ackedSourceSeq).toBe(0); + sendMaskedJson(socket, { protocol: "paperclip.runner", version: 1, kind: "auth_response", payload: { + credentialId: material.credentialId, clientNonce: challenge.clientNonce, + serverNonce: challenge.serverNonce, clientProof: "0".repeat(64), + } }); + } + expect(await reader.next()).toBeNull(); + expect(core.store.state.ackedSourceSeq).toBe(0); + expect(core.recoveredSourceSeq).toBe(0); + socket.destroy(); + } finally { + await core.stop(); + rmSync(root, { recursive: true, force: true }); + } +}); + +it("persists authentication and commands before the runner can observe them", async () => { + const root = mkdtempSync(resolve(tmpdir(), "runner-external-journal-")); + const core = new DurablePrpControlPlane({ stateDirectory: root, identity, expectedRunnerVersion, expectedRunnerDigest }); + let release!: () => void; + let pending: Promise = Promise.resolve(); + const snapshots: DurablePrpCheckpoint[] = []; + core.setCheckpointPersistence(async snapshot => { await pending; snapshots.push(snapshot); }); + try { + await core.start(); + const client = (await authenticate(core, core.issueBootstrapTicket()))!; + expect(snapshots.at(-1)).toMatchObject({ runnerVersion: expectedRunnerVersion, runId: identity.runId }); + expect(snapshots.at(-1)!.records.get("header")).toContain(client.welcome.connectionLeaseId); + pending = new Promise(resolveCommit => { release = resolveCommit; }); + core.queueCommand("session.snapshot", {}, "checkpoint-command", true); + let received = false; + const command = receiveSecure(client).then(value => { received = true; return value; }); + await new Promise(resolveTurn => setImmediate(resolveTurn)); + expect(received).toBe(false); + release(); + expect(await command).toMatchObject({ kind: "command", payload: { commandId: "checkpoint-command" } }); + expect([...snapshots.at(-1)!.records.values()].join("\n")).toContain("checkpoint-command"); + client.socket.destroy(); + } finally { + release?.(); + await core.stop(); + rmSync(root, { recursive: true, force: true }); + } +}); + +it("does not dispatch a semantic effect when external journal persistence fails", async () => { + const root = mkdtempSync(resolve(tmpdir(), "runner-external-journal-failure-")); + const dispatch = vi.fn(async () => ({ result: "written" })); + const core = new DurablePrpControlPlane({ stateDirectory: root, identity, expectedRunnerVersion, + expectedRunnerDigest, onSemanticToolInput: dispatch }); + core.setCheckpointPersistence(async snapshot => { + if (JSON.parse(snapshot.records.get("cursor")!).ackedSourceSeq > 0) throw new Error("database unavailable"); + }); + try { + await core.start(); + const client = (await authenticate(core, core.issueBootstrapTicket()))!; + sendSecure(client, semanticInputEvent()); + await vi.waitFor(() => expect(core.activeRunnerConnectionCount()).toBe(0)); + expect(dispatch).not.toHaveBeenCalled(); + await expect(core.flushCheckpoint()).rejects.toThrow("database unavailable"); + client.socket.destroy(); + } finally { + await core.stop(); + rmSync(root, { recursive: true, force: true }); + } +}); + +it.each([false, true])("recovers checkpoint persistence and the same connection lease after a failed write (committed=%s)", async committed => { + const root = mkdtempSync(resolve(tmpdir(), "runner-checkpoint-retry-")); + const core = new DurablePrpControlPlane({ stateDirectory: root, identity, expectedRunnerVersion, expectedRunnerDigest }); + const stored = new Map(); + const attempts: DurablePrpCheckpoint[] = []; + let fail = false; + core.setCheckpointPersistence(async snapshot => { + attempts.push(snapshot); + if (!fail || committed) { + if (snapshot.reset) stored.clear(); + for (const key of snapshot.removedRecords) stored.delete(key); + for (const [key, value] of snapshot.changedRecords) stored.set(key, value); + } + if (fail) throw new Error("checkpoint response lost"); + }); + try { + await core.start(); + const client = (await authenticate(core, core.issueBootstrapTicket()))!; + fail = true; + core.queueCommand("session.snapshot", {}, "retry-command", true); + await expect(core.flushCheckpoint()).rejects.toThrow("checkpoint response lost"); + expect(await receiveSecure(client)).toBeNull(); + fail = false; + await core.flushCheckpoint(); + expect(attempts.at(-1)!.reset).toBe(true); + expect(stored).toEqual(attempts.at(-1)!.records); + const settledAttempts = attempts.length; + await core.flushCheckpoint(); + expect(attempts).toHaveLength(settledAttempts); + const resumed = (await authenticate(core, client.leaseToken!))!; + expect(resumed).not.toBeNull(); + expect(resumed.welcome.payload).toMatchObject({ pendingCommands: [{ commandId: "retry-command" }] }); + expect(attempts.at(-1)!.reset).toBe(false); + expect(stored).toEqual(attempts.at(-1)!.records); + resumed.socket.destroy(); + client.socket.destroy(); + } finally { + await core.stop(); + rmSync(root, { recursive: true, force: true }); + } +}); + +it("rebuilds a queued checkpoint after the preceding write fails", async () => { + const root = mkdtempSync(resolve(tmpdir(), "runner-checkpoint-queued-retry-")); + const core = new DurablePrpControlPlane({ stateDirectory: root, identity, expectedRunnerVersion, expectedRunnerDigest }); + let reject!: (error: Error) => void; + let pending: Promise = Promise.resolve(); + const snapshots: DurablePrpCheckpoint[] = []; + core.setCheckpointPersistence(async snapshot => { await pending; snapshots.push(snapshot); }); + try { + await core.flushCheckpoint(); + pending = new Promise((_resolve, rejectWrite) => { reject = rejectWrite; }); + core.queueCommand("session.snapshot", {}, "first-command"); + const first = core.flushCheckpoint(); + await new Promise(resolveTurn => setImmediate(resolveTurn)); + core.queueCommand("session.snapshot", {}, "second-command"); + const second = core.flushCheckpoint(); + const failed = expect(first).rejects.toThrow("first write failed"); + reject(new Error("first write failed")); + pending = Promise.resolve(); + await failed; + await second; + expect(snapshots).toHaveLength(2); + expect(snapshots.at(-1)).toMatchObject({ reset: true }); + expect([...snapshots.at(-1)!.changedRecords.values()].join("\n")).toContain("first-command"); + expect([...snapshots.at(-1)!.changedRecords.values()].join("\n")).toContain("second-command"); + await core.flushCheckpoint(); + expect(snapshots).toHaveLength(2); + } finally { + reject?.(new Error("cleanup")); + await core.stop(); + rmSync(root, { recursive: true, force: true }); + } +}); + +it.each([true, false])("reconciles a saved semantic input only from an authoritative outcome (known=%s)", async known => { + const root = mkdtempSync(resolve(tmpdir(), "runner-recovered-semantic-")); + const dispatch = vi.fn(() => new Promise<{ result: unknown }>(() => undefined)); + const options = { stateDirectory: root, identity, expectedRunnerVersion, expectedRunnerDigest }; + const first = new DurablePrpControlPlane({ ...options, onSemanticToolInput: dispatch }); + let recovered: DurablePrpControlPlane | undefined; + try { + await first.start(); + const client = (await authenticate(first, first.issueBootstrapTicket()))!; + sendSecure(client, semanticInputEvent()); + expect((await receiveSecure(client))!.kind).toBe("ack"); + expect(dispatch).toHaveBeenCalledTimes(1); + await first.stop(); + const recovery = vi.fn(async () => known ? { result: { savedReceipt: "exact-effect" } } : null); + const redispatch = vi.fn(async () => ({ result: "must not execute again" })); + recovered = new DurablePrpControlPlane({ ...options, onSemanticToolInput: redispatch, + recoverSemanticToolInput: recovery }); + await recovered.start(); + const next = (await authenticate(recovered, client.leaseToken!))!; + await recovered.recoverPendingSemanticInputs(); + expect(recovery).toHaveBeenCalledTimes(1); + expect(redispatch).not.toHaveBeenCalled(); + if (known) { + expect(await receiveSecure(next)).toMatchObject({ kind: "command", payload: { + type: "semantic_tool.result", payload: { result: { savedReceipt: "exact-effect" } }, + } }); + } else { + expect(recovered.store.state.commands).toHaveLength(0); + expect(recovered.semanticToolResultsSettled()).toBe(false); + } + next.socket.destroy(); + } finally { + await recovered?.stop(); + await first.stop(); + rmSync(root, { recursive: true, force: true }); + } +}); + +it("resolves a saved semantic input only once across overlapping recovery calls", async () => { + const root = mkdtempSync(resolve(tmpdir(), "runner-saved-semantic-race-")); + const options = { stateDirectory: root, identity, expectedRunnerVersion, expectedRunnerDigest }; + const original = new DurablePrpControlPlane({ ...options, + onSemanticToolInput: () => new Promise<{ result: unknown }>(() => undefined) }); + let resolveOutcome!: (value: { result: unknown }) => void; + const outcome = new Promise<{ result: unknown }>(resolveResult => { resolveOutcome = resolveResult; }); + const recover = vi.fn(async () => outcome); + let replacement: DurablePrpControlPlane | undefined; + const pending: Promise[] = []; + try { + await original.start(); + const client = (await authenticate(original, original.issueBootstrapTicket()))!; + sendSecure(client, semanticInputEvent()); + expect((await receiveSecure(client))!.kind).toBe("ack"); + client.socket.destroy(); + await original.stop(); + replacement = new DurablePrpControlPlane({ ...options, recoverSemanticToolInput: recover }); + pending.push(replacement.recoverPendingSemanticInputs()); + await vi.waitFor(() => expect(recover).toHaveBeenCalledOnce()); + pending.push(replacement.recoverPendingSemanticInputs()); + await new Promise(resolveTurn => setImmediate(resolveTurn)); + expect(recover).toHaveBeenCalledOnce(); + resolveOutcome({ result: { receipt: "one-authoritative-outcome" } }); + await Promise.all(pending); + await replacement.recoverPendingSemanticInputs(); + expect(recover).toHaveBeenCalledOnce(); + expect(replacement.store.state.commands.filter(command => command.type === "semantic_tool.result")).toHaveLength(1); + } finally { + resolveOutcome({ result: { receipt: "one-authoritative-outcome" } }); + await Promise.allSettled(pending); + await replacement?.stop(); + await original.stop(); + rmSync(root, { recursive: true, force: true }); + } +}); + +it.each([false, true])("dispatches a fresh semantic input once when recovery overlaps its checkpoint (saved outcome: %s)", async known => { + const root = mkdtempSync(resolve(tmpdir(), "runner-fresh-semantic-race-")); + let releaseWrite!: () => void; + const writeBarrier = new Promise(resolveWrite => { releaseWrite = resolveWrite; }); + let releaseRecovery!: () => void; + const recoveryBarrier = new Promise(resolveRecovery => { releaseRecovery = resolveRecovery; }); + let releaseDispatch!: () => void; + const dispatchBarrier = new Promise(resolveDispatch => { releaseDispatch = resolveDispatch; }); + const recover = vi.fn(async () => { await recoveryBarrier; return known ? { result: "stale recovery outcome" } : null; }); + const dispatch = vi.fn(async () => { await dispatchBarrier; return { result: "original live outcome" }; }); + const core = new DurablePrpControlPlane({ stateDirectory: root, identity, expectedRunnerVersion, + expectedRunnerDigest, onSemanticToolInput: dispatch, recoverSemanticToolInput: recover }); + let writeHeld = false; + core.setCheckpointPersistence(async saved => { + if (JSON.parse(saved.records.get("cursor")!).ackedSourceSeq > 0) { writeHeld = true; await writeBarrier; } + }); + const pending: Promise[] = []; + let client: Awaited>; + try { + await core.start(); + client = (await authenticate(core, core.issueBootstrapTicket()))!; + sendSecure(client, semanticInputEvent()); + await vi.waitFor(() => expect(writeHeld).toBe(true)); + expect(core.store.state.committedEvents).toHaveLength(0); + pending.push(core.recoverPendingSemanticInputs()); + await new Promise(resolveTurn => setImmediate(resolveTurn)); + expect(dispatch).not.toHaveBeenCalled(); // The durability barrier still applies. + releaseWrite(); + expect((await receiveSecure(client))!.kind).toBe("ack"); + releaseRecovery(); + await Promise.all(pending); + expect(dispatch).toHaveBeenCalledOnce(); + expect(recover).not.toHaveBeenCalled(); + await core.recoverPendingSemanticInputs(); + expect(recover).not.toHaveBeenCalled(); + releaseDispatch(); + expect(await receiveSecure(client)).toMatchObject({ kind: "command", payload: { + type: "semantic_tool.result", payload: { result: "original live outcome" }, + } }); + expect(core.store.state.commands.filter(command => command.type === "semantic_tool.result")).toHaveLength(1); + } finally { + releaseWrite(); releaseRecovery(); releaseDispatch(); + await Promise.allSettled(pending); + client?.socket.destroy(); + await core.stop(); + rmSync(root, { recursive: true, force: true }); + } +}); +it.each(["turn.completed", "run.result.proposed", "tool.execution.progressed", "command_result"] as const)( + "publishes %s to transport readers only after its external checkpoint commits", async mode => { + for (const rejectWrite of [false, true]) { + const root = mkdtempSync(resolve(tmpdir(), "runner-published-checkpoint-")); + const core = new DurablePrpControlPlane({ stateDirectory: root, identity, expectedRunnerVersion, expectedRunnerDigest }); + let release!: () => void; + const gate = new Promise(resolveGate => { release = resolveGate; }); + let entered = false; + let hold = false; + let lastSaved!: DurablePrpCheckpoint; + core.setCheckpointPersistence(async checkpoint => { + if (hold) { + entered = true; + await gate; + if (rejectWrite) throw new Error("controlled checkpoint rejection"); + } + lastSaved = checkpoint; + }); + let client: AuthenticatedClient | null = null; + let pump: ReturnType | undefined; + let polls = 0; + const observed = vi.fn(); + try { + await core.start(); + client = (await authenticate(core, core.issueBootstrapTicket()))!; + core.queueCommand("session.snapshot", {}, "publication-command", true); + expect(await receiveSecure(client)).toMatchObject({ kind: "command" }); + await core.flushCheckpoint(); + hold = true; + pump = setInterval(() => { + polls++; + const visible = mode === "command_result" + ? core.getCommand("publication-command")?.status === "completed" + || core.store.state.commands.some(command => command.commandId === "publication-command" && command.status === "completed") + : core.store.state.committedEvents.some(event => event.eventType === mode); + if (visible && observed.mock.calls.length === 0) observed(); + }, 5); + if (mode === "command_result") { + sendSecure(client, { protocol: "paperclip.runner", version: 1, kind: "command_result", + payload: { commandId: "publication-command", commandType: "session.snapshot", controllerSeq: 1, + status: "completed", result: { snapshot: true } } }); + } else { + const event = semanticInputEvent(); + const payload = event.payload as Record; + payload.eventType = mode; + payload.priority = mode === "tool.execution.progressed" ? 1 : 0; + payload.payload = mode === "run.result.proposed" ? { + schema: "paperclip.run_result.v1", reportedWorkDisposition: "done", summary: "complete", + completionClaim: { contractRevision: "test-v1", objectiveSatisfied: true, criteria: [], remainingWork: [] }, + evidence: [], verification: [], attentionRequests: [], artifacts: [], + } : mode === "tool.execution.progressed" ? { + schema: "paperclip.tool.execution.v1", executionId: "publication-progress", transport: "process", operation: "execute", + status: "running", output: "working", outputBytes: 7, outputTruncated: false, outputDigest: null, + } : { status: "completed" }; + expect(validatePrpEvent(payload)).toMatchObject({ ok: true }); + sendSecure(client, event); + } + await vi.waitFor(() => expect(entered).toBe(true)); + await vi.waitFor(() => expect(polls).toBeGreaterThan(2)); + expect(observed).not.toHaveBeenCalled(); + release(); + if (rejectWrite) { + await expect(core.flushCheckpoint()).rejects.toThrow("controlled checkpoint rejection"); + expect(observed).not.toHaveBeenCalled(); + hold = false; + await core.flushCheckpoint(); + } else await core.flushCheckpoint(); + await vi.waitFor(() => expect(observed).toHaveBeenCalledOnce()); + if (mode === "command_result") { + expect(JSON.parse(lastSaved.records.get("command:1")!).status).toBe("completed"); + } else expect(lastSaved.records.has("event:1")).toBe(true); + } finally { + hold = false; + release(); + if (pump) clearInterval(pump); + client?.socket.destroy(); + await core.stop(); + await core.drainPendingConnectionProcessing(); + rmSync(root, { recursive: true, force: true }); + } + } + }, +); + it.each(["event", "progress", "lifecycle", "lifecycle_priority_one", "usage", "credentials", "newer_command", "command_burst", "command_failure", "authority_conflict", "replacement", "write_failure", "usage_write_failure", "drain", "usage_drain", "semantic_replacement"] as const)( "keeps HTTP responsive during a durable event write and preserves %s", async (mode) => { if (mode === "command_failure" && process.platform === "win32") return; @@ -606,6 +1208,26 @@ it.skipIf(process.platform === "win32")( }, ); +it("retains process birth identity published after an external launch returns", async () => { + let startedAt: string | undefined; + const birth = "2026-10-26T12:00:00.000Z"; + const handle = spawnRunner({ + connection: { mode: "connect", connectUrl: "ws://127.0.0.1:43127" }, + stateDirectory: "/tmp/paperclip-runner-deferred-identity", identity, ticket: "bootstrap-ticket", + maxOutboxBytes: 256 * 1024, p0ReserveBytes: 64 * 1024, + runnerVersion: expectedRunnerVersion, runnerDigest: expectedRunnerDigest, + processLauncher: () => ({ + child: { pid: 42, exitCode: null, kill: () => true }, + completion: new Promise(() => {}), + get startedAt() { return startedAt; }, + }), + }); + expect(handle.startedAt).toBeUndefined(); + // The remote process marker arrives after the provider dispatch returns. + await Promise.resolve().then(() => { startedAt = birth; }); + expect(handle.startedAt).toBe(birth); +}); + it("does not release or replace a remote runner until its exit is confirmed", async () => { let exited = false; const launch = vi.fn(() => ({ @@ -1258,6 +1880,8 @@ function authHello( protocolMin: 1, protocolMax: 1, warmTransitionVersion: 1, + recoveryCursorVersion: 1, + resume: { ackedSourceSeq: 0, nextSourceEventSeq: 1 }, ...selectedIdentity, runnerVersion: expectedRunnerVersion, runnerDigest: expectedRunnerDigest, @@ -1273,11 +1897,13 @@ async function authenticate( warmTransitionId?: string, withoutWarmCapability = false, protocolMax = 1, + acknowledgedSourceSeq = 0, ): Promise { const { socket, reader } = await upgradeSocket(controlPlane.connectUrl); const material = credentialMaterial(token); const hello = authHello(material.credentialId, selectedIdentity); (hello.payload as Record).protocolMax = protocolMax; + (hello.payload as Record).resume = { ackedSourceSeq: acknowledgedSourceSeq, nextSourceEventSeq: acknowledgedSourceSeq + 1 }; (hello.payload as Record).runnerDigest = runnerDigest; if (warmTransitionId !== undefined) (hello.payload as Record).warmTransitionId = @@ -3194,12 +3820,15 @@ describe("DurablePrpControlPlane", () => { const root = mkdtempSync( resolve(tmpdir(), "paperclip-prp-attach-transition-"), ); + let externalCheckpoint: DurablePrpCheckpoint | undefined; let core = new DurablePrpControlPlane({ - stateDirectory: root, + stateDirectory: resolve(root, "control-plane"), identity, expectedRunnerVersion, expectedRunnerDigest, }); + const usesExternalPersistence = mode === "activate" || mode === "activate-v2"; + if (usesExternalPersistence) core.setCheckpointPersistence(async checkpoint => { externalCheckpoint = checkpoint; }); const nextIdentity = { ...identity, runId: "00000000-0000-4000-8000-000000000002", @@ -3214,6 +3843,9 @@ describe("DurablePrpControlPlane", () => { peer = await authenticateVersion(core, core.issueBootstrapTicket()); const oldToken = peer!.leaseToken!; core.persistRunAttachTemplate({ provider: { kind: "mock", run: "prior" } }); + core.persistPreparedLaunchConfiguration(identity, "prior-configuration"); + core.persistPreparedLaunchConfiguration(nextIdentity, "next-configuration"); + if (usesExternalPersistence) rmSync(resolve(root, "control-plane"), { recursive: true }); const command = core.queueCommand( "run.attach", { @@ -3229,7 +3861,7 @@ describe("DurablePrpControlPlane", () => { await expect(receiveSecure(peer!)).resolves.toMatchObject({ kind: "command", }); - const beforeResult = readFileSync(core.store.path, "utf8"); + const beforeResult = usesExternalPersistence ? null : readFileSync(core.store.path, "utf8"); const result = { commandId: command.commandId, commandType: command.type, @@ -3271,9 +3903,10 @@ describe("DurablePrpControlPlane", () => { peer!.socket.destroy(); await core.stop(); if (mode === "bootstrap-before-result") - writeFileSync(core.store.path, beforeResult); + writeFileSync(core.store.path, beforeResult!); + externalCheckpoint?.restoreSessionDirectory(root); core = new DurablePrpControlPlane({ - stateDirectory: root, + stateDirectory: resolve(root, "control-plane"), identity, expectedRunnerVersion, expectedRunnerDigest, @@ -3438,7 +4071,7 @@ describe("DurablePrpControlPlane", () => { expect(core.getCommand(command.commandId)?.status).toBe("pending"); await core.stop(); core = new DurablePrpControlPlane({ - stateDirectory: root, + stateDirectory: resolve(root, "control-plane"), identity, expectedRunnerVersion, expectedRunnerDigest, @@ -3562,6 +4195,10 @@ describe("DurablePrpControlPlane", () => { ); expect(successor).not.toBeNull(); expect(core.store.state.identity).toEqual(nextIdentity); + expect(core.preparedLaunchConfigurationId(nextIdentity)).toBe("next-configuration"); + core.persistPreparedLaunchConfiguration({ ...nextIdentity, runId: "third-run" }, "third-configuration"); + expect(core.preparedLaunchConfigurationId(identity)).toBeUndefined(); + expect(core.store.state.preparedLaunchConfigurations).toHaveLength(2); expect(core.store.state.runAttachTemplate).toEqual({ provider: { kind: "mock", run: "next" } }); expect( (successor!.welcome.payload as Record) @@ -3598,6 +4235,9 @@ describe("DurablePrpControlPlane", () => { await vi.waitFor(() => expect(core.store.state.warmTransition).toBeUndefined(), ); + if (mode === "activate" || mode === "activate-v2") { + expect(core.store.state.checkpointEventPolicy).toBe("control-v1"); + } expect(core.store.state.schema).toBe( "paperclip.runner.durable.control-plane-state.v1", ); @@ -3665,7 +4305,7 @@ describe("DurablePrpControlPlane", () => { successor!.socket.destroy(); await core.stop(); core = new DurablePrpControlPlane({ - stateDirectory: root, + stateDirectory: resolve(root, "control-plane"), identity: nextIdentity, expectedRunnerVersion, expectedRunnerDigest, diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts index 4f09db52dd9..7fb21c34431 100644 --- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts +++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts @@ -54,6 +54,7 @@ const protocolVersion = RUNNER_PRP_PROTOCOL_MAX; const secureFrameSchema = "paperclip.runner.secure-frame.v1"; const websocketGuid = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"; const coreStateSchema = "paperclip.runner.durable.control-plane-state.v1"; +const externalCoreStateSchema = "paperclip.runner.external-control-plane-state.v1"; const transitionCoreStateSchema = "paperclip.runner.durable.control-plane-state.warm-transition.v1"; const maxFrameBytes = 1024 * 1024; @@ -142,6 +143,8 @@ interface ConnectionLeaseRecord { interface StoredCoreState { schema: typeof coreStateSchema | typeof transitionCoreStateSchema; identity: DurableRecoveryIdentity; + /** External checkpoints contain control state, not a duplicate output log. */ + checkpointEventPolicy?: "control-v1"; warmTransition?: { receipt: DurableWarmRunTransition; phase: "awaiting_result" | "prepared" | "activated"; @@ -161,6 +164,8 @@ interface StoredCoreState { * reconstruct another warm attachment. */ runAttachTemplate?: Record | null; + /** Opaque environment configuration bindings; bounded across a warm handoff. */ + preparedLaunchConfigurations?: Array<{ identity: DurableRecoveryIdentity; configurationId: string }>; tickets: Record; leases: Record; commands: DurableRecoveryCoreCommand[]; @@ -226,6 +231,7 @@ interface PendingChallenge { warmTransitionVersion?: 1; warmTransitionId?: string; requestedIdentity?: DurableRecoveryIdentity; + acknowledgedSourceSeq?: number; } interface SecureChannel { @@ -269,6 +275,9 @@ export interface DurablePrpControlPlaneOptions { }) => Promise<{ readonly result: unknown; readonly isError?: boolean }>; /** Persist the canonical event before the runner receives its cumulative ACK. */ onCommittedEvent?: (event: PrpEvent) => Promise; + recoverSemanticToolInput?: ( + input: Parameters>[0], + ) => Promise<{ readonly result: unknown; readonly isError?: boolean } | null>; /** Stop this exact owner after a proven, authenticated permanent integrity fault. */ onProtocolIntegrityError?: ( error: NativeSessionProtocolIntegrityError, @@ -276,6 +285,144 @@ export interface DurablePrpControlPlaneOptions { connectionLeaseTtlMs?: number; } +export interface DurablePrpCheckpoint { + /** Read-only protocol evidence. Does not grant attachment or execution authority. */ + readonly recoveryEvidence: { + readonly runnerInstanceId: string; + readonly normalizedSessionId: string; + readonly authenticatedConnectionCount: number; + readonly highestRecordedSourceSeq: number; + }; + readonly runnerVersion: string; + readonly runnerDigest: string; + readonly runId: string; + readonly predecessor: { readonly runId: string; readonly normalizedSessionId: string } | null; + readonly records: ReadonlyMap; + /** Atomic write set relative to the last successful callback for this run. */ + readonly changedRecords: ReadonlyMap; + readonly removedRecords: readonly string[]; + /** First write for a run, or retry after a failed write, replaces the checkpoint atomically. */ + readonly reset: boolean; + readonly workCommandIds: readonly string[]; + restoreSessionDirectory(root: string): void; +} + +type CheckpointHeader = { + schema: "paperclip.runner.controller-checkpoint.v2"; + runnerVersion: string; + runnerDigest: string; + state: Omit; +}; + +function checkpointRunId(state: StoredCoreState): string { + if (state.warmTransition) return state.warmTransition.receipt.newIdentity.runId; + const attachment = state.commands.findLast(command => command.type === "run.attach"); + const next = attachment?.payload.paperclipNextAuthority as { identity?: { runId?: string } } | undefined; + return next?.identity?.runId ?? state.identity.runId; +} + +function isStreamingEvent(event: Pick): boolean { + return event.priority !== 0 && ["item.delta", "usage.reported"].includes(event.eventType); +} + +function evictionCandidate(state: StoredCoreState): number { + // Keep the latest identity/turn/result anchors through arbitrarily long text + // streams. Unsettled semantic inputs retain their existing stronger fence. + const anchors = new Set(["harness.ready", "session.started", "session.resumed", "turn.started", + "turn.completed", "turn.failed", "turn.cancelled", "turn.interrupted", "run.result.proposed", "usage.reported"]); + const latest = new Map(); + for (const event of state.committedEvents) if (anchors.has(event.eventType)) latest.set(event.eventType, event); + return state.committedEvents.findIndex(event => latest.get(event.eventType) !== event && !unsettledSemanticInput(event, state)); +} + +function checkpointChanges(records: ReadonlyMap, runId: string, previous?: DurablePrpCheckpoint) { + const reset = !previous || previous.runId !== runId; + return { + changedRecords: new Map([...records].filter(([key, value]) => reset || previous.records.get(key) !== value)), + removedRecords: reset ? [] : [...previous.records.keys()].filter(key => !records.has(key)), + reset, + }; +} + +function captureCheckpoint(state: StoredCoreState, runnerVersion: string, runnerDigest: string): DurablePrpCheckpoint { + const runId = checkpointRunId(state); + const records = new Map(); + const { commands, committedEvents, ackedSourceSeq: _cursor, replayDeliveries: _replays, ...metadata } = state; + const latestUsage = committedEvents.findLast(event => event.eventType === "usage.reported"); + const recoveryEvents = committedEvents.filter(event => !isStreamingEvent(event) || event === latestUsage); + for (const command of commands) records.set(`command:${command.controllerSeq}`, JSON.stringify(command)); + for (const event of recoveryEvents) records.set(`event:${event.sourceSeq}`, JSON.stringify(event)); + const header: CheckpointHeader = { schema: "paperclip.runner.controller-checkpoint.v2", + runnerVersion, runnerDigest, state: { ...metadata, checkpointEventPolicy: "control-v1" } }; + records.set("header", JSON.stringify(header)); + records.set("cursor", JSON.stringify({ ackedSourceSeq: state.ackedSourceSeq, replayDeliveries: state.replayDeliveries, + commandCount: commands.length, eventCount: recoveryEvents.length })); + const maintenance = new Set(["runner.drain", "runner.suspend", "session.snapshot", "session.close"]); + return { + runnerVersion, runnerDigest, runId, records, ...checkpointChanges(records, runId), + recoveryEvidence: { runnerInstanceId: state.identity.runnerInstanceId, normalizedSessionId: state.identity.normalizedSessionId, + authenticatedConnectionCount: state.connectionCount, + highestRecordedSourceSeq: state.committedEvents.reduce((highest, event) => Math.max(highest, event.sourceSeq), state.ackedSourceSeq) }, + predecessor: state.identity.runId === runId ? null + : { runId: state.identity.runId, normalizedSessionId: state.identity.normalizedSessionId }, + workCommandIds: commands.filter(command => !maintenance.has(command.type)).map(command => command.commandId), + restoreSessionDirectory(root) { + mkdirSync(root, { recursive: true, mode: 0o700 }); + verifyPrivateDirectory(root); + const directory = resolve(root, "control-plane"); + mkdirSync(directory, { recursive: true, mode: 0o700 }); + verifyPrivateDirectory(directory); + atomicPrivateWrite(resolve(directory, "control-plane-state.json"), JSON.stringify(checkpointState(records).state)); + }, + }; +} + +function checkpointState(records: ReadonlyMap) { + const header: unknown = JSON.parse(records.get("header") ?? "null"); + if (!isRecord(header) || !["paperclip.runner.controller-checkpoint.v1", "paperclip.runner.controller-checkpoint.v2"].includes(String(header.schema)) + || typeof header.runnerVersion !== "string" || !header.runnerVersion + || typeof header.runnerDigest !== "string" || !runnerDigestPattern.test(header.runnerDigest) + || !isRecord(header.state) || !exactIdentity(header.state.identity)) throw new Error("native_runner_checkpoint_invalid"); + const read = (key: string): unknown => { + const value = records.get(key); + if (value === undefined) throw new Error("native_runner_checkpoint_incomplete"); + return JSON.parse(value); + }; + const v2 = header.schema === "paperclip.runner.controller-checkpoint.v2"; + const keys = (prefix: string) => [...records.keys()].filter(key => key.startsWith(prefix)) + .sort((a, b) => Number(a.slice(prefix.length)) - Number(b.slice(prefix.length))); + const commandKeys = v2 ? keys("command:") : header.commands; + const eventKeys = v2 ? keys("event:") : header.events; + if (!Array.isArray(commandKeys) || commandKeys.length > maxCommands + || !Array.isArray(eventKeys) || eventKeys.length > maxCommittedEventWindow + || commandKeys.some(key => typeof key !== "string" || !/^command:\d+$/.test(key)) + || eventKeys.some(key => typeof key !== "string" || !/^event:\d+$/.test(key)) + || new Set(commandKeys).size !== commandKeys.length || new Set(eventKeys).size !== eventKeys.length) { + throw new Error("native_runner_checkpoint_invalid"); + } + const cursor = v2 ? read("cursor") : header.state; + if (!isRecord(cursor) || (v2 && (cursor.commandCount !== commandKeys.length || cursor.eventCount !== eventKeys.length + || records.size !== commandKeys.length + eventKeys.length + 2))) throw new Error("native_runner_checkpoint_incomplete"); + const state = { ...header.state, ackedSourceSeq: cursor.ackedSourceSeq, replayDeliveries: cursor.replayDeliveries, + commands: commandKeys.map(read), committedEvents: eventKeys.map(read) }; + if (!isStoredCoreState(state, header.state.identity)) throw new Error("native_runner_checkpoint_invalid"); + if (state.commands.some((command, index) => commandKeys[index] !== `command:${command.controllerSeq}`) + || state.committedEvents.some((event, index) => eventKeys[index] !== `event:${event.sourceSeq}`)) { + throw new Error("native_runner_checkpoint_invalid"); + } + return { state, runnerVersion: header.runnerVersion, runnerDigest: header.runnerDigest }; +} + +export function decodeDurablePrpCheckpoint(records: ReadonlyMap): DurablePrpCheckpoint { + try { + const decoded = checkpointState(records); + return captureCheckpoint(decoded.state, decoded.runnerVersion, decoded.runnerDigest); + } catch (error) { + if (error instanceof SyntaxError) throw new Error("native_runner_checkpoint_invalid", { cause: error }); + throw error; + } +} + export interface RunnerProcessResult { code: number | null; signal: NodeJS.Signals | null; @@ -645,6 +792,7 @@ function isStoredCoreState( (value.schema !== coreStateSchema && value.schema !== transitionCoreStateSchema) || canonicalJson(value.identity) !== canonicalJson(identity) || + (value.checkpointEventPolicy !== undefined && value.checkpointEventPolicy !== "control-v1") || !isRecord(value.tickets) || !isRecord(value.leases) || !Array.isArray(commands) || @@ -706,6 +854,13 @@ function isStoredCoreState( ) { return false; } + if (value.preparedLaunchConfigurations !== undefined && ( + !Array.isArray(value.preparedLaunchConfigurations) || value.preparedLaunchConfigurations.length > 2 || + !value.preparedLaunchConfigurations.every(binding => isRecord(binding) && exactIdentity(binding.identity) + && typeof binding.configurationId === "string" && binding.configurationId.trim().length > 0 + && binding.configurationId.length <= 1024) || + new Set(value.preparedLaunchConfigurations.map(binding => canonicalJson(binding.identity))).size !== value.preparedLaunchConfigurations.length + )) return false; if ( !commands.every( (command, index) => @@ -1203,7 +1358,9 @@ class DurableCoreStore { readonly path: string; #state: StoredCoreState; #writeIndeterminate = false; - #revision = 0; + #writeRevision = 0; + #externallyPersisted = false; + revision = 0; #pendingEvent: { buildCandidate: (state: StoredCoreState) => StoredCoreState | null; isCurrentOwner: () => boolean; @@ -1228,6 +1385,9 @@ class DurableCoreStore { const stored = readPrivateFile(this.path); if (stored !== null) { const parsed = JSON.parse(stored) as unknown; + if (isRecord(parsed) && parsed.schema === externalCoreStateSchema) { + throw new Error("Restore the external checkpoint before opening this controller."); + } if (!isStoredCoreState(parsed, identity)) { throw new Error( "Control-plane state is invalid or does not match the requested PRP identity.", @@ -1244,9 +1404,29 @@ class DurableCoreStore { return this.#state; } + useExternalPersistence(): void { + this.assertWritable(); + this.#pendingEvent = null; + ++this.#writeRevision; + // Leave a fail-closed marker, never a stale recovery snapshot. The external + // checkpoint must be restored before another controller opens this root. + try { + atomicPrivateWrite(this.path, JSON.stringify({ + schema: externalCoreStateSchema, + identity: this.#state.identity, + })); + } catch (error) { + this.#writeIndeterminate = true; + throw error; + } + this.#state.checkpointEventPolicy = "control-v1"; + this.#externallyPersisted = true; + this.revision += 1; + } + save(): void { this.assertWritable(); - ++this.#revision; + ++this.#writeRevision; try { const pending = this.#pendingEvent; // A command already needs a synchronous write. Include the admitted @@ -1254,36 +1434,48 @@ class DurableCoreStore { const withEvent = pending && !pending.committed && pending.isCurrentOwner() ? pending.buildCandidate(this.#state) : null; const candidate = withEvent ?? this.#state; - atomicPrivateWrite(this.path, `${JSON.stringify(candidate)}\n`); + if (!this.#externallyPersisted) { + atomicPrivateWrite(this.path, `${JSON.stringify(candidate)}\n`); + } this.#state = candidate; if (pending && withEvent) pending.committed = true; + this.revision += 1; } catch (error) { this.#writeIndeterminate = true; throw error; } } - /** Stage event state privately; publish it only after its durable commit. */ + /** File-backed events publish after the staged commit. External delivery + * waits for its checkpoint callback in the controller. */ async commitEvent( buildCandidate: (state: StoredCoreState) => StoredCoreState | null, isCurrentOwner: () => boolean, + checkpointRequired: boolean, ): Promise { if (!isCurrentOwner()) return false; const pending = { buildCandidate, isCurrentOwner, committed: false }; this.#pendingEvent = pending; try { this.assertWritable(); - const revision = this.#revision; + const revision = this.#writeRevision; const candidate = buildCandidate(this.#state); if (candidate === null) return false; + if (this.#externallyPersisted) { + this.#state = candidate; + ++this.#writeRevision; + if (checkpointRequired) this.revision += 1; + return true; + } await withStagedPrivateWrite(this.path, `${JSON.stringify(candidate)}\n`, (temporary) => { this.assertWritable(); - if (pending.committed || !isCurrentOwner() || this.#revision !== revision) return; + if (pending.committed || !isCurrentOwner() || this.#writeRevision !== revision) return; renameSync(temporary, this.path); syncParentDirectory(this.path); this.#state = candidate; - ++this.#revision; + ++this.#writeRevision; pending.committed = true; + if (checkpointRequired) this.revision += 1; }); // Explicit authority commits do not absorb pending events. On conflict, // leave the event unacknowledged for replay instead of rewriting in a loop. @@ -1303,16 +1495,20 @@ class DurableCoreStore { ); } - /** Persist a complete candidate before publishing any new authority in memory. */ - commit(candidate: StoredCoreState): void { + /** Replace authority without allowing an older staged event to overwrite it. */ + commit(candidate: StoredCoreState, checkpointRequired = true): void { this.assertWritable(); // An explicit authority transaction owns its complete snapshot. Later // command writes must not absorb an event admitted before that boundary. this.#pendingEvent = null; - ++this.#revision; + ++this.#writeRevision; + candidate.checkpointEventPolicy ??= this.#state.checkpointEventPolicy; try { - atomicPrivateWrite(this.path, `${JSON.stringify(candidate)}\n`); + if (!this.#externallyPersisted) { + atomicPrivateWrite(this.path, `${JSON.stringify(candidate)}\n`); + } this.#state = candidate; + if (checkpointRequired) this.revision += 1; } catch (error) { // Rename may already have succeeded before directory fsync failed. // Never overwrite that possibly durable receipt using stale memory. @@ -1558,10 +1754,23 @@ export class DurablePrpControlPlane { #port: number | null = null; #onSemanticToolInput?: DurablePrpControlPlaneOptions["onSemanticToolInput"]; #onCommittedEvent?: DurablePrpControlPlaneOptions["onCommittedEvent"]; + #recoverSemanticToolInput?: DurablePrpControlPlaneOptions["recoverSemanticToolInput"]; #beforeAuthenticatedConnection?: DurablePrpControlPlaneOptions["beforeAuthenticatedConnection"]; #onProtocolIntegrityError?: DurablePrpControlPlaneOptions["onProtocolIntegrityError"]; #protocolIntegrityError: NativeSessionProtocolIntegrityError | null = null; #connectionLeaseTtlMs: number; + #persistCheckpoint: ((checkpoint: DurablePrpCheckpoint) => Promise) | null = null; + #persistedRevision = -1; + #queuedRevision = -1; + #lastCheckpoint?: DurablePrpCheckpoint; + #publishedState?: StoredCoreState; + #publishedStore?: DurablePrpControlPlaneStore; + #publicationCommands = new Map(); + #visibleStateCache?: { working: StoredCoreState; published: StoredCoreState; revision: number; state: StoredCoreState }; + #checkpointPersistenceFailed = false; + #persistenceTail = Promise.resolve(); + #sendQueues = new Map>(); + #recoveredCursor: { runId: string; sourceSeq: number } | null = null; constructor(options: DurablePrpControlPlaneOptions) { if ( @@ -1582,6 +1791,9 @@ export class DurablePrpControlPlane { options.stateDirectory, options.identity, ); + if (this.#store.state.checkpointEventPolicy === "control-v1") { + this.#recoveredCursor = { runId: options.identity.runId, sourceSeq: this.#store.state.ackedSourceSeq }; + } this.#expectedRunnerVersion = options.expectedRunnerVersion; this.#expectedRunnerDigest = options.expectedRunnerDigest; const transition = this.#store.state.warmTransition; @@ -1596,16 +1808,135 @@ export class DurablePrpControlPlane { } this.#onSemanticToolInput = options.onSemanticToolInput; this.#onCommittedEvent = options.onCommittedEvent; + this.#recoverSemanticToolInput = options.recoverSemanticToolInput; this.#beforeAuthenticatedConnection = options.beforeAuthenticatedConnection; this.#onProtocolIntegrityError = options.onProtocolIntegrityError; this.#connectionLeaseTtlMs = options.connectionLeaseTtlMs ?? 60_000; } get store(): DurablePrpControlPlaneStore { - return this.#store; + return this.#publishedStore ?? this.#store; + } + + #capturePublishedState(snapshot: DurablePrpCheckpoint): StoredCoreState { + const header = JSON.parse(snapshot.records.get("header")!) as CheckpointHeader; + const cursor = JSON.parse(snapshot.records.get("cursor")!) as { ackedSourceSeq: number; replayDeliveries: number }; + const cached = new Map(); + for (const [part, record] of snapshot.records) { + if (!part.startsWith("command:")) continue; + const prior = this.#publicationCommands.get(part); + cached.set(part, prior?.record === record ? prior : { record, command: JSON.parse(record) }); + } + this.#publicationCommands = cached; + return { ...header.state, ...cursor, commands: [...cached.values()].map(value => value.command), + // Event records are immutable after insertion. Retain their array, not + // another deep copy of the streaming payload window. + committedEvents: this.#store.state.committedEvents }; + } + + #visibleState(): StoredCoreState { + const published = this.#publishedState!; + const working = this.#store.state; + const cached = this.#visibleStateCache; + if (cached?.working === working && cached.published === published && cached.revision === this.#store.revision) { + return cached.state; + } + // A warm handoff cannot expose its new authority before persistence. + if (canonicalJson(working.identity) !== canonicalJson(published.identity)) return published; + const barrier = this.#persistedRevision === this.#store.revision ? -1 + : working.committedEvents.findIndex(event => event.sourceSeq > published.ackedSourceSeq && !isStreamingEvent(event)); + const state = { ...published, + committedEvents: barrier < 0 ? working.committedEvents : working.committedEvents.slice(0, barrier), + ackedSourceSeq: barrier < 0 ? working.ackedSourceSeq : Math.min(working.ackedSourceSeq, working.committedEvents[barrier].sourceSeq - 1), + }; + this.#visibleStateCache = { working, published, revision: this.#store.revision, state }; + return state; + } + + /** Sparse, already-acknowledged history; never permits gaps in new delivery. */ + get recoveredSourceSeq(): number { + return this.#recoveredCursor?.runId === this.#identity.runId ? this.#recoveredCursor.sourceSeq : 0; + } + + /** + * Make the callback the durable authority. Local state becomes a marker; + * restart must restore its checkpoint before constructing the controller. + * Outbound frames and semantic dispatch still await the callback's commit. + */ + setCheckpointPersistence(persist: (checkpoint: DurablePrpCheckpoint) => Promise): void { + if (this.#connections.size > 0 || this.#persistCheckpoint !== null) { + throw new Error("runner_controller_persistence_already_active"); + } + this.#store.useExternalPersistence(); + this.#persistCheckpoint = persist; + this.#publishedState = this.#capturePublishedState(captureCheckpoint(this.#store.state, + this.#expectedRunnerVersion, this.#expectedRunnerDigest)); + const core = this; + this.#publishedStore = { path: this.#store.path, get state() { return core.#visibleState(); } }; + } + + flushCheckpoint(): Promise { + if (!this.#persistCheckpoint || this.#persistedRevision === this.#store.revision + || this.#queuedRevision === this.#store.revision) { + return this.#persistenceTail; + } + const snapshot = captureCheckpoint(this.#store.state, + this.#expectedRunnerVersion, this.#expectedRunnerDigest); + const published = this.#capturePublishedState(snapshot); + const revision = this.#store.revision; + this.#queuedRevision = revision; + const persist = this.#persistCheckpoint; + const pending = this.#persistenceTail.catch(() => undefined).then(async () => { + // The previous write may have failed after committing. Rebuild the delta + // at execution time, and reset after uncertainty instead of assuming it failed. + const checkpoint = { ...snapshot, ...checkpointChanges(snapshot.records, snapshot.runId, + this.#checkpointPersistenceFailed ? undefined : this.#lastCheckpoint) }; + try { + await persist(checkpoint); + this.#lastCheckpoint = checkpoint; + this.#publishedState = published; + this.#persistedRevision = revision; + this.#checkpointPersistenceFailed = false; + } catch (error) { + this.#checkpointPersistenceFailed = true; + throw error; + } + }); + this.#persistenceTail = pending; + void pending.then(() => { + if (this.#persistenceTail === pending) this.#queuedRevision = -1; + }, () => { + if (this.#persistenceTail === pending) this.#queuedRevision = -1; + this.disconnectActiveRunner(); + }); + return pending; + } + + #sendJson(connection: AuthorityConnection, value: unknown): void { + if (!this.#persistCheckpoint) { + connection.sendJson(value); + return; + } + const checkpoint = this.flushCheckpoint(); + const frame = structuredClone(value); + const sending = (this.#sendQueues.get(connection) ?? Promise.resolve()) + .then(() => checkpoint) + .then(() => connection.sendJson(frame)) + .catch(() => connection.close()); + this.#sendQueues.set(connection, sending); + void sending.finally(() => { + if (this.#sendQueues.get(connection) === sending) this.#sendQueues.delete(connection); + }); } getCommand(commandId: string): DurableRecoveryCoreCommand | undefined { + const state = this.store.state; + return state.commands.find(command => command.commandId === commandId) + ?? (state.warmTransition?.command.commandId === commandId ? state.warmTransition.command : undefined) + ?? (state.completedWarmTransition?.command.commandId === commandId ? state.completedWarmTransition.command : undefined); + } + + #getCommand(commandId: string): DurableRecoveryCoreCommand | undefined { return ( this.#store.state.commands.find( (command) => command.commandId === commandId, @@ -1620,6 +1951,36 @@ export class DurablePrpControlPlane { ); } + async recoverPendingSemanticInputs(): Promise { + if (!this.#recoverSemanticToolInput) return; + const identity = this.#identity; + for (const saved of [...this.#store.state.committedEvents]) { + if (!unsettledSemanticInput(saved, this.#store.state)) continue; + const event = saved.envelope.payload as PrpEvent; + const semantic = (event.payload as Record).semantic_tool as Record; + if (!semantic || event.runId !== identity.runId || event.normalizedSessionId !== identity.normalizedSessionId) continue; + if (typeof semantic.callId !== "string" || typeof semantic.operationId !== "string") continue; + const commandId = `command_tool_${createHash("sha256").update(`${identity.runId}\0${semantic.callId}`).digest("hex").slice(0, 32)}`; + if (this.#pendingSemanticCalls.has(commandId) || this.#getCommand(commandId)) continue; + this.#pendingSemanticCalls.add(commandId); + try { + const call = { callId: semantic.callId, operationId: semantic.operationId, input: semantic.input, + sourceEventId: event.sourceEventId, sourceEventType: event.eventType, + correlation: { runId: identity.runId, normalizedSessionId: identity.normalizedSessionId, + turnId: identity.turnId, itemId: event.itemId ?? identity.itemId } }; + const outcome = await this.#recoverSemanticToolInput(call); + if (this.#identity !== identity || this.#semanticCallbacksRetired) return; + if (!outcome || this.#getCommand(commandId)) continue; + const { input, ...metadata } = call; + this.queueCommand("semantic_tool.result", { ...metadata, inputDigest: canonicalDigest(input), + result: outcome.result, isError: outcome.isError === true }, commandId, true); + await this.flushCheckpoint(); + } finally { + this.#pendingSemanticCalls.delete(commandId); + } + } + } + get connectUrl(): string { if (this.#port === null) { throw new Error("Durable PRP control plane is not listening."); @@ -1884,6 +2245,28 @@ export class DurablePrpControlPlane { this.#store.save(); } + preparedLaunchConfigurationId(identity: DurableRecoveryIdentity): string | undefined { + return this.#store.state.preparedLaunchConfigurations?.find(binding => + canonicalJson(binding.identity) === canonicalJson(identity))?.configurationId; + } + + /** Called only after the environment has validated a prepared launch. */ + persistPreparedLaunchConfiguration(identity: DurableRecoveryIdentity, configurationId: string): void { + this.#store.assertWritable(); + if (!exactIdentity(identity) || typeof configurationId !== "string" || !configurationId.trim() || configurationId.length > 1024) { + throw new Error("runner_prepared_launch_configuration_invalid"); + } + const existing = this.preparedLaunchConfigurationId(identity); + if (existing !== undefined) { + if (existing !== configurationId) throw new Error("runner_prepared_launch_configuration_changed"); + return; + } + const current = this.#store.state.preparedLaunchConfigurations?.filter(binding => + canonicalJson(binding.identity) === canonicalJson(this.#identity)) ?? []; + this.#store.state.preparedLaunchConfigurations = [...current, { identity: structuredClone(identity), configurationId }]; + this.#store.save(); + } + issueBootstrapTicket(ttlMs = 5_000): string { this.#store.assertWritable(); if (this.#store.state.warmTransition) { @@ -2053,6 +2436,7 @@ export class DurablePrpControlPlane { throw new CommandJournalLimitError("command_payload_too_large"); this.#store.state.commands.push(command); this.#store.save(); + void this.flushCheckpoint().catch(() => undefined); if (deliverImmediately) { for (const connection of this.#connections) { if (connection.secureChannel !== null) { @@ -2235,6 +2619,7 @@ export class DurablePrpControlPlane { ) connection.replayOnly = true; this.#commandResult(connection, envelope); + await this.flushCheckpoint(); return; } if (kind === "warm_transition_activated") { @@ -2275,7 +2660,7 @@ export class DurablePrpControlPlane { this.#store.commit(candidate); connection.lease = this.#store.state.leases[transition.credentialId]!; } - connection.sendJson( + this.#sendJson(connection, this.#controlEnvelope( connection, `activation_ack_${receipt.transitionId}`, @@ -2335,7 +2720,7 @@ export class DurablePrpControlPlane { this.#store.commit(candidate); connection.lease = this.#store.state.leases[lease.credentialId]!; } - connection.sendJson( + this.#sendJson(connection, this.#controlEnvelope( connection, `lease_renewed_${expectedExpiry}`, @@ -2567,6 +2952,16 @@ export class DurablePrpControlPlane { connection.close(); return; } + const requiresRecoveryCursor = this.#persistCheckpoint !== null || this.#store.state.checkpointEventPolicy === "control-v1"; + const resume = payload.resume; + const acknowledgedSourceSeq = payload.recoveryCursorVersion === 1 && isRecord(resume) + && Number.isSafeInteger(resume.ackedSourceSeq) && (resume.ackedSourceSeq as number) >= 0 + && Number.isSafeInteger(resume.nextSourceEventSeq) && (resume.nextSourceEventSeq as number) > (resume.ackedSourceSeq as number) + ? resume.ackedSourceSeq as number : undefined; + if ((requiresRecoveryCursor || payload.recoveryCursorVersion !== undefined) && acknowledgedSourceSeq === undefined) { + connection.close(); + return; + } const serverNonce = randomUUID(); const selectedVersion = authorization.kind === "lease" @@ -2598,6 +2993,9 @@ export class DurablePrpControlPlane { ...(typeof payload.warmTransitionId === "string" ? { warmTransitionId: payload.warmTransitionId } : {}), + ...(acknowledgedSourceSeq !== undefined + ? { recoveryCursorVersion: 1, recoveryAckedSourceSeq: acknowledgedSourceSeq } + : {}), }; const canonicalChallenge = canonicalJson(challengePayload); const serverProof = domainHmac( @@ -2623,8 +3021,9 @@ export class DurablePrpControlPlane { ? { warmTransitionId: payload.warmTransitionId } : {}), requestedIdentity: structuredClone(authorization.identity), + acknowledgedSourceSeq, }; - connection.sendJson({ + this.#sendJson(connection, { protocol, version: selectedVersion, kind: "auth_challenge", @@ -2777,6 +3176,7 @@ export class DurablePrpControlPlane { } if (transition.phase === "prepared") { const candidate = initialCoreState(transition.receipt.newIdentity); + candidate.preparedLaunchConfigurations = structuredClone(this.#store.state.preparedLaunchConfigurations); candidate.schema = transitionCoreStateSchema; candidate.warmTransition = { ...structuredClone(transition), @@ -2790,6 +3190,16 @@ export class DurablePrpControlPlane { lease = this.#store.state.leases[lease.credentialId]!; } } + // The authenticated peer's durable ACK covers only events this authority + // previously released. Every control event was checkpointed before that ACK; + // only display/usage streaming may be absent from the restored event window. + if (pending.acknowledgedSourceSeq !== undefined + && (this.#persistCheckpoint !== null || this.#store.state.checkpointEventPolicy === "control-v1") + && canonicalJson(requestedIdentity) === canonicalJson(this.#store.state.identity)) { + const cursor = Math.max(this.#store.state.ackedSourceSeq, pending.acknowledgedSourceSeq); + this.#store.state.ackedSourceSeq = cursor; + this.#recoveredCursor = { runId: this.#identity.runId, sourceSeq: cursor }; + } connection.pendingChallenge = null; connection.lease = lease; connection.identity = structuredClone(requestedIdentity); @@ -2844,7 +3254,7 @@ export class DurablePrpControlPlane { (this.#store.state.commandDeliveryCounts[command.commandId] ?? 0) + 1; } this.#store.save(); - connection.sendJson({ + this.#sendJson(connection, { protocol, version: lease.protocolVersion, envelopeId: `welcome_${this.#store.state.connectionCount}`, @@ -2955,7 +3365,7 @@ export class DurablePrpControlPlane { this.#store.state.commandDeliveryCounts[command.commandId] = (this.#store.state.commandDeliveryCounts[command.commandId] ?? 0) + 1; this.#store.save(); - connection.sendJson( + this.#sendJson(connection, this.#controlEnvelope( connection, `command_${command.commandId}_${this.#store.state.commandDeliveryCounts[command.commandId]}`, @@ -3093,7 +3503,7 @@ export class DurablePrpControlPlane { connection: AuthorityConnection, receipt: DurableWarmRunTransition, ): void { - connection.sendJson( + this.#sendJson(connection, this.#controlEnvelope( connection, `command_result_ack_${receipt.controllerSeq}`, @@ -3122,7 +3532,7 @@ export class DurablePrpControlPlane { if (!this.#isTerminalLifecycleCommand(command)) { return; } - connection.sendJson( + this.#sendJson(connection, this.#controlEnvelope( connection, `command_result_ack_${command.controllerSeq}`, @@ -3272,10 +3682,7 @@ export class DurablePrpControlPlane { const eventToEvict = existing === undefined && this.#store.state.committedEvents.length >= maxCommittedEventWindow - ? this.#store.state.committedEvents.findIndex( - (candidate) => - !unsettledSemanticInput(candidate, this.#store.state), - ) + ? evictionCandidate(this.#store.state) : null; if (eventToEvict === -1) { connection.close(); @@ -3323,9 +3730,7 @@ export class DurablePrpControlPlane { if (committedEvents.length >= maxCommittedEventWindow) { // A command can settle an input while this snapshot is staged. Use // the current response ledger to select an event that can be evicted. - const currentEviction = committedEvents.findIndex( - (candidate) => !unsettledSemanticInput(candidate, state), - ); + const currentEviction = evictionCandidate(state); if (currentEviction < 0) return null; committedEvents.splice(currentEviction, 1); } @@ -3345,49 +3750,34 @@ export class DurablePrpControlPlane { connection.close(); return; } - this.#store.commit(candidate); - } else if (!await this.#store.commitEvent(buildCandidate, isCurrentOwner)) { + this.#store.commit(candidate, !isStreamingEvent(event)); + } else if (!await this.#store.commitEvent(buildCandidate, isCurrentOwner, !isStreamingEvent(event))) { if (isCurrentOwner()) connection.close(); return; } if (!isCurrentOwner()) return; - if ( - isSemanticInput && - this.#onSemanticToolInput && - semantic !== undefined && - typeof semantic.callId === "string" && - typeof semantic.operationId === "string" - ) { - const call = { - callId: semantic.callId, - operationId: semantic.operationId, - input: semantic.input, - sourceEventId, - sourceEventType: eventType, - correlation: { - runId: this.#identity.runId, - normalizedSessionId: this.#identity.normalizedSessionId, - turnId: this.#identity.turnId, - itemId: - typeof event.itemId === "string" - ? event.itemId - : this.#identity.itemId, - }, - }; - const commandId = `command_tool_${createHash("sha256") - .update(`${this.#identity.runId}\0${call.callId}`) - .digest("hex") - .slice(0, 32)}`; - const alreadyQueued = this.#store.state.commands.some( - (command) => command.commandId === commandId, - ); - // A committed input without a result after restart may already have - // changed the outside world. Never redispatch it on event replay. Only - // the original admitted callback may publish its result; missing outcome - // evidence stays pending for authoritative reconciliation. - if (existing === undefined && !alreadyQueued && !this.#pendingSemanticCalls.has(commandId)) { - this.#pendingSemanticCalls.add(commandId); + const liveCall = isSemanticInput && this.#onSemanticToolInput && semantic !== undefined + && typeof semantic.callId === "string" && typeof semantic.operationId === "string" + ? { callId: semantic.callId, operationId: semantic.operationId, input: semantic.input, + sourceEventId, sourceEventType: eventType, + correlation: { runId: identity.runId, normalizedSessionId: identity.normalizedSessionId, + turnId: identity.turnId, itemId: typeof event.itemId === "string" ? event.itemId : identity.itemId } } + : null; + const liveCommandId = liveCall ? `command_tool_${createHash("sha256") + .update(`${identity.runId}\0${liveCall.callId}`).digest("hex").slice(0, 32)}` : null; + const ownsLiveDispatch = existing === undefined && liveCommandId !== null + && !this.#getCommand(liveCommandId) && !this.#pendingSemanticCalls.has(liveCommandId); + // Reserve the original callback before exposing its checkpoint wait to + // recovery. A missing saved outcome must never suppress first dispatch. + if (ownsLiveDispatch) this.#pendingSemanticCalls.add(liveCommandId!); + let dispatched = false; + try { + await this.flushCheckpoint(); + if (this.#eventOwner !== connection || this.#identity !== identity) return; + if (ownsLiveDispatch && liveCall && liveCommandId && this.#onSemanticToolInput && !this.#getCommand(liveCommandId)) { + const call = liveCall; + const commandId = liveCommandId; const recordFailure = (stage: "persist_result" | "dispatch", error: unknown) => { this.#semanticResultPersistenceFailed = true; // Keep diagnostics bounded and content-free. Exception messages can @@ -3420,7 +3810,9 @@ export class DurablePrpControlPlane { recordFailure("persist_result", error); } }; - void this.#onSemanticToolInput(call) + const outcome = this.#onSemanticToolInput(call); + dispatched = true; + void outcome .then((outcome) => queueResult(outcome.result, outcome.isError === true), ) @@ -3439,18 +3831,19 @@ export class DurablePrpControlPlane { }) .finally(() => this.#pendingSemanticCalls.delete(commandId)); } + this.#sendJson(connection, + this.#controlEnvelope( + connection, + `ack_${this.#store.state.ackedSourceSeq}`, + "ack", + { + ackedSourceSeq: this.#store.state.ackedSourceSeq, + }, + ), + ); + } finally { + if (ownsLiveDispatch && !dispatched) this.#pendingSemanticCalls.delete(liveCommandId!); } - - connection.sendJson( - this.#controlEnvelope( - connection, - `ack_${this.#store.state.ackedSourceSeq}`, - "ack", - { - ackedSourceSeq: this.#store.state.ackedSourceSeq, - }, - ), - ); } } @@ -3722,6 +4115,8 @@ export function spawnRunner(options: { const environment = runnerEnvironment(options.ticket, options.identity.normalizedSessionId, options.environment); const withRestart = (handle: RunnerProcessHandle): RunnerProcessHandle => handle.restart === null ? handle : ({ ...handle, + // Remote owners publish this identity after dispatch; keep it live through the wrapper. + get startedAt() { return handle.startedAt; }, restart: (ticket) => { if (handle.isExitConfirmed?.() === false) { throw new Error("runner_process_termination_unconfirmed"); diff --git a/packages/paperclip-runner/src/control-plane/runner-execution-driver.ts b/packages/paperclip-runner/src/control-plane/runner-execution-driver.ts index 7a55b20b843..f94fa464082 100644 --- a/packages/paperclip-runner/src/control-plane/runner-execution-driver.ts +++ b/packages/paperclip-runner/src/control-plane/runner-execution-driver.ts @@ -11,6 +11,12 @@ import type { DurableRecoveryIdentity } from "../contracts/durable-recovery.js"; export interface RunnerExecutionDriver { /** Own provider configuration and file preparation in the execution environment. */ prepareLaunch?: RunnerLaunchPreparer; + /** Reauthorize retained resources before adoption without preparing them again. + * The ID is bound to the exact execution in the durable controller checkpoint. + * Omission preserves prepareLaunch's existing recovery/authorization path. + * Adoption with this hook requires prepareLaunch and its retained configuration binding. + */ + authorizeRunnerAdoption?: (configurationId: string, identity: DurableRecoveryIdentity) => Promise; /** Prepare resources and register or acquire the connection before launch. */ controlPlaneRegistration?: ( authority: DurablePrpControlPlane, @@ -57,9 +63,9 @@ export interface RunnerExecutionDriver { * never calls the process launcher while the process remains alive. */ adoptExistingRunner?: { - pid: number; + pid: number | null; processGroupId: number | null; - startedAt: string; + startedAt: string | null; isAlive: () => Promise | boolean; signal?: (signal: NodeJS.Signals) => Promise | boolean; }; diff --git a/packages/paperclip-runner/src/control-plane/runner-recovery-fixture.test.ts b/packages/paperclip-runner/src/control-plane/runner-recovery-fixture.test.ts new file mode 100644 index 00000000000..e346eee823f --- /dev/null +++ b/packages/paperclip-runner/src/control-plane/runner-recovery-fixture.test.ts @@ -0,0 +1,142 @@ +import { execFileSync, spawn, type ChildProcess } from "node:child_process"; +import { once } from "node:events"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { beforeAll, expect, it, vi } from "vitest"; +import { + DurablePrpControlPlane, decodeDurablePrpCheckpoint, type DurablePrpCheckpoint, +} from "./durable-prp-control-plane.js"; + +const packageRoot = fileURLToPath(new URL("../..", import.meta.url)); +const binary = resolve(packageRoot, "runner/target/debug/examples/recovery-fixture"); +const identity = { + runnerInstanceId: "fixture-runner", environmentLeaseId: "fixture-lease", + runId: "fixture-run", normalizedSessionId: "fixture-session", turnId: "fixture-turn", itemId: "fixture-item", +}; +const expectedRunnerVersion = "0.3.0"; +const expectedRunnerDigest = `sha256:${"c".repeat(64)}`; + +beforeAll(() => { + execFileSync("cargo", ["build", "--locked", "--manifest-path", resolve(packageRoot, "runner/Cargo.toml"), + "-p", "paperclip-runner-core", "--example", "recovery-fixture"], { timeout: 120_000, stdio: "pipe" }); +}, 120_000); + +async function fixture(externalPersistence: boolean, graceMs = 20_000) { + const root = mkdtempSync(resolve(tmpdir(), "runner-in-process-recovery-")); + const stateDirectory = resolve(root, "control-plane"); + const receiptFile = resolve(root, "work.json"); + const releaseFile = resolve(root, "release"); + const scriptFile = resolve(root, "script.json"); + writeFileSync(scriptFile, JSON.stringify({ steps: 64, waitAfter: 32, retentionLimit: 8, receiptFile, releaseFile })); + const delivered: Array<{ sourceEventId: string; text: string }> = []; + let checkpoint: DurablePrpCheckpoint | undefined; + const create = () => { + const core = new DurablePrpControlPlane({ stateDirectory, identity, expectedRunnerVersion, expectedRunnerDigest, + onCommittedEvent: async event => { + if (event.eventType === "item.delta") delivered.push({ sourceEventId: event.sourceEventId, text: (event.payload as { text: string }).text }); + }, + }); + if (externalPersistence) core.setCheckpointPersistence(async saved => { checkpoint = saved; }); + return core; + }; + let core = create(); + let child: ChildProcess | undefined; + let exit: Promise | undefined; + const receipt = () => JSON.parse(readFileSync(receiptFile, "utf8")) as { + pid: number; phase: string; completedSteps: number; retainedEvents: number; peakRetainedEvents: number; acknowledgedEvents: number; + }; + await core.start(); + const port = Number(new URL(core.connectUrl).port); + const ticket = core.issueBootstrapTicket(); + await core.flushCheckpoint(); + child = spawn(binary, ["--fixture-script", scriptFile, "--connect-url", core.connectUrl, + "--state-dir", resolve(root, "runner"), "--runner-id", identity.runnerInstanceId, + "--environment-lease-id", identity.environmentLeaseId, "--run-id", identity.runId, + "--session-id", identity.normalizedSessionId, "--turn-id", identity.turnId, "--item-id", identity.itemId, + "--runner-version", expectedRunnerVersion, "--runner-digest", expectedRunnerDigest, + "--max-outbox-bytes", "65536", "--p0-reserve-bytes", "16384", "--reconnect-grace-ms", String(graceMs), + ], { env: { ...process.env, PAPERCLIP_RUNNER_BOOTSTRAP_TICKET: ticket }, stdio: ["ignore", "ignore", "pipe"] }); + let diagnostics = ""; + child.stderr!.on("data", bytes => { diagnostics = (diagnostics + String(bytes)).slice(-4096); }); + exit = once(child, "exit"); + async function command(type: string, commandId: string) { + core.queueCommand(type, {}, commandId, true); + await vi.waitFor(() => expect(core.getCommand(commandId)?.status, `${commandId}: exit=${child?.exitCode}; ${diagnostics}`).toBe("completed"), { timeout: 10_000 }); + } + return { + root, receipt, child, exit, delivered, command, + checkpoint: () => checkpoint, + release: () => writeFileSync(releaseFile, "compute"), + stopController: async () => { + await core.stop(); + await core.drainPendingConnectionProcessing(); + await core.flushCheckpoint(); + }, + replaceController: async () => { + if (externalPersistence) { + rmSync(stateDirectory, { recursive: true }); + decodeDurablePrpCheckpoint(checkpoint!.records).restoreSessionDirectory(root); + } + core = create(); + await core.start(port); + }, + cleanup: async () => { + await core.stop(); + if (child?.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); + await exit; + rmSync(root, { recursive: true, force: true }); + }, + }; +} + +it.each([false, true])("preserves useful in-process work through repeated replacement (external persistence: %s)", async external => { + const run = await fixture(external); + try { + await run.command("turn.start", "start-once"); + expect(run.receipt().completedSteps).toBe(0); + await run.stopController(); + run.release(); + await vi.waitFor(() => expect(run.receipt()).toMatchObject({ phase: "backpressured", completedSteps: 8, retainedEvents: 8 }), { timeout: 5000 }); + expect(run.receipt().pid).toBe(run.child.pid); + expect(run.child.exitCode).toBeNull(); + expect(run.delivered).toHaveLength(0); + await run.replaceController(); + await vi.waitFor(() => expect(run.delivered).toHaveLength(32), { timeout: 10_000 }); + expect(run.receipt()).toMatchObject({ phase: "awaiting_controller", completedSteps: 32 }); + await run.stopController(); + expect(run.receipt().completedSteps).toBe(32); + await run.replaceController(); + await run.command("session.snapshot", "after-second-replacement"); + expect(run.receipt().completedSteps).toBe(32); + await run.command("turn.steer", "controller-dependent-continuation"); + await vi.waitFor(() => expect(run.delivered).toHaveLength(64), { timeout: 10_000 }); + expect(new Set(run.delivered.map(event => event.sourceEventId)).size).toBe(64); + expect(run.delivered.map(event => Number(event.text.match(/^step=(\d+)/)?.[1]))).toEqual(Array.from({ length: 64 }, (_, i) => i + 1)); + // Controller delivery precedes the acknowledgement reaching the runner. + // Prove backlog settlement at the runner before ending the experiment. + await vi.waitFor(() => expect(run.receipt()).toMatchObject({ + phase: "completed", pid: run.child.pid, completedSteps: 64, acknowledgedEvents: 64, retainedEvents: 0, + }), { timeout: 10_000 }); + expect(run.receipt().peakRetainedEvents).toBeLessThanOrEqual(8); + if (external) expect([...run.checkpoint()!.records.values()].join("\n")).not.toContain("step="); + await run.command("runner.shutdown", "shutdown"); + expect((await run.exit)[0]).toBe(0); + } finally { await run.cleanup(); } +}, 35_000); + +it("stops after the original reconnect deadline while retaining recovery state", async () => { + const run = await fixture(true, 1500); + try { + await run.command("turn.start", "start-once"); + await run.stopController(); + run.release(); + await run.exit; + const state = JSON.parse(readFileSync(resolve(run.root, "runner/runner-state.json"), "utf8")); + expect(state.recoverableFailure).toBe("transport_reconnect_grace_exceeded"); + expect(run.receipt()).toMatchObject({ phase: "stopped", pid: run.child.pid }); + expect(run.receipt().completedSteps).toBeGreaterThan(0); + expect(run.receipt().peakRetainedEvents).toBeLessThanOrEqual(8); + } finally { await run.cleanup(); } +}, 15_000); diff --git a/packages/paperclip-runner/src/drivers/acpx/copilot-installation.test.ts b/packages/paperclip-runner/src/drivers/acpx/copilot-installation.test.ts index c8743107d1f..35d3225b5f5 100644 --- a/packages/paperclip-runner/src/drivers/acpx/copilot-installation.test.ts +++ b/packages/paperclip-runner/src/drivers/acpx/copilot-installation.test.ts @@ -87,9 +87,64 @@ vi.mock("./installation-integrity.js", () => ({ verifyNativeAcpxInstallation: vi describe("Copilot build-owned installation", () => { it("admits the current profile with environment launch preparation", () => { - expect(QUALIFIED_ACPX_PROFILES.copilot.agentProfileVersion).toBe(66); + expect(QUALIFIED_ACPX_PROFILES.copilot.agentProfileVersion).toBe(68); expect(QUALIFIED_ACPX_PROFILES.copilot.qualificationStatus).toBeUndefined(); - expect(QUALIFIED_ACPX_PROFILES.copilot.commandDigest).toBe("sha256:0cddc219af25b741b7138f38faa7514dd5ee3427b1d3168e8a4bfbe5180983e8"); + expect(QUALIFIED_ACPX_PROFILES.copilot.commandDigest).toBe("sha256:1ce8bb0e00df81fa9eb71cff5063ce769637f5db7b3aa0ada9ade651db089c63"); + }); + + it("preserves both lineages while binding the merged implementation in v68", async () => { + const published = readFileSync(new URL("../../../test/fixtures/copilot-profile-v66-identity.json", import.meta.url)); + const candidate = readFileSync(new URL("../../../test/fixtures/copilot-profile-v67-identity.json", import.meta.url)); + expect(createHash("sha256").update(published).digest("hex")).toBe("9eb0bfd31334210fd41ddb677a165ef7432a2951922e642dddff087e4db8b6bc"); + expect(createHash("sha256").update(candidate).digest("hex")).toBe("6c818541933b97bea896ca29179ad54c887db9b7d563021da1676a8126f960dd"); + const prior = JSON.parse(published.toString("utf8")); + const current = JSON.parse(readFileSync(new URL("../../../test/fixtures/copilot-profile-v68-identity.json", import.meta.url), "utf8")); + expect(Object.keys(current.declaration).filter(key => current.declaration[key] !== prior.declaration[key]).sort()).toEqual([ + "agentProfileVersion", "profileCompatibilitySourceSha256", "providerLaunchSourceSha256", "runtimeRequestBridgeSourceSha256", + ]); + for (const history of [prior, JSON.parse(candidate.toString("utf8"))]) { + expect(isSupportedAcpxProfileVersion("copilot", history.declaration.agentProfileVersion)).toBe(true); + await expect(verifyCopilotInstallation({ ...QUALIFIED_ACPX_PROFILES.copilot, + agentProfileVersion: history.declaration.agentProfileVersion, commandDigest: history.commandDigest })).rejects.toThrow("exact pinned"); + } + }); + + it("preserves published v65 while binding hot-upgrade adoption in v67", async () => { + const bytes = readFileSync(new URL("../../../test/fixtures/copilot-profile-v65-identity.json", import.meta.url)); + expect(createHash("sha256").update(bytes).digest("hex")).toBe("ca1d95c21b3f49c99b6d7c6bd3c2bf149715db3263927ad653cc0c6f33fb4462"); + const prior = JSON.parse(bytes.toString("utf8")); + const current = JSON.parse(readFileSync(new URL("../../../test/fixtures/copilot-profile-v67-identity.json", import.meta.url), "utf8")); + expect(Object.keys(current.declaration).filter(key => current.declaration[key] !== prior.declaration[key]).sort()).toEqual([ + "agentProfileVersion", "profileCompatibilitySourceSha256", "providerLaunchSourceSha256", "runtimeRequestBridgeSourceSha256", + ]); + expect(Object.keys(current.declaration).sort()).toEqual(Object.keys(prior.declaration).sort()); + expect(isSupportedAcpxProfileVersion("copilot", 65)).toBe(true); + await expect(verifyCopilotInstallation({ ...QUALIFIED_ACPX_PROFILES.copilot, + agentProfileVersion: 65, commandDigest: prior.commandDigest })).rejects.toThrow("exact pinned"); + }); + + it("retains candidate v66 identity when completion recovery changes in v67", async () => { + const bytes = readFileSync(new URL("../../../test/fixtures/copilot-profile-v66-hot-upgrade-candidate-historical-identity.json", import.meta.url)); + expect(createHash("sha256").update(bytes).digest("hex")).toBe("430f7a3bb4f51a22d6f18a515fb32eb4defa5bab60a6a4901f6750e59f2536c6"); + const prior = JSON.parse(bytes.toString("utf8")); + const current = JSON.parse(readFileSync(new URL("../../../test/fixtures/copilot-profile-v67-identity.json", import.meta.url), "utf8")); + expect(Object.keys(current.declaration).filter(key => current.declaration[key] !== prior.declaration[key]).sort()).toEqual([ + "agentProfileVersion", "profileCompatibilitySourceSha256", "runtimeRequestBridgeSourceSha256", + ]); + expect(isSupportedAcpxProfileVersion("copilot", 66)).toBe(true); + await expect(verifyCopilotInstallation({ ...QUALIFIED_ACPX_PROFILES.copilot, + agentProfileVersion: 66, commandDigest: prior.commandDigest })).rejects.toThrow("exact pinned"); + }); + + it("retains the hot-upgrade v59 development identity without replacing published history", async () => { + const bytes = readFileSync(new URL("../../../test/fixtures/copilot-profile-v59-hot-upgrade-candidate-historical-identity.json", import.meta.url)); + expect(createHash("sha256").update(bytes).digest("hex")).toBe("b61fe644383acbbaacd209836878243cb96b143807d90c521ef1fbee2a2fcb0c"); + const prior = JSON.parse(bytes.toString("utf8")); + const sorted = Object.fromEntries(Object.entries(prior.declaration).sort(([a], [b]) => a.localeCompare(b))); + expect(`sha256:${createHash("sha256").update(JSON.stringify(sorted)).digest("hex")}`).toBe(prior.commandDigest); + expect(isSupportedAcpxProfileVersion("copilot", 59)).toBe(true); + await expect(verifyCopilotInstallation({ ...QUALIFIED_ACPX_PROFILES.copilot, + agentProfileVersion: 59, commandDigest: prior.commandDigest })).rejects.toThrow("exact pinned"); }); it("retains v65 history while binding the shared native Grok effort transport in v66", async () => { diff --git a/packages/paperclip-runner/src/drivers/acpx/generated-profiles.ts b/packages/paperclip-runner/src/drivers/acpx/generated-profiles.ts index e8810030a7d..7b94907809f 100644 --- a/packages/paperclip-runner/src/drivers/acpx/generated-profiles.ts +++ b/packages/paperclip-runner/src/drivers/acpx/generated-profiles.ts @@ -48,12 +48,12 @@ export const QUALIFIED_ACPX_PROFILE_DATA = { "protocolVersion": 1, "acpxVersion": "0.13.1", "agent": "copilot", - "agentProfileVersion": 66, + "agentProfileVersion": 68, "agentServerPackage": "@github/copilot", "agentServerVersion": "1.0.88", "agentRuntimePackage": null, "agentRuntimeVersion": null, - "commandDigest": "sha256:0cddc219af25b741b7138f38faa7514dd5ee3427b1d3168e8a4bfbe5180983e8", + "commandDigest": "sha256:1ce8bb0e00df81fa9eb71cff5063ce769637f5db7b3aa0ada9ade651db089c63", "permissionPolicy": "interactive" }, "claude": { diff --git a/packages/paperclip-runner/src/drivers/acpx/profile-compatibility.ts b/packages/paperclip-runner/src/drivers/acpx/profile-compatibility.ts index 6d397e70882..25804b55c31 100644 --- a/packages/paperclip-runner/src/drivers/acpx/profile-compatibility.ts +++ b/packages/paperclip-runner/src/drivers/acpx/profile-compatibility.ts @@ -2,13 +2,13 @@ import { QUALIFIED_ACPX_PROFILE_DATA } from "./generated-profiles.js"; /** Decodable historical profile revisions, not permission to launch them. * Exact current profile/package/digest admission is checked separately. */ -type HistoricalAcpxProfileVersion = 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65; +type HistoricalAcpxProfileVersion = 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67; type CurrentAcpxProfileVersion = typeof QUALIFIED_ACPX_PROFILE_DATA[keyof typeof QUALIFIED_ACPX_PROFILE_DATA]["agentProfileVersion"]; export type AcpxProfileVersion = HistoricalAcpxProfileVersion | CurrentAcpxProfileVersion; const historicalVersions: Readonly> = { pi: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29], claude: [1, 2, 3, 4, 5], codex: [1, 2, 3, 4, 5], - grok: [1, 2, 3, 4, 5], copilot: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65], + grok: [1, 2, 3, 4, 5], copilot: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67], cursor: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], }; diff --git a/packages/paperclip-runner/src/drivers/dot/runnerd-dot-driver.test.ts b/packages/paperclip-runner/src/drivers/dot/runnerd-dot-driver.test.ts index 3cf534e40d5..9e14a1bdd2b 100644 --- a/packages/paperclip-runner/src/drivers/dot/runnerd-dot-driver.test.ts +++ b/packages/paperclip-runner/src/drivers/dot/runnerd-dot-driver.test.ts @@ -132,7 +132,7 @@ it.each([ } }, 45000); -it.each(["local", "custom local directory", "external launcher", "grok external launcher"])("reattaches the %s Rust bridge without duplicate operations and refuses lost checkpoints", async launch => { +it.each(["local", "local pending finish", "custom local directory", "external launcher", "grok external launcher", "external launcher empty controller"])("reattaches the %s Rust bridge without duplicate operations and refuses lost checkpoints", async launch => { const root = await mkdtemp(join(tmpdir(), "dot-driver-recovery-")); await writeFile(join(root, "AGENTS.md"), "Use only the synthetic counter."); const input = execution(root, launch === "grok external launcher" ? "grok_bot" : "openai_dot"); @@ -142,6 +142,11 @@ it.each(["local", "custom local directory", "external launcher", "grok external const settlements = new Map(); const dispatches: unknown[] = []; let controllerPort = 0; + const emptyController = launch === "external launcher empty controller"; + const pendingFinish = launch === "local pending finish"; + const firstCompletion = vi.fn(() => new Promise(() => {})); + const recoveredCompletion = vi.fn(async () => "Completion accepted"); + let checkpoint: controlPlane.DurablePrpCheckpoint | undefined; const options: RunnerdDotDriverOptions = { execution: input, stateDirectory: join(root, "state"), identity: { runnerInstanceId: randomUUID(), environmentLeaseId: randomUUID(), runId: input.binding.runId, @@ -149,18 +154,23 @@ it.each(["local", "custom local directory", "external launcher", "grok external runnerBinary: resolve("runner/target/debug/paperclip-runnerd"), onSpawn: process => { spawned = process; }, controlPlaneRegistration: async authority => { + if (emptyController) { + authority.setCheckpointPersistence(async saved => { checkpoint = controlPlane.decodeDurablePrpCheckpoint(new Map(saved.records)); }); + await authority.flushCheckpoint(); + } await authority.start(controllerPort); controllerPort = Number(new URL(authority.connectUrl).port); return { connectUrl: authority.connectUrl, release: () => authority.stop() }; }, dynamicTools: [{ name: "synthetic_counter", description: "Read a synthetic counter", inputSchema: { type: "object", properties: {}, additionalProperties: false } }], dynamicToolHandler: async () => ({ count: ++calls }), + ...(pendingFinish ? { completionFeedback: firstCompletion } : {}), port: { dispatch: async event => { dispatches.push(event); }, settle: async event => { settlements.set(String(event.payload.requestId), event.payload.outcome); }, attach: async callback => { send = callback; return async () => { send = undefined; }; } }, }; - const runnerState = launch === "local" ? join(root, "state/runner") : join(root, "external-runtime/runner"); + const runnerState = launch === "local" || pendingFinish ? join(root, "state/runner") : join(root, "external-runtime/runner"); if (launch === "custom local directory") options.runnerStateDirectory = runnerState; - if (["external launcher", "grok external launcher"].includes(launch)) { + if (launch.includes("external launcher")) { await mkdir(runnerState, { recursive: true }); options.runnerStateDirectory = runnerState; options.readProviderState = async () => { @@ -168,6 +178,7 @@ it.each(["local", "custom local directory", "external launcher", "grok external catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; throw error; } }; options.runnerProcessLauncher = spec => { + if (emptyController) expect(Object.keys(JSON.parse(checkpoint!.records.get("header")!).state.tickets)).toHaveLength(1); expect(spec.environment).not.toHaveProperty("OPENAI_API_KEY"); expect(spec.args).toContain(runnerState); const child = spawn(spec.command, [...spec.args], { cwd: runnerState, env: spec.environment, stdio: "ignore" }); @@ -196,13 +207,37 @@ it.each(["local", "custom local directory", "external launcher", "grok external await send!(renewal); await vi.waitFor(() => expect(settlements.get(renewal.requestId)).toMatchObject({ status: "renewed" }), { timeout: 10000 }); } + const finish = operation("tool", { name: "paperclip_finish", arguments: { + schema: "paperclip.run_result.v1", reportedWorkDisposition: "done", summary: "Read the counter", + completionClaim: { contractRevision: "1", objectiveSatisfied: true, criteria: [{ criterionId: "objective", status: "satisfied", evidenceRefs: [] }], remainingWork: [] }, + evidence: [], verification: [], attentionRequests: [], artifacts: [], + } }); + if (pendingFinish) { + await send!(finish); + await vi.waitFor(() => expect(firstCompletion).toHaveBeenCalledOnce(), { timeout: 10000 }); + expect(settlements.has(finish.requestId)).toBe(false); + } const snapshot = await session.snapshot(); expect(snapshot).toMatchObject({ providerSessionId: null, activeTurnId: options.identity.turnId, providerRecoveryPolicy: "same_session_only" }); await session.detachControllerForRestart!(); - const recoveredDriver = new RunnerdDotDriver({ ...options, adoptExistingRunner: { ...spawned!, isAlive: () => true } }); + if (emptyController) { + await rm(options.stateDirectory, { recursive: true }); + checkpoint!.restoreSessionDirectory(options.stateDirectory); + } + const recoveredDriver = new RunnerdDotDriver({ ...options, + ...(pendingFinish ? { completionFeedback: recoveredCompletion } : {}), + ...(emptyController ? { runnerBinary: join(root, "unavailable-old-runner"), + runnerArtifact: { version: checkpoint!.runnerVersion, digest: checkpoint!.runnerDigest } } : {}), + adoptExistingRunner: { ...spawned!, isAlive: () => true } }); const recovered = await recoveredDriver.recoverSession(snapshot, { signal: new AbortController().signal }); expect(recovered.recovered).toBe(true); session = recovered.session!; + if (pendingFinish) { + expect(options.recoverSemanticToolInput).toBeUndefined(); + await vi.waitFor(() => expect(settlements.get(finish.requestId)).toMatchObject({ status: "completed", isError: false }), { timeout: 10000 }); + expect(recoveredCompletion).toHaveBeenCalledOnce(); + expect(firstCompletion).toHaveBeenCalledOnce(); + } await send!(call); expect(calls).toBe(1); expect(dispatches).toHaveLength(1); diff --git a/packages/paperclip-runner/src/drivers/external/runnerd-external-driver.ts b/packages/paperclip-runner/src/drivers/external/runnerd-external-driver.ts index 6de8688950b..3daecd15d60 100644 --- a/packages/paperclip-runner/src/drivers/external/runnerd-external-driver.ts +++ b/packages/paperclip-runner/src/drivers/external/runnerd-external-driver.ts @@ -7,7 +7,7 @@ import type { NativeExecutionInputV6, NativeExecutionInputV8 } from "../../contr import type { ExternalProviderPort } from "../../contracts/external-provider.js"; import type { PrpEvent } from "../../protocol/replay-contract.js"; import { validatePrpStructuredRunResult } from "../../protocol/replay-contract.js"; -import { DurablePrpControlPlane, spawnRunner, type RunnerProcessHandle } from "../../control-plane/durable-prp-control-plane.js"; +import { DurablePrpControlPlane, spawnRunner, type RunnerProcessHandle, type DurablePrpControlPlaneOptions } from "../../control-plane/durable-prp-control-plane.js"; import type { DurableRecoveryIdentity } from "../../contracts/durable-recovery.js"; import { authorizedToolSetForProvider, defaultCapabilityRunnerdBinary, readRunnerdArtifactBinding, type CapabilityRunnerdCodexTransportOptions } from "../../live/runnerd-codex-transport.js"; import { codexSemanticToolSpecs } from "../codex/codex-driver-values.js"; @@ -21,6 +21,9 @@ export interface RunnerdExternalDriverOptions { identity: DurableRecoveryIdentity; port: ExternalProviderPort; runnerBinary?: string; + runnerArtifact?: { version: string; digest: string }; + runnerReconnectGraceMs?: number; + recoverSemanticToolInput?: DurablePrpControlPlaneOptions["recoverSemanticToolInput"]; runnerStateDirectory?: string; runnerProcessLauncher?: CapabilityRunnerdCodexTransportOptions["runnerProcessLauncher"]; /** Reads the checkpoint from the execution target. Missing state cannot authorize replay. */ @@ -172,7 +175,32 @@ class RunnerdExternalSession implements HarnessSession { const runnerState = o.runnerStateDirectory ?? resolve(o.stateDirectory, "runner"); if (!o.runnerProcessLauncher) mkdirSync(runnerState, { recursive: true, mode: 0o700 }); const binary = o.runnerBinary ?? defaultCapabilityRunnerdBinary(); - const artifact = readRunnerdArtifactBinding(binary); + const artifact = o.runnerArtifact ?? readRunnerdArtifactBinding(binary); + const handleSemanticInput: NonNullable = async call => { + if (call.operationId === "paperclip_finish" || call.operationId === "paperclip_block") { + let completionInput: unknown; + try { + completionInput = await resolveCompletionContinuation(call.input, o.identity.runId, + o.dynamicToolHandler ? () => o.dynamicToolHandler!({ + tool: "get_task_context", callId: `${call.callId}:continuation`, + threadId: o.identity.normalizedSessionId, turnId: o.identity.turnId, arguments: {}, + }) : undefined); + } catch (error) { + return { result: { error: error instanceof Error ? error.message : "Completion rejected" }, isError: true }; + } + const result = validatePrpStructuredRunResult(completionInput); + if (!result.ok || (call.operationId === "paperclip_block") !== (result.ok && result.result.reportedWorkDisposition === "blocked")) { + return { result: { error: "Invalid completion report. Use the admitted completion contract and the correct completion tool." }, isError: true }; + } + try { return { result: { accepted: true, completionReport: result.result, feedback: await o.completionFeedback?.(result.result) ?? "Completion accepted; finish the external turn." } }; } + catch (error) { return { result: { error: error instanceof Error ? error.message : "Completion rejected" }, isError: true }; } + } + if (!o.dynamicToolHandler) throw new Error("dot_semantic_authority_unavailable"); + const result = await o.dynamicToolHandler({ tool: call.operationId, callId: call.callId, + threadId: o.identity.normalizedSessionId, turnId: o.identity.turnId, arguments: call.input }); + const outcome = result && typeof result === "object" ? (result as Record).outcome : undefined; + return { result, isError: typeof outcome === "string" && !["completed", "succeeded", "success"].includes(outcome) }; + }; const core = this.#core = new DurablePrpControlPlane({ identity: o.identity, stateDirectory: resolve(o.stateDirectory, "control-plane"), expectedRunnerVersion: artifact.version, expectedRunnerDigest: artifact.digest, @@ -188,31 +216,10 @@ class RunnerdExternalSession implements HarnessSession { if (!this.#events.some(e => e.sourceEventId === event.sourceEventId)) this.#events.push(event); this.#wake(); }, - onSemanticToolInput: async call => { - if (call.operationId === "paperclip_finish" || call.operationId === "paperclip_block") { - let completionInput: unknown; - try { - completionInput = await resolveCompletionContinuation(call.input, o.identity.runId, - o.dynamicToolHandler ? () => o.dynamicToolHandler!({ - tool: "get_task_context", callId: `${call.callId}:continuation`, - threadId: o.identity.normalizedSessionId, turnId: o.identity.turnId, arguments: {}, - }) : undefined); - } catch (error) { - return { result: { error: error instanceof Error ? error.message : "Completion rejected" }, isError: true }; - } - const result = validatePrpStructuredRunResult(completionInput); - if (!result.ok || (call.operationId === "paperclip_block") !== (result.ok && result.result.reportedWorkDisposition === "blocked")) { - return { result: { error: "Invalid completion report. Use the admitted completion contract and the correct completion tool." }, isError: true }; - } - try { return { result: { accepted: true, completionReport: result.result, feedback: await o.completionFeedback?.(result.result) ?? "Completion accepted; finish the external turn." } }; } - catch (error) { return { result: { error: error instanceof Error ? error.message : "Completion rejected" }, isError: true }; } - } - if (!o.dynamicToolHandler) throw new Error("dot_semantic_authority_unavailable"); - const result = await o.dynamicToolHandler({ tool: call.operationId, callId: call.callId, - threadId: o.identity.normalizedSessionId, turnId: o.identity.turnId, arguments: call.input }); - const outcome = result && typeof result === "object" ? (result as Record).outcome : undefined; - return { result, isError: typeof outcome === "string" && !["completed", "succeeded", "success"].includes(outcome) }; - }, + onSemanticToolInput: handleSemanticInput, + recoverSemanticToolInput: async call => + call.operationId === "paperclip_finish" || call.operationId === "paperclip_block" + ? handleSemanticInput(call) : o.recoverSemanticToolInput?.(call) ?? null, }); // Rehydrate the projection from the authenticated transport journal. Broker // persistence happens before ACK, so these events have durable projections. @@ -222,15 +229,17 @@ class RunnerdExternalSession implements HarnessSession { if (!registration) await core.start(); signal?.throwIfAborted(); if (!o.adoptExistingRunner) { + const ticket = core.issueBootstrapTicket(60_000); + await core.flushCheckpoint(); this.#process = spawnRunner({ processLauncher: o.runnerProcessLauncher, connectUrl: registration?.connectUrl ?? (registration?.connection ? undefined : core.connectUrl), connection: registration?.connection, stateDirectory: runnerState, identity: o.identity, - ticket: core.issueBootstrapTicket(60_000), runnerBinaryPath: binary, + ticket, runnerBinaryPath: binary, runnerVersion: artifact.version, runnerDigest: artifact.digest, maxOutboxBytes: 16 * 1024 * 1024, p0ReserveBytes: 1024 * 1024, // A managed controller rollout can outlast a minute. Task authority // remains gated by the current controller lease throughout the gap. - maxRuntimeMs: 0, reconnectGraceMs: o.runnerProcessLauncher ? 300_000 : 60_000, + maxRuntimeMs: 0, reconnectGraceMs: o.runnerReconnectGraceMs ?? (o.runnerProcessLauncher ? 300_000 : 60_000), // No agent, OAuth, ChatGPT or provider API credentials are inherited. environment: { PATH: process.env.PATH }, }); @@ -275,6 +284,7 @@ class RunnerdExternalSession implements HarnessSession { criterionIds: o.execution.completionContract.contract.criteria.map(c => c.id) }, }, `${this.codec.commandPrefix}_prepare`); await this.#command("session.open", {}, `${this.codec.commandPrefix}_open`); + await core.recoverPendingSemanticInputs(); signal?.throwIfAborted(); } diff --git a/packages/paperclip-runner/src/index.ts b/packages/paperclip-runner/src/index.ts index a4d3769bb4c..45ad697cbc6 100644 --- a/packages/paperclip-runner/src/index.ts +++ b/packages/paperclip-runner/src/index.ts @@ -38,6 +38,8 @@ export { RUNNER_PRP_PROTOCOL_MAX, inspectWarmRunTransition, type DurablePrpControlPlaneOptions, + type DurablePrpCheckpoint, + decodeDurablePrpCheckpoint, type PrpWireConnection, type PrpWireAttachment, type TransportCloseReason, diff --git a/packages/paperclip-runner/src/live/runner-provider-launch.ts b/packages/paperclip-runner/src/live/runner-provider-launch.ts index 6f2a815e3d3..39fa635f1d6 100644 --- a/packages/paperclip-runner/src/live/runner-provider-launch.ts +++ b/packages/paperclip-runner/src/live/runner-provider-launch.ts @@ -157,7 +157,7 @@ export function validatePreparedRunnerLaunch( * Providers without a filesystem working directory return null. */ export function preparedRunnerLaunchWorkingDirectory( - prepared: PreparedRunnerProviderLaunch, + prepared: Pick, ): string | null { return prepared.provider.cwd === undefined ? null : validatePreparedLaunchWorkingDirectory(prepared.provider.cwd); diff --git a/packages/paperclip-runner/src/live/runnerd-acpx-permission-forwarding.test.ts b/packages/paperclip-runner/src/live/runnerd-acpx-permission-forwarding.test.ts index 14e6351c9c9..f45407609d5 100644 --- a/packages/paperclip-runner/src/live/runnerd-acpx-permission-forwarding.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-acpx-permission-forwarding.test.ts @@ -10,6 +10,7 @@ import type { HarnessRuntimeRequest } from "../contracts/harness-driver.js"; // event pump, registered handler, and request emission below all run unchanged. const peer = vi.hoisted(() => ({ current: null as TestPeer | null, readiness: {} as Record })); interface TestPeer { + readonly recoveredSourceSeq: number; store: { state: { identity: DurableRecoveryIdentity; committedEvents: DurableRecoveryCommittedEvent[]; commands: Array> } }; emit(type: string, payload: Record): void; } @@ -19,6 +20,7 @@ vi.mock("../control-plane/durable-prp-control-plane.js", async (importOriginal) ...actual, spawnRunner: vi.fn(() => ({ child: { exitCode: null }, completion: new Promise(() => {}) })), DurablePrpControlPlane: class implements TestPeer { + readonly recoveredSourceSeq = 0; store: TestPeer["store"]; connectUrl = "ws://fixture.invalid"; constructor(options: { identity: DurableRecoveryIdentity }) { @@ -30,7 +32,10 @@ vi.mock("../control-plane/durable-prp-control-plane.js", async (importOriginal) async drainPendingConnectionProcessing() {} disconnectActiveRunner() {} persistRunAttachTemplate() {} + persistPreparedLaunchConfiguration() {} issueBootstrapTicket() { return {}; } + async flushCheckpoint() {} + async recoverPendingSemanticInputs() {} getCommand(id: string) { return this.store.state.commands.find(c => c.commandId === id); } queueCommand(type: string, payload: Record, id = type) { this.store.state.commands.push({ type, commandId: id, status: "completed", result: { result: { providerTurnId: payload.turnId } } }); diff --git a/packages/paperclip-runner/src/live/runnerd-acpx-permission-ordering.test.ts b/packages/paperclip-runner/src/live/runnerd-acpx-permission-ordering.test.ts index 1dcd3de3875..bdab53a9376 100644 --- a/packages/paperclip-runner/src/live/runnerd-acpx-permission-ordering.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-acpx-permission-ordering.test.ts @@ -8,6 +8,7 @@ import type { DurableRecoveryCommittedEvent, DurableRecoveryIdentity } from "../ // event pump, registered handler, and request emission below all run unchanged. const peer = vi.hoisted(() => ({ current: null as TestPeer | null })); interface TestPeer { + readonly recoveredSourceSeq: number; store: { state: { identity: DurableRecoveryIdentity; committedEvents: DurableRecoveryCommittedEvent[]; commands: Array> } }; emit(type: string, payload: Record): void; } @@ -17,6 +18,7 @@ vi.mock("../control-plane/durable-prp-control-plane.js", async (importOriginal) ...actual, spawnRunner: vi.fn(() => ({ child: { exitCode: null }, completion: new Promise(() => {}) })), DurablePrpControlPlane: class implements TestPeer { + readonly recoveredSourceSeq = 0; store: TestPeer["store"]; connectUrl = "ws://fixture.invalid"; constructor(options: { identity: DurableRecoveryIdentity }) { @@ -28,7 +30,10 @@ vi.mock("../control-plane/durable-prp-control-plane.js", async (importOriginal) async drainPendingConnectionProcessing() {} disconnectActiveRunner() {} persistRunAttachTemplate() {} + persistPreparedLaunchConfiguration() {} issueBootstrapTicket() { return {}; } + async flushCheckpoint() {} + async recoverPendingSemanticInputs() {} getCommand(id: string) { return this.store.state.commands.find(c => c.commandId === id); } queueCommand(type: string, payload: Record, id = type) { this.store.state.commands.push({ type, commandId: id, status: "completed", result: { result: { providerTurnId: payload.turnId } } }); diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts index 26a3e0b20b4..855c41492a3 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts @@ -608,6 +608,24 @@ it("requires an explicit retained state directory before adopting a runner", () expect(signal).not.toHaveBeenCalled(); }); +it("rejects an adoption authorization hook without launch preparation", () => { + const launch = vi.fn(); + const registration = vi.fn(); + const authorizeRunnerAdoption = vi.fn(); + expect(() => createCapabilityRunnerdCodexTransport({ + stateDirectory: tmpdir(), + runnerProcessLauncher: launch, + controlPlaneRegistration: registration, + authorizeRunnerAdoption, + adoptExistingRunner: { + pid: 123, processGroupId: 123, startedAt: null, isAlive: () => true, + }, + })).toThrow("runner_adoption_authorization_requires_launch_preparation"); + expect(launch).not.toHaveBeenCalled(); + expect(registration).not.toHaveBeenCalled(); + expect(authorizeRunnerAdoption).not.toHaveBeenCalled(); +}); + it("reads valid control-plane history above 64 MiB and rejects it above 256 MiB", async () => { const root = await mkdtemp(join(tmpdir(), "runnerd-large-control-state-")); const stateDirectory = join(root, "control-plane"); @@ -1103,6 +1121,44 @@ it("refuses a reusable close checkpoint when the local provider snapshot is unre } }, 15_000); +it("uses the same API result envelope for live dispatch and recovered receipts", async () => { + const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-recovered-api-")); + const OriginalCore = durableControlPlane.DurablePrpControlPlane; + let callbacks!: ConstructorParameters[0]; + let core!: DurablePrpControlPlane; + const coreSpy = vi.spyOn(durableControlPlane, "DurablePrpControlPlane").mockImplementation(function (options) { + callbacks = options; + return core = new OriginalCore(options); + }); + let outcome: { result: unknown; isError?: boolean } | null = { + result: { ok: true, apiOperationId: "POST /api/example", data: { saved: true } }, isError: false, + }; + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), codexCommand: fakeCodex, + codexArgs: fakeCodexArgs(stateDirectory), stateDirectory, + recoverSemanticToolInput: async () => outcome, + }); + bundle.transport.setServerRequestHandler(async () => ({ success: !outcome?.isError, + contentItems: [{ type: "inputText", text: JSON.stringify(outcome?.result) }] })); + try { + await bundle.transport.request("thread/start", { cwd: tmpdir() }); + const call = { callId: "recovered-api", operationId: "call_api", input: { operationId: "POST /api/example" }, + sourceEventId: "saved-event", sourceEventType: "semantic_tool.input", correlation: core.store.state.identity }; + const live = await callbacks.onSemanticToolInput!(call); + expect(live).toMatchObject({ isError: false, result: { ok: true, operationId: "call_api", callId: call.callId, result: outcome!.result } }); + expect(await callbacks.recoverSemanticToolInput!(call)).toMatchObject({ result: live.result, isError: live.isError }); + outcome = { result: { error: "saved failure" }, isError: true }; + const liveError = await callbacks.onSemanticToolInput!(call); + expect(await callbacks.recoverSemanticToolInput!(call)).toMatchObject({ result: liveError.result, isError: liveError.isError }); + outcome = null; + expect(await callbacks.recoverSemanticToolInput!(call)).toBeNull(); + } finally { + await bundle.transport.close(); + coreSpy.mockRestore(); + await rm(stateDirectory, { recursive: true, force: true }); + } +}, 20_000); + it.each(["after_budget", "within_budget", "interrupted_within_budget", "persistence_failure"] as const)( "fences reusable suspension against late semantic completion (%s)", async (mode) => { @@ -8374,9 +8430,19 @@ async function verifyLiveRunnerAdoption( goalMidTurn = false, detachBeforeCleanup = false, startWithoutCheckpoint = false, + preparedTask = false, + adoptionAuthorization: "allow" | "deny" | "legacy" | "legacy-missing" | "missing-binding" = "allow", + toolHandlerOrder?: "before" | "after" | "completion-before" | "completion-after", uncertainAtClose = false, ) { const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-live-adopt-")); + const pendingCompletion = toolHandlerOrder?.startsWith("completion-") === true; + const completionPath = join(stateDirectory, "pending-completion.json"); + if (pendingCompletion) await writeFile(completionPath, JSON.stringify({ + schema: "paperclip.run_result.v1", reportedWorkDisposition: "done", summary: "Completed the synthetic work", + completionClaim: { contractRevision: "1", objectiveSatisfied: true, criteria: [], remainingWork: [] }, + evidence: [], verification: [], attentionRequests: [], artifacts: [], + })); const server = createServer(); let authority: DurablePrpControlPlane | null = null; const checkpoint = vi.fn(async () => undefined); @@ -8393,7 +8459,7 @@ async function verifyLiveRunnerAdoption( const address = server.address(); if (!address || typeof address === "string") throw new Error("Expected adoption test listener"); - const registration = async (next: DurablePrpControlPlane) => { + const registration = vi.fn(async (next: DurablePrpControlPlane) => { authority = next; return { connectUrl: `ws://127.0.0.1:${address.port}/runner`, @@ -8402,7 +8468,7 @@ async function verifyLiveRunnerAdoption( if (authority === next) authority = null; }, }; - }; + }); const identity = { runnerInstanceId: "runner-live-adopt", environmentLeaseId: "lease-live-adopt", @@ -8414,14 +8480,36 @@ async function verifyLiveRunnerAdoption( const sharedOptions = { runnerBinary: defaultCapabilityRunnerdBinary(), codexCommand: fakeCodex, - codexArgs: fakeCodexArgs(stateDirectory, ...(goalMidTurn ? ["--goal-autostart", "--goal-item-trigger", join(stateDirectory, "emit-goal-item")] : [])), + codexArgs: fakeCodexArgs(stateDirectory, ...(pendingCompletion ? ["--terminal-result-file", completionPath] : toolHandlerOrder ? ["--emit-tool-call"] : []), ...(goalMidTurn ? ["--goal-autostart", "--goal-item-trigger", join(stateDirectory, "emit-goal-item")] : [])), stateDirectory, prpIdentity: identity, lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 60_000 }, controlPlaneRegistration: registration, }; - const first = createCapabilityRunnerdCodexTransport(sharedOptions); + const prepareLaunch = vi.fn(async (requirements: RunnerProviderLaunchRequirements) => ({ + configurationId: "adopted-preparation", requirementsDigest: runnerLaunchRequirementsDigest(requirements), + provider: createRunnerProviderLaunch(requirements, { options: { codexArgs: sharedOptions.codexArgs }, + root: stateDirectory, cwd: tmpdir(), codexHome: join(stateDirectory, "codex-home"), + codexCommand: fakeCodex, providerNodeCommand: process.execPath, opencodeProxyPath: "", acpxSidecarPath: "" }), + })); + const options = { ...sharedOptions, ...(preparedTask ? { + prepareLaunch, runnerArtifact: readRunnerdArtifactBinding(defaultCapabilityRunnerdBinary()), + runnerProcessLauncher: (spec: durableControlPlane.RunnerProcessLaunchSpec) => { + const child = spawn(spec.command, [...spec.args], { cwd: spec.cwd, env: { ...process.env, ...spec.environment }, + detached: process.platform !== "win32", stdio: "ignore" }); + return { child, processGroupId: child.pid ?? null, completion: new Promise<{ code: number | null; signal: NodeJS.Signals | null; stdout: string; stderr: string }>((resolveExit, reject) => { + child.once("error", reject); + child.once("exit", (code, signal) => resolveExit({ code, signal, stdout: "", stderr: "" })); + }) }; + }, + } : {}) }; + const first = createCapabilityRunnerdCodexTransport(options); + const pendingTool = vi.fn(() => new Promise>(() => undefined)); + if (toolHandlerOrder) first.transport.setServerRequestHandler(pendingTool); + const recoveredTool = vi.fn(async () => ({ result: { receipt: "known-outcome" }, isError: false })); + const redispatchedTool = vi.fn(async () => ({ success: true, contentItems: [] })); let runnerPid: number | null = null; + let runnerStartedAt: string | null = null; let adopted: ReturnType | null = null; let restoreIntervalSpies: () => void = () => undefined; @@ -8430,7 +8518,7 @@ async function verifyLiveRunnerAdoption( try { opened = await first.transport.request("thread/start", { cwd: tmpdir(), - dynamicTools: codexSemanticToolSpecs(), + dynamicTools: toolHandlerOrder && !pendingCompletion ? [{ name: "get_task_context", description: "Read current task", inputSchema: { type: "object", properties: {}, additionalProperties: false } }] : codexSemanticToolSpecs(), }); } catch (error) { const stderr = await readFile( @@ -8442,6 +8530,7 @@ async function verifyLiveRunnerAdoption( ); } runnerPid = first.evidence().runnerPid; + runnerStartedAt = first.transport.processInfo().startedAt; expect(runnerPid).toEqual(expect.any(Number)); if (goalMidTurn) { @@ -8451,6 +8540,10 @@ async function verifyLiveRunnerAdoption( } } + if (toolHandlerOrder) { + await first.transport.request("turn/start", { input: [{ type: "text", text: "Read current task" }] }); + await vi.waitFor(() => expect(pendingTool).toHaveBeenCalledOnce(), { timeout: 5_000 }); + } await first.detachControllerForRestart(); expect(() => process.kill(runnerPid!, 0)).not.toThrow(); if (goalMidTurn) { @@ -8487,6 +8580,20 @@ async function verifyLiveRunnerAdoption( }; if (!startWithoutCheckpoint) await compactProviderIdentityEvents(); + if (preparedTask) { + if (adoptionAuthorization !== "legacy") await rm(join(stateDirectory, "launch-preparation.json")); + prepareLaunch.mockClear(); + registration.mockClear(); + if (adoptionAuthorization === "missing-binding") { + const retained = JSON.parse(await readFile(controlPlaneStatePath, "utf8")); + delete retained.preparedLaunchConfigurations; + await writeFile(controlPlaneStatePath, JSON.stringify(retained), { mode: 0o600 }); + } + } + const authorizeRunnerAdoption = vi.fn(async () => { + expect(registration).not.toHaveBeenCalled(); + if (adoptionAuthorization === "deny") throw new Error("project grant revoked"); + }); const duplicateLauncher = vi.fn(() => { throw new Error("duplicate runner spawn attempted"); }); @@ -8498,7 +8605,8 @@ async function verifyLiveRunnerAdoption( const clearedIntervals = uncertainAtClose ? vi.spyOn(globalThis, "clearInterval") : null; restoreIntervalSpies = () => { intervals?.mockRestore(); clearedIntervals?.mockRestore(); }; adopted = createCapabilityRunnerdCodexTransport({ - ...sharedOptions, + ...options, + ...(preparedTask && !adoptionAuthorization.startsWith("legacy") ? { authorizeRunnerAdoption } : {}), ...(uncertainAtClose ? { closeGraceMs: 2_000 } : {}), // Hash different stable bytes without replacing the real runner artifact // used by concurrent tests. Adoption must never execute this path. @@ -8508,7 +8616,8 @@ async function verifyLiveRunnerAdoption( runnerReconnectGraceMs: 150, } : {}), - resumeDynamicTools: [], + ...(toolHandlerOrder && !pendingCompletion ? { recoverSemanticToolInput: recoveredTool } : {}), + resumeDynamicTools: pendingCompletion ? codexSemanticToolSpecs() : toolHandlerOrder ? [{ name: "get_task_context", description: "Read current task", inputSchema: { type: "object", properties: {}, additionalProperties: false } }] : [], resumeProviderSession: startWithoutCheckpoint ? undefined : { driverSessionId: String(openedThread.id), providerSessionId: mismatchedCheckpoint @@ -8519,7 +8628,7 @@ async function verifyLiveRunnerAdoption( adoptExistingRunner: { pid: runnerPid!, processGroupId: runnerPid, - startedAt: new Date().toISOString(), + startedAt: runnerStartedAt, signal, isAlive: () => { if (rejectLiveness) return Promise.reject(probeError); @@ -8532,6 +8641,18 @@ async function verifyLiveRunnerAdoption( }, }, }); + if (preparedTask && ["deny", "legacy-missing", "missing-binding"].includes(adoptionAuthorization)) { + await expect(adopted.transport.request("thread/read", {})).rejects.toThrow( + adoptionAuthorization === "deny" ? "project grant revoked" : adoptionAuthorization === "legacy-missing" + ? "runner_launch_recovery_requirements_unavailable" : "runner_launch_recovery_configuration_unavailable"); + expect(registration).not.toHaveBeenCalled(); + expect(prepareLaunch).not.toHaveBeenCalled(); + expect(duplicateLauncher).not.toHaveBeenCalled(); + expect(signal).not.toHaveBeenCalled(); + expect(() => process.kill(runnerPid!, 0)).not.toThrow(); + expect(authorizeRunnerAdoption).toHaveBeenCalledTimes(adoptionAuthorization === "deny" ? 1 : 0); + return; + } if (mismatchedArtifact) { await expect( adopted.transport.request("thread/read", {}), @@ -8576,12 +8697,26 @@ async function verifyLiveRunnerAdoption( expect(() => process.kill(runnerPid!, 0)).not.toThrow(); return; } + if (toolHandlerOrder?.endsWith("before")) adopted.transport.setServerRequestHandler(redispatchedTool); await expect(adopted.transport.request(startWithoutCheckpoint ? "thread/start" : "thread/read", {})).resolves.toEqual( expect.objectContaining({ thread: expect.objectContaining({ id: "codex-thread-1" }), }), ); + if (toolHandlerOrder?.endsWith("after")) adopted.transport.setServerRequestHandler(redispatchedTool); + if (toolHandlerOrder) { + await vi.waitFor(() => { + expect(pendingCompletion ? redispatchedTool : recoveredTool).toHaveBeenCalledOnce(); + const result = authority!.store.state.commands.find(command => command.type === "semantic_tool.result"); + expect(result?.status).toBe("completed"); + }, { timeout: 5_000 }); + if (pendingCompletion) { + expect(recoveredTool).not.toHaveBeenCalled(); + expect(redispatchedTool).toHaveBeenCalledWith(expect.objectContaining({ params: expect.objectContaining({ tool: "paperclip_finish" }) })); + } else expect(redispatchedTool).not.toHaveBeenCalled(); + } expect(adopted.evidence().runnerPid).toBe(runnerPid); + expect(adopted.transport.processInfo().startedAt).toBe(runnerStartedAt); if (startWithoutCheckpoint) { const retained = JSON.parse(await readFile(controlPlaneStatePath, "utf8")); for (const type of ["run.prepare", "session.open"]) { @@ -8601,6 +8736,16 @@ async function verifyLiveRunnerAdoption( expect(observed.params).toMatchObject({ threadId: "codex-thread-1", turnId: "provider-goal-turn-1" }); } expect(duplicateLauncher).not.toHaveBeenCalled(); + if (preparedTask) { + if (adoptionAuthorization === "legacy") { + expect(prepareLaunch).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ identity }), expect.objectContaining({ configurationId: "adopted-preparation" })); + expect(authorizeRunnerAdoption).not.toHaveBeenCalled(); + } else { + expect(prepareLaunch).not.toHaveBeenCalled(); + expect(authorizeRunnerAdoption).toHaveBeenCalledExactlyOnceWith("adopted-preparation", identity); + } + expect((await adopted.transport.request("thread/read", {})).thread).toMatchObject({ cwd: tmpdir() }); + } if (uncertainAtClose) { const before = JSON.parse(await readFile(controlPlaneStatePath, "utf8")); const retainedAuthority = authority as DurablePrpControlPlane | null; @@ -8680,7 +8825,12 @@ it( 30_000, ); -it("retains authenticated adopted-runner ownership when a close probe rejects", () => verifyLiveRunnerAdoption(false, false, false, false, false, true), 30_000); +it("adopts a prepared runner after losing its controller launch cache without preparing resources again", + () => verifyLiveRunnerAdoption(false, false, false, false, false, true), 30_000); + +it.each(["deny", "legacy", "legacy-missing", "missing-binding"] as const)("preserves adoption authorization with %s", + mode => verifyLiveRunnerAdoption(false, false, false, false, false, true, mode), 30_000); +it("retains authenticated adopted-runner ownership when a close probe rejects", () => verifyLiveRunnerAdoption(false, false, false, false, false, false, "allow", undefined, true), 30_000); it( "blocks adopted runner artifact drift without duplicate launch, checkpoint replacement, or process signals", @@ -9027,9 +9177,10 @@ it.each([true, false])("preserves prepared OpenCode cleanup errors (primary fail }); it.each([ - { model: "openrouter/deepseek/deepseek-v4-flash-0731", supportsReasoning: true }, - { model: "anthropic/claude-sonnet-4", supportsReasoning: false }, -])("preserves prepared input and completion feedback through runnerd and the real OpenCode proxy boundary ($model)", async ({ model, supportsReasoning }) => { + { model: "openrouter/deepseek/deepseek-v4-flash-0731", supportsReasoning: true, adoptAfterTurn: false }, + { model: "openrouter/deepseek/deepseek-v4-flash-0731", supportsReasoning: true, adoptAfterTurn: true }, + { model: "anthropic/claude-sonnet-4", supportsReasoning: false, adoptAfterTurn: false }, +])("preserves prepared input and completion feedback through runnerd and the real OpenCode proxy boundary ($model, live adoption: $adoptAfterTurn)", async ({ model, supportsReasoning, adoptAfterTurn }) => { const root = await mkdtemp(join(tmpdir(), "runnerd-prepared-opencode-")); // GitHub-hosted Linux toolcache Node can be group-writable, unlike the AWS // fleet. Qualify an owned copy with strict permissions, never chmod the host @@ -9062,7 +9213,30 @@ it.each([ await writeFile(proxy, proxyBytes, { mode: 0o755 }); const digest = (file: string) => `sha256:${createHash("sha256").update(readFileSync(file)).digest("hex")}`; const runtime = join(root, "opencode"); + const server = createServer(); + let authority: DurablePrpControlPlane | null = null; + server.on("upgrade", (request, socket, head) => { + if (authority) authority.handleUpgrade(request, socket, "/runner", head); + else socket.destroy(); + }); + await new Promise(resolveListen => server.listen(0, "127.0.0.1", resolveListen)); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("Expected OpenCode adoption listener"); + const registration = async (next: DurablePrpControlPlane) => { + authority = next; + return { + connectUrl: `ws://127.0.0.1:${address.port}/runner`, + release: async () => { if (authority === next) authority = null; }, + }; + }; + const identity = { + runnerInstanceId: "runner-opencode-adopt", environmentLeaseId: "lease-opencode-adopt", + runId: "prepared-opencode", normalizedSessionId: "prepared-opencode", + turnId: "turn-opencode-adopt", itemId: "item-opencode-adopt", + }; const bundle = createCapabilityRunnerdCodexTransport({ + prpIdentity: identity, + controlPlaneRegistration: registration, provider: "opencode", runnerBinary: defaultCapabilityRunnerdBinary(), stateDirectory: join(root, "runner-state"), @@ -9097,6 +9271,8 @@ it.each([ }, }); let session: Awaited> | undefined; + let adopted: ReturnType | undefined; + let survivingPid: number | null = null; const prepared = JSON.stringify({ schema: "paperclip.native-model-envelope.v3", task: { prompt: "Keep this completion-feedback request unchanged." }, @@ -9139,6 +9315,54 @@ it.each([ expect(outcomes).toHaveLength(2); expect(outcomes[0].result).toMatchObject({ isError: true, content: [{ text: expect.stringContaining("required document link") }] }); expect(outcomes[1].result).toMatchObject({ content: [{ text: expect.stringContaining(feedback) }] }); + if (adoptAfterTurn) { + const snapshot = await session.snapshot(); + const runnerStartedAt = bundle.transport.processInfo().startedAt; + survivingPid = bundle.evidence().runnerPid; + expect(survivingPid).toEqual(expect.any(Number)); + const artifact = readRunnerdArtifactBinding(defaultCapabilityRunnerdBinary()); + await bundle.detachControllerForRestart(); + // These controller launch assets are gone. The original native proxy + // and harness remain alive; only authenticated adoption is allowed. + await rm(proxy); + await rm(executable); + const duplicateLauncher = vi.fn(() => { throw new Error("duplicate OpenCode runner launch"); }); + adopted = createCapabilityRunnerdCodexTransport({ + provider: "opencode", + stateDirectory: join(root, "runner-state"), + runnerFilesystemRoot: join(root, "retired-controller"), + runnerBinary: join(root, "retired-controller", "runnerd"), + runnerArtifact: artifact, + prpIdentity: identity, + controlPlaneRegistration: registration, + resumeProviderSession: { + driverSessionId: snapshot.driverSessionId, + providerSessionId: snapshot.providerSessionId, + providerIdentity: snapshot.providerIdentity, + }, + runnerProcessLauncher: duplicateLauncher, + adoptExistingRunner: { + pid: survivingPid!, processGroupId: survivingPid, startedAt: runnerStartedAt, + signal: () => true, + isAlive: () => { try { process.kill(survivingPid!, 0); return true; } catch { return false; } }, + }, + }); + const recoveryDriver = new CodexAppServerDriver({ + taskEnvelope: task, conversationMode: "prepared", model, + transportFactory: () => adopted!.transport, + workingDirectoryAuthority: "remote_runner", + environment: { PAPERCLIP_WORKSPACE_CWD: root }, + completionFeedback: async () => feedback, + }); + const recovery = await recoveryDriver.recoverSession(snapshot); + if (!recovery.recovered) throw new Error(`OpenCode live adoption failed: ${recovery.reason}`); + expect(recovery.session).toBeDefined(); + session = recovery.session!; + expect(adopted.evidence().runnerPid).toBe(survivingPid); + expect(adopted.transport.processInfo().startedAt).toBe(runnerStartedAt); + expect((await session.snapshot()).providerSessionId).toBe(snapshot.providerSessionId); + expect(duplicateLauncher).not.toHaveBeenCalled(); + } await session.startTurn({ message: { role: "user", text: "Reply Hi." } }); for await (const event of session.events()) if (event.eventType === "turn.completed") break; const allRequests = (await readFile(join(runtime, sessionRoots[0]!.name, "data/fake-prompt-requests.ndjson"), "utf8")).trim().split("\n").map((line) => JSON.parse(line)); @@ -9146,9 +9370,18 @@ it.each([ ? ["paperclip-no-reasoning", "paperclip-default"] : [undefined, undefined]); }, closeSession: async () => { await session?.close(); }, - closeTransport: () => bundle.transport.close(), - removeRoot: () => rm(root, { recursive: true, force: true }), - evidence: () => bundle.evidence(), + closeTransport: async () => { + try { await adopted?.transport.close(); } + finally { + await bundle.transport.close(); + if (survivingPid !== null) { try { process.kill(survivingPid, "SIGKILL"); } catch {} } + } + }, + removeRoot: async () => { + await new Promise((resolveClose, reject) => server.close(error => error ? reject(error) : resolveClose())); + await rm(root, { recursive: true, force: true }); + }, + evidence: () => adopted?.evidence() ?? bundle.evidence(), }); }, 30_000); @@ -9261,3 +9494,10 @@ it("launches and closes native Codex without materializing or editing the precon await rm(root, { force: true, recursive: true }); } }, 30_000); + + +it.each(["before", "after"] as const)("settles a saved tool outcome with the handler installed %s recovery startup", order => + verifyLiveRunnerAdoption(false, false, false, false, false, false, "allow", order), 30_000); + +it.each(["completion-before", "completion-after"] as const)("recovers a pending finish without an ordinary-tool lookup (%s)", order => + verifyLiveRunnerAdoption(false, false, false, false, false, false, "allow", order), 30_000); diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts index c05c6a14894..912fb0de809 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts @@ -1238,6 +1238,7 @@ export interface CapabilityRunnerdCodexTransportOptions extends RunnerExecutionD provider?: "codex" | "opencode" | "claude_managed" | "aws_agentcore" | "acpx"; observeCredentials?: boolean; onCredentialsUpdated?: (event: PrpEvent) => Promise; + recoverSemanticToolInput?: DurablePrpControlPlaneOptions["recoverSemanticToolInput"]; opencodePermissionMode?: NativeOpenCodePermissionMode; acpxAgent?: QualifiedAcpxAgent; /** Explicit evaluation-only candidate selection, never derived from persisted session input. */ @@ -3603,6 +3604,9 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { readonly #bridgedRuntimeInputs = new Map(); constructor(readonly options: CapabilityRunnerdCodexTransportOptions) { + if (options.adoptExistingRunner && options.authorizeRunnerAdoption && !options.prepareLaunch) { + throw new Error("runner_adoption_authorization_requires_launch_preparation"); + } if (options.prepareLaunch && (!options.runnerProcessLauncher || !options.controlPlaneRegistration || !options.runnerArtifact)) { throw new Error("runner_launch_preparation_requires_process_owner_and_artifact"); } @@ -4025,6 +4029,14 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { this.#handler = handler; this.#handlerInstalled = true; this.#resolveHandlerReady(); + this.#recoverPendingSemanticInputsWhenReady(); + } + + #recoverPendingSemanticInputsWhenReady(): void { + if (this.#startupComplete && this.#handlerInstalled) { + void Promise.resolve().then(() => this.#core?.recoverPendingSemanticInputs()) + .catch(error => this.#failTransport(error instanceof Error ? error : new Error(String(error)))); + } } async #awaitWarmRunAttachmentReady(): Promise { @@ -4312,7 +4324,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { return { pid: this.#evidence.runnerPid, processGroupId: this.#evidence.runnerProcessGroupId, - startedAt: this.#startedAt, + startedAt: this.options.adoptExistingRunner?.startedAt ?? this.#handle?.startedAt ?? this.#startedAt, exited: this.#evidence.runnerExited, exitCode: this.#evidence.runnerExitCode, signal: this.#evidence.runnerSignal, @@ -4327,7 +4339,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { await this.options.onSpawn?.({ pid: handle.child.pid, processGroupId: handle.processGroupId ?? null, - startedAt: this.#startedAt, + startedAt: handle.startedAt ?? this.#startedAt, }); } } @@ -4837,12 +4849,13 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { this.#assertStartupOpen(); template.provider = prepared.provider; new RunnerLaunchPreparationStore(this.#root).save(requirements, prepared, this.#core!.store.state.identity); + this.#core!.persistPreparedLaunchConfiguration(identity, prepared.configurationId); return prepared; } // The prepared launch is the only source of the provider directory. Core // records it in session metadata; the session cannot move to another one. - #recordPreparedWorkingDirectory(prepared: PreparedRunnerProviderLaunch): void { + #recordPreparedWorkingDirectory(prepared: Pick): void { const workingDirectory = preparedRunnerLaunchWorkingDirectory(prepared); if (workingDirectory === null) { throw new Error("runner_prepared_launch_working_directory_unavailable"); @@ -4876,6 +4889,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { this.#recordPreparedWorkingDirectory(prepared); this.#assertStartupOpen(); new RunnerLaunchPreparationStore(this.#root).save(requirements, prepared, this.#core!.store.state.identity); + this.#core!.persistPreparedLaunchConfiguration(identity, prepared.configurationId); const completionContract = record(params.completionContract); return { runAttachTemplate: { observeCredentials: this.options.observeCredentials !== false, @@ -5090,6 +5104,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { expectedRunnerDigest: runnerArtifact.digest, onProtocolIntegrityError: (error) => this.#failTransport(error), onSemanticToolInput: (call) => this.#handleSemanticToolInput(call), + recoverSemanticToolInput: call => this.#recoverSemanticToolInput(call), onCommittedEvent: async (event) => { if (event.eventType === "auth.credentials.updated") { await this.options.onCredentialsUpdated?.(event); @@ -5134,6 +5149,9 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { } this.#assertStartupOpen(); const admissionDeadline = this.#coldAdmissionDeadline(); + const bootstrapTicket = core.issueBootstrapTicket(RUNNER_BOOTSTRAP_TICKET_TTL_MS); + await core.flushCheckpoint(); + this.#assertStartupOpen(); const handle = spawnRunner({ connection: registration?.connection ?? { mode: "connect", @@ -5142,7 +5160,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { stateDirectory: this.options.runnerStateDirectory ?? resolve(this.#root, "runner"), identity, - ticket: core.issueBootstrapTicket(RUNNER_BOOTSTRAP_TICKET_TTL_MS), + ticket: bootstrapTicket, maxOutboxBytes: RUNNERD_MAX_OUTBOX_BYTES, p0ReserveBytes: RUNNERD_P0_RESERVE_BYTES, maxRuntimeMs: 0, @@ -5444,6 +5462,9 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { this.#durableTurnId = identity.turnId; this.#eventIdentity = structuredClone(identity); const provider = this.options.provider ?? "codex"; + // Adoption authenticates the original artifact and process. Launch profiles + // belong only to a new spawn and may reference a retired controller build. + const preparesProviderLaunch = !this.options.prepareLaunch && !this.options.adoptExistingRunner; if (!this.options.prepareLaunch && provider === "codex" && (this.options.homeMode === "preconfigured" || record(record(controlPlaneState!.runAttachTemplate).provider).preconfiguredHome !== undefined)) { const seed = rotatedRunAttachPayload(controlPlaneState!, desiredIdentity, this.#authorizedTools, this.options.resumeCompletionContract); const priorHome = record(seed.provider).preconfiguredHome; @@ -5465,14 +5486,14 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { ? resolve(this.options.runnerFilesystemRoot, "codex-home") : localCodexHome; const opencodeProxyPath = - (this.options.prepareLaunch ? "" : undefined) ?? this.options.opencodeProxyPath ?? + (preparesProviderLaunch ? undefined : "") ?? this.options.opencodeProxyPath ?? (provider === "opencode" && !this.options.runnerFilesystemRoot ? resolveBuildOwnedCliArtifact("opencode-app-server-proxy.cjs") : fileURLToPath( new URL("../cli/opencode-app-server-proxy.cjs", import.meta.url), )); const acpxSidecarPath = - (this.options.prepareLaunch ? "" : undefined) ?? this.options.acpxSidecarPath ?? + (preparesProviderLaunch ? undefined : "") ?? this.options.acpxSidecarPath ?? (provider === "acpx" && !this.options.runnerFilesystemRoot ? resolveBuildOwnedCliArtifact("acpx-runtime-sidecar.cjs") : fileURLToPath( @@ -5481,11 +5502,11 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { const providerNodeCommand = this.options.providerNodeCommand ?? process.execPath; const opencodeExecutable = - provider === "opencode" && !this.options.prepareLaunch + provider === "opencode" && preparesProviderLaunch ? resolveRunnerOpenCodeExecutable(this.options) : null; const runnerAcpxLaunchProfile = - provider === "acpx" && !this.options.prepareLaunch + provider === "acpx" && preparesProviderLaunch ? acpxRunnerLaunchProfile( this.options, providerNodeCommand, @@ -5493,7 +5514,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { ) : undefined; const runnerOpenCodeLaunchProfile = - provider === "opencode" && !this.options.prepareLaunch + provider === "opencode" && preparesProviderLaunch ? opencodeRunnerLaunchProfile( this.options, providerNodeCommand, @@ -5508,6 +5529,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { expectedRunnerDigest: runnerArtifact.digest, onProtocolIntegrityError: (error) => this.#failTransport(error), onSemanticToolInput: (call) => this.#handleSemanticToolInput(call), + recoverSemanticToolInput: call => this.#recoverSemanticToolInput(call), onCommittedEvent: async (event) => { if (event.eventType === "auth.credentials.updated") { await this.options.onCredentialsUpdated?.(event); @@ -5535,8 +5557,19 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { connectionLeaseTtlMs: 60 * 60 * 1_000, }); this.#core = core; + // Reusing resources still requires current environment authorization. The + // optional hook avoids re-preparation after controller storage is lost. + // Existing consumers retain the preparation callback's authorization path. + const authorizeAdoption = this.options.adoptExistingRunner && this.options.authorizeRunnerAdoption; + if (this.options.prepareLaunch && authorizeAdoption) { + const configurationId = core.preparedLaunchConfigurationId(desiredIdentity); + if (!configurationId) throw new Error("runner_launch_recovery_configuration_unavailable"); + await authorizeAdoption(configurationId, structuredClone(desiredIdentity)); + this.#assertStartupOpen(); + this.#recordPreparedWorkingDirectory({ provider: record(record(controlPlaneState!.runAttachTemplate).provider) }); + } let environmentPrepared: PreparedRunnerProviderLaunch | undefined; - if (this.options.prepareLaunch) { + if (this.options.prepareLaunch && !authorizeAdoption) { const preparations = new RunnerLaunchPreparationStore(this.#root); const saved = preparations.load(desiredIdentity); const previous = preparations.load(controlPlaneIdentity(controlPlaneState!)); @@ -5812,6 +5845,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { RUNNER_BOOTSTRAP_TICKET_TTL_MS, ) : core.issueBootstrapTicket(RUNNER_BOOTSTRAP_TICKET_TTL_MS); + await core.flushCheckpoint(); if (warmRecovery) { await this.options.authorizeWarmTransitionRecovery?.("before_spawn"); } @@ -5963,6 +5997,15 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { ? "runnerd attached the durable provider session to a fresh PRP run authority" : "runnerd restored its durable PRP session and provider thread", ); + this.#recoverPendingSemanticInputsWhenReady(); + } + + async #recoverSemanticToolInput(call: Parameters>[0]) { + if (call.operationId === "paperclip_finish" || call.operationId === "paperclip_block") { + return this.#handleSemanticToolInput(call); + } + const outcome = await this.options.recoverSemanticToolInput?.(call); + return outcome ? this.#bindSemanticToolOutcome(call, outcome) : null; } async #handleSemanticToolInput( @@ -6042,15 +6085,16 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { this.options.provider === "acpx" && (call.operationId === "paperclip_finish" || call.operationId === "paperclip_block"), ); + return this.#bindSemanticToolOutcome(call, outcome); + } + + #bindSemanticToolOutcome(call: { operationId: string; callId: string }, outcome: { result: unknown; isError?: boolean }) { if (call.operationId === "call_api") { const result = record(outcome.result); if ( result.operationId !== call.operationId || result.callId !== call.callId ) { - // HTTP receipts also contain `ok` and `operationId` (the HTTP route). - // Bind that application value inside a real semantic envelope so the - // runner cannot mistake the route for the provider tool's identity. return { ...outcome, result: { @@ -6378,8 +6422,14 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { const event = deferredEvent ?? events.find((candidate) => candidate.sourceSeq > this.#eventSourceSeq); - if (event === undefined) return; + if (event === undefined) { + this.#eventSourceSeq = Math.max(this.#eventSourceSeq, core.recoveredSourceSeq); + return; + } const fromDeferredQueue = deferredEvent !== undefined; + if (!fromDeferredQueue && event.sourceSeq > this.#eventSourceSeq + 1) { + this.#eventSourceSeq = Math.max(this.#eventSourceSeq, Math.min(event.sourceSeq - 1, core.recoveredSourceSeq)); + } if (!fromDeferredQueue && event.sourceSeq !== this.#eventSourceSeq + 1) { throw new Error( `PRP provider event window advanced past source sequence ${this.#eventSourceSeq + 1}`, @@ -7118,9 +7168,10 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { const priorConnectionCount = this.#core.store.state.connectionCount; let recoveredHandle: RunnerProcessHandle; try { - recoveredHandle = restart( - this.#core.issueBootstrapTicket(RUNNER_BOOTSTRAP_TICKET_TTL_MS), - ); + const ticket = this.#core.issueBootstrapTicket(RUNNER_BOOTSTRAP_TICKET_TTL_MS); + await this.#core.flushCheckpoint(); + if (this.#closed || this.#failure !== null) return; + recoveredHandle = restart(ticket); } catch (error) { lastDetail = error instanceof Error ? error.message : String(error); attempt += 1; diff --git a/packages/paperclip-runner/test/fixtures/copilot-profile-v59-hot-upgrade-candidate-historical-identity.json b/packages/paperclip-runner/test/fixtures/copilot-profile-v59-hot-upgrade-candidate-historical-identity.json new file mode 100644 index 00000000000..a623aa0d80c --- /dev/null +++ b/packages/paperclip-runner/test/fixtures/copilot-profile-v59-hot-upgrade-candidate-historical-identity.json @@ -0,0 +1,85 @@ +{ + "commandDigest": "sha256:4418740ed8d2ada631094e37b25df5e2fa374fda3c7b66d5045c35fd79ca19b1", + "declaration": { + "schema": "paperclip.acpx_profile_declaration.v1", + "agent": "copilot", + "agentProfileVersion": 59, + "acpxVersion": "0.13.1", + "agentServerVersion": "1.0.88", + "protocolPolicyRevision": "copilot-agent-manual-v1", + "acpxPatchSha256": "00d6af17216cb0cad902b48633eb19bf0ec6b05837a984cb24661fca593f09c1", + "policySha256": "a4be69be43db84052560420a3ca546a665c9165573c24d543eb33d144ba138a3", + "distributionSourceSha256": "c0a65e247fc47b263bd369a3d5ab505c4b323d4e6493be8a9ecbeb0e9dbb9107", + "agentFilesBinding": "registered-runtime-context-v1", + "systemInstructionDelivery": "COPILOT_HOME/copilot-instructions.md:replace-under-lifetime-lease-before-launch:v1", + "sharedRuntimeContract": "paperclip.acpx-runtime-contract.v1", + "innerDistributionSourceSha256": "9bb08347eed01f1ab44e065c8b06ead9be3a25973d6e6329dc4c0a099096279a", + "upstreamAppSha256": "7b48282a19b5b0814a0c96ad5e3a125173d792cc55f9525b2effea962f6063c0", + "patchedAppSha256": "61f8eebe0c30cc03b311c6818b2acb158a6d16ca58c8bfac5ebd42db095c2679", + "messageIdentityContract": "copilot-native-message-id-v1", + "ownedDistributionDelivery": "COPILOT_CLI_DIST_DIR:lease-owned-guarded-inner-distribution:v1", + "permissionContextContract": "copilot-edit-permission-context-v1", + "permissionContextSourceSha256": "de2d38d702b3c574161af0c0c04790fb6f6347916eb3916ce7a131a8c229121b", + "permissionAdapterSourceSha256": "ee5929ed2ee647ad0369465dee9d51a00b2dd28e1ff735b43e969ddc9c164668", + "permissionLocationsSourceSha256": "f39a81da395b6e23aa2a1a45572a28846e5e81bfd3d764d9a456f74a79c752e9", + "permissionRedactionSourceSha256": "ad6770c69269087951790e2e87e22ac088e3d432769adf17ebc865ed512e6c4f", + "permissionClassifierSourceSha256": "1161eabd82e26133e3b3db3ab567f65d52808b03f176e93497e02c8330349fb4", + "semanticToolReceiptContract": "paperclip.semantic_tool_receipt.v2", + "semanticReceiptSourceSha256": "b2795726b2234a663fec1a6ca11f0551ae69882fe9e82aa39686e952d036e791", + "semanticBridgeSourceSha256": "8764b8c9032fdc7d1415b01e75a9fc57c550e7a5e11da4bed7d9100082872ae5", + "toolEvidenceSourceSha256": "6eef5039afc75bb851eaf9cd82a6f099e4bd398ca2deb59dd66f5b2a81a7a706", + "semanticNormalizedInputContract": "validated-forwarded-input-commit-v1", + "semanticSidecarSourceSha256": "17361d1782c0dcebb2b7972635f45c890f2a4081f8bc0ef2d2276550e0b91629", + "semanticDirectDriverSourceSha256": "ff572638f2cb5d9f8249ff8e2f412311c1e3af209ef0bf7e5ea9f10a21d91103", + "semanticValidationSourceSha256": "02fbb92681c78c5de5d2f1aae25aabc25a0e5df6b0ff4baccac652192a8da975", + "semanticNormalizationSourceSha256": "8b628b82e8eacc91cdcc6fdca2210ba4cf282f01e9c6f74e95ff271655b5047e", + "semanticCompletionContractSourceSha256": "3343df6362252d44d12b079ff78c8f6539d1544f3ce5ec36ff36db70ab0fac31", + "semanticValidatorsSourceSha256": "f22d9caf2035b600abc1a6625ae23ca97f1409d7e29ed5b7264bb8f9784c7dc9", + "semanticSchemaBundleSourceSha256": "016391a087cc217aafed4ba1f4efafc6e4881a7525952dce824b7f4d24530603", + "semanticSidecarProtocolSourceSha256": "03c0c95b900f9dc18f29f6be27b8c79bc0bcab29780190d46a74e95328081236", + "semanticSidecarReceiptCommitContract": "correlated-tool-resolve-v1", + "displayEventSourceSha256": "d72f9a6035c247bc6a42c067c05753076c54322c4b58984e93922f3325dc6877", + "displayProjectionSourceSha256": "75952606b26dd93bbc582c1f0a4424487377272ea745fb3a27ceae031fd5dc7e", + "configuredEnvironmentSourceSha256": "5c9008d121979a3a8987e6b8a5752172eafdc4a2b5d725a73f3bbb8fb83be8ff", + "configuredEnvironmentRustSourceSha256": "85c87fae62b8a9377e4f0a4c00302c7519fae4397b1c48253f8f2b7ba7faf994", + "profileActivitySourceSha256": "4fdb6b0f9125fd8372ba6f27945fa5c8a3d3b2ec891df544092e3dc7a75d5282", + "profileExtensionsSourceSha256": "a80ddc95eb4505095bfdbbf85c758199949ab20be9c84b34c77f762eb7180571", + "profileInstallationSourceSha256": "3e61c4cdd4b78e38fcb16eaadae4f1fd65ce3c7127184ea7a80948c8e5d77f88", + "attachmentPolicySourceSha256": "4f7638b7cc64e18f6a4ba33fd46de2b3d49225e32aeab41f5c695b268fa4f8e2", + "singleReadEvidenceSourceSha256": "4069393405089bde613cecec04f5123c426594811da4c668bd96f0e7e283550f", + "nativeSessionRuntimeSourceSha256": "1fe6fa9e6b96d84d70432b85a57c473f833606d4a521563d5022b4d3c41576ce", + "nativeRuntimeContextSourceSha256": "71607f80dbb8dc5cc6c123d0b8f3995d6c5407277695a8c82cbedc90fb8f4842", + "nativeRuntimePromptsSourceSha256": "103b83066da72434602e9e461e6278dfb40bb993d2f3b49b2e93df81bbb08b72", + "nativeSessionContractSourceSha256": "22a8c96ce41ba4f03e976fc37f0cc7c5d05cee99fa789941da31ee1dab440b90", + "harnessSessionContractSourceSha256": "03f4cd7f76f34f6d82e009b66aa51f18f6e23dc235b660778060e35b6ea07412", + "harnessBackendSourceSha256": "dc827e281ae6731545b166ee4d709a4f67f594a556dcbe63c70fe8f1aca26249", + "harnessSessionSourceSha256": "81361733e86b56dd5a66415aefc6f72e69be24acc8d3417370154ebb7fef6e94", + "transportContractSourceSha256": "03fca93d36a89b0cc0917335e293ad097fdf04356d21b307827bb0ea4c13f523", + "nativeBackendSourceSha256": "972c08e6e320d1384221e180391fe817ea1ba632db687201e457d64e68ef4360", + "nativeBackendFactorySourceSha256": "7e431c926b76d53c4c08e3a4105353dcd34bd940f6588a1a68f79631f6feeb44", + "nativeInstructionsSourceSha256": "9173ebccf09cb56a781d2bf5c7eb27fdafb13d3450d99342c6aff86b42d6b43a", + "providerLaunchSourceSha256": "fb6305de1473a51efa25eff83dcea2ab124dd60cfb8036e5ba56ad6785fc1929", + "launchPreparationStoreSourceSha256": "ff6025568bbc0aa6ce545772d73dfca2d3da226740bd3a7b386b069e76cff296", + "runtimeRequestBridgeSourceSha256": "2e940cf2d55211a46e08fb8575cf6157c438d5acd5e0a46cb72d8dd0ad8951c4", + "runtimeRequestFacadeSourceSha256": "d0ed28e0c5ddad34db8f437e704c8a755eb0b47f27e08e44185f66d1d2205295", + "runtimeRequestNotificationSourceSha256": "6196c6175f11c69295dc00fc773201f4fdeca26c500653870253c43b81e65211", + "guardedInstallationSourceSha256": "56a177d3e702914818dca4ca50e061f0d25f988bfc4c0582c36ce23c1696d82f", + "nativeExecutionSourceSha256": "fc7c6568feb0d8d709e9baa51839420c9b90336b4865a9b8405aba06e4d1e9c4", + "profileCompatibilitySourceSha256": "6a26b102b78f282137ff93b0f75eb8788c603fc177802c58715c5ad7b2f3b9e3", + "installationSourceSha256": "a1c7c6f103b898216973e77ffbac0ef7572e250aa574d1a4065270e9dfa0dc2d", + "assetOwnershipSourceSha256": "1785e3f31e5dad5a13b53e25d8cf9482a9bd89a27b6c127830aed6b81acc605c", + "assetInstallerSourceSha256": "d3bd9514e5920d0cc3567aa50487fb534c57a6f66b8da0d3f22c5f6e499c841f", + "credentialBindingSourceSha256": "7586d44b47c83395498dc8806287cd3538ef58177ba7c5949f5766ed105a0161", + "sidecarErrorSourceSha256": "b09af4577f8bf42eee7036b2ad9842202cfc763e72b53c3287f82c6090cdad80", + "profileSourceSha256": "f5538b3dbefa7634a332b87d5ededcefb56aa7e5790cede1c7a0d948a548a456", + "metadataProbeSourceSha256": "be4c8416edd27061109bc1f33d0c26071b4facbb3f651beaae6b3c0275cce727", + "permissionToolIdentitySourceSha256": "770b47305e571a81f344883057d0e4a482a1cd59ac77d045836593a8e739deb1", + "semanticContinuationSourceSha256": "d248e47a0fe7f2536d1e64cea0b0b410f34104800cd643d995b4473104ecc8e0", + "runtimeHostSourceSha256": "94a73749267f5a80b5df19413a65b4b9263c00a9bf7e98fda00f00b8f9fd32ec", + "runtimeAdapterSourceSha256": "e4c764c2c3c7dd655a965b39472bd1ad8d577d1e7258b25594b171c9429752b8", + "runtimeSandboxSourceSha256": "92dfa53f5a3ed0062913ae417eeccec243b430d444495efde2a8e9c4c02e78ea", + "cancellationSourceSha256": "a07d1a9e982dec0ae130c1ad1efa5bc75c6a39e013b730ba3ee45256f09ceeca", + "agentFilesSourceSha256": "fb32f6badc13dfa22f231479509b631b2f0eba0c99b1ab9f3cd5c0c601aa03ba", + "nativeDistributionSourceSha256": "aed327c389e957483a503cb1bb8b247ee7cd7ff4b528fb4567c39bcb4577690b" + } +} diff --git a/packages/paperclip-runner/test/fixtures/copilot-profile-v66-hot-upgrade-candidate-historical-identity.json b/packages/paperclip-runner/test/fixtures/copilot-profile-v66-hot-upgrade-candidate-historical-identity.json new file mode 100644 index 00000000000..d6b3c24d98d --- /dev/null +++ b/packages/paperclip-runner/test/fixtures/copilot-profile-v66-hot-upgrade-candidate-historical-identity.json @@ -0,0 +1,85 @@ +{ + "commandDigest": "sha256:5e9c2d1ca9c1a915ab449871eabd679bc6ac7b62afd96109073b9bbcd8183351", + "declaration": { + "schema": "paperclip.acpx_profile_declaration.v1", + "agent": "copilot", + "agentProfileVersion": 66, + "acpxVersion": "0.13.1", + "agentServerVersion": "1.0.88", + "protocolPolicyRevision": "copilot-agent-manual-v1", + "acpxPatchSha256": "00d6af17216cb0cad902b48633eb19bf0ec6b05837a984cb24661fca593f09c1", + "policySha256": "a4be69be43db84052560420a3ca546a665c9165573c24d543eb33d144ba138a3", + "distributionSourceSha256": "c0a65e247fc47b263bd369a3d5ab505c4b323d4e6493be8a9ecbeb0e9dbb9107", + "agentFilesBinding": "registered-runtime-context-v1", + "systemInstructionDelivery": "COPILOT_HOME/copilot-instructions.md:replace-under-lifetime-lease-before-launch:v1", + "sharedRuntimeContract": "paperclip.acpx-runtime-contract.v1", + "innerDistributionSourceSha256": "9bb08347eed01f1ab44e065c8b06ead9be3a25973d6e6329dc4c0a099096279a", + "upstreamAppSha256": "7b48282a19b5b0814a0c96ad5e3a125173d792cc55f9525b2effea962f6063c0", + "patchedAppSha256": "61f8eebe0c30cc03b311c6818b2acb158a6d16ca58c8bfac5ebd42db095c2679", + "messageIdentityContract": "copilot-native-message-id-v1", + "ownedDistributionDelivery": "COPILOT_CLI_DIST_DIR:lease-owned-guarded-inner-distribution:v1", + "permissionContextContract": "copilot-edit-permission-context-v1", + "permissionContextSourceSha256": "de2d38d702b3c574161af0c0c04790fb6f6347916eb3916ce7a131a8c229121b", + "permissionAdapterSourceSha256": "ee5929ed2ee647ad0369465dee9d51a00b2dd28e1ff735b43e969ddc9c164668", + "permissionLocationsSourceSha256": "f39a81da395b6e23aa2a1a45572a28846e5e81bfd3d764d9a456f74a79c752e9", + "permissionRedactionSourceSha256": "ad6770c69269087951790e2e87e22ac088e3d432769adf17ebc865ed512e6c4f", + "permissionClassifierSourceSha256": "1161eabd82e26133e3b3db3ab567f65d52808b03f176e93497e02c8330349fb4", + "semanticToolReceiptContract": "paperclip.semantic_tool_receipt.v2", + "semanticReceiptSourceSha256": "b2795726b2234a663fec1a6ca11f0551ae69882fe9e82aa39686e952d036e791", + "semanticBridgeSourceSha256": "8764b8c9032fdc7d1415b01e75a9fc57c550e7a5e11da4bed7d9100082872ae5", + "toolEvidenceSourceSha256": "6eef5039afc75bb851eaf9cd82a6f099e4bd398ca2deb59dd66f5b2a81a7a706", + "semanticNormalizedInputContract": "validated-forwarded-input-commit-v1", + "semanticSidecarSourceSha256": "17361d1782c0dcebb2b7972635f45c890f2a4081f8bc0ef2d2276550e0b91629", + "semanticDirectDriverSourceSha256": "ff572638f2cb5d9f8249ff8e2f412311c1e3af209ef0bf7e5ea9f10a21d91103", + "semanticValidationSourceSha256": "02fbb92681c78c5de5d2f1aae25aabc25a0e5df6b0ff4baccac652192a8da975", + "semanticNormalizationSourceSha256": "8b628b82e8eacc91cdcc6fdca2210ba4cf282f01e9c6f74e95ff271655b5047e", + "semanticCompletionContractSourceSha256": "f49398b682a5623812f5d3c33d6bb9a749008e47cc0ed969072041615f66df59", + "semanticValidatorsSourceSha256": "f22d9caf2035b600abc1a6625ae23ca97f1409d7e29ed5b7264bb8f9784c7dc9", + "semanticSchemaBundleSourceSha256": "016391a087cc217aafed4ba1f4efafc6e4881a7525952dce824b7f4d24530603", + "semanticSidecarProtocolSourceSha256": "03c0c95b900f9dc18f29f6be27b8c79bc0bcab29780190d46a74e95328081236", + "semanticSidecarReceiptCommitContract": "correlated-tool-resolve-v1", + "displayEventSourceSha256": "d72f9a6035c247bc6a42c067c05753076c54322c4b58984e93922f3325dc6877", + "displayProjectionSourceSha256": "75952606b26dd93bbc582c1f0a4424487377272ea745fb3a27ceae031fd5dc7e", + "configuredEnvironmentSourceSha256": "5c9008d121979a3a8987e6b8a5752172eafdc4a2b5d725a73f3bbb8fb83be8ff", + "configuredEnvironmentRustSourceSha256": "85c87fae62b8a9377e4f0a4c00302c7519fae4397b1c48253f8f2b7ba7faf994", + "profileActivitySourceSha256": "4fdb6b0f9125fd8372ba6f27945fa5c8a3d3b2ec891df544092e3dc7a75d5282", + "profileExtensionsSourceSha256": "a80ddc95eb4505095bfdbbf85c758199949ab20be9c84b34c77f762eb7180571", + "profileInstallationSourceSha256": "3e61c4cdd4b78e38fcb16eaadae4f1fd65ce3c7127184ea7a80948c8e5d77f88", + "attachmentPolicySourceSha256": "4f7638b7cc64e18f6a4ba33fd46de2b3d49225e32aeab41f5c695b268fa4f8e2", + "singleReadEvidenceSourceSha256": "4069393405089bde613cecec04f5123c426594811da4c668bd96f0e7e283550f", + "nativeSessionRuntimeSourceSha256": "1fe6fa9e6b96d84d70432b85a57c473f833606d4a521563d5022b4d3c41576ce", + "nativeRuntimeContextSourceSha256": "71607f80dbb8dc5cc6c123d0b8f3995d6c5407277695a8c82cbedc90fb8f4842", + "nativeRuntimePromptsSourceSha256": "fff5755dba9b7dce8a5f10ff6ac6d5ed06b92ed079ce1e0a44ac72f2ce7869c1", + "nativeSessionContractSourceSha256": "22a8c96ce41ba4f03e976fc37f0cc7c5d05cee99fa789941da31ee1dab440b90", + "harnessSessionContractSourceSha256": "03f4cd7f76f34f6d82e009b66aa51f18f6e23dc235b660778060e35b6ea07412", + "harnessBackendSourceSha256": "dc827e281ae6731545b166ee4d709a4f67f594a556dcbe63c70fe8f1aca26249", + "harnessSessionSourceSha256": "81361733e86b56dd5a66415aefc6f72e69be24acc8d3417370154ebb7fef6e94", + "transportContractSourceSha256": "03fca93d36a89b0cc0917335e293ad097fdf04356d21b307827bb0ea4c13f523", + "nativeBackendSourceSha256": "e7b12d4418c7a71f5f2474b046697d3009eae3613b284bec381df5e9965a1c37", + "nativeBackendFactorySourceSha256": "7e431c926b76d53c4c08e3a4105353dcd34bd940f6588a1a68f79631f6feeb44", + "nativeInstructionsSourceSha256": "9173ebccf09cb56a781d2bf5c7eb27fdafb13d3450d99342c6aff86b42d6b43a", + "providerLaunchSourceSha256": "63c14082a11f9b02dea9563408bf22fff9f73b8cec0f9f78e6147572190fd2b4", + "launchPreparationStoreSourceSha256": "ff6025568bbc0aa6ce545772d73dfca2d3da226740bd3a7b386b069e76cff296", + "runtimeRequestBridgeSourceSha256": "99295623f20124617ba96a2ae9f10a39771de1c6afd540c05e82d6b8a3c6def1", + "runtimeRequestFacadeSourceSha256": "d0ed28e0c5ddad34db8f437e704c8a755eb0b47f27e08e44185f66d1d2205295", + "runtimeRequestNotificationSourceSha256": "6196c6175f11c69295dc00fc773201f4fdeca26c500653870253c43b81e65211", + "guardedInstallationSourceSha256": "56a177d3e702914818dca4ca50e061f0d25f988bfc4c0582c36ce23c1696d82f", + "nativeExecutionSourceSha256": "3a30d52bffad68c7f02349193be84761050eb99039fc0556c2b449891a36e918", + "profileCompatibilitySourceSha256": "d96c3b809d083f9d43936af05e6bc14b1755c55f255c8e5967b8c16661f34ffa", + "installationSourceSha256": "a1c7c6f103b898216973e77ffbac0ef7572e250aa574d1a4065270e9dfa0dc2d", + "assetOwnershipSourceSha256": "1785e3f31e5dad5a13b53e25d8cf9482a9bd89a27b6c127830aed6b81acc605c", + "assetInstallerSourceSha256": "d3bd9514e5920d0cc3567aa50487fb534c57a6f66b8da0d3f22c5f6e499c841f", + "credentialBindingSourceSha256": "7586d44b47c83395498dc8806287cd3538ef58177ba7c5949f5766ed105a0161", + "sidecarErrorSourceSha256": "b09af4577f8bf42eee7036b2ad9842202cfc763e72b53c3287f82c6090cdad80", + "profileSourceSha256": "f5538b3dbefa7634a332b87d5ededcefb56aa7e5790cede1c7a0d948a548a456", + "metadataProbeSourceSha256": "be4c8416edd27061109bc1f33d0c26071b4facbb3f651beaae6b3c0275cce727", + "permissionToolIdentitySourceSha256": "770b47305e571a81f344883057d0e4a482a1cd59ac77d045836593a8e739deb1", + "semanticContinuationSourceSha256": "c9cc0fd516b3df92a6655f56ee9bef37615a889c371e2e137d8ac5c1176de811", + "runtimeHostSourceSha256": "94a73749267f5a80b5df19413a65b4b9263c00a9bf7e98fda00f00b8f9fd32ec", + "runtimeAdapterSourceSha256": "e4c764c2c3c7dd655a965b39472bd1ad8d577d1e7258b25594b171c9429752b8", + "runtimeSandboxSourceSha256": "92dfa53f5a3ed0062913ae417eeccec243b430d444495efde2a8e9c4c02e78ea", + "cancellationSourceSha256": "a07d1a9e982dec0ae130c1ad1efa5bc75c6a39e013b730ba3ee45256f09ceeca", + "agentFilesSourceSha256": "fb32f6badc13dfa22f231479509b631b2f0eba0c99b1ab9f3cd5c0c601aa03ba", + "nativeDistributionSourceSha256": "aed327c389e957483a503cb1bb8b247ee7cd7ff4b528fb4567c39bcb4577690b" + } +} diff --git a/packages/paperclip-runner/test/fixtures/copilot-profile-v67-identity.json b/packages/paperclip-runner/test/fixtures/copilot-profile-v67-identity.json new file mode 100644 index 00000000000..d105cf07a6e --- /dev/null +++ b/packages/paperclip-runner/test/fixtures/copilot-profile-v67-identity.json @@ -0,0 +1,85 @@ +{ + "commandDigest": "sha256:7f5897a354d5cd8bdcd165d3e3bab4d6908aaca3981af384f4dfeb4bc0838fd8", + "declaration": { + "schema": "paperclip.acpx_profile_declaration.v1", + "agent": "copilot", + "agentProfileVersion": 67, + "acpxVersion": "0.13.1", + "agentServerVersion": "1.0.88", + "protocolPolicyRevision": "copilot-agent-manual-v1", + "acpxPatchSha256": "00d6af17216cb0cad902b48633eb19bf0ec6b05837a984cb24661fca593f09c1", + "policySha256": "a4be69be43db84052560420a3ca546a665c9165573c24d543eb33d144ba138a3", + "distributionSourceSha256": "c0a65e247fc47b263bd369a3d5ab505c4b323d4e6493be8a9ecbeb0e9dbb9107", + "agentFilesBinding": "registered-runtime-context-v1", + "systemInstructionDelivery": "COPILOT_HOME/copilot-instructions.md:replace-under-lifetime-lease-before-launch:v1", + "sharedRuntimeContract": "paperclip.acpx-runtime-contract.v1", + "innerDistributionSourceSha256": "9bb08347eed01f1ab44e065c8b06ead9be3a25973d6e6329dc4c0a099096279a", + "upstreamAppSha256": "7b48282a19b5b0814a0c96ad5e3a125173d792cc55f9525b2effea962f6063c0", + "patchedAppSha256": "61f8eebe0c30cc03b311c6818b2acb158a6d16ca58c8bfac5ebd42db095c2679", + "messageIdentityContract": "copilot-native-message-id-v1", + "ownedDistributionDelivery": "COPILOT_CLI_DIST_DIR:lease-owned-guarded-inner-distribution:v1", + "permissionContextContract": "copilot-edit-permission-context-v1", + "permissionContextSourceSha256": "de2d38d702b3c574161af0c0c04790fb6f6347916eb3916ce7a131a8c229121b", + "permissionAdapterSourceSha256": "ee5929ed2ee647ad0369465dee9d51a00b2dd28e1ff735b43e969ddc9c164668", + "permissionLocationsSourceSha256": "f39a81da395b6e23aa2a1a45572a28846e5e81bfd3d764d9a456f74a79c752e9", + "permissionRedactionSourceSha256": "ad6770c69269087951790e2e87e22ac088e3d432769adf17ebc865ed512e6c4f", + "permissionClassifierSourceSha256": "1161eabd82e26133e3b3db3ab567f65d52808b03f176e93497e02c8330349fb4", + "semanticToolReceiptContract": "paperclip.semantic_tool_receipt.v2", + "semanticReceiptSourceSha256": "b2795726b2234a663fec1a6ca11f0551ae69882fe9e82aa39686e952d036e791", + "semanticBridgeSourceSha256": "8764b8c9032fdc7d1415b01e75a9fc57c550e7a5e11da4bed7d9100082872ae5", + "toolEvidenceSourceSha256": "6eef5039afc75bb851eaf9cd82a6f099e4bd398ca2deb59dd66f5b2a81a7a706", + "semanticNormalizedInputContract": "validated-forwarded-input-commit-v1", + "semanticSidecarSourceSha256": "17361d1782c0dcebb2b7972635f45c890f2a4081f8bc0ef2d2276550e0b91629", + "semanticDirectDriverSourceSha256": "ff572638f2cb5d9f8249ff8e2f412311c1e3af209ef0bf7e5ea9f10a21d91103", + "semanticValidationSourceSha256": "02fbb92681c78c5de5d2f1aae25aabc25a0e5df6b0ff4baccac652192a8da975", + "semanticNormalizationSourceSha256": "8b628b82e8eacc91cdcc6fdca2210ba4cf282f01e9c6f74e95ff271655b5047e", + "semanticCompletionContractSourceSha256": "f49398b682a5623812f5d3c33d6bb9a749008e47cc0ed969072041615f66df59", + "semanticValidatorsSourceSha256": "f22d9caf2035b600abc1a6625ae23ca97f1409d7e29ed5b7264bb8f9784c7dc9", + "semanticSchemaBundleSourceSha256": "016391a087cc217aafed4ba1f4efafc6e4881a7525952dce824b7f4d24530603", + "semanticSidecarProtocolSourceSha256": "03c0c95b900f9dc18f29f6be27b8c79bc0bcab29780190d46a74e95328081236", + "semanticSidecarReceiptCommitContract": "correlated-tool-resolve-v1", + "displayEventSourceSha256": "d72f9a6035c247bc6a42c067c05753076c54322c4b58984e93922f3325dc6877", + "displayProjectionSourceSha256": "75952606b26dd93bbc582c1f0a4424487377272ea745fb3a27ceae031fd5dc7e", + "configuredEnvironmentSourceSha256": "5c9008d121979a3a8987e6b8a5752172eafdc4a2b5d725a73f3bbb8fb83be8ff", + "configuredEnvironmentRustSourceSha256": "85c87fae62b8a9377e4f0a4c00302c7519fae4397b1c48253f8f2b7ba7faf994", + "profileActivitySourceSha256": "4fdb6b0f9125fd8372ba6f27945fa5c8a3d3b2ec891df544092e3dc7a75d5282", + "profileExtensionsSourceSha256": "a80ddc95eb4505095bfdbbf85c758199949ab20be9c84b34c77f762eb7180571", + "profileInstallationSourceSha256": "3e61c4cdd4b78e38fcb16eaadae4f1fd65ce3c7127184ea7a80948c8e5d77f88", + "attachmentPolicySourceSha256": "4f7638b7cc64e18f6a4ba33fd46de2b3d49225e32aeab41f5c695b268fa4f8e2", + "singleReadEvidenceSourceSha256": "4069393405089bde613cecec04f5123c426594811da4c668bd96f0e7e283550f", + "nativeSessionRuntimeSourceSha256": "1fe6fa9e6b96d84d70432b85a57c473f833606d4a521563d5022b4d3c41576ce", + "nativeRuntimeContextSourceSha256": "71607f80dbb8dc5cc6c123d0b8f3995d6c5407277695a8c82cbedc90fb8f4842", + "nativeRuntimePromptsSourceSha256": "fff5755dba9b7dce8a5f10ff6ac6d5ed06b92ed079ce1e0a44ac72f2ce7869c1", + "nativeSessionContractSourceSha256": "22a8c96ce41ba4f03e976fc37f0cc7c5d05cee99fa789941da31ee1dab440b90", + "harnessSessionContractSourceSha256": "03f4cd7f76f34f6d82e009b66aa51f18f6e23dc235b660778060e35b6ea07412", + "harnessBackendSourceSha256": "dc827e281ae6731545b166ee4d709a4f67f594a556dcbe63c70fe8f1aca26249", + "harnessSessionSourceSha256": "81361733e86b56dd5a66415aefc6f72e69be24acc8d3417370154ebb7fef6e94", + "transportContractSourceSha256": "03fca93d36a89b0cc0917335e293ad097fdf04356d21b307827bb0ea4c13f523", + "nativeBackendSourceSha256": "e7b12d4418c7a71f5f2474b046697d3009eae3613b284bec381df5e9965a1c37", + "nativeBackendFactorySourceSha256": "7e431c926b76d53c4c08e3a4105353dcd34bd940f6588a1a68f79631f6feeb44", + "nativeInstructionsSourceSha256": "9173ebccf09cb56a781d2bf5c7eb27fdafb13d3450d99342c6aff86b42d6b43a", + "providerLaunchSourceSha256": "63c14082a11f9b02dea9563408bf22fff9f73b8cec0f9f78e6147572190fd2b4", + "launchPreparationStoreSourceSha256": "ff6025568bbc0aa6ce545772d73dfca2d3da226740bd3a7b386b069e76cff296", + "runtimeRequestBridgeSourceSha256": "5f2255933fb9974063c9f0cf312333f2d283cc3ec879a217834759c7d00c37b0", + "runtimeRequestFacadeSourceSha256": "d0ed28e0c5ddad34db8f437e704c8a755eb0b47f27e08e44185f66d1d2205295", + "runtimeRequestNotificationSourceSha256": "6196c6175f11c69295dc00fc773201f4fdeca26c500653870253c43b81e65211", + "guardedInstallationSourceSha256": "56a177d3e702914818dca4ca50e061f0d25f988bfc4c0582c36ce23c1696d82f", + "nativeExecutionSourceSha256": "3a30d52bffad68c7f02349193be84761050eb99039fc0556c2b449891a36e918", + "profileCompatibilitySourceSha256": "94aaea13c8a6957097cfcf5794a6fd5035a00a3f5c7aabee1019a1ca6b9a1670", + "installationSourceSha256": "a1c7c6f103b898216973e77ffbac0ef7572e250aa574d1a4065270e9dfa0dc2d", + "assetOwnershipSourceSha256": "1785e3f31e5dad5a13b53e25d8cf9482a9bd89a27b6c127830aed6b81acc605c", + "assetInstallerSourceSha256": "d3bd9514e5920d0cc3567aa50487fb534c57a6f66b8da0d3f22c5f6e499c841f", + "credentialBindingSourceSha256": "7586d44b47c83395498dc8806287cd3538ef58177ba7c5949f5766ed105a0161", + "sidecarErrorSourceSha256": "b09af4577f8bf42eee7036b2ad9842202cfc763e72b53c3287f82c6090cdad80", + "profileSourceSha256": "f5538b3dbefa7634a332b87d5ededcefb56aa7e5790cede1c7a0d948a548a456", + "metadataProbeSourceSha256": "be4c8416edd27061109bc1f33d0c26071b4facbb3f651beaae6b3c0275cce727", + "permissionToolIdentitySourceSha256": "770b47305e571a81f344883057d0e4a482a1cd59ac77d045836593a8e739deb1", + "semanticContinuationSourceSha256": "c9cc0fd516b3df92a6655f56ee9bef37615a889c371e2e137d8ac5c1176de811", + "runtimeHostSourceSha256": "94a73749267f5a80b5df19413a65b4b9263c00a9bf7e98fda00f00b8f9fd32ec", + "runtimeAdapterSourceSha256": "e4c764c2c3c7dd655a965b39472bd1ad8d577d1e7258b25594b171c9429752b8", + "runtimeSandboxSourceSha256": "92dfa53f5a3ed0062913ae417eeccec243b430d444495efde2a8e9c4c02e78ea", + "cancellationSourceSha256": "a07d1a9e982dec0ae130c1ad1efa5bc75c6a39e013b730ba3ee45256f09ceeca", + "agentFilesSourceSha256": "fb32f6badc13dfa22f231479509b631b2f0eba0c99b1ab9f3cd5c0c601aa03ba", + "nativeDistributionSourceSha256": "aed327c389e957483a503cb1bb8b247ee7cd7ff4b528fb4567c39bcb4577690b" + } +} diff --git a/packages/paperclip-runner/test/fixtures/copilot-profile-v68-identity.json b/packages/paperclip-runner/test/fixtures/copilot-profile-v68-identity.json new file mode 100644 index 00000000000..397ee98de59 --- /dev/null +++ b/packages/paperclip-runner/test/fixtures/copilot-profile-v68-identity.json @@ -0,0 +1,85 @@ +{ + "commandDigest": "sha256:1ce8bb0e00df81fa9eb71cff5063ce769637f5db7b3aa0ada9ade651db089c63", + "declaration": { + "schema": "paperclip.acpx_profile_declaration.v1", + "agent": "copilot", + "agentProfileVersion": 68, + "acpxVersion": "0.13.1", + "agentServerVersion": "1.0.88", + "protocolPolicyRevision": "copilot-agent-manual-v1", + "acpxPatchSha256": "00d6af17216cb0cad902b48633eb19bf0ec6b05837a984cb24661fca593f09c1", + "policySha256": "a4be69be43db84052560420a3ca546a665c9165573c24d543eb33d144ba138a3", + "distributionSourceSha256": "c0a65e247fc47b263bd369a3d5ab505c4b323d4e6493be8a9ecbeb0e9dbb9107", + "agentFilesBinding": "registered-runtime-context-v1", + "systemInstructionDelivery": "COPILOT_HOME/copilot-instructions.md:replace-under-lifetime-lease-before-launch:v1", + "sharedRuntimeContract": "paperclip.acpx-runtime-contract.v1", + "innerDistributionSourceSha256": "9bb08347eed01f1ab44e065c8b06ead9be3a25973d6e6329dc4c0a099096279a", + "upstreamAppSha256": "7b48282a19b5b0814a0c96ad5e3a125173d792cc55f9525b2effea962f6063c0", + "patchedAppSha256": "61f8eebe0c30cc03b311c6818b2acb158a6d16ca58c8bfac5ebd42db095c2679", + "messageIdentityContract": "copilot-native-message-id-v1", + "ownedDistributionDelivery": "COPILOT_CLI_DIST_DIR:lease-owned-guarded-inner-distribution:v1", + "permissionContextContract": "copilot-edit-permission-context-v1", + "permissionContextSourceSha256": "de2d38d702b3c574161af0c0c04790fb6f6347916eb3916ce7a131a8c229121b", + "permissionAdapterSourceSha256": "ee5929ed2ee647ad0369465dee9d51a00b2dd28e1ff735b43e969ddc9c164668", + "permissionLocationsSourceSha256": "f39a81da395b6e23aa2a1a45572a28846e5e81bfd3d764d9a456f74a79c752e9", + "permissionRedactionSourceSha256": "ad6770c69269087951790e2e87e22ac088e3d432769adf17ebc865ed512e6c4f", + "permissionClassifierSourceSha256": "1161eabd82e26133e3b3db3ab567f65d52808b03f176e93497e02c8330349fb4", + "semanticToolReceiptContract": "paperclip.semantic_tool_receipt.v2", + "semanticReceiptSourceSha256": "b2795726b2234a663fec1a6ca11f0551ae69882fe9e82aa39686e952d036e791", + "semanticBridgeSourceSha256": "8764b8c9032fdc7d1415b01e75a9fc57c550e7a5e11da4bed7d9100082872ae5", + "toolEvidenceSourceSha256": "6eef5039afc75bb851eaf9cd82a6f099e4bd398ca2deb59dd66f5b2a81a7a706", + "semanticNormalizedInputContract": "validated-forwarded-input-commit-v1", + "semanticSidecarSourceSha256": "7d6e948b7cff4df52a9afc916db46ace0f42b4b2d3134ba0c21ee0a928857e62", + "semanticDirectDriverSourceSha256": "d683429427dd8a1a5efe50952018ea486f808bdbf0a92fc5946ee267fe2c72ee", + "semanticValidationSourceSha256": "02fbb92681c78c5de5d2f1aae25aabc25a0e5df6b0ff4baccac652192a8da975", + "semanticNormalizationSourceSha256": "8b628b82e8eacc91cdcc6fdca2210ba4cf282f01e9c6f74e95ff271655b5047e", + "semanticCompletionContractSourceSha256": "f49398b682a5623812f5d3c33d6bb9a749008e47cc0ed969072041615f66df59", + "semanticValidatorsSourceSha256": "f22d9caf2035b600abc1a6625ae23ca97f1409d7e29ed5b7264bb8f9784c7dc9", + "semanticSchemaBundleSourceSha256": "4cae579aac836762fdbf92c5447425d8b756e87a5766bc0c7a1885a90541c8d6", + "semanticSidecarProtocolSourceSha256": "26dd0cfbef8b2b401de6b7862a4af7f243fb703e23678a1a5ed8fa68d4784d42", + "semanticSidecarReceiptCommitContract": "correlated-tool-resolve-v1", + "displayEventSourceSha256": "d72f9a6035c247bc6a42c067c05753076c54322c4b58984e93922f3325dc6877", + "displayProjectionSourceSha256": "75952606b26dd93bbc582c1f0a4424487377272ea745fb3a27ceae031fd5dc7e", + "configuredEnvironmentSourceSha256": "5c9008d121979a3a8987e6b8a5752172eafdc4a2b5d725a73f3bbb8fb83be8ff", + "configuredEnvironmentRustSourceSha256": "85c87fae62b8a9377e4f0a4c00302c7519fae4397b1c48253f8f2b7ba7faf994", + "profileActivitySourceSha256": "4fdb6b0f9125fd8372ba6f27945fa5c8a3d3b2ec891df544092e3dc7a75d5282", + "profileExtensionsSourceSha256": "a80ddc95eb4505095bfdbbf85c758199949ab20be9c84b34c77f762eb7180571", + "profileInstallationSourceSha256": "3e61c4cdd4b78e38fcb16eaadae4f1fd65ce3c7127184ea7a80948c8e5d77f88", + "attachmentPolicySourceSha256": "4f7638b7cc64e18f6a4ba33fd46de2b3d49225e32aeab41f5c695b268fa4f8e2", + "singleReadEvidenceSourceSha256": "4069393405089bde613cecec04f5123c426594811da4c668bd96f0e7e283550f", + "nativeSessionRuntimeSourceSha256": "1fe6fa9e6b96d84d70432b85a57c473f833606d4a521563d5022b4d3c41576ce", + "nativeRuntimeContextSourceSha256": "71607f80dbb8dc5cc6c123d0b8f3995d6c5407277695a8c82cbedc90fb8f4842", + "nativeRuntimePromptsSourceSha256": "fff5755dba9b7dce8a5f10ff6ac6d5ed06b92ed079ce1e0a44ac72f2ce7869c1", + "nativeSessionContractSourceSha256": "22a8c96ce41ba4f03e976fc37f0cc7c5d05cee99fa789941da31ee1dab440b90", + "harnessSessionContractSourceSha256": "baba1a210b863518f01e0c875606ed37f9467c433222251438578bd999bc43b5", + "harnessBackendSourceSha256": "dc827e281ae6731545b166ee4d709a4f67f594a556dcbe63c70fe8f1aca26249", + "harnessSessionSourceSha256": "81361733e86b56dd5a66415aefc6f72e69be24acc8d3417370154ebb7fef6e94", + "transportContractSourceSha256": "03fca93d36a89b0cc0917335e293ad097fdf04356d21b307827bb0ea4c13f523", + "nativeBackendSourceSha256": "7495d9898d22dca1ebf6336361d332bafefc4498f6e49f1135025fad846c8586", + "nativeBackendFactorySourceSha256": "7e431c926b76d53c4c08e3a4105353dcd34bd940f6588a1a68f79631f6feeb44", + "nativeInstructionsSourceSha256": "82ac6e5f7275b4ef79fb166c7330efc504e80cab3d7e9081e4e40a8b570bc50b", + "providerLaunchSourceSha256": "c744235b3067ea75973523f78f1e13399337bc155f1775af694e34782d5c9bef", + "launchPreparationStoreSourceSha256": "ff6025568bbc0aa6ce545772d73dfca2d3da226740bd3a7b386b069e76cff296", + "runtimeRequestBridgeSourceSha256": "93a631b8439df1590b0cf442cf84a9b05f004f12bfee90291651ccfc0861effe", + "runtimeRequestFacadeSourceSha256": "d0ed28e0c5ddad34db8f437e704c8a755eb0b47f27e08e44185f66d1d2205295", + "runtimeRequestNotificationSourceSha256": "6196c6175f11c69295dc00fc773201f4fdeca26c500653870253c43b81e65211", + "guardedInstallationSourceSha256": "56a177d3e702914818dca4ca50e061f0d25f988bfc4c0582c36ce23c1696d82f", + "nativeExecutionSourceSha256": "d60b8ea63bad975c0144f0702b2749d7db9a7da50680a6147f20d4d11f2d5297", + "profileCompatibilitySourceSha256": "7b4706f8af55f0359b28d2f941a7774f2932bd8ca4f0aa883559f7aa9e781253", + "installationSourceSha256": "a1c7c6f103b898216973e77ffbac0ef7572e250aa574d1a4065270e9dfa0dc2d", + "assetOwnershipSourceSha256": "1785e3f31e5dad5a13b53e25d8cf9482a9bd89a27b6c127830aed6b81acc605c", + "assetInstallerSourceSha256": "d3bd9514e5920d0cc3567aa50487fb534c57a6f66b8da0d3f22c5f6e499c841f", + "credentialBindingSourceSha256": "7586d44b47c83395498dc8806287cd3538ef58177ba7c5949f5766ed105a0161", + "sidecarErrorSourceSha256": "b09af4577f8bf42eee7036b2ad9842202cfc763e72b53c3287f82c6090cdad80", + "profileSourceSha256": "f5538b3dbefa7634a332b87d5ededcefb56aa7e5790cede1c7a0d948a548a456", + "metadataProbeSourceSha256": "be4c8416edd27061109bc1f33d0c26071b4facbb3f651beaae6b3c0275cce727", + "permissionToolIdentitySourceSha256": "770b47305e571a81f344883057d0e4a482a1cd59ac77d045836593a8e739deb1", + "semanticContinuationSourceSha256": "c9cc0fd516b3df92a6655f56ee9bef37615a889c371e2e137d8ac5c1176de811", + "runtimeHostSourceSha256": "976fef21ec9f3ae9dd5eabb133a55ae578f1c0729c657c30e42562a80db237d8", + "runtimeAdapterSourceSha256": "e2dd18de4b134a5492de93c88b4add479f63fb650c655d60a17c04af722c7563", + "runtimeSandboxSourceSha256": "981449a4c6d10f682ecf86d834eed07faca30a7d9b419fb8db4973a442a343ec", + "cancellationSourceSha256": "a07d1a9e982dec0ae130c1ad1efa5bc75c6a39e013b730ba3ee45256f09ceeca", + "agentFilesSourceSha256": "fb32f6badc13dfa22f231479509b631b2f0eba0c99b1ab9f3cd5c0c601aa03ba", + "nativeDistributionSourceSha256": "aed327c389e957483a503cb1bb8b247ee7cd7ff4b528fb4567c39bcb4577690b" + } +}