diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts index bd4f062de21..cd70ff1c0b2 100644 --- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts +++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts @@ -24,7 +24,7 @@ import { describe, expect, it, vi } from "vitest"; import { configuredEnvironmentProjection } from "../configured-environment.js"; import { NativeSessionProtocolIntegrityError } from "../contracts/native-session-backend.js"; -import { createCapabilityRunnerdCodexTransport, createCapabilityRunnerdProviderEnvironment } from "../live/runnerd-codex-transport.js"; +import { createCapabilityRunnerdCodexTransport, createCapabilityRunnerdProviderEnvironment, waitForRunnerCommand } from "../live/runnerd-codex-transport.js"; import { ACPX_CREDENTIAL_BINDING_ENV, createAcpxSidecarHostEnvironment } from "../drivers/acpx/environment.js"; import { validatePrpEvent } from "../protocol/replay-contract.js"; import { digestPaperclipSemanticContent } from "../semantic-tools/receipts.js"; @@ -1863,6 +1863,245 @@ it("answers lease renewal while an earlier event commit is still in flight", asy } }); +function replayBacklogCore(root: string, commitMs: number) { + let stopped = false; + const commits = vi.fn(async () => { + if (stopped) throw new Error("replay_test_stopped"); + await new Promise(resolveCommit => setTimeout(resolveCommit, commitMs)); + }); + const core = new DurablePrpControlPlane({ stateDirectory: root, identity, + expectedRunnerVersion, expectedRunnerDigest, onCommittedEvent: commits }); + return { core, commits, stop: () => { stopped = true; } }; +} + +function commandResultFrame(client: AuthenticatedClient, command: { commandId: string; type: string; controllerSeq: number }, + result: Record = { status: "completed" }, status = "completed"): Record { + return { protocol: "paperclip.runner", version: client.welcome.version, kind: "command_result", + payload: { commandId: command.commandId, commandType: command.type, controllerSeq: command.controllerSeq, status, result } }; +} + +async function receiveCommand(client: AuthenticatedClient): Promise> { + for (;;) { + const frame = await receiveSecure(client); + if (frame === null) throw new Error("connection closed before the command arrived"); + if (frame.kind === "command") return frame; + } +} + +it("answers a command issued during a 1,000-event replay within its deadline", async () => { + const root = mkdtempSync(resolve(tmpdir(), "runner-command-under-replay-")); + const { core, commits, stop } = replayBacklogCore(root, 50); + try { + await core.start(); + const client = (await authenticate(core, core.issueBootstrapTicket()))!; + for (let seq = 1; seq <= 1_000; seq++) sendSecure(client, ordinaryOutput(seq)); + await vi.waitFor(() => expect(commits).toHaveBeenCalled()); + const snapshot = core.queueCommand("session.snapshot", {}, "snapshot-during-replay", true); + expect((await receiveCommand(client)).payload).toMatchObject({ commandId: snapshot.commandId }); + sendSecure(client, commandResultFrame(client, snapshot)); + await vi.waitFor(() => expect(core.commandOutcome(snapshot.commandId)?.status).toBe("completed"), + { timeout: 30_000, interval: 10 }); + expect(core.store.state.ackedSourceSeq).toBeLessThan(1_000); + client.socket.destroy(); + } finally { + stop(); + await core.stop(); + await core.drainPendingConnectionProcessing(); + rmSync(root, { recursive: true, force: true }); + } +}, 60_000); + +it("keeps a failed command result behind the events the Runner sent before it", async () => { + const root = mkdtempSync(resolve(tmpdir(), "runner-failed-result-order-")); + const { core, stop } = replayBacklogCore(root, 20); + try { + await core.start(); + const client = (await authenticate(core, core.issueBootstrapTicket()))!; + const snapshot = core.queueCommand("session.snapshot", {}, "failed-after-evidence", true); + await receiveCommand(client); + for (let seq = 1; seq <= 20; seq++) sendSecure(client, ordinaryOutput(seq)); + sendSecure(client, commandResultFrame(client, snapshot, { code: "command_execution_failed" }, "failed")); + await vi.waitFor(() => expect(core.commandOutcome(snapshot.commandId)?.status).toBe("failed"), { timeout: 10_000 }); + expect(core.store.state.ackedSourceSeq).toBe(20); + client.socket.destroy(); + } finally { + stop(); + await core.stop(); + await core.drainPendingConnectionProcessing(); + rmSync(root, { recursive: true, force: true }); + } +}, 30_000); + +function staleToolResult(core: DurablePrpControlPlane) { + return core.queueCommand("semantic_tool.result", { callId: "call-earlier-generation", operationId: "get_task_context", + sourceEventId: "tool-input-earlier-generation", sourceEventType: "semantic_tool.input", inputDigest: "digest", + correlation: { ...identity }, result: { ok: true }, isError: false }, "command_tool_earlier_generation"); +} + +it("delivers every command pending at attach so an earlier generation's result cannot hold the recovery probe", async () => { + const root = mkdtempSync(resolve(tmpdir(), "runner-stale-pending-at-attach-")); + const { core, stop } = replayBacklogCore(root, 50); + try { + const stale = staleToolResult(core); + const probe = core.queueCommand("runner.drain", {}, "command_resume_probe_attach"); + await core.start(); + const client = (await authenticate(core, core.issueBootstrapTicket()))!; + const pending = (client.welcome.payload as { pendingCommands: Array<{ commandId: string }> }).pendingCommands; + expect(pending.map(command => command.commandId)).toEqual([stale.commandId, probe.commandId]); + sendSecure(client, commandResultFrame(client, stale)); + sendSecure(client, commandResultFrame(client, probe, { status: "completed", retainedEventsDrained: false })); + for (let seq = 1; seq <= 1_000; seq++) sendSecure(client, ordinaryOutput(seq)); + await vi.waitFor(() => expect(core.commandOutcome(probe.commandId)?.status).toBe("completed"), { timeout: 5_000 }); + expect(core.commandOutcome(stale.commandId)?.status).toBe("completed"); + expect(core.store.state.commandDeliveryCounts).toEqual({ [stale.commandId]: 1, [probe.commandId]: 1 }); + client.socket.destroy(); + } finally { + stop(); + await core.stop(); + await core.drainPendingConnectionProcessing(); + rmSync(root, { recursive: true, force: true }); + } +}, 30_000); + +it("never delivers a lifecycle command ahead of its turn at attach", async () => { + const root = mkdtempSync(resolve(tmpdir(), "runner-attach-lifecycle-fence-")); + const { core, stop } = replayBacklogCore(root, 1); + try { + const stale = staleToolResult(core); + const drain = core.queueCommand("runner.drain", {}, "command_drain_before_suspend"); + core.queueCommand("runner.suspend", {}, "command_suspend_after_drain"); + core.queueCommand("session.snapshot", {}, "command_snapshot_after_suspend"); + await core.start(); + const client = (await authenticate(core, core.issueBootstrapTicket()))!; + const pending = (client.welcome.payload as { pendingCommands: Array<{ commandId: string }> }).pendingCommands; + expect(pending.map(command => command.commandId)).toEqual([stale.commandId, drain.commandId]); + sendSecure(client, commandResultFrame(client, stale)); + sendSecure(client, commandResultFrame(client, drain, { status: "completed", retainedEventsDrained: true })); + expect((await receiveCommand(client)).payload).toMatchObject({ commandId: "command_suspend_after_drain" }); + client.socket.destroy(); + } finally { + stop(); + await core.stop(); + await core.drainPendingConnectionProcessing(); + rmSync(root, { recursive: true, force: true }); + } +}, 30_000); + +function completedItem(sourceSeq: number): Record { + const envelope = ordinaryOutput(sourceSeq); + Object.assign(envelope.payload as Record, { + sourceEventId: `completed-${sourceSeq}`, eventType: "item.completed", priority: 1, + payload: { item: { id: `item-${sourceSeq}` } }, + }); + return envelope; +} + +it("acknowledges a replayed burst once per batch, never ahead of its durable checkpoint", async () => { + const root = mkdtempSync(resolve(tmpdir(), "runner-batched-ack-")); + const { core, stop } = replayBacklogCore(root, 5); + let persistedCursor = 0; + const persisted: number[] = []; + core.setCheckpointPersistence(async saved => { + await new Promise(resolveWrite => setTimeout(resolveWrite, 5)); + persistedCursor = JSON.parse(saved.records.get("cursor")!).ackedSourceSeq; + persisted.push(persistedCursor); + }); + try { + await core.start(); + const client = (await authenticate(core, core.issueBootstrapTicket()))!; + const before = persisted.length; + for (let seq = 1; seq <= 300; seq++) sendSecure(client, completedItem(seq)); + const acks: number[] = []; + while (acks.at(-1) !== 300) { + const frame = await receiveSecure(client); + if (frame === null) throw new Error("connection closed before the burst was acknowledged"); + if (frame.kind !== "ack") continue; + const acked = (frame.payload as { ackedSourceSeq: number }).ackedSourceSeq; + expect(acked).toBeLessThanOrEqual(persistedCursor); + acks.push(acked); + } + expect(acks).toEqual([...acks].sort((left, right) => left - right)); + expect(acks.length).toBeLessThan(300 / 16); + expect(persisted.length - before).toBeLessThan(300 / 16); + expect(core.store.state.committedEvents.map(event => event.sourceSeq)).toEqual(Array.from({ length: 300 }, (_, index) => index + 1)); + client.socket.destroy(); + } finally { + stop(); + await core.stop(); + await core.drainPendingConnectionProcessing(); + rmSync(root, { recursive: true, force: true }); + } +}, 60_000); + +it("still acknowledges a lifecycle event inside a burst before the next frame", async () => { + const root = mkdtempSync(resolve(tmpdir(), "runner-batched-ack-lifecycle-")); + const { core, stop } = replayBacklogCore(root, 5); + try { + await core.start(); + const client = (await authenticate(core, core.issueBootstrapTicket()))!; + const turn = completedItem(11); + Object.assign(turn.payload as Record, { sourceEventId: "turn-11", eventType: "turn.completed", priority: 0, + payload: { turnId: identity.turnId } }); + for (let seq = 1; seq <= 20; seq++) sendSecure(client, seq === 11 ? turn : completedItem(seq)); + const acks: number[] = []; + while (acks.at(-1) !== 20) { + const frame = await receiveSecure(client); + if (frame === null) throw new Error("connection closed before the burst was acknowledged"); + if (frame.kind === "ack") acks.push((frame.payload as { ackedSourceSeq: number }).ackedSourceSeq); + } + expect(acks).toContain(11); + client.socket.destroy(); + } finally { + stop(); + await core.stop(); + await core.drainPendingConnectionProcessing(); + rmSync(root, { recursive: true, force: true }); + } +}, 60_000); + +it("answers a snapshot from a Runner whose provider stays wedged until its whole replay is acknowledged", async () => { + const root = mkdtempSync(resolve(tmpdir(), "runner-wedged-provider-replay-")); + const { core, stop } = replayBacklogCore(root, 20); + try { + await core.start(); + const client = (await authenticate(core, core.issueBootstrapTicket()))!; + const backlog = 250; + for (let seq = 1; seq <= backlog; seq++) sendSecure(client, completedItem(seq)); + const snapshot = core.queueCommand("session.snapshot", {}, "snapshot-behind-wedged-provider", true); + const runner = (async () => { + let delivered = false; + let acked = 0; + while (!delivered || acked < backlog) { + const frame = await receiveSecure(client, null); + if (frame === null) return; + if (frame.kind === "command") delivered = true; + if (frame.kind === "ack") acked = (frame.payload as { ackedSourceSeq: number }).ackedSourceSeq; + } + sendSecure(client, commandResultFrame(client, snapshot)); + })(); + await waitForRunnerCommand({ + type: "session.snapshot", + deadline: Date.now() + 2_000, + abortOnClose: false, + isClosed: () => false, + throwIfFailed: () => undefined, + command: () => core.getCommand(snapshot.commandId), + runnerHasExited: async () => false, + failureCode: () => "provider_transport_failed", + answeredProgressAt: () => core.queuedCommandResultProgressAt(snapshot.commandId), + replayProgressAt: () => core.committedEventProgressAt?.() ?? null, + }); + expect(core.store.state.ackedSourceSeq).toBe(backlog); + await runner; + client.socket.destroy(); + } finally { + stop(); + await core.stop(); + await core.drainPendingConnectionProcessing(); + rmSync(root, { recursive: true, force: true }); + } +}, 60_000); + it("keeps a staged tool-progress write when a lease renewal lands mid-write", async () => { const root = mkdtempSync(resolve(tmpdir(), "runner-lease-renewal-staged-write-")); let release!: () => void; @@ -2690,13 +2929,13 @@ class ServerFrameReader { socket.once("close", () => this.#publish(null)); } - next(): Promise | null> { + next(timeoutMs: number | null = 2_000): Promise | null> { const queued = this.#frames.shift(); if (queued !== undefined) return Promise.resolve(queued); return new Promise((resolveFrame, rejectFrame) => { - const timer = setTimeout( + const timer = timeoutMs === null ? undefined : setTimeout( () => rejectFrame(new Error("Server WebSocket frame timed out.")), - 2_000, + timeoutMs, ); this.#waiters.push((value) => { clearTimeout(timer); @@ -3131,8 +3370,9 @@ function sendSecure( async function receiveSecure( client: AuthenticatedClient, + frameTimeoutMs: number | null = 2_000, ): Promise | null> { - const frame = await client.reader.next(); + const frame = await client.reader.next(frameTimeoutMs); if (frame === null) return null; const counter = BigInt(frame.counter as number); expect(counter).toBe(client.receiveCounter); @@ -4619,11 +4859,6 @@ describe("DurablePrpControlPlane", () => { { callId: "call-1" }, "command-tool-1", ); - controlPlane.queueCommand( - "turn.interrupt", - { turnId: "turn-1" }, - "command-interrupt-1", - ); const client = await authenticate( controlPlane, controlPlane.issueBootstrapTicket(), @@ -4633,6 +4868,11 @@ describe("DurablePrpControlPlane", () => { expect.objectContaining({ commandId: "command-tool-1" }), ], }); + controlPlane.queueCommand( + "turn.interrupt", + { turnId: "turn-1" }, + "command-interrupt-1", + ); // Exactly what runnerd replays after it is killed between journaling a // command and confirming its effect. Its durable contract promotes such a diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts index 8e75bf1ff8c..3b1d71514ba 100644 --- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts +++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts @@ -65,6 +65,8 @@ const maxFrameBytes = 1024 * 1024; // Secure frames hex-encode ciphertext. Reserve envelope/tag space as well. const maxCommandBytes = Math.floor((maxFrameBytes - 4 * 1024) / 2); const maxCommands = 500; +const maxCommandsAtAttach = 32; +const maxEventsPerAck = 64; class CommandJournalLimitError extends Error { constructor(readonly code: "command_payload_too_large" | "command_journal_full") { super(`Durable PRP command journal bound exceeded: ${code}.`); @@ -83,6 +85,13 @@ const runnerDigestPattern = /^sha256:[0-9a-f]{64}$/; export function isRunnerRecoveryMaintenanceCommand(type: string): boolean { return ["session.snapshot", "session.goal.get", "runner.drain", "runner.suspend", "runner.shutdown", "turn.stop"].includes(type); } +const orderIndependentResultTypes = new Set(["session.snapshot", "session.goal.get", "runner.drain", "semantic_tool.result"]); +function answersAheadOfEvents(command: DurableRecoveryCoreCommand | undefined, result: unknown): boolean { + if (command === undefined || command.status !== "pending" || !orderIndependentResultTypes.has(command.type)) return false; + if (command.type === "session.snapshot" && command.payload.quiesceForWarmAttach === true) return false; + const status = isRecord(result) ? result.status : undefined; + return status === "completed" || status === "rejected"; +} /** The Runner journaled this command before its effect, then died. It sends * that `indeterminate` entry as `failed` with `execution_indeterminate`, never @@ -2033,6 +2042,9 @@ class AuthorityConnection { warmTransitionVersion: 1 | null = null; identity: DurableRecoveryIdentity | null = null; replayOnly = false; + readonly deliveredAtAttach = new Set(); + queuedInboundFrames = 0; + unacknowledgedEvents = 0; activationReceipt: DurableWarmRunTransition | null = null; readonly wire: PrpWireConnection; #closed = false; @@ -2100,6 +2112,7 @@ export class DurablePrpControlPlane { #semanticCallbacksRetired = false; #queuedCommandResults = new Map(); #inboundFrameSettledAt = 0; + #eventCommittedAt: number | null = null; #semanticResultPersistenceFailed = false; #semanticResultFailures: Array<{ callId: string; @@ -2708,6 +2721,10 @@ export class DurablePrpControlPlane { return queued === undefined ? null : Math.max(queued.queuedAt, this.#inboundFrameSettledAt); } + committedEventProgressAt(): number | null { + return this.#eventCommittedAt; + } + withdrawUndeliveredTerminalCommands(): string[] { if (this.#store.state.warmTransition) return []; const commands = this.#store.state.commands; @@ -3139,6 +3156,7 @@ export class DurablePrpControlPlane { } return; } + if (decrypted.kind === "command_result" && this.#answerOnArrival(connection, decrypted)) return; frame = { decrypted }; } else { undecryptedFrames += 1; @@ -3151,12 +3169,15 @@ export class DurablePrpControlPlane { const queued = this.#queuedCommandResults.get(queuedResult); this.#queuedCommandResults.set(queuedResult, { count: (queued?.count ?? 0) + 1, queuedAt: queued?.queuedAt ?? Date.now() }); } + connection.queuedInboundFrames += 1; processing = processing .then(() => "raw" in frame ? this.#handleJson(connection, frame.raw).finally(() => { undecryptedFrames -= 1; }) : this.#handleDecryptedFrame(connection, frame.decrypted)) + .then(() => connection.queuedInboundFrames === 1 ? this.#acknowledgeDeferredEvents(connection) : undefined) .catch(() => connection.close()) .finally(() => { + connection.queuedInboundFrames -= 1; this.#inboundFrameSettledAt = Date.now(); if (queuedResult === null) return; const queued = this.#queuedCommandResults.get(queuedResult); @@ -3237,6 +3258,31 @@ export class DurablePrpControlPlane { if (envelope.kind === "lease_renew") this.#renewLease(connection, envelope); } + #answerOnArrival( + connection: AuthorityConnection, + envelope: Record, + ): boolean { + const result = envelope.payload; + const commandId = isRecord(result) ? result.commandId : undefined; + if ( + typeof commandId !== "string" || + connection.replayOnly || + this.#store.state.warmTransition !== undefined || + !answersAheadOfEvents(this.#store.state.commands.find(command => command.commandId === commandId), result) + ) return false; + try { + this.#store.assertWritable(); + if (!this.#admitEnvelopeVersion(connection, envelope) || !this.#admitAuthenticatedFrame(connection) + || connection.lease === null) return true; + this.#commandResult(connection, envelope); + } catch { + connection.close(); + return true; + } + void this.flushCheckpoint().catch(() => connection.close()); + return true; + } + #admitEnvelopeVersion( connection: AuthorityConnection, envelope: Record, @@ -3982,12 +4028,13 @@ export class DurablePrpControlPlane { this.#store.state.lastLeaseId = lease.leaseId; this.#store.state.lastLeaseExpiresAt = lease.expiresAt; - const pending = connection.replayOnly ? [] : this.#nextPendingCommand(); + const pending = connection.replayOnly ? [] : this.#pendingCommandsAtAttach(); const [pendingCommand] = pending; connection.terminalLifecycleCommandId = pendingCommand && this.#isTerminalLifecycleCommand(pendingCommand) ? pendingCommand.commandId : null; + for (const command of pending.slice(1)) connection.deliveredAtAttach.add(command.commandId); for (const command of pending) { this.#store.state.commandDeliveryCounts[command.commandId] = (this.#store.state.commandDeliveryCounts[command.commandId] ?? 0) + 1; @@ -4065,6 +4112,24 @@ export class DurablePrpControlPlane { return command === undefined ? [] : [command]; } + #pendingCommandsAtAttach(): DurableRecoveryCoreCommand[] { + const [head] = this.#nextPendingCommand(); + if (head === undefined) return []; + const batch = [head]; + if (this.#isTerminalLifecycleCommand(head) || head.type === "run.attach") return batch; + const commands = this.#store.state.commands; + let bytes = Buffer.byteLength(JSON.stringify(this.#wireCommand(head))); + for (let index = commands.indexOf(head) + 1; index < commands.length && batch.length < maxCommandsAtAttach; index += 1) { + const command = commands[index]!; + if (command.status !== "pending" || command.controllerSeq !== batch.at(-1)!.controllerSeq + 1 + || this.#isTerminalLifecycleCommand(command) || command.type === "run.attach") break; + bytes += Buffer.byteLength(JSON.stringify(this.#wireCommand(command))); + if (bytes > maxCommandBytes) break; + batch.push(command); + } + return batch; + } + #controlEnvelope( connection: AuthorityConnection, envelopeId: string, @@ -4102,7 +4167,7 @@ export class DurablePrpControlPlane { ) return; const [command] = this.#nextPendingCommand(); - if (command === undefined) return; + if (command === undefined || connection.deliveredAtAttach.has(command.commandId)) return; if (this.#isTerminalLifecycleCommand(command)) { connection.terminalLifecycleCommandId = command.commandId; } @@ -4330,6 +4395,24 @@ export class DurablePrpControlPlane { ); } + async #acknowledgeDeferredEvents(connection: AuthorityConnection): Promise { + if (connection.unacknowledgedEvents === 0) return; + connection.unacknowledgedEvents = 0; + const identity = this.#identity; + if (this.#eventOwner !== connection || this.#protocolIntegrityError !== null) return; + try { + await this.flushCheckpoint(); + } catch (error) { + if (this.#eventOwner === connection && this.#identity === identity) { + this.#eventCommitFailed(connection, `checkpoint_flush_failed:${safeErrorCauseCode(error)}`); + } + return; + } + if (this.#eventOwner !== connection || this.#identity !== identity || this.#protocolIntegrityError !== null) return; + this.#sendJson(connection, this.#controlEnvelope(connection, `ack_${this.#store.state.ackedSourceSeq}`, "ack", + { ackedSourceSeq: this.#store.state.ackedSourceSeq })); + } + #eventCommitFailed(connection: AuthorityConnection, cause: string, sourceSeq?: number, now = Date.now()): void { const acked = this.#store.state.ackedSourceSeq; const runId = this.#identity.runId; @@ -4546,7 +4629,8 @@ export class DurablePrpControlPlane { }; const isCurrentOwner = () => this.#eventOwner === connection && this.#identity === identity && this.#protocolIntegrityError === null; - if (priority === 0 || isSemanticInput || eventType.startsWith("turn.") || eventType.startsWith("run.")) { + const critical = priority === 0 || isSemanticInput || eventType.startsWith("turn.") || eventType.startsWith("run."); + if (critical) { // Critical lifecycle suffixes and tool admission must finish in this // turn before readers react or a replacement owner adopts their cursor. const candidate = buildCandidate(this.#store.state); @@ -4575,6 +4659,12 @@ export class DurablePrpControlPlane { } } if (!isCurrentOwner()) return; + this.#eventCommittedAt = Date.now(); + if (!critical && connection.queuedInboundFrames > 1 && connection.unacknowledgedEvents + 1 < maxEventsPerAck) { + connection.unacknowledgedEvents += 1; + return; + } + connection.unacknowledgedEvents = 0; const liveCall = !this.#retainsSourceInputs() && isSemanticInput && this.#onSemanticToolInput && semantic !== undefined && typeof semantic.callId === "string" && typeof semantic.operationId === "string" diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport-orphaned-tool-at-attach.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport-orphaned-tool-at-attach.test.ts new file mode 100644 index 00000000000..c946377453d --- /dev/null +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport-orphaned-tool-at-attach.test.ts @@ -0,0 +1,122 @@ +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { createServer } from "node:http"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, it, vi } from "vitest"; +import { DurablePrpControlPlane } from "../control-plane/durable-prp-control-plane.js"; +import { recoverToolCallback, TOOL_CALLBACK_INTERRUPTED_CODE } from "../control-plane/orphaned-tool-callback.js"; +import { createCapabilityRunnerdCodexTransport } from "./runnerd-codex-transport.js"; +import { defaultCapabilityRunnerdBinary, fakeCodex, fakeCodexArgs } from "../../test/support/runnerd-transport.js"; + +const taskTool = { name: "get_task_context", description: "Read current task", inputSchema: { type: "object", properties: {}, additionalProperties: false } }; + +type SavedCommand = { commandId: string; type: string; status: string; payload: Record }; + +it("settles an earlier generation's in-flight tool call at attach even when that attach never finishes startup", async () => { + const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-orphaned-tool-at-attach-")); + const controlPlaneStatePath = join(stateDirectory, "control-plane", "control-plane-state.json"); + const server = createServer(); + let authority: DurablePrpControlPlane | null = null; + let admitConnections = true; + server.on("upgrade", (request, socket, head) => { + if (!authority || !admitConnections) { + socket.destroy(); + return; + } + authority.handleUpgrade(request, socket, "/runner", head); + }); + await new Promise(resolveListen => server.listen(0, "127.0.0.1", resolveListen)); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("Expected a test listener"); + const registration = vi.fn(async (next: DurablePrpControlPlane) => { + authority = next; + return { + connectUrl: `ws://127.0.0.1:${address.port}/runner`, + release: async () => { if (authority === next) authority = null; }, + }; + }); + const identity = { + runnerInstanceId: "runner-orphaned-tool", + environmentLeaseId: "lease-orphaned-tool", + runId: "run-orphaned-tool", + normalizedSessionId: "session-orphaned-tool", + turnId: "turn-orphaned-tool", + itemId: "item-orphaned-tool", + }; + const options = { + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs(stateDirectory, "--emit-tool-call"), + stateDirectory, + prpIdentity: identity, + lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 60_000 }, + controlPlaneRegistration: registration, + }; + const recovered = vi.fn(async (call: { operationId: string; callId: string; input: unknown; admission?: "this_controller" | "earlier_controller" }) => + recoverToolCallback({ runId: identity.runId, call: { tool: call.operationId, callId: call.callId, arguments: call.input }, + admission: call.admission, recover: async () => null })); + const first = createCapabilityRunnerdCodexTransport(options); + first.transport.setServerRequestHandler(() => new Promise>(() => undefined)); + const generations: Array> = []; + let runnerPid: number | null = null; + const savedToolResults = async () => (JSON.parse(await readFile(controlPlaneStatePath, "utf8")) as { commands: SavedCommand[] }) + .commands.filter(command => command.type === "semantic_tool.result"); + try { + const opened = (await first.transport.request("thread/start", { cwd: tmpdir(), dynamicTools: [taskTool] })).thread as { id: string; sessionId: string }; + runnerPid = first.evidence().runnerPid; + const runnerStartedAt = first.transport.processInfo!().startedAt; + await first.transport.request("turn/start", { input: [{ type: "text", text: "Read current task" }] }); + await vi.waitFor(async () => { + const saved = JSON.parse(await readFile(controlPlaneStatePath, "utf8")) as { committedEvents: Array<{ eventType: string }> }; + expect(saved.committedEvents.some(event => event.eventType === "semantic_tool.input")).toBe(true); + }, { timeout: 10_000 }); + await first.detachControllerForRestart(); + expect(await savedToolResults()).toEqual([]); + + const adopt = (graceMs?: number) => { + const generation = createCapabilityRunnerdCodexTransport({ + ...options, + ...(graceMs === undefined ? {} : { runnerReconnectGraceMs: graceMs }), + recoverSemanticToolInput: recovered, + resumeDynamicTools: [taskTool], + resumeProviderSession: { driverSessionId: opened.id, providerSessionId: opened.sessionId }, + runnerProcessLauncher: () => { throw new Error("duplicate runner spawn attempted"); }, + adoptExistingRunner: { + pid: runnerPid!, processGroupId: runnerPid, startedAt: runnerStartedAt, signal: () => true, + isAlive: () => { try { process.kill(runnerPid!, 0); return true; } catch { return false; } }, + }, + }); + generation.transport.setServerRequestHandler(vi.fn(async () => ({ success: true, contentItems: [] }))); + generations.push(generation); + return generation; + }; + + admitConnections = false; + const stalled = adopt(1_500); + await expect(stalled.transport.request("thread/read", {})).rejects.toThrow(); + await stalled.transport.close().catch(() => undefined); + const settled = await savedToolResults(); + expect(settled).toHaveLength(1); + expect(JSON.stringify(settled[0]!.payload)).toContain(TOOL_CALLBACK_INTERRUPTED_CODE); + + admitConnections = true; + const next = adopt(); + await expect(next.transport.request("thread/read", {})).resolves.toEqual( + expect.objectContaining({ thread: expect.objectContaining({ id: opened.id }) })); + await vi.waitFor(async () => { + const results = await savedToolResults(); + expect(results).toHaveLength(1); + expect(results[0]).toMatchObject({ commandId: settled[0]!.commandId, status: "completed" }); + }, { timeout: 10_000 }); + expect(recovered).toHaveBeenCalledTimes(1); + } finally { + for (const generation of generations.reverse()) await generation.transport.close().catch(() => undefined); + await first.transport.close().catch(() => undefined); + if (runnerPid) { + try { process.kill(-runnerPid, "SIGKILL"); } catch { runnerPid = null; } + } + server.closeAllConnections(); + if (server.listening) await new Promise(resolveClose => server.close(() => resolveClose())); + await rm(stateDirectory, { recursive: true, force: true }); + } +}, 120_000); diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport-reattach-carried-commands.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport-reattach-carried-commands.test.ts index f57d46636a6..bed9654e2df 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport-reattach-carried-commands.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport-reattach-carried-commands.test.ts @@ -4,7 +4,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { expect, it, vi } from "vitest"; import { DurablePrpControlPlane } from "../control-plane/durable-prp-control-plane.js"; -import { createCapabilityRunnerdCodexTransport, RunnerStallLedger, waitForRunnerCommand } from "./runnerd-codex-transport.js"; +import { createCapabilityRunnerdCodexTransport, RUNNER_REPLAY_PROGRESS_MAX_EXTENSION_MS, RunnerStallLedger, waitForRunnerCommand } from "./runnerd-codex-transport.js"; import { defaultCapabilityRunnerdBinary, fakeCodex, fakeCodexArgs } from "../../test/support/runnerd-transport.js"; type Carried = "late-tool-result" | "close-path-suspend" | "suspend-then-tool-result" | "event-backlog"; @@ -222,6 +222,73 @@ it("times out when the runner's queued answer stops advancing", async () => { })).rejects.toThrow("provider_transport_failed: PRP command session.snapshot timed out"); }); +it("keeps waiting past the deadline while the controller is still committing the Runner's replayed events", async () => { + let replayedAt = Date.now(); + const command = pendingCommand(); + const replaying = setInterval(() => { replayedAt = Date.now(); }, 5); + setTimeout(() => { command.status = "completed"; }, 150); + try { + await expect(waitForRunnerCommand({ + type: "runner.drain", + deadline: Date.now() + 20, + abortOnClose: false, + isClosed: () => false, + throwIfFailed: () => undefined, + command: () => command, + runnerHasExited: async () => false, + failureCode: () => "runner_ingress_unavailable", + answeredProgressAt: () => null, + replayProgressAt: () => replayedAt, + })).resolves.toBeUndefined(); + } finally { + clearInterval(replaying); + } +}); + +it("times out a Runner that keeps replaying but never answers once the replay extension is spent", async () => { + vi.useFakeTimers(); + try { + const deadline = Date.now() + 30_000; + const waited = waitForRunnerCommand({ + type: "session.snapshot", + deadline, + abortOnClose: false, + isClosed: () => false, + throwIfFailed: () => undefined, + command: () => pendingCommand(), + runnerHasExited: async () => false, + failureCode: () => "provider_transport_failed", + answeredProgressAt: () => null, + replayProgressAt: () => Date.now(), + }); + const outcome = waited.then(() => "completed", (error: unknown) => String(error)); + await vi.advanceTimersByTimeAsync(30_000 + RUNNER_REPLAY_PROGRESS_MAX_EXTENSION_MS - 1_000); + let settled = false; + void outcome.then(() => { settled = true; }); + await vi.advanceTimersByTimeAsync(0); + expect(settled).toBe(false); + await vi.advanceTimersByTimeAsync(2_000); + await expect(outcome).resolves.toContain("provider_transport_failed: PRP command session.snapshot timed out"); + } finally { + vi.useRealTimers(); + } +}); + +it("does not extend a deadline for a Runner whose replay has stopped advancing", async () => { + await expect(waitForRunnerCommand({ + type: "runner.drain", + deadline: Date.now() + 20, + abortOnClose: false, + isClosed: () => false, + throwIfFailed: () => undefined, + command: () => pendingCommand(), + runnerHasExited: async () => false, + failureCode: () => "runner_ingress_unavailable", + answeredProgressAt: () => null, + replayProgressAt: () => Date.now() - 31_000, + })).rejects.toThrow("runner_ingress_unavailable: PRP command runner.drain timed out"); +}); + it("leaves later barriers that share an admission deadline their time after an answered barrier ran past it", async () => { const clock = new RunnerStallLedger(); const deadline = clock.now() + 20; diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts index 53dd42c4053..39714fba6a3 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts @@ -4690,8 +4690,10 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { this.#recoverPendingSemanticInputsWhenReady(); } - #recoverPendingSemanticInputsWhenReady(): void { - if (!this.#retainsSourceInputs() && this.options.recoveryPurpose !== "stop_cleanup" && this.#startupComplete && this.#handlerInstalled && this.#sourceRecoveryIdentity === null && this.#pendingWarmRecoveryCompletion === null) { + #recoverPendingSemanticInputsWhenReady(atAttach = false): void { + if (!this.#retainsSourceInputs() && this.options.recoveryPurpose !== "stop_cleanup" + && (atAttach || (this.#startupComplete && this.#handlerInstalled)) + && this.#sourceRecoveryIdentity === null && this.#pendingWarmRecoveryCompletion === null) { void Promise.resolve().then(() => this.#core?.recoverPendingSemanticInputs()) .catch(error => this.#failTransport(error instanceof Error ? error : new Error(String(error)))); } @@ -5879,6 +5881,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { let checkpointFailed = false; let releaseFailed = false; let processStopFailed = false; + let retirementFailed = false; try { await releaseRunnerProcessOwnership({ runnerSettled, @@ -5920,7 +5923,14 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { throw error; } }), - retireController: () => this.#stopAndDrainAuthority(closeDeadline), + retireController: async () => { + try { + await this.#stopAndDrainAuthority(closeDeadline); + } catch (error) { + retirementFailed = true; + throw error; + } + }, }); } catch (error) { if (checkpoint && checkpointFailed && !releaseFailed && !processStopFailed) { @@ -5928,6 +5938,14 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { checkpoint: () => checkpoint(runnerSettled ? "settled" : "unsettled"), finish: finishClose, }; + } else if (retirementFailed && !releaseFailed && !processStopFailed) { + this.#pendingCloseCheckpointRetry = { + checkpoint: async () => { + await this.#stopAndDrainAuthority(Date.now() + runnerCloseGraceMs(this.options)); + await checkpoint?.(runnerSettled ? "settled" : "unsettled"); + }, + finish: finishClose, + }; } throw error; } finally { @@ -7386,6 +7404,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { } this.#assertStartupOpen(); this.#startPump(); + this.#recoverPendingSemanticInputsWhenReady(true); if (adoptedRunner) { await this.#awaitAdoptedRunnerConnection( adoptedRunner, @@ -7469,7 +7488,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { if (this.#retainsSourceInputs() || (this.#authorityRunId() !== undefined && call.correlation.runId !== this.#authorityRunId())) return null; if (this.#sourceRecoveryIdentity) throw new NativeRunnerSourceReconciliationRequiredError("semantic"); if (call.operationId === "paperclip_finish" || call.operationId === "paperclip_block") { - return this.#handleSemanticToolInput(call); + return this.#startupComplete && this.#handlerInstalled ? this.#handleSemanticToolInput(call) : null; } const outcome = await this.options.recoverSemanticToolInput?.(call); return outcome ? this.#bindSemanticToolOutcome(call, outcome) : null; @@ -8081,6 +8100,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { } return progressAt; }, + replayProgressAt: () => this.#core?.committedEventProgressAt() ?? null, describeStall: () => this.#core === null ? null : describeRunnerCommandStall({ commands: this.#core.store.state.commands, commandId: commandId ?? this.#core.store.state.commands.find(candidate => candidate.type === type)?.commandId, @@ -9727,6 +9747,7 @@ export class RunnerStallLedger { /** Shared polling path keeps startup cancellation separate from owned cleanup commands. */ const RUNNER_ANSWERED_RESULT_STALL_MS = 30_000; +export const RUNNER_REPLAY_PROGRESS_MAX_EXTENSION_MS = 300_000; export async function waitForRunnerCommand(input: { type: string; @@ -9742,12 +9763,19 @@ export async function waitForRunnerCommand(input: { /** Bounded, content-free evidence of what the command waited behind. */ describeStall?: () => string | null; answeredProgressAt?: () => number | null; + replayProgressAt?: () => number | null; }): Promise { const now = () => input.clock ? input.clock.now() : Date.now(); const answeredAndAdvancing = () => { const progressAt = input.answeredProgressAt?.() ?? null; return progressAt !== null && Date.now() - progressAt < RUNNER_ANSWERED_RESULT_STALL_MS; }; + const replayExtensionEndsAt = Date.now() + Math.max(0, input.deadline - now()) + RUNNER_REPLAY_PROGRESS_MAX_EXTENSION_MS; + const replayAdvancing = () => { + const progressAt = input.replayProgressAt?.() ?? null; + return progressAt !== null && Date.now() < replayExtensionEndsAt + && Date.now() - progressAt < RUNNER_ANSWERED_RESULT_STALL_MS; + }; let extended = false; const settled = () => { const command = input.command(); @@ -9768,7 +9796,7 @@ export async function waitForRunnerCommand(input: { input.clock?.credit(Math.max(0, now() - input.deadline) + RUNNER_POST_STOP_GRACE_MS); }; let observed = false; - while (now() < input.deadline || answeredAndAdvancing()) { + while (now() < input.deadline || answeredAndAdvancing() || replayAdvancing()) { observed = true; if (now() >= input.deadline) extended = true; input.throwIfFailed(); diff --git a/packages/paperclip-runner/src/live/runnerd-detach-durability.test.ts b/packages/paperclip-runner/src/live/runnerd-detach-durability.test.ts index add7ff63fa8..2dbea27c294 100644 --- a/packages/paperclip-runner/src/live/runnerd-detach-durability.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-detach-durability.test.ts @@ -4,6 +4,7 @@ import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; import { expect, it, vi } from "vitest"; import { DurablePrpControlPlane, type RunnerProcessResult } from "../control-plane/durable-prp-control-plane.js"; +import { NativeSessionCloseUnrecoverableError } from "../contracts/native-session-backend.js"; import { createCapabilityRunnerdCodexTransport, defaultCapabilityRunnerdBinary, readRunnerdArtifactBinding, runnerdRecoveryInternals } from "./runnerd-codex-transport.js"; it.each(["resolved", "rejected"] as const)("keeps the checkpoint behind controller retirement (%s)", async outcome => { @@ -32,7 +33,7 @@ it.each(["resolved", "rejected"] as const)("keeps the checkpoint behind controll } }); -it.each(["drain", "timeout", "external_close"] as const)("joins an output flush before controller retirement permits a successor writer (%s)", async mode => { +it.each(["drain", "timeout", "external_close", "close_timeout"] as const)("joins an output flush before controller retirement permits a successor writer (%s)", async mode => { const root = await fs.mkdtemp(join(tmpdir(), "runnerd-detach-write-")); await fs.mkdir(join(root, "diagnostics"), { mode: 0o700 }); const binary = defaultCapabilityRunnerdBinary(); @@ -64,7 +65,7 @@ it.each(["drain", "timeout", "external_close"] as const)("joins an output flush codexCommand: resolve(import.meta.dirname, "../../runner/target/debug/fake-codex-app-server"), codexArgs: ["--state-file", join(root, "provider-state.json"), "--split-event-burst", "--split-event-prefix-count", "1"], stateDirectory: root, - closeGraceMs: mode === "timeout" ? 200 : 1_500, + closeGraceMs: mode === "timeout" || mode === "close_timeout" ? 200 : 1_500, ...(mode === "external_close" ? { controlPlaneRegistration: async (authority: DurablePrpControlPlane) => { await authority.start(); const stop = authority.stop.bind(authority); @@ -105,6 +106,15 @@ it.each(["drain", "timeout", "external_close"] as const)("joins an output flush expect(closeSteps.indexOf("release")).toBeLessThan(closeSteps.indexOf("stop")); return; } + if (mode === "close_timeout") { + await expect(bundle.transport.close()).rejects.toThrow("journal drain timed out"); + release(); + await core.drainPendingConnectionProcessing(); + const retried = await bundle.transport.close().then(() => null, (error: Error) => error); + expect(retried).toBeInstanceOf(NativeSessionCloseUnrecoverableError); + expect(retried?.message ?? "").not.toContain("journal drain timed out"); + return; + } let settled = false; detached = bundle.detachControllerForRestart().then(() => { settled = true; }); await vi.waitFor(() => expect(core.activeRunnerConnectionCount()).toBe(0));