diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts index 492e7534800..2ddf31f66b5 100644 --- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts +++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts @@ -293,6 +293,12 @@ export class SemanticToolNotDispatchedError extends Error { } } +export class SemanticToolHandedOffError extends Error { + constructor() { + super("semantic_tool_handed_off"); + } +} + export interface DurablePrpControlPlaneOptions { stateDirectory: string; identity: DurableRecoveryIdentity; @@ -4798,6 +4804,7 @@ export class DurablePrpControlPlane { queueResult(outcome.result, outcome.isError === true), ) .catch((error: unknown) => { + if (error instanceof SemanticToolHandedOffError) return; if (error instanceof SemanticToolNotDispatchedError) { queueResult({ error: { code: "semantic_tool_not_dispatched", diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport-orphaned-tool-at-attach.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport-orphaned-tool-at-attach.test.ts index c946377453d..080305097ad 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport-orphaned-tool-at-attach.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport-orphaned-tool-at-attach.test.ts @@ -3,7 +3,8 @@ import { createServer } from "node:http"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { expect, it, vi } from "vitest"; -import { DurablePrpControlPlane } from "../control-plane/durable-prp-control-plane.js"; +import * as durableControlPlane from "../control-plane/durable-prp-control-plane.js"; +import type { DurablePrpControlPlane } from "../control-plane/durable-prp-control-plane.js"; import { recoverToolCallback, TOOL_CALLBACK_INTERRUPTED_CODE } from "../control-plane/orphaned-tool-callback.js"; import { createCapabilityRunnerdCodexTransport } from "./runnerd-codex-transport.js"; import { defaultCapabilityRunnerdBinary, fakeCodex, fakeCodexArgs } from "../../test/support/runnerd-transport.js"; @@ -12,7 +13,7 @@ const taskTool = { name: "get_task_context", description: "Read current task", i type SavedCommand = { commandId: string; type: string; status: string; payload: Record }; -it("settles an earlier generation's in-flight tool call at attach even when that attach never finishes startup", async () => { +it.each(["dispatched to its handler", "still before dispatch"] as const)("settles an earlier generation's in-flight tool call at attach even when that attach never finishes startup (%s at detach)", async stage => { const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-orphaned-tool-at-attach-")); const controlPlaneStatePath = join(stateDirectory, "control-plane", "control-plane-state.json"); const server = createServer(); @@ -55,8 +56,28 @@ it("settles an earlier generation's in-flight tool call at attach even when that const recovered = vi.fn(async (call: { operationId: string; callId: string; input: unknown; admission?: "this_controller" | "earlier_controller" }) => recoverToolCallback({ runId: identity.runId, call: { tool: call.operationId, callId: call.callId, arguments: call.input }, admission: call.admission, recover: async () => null })); + let reached!: () => void; + const callReached = new Promise(resolveReached => { reached = resolveReached; }); + let releaseDispatch!: () => void; + const dispatchGate = new Promise(resolveGate => { releaseDispatch = resolveGate; }); + let gatedOutcome: Promise | null = null; + const OriginalCore = durableControlPlane.DurablePrpControlPlane; + const coreSpy = vi.spyOn(durableControlPlane, "DurablePrpControlPlane").mockImplementation(function (coreOptions) { + const dispatch = coreOptions.onSemanticToolInput; + if (stage === "still before dispatch" && dispatch && !gatedOutcome) { + coreOptions = { ...coreOptions, onSemanticToolInput: call => { + reached(); + gatedOutcome = dispatchGate.then(() => dispatch(call)); + return gatedOutcome as ReturnType; + } }; + } + return new OriginalCore(coreOptions); + }); const first = createCapabilityRunnerdCodexTransport(options); - first.transport.setServerRequestHandler(() => new Promise>(() => undefined)); + first.transport.setServerRequestHandler(() => { + reached(); + return new Promise>(() => undefined); + }); const generations: Array> = []; let runnerPid: number | null = null; const savedToolResults = async () => (JSON.parse(await readFile(controlPlaneStatePath, "utf8")) as { commands: SavedCommand[] }) @@ -66,11 +87,11 @@ it("settles an earlier generation's in-flight tool call at attach even when that runnerPid = first.evidence().runnerPid; const runnerStartedAt = first.transport.processInfo!().startedAt; await first.transport.request("turn/start", { input: [{ type: "text", text: "Read current task" }] }); - await vi.waitFor(async () => { - const saved = JSON.parse(await readFile(controlPlaneStatePath, "utf8")) as { committedEvents: Array<{ eventType: string }> }; - expect(saved.committedEvents.some(event => event.eventType === "semantic_tool.input")).toBe(true); - }, { timeout: 10_000 }); + await callReached; await first.detachControllerForRestart(); + releaseDispatch(); + await (gatedOutcome as Promise | null)?.catch(() => undefined); + await new Promise(resolveTurn => setImmediate(resolveTurn)); expect(await savedToolResults()).toEqual([]); const adopt = (graceMs?: number) => { @@ -110,6 +131,8 @@ it("settles an earlier generation's in-flight tool call at attach even when that }, { timeout: 10_000 }); expect(recovered).toHaveBeenCalledTimes(1); } finally { + releaseDispatch(); + coreSpy.mockRestore(); for (const generation of generations.reverse()) await generation.transport.close().catch(() => undefined); await first.transport.close().catch(() => undefined); if (runnerPid) { @@ -120,3 +143,55 @@ it("settles an earlier generation's in-flight tool call at attach even when that await rm(stateDirectory, { recursive: true, force: true }); } }, 120_000); + +it("still settles a call stopped before dispatch as not dispatched when the controller closes without a restart", async () => { + const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-not-dispatched-at-close-")); + const controlPlaneStatePath = join(stateDirectory, "control-plane", "control-plane-state.json"); + let reached!: () => void; + const callReached = new Promise(resolveReached => { reached = resolveReached; }); + let releaseDispatch!: () => void; + const dispatchGate = new Promise(resolveGate => { releaseDispatch = resolveGate; }); + let gatedOutcome: Promise | null = null; + const OriginalCore = durableControlPlane.DurablePrpControlPlane; + const coreSpy = vi.spyOn(durableControlPlane, "DurablePrpControlPlane").mockImplementation(function (coreOptions) { + const dispatch = coreOptions.onSemanticToolInput; + if (dispatch && !gatedOutcome) { + coreOptions = { ...coreOptions, onSemanticToolInput: call => { + reached(); + gatedOutcome = dispatchGate.then(() => dispatch(call)); + return gatedOutcome as ReturnType; + } }; + } + return new OriginalCore(coreOptions); + }); + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs(stateDirectory, "--emit-tool-call"), + stateDirectory, + }); + bundle.transport.setServerRequestHandler(() => new Promise>(() => undefined)); + let runnerPid: number | null = null; + try { + await bundle.transport.request("thread/start", { cwd: tmpdir(), dynamicTools: [taskTool] }); + runnerPid = bundle.evidence().runnerPid; + await bundle.transport.request("turn/start", { input: [{ type: "text", text: "Read current task" }] }); + await callReached; + const closing = bundle.transport.close().catch(() => undefined); + releaseDispatch(); + await (gatedOutcome as Promise | null)?.catch(() => undefined); + await closing; + const results = (JSON.parse(await readFile(controlPlaneStatePath, "utf8")) as { commands: SavedCommand[] }) + .commands.filter(command => command.type === "semantic_tool.result"); + expect(results).toHaveLength(1); + expect(JSON.stringify(results[0]!.payload)).toContain("semantic_tool_not_dispatched"); + } finally { + releaseDispatch(); + coreSpy.mockRestore(); + await bundle.transport.close().catch(() => undefined); + if (runnerPid) { + try { process.kill(-runnerPid, "SIGKILL"); } catch { runnerPid = null; } + } + await rm(stateDirectory, { recursive: true, force: true }); + } +}, 120_000); diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts index a9e56ff8f2e..338eedbcd6f 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts @@ -76,6 +76,7 @@ import { NativeRunnerResumeStateUnavailableError, NativeRunnerSourceReconciliati import { DurablePrpControlPlane, inspectStagedRunAttachment, + SemanticToolHandedOffError, SemanticToolNotDispatchedError, durableRecoveryInternals, inspectWarmRunTransition, @@ -6518,7 +6519,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { onAuthenticatedConnection: () => this.#startupSteps?.advance("runner.process.launch", "provider.session.load"), get authorityRunId() { return authorityRunId(); }, semanticInputMode: this.#retainsSourceInputs() || this.options.recoveryPurpose === "stop_cleanup" ? "retain_only" : "dispatch", - onSemanticToolInput: this.options.recoveryPurpose === "stop_cleanup" ? undefined : (call) => this.#handleSemanticToolInput(call), + onSemanticToolInput: this.options.recoveryPurpose === "stop_cleanup" ? undefined : (call) => this.#dispatchSemanticToolInput(call), recoverSemanticToolInput: this.options.recoveryPurpose === "stop_cleanup" ? undefined : call => this.#recoverSemanticToolInput(call), onCommittedEvent: async (event) => { const source = this.#sourceAccountingIdentity(); @@ -7079,7 +7080,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { onAuthenticatedConnection: () => this.#startupSteps?.advance("runner.process.launch", "provider.session.load"), get authorityRunId() { return authorityRunId(); }, semanticInputMode: this.#retainsSourceInputs() || this.options.recoveryPurpose === "stop_cleanup" ? "retain_only" : "dispatch", - onSemanticToolInput: this.options.recoveryPurpose === "stop_cleanup" ? undefined : (call) => this.#handleSemanticToolInput(call), + onSemanticToolInput: this.options.recoveryPurpose === "stop_cleanup" ? undefined : (call) => this.#dispatchSemanticToolInput(call), recoverSemanticToolInput: this.options.recoveryPurpose === "stop_cleanup" ? undefined : call => this.#recoverSemanticToolInput(call), onCommittedEvent: async (event) => { const source = this.#sourceAccountingIdentity(); @@ -7680,6 +7681,15 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { return outcome ? this.#bindSemanticToolOutcome(call, outcome) : null; } + async #dispatchSemanticToolInput(call: Parameters>[0]) { + try { + return await this.#handleSemanticToolInput(call); + } catch (error) { + if (error instanceof SemanticToolNotDispatchedError && this.#controllerDetachedForRestart) throw new SemanticToolHandedOffError(); + throw error; + } + } + async #handleSemanticToolInput( call: Parameters< NonNullable