diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts index eaa91d2006e..e2e61d45b72 100644 --- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts +++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts @@ -2262,7 +2262,20 @@ export class DurablePrpControlPlane { this.#semanticInputSourceRunId = null; } - async recoverPendingSemanticInputs(): Promise { + #semanticRecoveryPass: Promise | null = null; + + recoverPendingSemanticInputs(): Promise { + const pass = (this.#semanticRecoveryPass ?? Promise.resolve()) + .catch(() => undefined) + .then(() => this.#recoverPendingSemanticInputsOnce()); + this.#semanticRecoveryPass = pass; + void pass.finally(() => { + if (this.#semanticRecoveryPass === pass) this.#semanticRecoveryPass = null; + }).catch(() => undefined); + return pass; + } + + async #recoverPendingSemanticInputsOnce(): Promise { if (this.#retainsSourceInputs() || !this.#recoverSemanticToolInput) return; const identity = this.#identity; for (const saved of [...this.#store.state.committedEvents]) { diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts index c772fb40098..d9627e81fa6 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts @@ -1398,6 +1398,60 @@ it("refuses a terminal call that arrives before a held call's result (restart ch } }, 20_000); +it("answers a held call before starting the next turn (restart chaos t101)", async () => { + const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-held-before-turn-")); + const OriginalCore = durableControlPlane.DurablePrpControlPlane; + let core!: DurablePrpControlPlane; + const coreSpy = vi.spyOn(durableControlPlane, "DurablePrpControlPlane").mockImplementation(function (options) { + return core = new OriginalCore(options); + }); + const order: string[] = []; + const recovered = vi.fn(async (call: { callId: string }) => { + order.push(`recover:${call.callId}`); + await new Promise((resolveRead) => setTimeout(resolveRead, 300)); + return { result: { success: true, contentItems: [{ type: "inputText", text: "brief body" }] } }; + }); + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), codexCommand: fakeCodex, + codexArgs: fakeCodexArgs(stateDirectory), stateDirectory, recoverSemanticToolInput: recovered, + }); + bundle.transport.setServerRequestHandler(vi.fn(async () => ({ success: true, contentItems: [] }))); + try { + await bundle.transport.request("thread/start", { cwd: tmpdir(), dynamicTools: codexSemanticToolSpecs() }); + const identity = core.store.state.identity; + const correlation = { runId: identity.runId, normalizedSessionId: identity.normalizedSessionId, + turnId: identity.turnId, itemId: identity.itemId }; + core.store.state.committedEvents.unshift({ sourceSeq: 0, sourceEventId: "held-read-input", eventType: "semantic_tool.input", + priority: 1, deliveryCount: 1, logicalEffectCount: 1, envelope: { payload: { runId: identity.runId, + normalizedSessionId: identity.normalizedSessionId, sourceEventId: "held-read-input", eventType: "semantic_tool.input", + itemId: identity.itemId, payload: { semantic_tool: { callId: "exec-held-read", operationId: "read_document", + input: { key: "brief" }, correlation } } } } }); + expect(core.semanticToolResultsSettled()).toBe(false); + const queueCommand = core.queueCommand.bind(core); + vi.spyOn(core, "queueCommand").mockImplementation((type, ...rest) => { + order.push(type); + return queueCommand(type, ...rest); + }); + const startupPass = core.recoverPendingSemanticInputs(); + await vi.waitFor(() => expect(recovered).toHaveBeenCalledOnce()); + const turnStarted = expectTurnStarted(bundle.transport.notifications()[Symbol.asyncIterator]()); + void turnStarted.catch(() => undefined); + await bundle.transport.request("turn/start", { input: [{ type: "text", text: "Continue after the restart." }] }); + await turnStarted; + await startupPass; + expect(recovered).toHaveBeenCalledOnce(); + expect(order.slice(0, 2)).toEqual(["recover:exec-held-read", "semantic_tool.result"]); + expect(order.indexOf("semantic_tool.result")).toBeLessThan(order.indexOf("turn.start")); + const result = core.store.state.commands.find(command => command.type === "semantic_tool.result"); + expect(result).toMatchObject({ payload: { callId: "exec-held-read", operationId: "read_document" } }); + expect(result!.status).not.toBe("pending"); + } finally { + coreSpy.mockRestore(); + await bundle.transport.close().catch(() => undefined); + await rm(stateDirectory, { recursive: true, force: true }); + } +}, 30_000); + it.each(["after_budget", "within_budget", "interrupted_within_budget", "persistence_failure"] as const)( "fences reusable suspension against late semantic completion (%s)", async (mode) => { diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts index 62b46db025e..700127cbc2e 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts @@ -7262,7 +7262,23 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { if (!Number.isSafeInteger(turnStartTimeoutMs) || turnStartTimeoutMs <= 0) { throw new Error("turnStartTimeoutMs must be a positive safe integer"); } - const commandDeadline = Date.now() + turnStartTimeoutMs; + let commandDeadline = Date.now() + turnStartTimeoutMs; + const heldCalls = this.#core; + if (typeof heldCalls?.semanticToolResultsSettled === "function" && !heldCalls.semanticToolResultsSettled()) { + let recovering = true; + void Promise.resolve(heldCalls.recoverPendingSemanticInputs()) + .catch(error => this.#failTransport(error instanceof Error ? error : new Error(String(error)))) + .finally(() => { recovering = false; }); + while (recovering && Date.now() < commandDeadline) { + this.#assertStartupOpen(); + this.#throwIfFailed(); + await new Promise((resolveWait) => setTimeout(resolveWait, 10)); + } + this.#assertStartupOpen(); + this.#throwIfFailed(); + if (recovering) throw new Error("native_turn_start_held_tool_calls_unresolved: recovering an earlier turn's held Paperclip tool calls did not finish before the turn-start deadline"); + commandDeadline = Date.now() + turnStartTimeoutMs; + } const compactionTurn = params[COMPACTION_TURN] === true; const compactionPayload = compactionTurn ? runnerdCompactionTurnPayload(this.options.provider ?? "codex") : null; const input = Array.isArray(params.input) ? params.input.map(record) : []; diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index d96451efb43..5a577b4ab17 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -764,6 +764,14 @@ describe("remote controller restart adoption", () => { await rm(stateBase, { recursive: true, force: true }); } }); + it("answers a local run's held Paperclip tool calls after a restart (restart chaos t101)", async () => { + state.createBackend.mockClear(); + state.createTransport.mockClear(); + await createRunnerdBackend({ db: leaseDb(execution), execution, runnerInstanceId: "runner" }); + state.createBackend.mock.calls.at(-1)![1].codexTransportFactory!(); + expect((state.createTransport.mock.calls.at(-1)![0] as { recoverSemanticToolInput?: unknown }).recoverSemanticToolInput) + .toEqual(expect.any(Function)); + }); it("rejects a replaced sandbox before reading or mutating it", async () => { const { execute, target } = fixture(); target.sandboxLeaseAcquisition.providerLeaseId = "replacement"; diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 74ae7795aca..9dfa84c231a 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -14904,8 +14904,8 @@ async function createRunnerdBackendWithinSessionClaim( stateDirectory: root, runnerBinary: remoteDot?.runnerBinary ?? resolvePaperclipRunnerBinary(), runnerArtifact: input.controllerPersistence?.runnerArtifact, runnerReconnectGraceMs: remoteDot ? REMOTE_RUNNER_RECONNECT_GRACE_MS : undefined, - recoverSemanticToolInput: input.controllerPersistence ? (call: { operationId: string; callId: string; input: unknown }) => - authorityEpoch.recover({ tool: call.operationId, callId: call.callId, arguments: call.input }) : undefined, + recoverSemanticToolInput: (call: { operationId: string; callId: string; input: unknown }) => + authorityEpoch.recover({ tool: call.operationId, callId: call.callId, arguments: call.input }), ...(remoteDot ?? {}), identity: { runnerInstanceId: effectiveRunnerInstanceId, environmentLeaseId: effectiveEnvironmentLeaseId, runId: input.execution.binding.runId, normalizedSessionId: nativeSessionKey(input.execution), @@ -16499,11 +16499,11 @@ async function createRunnerdBackendWithinSessionClaim( stateDirectory: root, runnerBinary: controllerRunnerBinary, runnerArtifact: input.controllerPersistence?.runnerArtifact ?? providerPackAuthority?.runnerArtifact, - recoverSemanticToolInput: input.controllerPersistence ? call => { + recoverSemanticToolInput: call => { const current = sessionToolAuthorityEpochs.get(sessionScopeId); if (!current || current.runId !== call.correlation.runId) throw new Error("native_tool_authority_epoch_revoked"); return current.recover({ tool: call.operationId, callId: call.callId, arguments: call.input }); - } : undefined, + }, codexCommand: remoteCodexBinary ?? undefined, sourceCodexHome: remoteTarget ? resolveSourceCodexHome(resolveNativeProviderEnvironment(input.execution.provider, runnerEnvironment))