diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index c4777023819..3e07b832df4 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1772,6 +1772,22 @@ in-flight turn stay for the next same-run reattachment. An abort (a Stop or the recovery deadline), any identity or authority refusal, any other recovery failure, and a protocol integrity fault still stop it. +A failed close can leave its `runner.suspend` (or `runner.shutdown`) pending in +the saved journal of a Runner it kept alive. The controller sends one command at +a time, so neither that command nor any command queued after it reached the +Runner. A reattachment that adopts the live Runner withdraws those terminal +commands before it sends anything. It keeps every other pending command, such +as a tool result that arrived after the close, and renumbers it so the Runner's +command sequence stays contiguous. The adopted Runner continues without exiting. + +At reconnect, a Runner also sends the events it collected while it was +disconnected. The controller commits a connection's frames in order, so a +recovery command's answer can wait behind that backlog. A recovery barrier does +not time out while the Runner's answer to it, or to a command ahead of it, is +queued and the controller committed a frame in the last 30 s. The time spent +past the deadline is credited to the admission clock, so later barriers that +share the deadline keep their time. + A remote native run can lose its original runner or sandbox before reattachment. This happens after a server restart, and for a same-run retry after a turn ends without a semantic result. The reattachment then fails with diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts index 9e77d4c4b25..d928f3d40f0 100644 --- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts +++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts @@ -2098,6 +2098,8 @@ export class DurablePrpControlPlane { * in-flight and failed dispatches remain. */ #liveDispatchedSemanticCalls = new Set(); #semanticCallbacksRetired = false; + #queuedCommandResults = new Map(); + #inboundFrameSettledAt = 0; #semanticResultPersistenceFailed = false; #semanticResultFailures: Array<{ callId: string; @@ -2675,6 +2677,27 @@ export class DurablePrpControlPlane { * source for a later run.attach. Repeating the same write is idempotent; * changing an established seed fails closed. */ + queuedCommandResultProgressAt(commandId: string): number | null { + const queued = this.#queuedCommandResults.get(commandId); + return queued === undefined ? null : Math.max(queued.queuedAt, this.#inboundFrameSettledAt); + } + + withdrawUndeliveredTerminalCommands(): string[] { + if (this.#store.state.warmTransition) return []; + const commands = this.#store.state.commands; + const firstPending = commands.findIndex((command) => command.status === "pending"); + if (firstPending < 0) return []; + const suffix = commands.slice(firstPending); + const kept = suffix.filter((command) => !this.#isTerminalLifecycleCommand(command)); + if (kept.length === suffix.length) return []; + const withdrawn = suffix.filter((command) => this.#isTerminalLifecycleCommand(command)); + for (const command of withdrawn) delete this.#store.state.commandDeliveryCounts[command.commandId]; + kept.forEach((command, index) => { command.controllerSeq = firstPending + index + 1; }); + commands.splice(firstPending, suffix.length, ...kept); + this.#store.save(); + return withdrawn.map((command) => command.type); + } + persistRunAttachTemplate(runAttachTemplate: Record): void { if (!isRecord(runAttachTemplate.provider)) { throw new Error("Durable PRP run attachment template is invalid."); @@ -2950,14 +2973,18 @@ export class DurablePrpControlPlane { } } const controllerSeq = this.#store.state.commands.length + 1; + const sequenceId = `command_prp_${controllerSeq.toString().padStart(8, "0")}`; + let generatedId = sequenceId; + for (let reuse = 1; this.#store.state.commands.some((candidate) => candidate.commandId === generatedId); reuse += 1) { + generatedId = `${sequenceId}_${reuse}`; + } const command: DurableRecoveryCoreCommand = { schema: type === "external_provider.operation" || type === "runner.attachment.download" ? "paperclip.prp.command.v3" : type.startsWith("session.goal.") ? "paperclip.prp.command.v2" : "paperclip.prp.command.v1", - commandId: - commandId ?? `command_prp_${controllerSeq.toString().padStart(8, "0")}`, + commandId: commandId ?? generatedId, controllerSeq, type, issuedAt: new Date().toISOString(), @@ -3091,11 +3118,25 @@ export class DurablePrpControlPlane { undecryptedFrames += 1; frame = { raw: value }; } + const answeredCommandId = "decrypted" in frame && frame.decrypted.kind === "command_result" + ? (frame.decrypted.payload as Record | undefined)?.commandId : undefined; + const queuedResult = typeof answeredCommandId === "string" ? answeredCommandId : null; + if (queuedResult !== null) { + const queued = this.#queuedCommandResults.get(queuedResult); + this.#queuedCommandResults.set(queuedResult, { count: (queued?.count ?? 0) + 1, queuedAt: queued?.queuedAt ?? Date.now() }); + } processing = processing .then(() => "raw" in frame ? this.#handleJson(connection, frame.raw).finally(() => { undecryptedFrames -= 1; }) : this.#handleDecryptedFrame(connection, frame.decrypted)) - .catch(() => connection.close()); + .catch(() => connection.close()) + .finally(() => { + this.#inboundFrameSettledAt = Date.now(); + if (queuedResult === null) return; + const queued = this.#queuedCommandResults.get(queuedResult); + if (queued !== undefined && queued.count > 1) queued.count -= 1; + else this.#queuedCommandResults.delete(queuedResult); + }); const tail = processing; this.#connectionProcessing.set(connection, tail); const release = () => { diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport-reattach-carried-commands.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport-reattach-carried-commands.test.ts new file mode 100644 index 00000000000..f57d46636a6 --- /dev/null +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport-reattach-carried-commands.test.ts @@ -0,0 +1,279 @@ +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { createServer } from "node:http"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, it, vi } from "vitest"; +import { DurablePrpControlPlane } from "../control-plane/durable-prp-control-plane.js"; +import { createCapabilityRunnerdCodexTransport, RunnerStallLedger, waitForRunnerCommand } from "./runnerd-codex-transport.js"; +import { defaultCapabilityRunnerdBinary, fakeCodex, fakeCodexArgs } from "../../test/support/runnerd-transport.js"; + +type Carried = "late-tool-result" | "close-path-suspend" | "suspend-then-tool-result" | "event-backlog"; + +const BACKLOG = 400; +const taskTool = { name: "get_task_context", description: "Read current task", inputSchema: { type: "object", properties: {}, additionalProperties: false } }; + +async function reattachWithCarriedCommands(carried: Carried) { + const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-reattach-carried-")); + const controlPlaneStatePath = join(stateDirectory, "control-plane", "control-plane-state.json"); + const server = createServer(); + let authority: DurablePrpControlPlane | null = null; + server.on("upgrade", (request, socket, head) => { + if (!authority) { + socket.destroy(); + return; + } + authority.handleUpgrade(request, socket, "/runner", head); + }); + await new Promise(resolveListen => server.listen(0, "127.0.0.1", resolveListen)); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("Expected a reattachment test listener"); + const registration = vi.fn(async (next: DurablePrpControlPlane) => { + authority = next; + return { + connectUrl: `ws://127.0.0.1:${address.port}/runner`, + release: async () => { + if (authority === next) authority = null; + }, + }; + }); + const identity = { + runnerInstanceId: "runner-reattach-carried", + environmentLeaseId: "lease-reattach-carried", + runId: "run-reattach-carried", + normalizedSessionId: "session-reattach-carried", + turnId: "turn-reattach-carried", + itemId: "item-reattach-carried", + }; + const toolCall = carried === "late-tool-result" || carried === "suspend-then-tool-result"; + const options = { + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs(stateDirectory, ...(toolCall ? ["--emit-tool-call"] : carried === "event-backlog" + ? ["--hold-turn", "--held-output", "--held-output-count", String(BACKLOG), "--held-output-release-file", join(stateDirectory, "release-output")] : [])), + stateDirectory, + prpIdentity: identity, + lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 60_000 }, + controlPlaneRegistration: registration, + }; + const first = createCapabilityRunnerdCodexTransport(options); + let finishTool!: (value: Record) => void; + const lateTool = vi.fn(() => new Promise>(resolveTool => { finishTool = resolveTool; })); + first.transport.setServerRequestHandler(lateTool); + let adopted: ReturnType | null = null; + let runnerPid: number | null = null; + try { + const opened = (await first.transport.request("thread/start", { + cwd: tmpdir(), + dynamicTools: toolCall ? [taskTool] : [], + })).thread as { id: string; sessionId: string }; + runnerPid = first.evidence().runnerPid; + const runnerStartedAt = first.transport.processInfo!().startedAt; + if (carried === "event-backlog") { + await first.transport.request("turn/start", { input: [{ type: "text", text: "Keep working" }] }); + } + if (toolCall) { + await first.transport.request("turn/start", { input: [{ type: "text", text: "Read current task" }] }); + await vi.waitFor(() => expect(lateTool).toHaveBeenCalledOnce(), { timeout: 5_000 }); + } + await first.detachControllerForRestart(); + expect(() => process.kill(runnerPid!, 0)).not.toThrow(); + if (toolCall) { + finishTool({ success: true, contentItems: [{ type: "inputText", text: "task context" }] }); + await vi.waitFor(async () => { + const saved = JSON.parse(await readFile(controlPlaneStatePath, "utf8")) as { commands: Array<{ type: string; status: string }> }; + expect(saved.commands.at(-1)).toMatchObject({ type: "semantic_tool.result", status: "pending" }); + }, { timeout: 5_000 }); + if (carried === "suspend-then-tool-result") { + const saved = JSON.parse(await readFile(controlPlaneStatePath, "utf8")) as { commands: Array> }; + const result = saved.commands.pop()!; + for (const type of ["runner.drain", "runner.suspend"]) { + const controllerSeq = saved.commands.length + 1; + saved.commands.push({ + schema: "paperclip.prp.command.v1", + commandId: `command_prp_${controllerSeq.toString().padStart(8, "0")}`, + controllerSeq, + type, + issuedAt: new Date().toISOString(), + payload: {}, + status: "pending", + result: null, + }); + } + saved.commands.push({ ...result, controllerSeq: saved.commands.length + 1 }); + await writeFile(controlPlaneStatePath, JSON.stringify(saved), { mode: 0o600 }); + } + } else if (carried === "event-backlog") { + await writeFile(join(stateDirectory, "release-output"), "release"); + await new Promise(resolveWait => setTimeout(resolveWait, 3_000)); + } else { + const saved = JSON.parse(await readFile(controlPlaneStatePath, "utf8")) as { commands: Array> }; + for (const type of ["runner.drain", "runner.suspend"]) { + const controllerSeq = saved.commands.length + 1; + saved.commands.push({ + schema: "paperclip.prp.command.v1", + commandId: `command_prp_${controllerSeq.toString().padStart(8, "0")}`, + controllerSeq, + type, + issuedAt: new Date().toISOString(), + payload: {}, + status: "pending", + result: null, + }); + } + await writeFile(controlPlaneStatePath, JSON.stringify(saved), { mode: 0o600 }); + } + + const redispatched = vi.fn(async () => ({ success: true, contentItems: [] })); + adopted = createCapabilityRunnerdCodexTransport({ + ...options, + onAccountingEvent: async () => { + if (carried === "event-backlog") await new Promise(resolveWait => setTimeout(resolveWait, 100)); + }, + resumeDynamicTools: toolCall ? [taskTool] : [], + resumeProviderSession: { driverSessionId: opened.id, providerSessionId: opened.sessionId }, + runnerProcessLauncher: () => { + throw new Error("duplicate runner spawn attempted"); + }, + adoptExistingRunner: { + pid: runnerPid!, + processGroupId: runnerPid, + startedAt: runnerStartedAt, + signal: () => true, + isAlive: () => { + try { + process.kill(runnerPid!, 0); + return true; + } catch { + return false; + } + }, + }, + }); + adopted.transport.setServerRequestHandler(redispatched); + await expect(adopted.transport.request("thread/read", {})).resolves.toEqual( + expect.objectContaining({ thread: expect.objectContaining({ id: opened.id }) }), + ); + expect(() => process.kill(runnerPid!, 0)).not.toThrow(); + const commands = authority!.store.state.commands; + expect(commands.map(command => command.controllerSeq)).toEqual(commands.map((_command, index) => index + 1)); + expect(commands.some(command => command.type === "runner.suspend")).toBe(false); + if (toolCall) { + expect(commands.filter(command => command.type === "semantic_tool.result")).toMatchObject([{ status: "completed" }]); + expect(redispatched).not.toHaveBeenCalled(); + } + } finally { + await adopted?.transport.close().catch(() => undefined); + await first.transport.close().catch(() => undefined); + if (runnerPid) { + try { process.kill(-runnerPid, "SIGKILL"); } catch { /* already exited */ } + } + server.closeAllConnections(); + if (server.listening) await new Promise(resolveClose => server.close(() => resolveClose())); + await rm(stateDirectory, { recursive: true, force: true }); + } +} + +it.each([ + ["a tool result its predecessor queued after the restart", "late-tool-result"], + ["a runner.suspend left by an earlier attempt's close path", "close-path-suspend"], + ["a runner.suspend with a tool result queued after it", "suspend-then-tool-result"], + ["an event backlog ahead of its recovery snapshot result", "event-backlog"], +] as const)("reattaches a live runner past %s", async (_label, carried) => { + await reattachWithCarriedCommands(carried); +}, 120_000); + +function pendingCommand() { + return { status: "pending" }; +} + +it("keeps waiting past the deadline while the runner's answer is queued behind frames the controller is still committing", async () => { + let progressAt = Date.now(); + const command = pendingCommand(); + const advancing = setInterval(() => { progressAt = Date.now(); }, 5); + setTimeout(() => { command.status = "completed"; }, 150); + try { + await expect(waitForRunnerCommand({ + type: "session.snapshot", + deadline: Date.now() + 20, + abortOnClose: false, + isClosed: () => false, + throwIfFailed: () => undefined, + command: () => command, + runnerHasExited: async () => false, + failureCode: () => "provider_transport_failed", + answeredProgressAt: () => progressAt, + })).resolves.toBeUndefined(); + } finally { + clearInterval(advancing); + } +}); + +it("times out when the runner's queued answer stops advancing", async () => { + await expect(waitForRunnerCommand({ + type: "session.snapshot", + deadline: Date.now() + 20, + abortOnClose: false, + isClosed: () => false, + throwIfFailed: () => undefined, + command: () => pendingCommand(), + runnerHasExited: async () => false, + failureCode: () => "provider_transport_failed", + answeredProgressAt: () => Date.now() - 31_000, + })).rejects.toThrow("provider_transport_failed: PRP command session.snapshot timed out"); +}); + +it("leaves later barriers that share an admission deadline their time after an answered barrier ran past it", async () => { + const clock = new RunnerStallLedger(); + const deadline = clock.now() + 20; + let progressAt = Date.now(); + const drain = pendingCommand(); + const snapshot = pendingCommand(); + const advancing = setInterval(() => { progressAt = Date.now(); }, 5); + setTimeout(() => { drain.status = "completed"; }, 150); + const wait = (type: string, command: { status: string }, answered: () => number | null) => waitForRunnerCommand({ + type, + deadline, + clock, + abortOnClose: false, + isClosed: () => false, + throwIfFailed: () => undefined, + command: () => command, + runnerHasExited: async () => false, + failureCode: () => "provider_transport_failed", + answeredProgressAt: answered, + }); + try { + await expect(wait("runner.drain", drain, () => progressAt)).resolves.toBeUndefined(); + } finally { + clearInterval(advancing); + } + setTimeout(() => { snapshot.status = "completed"; }, 50); + await expect(wait("session.snapshot", snapshot, () => null)).resolves.toBeUndefined(); +}); + +it("completes a command whose answer arrives during the last sleep before its deadline", async () => { + vi.useFakeTimers(); + try { + const command = pendingCommand(); + const deadline = Date.now() + 25; + const waited = waitForRunnerCommand({ + type: "runner.drain", + deadline, + abortOnClose: false, + isClosed: () => false, + throwIfFailed: () => undefined, + command: () => command, + runnerHasExited: async () => false, + failureCode: () => "provider_transport_failed", + answeredProgressAt: () => null, + }); + const outcome = waited.then(() => "completed", (error: unknown) => String(error)); + await vi.advanceTimersByTimeAsync(20); + command.status = "completed"; + await vi.advanceTimersByTimeAsync(10); + expect(Date.now()).toBeGreaterThanOrEqual(deadline); + await expect(outcome).resolves.toBe("completed"); + } finally { + vi.useRealTimers(); + } +}); diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport-remote-retention.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport-remote-retention.test.ts index c68c0f131e5..dd23825af1c 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport-remote-retention.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport-remote-retention.test.ts @@ -95,7 +95,7 @@ it.each([ expect(result.runnerState).toBe("gone"); }, 30_000); -it("keeps a retained remote Runner for the reattaching controller, which settles it into a journal the next attempt can relaunch", async () => { +it("keeps a retained remote Runner for the reattaching controller, which continues it without the failed close's suspend", async () => { const root = await mkdtemp(join(tmpdir(), "runnerd-remote-retention-adopt-")); const server = createServer(); let authority: DurablePrpControlPlane | null = null; @@ -172,9 +172,12 @@ it("keeps a retained remote Runner for the reattaching controller, which settles }, }); adopted.transport.setServerRequestHandler(async () => ({ success: true, contentItems: [] })); - await adopted.transport.request("thread/read", {}).catch(() => undefined); - await vi.waitFor(() => expect(processState(adoptedPid)).toBe("gone"), { timeout: 10_000, interval: 50 }); - expect(await journal()).toMatchObject({ lifecycle: "suspended", outbox: [], pendingTerminalDelivery: null }); + await expect(adopted.transport.request("thread/read", {})).resolves.toEqual( + expect.objectContaining({ thread: expect.objectContaining({ id: thread.id }) })); + expect(processState(adoptedPid)).toBe("alive"); + expect(await journal()).toMatchObject({ lifecycle: "ready", pendingTerminalDelivery: null }); + expect(adopted.evidence().diagnostics.join("\n")).toContain("before reattaching the live runner"); + expect(authority!.store.state.commands.some(command => command.type === "runner.suspend" && command.status === "pending")).toBe(false); expect(duplicateLaunch).not.toHaveBeenCalled(); expect(launch).toHaveBeenCalledTimes(1); } finally { diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts index 71a37b75020..20223a7f1bd 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts @@ -6822,6 +6822,12 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { }); this.#core = core; this.#followCommittedEvents(core); + if (this.options.adoptExistingRunner) { + const withdrawn = core.withdrawUndeliveredTerminalCommands(); + if (withdrawn.length > 0) { + this.#diagnostic(`withdrew ${withdrawn.join(", ")} queued by an earlier attempt before reattaching the live runner`); + } + } this.#carriedCommandCount = core.store.state.commands.length; // Reusing resources still requires current environment authorization. The // optional hook avoids re-preparation after controller storage is lost. @@ -7914,7 +7920,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { expectedEventType?: "harness.ready" | "session.started" | "session.resumed", deadline = this.#waitNow() + 30_000, ): Promise { - while (this.#waitNow() < deadline) { + for (;;) { this.#throwIfFailed(); if (this.#closed) throw new Error("runnerd transport closed during provider startup"); this.#pumpEvents(); @@ -7929,6 +7935,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { return; if (await this.#runnerHasExited()) throw new Error("runnerd exited before provider startup"); + if (this.#waitNow() >= deadline) break; await new Promise((resolveWait) => setTimeout(resolveWait, 10)); } throw new Error("runnerd did not report its provider identity"); @@ -7962,6 +7969,20 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { : this.#core?.getCommand(commandId), runnerHasExited: () => this.#runnerHasExited(), failureCode: () => this.#startupComplete ? "provider_transport_failed" : this.#startupFailureCode, + answeredProgressAt: () => { + const core = this.#core; + if (core === null) return null; + const commands = core.store.state.commands; + const target = commandId === undefined ? commands.find(candidate => candidate.type === type) : core.getCommand(commandId); + if (target === undefined) return null; + let progressAt: number | null = null; + for (const command of commands) { + if (command.controllerSeq > target.controllerSeq || command.status !== "pending") continue; + const at = core.queuedCommandResultProgressAt(command.commandId); + if (at !== null && (progressAt === null || at > progressAt)) progressAt = at; + } + return progressAt; + }, describeStall: () => this.#core === null ? null : describeRunnerCommandStall({ commands: this.#core.store.state.commands, commandId: commandId ?? this.#core.store.state.commands.find(candidate => candidate.type === type)?.commandId, @@ -9583,6 +9604,11 @@ export class RunnerStallLedger { return this.#creditedMs; } + credit(ms: number): void { + this.now(); + this.#creditedMs += Math.max(0, ms); + } + now(): number { const at = this.wallNow(); if (at - this.#last > RUNNER_STALL_THRESHOLD_MS) { @@ -9602,6 +9628,8 @@ export class RunnerStallLedger { } /** Shared polling path keeps startup cancellation separate from owned cleanup commands. */ +const RUNNER_ANSWERED_RESULT_STALL_MS = 30_000; + export async function waitForRunnerCommand(input: { type: string; deadline: number; @@ -9615,15 +9643,17 @@ export async function waitForRunnerCommand(input: { failureCode: () => string; /** Bounded, content-free evidence of what the command waited behind. */ describeStall?: () => string | null; + answeredProgressAt?: () => number | null; }): Promise { const now = () => input.clock ? input.clock.now() : Date.now(); - while (now() < input.deadline) { - input.throwIfFailed(); - if (input.abortOnClose && input.isClosed()) { - throw new Error(`runnerd transport closed while waiting for ${input.type}`); - } + const answeredAndAdvancing = () => { + const progressAt = input.answeredProgressAt?.() ?? null; + return progressAt !== null && Date.now() - progressAt < RUNNER_ANSWERED_RESULT_STALL_MS; + }; + let extended = false; + const settled = () => { const command = input.command(); - if (command?.status === "completed") return; + if (command?.status === "completed") return true; if (command !== undefined && command.status !== "pending") { const detail = record(record(command.result).result); const reportedResult = readProviderFailure(detail.providerFailure) @@ -9634,11 +9664,32 @@ export async function waitForRunnerCommand(input: { Object.defineProperty(error, "runnerCommandResult", { value: structuredClone(command.result), enumerable: false }); throw error; } + return false; + }; + const completeExtended = () => { + input.clock?.credit(Math.max(0, now() - input.deadline) + RUNNER_POST_STOP_GRACE_MS); + }; + let observed = false; + while (now() < input.deadline || answeredAndAdvancing()) { + observed = true; + if (now() >= input.deadline) extended = true; + input.throwIfFailed(); + if (input.abortOnClose && input.isClosed()) { + throw new Error(`runnerd transport closed while waiting for ${input.type}`); + } + if (settled()) { + if (extended) completeExtended(); + return; + } if (await input.runnerHasExited()) { throw new Error(`runnerd exited while waiting for ${input.type}`); } await new Promise((resolveWait) => setTimeout(resolveWait, 10)); } + if (observed && settled()) { + if (extended) completeExtended(); + return; + } let stall: string | null = null; try { stall = input.describeStall?.() ?? null; } catch { stall = null; } throw new Error(`${input.failureCode()}: PRP command ${input.type} timed out${stall ? ` (${stall})` : ""}`);