diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport-close-proof.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport-close-proof.test.ts new file mode 100644 index 00000000000..fccdbdcd46d --- /dev/null +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport-close-proof.test.ts @@ -0,0 +1,211 @@ +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { createServer } from "node:http"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, it, vi } from "vitest"; +import { NativeSessionCloseUnrecoverableError } from "../contracts/native-session-backend.js"; +import type { DurablePrpControlPlane } from "../control-plane/durable-prp-control-plane.js"; +import { codexSemanticToolSpecs } from "../drivers/codex/codex-app-server-driver.js"; +import { createCapabilityRunnerdCodexTransport } from "./runnerd-codex-transport.js"; +import { defaultCapabilityRunnerdBinary, fakeCodex, fakeCodexArgs } from "../../test/support/runnerd-transport.js"; + +const pause = (ms: number) => new Promise(resolveWait => setTimeout(resolveWait, ms)); + +function identityFor(label: string) { + return { + runnerInstanceId: `runner-${label}`, + environmentLeaseId: `lease-${label}`, + runId: `run-${label}`, + normalizedSessionId: `session-${label}`, + turnId: `turn-${label}`, + itemId: `item-${label}`, + }; +} + +function settlementOf(diagnostics: readonly string[]) { + const line = diagnostics.find(message => message.startsWith("native_session_settlement_incomplete ")); + return line ? JSON.parse(line.slice("native_session_settlement_incomplete ".length)) as Record : null; +} + +it("proves a completed suspend on its own read window when the durable state read outlasts the close grace", async () => { + const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-close-proof-slow-")); + const identity = identityFor("close-proof-slow"); + const diagnostics: string[] = []; + const readRunnerState = vi.fn(async () => { + await pause(9_000); + return { schema: "paperclip.runner.durable.state.v1", ...identity, lifecycle: "suspended" }; + }); + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs(stateDirectory), + stateDirectory, + closeGraceMs: 6_000, + lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, + prpIdentity: identity, + readRunnerState, + onDiagnostic: message => diagnostics.push(message), + }); + bundle.transport.setServerRequestHandler(async () => ({ success: true, contentItems: [] })); + try { + await bundle.transport.request("thread/start", { cwd: tmpdir(), dynamicTools: [] }); + await expect(bundle.transport.close("warm native session idle timeout")).resolves.toBeUndefined(); + expect(settlementOf(diagnostics)).toBeNull(); + expect(readRunnerState).toHaveBeenCalled(); + } finally { + await bundle.transport.close().catch(() => undefined); + await rm(stateDirectory, { recursive: true, force: true }); + } +}, 30_000); + +it.each([ + { title: "retires a predecessor whose earlier close could not read its completed suspension", later: "suspended", retired: true }, + { title: "keeps a predecessor unretired when its durable state still is not suspended", later: "ready", retired: false }, +] as const)("$title", async ({ later, retired }) => { + const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-close-proof-retry-")); + const identity = identityFor("close-proof-retry"); + const diagnostics: string[] = []; + let readable = false; + const readRunnerState = vi.fn(async () => { + if (!readable) throw new Error("remote runner state read failed"); + return { schema: "paperclip.runner.durable.state.v1", ...identity, lifecycle: later }; + }); + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs(stateDirectory), + stateDirectory, + closeGraceMs: 6_000, + closeProofReadMs: 500, + lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, + prpIdentity: identity, + readRunnerState, + onDiagnostic: message => diagnostics.push(message), + }); + bundle.transport.setServerRequestHandler(async () => ({ success: true, contentItems: [] })); + try { + await bundle.transport.request("thread/start", { cwd: tmpdir(), dynamicTools: [] }); + const failed = await bundle.transport.close("warm native session idle timeout").catch((error: unknown) => error); + expect(failed).toBeInstanceOf(NativeSessionCloseUnrecoverableError); + expect((failed as NativeSessionCloseUnrecoverableError).settlement).toMatchObject({ + runnerSuspended: false, + suspensionState: { commandStatus: "completed" }, + }); + readable = true; + const successor = bundle.transport.close("warm Runner held the sandbox Runner port for another session"); + if (retired) { + await expect(successor).resolves.toBeUndefined(); + expect(diagnostics.join("\n")).toContain("runner suspension proven from its durable state"); + await expect(bundle.transport.close()).resolves.toBeUndefined(); + } else { + const unproven = await successor.catch((error: unknown) => error); + expect(unproven).toBeInstanceOf(NativeSessionCloseUnrecoverableError); + expect((unproven as NativeSessionCloseUnrecoverableError).settlement).toMatchObject({ + runnerSuspended: false, + timings: expect.objectContaining({ suspendMs: expect.any(Number), lateProofReadMs: expect.any(Number), totalMs: expect.any(Number) }), + }); + } + } finally { + await bundle.transport.close().catch(() => undefined); + await rm(stateDirectory, { recursive: true, force: true }); + } +}, 30_000); + +async function adoptedIdleRunnerClosesWithSlowLivenessProbe(label: string, probeDelayMs: number) { + const stateDirectory = await mkdtemp(join(tmpdir(), `runnerd-close-proof-${label}-`)); + const server = createServer(); + let authority = null as DurablePrpControlPlane | null; + server.on("upgrade", (request, socket, head) => { + if (!authority) { + socket.destroy(); + return; + } + authority.handleUpgrade(request, socket, "/runner", head); + }); + await new Promise(resolveListen => server.listen(0, "127.0.0.1", resolveListen)); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("Expected close proof listener"); + const registration = async (next: DurablePrpControlPlane) => { + authority = next; + return { + connectUrl: `ws://127.0.0.1:${address.port}/runner`, + release: async () => { + if (authority === next) authority = null; + }, + }; + }; + const options = { + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs(stateDirectory), + stateDirectory, + prpIdentity: identityFor(label), + lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 60_000 }, + controlPlaneRegistration: registration, + }; + const first = createCapabilityRunnerdCodexTransport(options); + let adopted: ReturnType | null = null; + let runnerPid: number | null = null; + const diagnostics: string[] = []; + try { + const opened = await first.transport.request("thread/start", { cwd: tmpdir(), dynamicTools: codexSemanticToolSpecs() }); + runnerPid = first.evidence().runnerPid; + const startedAt = first.transport.processInfo!().startedAt; + await first.detachControllerForRestart(); + const controlPlaneStatePath = join(stateDirectory, "control-plane", "control-plane-state.json"); + const controlPlaneState = JSON.parse(await readFile(controlPlaneStatePath, "utf8")) as { committedEvents: Array<{ eventType: string }> }; + controlPlaneState.committedEvents = controlPlaneState.committedEvents.filter(event => + !["harness.ready", "session.started", "session.resumed"].includes(event.eventType)); + await writeFile(controlPlaneStatePath, `${JSON.stringify(controlPlaneState, null, 2)}\n`, { mode: 0o600 }); + const thread = opened.thread as Record; + let slowProbes = false; + const pid = runnerPid!; + adopted = createCapabilityRunnerdCodexTransport({ + ...options, + closeGraceMs: 6_000, + onDiagnostic: message => diagnostics.push(message), + resumeDynamicTools: [], + resumeProviderSession: { driverSessionId: String(thread.id), providerSessionId: String(thread.sessionId) }, + runnerProcessLauncher: () => { throw new Error("duplicate runner spawn attempted"); }, + adoptExistingRunner: { + pid, + processGroupId: pid, + startedAt, + signal: () => true, + isAlive: async () => { + if (slowProbes) await pause(probeDelayMs); + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } + }, + }, + }); + await adopted.transport.request("thread/read", {}); + slowProbes = true; + const closed = await adopted.transport.close("warm native session idle timeout", { retainUnsettledRemoteRunner: true }) + .then(() => null, (error: unknown) => error); + const state = JSON.parse(await readFile(join(stateDirectory, "runner", "runner-state.json"), "utf8")) as Record; + return { closed, lifecycle: state.lifecycle, settlement: settlementOf(diagnostics) }; + } finally { + await adopted?.transport.close().catch(() => undefined); + if (runnerPid) { + try { process.kill(runnerPid, "SIGKILL"); } catch { /* already gone */ } + } + server.closeAllConnections(); + if (server.listening) await new Promise(resolveClose => server.close(() => resolveClose())); + await rm(stateDirectory, { recursive: true, force: true }); + } +} + +it("suspends five adopted idle Runners whose liveness probes stall together, instead of spending the close on the probe", async () => { + const results = await Promise.all(Array.from({ length: 5 }, (_, index) => + adoptedIdleRunnerClosesWithSlowLivenessProbe(`adopted-${index}`, 10_000))); + for (const result of results) { + expect(result.closed).toBeNull(); + expect(result.lifecycle).toBe("suspended"); + expect(result.settlement).toBeNull(); + } +}, 120_000); diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts index 237a1874cce..b983446b565 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts @@ -2450,6 +2450,136 @@ it("waits for the exact durable suspension command behind prior close work", asy expect(pumpCount).toBeGreaterThanOrEqual(2); }); +it("gives a suspend that completes near the close deadline its whole proof window from the completion", async () => { + const commands: Array<{ commandId: string; type: string; status: string }> = []; + const startedAt = Date.now(); + const closeDeadline = startedAt + 300; + let completedAt = 0; + const result = await runnerdRecoveryInternals.awaitRunnerSuspensionBarrier({ + commands: () => commands, + queueSuspend: commandId => { commands.push({ commandId, type: "runner.suspend", status: "pending" }); }, + readRunnerState: async () => { + await new Promise(resolve => setTimeout(resolve, 200)); + return { lifecycle: "suspended" }; + }, + runnerHasExited: async () => true, + pump: () => { + if (Date.now() - startedAt >= 250 && commands[0]?.status === "pending") { + commands[0].status = "completed"; + completedAt = Date.now(); + } + }, + deadline: closeDeadline, + proofWindowMs: 400, + pollIntervalMs: 10, + }); + expect(result).toBe(true); + expect(completedAt).toBeGreaterThan(0); + expect(Date.now()).toBeGreaterThan(closeDeadline); + expect(Date.now()).toBeLessThanOrEqual(completedAt + 400 + 50); +}); + +async function suspendProofAfterCompletion(input: { lifecycleAt: (sinceCompletionMs: number) => string; runId?: string; + deadlineMs: number; proofWindowMs: number; proofSettleMs: number }) { + const commands: Array<{ commandId: string; type: string; status: string }> = []; + const startedAt = Date.now(); + let completedAt: number | null = null; + const proven = await runnerdRecoveryInternals.awaitRunnerSuspensionBarrier({ + commands: () => commands, + queueSuspend: commandId => { commands.push({ commandId, type: "runner.suspend", status: "pending" }); }, + readRunnerState: async () => { + const lifecycle = input.lifecycleAt(completedAt === null ? 0 : Date.now() - completedAt); + const refutation = input.runId ? "identity" : lifecycle === "suspended" ? null : "lifecycle"; + if (refutation) throw new runnerdRecoveryInternals.RunnerSuspensionProofRefutedError(refutation); + return { lifecycle }; + }, + runnerHasExited: async () => true, + pump: () => { + if (commands[0] && completedAt === null) { + commands[0].status = "completed"; + completedAt = Date.now(); + } + }, + deadline: startedAt + input.deadlineMs, + proofWindowMs: input.proofWindowMs, + proofSettleMs: input.proofSettleMs, + pollIntervalMs: 20, + }); + return { proven, elapsedMs: Date.now() - startedAt }; +} + +it("proves a suspension whose suspended state appears 1.5 s after the command completed", async () => { + const { proven } = await suspendProofAfterCompletion({ lifecycleAt: since => since >= 1_500 ? "suspended" : "ready", + deadlineMs: 200, proofWindowMs: 30_000, proofSettleMs: 3_000 }); + expect(proven).toBe(true); +}); + +it("fails a suspension whose state stays ready after the settle window, not after the whole proof window", async () => { + const { proven, elapsedMs } = await suspendProofAfterCompletion({ lifecycleAt: () => "ready", + deadlineMs: 200, proofWindowMs: 30_000, proofSettleMs: 1_000 }); + expect(proven).toBe(false); + expect(elapsedMs).toBeGreaterThanOrEqual(900); + expect(elapsedMs).toBeLessThan(3_000); +}); + +it("ignores a not-suspended read that started before the suspend completed, even when it answers after the proof window opened", async () => { + const commands: Array<{ commandId: string; type: string; status: string }> = []; + let completedAt: number | null = null; + let reads = 0; + const refutations: Array = []; + const proven = await runnerdRecoveryInternals.awaitRunnerSuspensionBarrier({ + commands: () => commands, + queueSuspend: commandId => { commands.push({ commandId, type: "runner.suspend", status: "pending" }); }, + readRunnerState: async () => { + reads += 1; + if (reads === 1) { + await new Promise(resolve => setTimeout(resolve, 150)); + throw new runnerdRecoveryInternals.RunnerSuspensionProofRefutedError("lifecycle"); + } + if (Date.now() - completedAt! < 1_500) throw new Error("remote runner state read failed"); + return { lifecycle: "suspended" }; + }, + runnerHasExited: async () => true, + pump: () => { + if (commands[0] && completedAt === null && reads >= 1) { + commands[0].status = "completed"; + completedAt = Date.now(); + } + }, + deadline: Date.now() + 100, + proofWindowMs: 30_000, + proofSettleMs: 500, + onProofRefutation: refutation => refutations.push(refutation), + pollIntervalMs: 20, + }); + expect(proven).toBe(true); + expect(refutations).not.toContain("lifecycle"); +}); + +it("fails at the close deadline when the durable state belongs to another run", async () => { + const { proven, elapsedMs } = await suspendProofAfterCompletion({ lifecycleAt: () => "suspended", runId: "another-run", + deadlineMs: 200, proofWindowMs: 30_000, proofSettleMs: 3_000 }); + expect(proven).toBe(false); + expect(elapsedMs).toBeLessThan(1_000); +}); + +it("bounds the proof window from the observed completion, so a proof read that never answers still ends", async () => { + const commands: Array<{ commandId: string; type: string; status: string }> = []; + const startedAt = Date.now(); + const result = await runnerdRecoveryInternals.awaitRunnerSuspensionBarrier({ + commands: () => commands, + queueSuspend: commandId => { commands.push({ commandId, type: "runner.suspend", status: "pending" }); }, + readRunnerState: () => new Promise(() => undefined), + runnerHasExited: async () => true, + pump: () => { if (commands[0]) commands[0].status = "completed"; }, + deadline: startedAt + 100, + proofWindowMs: 200, + pollIntervalMs: 10, + }); + expect(result).toBe(false); + expect(Date.now() - startedAt).toBeLessThan(100 + 200 + 150); +}); + it("retries a rejected suspension within the close deadline instead of failing a settled session", async () => { // An ACPX sidecar can reject session.suspend while the just-completed turn // is still settling. One rejected command must not leave a successful run's diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts index 8e47047e03f..0fbecce161f 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts @@ -1085,28 +1085,59 @@ async function awaitRunnerSuspensionBarrier(input: { /** Called once the Runner's exit is certain: its suspend command completed. */ onExitCommitted?: () => void; deadline: number; + proofWindowMs?: number; + proofSettleMs?: number; + onProofRefutation?: (refutation: RunnerSuspensionProofRefutation | null) => void; pollIntervalMs?: number; }): Promise { - if (Date.now() >= input.deadline) return false; - let exitCommitted = false; + let proofDeadline: number | null = null; + let notSuspendedSince: number | null = null; + let refutation: RunnerSuspensionProofRefutation | null = null; + let readGeneration = 0; + const deadline = () => { + if (proofDeadline === null) return input.deadline; + if (refutation === "identity") return input.deadline; + if (refutation === "lifecycle") { + notSuspendedSince ??= Date.now(); + return Math.max(input.deadline, Math.min(proofDeadline, notSuspendedSince + (input.proofSettleMs ?? 0))); + } + return Math.max(input.deadline, proofDeadline); + }; + if (Date.now() >= deadline()) return false; const commitExit = () => { - if (exitCommitted) return; - exitCommitted = true; + if (proofDeadline !== null) return; + proofDeadline = Date.now() + (input.proofWindowMs ?? 0); input.onExitCommitted?.(); }; const readRunnerState = async (): Promise | undefined> => { - const remainingMs = input.deadline - Date.now(); + const remainingMs = deadline() - Date.now(); if (remainingMs <= 0) return undefined; + const generation = ++readGeneration; + const startedAfterCompletion = proofDeadline !== null; + let expired = false; + const observe = (observed: RunnerSuspensionProofRefutation | null) => { + if (expired || !startedAfterCompletion || generation !== readGeneration) return; + refutation = observed; + input.onProofRefutation?.(observed); + }; + const read = input.readRunnerState().then( + (state) => { observe(null); return state; }, + (error: unknown) => { + if (error instanceof RunnerSuspensionProofRefutedError) observe(error.refutation); + throw error; + }, + ); let timer: ReturnType | undefined; try { return await Promise.race([ - input.readRunnerState(), + read, new Promise(resolveDeadline => { - timer = setTimeout(() => resolveDeadline(undefined), remainingMs); + timer = setTimeout(() => { expired = true; resolveDeadline(undefined); }, remainingMs); }), ]); } finally { clearTimeout(timer); + read.catch(() => undefined); } }; let existing = [...input.commands()] @@ -1125,25 +1156,27 @@ async function awaitRunnerSuspensionBarrier(input: { if (state === undefined) return false; if (state.lifecycle === "suspended") { commitExit(); - return Date.now() < input.deadline; + return Date.now() < deadline(); } } catch { // The normal bounded barrier below handles unavailable state. } existing = undefined; } - if (Date.now() >= input.deadline) return false; + if (Date.now() >= deadline()) return false; let commandId = existing?.commandId ?? `command_close_suspend_${randomUUID().replaceAll("-", "")}`; if (!existing) input.queueSuspend(commandId); let suspendAttempts = 1; - while (Date.now() < input.deadline) { + for (;;) { input.pump(); const command = input .commands() .find((candidate) => candidate.commandId === commandId); + if (command?.status === "completed") commitExit(); + if (Date.now() >= deadline()) break; if ( command !== undefined && command.status !== "pending" && @@ -1163,7 +1196,7 @@ async function awaitRunnerSuspensionBarrier(input: { await new Promise((resolveWait) => setTimeout(resolveWait, input.pollIntervalMs ?? RUNNER_SUSPEND_RETRY_DELAY_MS), ); - if (Date.now() >= input.deadline) return false; + if (Date.now() >= deadline()) return false; suspendAttempts += 1; commandId = `command_close_suspend_${randomUUID().replaceAll("-", "")}`; input.queueSuspend(commandId); @@ -1175,17 +1208,24 @@ async function awaitRunnerSuspensionBarrier(input: { let lifecycle: unknown; try { const state = await readRunnerState(); - if (state === undefined) return false; + if (state === undefined) { + input.pump(); + if (input.commands().find((candidate) => candidate.commandId === commandId)?.status === "completed") commitExit(); + if (Date.now() >= deadline()) return false; + continue; + } lifecycle = state.lifecycle; } catch { // A remote filesystem can lag the command-result delivery by a small // amount. Keep the single close deadline as the fail-closed bound. } - if (Date.now() >= input.deadline) return false; + input.pump(); + if (input.commands().find((candidate) => candidate.commandId === commandId)?.status === "completed") commitExit(); + if (Date.now() >= deadline()) return false; if ( command?.status === "completed" && lifecycle === "suspended" && - Date.now() < input.deadline + Date.now() < deadline() ) { return true; } @@ -1200,6 +1240,19 @@ async function awaitRunnerSuspensionBarrier(input: { return false; } +export type RunnerSuspensionProofRefutation = "identity" | "lifecycle"; + +export class RunnerSuspensionProofRefutedError extends Error { + constructor(readonly refutation: RunnerSuspensionProofRefutation) { + super("native_runner_authority_rotation_requires_settled_state"); + } +} + +function runnerSuspensionProofRefutation(state: Record, expected: DurableRecoveryIdentity): RunnerSuspensionProofRefutation | null { + if (state.schema !== "paperclip.runner.durable.state.v1" || !recoveryIdentityMatches(state, expected)) return "identity"; + return state.lifecycle === "suspended" ? null : "lifecycle"; +} + function runnerCloseGraceMs(options: Pick): number { // Native Pi retirement allows thirty seconds for snapshot cleanup and five // for its lifetime fence. Keep a drain round trip and suspension reserve too. @@ -1208,6 +1261,39 @@ function runnerCloseGraceMs(options: Pick; + +async function closeStepWithin( + timings: CloseTimings, + step: CloseTimingStep, + operation: () => Promise, + deadline: number, +): Promise<{ answered: true; value: T } | { answered: false }> { + const startedAt = Date.now(); + let timer: ReturnType | undefined; + const answer = operation().then(value => ({ answered: true as const, value })); + answer.catch(() => undefined); + try { + const outcome = await Promise.race([ + answer, + new Promise<{ answered: false }>(resolveDeadline => { + timer = setTimeout(() => resolveDeadline({ answered: false }), Math.max(0, deadline - Date.now())); + }), + ]); + if (!outcome.answered) timings[`${step}Unanswered`] = (timings[`${step}Unanswered`] ?? 0) + 1; + return outcome; + } finally { + clearTimeout(timer); + timings[`${step}Ms`] = (timings[`${step}Ms`] ?? 0) + Date.now() - startedAt; + } +} + function runnerProcessReconnectGraceMs(options: CapabilityRunnerdCodexTransportOptions): number | undefined { if (options.runnerProcessReconnectGraceMs === null) return undefined; return options.runnerProcessReconnectGraceMs ?? options.runnerReconnectGraceMs; @@ -1668,6 +1754,8 @@ export interface CapabilityRunnerdCodexTransportOptions extends RunnerExecutionD /** Provider system instructions supplied by a native execution caller. */ baseInstructions?: string; closeGraceMs?: number; + closeProofReadMs?: number; + closeProofSettleMs?: number; /** Bounded provider turn-admission wait; defaults to 30 seconds. */ turnStartTimeoutMs?: number; turnStartProgressCeilingMs?: number; @@ -4118,6 +4206,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { readonly #closedSignal: Promise; #resolveClosedSignal!: () => void; #closePromise: Promise | null = null; + #closeTimings: CloseTimings = {}; #retainUnsettledRemoteRunner = false; #pendingCloseCheckpointRetry: CloseCheckpointRetry | null = null; #failedCloseCheckpoint: CloseCheckpointRetry | null = null; @@ -5601,10 +5690,17 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { } } + async #runnerExitObservedWithin(deadline: number, step: CloseTimingStep): Promise { + const observed = await closeStepWithin(this.#closeTimings, step, () => this.#runnerHasExited(), deadline); + return observed.answered && observed.value === true; + } + async #closeAndSettle(): Promise { this.#closed = true; this.#resolveClosedSignal(); - const { preparationDeadline, closeDeadline } = runnerCloseDeadlines(Date.now(), runnerCloseGraceMs(this.options)); + const closeStartedAt = Date.now(); + const timings = this.#closeTimings; + const { preparationDeadline, closeDeadline } = runnerCloseDeadlines(closeStartedAt, runnerCloseGraceMs(this.options)); let preparationConfirmed = true; const attachmentPending = this.#attachmentOperation !== null; if (this.#startupOperation !== null || attachmentPending) { @@ -5650,6 +5746,9 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { && this.options.recoveryPurpose !== "stop_cleanup" && (this.options.runnerProcessLauncher !== undefined || adoptedRunner !== undefined); let lastSuspensionState: Record | null = null; + let suspensionExitCommitted = false; + let proofWindowEnd: string | null = null; + let suspensionProofRefutation: RunnerSuspensionProofRefutation | null = null; let suspensionState: { commandStatus: string | null; runnerLifecycle: string | null; @@ -5665,7 +5764,10 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { // its durable provider suffix is ACKed. Drain it before suspension so a // fresh run authority never inherits the prior run's pending events. - if (!(await this.#runnerHasExited())) { + const exitProbeDeadline = this.#core.activeRunnerConnectionCount() > 0 + ? Math.min(preparationDeadline, Date.now() + RUNNER_CONNECTED_EXIT_PROBE_SLICE_MS) : preparationDeadline; + if (!(await this.#runnerExitObservedWithin(exitProbeDeadline, "exitProbe"))) { + const stopStartedAt = Date.now(); if (this.options.recoveryPurpose === "stop_cleanup") { cleanupProviderStopConfirmed = await this.#stopProviderBeforeSuspend(preparationDeadline); } @@ -5688,10 +5790,13 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { if (this.options.recoveryPurpose !== "stop_cleanup") { await this.#stopProviderBeforeSuspend(preparationDeadline); } + timings.stopMs = Date.now() - stopStartedAt; + const drainStartedAt = Date.now(); providerDrained = await this.#drainSettledProviderEventsBeforeSuspend( Math.min(5_000, Math.max(0, preparationDeadline - Date.now())), cleanupProviderStopConfirmed, ); + timings.drainMs = Date.now() - drainStartedAt; } // Local durable roots are reused too. Process exit alone cannot prove // their authority is safe to rotate; require the same exact suspension @@ -5707,26 +5812,45 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { // still reading durable state, and a Core-dialed route would otherwise // redial it. The release path quiesces its route the same way. const stopObservingExit = this.#core.onRunnerExitCommitted(() => this.#quiesceControlPlaneRoutes()); + const suspendStartedAt = Date.now(); try { runnerSuspended = !lostLocalRunner && await awaitRunnerSuspensionBarrier({ commands: () => this.#core?.store.state.commands ?? [], queueSuspend: (commandId) => { + timings.suspendQueuedAfterMs ??= Date.now() - closeStartedAt; this.#core?.queueCommand("runner.suspend", {}, commandId, true); }, readRunnerState: async () => { - const state = await this.#readDurableRunnerState(); - lastSuspensionState = state; - assertSuspendedRunnerState(state, this.#core!.store.state.identity); - return state; + const readStartedAt = Date.now(); + try { + const state = await this.#readDurableRunnerState(); + lastSuspensionState = state; + const refuted = runnerSuspensionProofRefutation(state, this.#core!.store.state.identity); + if (refuted) throw new RunnerSuspensionProofRefutedError(refuted); + return state; + } finally { + timings.proofReadMs = (timings.proofReadMs ?? 0) + Date.now() - readStartedAt; + } }, - runnerHasExited: () => this.#runnerHasExited(), + runnerHasExited: () => this.#runnerExitObservedWithin( + Math.min(closeDeadline, Date.now() + RUNNER_CLOSE_EXIT_PROBE_SLICE_MS), "barrierExitProbe"), pump: () => this.#pumpEventsSafely(), - // Also covers a completed receipt acknowledged before this close. - onExitCommitted: () => this.#quiesceControlPlaneRoutes(), + onExitCommitted: () => { + suspensionExitCommitted = true; + suspensionProofRefutation = null; + this.#quiesceControlPlaneRoutes(); + }, deadline: closeDeadline, + proofWindowMs: this.options.closeProofReadMs ?? RUNNER_SUSPENSION_PROOF_READ_MS, + proofSettleMs: this.options.closeProofSettleMs ?? RUNNER_SUSPENSION_PROOF_SETTLE_MS, + onProofRefutation: (observed) => { suspensionProofRefutation = observed; }, }); } finally { stopObservingExit(); + timings.suspendMs = Date.now() - suspendStartedAt; + proofWindowEnd = runnerSuspended ? "proven" : !suspensionExitCommitted ? "suspend_not_completed" + : suspensionProofRefutation === "identity" ? "wrong_identity" + : suspensionProofRefutation === "lifecycle" ? "not_suspended_after_settle" : "unreadable_after_window"; } runnerLost = !runnerSuspended && this.#handle !== null && this.#lostRunnerHandle === this.#handle; if (!runnerSuspended) { @@ -5748,7 +5872,8 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { this.#diagnostic( "runner did not prove durable suspension before checkpoint", ); - runnerRetained = remoteRunnerRetainable && !runnerLost && !(await this.#runnerHasExited()); + runnerRetained = remoteRunnerRetainable && !runnerLost + && !(await this.#runnerExitObservedWithin(Date.now() + RUNNER_CLOSE_EXIT_PROBE_SLICE_MS, "retainProbe")); if (runnerRetained) this.#diagnostic("remote runner left alive unsettled for same-run reattachment"); } try { @@ -5765,16 +5890,22 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { if (result.stderr.trim()) this.#diagnostic(result.stderr.trim().slice(-4_096)); } else if (adoptedRunner) { + const adoptedAlive = async () => { + const probe = await closeStepWithin(timings, "adoptedExitProbe", async () => adoptedRunner.isAlive(), + Math.max(closeDeadline, Date.now() + RUNNER_CLOSE_EXIT_PROBE_SLICE_MS)); + return !probe.answered || probe.value; + }; while ( - (await adoptedRunner.isAlive()) && + (await adoptedAlive()) && Date.now() < closeDeadline ) { await new Promise((resolveWait) => setTimeout(resolveWait, 25)); } - if (await adoptedRunner.isAlive()) { - await adoptedRunner.signal?.("SIGKILL"); + if (await adoptedAlive()) { + await closeStepWithin(timings, "adoptedExitProbe", async () => adoptedRunner.signal?.("SIGKILL"), + Date.now() + RUNNER_CLOSE_EXIT_PROBE_SLICE_MS); } - this.#evidence.runnerExited = !(await adoptedRunner.isAlive()); + this.#evidence.runnerExited = !(await adoptedAlive()); } } catch (error) { this.#diagnostic(`runner shutdown failed: ${String(error)}`); @@ -5818,14 +5949,32 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { // independently verified durable state before releasing the process owner; // an incomplete or identity-conflicting state remains fail-closed. const finalProviderState = this.#providerDrainState(); - const runnerSettled = - preparationConfirmed && runnerSuspended && + const settledApartFromSuspension = () => + preparationConfirmed && providerDrained && (this.#core?.semanticToolResultsSettled() === true || cleanupProviderStopConfirmed) && (finalProviderState === null || (finalProviderState !== "unreadable" && finalProviderState.pendingEventCount === 0 && finalProviderState.providerSettled)); + let runnerSettled = runnerSuspended && settledApartFromSuspension(); + const closeIdentity = this.#core ? structuredClone(this.#core.store.state.identity) : null; + const lateSuspensionProvable = () => suspensionRequired && !runnerSuspended && !runnerLost && suspensionProofRefutation === null + && suspensionState?.commandStatus === "completed" && closeIdentity !== null && settledApartFromSuspension(); + const proveSuspensionLate = async () => { + if (!lateSuspensionProvable()) return; + const proof = await closeStepWithin(timings, "lateProofRead", async () => { + const state = await this.#readDurableRunnerState(); + assertSuspendedRunnerState(state, closeIdentity!); + return state; + }, Date.now() + (this.options.closeProofReadMs ?? RUNNER_SUSPENSION_PROOF_READ_MS)).catch(() => ({ answered: false as const })); + if (!proof.answered) return; + runnerSuspended = true; + runnerRetained = false; + runnerSettled = settledApartFromSuspension(); + suspensionState = null; + this.#diagnostic("runner suspension proven from its durable state after an earlier close could not read it"); + }; const finishClose = async (finishDeadline: number) => { // A lost local runnerd cannot suspend, and its provider and tools were // re-parented when it died. When runnerd's own lineage journal proves @@ -5843,6 +5992,8 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { suspensionState, semanticTools: this.#core?.semanticToolSettlementDiagnostics(), finalProviderState, + timings: { ...timings, totalMs: Date.now() - closeStartedAt }, + proofWindowEnd, }; try { this.options.onDiagnostic?.(`native_session_settlement_incomplete ${JSON.stringify(settlement)}`); @@ -5879,11 +6030,14 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { // that never authenticated to this controller (for example after an // executable upgrade). Retain its prior checkpoint without rewriting it. checkpoint: checkpoint && (async (settlement) => { + const checkpointStartedAt = Date.now(); try { await checkpoint(settlement); } catch (error) { checkpointFailed = true; throw error; + } finally { + timings.checkpointMs = Date.now() - checkpointStartedAt; } }), forceKill: async () => { @@ -5906,32 +6060,42 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { this.#evidence.runnerSignal = result.signal as NodeJS.Signals | null; }, release: release && (async () => { + const releaseStartedAt = Date.now(); try { await release(); } catch (error) { releaseFailed = true; throw error; + } finally { + timings.releaseMs = Date.now() - releaseStartedAt; } }), retireController: async () => { + const retireStartedAt = Date.now(); try { await this.#stopAndDrainAuthority(closeDeadline); } catch (error) { retirementFailed = true; throw error; + } finally { + timings.retireMs = Date.now() - retireStartedAt; } }, }); } catch (error) { if (checkpoint && checkpointFailed && !releaseFailed && !processStopFailed) { this.#pendingCloseCheckpointRetry = { - checkpoint: () => checkpoint(runnerSettled ? "settled" : "unsettled"), + checkpoint: async () => { + await proveSuspensionLate(); + await checkpoint(runnerSettled ? "settled" : "unsettled"); + }, finish: finishClose, }; } else if (retirementFailed && !releaseFailed && !processStopFailed) { this.#pendingCloseCheckpointRetry = { checkpoint: async () => { await this.#stopAndDrainAuthority(Date.now() + runnerCloseGraceMs(this.options)); + await proveSuspensionLate(); await checkpoint?.(runnerSettled ? "settled" : "unsettled"); }, finish: finishClose, @@ -5943,7 +6107,21 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { this.#controlPlaneCheckpoint = null; this.#controlPlaneRelease = null; } - await finishClose(closeDeadline); + try { + await finishClose(closeDeadline); + } catch (error) { + if (error instanceof NativeSessionCloseUnrecoverableError && !releaseFailed && !processStopFailed && lateSuspensionProvable()) { + this.#pendingCloseCheckpointRetry = { + checkpoint: async () => { + await proveSuspensionLate(); + if (!runnerSuspended) await finishClose(Date.now()); + await checkpoint?.("settled"); + }, + finish: finishClose, + }; + } + throw error; + } } async #prepareOwnedProvider( @@ -9597,6 +9775,7 @@ export const runnerdRecoveryInternals = Object.freeze({ awaitProviderDrainBarrier, awaitAdoptedRunnerAuthentication, awaitRunnerSuspensionBarrier, + RunnerSuspensionProofRefutedError, providerDrainStateFromSnapshot, providerTurnIsActiveFromCommittedEvents, recoveredRunAttachment, diff --git a/server/src/__tests__/heartbeat-native-predispatch-retry.test.ts b/server/src/__tests__/heartbeat-native-predispatch-retry.test.ts index 5c715bd3a43..a99d4ca48c1 100644 --- a/server/src/__tests__/heartbeat-native-predispatch-retry.test.ts +++ b/server/src/__tests__/heartbeat-native-predispatch-retry.test.ts @@ -3,7 +3,7 @@ import { existsSync, mkdirSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { and, eq, inArray } from "drizzle-orm"; import { activityLog, agents, authUsers, companyMemberships, createDb, environmentLeases, environments, heartbeatRuns, issueComments, - issueRecoveryActions, issues } from "@paperclipai/db"; + issueRecoveryActions, issueRelations, issues } from "@paperclipai/db"; import { NativePredecessorRetirementUnprovenError } from "../services/native-runtime/predecessor-retirement.js"; import { NativeProviderArtifactTransferTimeoutError } from "../services/native-runtime/native-provider-artifacts.js"; import { taskInterruptionPresentation } from "../services/task-interruption-presentation.js"; @@ -44,7 +44,7 @@ type AdmissionFailure = { remaining: number; turnNeverIssued: boolean; environme sessionRoots: string[] }; function nativeBackend(db: ReturnType, options: { failDescriptorOnce?: { failed: boolean }; - unretiredPredecessorOnce?: { failed: boolean; environmentId: string }; turnMessages?: Map; + unretiredPredecessorOnce?: { failed: boolean; environmentId: string; taskBlocked?: boolean }; turnMessages?: Map; admissionFailure?: AdmissionFailure } = {}): NonNullable { return execution => { @@ -92,6 +92,7 @@ function nativeBackend(db: ReturnType, options: { failDescripto // the previous run's warm Runner could not be retired before any // provider work. The executor reports that after its coordinator claim. unretired.failed = true; + if (unretired.taskBlocked) await db.update(issues).set({ status: "blocked" }).where(eq(issues.id, execution.binding.issueId)); const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, execution.binding.runId)); await db.insert(environmentLeases).values({ companyId: run!.companyId, environmentId: unretired.environmentId, issueId: execution.binding.issueId, heartbeatRunId: run!.id, provider: "daytona", providerLeaseId: "sandbox-reused", @@ -136,7 +137,8 @@ function nativeBackend(db: ReturnType, options: { failDescripto describe.skipIf(!support.supported)("heartbeat native pre-dispatch failure retry", () => { const fixture = nativeRestartFixture({}); - async function startTask(db: ReturnType, heartbeat: ReturnType) { + async function startTask(db: ReturnType, heartbeat: ReturnType, + status: "in_progress" | "blocked" = "in_progress") { const { companyId, agentId } = fixture.current; const userId = randomUUID(); await db.insert(authUsers).values({ id: userId, name: "Task owner", email: `${userId}@example.invalid`, @@ -144,9 +146,13 @@ describe.skipIf(!support.supported)("heartbeat native pre-dispatch failure retry await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: userId, membershipRole: "owner" }); await db.update(agents).set({ status: "idle", adapterConfig: { provider: "codex" } }).where(eq(agents.id, agentId)); await instanceSettingsService(db).updateExperimental({ enableAgentChat: true, enableNativeRunner: true }); - const [issue] = await db.insert(issues).values({ companyId, title: "Survive a database fault", status: "in_progress", + const [issue] = await db.insert(issues).values({ companyId, title: "Survive a database fault", status, assigneeAgentId: agentId, responsibleUserId: userId, conversationAgentId: agentId, conversationUserId: userId, conversationState: "active" }).returning(); + if (status === "blocked") { + const [blocker] = await db.insert(issues).values({ companyId, title: "Open subtask", status: "todo" }).returning(); + await db.insert(issueRelations).values({ companyId, issueId: blocker!.id, relatedIssueId: issue!.id, type: "blocks" }); + } const [message] = await db.insert(issueComments).values({ companyId, issueId: issue!.id, authorType: "user", authorUserId: userId, body: "Start." }).returning(); const run = await heartbeat.wakeup(agentId, { source: "on_demand", triggerDetail: "manual", reason: "issue_commented", @@ -236,6 +242,50 @@ describe.skipIf(!support.supported)("heartbeat native pre-dispatch failure retry } }, 90_000); + it.each([ + { title: "continues a follow-up on a blocked task whose predecessor could not be retired", blockedAfter: false }, + { title: "does not continue a task that was blocked only after its run failed", blockedAfter: true }, + ])("$title", async ({ blockedAfter }) => { + const { companyId } = fixture.current; + const db = createDb(fixture.current.temporary.connectionString, { maxConnections: 4 }); + const [environment] = await db.insert(environments).values({ name: `reused-${randomUUID()}`, driver: "sandbox" }).returning(); + const heartbeat = heartbeatService(db, { nativeSessionBackendFactory: nativeBackend(db, + { unretiredPredecessorOnce: { failed: false, environmentId: environment!.id, taskBlocked: !blockedAfter } }) }); + try { + const { issueId, runId } = await startTask(db, heartbeat, blockedAfter ? "in_progress" : "blocked"); + const [failed] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + expect(failed).toMatchObject({ status: "failed", resultJson: expect.objectContaining({ + failureCause: "native_predecessor_retirement_unproven", + predecessorRetirementFallback: expect.objectContaining({ issueStatusAtFailure: blockedAfter ? "in_progress" : "blocked" }) }) }); + if (blockedAfter) await db.update(issues).set({ status: "blocked" }).where(eq(issues.id, issueId)); + const [task] = await db.select().from(issues).where(eq(issues.id, issueId)); + expect(task?.status).toBe("blocked"); + const successors = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, companyId), + eq(heartbeatRuns.retryOfRunId, runId))); + expect(successors).toEqual([expect.objectContaining({ status: "scheduled_retry", scheduledRetryReason: "interrupted_preparation" })]); + expect(await taskInterruptionPresentation(db, task!)).toMatchObject({ state: "working", canRetry: false }); + if (blockedAfter) { + await heartbeat.promoteDueScheduledRetries(); + await heartbeat.resumeQueuedRuns(); + await waitForCondition("the stale retry to settle", () => heartbeat.getTaskDrainStatus().quiescent); + const [stale] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, successors[0]!.id)); + expect(["cancelled", "scheduled_retry"]).toContain(stale?.status); + return; + } + expect(await heartbeat.promoteDueScheduledRetries()).toMatchObject({ promoted: 1 }); + await heartbeat.resumeQueuedRuns(); + await waitForCondition("the retry to settle", () => heartbeat.getTaskDrainStatus().quiescent); + const [retried] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, successors[0]!.id)); + expect({ status: retried?.status, errorCode: retried?.errorCode, error: retried?.error }).toEqual({ status: "succeeded", errorCode: null, error: null }); + } finally { + await heartbeat.drainActiveRunExecutions(); + await closeIdleWarmNativeSessionsForRestart(); + await db.update(heartbeatRuns).set({ status: "cancelled", finishedAt: new Date() }).where(and( + eq(heartbeatRuns.companyId, companyId), inArray(heartbeatRuns.status, ["queued", "scheduled_retry"]))); + await db.$client.end(); + } + }, 90_000); + it("delivers the monitor check to the automatic retry of a monitor run that failed before provider work", async () => { const { companyId, agentId } = fixture.current; const db = createDb(fixture.current.temporary.connectionString, { maxConnections: 4 }); diff --git a/server/src/modules/run-dispatch/adapters/postgres.ts b/server/src/modules/run-dispatch/adapters/postgres.ts index 96988c8a8a1..e526d078ded 100644 --- a/server/src/modules/run-dispatch/adapters/postgres.ts +++ b/server/src/modules/run-dispatch/adapters/postgres.ts @@ -56,6 +56,7 @@ import { appendHeartbeatRunEvent } from "../../../services/heartbeat-run-events. import { emitAgentTaskRun } from "../../../services/agent-task-run-telemetry.js"; import { issueService } from "../../../services/issues.js"; import { verifyBlockerStallWake } from "../../../services/recovery/blocker-stall-wakeups.js"; +import { readPredecessorRetirementFallback } from "../../../services/native-runtime/predecessor-retirement.js"; import { issueTreeControlService, isVerifiedIssueTreeControlInteractionWake, @@ -162,6 +163,19 @@ function readNonEmptyString(value: unknown): string | null { return typeof value === "string" && value.trim().length > 0 ? value : null; } +async function continuesNoWorkRunOnBlockedTask(dbOrTx: Db, + input: { companyId: string; runId: string }): Promise { + const [run] = await dbOrTx.select({ resultJson: heartbeatRuns.resultJson, retryOfRunId: heartbeatRuns.retryOfRunId }) + .from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.id, input.runId))); + let fallback = readPredecessorRetirementFallback(run?.resultJson); + if (!fallback && run?.retryOfRunId) { + const [source] = await dbOrTx.select({ resultJson: heartbeatRuns.resultJson }).from(heartbeatRuns) + .where(and(eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.id, run.retryOfRunId))); + fallback = readPredecessorRetirementFallback(source?.resultJson); + } + return fallback?.issueStatusAtFailure === "blocked"; +} + function classifyRetryReasonKind(retryReason: string | null): RetryReasonKind { if (retryReason === MAX_TURN_CONTINUATION_RETRY_REASON) return "max_turn_continuation"; if (retryReason === ISSUE_DISPOSITION_REPAIR_RETRY_REASON) return "disposition_repair"; @@ -431,6 +445,8 @@ export function createPostgresRunDispatchAdapter( if (!facts.issueFound) return { agentFound: true, facts }; facts.issueStatus = issue.status; + facts.continuesNoWorkRunOnBlockedTask = retryReasonKind === "native_safe_replacement" && issue.status === "blocked" + && await continuesNoWorkRunOnBlockedTask(dbOrTx as unknown as Db, { companyId: input.companyId, runId: input.runId }); facts.issueAssigneeAgentId = issue.assigneeAgentId; facts.issueExecutionRunId = issue.executionRunId; facts.issueCheckoutRunId = issue.checkoutRunId; @@ -514,6 +530,7 @@ export function createPostgresRunDispatchAdapter( && typeof (preparationWait as Record).kind === "string"); facts.dependenciesBlocked = readiness && !readiness.isDependencyReady && !stallAdmitted && !responseAdmitted && !preparationWaitAdmitted + && !facts.continuesNoWorkRunOnBlockedTask ? { unresolvedBlockerIssueIds: readiness.unresolvedBlockerIssueIds, unresolvedBlockerCount: readiness.unresolvedBlockerCount, @@ -686,12 +703,15 @@ export function createPostgresRunDispatchAdapter( && await verifyBlockerStallWake(dbOrTx as unknown as Db, { companyId: input.companyId, issueId, contextSnapshot: context, requestedByActorType: null }) ? null : replacementReadiness; + const continuesNoWorkRun = retryReasonKind === "native_safe_replacement" && issue?.status === "blocked" + && await continuesNoWorkRunOnBlockedTask(dbOrTx as unknown as Db, { companyId: input.companyId, runId: input.runId }); return { runId: input.runId, runAgentId: input.agentId, issueId, retryReasonKind, - dependenciesBlocked: readiness && !readiness.isDependencyReady ? { + continuesNoWorkRunOnBlockedTask: continuesNoWorkRun, + dependenciesBlocked: readiness && !readiness.isDependencyReady && !continuesNoWorkRun ? { unresolvedBlockerIssueIds: readiness.unresolvedBlockerIssueIds, unresolvedBlockerCount: readiness.unresolvedBlockerCount, } : null, diff --git a/server/src/modules/run-dispatch/domain/policy.ts b/server/src/modules/run-dispatch/domain/policy.ts index 720d0651203..9fac93e57ee 100644 --- a/server/src/modules/run-dispatch/domain/policy.ts +++ b/server/src/modules/run-dispatch/domain/policy.ts @@ -105,6 +105,7 @@ export type ScheduledRetryFacts = { dispositionRepair: DispositionRepairFacts | null; /** A conversation retry must wait for unresolved questions and approvals. */ pendingResponse?: "interaction" | "approval" | null; + continuesNoWorkRunOnBlockedTask?: boolean; }; export type QueuedRunStalenessErrorCode = @@ -132,6 +133,7 @@ export type StalenessDecision = export type QueuedRunFacts = { pendingResponse?: "interaction" | "approval" | null; + continuesNoWorkRunOnBlockedTask?: boolean; /** Rechecked for automatic native replacements immediately before dispatch. */ dependenciesBlocked?: DependencyBlockFacts | null; runId: string; @@ -369,7 +371,7 @@ export function decideScheduledRetryGate( }; } - if (facts.retryReasonKind === "native_safe_replacement" && facts.issueStatus === "blocked") { + if (facts.retryReasonKind === "native_safe_replacement" && facts.issueStatus === "blocked" && !facts.continuesNoWorkRunOnBlockedTask) { return { allowed: false, issueId: facts.issueId, errorCode: "issue_blocked", reason: "Scheduled replacement suppressed because the task was blocked after recovery", details: { issueId: facts.issueId } }; @@ -613,7 +615,7 @@ export function decideQueuedRunStaleness( }; } - if (facts.retryReasonKind === "native_safe_replacement" && facts.dependenciesBlocked) { + if (facts.retryReasonKind === "native_safe_replacement" && facts.dependenciesBlocked && !facts.continuesNoWorkRunOnBlockedTask) { return { stale: true, errorCode: "issue_dependencies_blocked", reason: "Cancelled because issue dependencies became blocked before replacement dispatch", details: { issueId: facts.issueId, @@ -621,7 +623,7 @@ export function decideQueuedRunStaleness( unresolvedBlockerCount: facts.dependenciesBlocked.unresolvedBlockerCount } }; } - if (facts.retryReasonKind === "native_safe_replacement" && facts.issueStatus === "blocked") { + if (facts.retryReasonKind === "native_safe_replacement" && facts.issueStatus === "blocked" && !facts.continuesNoWorkRunOnBlockedTask) { return { stale: true, errorCode: "issue_blocked", reason: "Cancelled because the task was blocked before replacement dispatch", details: { issueId: facts.issueId } }; diff --git a/server/src/services/heartbeat/run-completion.ts b/server/src/services/heartbeat/run-completion.ts index 78356f15303..233e7925ee1 100644 --- a/server/src/services/heartbeat/run-completion.ts +++ b/server/src/services/heartbeat/run-completion.ts @@ -57,7 +57,8 @@ import { hasChatRunOwnedProviderInteraction } from "../chat-interaction-arbitrat import { conversationTurnPresentsAnswer } from "../native-runtime/native-governance.js"; import { readAcceptedTaskContinuationSummary } from "../heartbeat-run-summary.js"; import { SANDBOX_PROVIDER_PLUGIN_NOT_READY_REASON } from "../recovery/stranded-notice.js"; -import { MAX_TURN_CONTINUATION_RETRY_REASON, isResolvedInteractionContinuationWakeContext } from "../../modules/run-dispatch/index.js"; +import { MAX_TURN_CONTINUATION_RETRY_REASON, allowsIssueInteractionWake, isResolvedInteractionContinuationWakeContext } from "../../modules/run-dispatch/index.js"; +import { ISSUE_TREE_CONTROL_INTERACTION_WAKE_REASONS } from "../issue-tree-control.js"; import { redactCurrentUserText } from "../../log-redaction.js"; import type { createHeartbeatLifecycle } from "./run-lifecycle.js"; import type { createHeartbeatRunPreparation } from "./run-preparation.js"; @@ -1203,6 +1204,16 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC */ async function schedulePredecessorRetirementFallbackRetry(run: Run, agent: Agent, fallback: PredecessorRetirementFallback | null | undefined) { if (!fallback || run.status !== "failed") return null; + const scheduled = await schedulePredecessorRetirementContinuation(run, agent, fallback); + if (scheduled?.outcome !== "scheduled") { + await db.update(heartbeatRuns).set({ resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || '{"continuationScheduling":"refused"}'::jsonb` }) + .where(and(eq(heartbeatRuns.companyId, run.companyId), eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, "failed"))) + .catch(error => logger.warn({ err: error, runId: run.id }, "could not record that a run's continuation was not scheduled")); + } + return scheduled; + } + + async function schedulePredecessorRetirementContinuation(run: Run, agent: Agent, fallback: PredecessorRetirementFallback) { // Files this run's setup may have written are never dropped: without the // shared unchanged-files proof the workspace stays dirty and preserved. const released = await releaseUnchangedFallbackWorkspace(db, run).catch((error) => { @@ -1213,7 +1224,10 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC try { // Recorded only once the workspace is settled: it makes the run eligible for retry. if (!readPredecessorRetirementFallback(run.resultJson)) { - const record = { [PREDECESSOR_RETIREMENT_FALLBACK_KEY]: fallback, executionRecovery: { kind: "bootstrap", providerWorkStarted: false } }; + const [task] = run.issueId ? await db.select({ status: issues.status }).from(issues) + .where(and(eq(issues.companyId, run.companyId), eq(issues.id, run.issueId))) : []; + const record = { [PREDECESSOR_RETIREMENT_FALLBACK_KEY]: { ...fallback, ...(task ? { issueStatusAtFailure: task.status } : {}) }, + executionRecovery: { kind: "bootstrap", providerWorkStarted: false } }; await db.update(heartbeatRuns).set({ resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || ${JSON.stringify(record)}::jsonb` }) .where(and(eq(heartbeatRuns.companyId, run.companyId), eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, "failed"))); run = { ...run, resultJson: { ...parseObject(run.resultJson), ...record } }; @@ -1229,6 +1243,7 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC // An answered interaction keeps its own wake, so the retry still carries // the answer as a resolved-interaction continuation, in the uncharged lane. const wakeReason = isResolvedInteractionContinuationWakeContext(run.contextSnapshot) + || allowsIssueInteractionWake(parseObject(run.contextSnapshot), ISSUE_TREE_CONTROL_INTERACTION_WAKE_REASONS) ? readNonEmptyString(parseObject(run.contextSnapshot).wakeReason) : null; return scheduleBoundedRetryForRun(run, agent, { retryReason: INTERRUPTED_PREPARATION_RETRY_REASON, ...(wakeReason ? { wakeReason } : {}) }).catch((retryError) => { diff --git a/server/src/services/native-runtime/predecessor-retirement.ts b/server/src/services/native-runtime/predecessor-retirement.ts index 0ec31ba178c..8882a12f3e1 100644 --- a/server/src/services/native-runtime/predecessor-retirement.ts +++ b/server/src/services/native-runtime/predecessor-retirement.ts @@ -56,6 +56,7 @@ export type PredecessorRetirementFallback = { providerWorkStarted: false; predecessorRunId: string | null; replacedProviderLeaseIds: string[]; + issueStatusAtFailure?: string; }; export function readPredecessorRetirementFallback(resultJson: unknown): PredecessorRetirementFallback | null { diff --git a/server/src/services/task-interruption-presentation.test.ts b/server/src/services/task-interruption-presentation.test.ts index 2d7378f5658..4a8bd737803 100644 --- a/server/src/services/task-interruption-presentation.test.ts +++ b/server/src/services/task-interruption-presentation.test.ts @@ -566,3 +566,32 @@ describe("automatic scheduled retry", () => { .toMatchObject({ state: "failed", canRetry: true }); }); }); + +describe("hand-off of a run that failed before any provider work", () => { + const now = Date.parse("2026-11-13T15:08:12Z"); + const failed = { ...run, runtimeMode: "native" as const, errorCode: "adapter_failed", finishedAt: new Date(now - 3_000), + resultJson: { failureCause: "native_predecessor_retirement_unproven", executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } }; + const inProgress = { ...issue, status: "in_progress" }; + + it("shows the task as working while its fresh-sandbox continuation is being queued", () => { + for (const failureCause of ["native_predecessor_retirement_unproven", "native_failed_before_provider_turn"]) { + expect(projectTaskInterruption(inProgress, { run: { ...failed, resultJson: { ...failed.resultJson, failureCause } }, + agent, paused: false, approval: false }, now)).toMatchObject({ state: "working", canRetry: false }); + } + }); + + it("keeps the task working after the fallback is recorded while its continuation is still being scheduled", () => { + const recorded = { ...failed, resultJson: { ...failed.resultJson, predecessorRetirementFallback: { version: 1, providerWorkStarted: false } } }; + expect(projectTaskInterruption(inProgress, { run: recorded, agent, paused: false, approval: false }, now)) + .toMatchObject({ state: "working", canRetry: false }); + }); + + it("offers Retry when the continuation was refused or never queued", () => { + const refused = { ...failed, resultJson: { ...failed.resultJson, predecessorRetirementFallback: { version: 1, providerWorkStarted: false }, + continuationScheduling: "refused" } }; + expect(projectTaskInterruption(inProgress, { run: refused, agent, paused: false, approval: false }, now)) + .toMatchObject({ state: "failed", canRetry: true }); + expect(projectTaskInterruption(inProgress, { run: { ...failed, finishedAt: new Date(now - 31_000) }, agent, paused: false, approval: false }, now)) + .toMatchObject({ state: "failed", canRetry: true }); + }); +}); diff --git a/server/src/services/task-interruption-presentation.ts b/server/src/services/task-interruption-presentation.ts index 9374d564e6e..c7d8d85fb40 100644 --- a/server/src/services/task-interruption-presentation.ts +++ b/server/src/services/task-interruption-presentation.ts @@ -43,6 +43,8 @@ const QUEUED_INTERRUPT_HANDOFF_GRACE_MS = 5 * 60_000; const QUEUED_INPUT_DISPATCH_GRACE_MS = 2 * 60_000; const AUTOMATIC_DISPOSITION_GRACE_MS = 2 * 60_000; const WAITING_CONTINUATION_MOVE_FORWARD_MS = 15 * 60_000; +const NO_WORK_CONTINUATION_HANDOFF_GRACE_MS = 30_000; +const NO_WORK_CONTINUATION_CAUSES = new Set(["native_predecessor_retirement_unproven", "native_failed_before_provider_turn"]); type ProjectionIssue = Pick; type ProjectionRun = Pick