diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 40903c46bee..61125608984 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1358,8 +1358,12 @@ same-run bootstrap retry, and leaves an active board recovery action that asks for a supported model. A definite rejection whose startup `providerFailure` is `requires_action` for `authentication`, `permission` or `invalid_request` stops on its first attempt as `native_provider_startup_rejected`. On a remote -execution target, any definite rejection with a startup `providerFailure` stops -the same way, because a remote same-run bootstrap cannot prove ownership. No +execution target, another definite rejection with a startup `providerFailure` +retries on the same run only when the Runner transport proved, at close, that +the failed runner settled and its process exited (`runnerExitConfirmed` in the +saved failure). Restart recovery then launches a new runner. A rejection +without that proof, or one that repeats the previous attempt's startup error, +stops as `native_provider_startup_rejected`; the repeat names that error. No provider work began, so the board offers Retry, which starts a new run with a fresh provider session. Other local `session.open` rejections keep the bounded startup retry and end with Retry when it is spent. An diff --git a/packages/paperclip-runner/docs/durable-recovery.md b/packages/paperclip-runner/docs/durable-recovery.md index 30423e9983e..92dd99e1d03 100644 --- a/packages/paperclip-runner/docs/durable-recovery.md +++ b/packages/paperclip-runner/docs/durable-recovery.md @@ -159,10 +159,13 @@ which proves that the sidecar and the provider are gone. `authentication`, `permission` or `invalid_request` records `native_provider_startup_rejected`. No provider work began, so the controller stops same-run bootstrap retries and the board offers Retry at once - with the provider message. On a remote execution target, every definite - `session.open` failure with a startup `providerFailure` records the same code, - because a remote same-run bootstrap after the runner launched cannot prove - ownership and would stop on `ambiguous_remote_recovery_evidence`. On a local + with the provider message. On a remote execution target, a definite + `session.open` failure with a startup `providerFailure` keeps a same-run + bootstrap retry only when the transport's close settled the runner and + observed its exit (`runnerExitConfirmed`); restart recovery accepts that proof + as it accepts a runner's own exit. Without it, or when the failure repeats the + previous attempt's startup error, it records the same code, because a remote + same-run bootstrap cannot then prove ownership. On a local target, other startup failures (unknown, transient, rate limit, quota, model) keep the bounded same-run startup retries and end as `native_session_retry_exhausted` with Retry. A run an older release held on diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs b/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs index 194f81ad1a4..f3bd0906d73 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs @@ -1327,6 +1327,12 @@ fn run() -> Result<(), Box> { if let Some(delay) = argument(&args, "--thread-start-delay-ms") { thread::sleep(Duration::from_millis(delay.parse()?)); } + // A definite provider startup rejection, as Codex answers when + // its sandbox helper cannot start (no thread is created). + if let Some(reason) = argument(&args, "--reject-thread-start") { + send(json!({"id": id, "error": {"code": -32603, "message": reason}}))?; + continue; + } if require_external_sandbox && (message.pointer("/params/sandbox") != Some(&json!("danger-full-access")) || message.pointer("/params/permissions").is_some()) diff --git a/packages/paperclip-runner/src/drivers/codex/app-server-transport.ts b/packages/paperclip-runner/src/drivers/codex/app-server-transport.ts index ff0affb416f..2502927f526 100644 --- a/packages/paperclip-runner/src/drivers/codex/app-server-transport.ts +++ b/packages/paperclip-runner/src/drivers/codex/app-server-transport.ts @@ -76,6 +76,8 @@ export interface CodexAppServerTransport { * diagnostic context; transports must not forward it to the provider. */ close(reason?: string): Promise; + /** After close: the runner settled and its process exit was observed. */ + runnerExitConfirmed?(): boolean; /** Return the confirmed released source, or false when active use was not released. */ release?(): Promise; /** Relinquish controller authority while leaving durable runner work alive. */ diff --git a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts index 633616a30eb..6b3a67ccc6d 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts @@ -373,6 +373,11 @@ export class CodexAppServerDriver implements HarnessDriver { await cancellation.close().catch(() => {}); if (error instanceof NativeSessionProtocolIntegrityError) throw error; if (input.signal?.aborted) input.signal.throwIfAborted(); + // The controller may retry this bootstrap on the same run only when the + // failed runner is proven gone; record that proof on the failure. + if (error && typeof error === "object" && transport.runnerExitConfirmed?.() === true) { + Object.defineProperty(error, "runnerExitConfirmed", { value: true, enumerable: false }); + } if (error instanceof CodexRpcError && (this.#options.driverIdentity?.kind ?? DRIVER_KIND) === DRIVER_KIND) { const providerFailure = codexProviderFailure(error, "startup", this.#options.environment); if (providerFailure) Object.defineProperty(error, "providerFailure", { value: providerFailure, enumerable: false }); diff --git a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.lifecycle.test.ts b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.lifecycle.test.ts index 62e97dc75e4..8e5d466c6d8 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.lifecycle.test.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.lifecycle.test.ts @@ -126,6 +126,21 @@ describe("Codex app-server Codex driver", () => { else expect(error.providerFailure).toBeUndefined(); }); + it.each([true, false])("records the transport's runner exit proof (%s) on a failed session open after close", async (proven) => { + class ExitProofTransport extends FakeCodexTransport { + closed = false; + override async close(): Promise { await super.close(); this.closed = true; } + // Proof is read only after the driver's close completed. + runnerExitConfirmed(): boolean { return proven && this.closed; } + } + const transport = new ExitProofTransport(); + transport.rejectMethods.set("thread/start", new CodexRpcError("error creating thread: bwrap: loopback: Failed RTM_NEWADDR", -32603)); + const error = await makeDriver([transport]).openSession({ runId: "exit-proof", normalizedSessionId: "exit-proof", + workingDirectory: WORKSPACE }).catch(error => error); + expect(transport.closed).toBe(true); + expect(error.runnerExitConfirmed === true).toBe(proven); + }); + it("accepts runner-proven warm attachment when the host active-turn reducer is stale", async () => { const transport = new WarmAttachTransport(); const driver = makeDriver([transport]); diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts index 0275f959d0d..d0c16c5a7de 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts @@ -4045,6 +4045,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { #resolveClosedSignal!: () => void; #closePromise: Promise | null = null; #shutdownSettled = false; + #closeFinished = false; #probeRetirement: Promise | null = null; readonly prepareConversationResume?: CodexAppServerTransport["prepareConversationResume"]; #detachPromise: Promise | null = null; @@ -4275,6 +4276,20 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { return structuredClone(this.#evidence); } + /** + * True only when this transport's startup failed before any provider + * identity existed, its close has finished, and the runner process's exit + * was observed (it suspended and exited, or close killed it and waited). + * A runner that failed session.open cannot always suspend, and its + * controller checkpoint can be refused for lack of a provider identity, so + * neither is required. A remote same-run bootstrap retry may then launch a + * new runner: the old one is gone, and its provider never opened a thread. + */ + runnerExitConfirmed(): boolean { + return this.#closeFinished && !this.#startupComplete && this.#providerIdentity === null + && this.#evidence.runnerExited === true && this.#handle?.isExitConfirmed?.() !== false; + } + async request( method: string, params: Record, @@ -5421,7 +5436,9 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { const deadline = Date.now() + runnerCloseGraceMs(this.options); try { await this.#closeAndSettle(); + this.#closeFinished = true; } catch (error) { + this.#closeFinished = true; this.#queue.close(); try { await this.#retireControllerForRecovery(deadline); } catch (retirementError) { diff --git a/packages/paperclip-runner/src/live/runnerd-startup-rejection-exit.test.ts b/packages/paperclip-runner/src/live/runnerd-startup-rejection-exit.test.ts new file mode 100644 index 00000000000..99e8038cf1d --- /dev/null +++ b/packages/paperclip-runner/src/live/runnerd-startup-rejection-exit.test.ts @@ -0,0 +1,81 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { expect, it } from "vitest"; +import { createCapabilityRunnerdCodexTransport, defaultCapabilityRunnerdBinary } from "./runnerd-codex-transport.js"; + +const REJECTION = "error creating thread: fs sandbox helper failed: bwrap: loopback: Failed RTM_NEWADDR: Operation not permitted"; + +function processAbsent(pid: number): boolean { + try { process.kill(pid, 0); return false; } catch { return true; } +} + +it.each([ + ["saves an unsettled checkpoint", false], + // The controller's real remote checkpoint refuses a session with no provider + // identity, which is exactly the state a rejected session.open leaves. + ["refuses the checkpoint for lack of a provider identity", true], +])("proves a runner whose provider rejected session.open stopped, when the controller %s", async (_label, refuseCheckpoint) => { + const root = await mkdtemp(join(tmpdir(), "runner-startup-rejection-")); + const settlements: string[] = []; + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: resolve(import.meta.dirname, "../../runner/target/debug/fake-codex-app-server"), + codexArgs: ["--state-file", join(root, "provider.json"), "--reject-thread-start", REJECTION], + stateDirectory: root, + controlPlaneRegistration: async authority => { + await authority.start(); + return { + release: () => undefined, + checkpoint: (settlement: "settled" | "unsettled") => { + settlements.push(settlement); + if (refuseCheckpoint) throw new Error("runnerd did not report its provider identity"); + }, + }; + }, + }); + let pid: number | null = null; + try { + const failure = await bundle.transport.request("thread/start", { cwd: root }).then(() => null, (error: unknown) => error); + expect(String(failure)).toContain("session.open"); + pid = bundle.evidence().runnerPid; + expect(pid).not.toBeNull(); + expect(bundle.transport.runnerExitConfirmed?.()).toBe(false); + // The runner cannot suspend without a session, so close kills it and waits + // for its exit; the checkpoint is "unsettled" and may be refused. + await bundle.transport.close().catch(() => undefined); + expect(settlements).toEqual(["unsettled"]); + expect(bundle.transport.runnerExitConfirmed?.()).toBe(true); + expect(processAbsent(pid!)).toBe(true); + } finally { + await bundle.transport.close().catch(() => undefined); + if (pid) { try { process.kill(-pid, "SIGKILL"); } catch { /* already stopped */ } } + await rm(root, { recursive: true, force: true }); + } +}, 30_000); + +it("gives no exit proof for a runner whose session opened", async () => { + const root = await mkdtemp(join(tmpdir(), "runner-startup-opened-")); + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: resolve(import.meta.dirname, "../../runner/target/debug/fake-codex-app-server"), + codexArgs: ["--state-file", join(root, "provider.json")], + stateDirectory: root, + controlPlaneRegistration: async authority => { + await authority.start(); + return { release: () => undefined }; + }, + }); + let pid: number | null = null; + try { + await bundle.transport.request("thread/start", { cwd: root }); + pid = bundle.evidence().runnerPid; + await bundle.transport.close().catch(() => undefined); + // Provider work could follow an opened session; this proof never covers it. + expect(bundle.transport.runnerExitConfirmed?.()).toBe(false); + } finally { + await bundle.transport.close().catch(() => undefined); + if (pid) { try { process.kill(-pid, "SIGKILL"); } catch { /* already stopped */ } } + await rm(root, { recursive: true, force: true }); + } +}, 30_000); diff --git a/server/src/services/native-runtime/native-restart-recovery.ts b/server/src/services/native-runtime/native-restart-recovery.ts index 244d7120c77..88dbb08f7f2 100644 --- a/server/src/services/native-runtime/native-restart-recovery.ts +++ b/server/src/services/native-runtime/native-restart-recovery.ts @@ -1571,7 +1571,12 @@ export async function claimNativeRestartRecoveries(input: { && row.run.runnerProfileJson?.nativePreparationFencingVersion === NATIVE_PREPARATION_FENCING_VERSION && row.coordinator.phase === "retryable_failure" && row.run.status === "failed" && !row.coordinator.leaseOwner && settledFailure?.recoveryMode === "bootstrap_retry" - && settledFailure?.originalFailureCode === "native_runner_process_exited" + // The runner exited on its own, or it rejected session.open and the + // controller's close then proved it settled and exited. A failure + // saved without that proof (older releases) stays held. + && (settledFailure?.originalFailureCode === "native_runner_process_exited" + || (settledFailure?.originalFailureCode === "provider_session_open_rejected" + && settledFailure?.runnerExitConfirmed === true)) && settledFailure?.providerSessionEstablished === false && settledFailure?.providerEventsExist === false && settledFailure?.checkpointExists === false && !settledFailure?.successorRunId && !row.run.sessionIdBefore && !row.run.sessionIdAfter diff --git a/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts b/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts index 63678a59f2a..f58fa27e351 100644 --- a/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts +++ b/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts @@ -3157,6 +3157,28 @@ process.on("SIGTERM", () => child.kill("SIGTERM")); kind: "blocked", runId: fixture.runId, reason })]); }); + it("bootstraps again after a remote session.open rejection whose runner exit the controller proved", async () => { + // Staging run 61d2e06e with the new proof: the runner answered session.open + // with a definite startup failure, then the controller's close settled it + // and observed its exit. No provider session, event or checkpoint exists. + const fixture = await seedExitedRemoteRunner("STARTUP-REJECTED-EXIT-PROVEN", { + originalFailureCode: "provider_session_open_rejected", runnerExitConfirmed: true }, 1); + expect(await claimNativeRestartRecoveries({ db: fixture.db, controller: successor, + restartKind: "hard", runIds: [fixture.runId] })).toEqual([expect.objectContaining({ + kind: "bootstrap_incomplete", runId: fixture.runId })]); + const [coordinator] = await fixture.db.select().from(nativeRunFinalizations) + .where(eq(nativeRunFinalizations.runId, fixture.runId)); + expect(coordinator.recoveryHistory.at(-1)).toMatchObject({ reason: "remote_runner_exited_before_provider_work" }); + }); + + it("keeps an older release's remote session.open rejection held without the runner exit proof", async () => { + const fixture = await seedExitedRemoteRunner("STARTUP-REJECTED-NO-PROOF", { + originalFailureCode: "provider_session_open_rejected" }, 1); + expect(await claimNativeRestartRecoveries({ db: fixture.db, controller: successor, + restartKind: "hard", runIds: [fixture.runId] })).toEqual([expect.objectContaining({ + kind: "blocked", runId: fixture.runId, reason: "ambiguous_remote_recovery_evidence" })]); + }); + it("records that a held same-run retry was not a server restart", async () => { // Staging run 61d2e06e: a remote session.open rejection saved by an older // release, held when its due retry re-entered restart recovery. No server diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index b723fd1d0b5..c5f40043e9c 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -446,6 +446,9 @@ import { NATIVE_PROVIDER_SESSION_CLOSED, NATIVE_PROVIDER_STARTUP_REJECTED, nativeRemoteStartupRejectionCode, + nativeRepeatedStartupFailureAction, + nativePreviousStartupFailureSignature, + nativeStartupFailureSignature, nativeSessionRecoveryProjection, nativeGovernedWaitResult, nativeConversationReplyResult, @@ -14169,6 +14172,33 @@ describe("native session bounded recovery", () => { .toBe("provider_session_open_rejected"); expect(nativeRemoteStartupRejectionCode("native_session_interrupted", failure, true)).toBe("native_session_interrupted"); }); + + it("retries a remote startup rejection on the same run once the runner's exit is proven, but not a repeat", () => { + const confirmed = (fields: Record = {}) => + Object.defineProperty(sessionOpenFailure(startupFailure(fields)), "runnerExitConfirmed", { value: true, enumerable: false }); + const first = confirmed(); + expect(nativeRemoteStartupRejectionCode(nativeSessionFailureSourceCode(first), first, true)) + .toBe("provider_session_open_rejected"); + // Wrapped by a caller: the proof still reads through `cause`. + const wrapped = new Error("native session bootstrap failed", { cause: first }); + expect(nativeRemoteStartupRejectionCode("provider_session_open_rejected", wrapped, true)) + .toBe("provider_session_open_rejected"); + // The same startup error as the previous attempt stops, with the message named. + const previous = nativePreviousStartupFailureSignature([ + { disposition: "retryable_failure", startupFailureSignature: nativeStartupFailureSignature(startupFailure({})) }, + { disposition: "bootstrap_incomplete", reason: "remote_runner_exited_before_provider_work" }, + ]); + expect(previous).not.toBeNull(); + expect(nativeRemoteStartupRejectionCode("provider_session_open_rejected", confirmed(), true, previous)) + .toBe(NATIVE_PROVIDER_STARTUP_REJECTED); + expect(nativeRepeatedStartupFailureAction(startupFailure({}).message as string)).toContain("bwrap: loopback"); + // A different startup error is not a repeat. + expect(nativeRemoteStartupRejectionCode("provider_session_open_rejected", + confirmed({ message: "Query closed before response received" }), true, previous)).toBe("provider_session_open_rejected"); + // A deterministic rejection never retries, even with exit proof. + const deterministic = confirmed({ category: "authentication", retryability: "requires_action" }); + expect(nativeSessionFailureSourceCode(deterministic)).toBe(NATIVE_PROVIDER_STARTUP_REJECTED); + }); }); it("stops same-run retries for an operator while a saved side-effecting call has no result", () => { diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index f59b41980fa..077ce813b59 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -8517,14 +8517,62 @@ function deterministicProviderStartupFailure(error: unknown): boolean { && ["authentication", "permission", "invalid_request"].includes(failure.category); } +/** The Runner transport proved the failed runner settled and its process exited. */ +export function nativeRunnerExitConfirmed(error: unknown, depth = 0): boolean { + if (!error || typeof error !== "object" || depth > 5) return false; + const value = error as Record; + if (value.runnerExitConfirmed === true) return true; + return nativeRunnerExitConfirmed(value.cause, depth + 1) + || (Array.isArray(value.errors) && value.errors.slice(0, 3).some(item => nativeRunnerExitConfirmed(item, depth + 1))); +} + +/** Same provider, category and message (numbers ignored): a retry would meet the same fault. */ +export function nativeStartupFailureSignature(failure: unknown): string | null { + const value = record(failure); + if (value.phase !== "startup" || typeof value.message !== "string" || !value.message.trim()) return null; + return JSON.stringify([value.provider ?? null, value.category ?? null, value.message.replace(/\d+/g, "#").trim()]); +} + +/** + * On a remote target, a definite startup rejection retries on the same run + * only when the Runner proved its failed runner exited (so no second runner + * can own the sandbox) and the previous attempt did not fail with the same + * startup error. Otherwise it ends at once with Retry. A deterministic + * rejection never reaches here: it is already NATIVE_PROVIDER_STARTUP_REJECTED. + */ export function nativeRemoteStartupRejectionCode( sourceFailureCode: Code, error: unknown, remoteTarget: boolean, + previousStartupSignature?: string | null, ): Code | typeof NATIVE_PROVIDER_STARTUP_REJECTED { - return remoteTarget && sourceFailureCode === "provider_session_open_rejected" - && providerFailureFromError(error)?.phase === "startup" - ? NATIVE_PROVIDER_STARTUP_REJECTED : sourceFailureCode; + if (!remoteTarget || sourceFailureCode !== "provider_session_open_rejected") return sourceFailureCode; + const failure = providerFailureFromError(error); + if (failure?.phase !== "startup") return sourceFailureCode; + return nativeRunnerExitConfirmed(error) && !nativeRepeatedStartupFailure(error, previousStartupSignature) + ? sourceFailureCode : NATIVE_PROVIDER_STARTUP_REJECTED; +} + +export function nativeRepeatedStartupFailure(error: unknown, previousStartupSignature: string | null | undefined): boolean { + const current = nativeStartupFailureSignature(providerFailureFromError(error)); + return current !== null && current === previousStartupSignature; +} + +/** The startup failure signature this run's previous failed attempt recorded. The + * execution claim clears the saved failure, but its history entry remains. */ +export function nativePreviousStartupFailureSignature(recoveryHistory: unknown): string | null { + if (!Array.isArray(recoveryHistory)) return null; + for (let index = recoveryHistory.length - 1; index >= 0; index -= 1) { + const entry = record(recoveryHistory[index]); + if (entry.disposition === "retryable_failure" || entry.disposition === "terminal_failure") { + return typeof entry.startupFailureSignature === "string" ? entry.startupFailureSignature : null; + } + } + return null; +} + +export function nativeRepeatedStartupFailureAction(message: string): string { + return `Provider startup failed the same way on consecutive attempts, so automatic retries stopped: "${message.slice(0, 500)}". No provider work began. Fix the cause in the agent's environment if needed, then Retry. Retry starts a new run with a fresh provider session.`; } /** Reads the Runner's closed class from an authenticated turn.start failure. */ @@ -12330,12 +12378,27 @@ async function executePaperclipNativeSessionAttempt( !prepareNativeSessionOpenTimeoutRetry(input.execution, input.runnerExecutionTarget?.kind !== "remote") ? "native_session_interrupted" as const : startupClassifiedFailureCode; + // The previous attempt's saved failure survives the retry claim; a + // remote startup rejection compares against it to stop a repeat. + const remoteStartupRejection = classifiedFailureCode === "provider_session_open_rejected" + && input.runnerExecutionTarget?.kind === "remote"; + const previousStartupSignature = remoteStartupRejection + ? await input.db.select({ recoveryHistory: nativeRunFinalizations.recoveryHistory }).from(nativeRunFinalizations) + .where(and(eq(nativeRunFinalizations.runId, input.execution.binding.runId), + eq(nativeRunFinalizations.companyId, input.execution.binding.companyId))) + .limit(1).then(rows => nativePreviousStartupFailureSignature(rows[0]?.recoveryHistory)) + // Without the history no repeat is provable; the attempt budget still bounds retries. + .catch(() => null) + : null; + const repeatedRemoteStartupFailure = remoteStartupRejection + && nativeRepeatedStartupFailure(error, previousStartupSignature); const sourceFailureCode = classifiedFailureCode === "native_event_replay_conflict" ? classifiedFailureCode : providerUsageLimitObserved ? "native_provider_usage_limit" - : nativeRemoteStartupRejectionCode(classifiedFailureCode, error, input.runnerExecutionTarget?.kind === "remote"); + : nativeRemoteStartupRejectionCode(classifiedFailureCode, error, input.runnerExecutionTarget?.kind === "remote", + previousStartupSignature); const bootstrap = input.useRunnerd && input.runnerExecutionTarget?.kind === "remote" && !input.restartRecovery && !retainedTransition && !remoteWarmRecovery ? { @@ -12368,6 +12431,10 @@ async function executePaperclipNativeSessionAttempt( ? sourceFailureCode : disposition.failureCode; const providerFailure = providerFailureFromError(error); + const startupFailureSignature = recoveryEvidence.recoveryMode === "bootstrap_retry" + ? nativeStartupFailureSignature(providerFailure) : null; + const startupRejectedAction = repeatedRemoteStartupFailure && providerFailure + ? nativeRepeatedStartupFailureAction(providerFailure.message) : NATIVE_PROVIDER_STARTUP_REJECTED_ACTION; // Pre-provider startup retries back off and wait out a known provider // limit; their exhaustion explains the startup failure. const startupRetryExhausted = @@ -12482,6 +12549,10 @@ async function executePaperclipNativeSessionAttempt( recoveryEvidence.providerSessionEstablished, providerEventsExist: recoveryEvidence.providerEventsExist, checkpointExists: recoveryEvidence.checkpointExists, + // Restart recovery relaunches a remote runner for this retry only + // with this proof that the failed runner is gone. + ...(recoveryEvidence.recoveryMode === "bootstrap_retry" && nativeRunnerExitConfirmed(error) + ? { runnerExitConfirmed: true } : {}), ...(providerCooldownAt ? { providerCooldownAt: providerCooldownAt.toISOString() } : {}), ...(providerLimit?.plan.resetAt ? { providerLimitResetAt: providerLimit.plan.resetAt.toISOString() } : {}), recoveryOwner: recoveryProjection.recoveryOwner, @@ -12495,7 +12566,7 @@ async function executePaperclipNativeSessionAttempt( : sourceFailureCode === NATIVE_PROVIDER_SESSION_CLOSED ? NATIVE_PROVIDER_SESSION_CLOSED_ACTION : sourceFailureCode === NATIVE_PROVIDER_STARTUP_REJECTED - ? NATIVE_PROVIDER_STARTUP_REJECTED_ACTION + ? startupRejectedAction : sourceFailureCode === "native_session_cleanup_quarantined" ? nativeCleanupRefusalAction(completedTurnCleanup) : sourceFailureCode === "runner_remote_provider_verification_unavailable" @@ -12552,6 +12623,7 @@ async function executePaperclipNativeSessionAttempt( providerSessionEstablished: recoveryEvidence.providerSessionEstablished, checkpointExists: recoveryEvidence.checkpointExists, + ...(startupFailureSignature ? { startupFailureSignature } : {}), })}::jsonb) ) with ordinality as history(item, ordinal) where ordinal > greatest( @@ -12569,6 +12641,7 @@ async function executePaperclipNativeSessionAttempt( providerSessionEstablished: recoveryEvidence.providerSessionEstablished, checkpointExists: recoveryEvidence.checkpointExists, + ...(startupFailureSignature ? { startupFailureSignature } : {}), })}::jsonb) ) - 20, 0 @@ -12772,7 +12845,7 @@ async function executePaperclipNativeSessionAttempt( : sourceFailureCode === NATIVE_PROVIDER_SESSION_CLOSED ? NATIVE_PROVIDER_SESSION_CLOSED_ACTION : sourceFailureCode === NATIVE_PROVIDER_STARTUP_REJECTED - ? NATIVE_PROVIDER_STARTUP_REJECTED_ACTION + ? startupRejectedAction : sourceFailureCode === "native_session_cleanup_quarantined" ? nativeCleanupRefusalAction(completedTurnCleanup) : sourceFailureCode === "runner_task_admission_rejected" diff --git a/server/src/services/native-runtime/native-startup-rejected-retry.integration.test.ts b/server/src/services/native-runtime/native-startup-rejected-retry.integration.test.ts index 20586d0b943..b2421734634 100644 --- a/server/src/services/native-runtime/native-startup-rejected-retry.integration.test.ts +++ b/server/src/services/native-runtime/native-startup-rejected-retry.integration.test.ts @@ -46,7 +46,7 @@ describe("native session.open the provider rejected before any provider work", ( message: "error creating thread: failed to load AGENTS.md instructions for environment `local`: fs sandbox helper failed with status exit status: 1: bwrap: loopback: Failed RTM_NEWADDR: Operation not permitted", ...fields }); - async function rejectSessionOpen(failure: Error, attempt = 0) { + async function rejectSessionOpen(failure: Error, attempt = 0, options: { remote?: boolean; recoveryHistory?: Record[] } = {}) { const issueId = randomUUID(); await db.insert(issues).values({ id: issueId, companyId, title: "Plan the change", status: "in_progress", workMode: "standard", assigneeAgentId: agentId, createdByUserId: "board-user", responsibleUserId: "board-user" }); @@ -58,7 +58,8 @@ describe("native session.open the provider rejected before any provider work", ( await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, issueId)); const native = await prepareNativeHeartbeatRun({ db, run, issue, environmentLeaseId: randomUUID() }); const [contract] = await db.select().from(completionContracts).where(eq(completionContracts.issueId, issueId)); - await db.insert(nativeRunFinalizations).values({ runId, companyId, issueId, phase: "observed", attempt }); + await db.insert(nativeRunFinalizations).values({ runId, companyId, issueId, phase: "observed", attempt, + ...(options.recoveryHistory ? { recoveryHistory: options.recoveryHistory } : {}) }); const execution: NativeExecutionInputV1 = { schema: "paperclip.native-execution-input.v1", binding: { companyId, runId, issueId, agentId, executionWorkspaceId: runId }, @@ -72,7 +73,9 @@ describe("native session.open the provider rejected before any provider work", ( }; await db.update(heartbeatRuns).set({ runnerProfileJson: { nativeExecutionInput: execution } }).where(eq(heartbeatRuns.id, runId)); provider.execute.mockReset().mockRejectedValue(failure); - await executePaperclipNativeSession({ db, execution, runnerInstanceId: native.runnerInstanceId, backend: { + await executePaperclipNativeSession({ db, execution, runnerInstanceId: native.runnerInstanceId, + ...(options.remote ? { runnerExecutionTarget: { kind: "remote", transport: "sandbox", providerKey: "test", remoteCwd: "/workspace", + runner: { execute: vi.fn() } } as never } : {}), backend: { descriptor: async () => { throw new Error("Unexpected provider call"); }, openSession: async () => { throw new Error("Unexpected provider call"); }, } }).catch(() => undefined); @@ -118,4 +121,32 @@ describe("native session.open the provider rejected before any provider work", ( const [issue] = await db.select().from(issues).where(eq(issues.id, issueId)); await expect(taskInterruptionPresentation(db, issue)).resolves.toMatchObject({ state: "failed", canRetry: true }); }, 120_000); + + const exitProven = (error: Error) => Object.defineProperty(error, "runnerExitConfirmed", { value: true, enumerable: false }); + + it("ends a remote startup rejection at once when the runner exit is not proven", async () => { + const { coordinator } = await rejectSessionOpen(rejection(startupFailure()), 0, { remote: true }); + expect(coordinator).toMatchObject({ phase: "terminal_failure", failureCode: NATIVE_PROVIDER_STARTUP_REJECTED }); + expect(coordinator.failureDetail).not.toHaveProperty("runnerExitConfirmed"); + }, 120_000); + + it("retries a remote startup rejection on the same run when the runner exit is proven", async () => { + const { coordinator } = await rejectSessionOpen(exitProven(rejection(startupFailure())), 0, { remote: true }); + expect(coordinator).toMatchObject({ phase: "retryable_failure", failureCode: "native_session_interrupted", + failureDetail: expect.objectContaining({ recoveryMode: "bootstrap_retry", originalFailureCode: "provider_session_open_rejected", + runnerExitConfirmed: true, providerSessionEstablished: false, providerEventsExist: false, checkpointExists: false }) }); + }, 120_000); + + it("stops a remote startup rejection that repeats the previous attempt's error, naming it, with Retry", async () => { + // The first attempt's own failure write records the signature the second compares with. + const first = await rejectSessionOpen(exitProven(rejection(startupFailure())), 0, { remote: true }); + const { issueId, coordinator } = await rejectSessionOpen(exitProven(rejection(startupFailure())), 1, { remote: true, + recoveryHistory: [...(first.coordinator.recoveryHistory as Record[]), + { disposition: "bootstrap_incomplete", reason: "remote_runner_exited_before_provider_work" }] }); + expect(coordinator).toMatchObject({ phase: "terminal_failure", failureCode: NATIVE_PROVIDER_STARTUP_REJECTED, nextAttemptAt: null, + failureDetail: expect.objectContaining({ nextAction: expect.stringContaining("bwrap: loopback: Failed RTM_NEWADDR") }) }); + expect(coordinator.failureDetail?.nextAction).toContain("Retry starts a new run"); + const [issue] = await db.select().from(issues).where(eq(issues.id, issueId)); + await expect(taskInterruptionPresentation(db, issue)).resolves.toMatchObject({ state: "failed", canRetry: true }); + }, 120_000); });