diff --git a/doc/run-log-events.md b/doc/run-log-events.md index 26d6e9782b3..c21734be604 100644 --- a/doc/run-log-events.md +++ b/doc/run-log-events.md @@ -361,6 +361,9 @@ runner-sourced event or controller checkpoint, records `remote_runner_never_launched` with the same disposition and budget. The executor records `native.process_start_requested` before a backend may spawn a runner, so its absence (with no checkpoint) proves nothing reached a runner. +The same rule covers a remote run that its executor already settled as a +same-run `bootstrap_retry` (for example when a shutdown closed the local +runtime listener during startup) and that is due when the server restarts. For a local Codex, OpenCode, or ACPX turn lost while runnerd was stopped, the canonical `turn.failed` event retains `error.code: "provider_turn_lost_on_restore"` and diff --git a/server/src/services/native-runtime/native-restart-recovery.ts b/server/src/services/native-runtime/native-restart-recovery.ts index b58403e4803..b72a164a1cf 100644 --- a/server/src/services/native-runtime/native-restart-recovery.ts +++ b/server/src/services/native-runtime/native-restart-recovery.ts @@ -1128,15 +1128,29 @@ export async function claimNativeRestartRecoveries(input: { const remoteRunnerNeverLaunched = !unclaimedBootstrap && remoteWorkspace?.state === "prepared" && !pluginTask && !durableAuthority && stagedAttachment === null && row.run.runnerProfileJson?.nativePreparationFencingVersion === NATIVE_PREPARATION_FENCING_VERSION - && row.coordinator.phase === "observed" && row.coordinator.attempt >= 1 && row.coordinator.controllerGeneration >= 1 - && Boolean(row.coordinator.controllerBootId && row.coordinator.controllerPid - && row.coordinator.controllerProcessStartedAt && row.coordinator.leaseOwner) + && row.coordinator.attempt >= 1 && row.coordinator.controllerGeneration >= 1 + && Boolean(row.coordinator.controllerBootId && row.coordinator.controllerPid && row.coordinator.controllerProcessStartedAt) + // Either the executor still held the claim, or it settled a same-run + // bootstrap retry itself (for example after a shutdown closed its local + // runtime listener) and released the lease. That retry already proved + // no provider session, event or checkpoint; the checks below re-prove it. + && (row.coordinator.phase === "observed" ? Boolean(row.coordinator.leaseOwner) + : row.coordinator.phase === "retryable_failure" && row.run.status === "failed" + && row.coordinator.failureDetail?.recoveryMode === "bootstrap_retry" && !row.coordinator.leaseOwner) && !row.coordinator.failureDetail?.successorRunId && !row.run.processPid && !row.run.processGroupId && !row.run.processStartedAt && !row.run.sessionIdBefore && !row.run.sessionIdAfter && Boolean(row.run.nativeSessionId && row.run.runnerInstanceId) && !hasCheckpoint && !hasProviderEvidence && !(await hasNativeExecutionStartEvidence(tx, row.run)); + // A settled retry starts new work, so it needs the same current + // admission as the resumption reconciler: task pause, agent status, + // dependencies, ownership and competing runs. Keep it waiting otherwise. + if (remoteRunnerNeverLaunched && row.coordinator.phase === "retryable_failure" && !admission.exhausted + && !await readNativeSessionResumptionAgent(tx as unknown as Db, { issue, run: row.run, coordinator: row.coordinator })) { + const deferred = await deferRemoteRetry("native_retry_admission_ineligible", ["done", "cancelled"].includes(issue.status)); + if (deferred) return deferred; + } // A claimed remote bootstrap (runner launched, perhaps connected) has no // provider evidence when its controller dies before session.open diff --git a/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts b/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts index 29881952bc1..92e8c5517a2 100644 --- a/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts +++ b/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts @@ -2374,6 +2374,8 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", ( attempt?: number; leaseExpired?: boolean; liveController?: boolean; event?: { eventType: string; sourceEventId?: string }; run?: Partial; + /** RC-29c: the executor settled a same-run bootstrap retry before the restart. */ + bootstrapRetry?: boolean; } = {}) { const fixture = await seedRun(`REMOTE-UNLAUNCHED-${label}`); await fixture.db.update(heartbeatRuns).set({ @@ -2391,9 +2393,78 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", ( leaseOwner: `previous:${randomUUID()}`, leaseExpiresAt: new Date(Date.now() + (options.leaseExpired ? -1_000 : 20 * 60_000)), }).where(eq(nativeRunFinalizations.runId, fixture.runId)); + if (options.bootstrapRetry) { + // Shape persisted by the executor when shutdown closed the local runtime + // listener during remote startup (local_runtime_api_listener_unavailable). + await fixture.db.update(nativeRunFinalizations).set({ + phase: "retryable_failure", leaseOwner: null, leaseExpiresAt: null, + nextAttemptAt: new Date(Date.now() - 1_000), failureCode: "local_runtime_api_listener_unavailable", + failureDetail: { message: "local_runtime_api_listener_unavailable", recoverable: true, + recoveryMode: "bootstrap_retry", checkpointExists: false, providerEventsExist: false, + providerSessionEstablished: false, originalFailureCode: "local_runtime_api_listener_unavailable" }, + }).where(eq(nativeRunFinalizations.runId, fixture.runId)); + await fixture.db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date(), + errorCode: "local_runtime_api_listener_unavailable" }).where(eq(heartbeatRuns.id, fixture.runId)); + } return fixture; } + it("restarts a remote bootstrap retry that shutdown cut before any runner was requested", async () => { + const fixture = await seedUnlaunchedRemoteClaim("BOOTSTRAP-RETRY", { bootstrapRetry: true }); + expect(await claimNativeRestartRecoveries({ db: fixture.db, controller: successor, + restartKind: "hard", runIds: [fixture.runId] })).toEqual([expect.objectContaining({ + kind: "bootstrap_incomplete", runId: fixture.runId, providerAttempt: 1, controllerGeneration: 2 })]); + const [coordinator] = await fixture.db.select().from(nativeRunFinalizations) + .where(eq(nativeRunFinalizations.runId, fixture.runId)); + expect(coordinator).toMatchObject({ phase: "observed", recoveryState: "bootstrap_incomplete", failureCode: null, + nextAttemptAt: null }); + expect(coordinator.recoveryHistory.at(-1)).toMatchObject({ reason: "remote_runner_never_launched" }); + const [run] = await fixture.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, fixture.runId)); + expect(run).toMatchObject({ status: "running", errorCode: null, finishedAt: null }); + expect(await fixture.db.select().from(issueRecoveryActions) + .where(eq(issueRecoveryActions.sourceIssueId, fixture.issueId))).toHaveLength(0); + }); + + it.each<[string, Parameters[1], { kind: string; reason: string }]>([ + ["a runner process start was requested", { bootstrapRetry: true, event: { eventType: "native.process_start_requested" } }, + { kind: "blocked", reason: "ambiguous_remote_recovery_evidence" }], + ["a runner event was committed", { bootstrapRetry: true, event: { eventType: "item.updated", sourceEventId: "runner-event-1" } }, + { kind: "blocked", reason: "ambiguous_remote_recovery_evidence" }], + ["the bootstrap budget is spent", { bootstrapRetry: true, attempt: 3 }, + { kind: "blocked", reason: "execution_recovery_budget_exhausted" }], + ])("keeps a remote bootstrap retry held when %s", async (label, options, expected) => { + const fixture = await seedUnlaunchedRemoteClaim(`RETRY-${label.replace(/\W+/g, "-")}`, options); + expect(await claimNativeRestartRecoveries({ db: fixture.db, controller: successor, + restartKind: "hard", runIds: [fixture.runId] })).toEqual([expect.objectContaining({ runId: fixture.runId, ...expected })]); + }); + + it("keeps a paused agent's remote bootstrap retry waiting instead of starting it", async () => { + const fixture = await seedUnlaunchedRemoteClaim("PAUSED-RETRY", { bootstrapRetry: true }); + await fixture.db.update(agents).set({ status: "paused" }).where(eq(agents.id, agentId)); + try { + expect(await claimNativeRestartRecoveries({ db: fixture.db, controller: successor, + restartKind: "hard", runIds: [fixture.runId] })).toEqual([expect.objectContaining({ + kind: "blocked", runId: fixture.runId, reason: "native_retry_admission_ineligible" })]); + const [run] = await fixture.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, fixture.runId)); + expect(run.status).toBe("failed"); + const [coordinator] = await fixture.db.select().from(nativeRunFinalizations) + .where(eq(nativeRunFinalizations.runId, fixture.runId)); + expect(coordinator).toMatchObject({ phase: "retryable_failure", leaseOwner: null }); + expect(coordinator.nextAttemptAt!.getTime()).toBeGreaterThan(Date.now()); + } finally { + await fixture.db.update(agents).set({ status: "active" }).where(eq(agents.id, agentId)); + } + }); + + it("leaves a remote exact-checkpoint retry to its own admission path", async () => { + const fixture = await seedUnlaunchedRemoteClaim("EXACT-RETRY", { bootstrapRetry: true }); + await fixture.db.update(nativeRunFinalizations).set({ failureDetail: { recoveryMode: "exact_checkpoint_resume" } }) + .where(eq(nativeRunFinalizations.runId, fixture.runId)); + const [disposition] = await claimNativeRestartRecoveries({ db: fixture.db, controller: successor, + restartKind: "hard", runIds: [fixture.runId] }); + expect(disposition.kind).not.toBe("bootstrap_incomplete"); + }); + it.each([false, true])("restarts a claimed remote run that never requested a runner (lease expired=%s)", async (leaseExpired) => { const fixture = await seedUnlaunchedRemoteClaim(`PROVEN-${leaseExpired}`, { leaseExpired }); expect(await claimNativeRestartRecoveries({ db: fixture.db, controller: successor,