diff --git a/server/src/services/native-runtime/native-restart-recovery.ts b/server/src/services/native-runtime/native-restart-recovery.ts index 2dbeea7c9f5..afc86c3ec3d 100644 --- a/server/src/services/native-runtime/native-restart-recovery.ts +++ b/server/src/services/native-runtime/native-restart-recovery.ts @@ -668,7 +668,8 @@ export function classifyNativeRunnerRecoveryEvidence(input: { unclaimedBootstrap?: boolean; /** Durable evidence proves a claimed remote bootstrap reached no provider work. */ remoteBootstrapProvenIncomplete?: boolean; - remoteBootstrapIncompleteReason?: "remote_runner_never_launched" | "remote_session_opened_without_turn"; + remoteBootstrapIncompleteReason?: "remote_runner_never_launched" | "remote_session_opened_without_turn" + | "remote_runner_exited_before_provider_work"; processStartMatches: boolean; knownProviderProcessAlive?: boolean; knownProviderProcessIdentityAmbiguous?: boolean; @@ -768,6 +769,21 @@ export function classifyNativeRunnerRecoveryEvidence(input: { /** Any record that execution reached a runner: a requested or identified * process, an adapter invocation, a tool start, or a runner-sourced event. */ +/** Whether a runner or provider produced any event for this run. Unlike + * hasNativeExecutionStartEvidence, the controller's own process-start and + * identity records do not count: a relaunched runner that exited before run.prepare + * has those, but nothing the runner itself sent. */ +async function hasNativeRunnerOriginatedEvidence( + tx: Pick, run: { companyId: string; id: string }, +): Promise { + const [event] = await tx.select({ id: heartbeatRunEvents.id }).from(heartbeatRunEvents) + .where(and(eq(heartbeatRunEvents.companyId, run.companyId), eq(heartbeatRunEvents.runId, run.id), + or(isNotNull(heartbeatRunEvents.sourceEventId), + inArray(heartbeatRunEvents.eventType, ["adapter.invoke", "tool.execution.started"])))) + .limit(1); + return event !== undefined; +} + export async function hasNativeExecutionStartEvidence( tx: Pick, run: { companyId: string; id: string }, ): Promise { @@ -1333,7 +1349,28 @@ export async function claimNativeRestartRecoveries(input: { // A settled retry starts new work, so it needs the same current // admission as the resumption reconciler: task pause, agent status, // dependencies, ownership and competing runs. Keep it waiting otherwise. - if (remoteRunnerNeverLaunched && row.coordinator.phase === "retryable_failure" && !admission.exhausted + // The executor relaunched a remote runner for a same-run bootstrap + // retry, and that runner exited (observed by its transport) before any + // provider work: the executor's settled failure durably records no + // provider session, no provider event and no checkpoint, and none exists + // now. The runner's own exit is the remote proof that it is gone; a + // recorded runner PID alone would not be (Daytona wave s9301). + const settledFailure = row.coordinator.failureDetail; + const remoteRunnerExitedBeforeProviderWork = !unclaimedBootstrap && !remoteRunnerNeverLaunched + && remoteWorkspace?.state === "prepared" && !pluginTask && stagedAttachment === null + && row.run.runnerProfileJson?.nativePreparationFencingVersion === NATIVE_PREPARATION_FENCING_VERSION + && row.coordinator.phase === "retryable_failure" && row.run.status === "failed" && !row.coordinator.leaseOwner + && settledFailure?.recoveryMode === "bootstrap_retry" + && settledFailure?.originalFailureCode === "native_runner_process_exited" + && settledFailure?.providerSessionEstablished === false && settledFailure?.providerEventsExist === false + && settledFailure?.checkpointExists === false && !settledFailure?.successorRunId + && !row.run.sessionIdBefore && !row.run.sessionIdAfter + && Boolean(row.run.nativeSessionId && row.run.runnerInstanceId) + && !hasCheckpoint && !hasProviderEvidence + // Fresh evidence wins over the saved failure: any event the runner or + // provider sent contradicts "no provider work". + && !(await hasNativeRunnerOriginatedEvidence(tx, row.run)); + if ((remoteRunnerNeverLaunched || remoteRunnerExitedBeforeProviderWork) && row.coordinator.phase === "retryable_failure" && !admission.exhausted && !await readNativeSessionResumptionAgent(tx as unknown as Db, { issue, run: row.run, coordinator: row.coordinator })) { const deferred = await deferRemoteRetry("native_retry_admission_ineligible", ["done", "cancelled"].includes(issue.status)); if (deferred) return deferred; @@ -1370,8 +1407,10 @@ export async function claimNativeRestartRecoveries(input: { const classification = classifyNativeRunnerRecoveryEvidence({ remoteSandbox: remoteSandbox || Boolean(pluginTask && durableAuthority), unclaimedBootstrap, - remoteBootstrapProvenIncomplete: remoteBootstrapProvenIncomplete || remoteRunnerNeverLaunched, + remoteBootstrapProvenIncomplete: remoteBootstrapProvenIncomplete || remoteRunnerNeverLaunched + || remoteRunnerExitedBeforeProviderWork, remoteBootstrapIncompleteReason: remoteRunnerNeverLaunched ? "remote_runner_never_launched" + : remoteRunnerExitedBeforeProviderWork ? "remote_runner_exited_before_provider_work" : remoteBootstrapProof === "session_opened_without_turn" ? "remote_session_opened_without_turn" : undefined, runnerPidAlive, runnerGroupAlive, diff --git a/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts b/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts index b2f50dfc521..6db00dd5178 100644 --- a/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts +++ b/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts @@ -2678,6 +2678,66 @@ process.on("SIGTERM", () => child.kill("SIGTERM")); .where(eq(issueRecoveryActions.sourceIssueId, fixture.issueId))).toHaveLength(0); }); + /** Daytona wave s9301: the relaunched remote runner exited before run.prepare. */ + async function seedExitedRemoteRunner(label: string, failure: Record = {}, attempt = 2) { + const fixture = await seedUnlaunchedRemoteClaim(label, { bootstrapRetry: true, attempt, + event: { eventType: "native.process_start_requested" }, run: { processPid: 955, processStartedAt: new Date(Date.now() - 30_000) } }); + await fixture.db.insert(heartbeatRunEvents).values({ companyId, runId: fixture.runId, agentId, seq: 901, stream: "system", + level: "info", eventType: "native.process_identity_recorded" }); + await fixture.db.update(nativeRunFinalizations).set({ failureCode: "native_runner_process_exited", + failureDetail: { message: "runnerd exited while waiting for run.prepare", recoverable: true, recoveryMode: "bootstrap_retry", + checkpointExists: false, providerEventsExist: false, providerSessionEstablished: false, + originalFailureCode: "native_runner_process_exited", ...failure } }) + .where(eq(nativeRunFinalizations.runId, fixture.runId)); + await fixture.db.update(heartbeatRuns).set({ errorCode: "native_runner_process_exited" }).where(eq(heartbeatRuns.id, fixture.runId)); + return fixture; + } + + it("bootstraps again when the relaunched remote runner exited before any provider work", async () => { + // The runner's own exit, observed and recorded by the controller with no + // provider session, events or checkpoint, proves the remote runner gone. + const fixture = await seedExitedRemoteRunner("EXITED-RUNNER"); + expect(await claimNativeRestartRecoveries({ db: fixture.db, controller: successor, + restartKind: "hard", runIds: [fixture.runId] })).toEqual([expect.objectContaining({ + kind: "bootstrap_incomplete", runId: fixture.runId, providerAttempt: 2 })]); + const [coordinator] = await fixture.db.select().from(nativeRunFinalizations) + .where(eq(nativeRunFinalizations.runId, fixture.runId)); + expect(coordinator).toMatchObject({ phase: "observed", recoveryState: "bootstrap_incomplete", failureCode: null }); + expect(coordinator.recoveryHistory.at(-1)).toMatchObject({ reason: "remote_runner_exited_before_provider_work" }); + const [run] = await fixture.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, fixture.runId)); + expect(run).toMatchObject({ status: "running", errorCode: null }); + }); + + it.each(["a provider event committed after the failure", "a session checkpoint saved after the failure"])( + "keeps an exited remote runner's retry held when %s contradicts the saved failure", async (label) => { + // The saved failure still says no provider work; fresh evidence wins. + const fixture = await seedExitedRemoteRunner(`EXITED-FRESH-${label.split(" ")[1]}`); + if (label.includes("event")) { + await fixture.db.insert(heartbeatRunEvents).values({ companyId, runId: fixture.runId, agentId, seq: 902, + stream: "system", level: "info", eventType: "item.updated", sourceEventId: "runner-event-after-failure" }); + } else { + const [run] = await fixture.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, fixture.runId)); + await fixture.db.update(heartbeatRuns).set({ runnerProfileJson: { ...run.runnerProfileJson, sessionCheckpoint: {} } }) + .where(eq(heartbeatRuns.id, fixture.runId)); + } + expect(await claimNativeRestartRecoveries({ db: fixture.db, controller: successor, + restartKind: "hard", runIds: [fixture.runId] })).toEqual([expect.objectContaining({ + kind: "blocked", runId: fixture.runId, reason: "ambiguous_remote_recovery_evidence" })]); + }); + + it.each<[string, Record, number, string]>([ + ["the failure was not a runner exit", { originalFailureCode: "local_runtime_api_listener_unavailable" }, 2, "ambiguous_remote_recovery_evidence"], + ["the provider session was established", { providerSessionEstablished: true }, 2, "ambiguous_remote_recovery_evidence"], + ["provider events exist", { providerEventsExist: true }, 2, "ambiguous_remote_recovery_evidence"], + ["a checkpoint exists", { checkpointExists: true }, 2, "ambiguous_remote_recovery_evidence"], + ["the bootstrap budget is spent", {}, 3, "execution_recovery_budget_exhausted"], + ])("keeps an exited remote runner's retry held when %s", async (label, failure, attempt, reason) => { + const fixture = await seedExitedRemoteRunner(`EXITED-${label.replace(/\W+/g, "-")}`, failure, attempt); + expect(await claimNativeRestartRecoveries({ db: fixture.db, controller: successor, + restartKind: "hard", runIds: [fixture.runId] })).toEqual([expect.objectContaining({ + kind: "blocked", runId: fixture.runId, reason })]); + }); + it.each<[string, Parameters[1], { kind: string; reason: string }]>([ ["a runner process start was requested", { bootstrapRetry: true, event: { eventType: "native.process_start_requested" } }, { kind: "blocked", reason: "ambiguous_remote_recovery_evidence" }],