diff --git a/server/src/services/native-runtime/native-restart-recovery.ts b/server/src/services/native-runtime/native-restart-recovery.ts index d2ae6c18e23..544940c183a 100644 --- a/server/src/services/native-runtime/native-restart-recovery.ts +++ b/server/src/services/native-runtime/native-restart-recovery.ts @@ -5,7 +5,7 @@ import { CONTROLLER_LEASE_RENEW_MS } from "../controller-instance-lease.js"; import { currentControllerProcessStamp, UNKNOWN_PROCESS_SPACE } from "../controller-process-identity.js"; import { readNativeSessionResumptionAgent, restoreNativeSessionExecution } from "./native-session-resumption-admission.js"; import { notifyLiveRunRecovery } from "../live-run-recovery-notifications.js"; -import { NATIVE_RETAINED_OWNER_WAIT } from "./native-controller-recovery.js"; +import { NATIVE_AGENT_DIRECTORY_BUSY_WAIT, NATIVE_RETAINED_OWNER_WAIT } from "./native-controller-recovery.js"; import { notifyNativeEnvironmentSettlement } from "../native-environment-settlement-notifications.js"; import { remoteWorkspaceRunReferenceSchema } from "@paperclipai/shared"; import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js"; @@ -223,6 +223,12 @@ function previousNoProgressRestarts(recoveryHistory: ReadonlyArray): number | null { + const last = [...recoveryHistory].reverse().find(entry => RECOVERY_CLAIM_DISPOSITIONS.has(entry?.disposition)); + const previous = previousRecoveryClaim(recoveryHistory); + return last?.disposition === "bootstrap_incomplete" && typeof previous === "object" ? previous.watermark : null; +} + /** * The most recent recovery claim's event-log position and the provider attempt * its resume runs as. Entries from releases that did not record them give none. @@ -759,7 +765,7 @@ export function classifyNativeRunnerRecoveryEvidence(input: { remoteBootstrapProvenIncomplete?: boolean; remoteBootstrapIncompleteReason?: "remote_runner_never_launched" | "remote_session_opened_without_turn" | "remote_runner_exited_before_provider_work" | "remote_runner_never_received_commands" - | "remote_follow_up_never_attached"; + | "remote_follow_up_never_attached" | "remote_bootstrap_restart_deferred_before_launch"; processStartMatches: boolean; knownProviderProcessAlive?: boolean; knownProviderProcessIdentityAmbiguous?: boolean; @@ -1735,7 +1741,25 @@ export async function claimNativeRestartRecoveries(input: { && await carriesSettledPredecessorSession(tx, row.run, checkpointRecord.providerSessionId) && !(await hasNativeRunnerOriginatedEvidence(tx, row.run)) && !(await hasNativeRunEvent(tx, row.run, PROCESS_IDENTITY_RECORDED)); - if ((remoteRunnerNeverLaunched || remoteRunnerExitedBeforeProviderWork || remoteFollowUpNeverAttached) + const bootstrapRestartWatermark = lastBootstrapRestartWatermark(row.coordinator.recoveryHistory); + const remoteBootstrapRestartDeferredBeforeLaunch = !unclaimedBootstrap && !remoteRunnerNeverLaunched + && !remoteRunnerExitedBeforeProviderWork && !remoteFollowUpNeverAttached + && remoteWorkspace?.state === "prepared" && !pluginTask && !durableAuthority && stagedAttachment === null + && row.run.runnerProfileJson?.nativePreparationFencingVersion === NATIVE_PREPARATION_FENCING_VERSION + && row.coordinator.phase === "retryable_failure" && row.run.status === "running" && !row.coordinator.leaseOwner + && row.coordinator.failureCode === NATIVE_AGENT_DIRECTORY_BUSY_WAIT + && settledFailure?.recoveryMode === "bootstrap_retry" + && settledFailure?.providerSessionEstablished === false && settledFailure?.providerEventsExist === false + && !settledFailure?.successorRunId + && bootstrapRestartWatermark !== null + && providerEvents.length === 0 && (!hasCheckpoint || continuedSessionOnly) + && (inheritedRunnerIdentity || (!row.run.processPid && !row.run.processGroupId && !row.run.processStartedAt)) + && (!row.run.sessionIdBefore || continuedSessionOnly) && !row.run.sessionIdAfter + && Boolean(row.run.nativeSessionId && row.run.runnerInstanceId) + && !(await nativeExecutionStartedSince(tx, row.run.id, bootstrapRestartWatermark)) + && !(await hasNativeRunEvent(tx, row.run, PROCESS_IDENTITY_RECORDED)); + if ((remoteRunnerNeverLaunched || remoteRunnerExitedBeforeProviderWork || remoteFollowUpNeverAttached + || remoteBootstrapRestartDeferredBeforeLaunch) && row.coordinator.phase === "retryable_failure" && !admission.exhausted && !await readNativeSessionResumptionAgent(tx as unknown as Db, { issue, run: row.run, coordinator: row.coordinator })) { const deferred = await deferRemoteRetry("native_retry_admission_ineligible", ["done", "cancelled"].includes(issue.status)); @@ -1809,11 +1833,13 @@ export async function claimNativeRestartRecoveries(input: { remoteSandbox: remoteSandbox || Boolean(pluginTask && durableAuthority) || portablePreparation, unclaimedBootstrap: unclaimedBootstrap || portablePreparation && row.coordinator.attempt === 0, remoteBootstrapProvenIncomplete: remoteBootstrapProvenIncomplete || remoteRunnerNeverLaunched - || remoteRunnerExitedBeforeProviderWork || remoteRunnerNeverReceivedCommands || remoteFollowUpNeverAttached || portablePreparation, + || remoteRunnerExitedBeforeProviderWork || remoteRunnerNeverReceivedCommands || remoteFollowUpNeverAttached || portablePreparation + || remoteBootstrapRestartDeferredBeforeLaunch, remoteBootstrapIncompleteReason: remoteRunnerNeverLaunched || portablePreparation ? "remote_runner_never_launched" : remoteRunnerExitedBeforeProviderWork ? "remote_runner_exited_before_provider_work" : remoteRunnerNeverReceivedCommands ? "remote_runner_never_received_commands" : remoteFollowUpNeverAttached ? "remote_follow_up_never_attached" + : remoteBootstrapRestartDeferredBeforeLaunch ? "remote_bootstrap_restart_deferred_before_launch" : remoteBootstrapProof === "session_opened_without_turn" ? "remote_session_opened_without_turn" : undefined, runnerPidAlive, runnerGroupAlive, @@ -1822,7 +1848,8 @@ export async function claimNativeRestartRecoveries(input: { knownProviderProcessIdentityAmbiguous: providerProcesses.ambiguousLivePids.length > 0, hasCheckpoint: hasCheckpoint && !portablePreparation && !remoteRunnerNeverReceivedCommands && !(unclaimedBootstrap && inheritedCheckpointOnly) - && !remoteRunnerNeverLaunched && !remoteFollowUpNeverAttached && !(remoteBootstrapProvenIncomplete && reboundSessionOnly), + && !remoteRunnerNeverLaunched && !remoteFollowUpNeverAttached && !(remoteBootstrapProvenIncomplete && reboundSessionOnly) + && !remoteBootstrapRestartDeferredBeforeLaunch, stagedAttachmentVerified: stagedAttachment !== null, checkpointIdentityMatches: checkpointIdentityMatches && diff --git a/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts b/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts index 12f75658633..150cccd0938 100644 --- a/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts +++ b/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts @@ -3147,6 +3147,52 @@ process.on("SIGTERM", () => child.kill("SIGTERM")); expect(coordinator.recoveryHistory.at(-1)).toMatchObject({ reason: "remote_runner_never_received_commands" }); }); + it.each([ + { label: "restarts it again", evidence: "none" }, + { label: "keeps a runner process recorded during the wait held", evidence: "native.process_identity_recorded" }, + { label: "keeps a provider event recorded during the wait held", evidence: "turn.started" }, + { label: "keeps a checkpoint saved during the wait held", evidence: "checkpoint" }, + ] as const)("$label when a restarted remote bootstrap waited for a busy agent directory", async ({ evidence }) => { + const fixture = await seedUnlaunchedRemoteClaim(`BUSY-DIRECTORY-BOOTSTRAP-${evidence}`, + { event: { eventType: "native.process_start_requested" } }); + const [claim] = await claimNativeRestartRecoveries({ db: fixture.db, controller: successor, + restartKind: "hard", runIds: [fixture.runId] }); + expect(claim).toMatchObject({ kind: "bootstrap_incomplete", runId: fixture.runId, controllerGeneration: 2 }); + if (claim?.kind !== "bootstrap_incomplete") throw new Error("restart recovery did not restart the bootstrap"); + const schedule = vi.fn(); + const deferral = await prepareWithAgentDirectoryDeferral(async () => { throw new AgentDirectoryOperationBusyError(); }) + .catch(error => error); + expect(await deferRunForBusyAgentDirectory(fixture.db, { run: { id: fixture.runId, companyId }, error: deferral, + restartRecovery: claim, schedule, record: async () => undefined, random: () => 0 })).toBe("deferred"); + const [, retryAt, mode] = schedule.mock.calls[0]! as [string, Date, string | undefined]; + expect(mode).toBe("restart"); + if (evidence === "checkpoint") { + const [seeded] = await fixture.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, fixture.runId)); + await fixture.db.update(heartbeatRuns).set({ runnerProfileJson: { ...seeded.runnerProfileJson, sessionCheckpoint: { + identity: { companyId, issueId: fixture.issueId, runId: fixture.runId, agentId, sessionId: fixture.native.normalizedSessionId }, + providerSessionId: "provider-thread", activeTurnId: "turn-1", cursor: null, terminal: null, terminalTurns: [], + semanticResult: null, pendingRuntimeRequests: [], + } } }).where(eq(heartbeatRuns.id, fixture.runId)); + } else if (evidence !== "none") { + await fixture.db.insert(heartbeatRunEvents).values({ companyId, runId: fixture.runId, agentId, seq: 950, + stream: "system", level: "info", eventType: evidence }); + } + const [resumed] = await claimNativeRestartRecoveries({ db: fixture.db, controller: successor, + restartKind: "hard", runIds: [fixture.runId], now: retryAt }); + if (evidence !== "none") { + expect(resumed).not.toMatchObject({ kind: "bootstrap_incomplete" }); + return; + } + expect(resumed).toMatchObject({ kind: "bootstrap_incomplete", runId: fixture.runId, controllerGeneration: 3 }); + if (resumed?.kind !== "bootstrap_incomplete") throw new Error("deferred restart recovery did not restart the bootstrap"); + const [coordinator] = await fixture.db.select().from(nativeRunFinalizations) + .where(eq(nativeRunFinalizations.runId, fixture.runId)); + expect(coordinator).toMatchObject({ phase: "observed", leaseOwner: resumed.leaseOwner }); + expect(coordinator.recoveryHistory.at(-1)).toMatchObject({ reason: "remote_bootstrap_restart_deferred_before_launch" }); + const [run] = await fixture.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, fixture.runId)); + expect(run).toMatchObject({ status: "running", errorCode: null }); + }); + // Daytona waves s9202 t02 (B1) and s9205 t04 (B2): a follow-up run on a // continued session was killed after native.process_start_requested but // before its remote runner launched. Admission rebinds the previous turn's