diff --git a/server/src/services/native-runtime/native-recovered-owner-followup.test.ts b/server/src/services/native-runtime/native-recovered-owner-followup.test.ts index e1887754d71..3e6af4c6106 100644 --- a/server/src/services/native-runtime/native-recovered-owner-followup.test.ts +++ b/server/src/services/native-runtime/native-recovered-owner-followup.test.ts @@ -670,6 +670,27 @@ describe.skipIf(!recoveryPostgresSupport.supported)("follow-up after a recovered expect(await archivedOwners(root)).toEqual([prior.runId]); }, 60_000); + it.each([ + { label: "ended on another sandbox", status: "expired", sameSandbox: false, recorded: true, outcome: ADMITTED }, + { label: "released on another sandbox", status: "released", sameSandbox: false, recorded: true, outcome: ADMITTED }, + { label: "still retained", status: "retained", sameSandbox: false, recorded: true, + outcome: stranded("journal_commands_unsettled(runner.suspend.pending)") }, + { label: "ended on this sandbox", status: "expired", sameSandbox: true, recorded: true, + outcome: stranded("journal_commands_unsettled(runner.suspend.pending)") }, + { label: "not recorded", status: "expired", sameSandbox: false, recorded: false, + outcome: stranded("journal_commands_unsettled(runner.suspend.pending)") }, + ])("retires a finished remote owner whose suspend never finished when its sandbox lease is $label", async ({ status, sameSandbox, recorded, outcome }) => { + const prior = await seedRecoveredPriorRun("pending-command"); + const followUp = await admitFollowUp(prior); + if (recorded) { + await db.insert(environments).values({ id: followUp.environmentId, name: `ended-${followUp.environmentId}`, driver: "sandbox" }); + await db.insert(environmentLeases).values({ companyId, environmentId: followUp.environmentId, issueId: prior.issueId, + heartbeatRunId: prior.runId, provider: "daytona", providerLeaseId: sameSandbox ? followUp.providerLeaseId : `sandbox-${prior.runId}`, + status, releasedAt: status === "retained" ? null : new Date(), metadata: { remoteCwd: "/workspace" } }); + } + expect(await followUp.start()).toBe(outcome); + }, 60_000); + it("admits again when a retry rebuilds the same projection from the database", async () => { const prior = await seedRecoveredPriorRun(); const followUp = await admitFollowUp(prior); diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 8512171085d..7a4bc5ed81f 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -5429,7 +5429,7 @@ type SettledRunnerdProjection = { bytes: string; refused?: never } | { bytes?: n const LOCAL_RUNNER_LIFECYCLE_COMMANDS = new Set(["runner.suspend", "runner.drain", "runner.shutdown"]); function readSettledRunnerdProjection(root: string, identity: Record, - options: { localRunner?: boolean; controlBytes?: string } = {}): SettledRunnerdProjection { + options: { localRunner?: boolean; controlBytes?: string; remoteRunnerGone?: boolean } = {}): SettledRunnerdProjection { const refuse = (refused: string): SettledRunnerdProjection => ({ refused }); try { if (!isSafeNativeStateDirectory(root)) return refuse("root_unsafe"); @@ -5449,7 +5449,7 @@ function readSettledRunnerdProjection(root: string, identity: Record !LOCAL_RUNNER_LIFECYCLE_COMMANDS.has(String(command.type)) || command.status === "completed") : commands; const settledControl = settledCommands === commands ? control : { ...control, commands: settledCommands }; @@ -5553,11 +5553,16 @@ async function inspectFinishedPerTurnRunnerdOwner(input: { let originalBytes: string; try { originalBytes = readBoundedNativeFile(statePath, NATIVE_CONTROL_PLANE_STATE_MAX_BYTES, "runner_durable_identity_too_large").toString("utf8"); } catch { return refuse("journal_unreadable"); } - const projectionOptions = { localRunner, controlBytes: input.projectedControlBytes }; + const { binding } = input.execution; + const remoteRunnerGone = !localRunner && await priorRemoteSandboxLeasesEnded(input.db, { + companyId: binding.companyId, runId: identity.runId, + currentProviderLeaseId: target?.kind === "remote" && target.transport === "sandbox" + ? target.sandboxLeaseAcquisition?.providerLeaseId : undefined, + }); + const projectionOptions = { localRunner, controlBytes: input.projectedControlBytes, remoteRunnerGone }; const projection = readSettledRunnerdProjection(input.root, identity, projectionOptions); if (projection.refused !== undefined) return refuse(projection.refused); const controlBytes = projection.bytes; - const { binding } = input.execution; let prior: { run: typeof heartbeatRuns.$inferSelect; owner: typeof nativeRunFinalizations.$inferSelect } | undefined; try { [prior] = await input.db.select({ run: heartbeatRuns, owner: nativeRunFinalizations }).from(heartbeatRuns) @@ -5619,7 +5624,7 @@ async function inspectFinishedPerTurnRunnerdOwner(input: { for (const name of readdirSync(quarantine)) { if (!name.startsWith(`${basename(input.root)}.`)) continue; if (quarantinedBootstrapOfOwnerHeldNoProviderWork(resolve(quarantine, name), identity, localRunner)) continue; - const copy = readSettledRunnerdProjection(resolve(quarantine, name), identity, { localRunner }); + const copy = readSettledRunnerdProjection(resolve(quarantine, name), identity, { localRunner, remoteRunnerGone }); if (copy.refused !== undefined) return refuse(`quarantine_${copy.refused}`); } } @@ -5643,6 +5648,20 @@ export function quarantinedBootstrapOfOwnerHeldNoProviderWork(root: string, iden || (!localRunner && runnerdStateProvesNoRemoteProviderTurn(root)); } +async function priorRemoteSandboxLeasesEnded(db: Db, input: { + companyId: string; runId: string; currentProviderLeaseId?: string; +}): Promise { + try { + const leases = await db.select({ status: environmentLeases.status, releasedAt: environmentLeases.releasedAt, + providerLeaseId: environmentLeases.providerLeaseId }).from(environmentLeases) + .where(and(eq(environmentLeases.companyId, input.companyId), eq(environmentLeases.heartbeatRunId, input.runId))); + return leases.length > 0 && leases.every(lease => (lease.status === "released" || lease.status === "expired") + && lease.releasedAt !== null && lease.providerLeaseId !== input.currentProviderLeaseId); + } catch { + return false; + } +} + async function retireFinishedPerTurnRunnerdOwner(input: Parameters[0]) : Promise<{ retired: true } | { retired: false; refused: string }> { const proof = await inspectFinishedPerTurnRunnerdOwner(input);