diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 8e463045596..c0011b7afc0 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -434,6 +434,8 @@ import { nativeSessionWorkspaceScope, reserveWarmNativeInstructionDirectory, closeIdleWarmNativeSessionsForRestart, + WARM_IDLE_CLOSE_MAX_RETRIES, + WARM_IDLE_CLOSE_RETRY_MS, readNativeSessionIdleWork, createGovernedWaitEventObservation, createRemoteRunnerProcessLauncher, @@ -11614,10 +11616,12 @@ describe("native warm session supervision", () => { }); it.each([ - { lease: "pending_cleanup", forgotten: true }, - { lease: "released", forgotten: true }, + { lease: "stopped-receipt", forgotten: true }, + { lease: "destroyed-receipt", forgotten: true }, + { lease: "pending_cleanup", forgotten: false }, + { lease: "released", forgotten: false }, { lease: "retained", forgotten: false }, - ])("forgets a warm owner whose idle close failed only once the host gave up its sandbox: $lease", async ({ lease, forgotten }) => { + ])("forgets a warm owner whose idle close failed only once its sandbox has a stop or destroy receipt: $lease", async ({ lease, forgotten }) => { const identity = `warm-idle-close-sandbox-${lease}`; const warmExecution = { ...execution, binding: { ...execution.binding, runId: identity, executionWorkspaceId: `${identity}-workspace` }, session: { ...execution.session, normalizedSessionId: identity, lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 50 } }, @@ -11632,7 +11636,7 @@ describe("native warm session supervision", () => { }); const copy = (runId: string) => ({ runId, root: `/tmp/${identity}`, preparationKey: "same", hasChanges: async () => false, collectStopped: async () => undefined, retirementFailed: async () => undefined }); - let leaseRows: Array<{ status: string }> = []; + let leaseRows: Array> = []; const base = leaseDb(warmExecution); const db = { ...base, select: (fields?: unknown) => ({ from: (table: unknown) => table === environmentLeases ? { where: async () => leaseRows } : base.select(fields as never).from(table as never) }) } as unknown as Db; @@ -11641,7 +11645,14 @@ describe("native warm session supervision", () => { runnerExecutionTarget: { kind: "remote", transport: "sandbox", environmentId: identity, remoteCwd: `/tmp/${identity}`, sandboxLeaseAcquisition: { providerLeaseId: `${identity}-sandbox` } } as never, instructionWorkingCopy: copy(identity) }); - leaseRows = [{ status: lease }]; + const receipt = lease.endsWith("-receipt") ? lease.slice(0, -"-receipt".length) : null; + leaseRows = [{ id: `${identity}-lease`, companyId: warmExecution.binding.companyId, environmentId: identity, + heartbeatRunId: identity, provider: "daytona", providerLeaseId: `${identity}-sandbox`, + status: receipt ? "released" : lease, releasedAt: lease === "retained" ? null : new Date(), + cleanupStatus: receipt || lease === "released" ? "success" : "failed", leasePolicy: "reuse_by_environment", + metadata: receipt ? { remoteExecutionTermination: { schema: "paperclip.remote-termination.v1", + companyId: warmExecution.binding.companyId, runId: identity, leaseId: `${identity}-lease`, provider: "daytona", + providerLeaseId: `${identity}-sandbox`, state: receipt, confirmedAt: new Date().toISOString() } } : {} }]; await vi.waitFor(() => expect(session.close).toHaveBeenCalledTimes(1)); const followUpRunId = `${identity}-follow-up`; const adopt = vi.fn(async () => null); @@ -12197,6 +12208,79 @@ describe("native warm session supervision", () => { expect(collectStopped).toHaveBeenCalledOnce(); }); + it("an idle warm session whose close fails with runner_remote_process_identity_unavailable and whose Runner is proven gone is released at controller restart", async () => { + const name = "warm-close-runner-gone"; + const current = { ...execution, + binding: { ...execution.binding, runId: name, executionWorkspaceId: name }, + session: { ...execution.session, normalizedSessionId: name, lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 60_000 } }, + } as NativeExecutionInputV1; + const close = vi.fn<() => Promise>(async () => { + throw new RemoteRunnerRecoveryError({ stage: "process_verification", reason: "process_not_running" }); + }); + state.execute.mockReset().mockImplementationOnce(async options => { + await options.onSession?.({ close }); + return { result: { summary: "completed" }, terminal: { runTerminalState: "succeeded" }, + turnId: name, normalizedSessionId: name, providerSessionId: name, driverKind: "test", driverVersion: "1", + nativeEventCount: 1, highestContiguousSourceSeq: 1, usage: null }; + }); + await expect(executePaperclipNativeSession({ db: leaseDb(current), execution: current, runnerInstanceId: name })) + .resolves.toMatchObject({ exitCode: 0 }); + expect(close).not.toHaveBeenCalled(); + expect(await closeIdleWarmNativeSessionsForRestart()).toEqual({ closed: 1, busy: 0, failed: 0, failures: [] }); + expect(close).toHaveBeenCalledOnce(); + expect(await closeIdleWarmNativeSessionsForRestart()).toEqual({ closed: 0, busy: 0, failed: 0, failures: [] }); + }); + + it("names the Runner check in a warm close failure and keeps an unproven owner registered at controller restart", async () => { + const name = "warm-close-marker-missing"; + const current = { ...execution, + binding: { ...execution.binding, runId: name, executionWorkspaceId: name }, + session: { ...execution.session, normalizedSessionId: name, lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 60_000 } }, + } as NativeExecutionInputV1; + const close = vi.fn<() => Promise>(async () => { + throw new RemoteRunnerRecoveryError({ stage: "process_verification", reason: "marker_missing" }); + }); + state.execute.mockReset().mockImplementationOnce(async options => { + await options.onSession?.({ close }); + return { result: { summary: "completed" }, terminal: { runTerminalState: "succeeded" }, + turnId: name, normalizedSessionId: name, providerSessionId: name, driverKind: "test", driverVersion: "1", + nativeEventCount: 1, highestContiguousSourceSeq: 1, usage: null }; + }); + await executePaperclipNativeSession({ db: leaseDb(current), execution: current, runnerInstanceId: name }); + expect(await closeIdleWarmNativeSessionsForRestart()).toMatchObject({ closed: 0, failed: 1, failures: [{ runId: name, + reason: "close failed: runner_remote_process_identity_unavailable: runner_remote_process_identity_unavailable [process_verification/marker_missing]" }] }); + close.mockResolvedValue(undefined); + expect(await closeIdleWarmNativeSessionsForRestart()).toMatchObject({ closed: 1, failed: 0 }); + }); + + it("retries a failed idle warm close a bounded number of times", async () => { + const name = "warm-idle-close-retry"; + const current = { ...execution, + binding: { ...execution.binding, runId: name, executionWorkspaceId: name }, + session: { ...execution.session, normalizedSessionId: name, lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 20 } }, + } as NativeExecutionInputV1; + const close = vi.fn<() => Promise>(async () => { throw new Error("native_runner_checkpoint_owner_lost"); }); + state.execute.mockReset().mockImplementationOnce(async options => { + await options.onSession?.({ close }); + return { result: { summary: "completed" }, terminal: { runTerminalState: "succeeded" }, + turnId: name, normalizedSessionId: name, providerSessionId: name, driverKind: "test", driverVersion: "1", + nativeEventCount: 1, highestContiguousSourceSeq: 1, usage: null }; + }); + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + try { + await executePaperclipNativeSession({ db: leaseDb(current), execution: current, runnerInstanceId: name }); + await vi.advanceTimersByTimeAsync(20); + expect(close).toHaveBeenCalledTimes(1); + await vi.advanceTimersByTimeAsync(WARM_IDLE_CLOSE_RETRY_MS); + expect(close).toHaveBeenCalledTimes(2); + await vi.advanceTimersByTimeAsync(WARM_IDLE_CLOSE_RETRY_MS * 10); + expect(close).toHaveBeenCalledTimes(1 + WARM_IDLE_CLOSE_MAX_RETRIES); + expect(readNativeSessionIdleWork()).toBe("present"); + } finally { vi.useRealTimers(); } + close.mockResolvedValue(undefined); + expect(await closeIdleWarmNativeSessionsForRestart()).toMatchObject({ closed: 1, failed: 0 }); + }); + it.each(["failed", "cancelled"] as const)("retires a %s session even if its warm checkpoint rejects", async (runTerminalState) => { const name = `warm-checkpoint-rejects-${runTerminalState}`; const current = { ...execution, @@ -13245,6 +13329,48 @@ describe("native warm session supervision", () => { }, ); + it.each([ + { lease: "released-stop-receipt", closed: 1 }, + { lease: "pending_cleanup", closed: 0 }, + { lease: "released-without-receipt", closed: 0 }, + { lease: "stop-receipt-for-another-run", closed: 0 }, + ] as const)("reads the warm owner's own lease rows at controller restart: $lease", async ({ lease: kind, closed }) => { + const { base } = fixture(`restart-lease-${kind}`); + const environmentId = `restart-lease-env-${kind}`, providerLeaseId = `restart-lease-sandbox-${kind}`; + const close = vi.fn(async (): Promise => { throw new Error("owned close not yet confirmed"); }); + state.execute.mockReset().mockImplementationOnce(async options => { + await options.onSession?.({ close }); return result; + }); + const leaseRunId = kind === "stop-receipt-for-another-run" ? "earlier-run" : base.binding.runId; + const terminal = kind !== "pending_cleanup"; + const row = { id: `lease-${kind}`, companyId: base.binding.companyId, environmentId, heartbeatRunId: leaseRunId, + provider: "daytona", providerLeaseId, status: terminal ? "released" : "pending_cleanup", releasedAt: new Date(), + cleanupStatus: terminal ? "success" : "failed", leasePolicy: "reuse_by_environment", + metadata: kind === "released-stop-receipt" || kind === "stop-receipt-for-another-run" + ? { remoteExecutionTermination: { schema: "paperclip.remote-termination.v1", companyId: base.binding.companyId, + runId: leaseRunId, leaseId: `lease-${kind}`, provider: "daytona", providerLeaseId, state: "stopped", + confirmedAt: new Date().toISOString() } } + : { failureReason: "sandbox_stop_pending" } }; + let leaseRows: unknown[] = []; + const ownerDb = leaseDb(base), select = ownerDb.select.bind(ownerDb); + ownerDb.select = ((fields?: Record) => ({ from: (table: unknown) => { + if (table !== environmentLeases) return select(fields as never).from(table as never); + const query = { then: (...args: Parameters["then"]>) => Promise.resolve(leaseRows).then(...args), + where: () => query, limit: () => query }; + return query; + } })) as unknown as Db["select"]; + try { + await executePaperclipNativeSession({ db: ownerDb, execution: base, runnerInstanceId: "runner", + runnerExecutionTarget: { kind: "remote", transport: "sandbox", environmentId, remoteCwd: `/tmp/${environmentId}`, + sandboxLeaseAcquisition: { outcome: "created", providerLeaseId } } }); + leaseRows = [row]; + expect(await closeIdleWarmNativeSessionsForRestart()).toMatchObject({ closed, failed: 1 - closed }); + } finally { + close.mockResolvedValue(undefined); + await closeIdleWarmNativeSessionsForRestart(); + } + }); + it("retains failed retirement for retry and never launches over an uncontained owner", async () => { const { base, next, run } = fixture("identity-close-failure"); const close = vi.fn().mockRejectedValueOnce(new Error("containment unavailable")).mockResolvedValue(undefined); diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index cffe9cd56c0..26badf5ba56 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -743,12 +743,15 @@ type WarmNativeSession = { controllerDetachedForRestart?: boolean; finishRelease?: () => Promise; closeOnReleaseReason?: string; + idleCloseRetries?: number; idleTimer: ReturnType | null; lastActivityAt: string; }; const WARM_CLOSE_FAILURE_DETAIL_MAX = 2_000; const WARM_NATIVE_SESSION_CLOSE_CONCURRENCY = 4; +export const WARM_IDLE_CLOSE_RETRY_MS = 5_000; +export const WARM_IDLE_CLOSE_MAX_RETRIES = 3; export function describeWarmCloseFailure(error: unknown): string { const describe = (value: unknown): string => { @@ -757,8 +760,10 @@ export function describeWarmCloseFailure(error: unknown): string { const code = "code" in value && typeof value.code === "string" ? `${value.code}: ` : ""; const settlement = value instanceof NativeSessionCloseUnrecoverableError && value.settlement ? ` ${JSON.stringify(value.settlement)}` : ""; + const remote = readRemoteRunnerRecoveryError(value); + const recovery = remote ? ` [${remote.stage}/${remote.reason}${remote.runnerLifecycle ? `/${remote.runnerLifecycle}` : ""}]` : ""; const cause = value.cause === undefined ? "" : ` (cause: ${describe(value.cause)})`; - return `${code}${value.message}${settlement}${cause}`; + return `${code}${value.message}${recovery}${settlement}${cause}`; }; const text = describe(error).replaceAll(/[\r\n]+/g, " "); return text.length > WARM_CLOSE_FAILURE_DETAIL_MAX ? `${text.slice(0, WARM_CLOSE_FAILURE_DETAIL_MAX)}…` : text; @@ -1107,17 +1112,7 @@ export async function closeWarmNativeSessionsForEnvironment(input: { return closeIdleWarmNativeSessions(input); } -async function warmSandboxGivenUpByHost(entry: WarmNativeSession): Promise { - if (!entry.sandboxLeaseDb || !entry.providerLeaseId || !entry.environmentId) return false; - const leases = await entry.sandboxLeaseDb.select({ status: environmentLeases.status }).from(environmentLeases).where(and( - eq(environmentLeases.companyId, entry.companyId), - eq(environmentLeases.environmentId, entry.environmentId), - eq(environmentLeases.providerLeaseId, entry.providerLeaseId), - )); - return leases.length > 0 && leases.every(lease => lease.status !== "active" && lease.status !== "retained"); -} - -async function warmSandboxDestroyed(db: Db, entry: WarmNativeSession): Promise { +async function warmSandboxTerminated(db: Db, entry: WarmNativeSession): Promise { if (!entry.providerLeaseId || !entry.environmentId) return false; const leases = await db.select().from(environmentLeases).where(and( eq(environmentLeases.companyId, entry.companyId), @@ -1125,8 +1120,23 @@ async function warmSandboxDestroyed(db: Db, entry: WarmNativeSession): Promise resolveRemoteTerminationLease(db, lease))); - return resolved.some(lease => hasRemoteTerminationReceipt(lease) - && (lease.metadata?.remoteExecutionTermination as { state?: unknown } | undefined)?.state === "destroyed"); + return resolved.some(lease => { + if (!hasRemoteTerminationReceipt(lease)) return false; + const state = (lease.metadata?.remoteExecutionTermination as { state?: unknown } | undefined)?.state; + return state === "destroyed" || (state === "stopped" && lease.heartbeatRunId === entry.runId); + }); +} + +/** A failed close is complete when its Runner is proven gone, its sandbox is destroyed, or the run's own sandbox lease has a stop receipt. */ +async function warmCloseFailureProvesOwnerGone(entry: WarmNativeSession, error: unknown, db = entry.sandboxLeaseDb): Promise { + const diagnostic = readRemoteRunnerRecoveryError(error); + if (diagnostic?.stage === "process_verification" && diagnostic.reason === "process_not_running") return true; + try { + return db ? await warmSandboxTerminated(db, entry) : false; + } catch (proofError) { + logger.warn({ err: proofError, runId: entry.runId }, "could not read the warm session's sandbox lease"); + return false; + } } /** Detach released owners and checkpoint unreleased owners before a controller @@ -1195,16 +1205,10 @@ async function closeIdleWarmNativeSessions(input: { busy += 1; return; } - const destroyed = !detach && input.forceStopUnsettled && input.db - ? await warmSandboxDestroyed(input.db, entry).catch(proofError => { - logger.warn({ err: proofError, runId: entry.runId }, "could not read the warm session's sandbox lease"); - return false; - }) - : false; - if (destroyed) { + if (!detach && await warmCloseFailureProvesOwnerGone(entry, error, input.db ?? entry.sandboxLeaseDb)) { logger.warn({ runId: entry.runId, issueId: entry.issueId, environmentId: entry.environmentId, providerLeaseId: entry.providerLeaseId ?? null, reason: input.reason, closeFailure: describeWarmCloseFailure(error) }, - "warm native session's sandbox was already destroyed; released its owner"); + "warm native session's Runner or sandbox is gone; released its owner"); forgetWarmNativeSession(sessionId, entry); closed += 1; return; @@ -8860,17 +8864,21 @@ function armWarmNativeSessionIdleTimer( return; } logger.warn({ runId: current.runId, issueId: current.issueId, environmentId: current.environmentId, - providerLeaseId: current.providerLeaseId ?? null, closeFailure: describeWarmCloseFailure(error) }, + providerLeaseId: current.providerLeaseId ?? null, closeFailure: describeWarmCloseFailure(error), + closeAttempt: (current.idleCloseRetries ?? 0) + 1 }, released ? "idle warm native session detach failed" : "idle warm native session close failed"); if (released) return; - return warmSandboxGivenUpByHost(current).then(givenUp => { - if (!givenUp) return; - logger.warn({ runId: current.runId, issueId: current.issueId, environmentId: current.environmentId, - providerLeaseId: current.providerLeaseId ?? null }, - "warm native session's sandbox was stopped by the host; released its owner"); - forgetWarmNativeSession(sessionId, current); - }, proofError => { - logger.warn({ err: proofError, runId: current.runId }, "could not read the warm session's sandbox lease"); + return warmCloseFailureProvesOwnerGone(current, error).then(gone => { + if (gone) { + logger.warn({ runId: current.runId, issueId: current.issueId, environmentId: current.environmentId, + providerLeaseId: current.providerLeaseId ?? null }, + "warm native session's Runner or sandbox is gone; released its owner"); + forgetWarmNativeSession(sessionId, current); + return; + } + if ((current.idleCloseRetries ?? 0) >= WARM_IDLE_CLOSE_MAX_RETRIES) return; + current.idleCloseRetries = (current.idleCloseRetries ?? 0) + 1; + armWarmNativeSessionIdleTimer(sessionId, current, WARM_IDLE_CLOSE_RETRY_MS); }); }) .finally(() => {