diff --git a/doc/plugins/PLUGIN_RUNNER_EXECUTION.md b/doc/plugins/PLUGIN_RUNNER_EXECUTION.md index afc0704453b..2fb9c87b637 100644 --- a/doc/plugins/PLUGIN_RUNNER_EXECUTION.md +++ b/doc/plugins/PLUGIN_RUNNER_EXECUTION.md @@ -390,6 +390,17 @@ collection, another close error from an owner without a task, or continuation of the same session still blocks replacement. A controller restart is not required for this stopped-owner reset path. +Deleting an environment (with consent to destroy its reusable sandboxes) closes its +idle warm owners first. An idle owner whose close cannot settle is retired the same +way: one confirmed `stop`, the credential snapshot, `complete`, then instruction +collection. An owner whose sandbox lease already carries a `destroyed` termination +receipt has no process left to stop, so Core releases it without contacting the +sandbox. Only then are the environment's sandboxes destroyed, through the usual +workspace-export and restore-source guards. When an owner still cannot be retired, +the delete returns 409 and names each run and the close and stop errors. Idle-timeout, +restart and sleep sweeps do not force a stop. They log the exact close error for each +owner instead of a count. + When retiring the previous owner stays unproven anyway (any close failure, on any of the paths a new run uses to retire it: instruction preparation, configuration change, session reset, or a prior-owner fence on a resumed diff --git a/server/src/__tests__/environment-routes.test.ts b/server/src/__tests__/environment-routes.test.ts index 235621dc517..1cf1f17f089 100644 --- a/server/src/__tests__/environment-routes.test.ts +++ b/server/src/__tests__/environment-routes.test.ts @@ -31,7 +31,8 @@ const mockEnvironmentRuntimeService = vi.hoisted(() => ({ releaseRestoreSourcesForEnvironmentDelete: vi.fn(async () => 0), })); const mockCloseWarmNativeSessionsForEnvironment = vi.hoisted(() => - vi.fn(async () => ({ closed: 0, busy: 0, failed: 0 })), + vi.fn(async (): Promise<{ closed: number; busy: number; failed: number; failures: Array> }> => + ({ closed: 0, busy: 0, failed: 0, failures: [] })), ); const mockInstanceSettingsService = vi.hoisted(() => ({ get: vi.fn(async () => ({ defaultEnvironmentId: null })), @@ -359,6 +360,7 @@ describe("environment routes", () => { closed: 0, busy: 0, failed: 0, + failures: [], }); mockInstanceSettingsService.listCompanyIds.mockReset(); mockInstanceSettingsService.getGeneral.mockReset(); @@ -1823,6 +1825,8 @@ describe("environment routes", () => { ).toHaveBeenCalledExactlyOnceWith({ environmentId: "env-1", reason: "environment deleted", + forceStopUnsettled: true, + db: expect.anything(), }); expect(mockEnvironmentService.removeIfDeletable).toHaveBeenCalledWith( "env-1", @@ -1830,6 +1834,24 @@ describe("environment routes", () => { expect(res.body.destroyedReusableSandboxLeaseCount).toBe(2); }); + it("names the warm runner session that could not be stopped and why", async () => { + const environment = createEnvironment(); + mockEnvironmentService.getById.mockResolvedValue(environment); + mockEnvironmentService.getDeleteBlastRadius.mockResolvedValue(createDeleteBlastRadius({ reusableSandboxLeaseCount: 1 })); + const failure = { runId: "run-parked", issueId: "issue-1", environmentId: "env-1", providerLeaseId: "sandbox-1", + reason: "close failed: native_session_close_unrecoverable: runner did not durably suspend; forced Runner stop failed: handoff unconfirmed" }; + mockCloseWarmNativeSessionsForEnvironment.mockResolvedValue({ closed: 0, busy: 0, failed: 1, failures: [failure] }); + const app = createApp(localBoardActor("admin-1")); + + const res = await request(app).delete("/api/environments/env-1?destroyReusableSandboxLeases=true"); + + expect(res.status).toBe(409); + expect(res.body.error).toContain(`run run-parked: ${failure.reason}`); + expect(res.body.details.nativeRunnerSessions.failures).toEqual([failure]); + expect(mockEnvironmentRuntimeService.destroyReusableSandboxLeasesForEnvironment).not.toHaveBeenCalled(); + expect(mockEnvironmentService.removeIfDeletable).not.toHaveBeenCalled(); + }); + it("keeps rejecting a consented delete when a non-lease blocker remains", async () => { const environment = createEnvironment(); mockEnvironmentService.getById.mockResolvedValue(environment); diff --git a/server/src/routes/environments.ts b/server/src/routes/environments.ts index 506fe19ee8f..10bbbf54610 100644 --- a/server/src/routes/environments.ts +++ b/server/src/routes/environments.ts @@ -1387,12 +1387,15 @@ export function environmentRoutes( const warmSessions = await closeWarmNativeSessionsForEnvironment({ environmentId: existing.id, reason: "environment deleted", + forceStopUnsettled: true, + db, }); if (warmSessions.busy > 0 || warmSessions.failed > 0) { throw conflict( warmSessions.busy > 0 ? "Cannot delete this environment while a native runner session is active. Wait for its run to finish, then retry." - : "Cannot delete this environment because its warm native runner did not shut down cleanly. Retry after the runner exits.", + : `Cannot delete this environment because its warm native runner could not be stopped (${warmSessions.failures + .map((failure) => `run ${failure.runId}: ${failure.reason}`).join("; ")}). Retry after the runner exits.`, { nativeRunnerSessions: warmSessions }, ); } diff --git a/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts b/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts index b67a49363e6..5b1251bb68d 100644 --- a/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts +++ b/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts @@ -614,7 +614,7 @@ process.on("SIGTERM", () => child.kill("SIGTERM")); const first = hardRestart ? await executeInController(firstOptions) : await executeFirst(); expect(first.exitCode).toBe(0); if (scenario === "warm-suspended") expect(await closeIdleWarmNativeSessionsForRestart()) - .toEqual({ closed: 1, busy: 0, failed: 0 }); + .toEqual({ closed: 1, busy: 0, failed: 0, failures: [] }); await finalizeNativeRun({ db: fixture.db, runId: fixture.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true }); const snapshot = await runner.execute({ command: "sh", args: ["-c", `cat '${remoteCwd}'/.paperclip-runtime/paperclip-runner/sessions/*/runner/runner-state.json`] }); expect(snapshot.exitCode).toBe(0); diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 9f3ce96b9d1..f798686a118 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -11433,7 +11433,7 @@ describe("native warm session supervision", () => { expect(collectStopped).not.toHaveBeenCalled(); expect(readNativeSessionIdleWork()).toBe("present"); close.mockResolvedValue(undefined); - expect(await closeIdleWarmNativeSessionsForRestart()).toEqual({ closed: 1, busy: 0, failed: 0 }); + expect(await closeIdleWarmNativeSessionsForRestart()).toEqual({ closed: 1, busy: 0, failed: 0, failures: [] }); expect(collectStopped).toHaveBeenCalledOnce(); }); @@ -11459,7 +11459,7 @@ describe("native warm session supervision", () => { expect(close).toHaveBeenCalledOnce(); expect(collectStopped).toHaveBeenCalledOnce(); expect(close.mock.invocationCallOrder[0]).toBeLessThan(collectStopped.mock.invocationCallOrder[0]!); - expect(await closeIdleWarmNativeSessionsForRestart()).toEqual({ closed: 0, busy: 0, failed: 0 }); + expect(await closeIdleWarmNativeSessionsForRestart()).toEqual({ closed: 0, busy: 0, failed: 0, failures: [] }); }); it.each(["failed", "cancelled"] as const)("collects edits written during %s provider shutdown after the warm checkpoint", async (runTerminalState) => { @@ -11716,7 +11716,7 @@ describe("native warm session supervision", () => { releaseClose(); close.mockResolvedValue(undefined); expect(await closeWarmNativeSessionsForEnvironment({ environmentId: identity, reason: "test cleanup" })) - .toEqual({ closed: changed && !closeFails ? 0 : 1, busy: 0, failed: 0 }); + .toEqual({ closed: changed && !closeFails ? 0 : 1, busy: 0, failed: 0, failures: [] }); expect(collectStopped).toHaveBeenCalledOnce(); } }); @@ -12292,6 +12292,122 @@ describe("native warm session supervision", () => { }, ); + it.each(["contained", "handoff-unconfirmed"] as const)( + "stops an idle warm owner whose close cannot settle when its environment is deleted (%s)", async mode => { + const { base } = fixture(`idle-unsettled-${mode}`); + const environmentId = `idle-unsettled-env-${mode}`; + const identity = { runnerInstanceId: "runner", environmentLeaseId: "lease", runId: base.binding.runId, + normalizedSessionId: base.session.normalizedSessionId!, turnId: "turn", itemId: "item" }; + let stopped = false; + const task = vi.fn(async (operation: any) => { + if (operation.kind === "describe") return { kind: "description", taskId: "parked-task", launchPreparationVersion: 1, + projectAuthCheckVersion: 1, runner: { version: "fixture", digest: `sha256:${"a".repeat(64)}` } }; + if (operation.kind === "stop") stopped = true; + if (operation.kind === "status") return { kind: "status", taskId: "parked-task", phase: stopped ? "cancelled" : "running", executionStopped: stopped }; + if (operation.kind === "complete" && mode === "handoff-unconfirmed") throw new Error("handoff unconfirmed"); + return { kind: "accepted", taskId: "parked-task" }; + }); + const createOwner = pluginTaskRuntime.createPluginRunnerTaskOwner; + let owner!: ReturnType; + const ownerFactory = vi.spyOn(pluginTaskRuntime, "createPluginRunnerTaskOwner").mockImplementation(input => { + owner = createOwner({ ...input, adoptIdentity: identity }); + return owner; + }); + onTestFinished(() => ownerFactory.mockRestore()); + const backend = await createRunnerdBackend({ db: leaseDb(base), + execution: { ...base, session: { ...base.session, lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } } }, + runnerInstanceId: "runner", runnerIngressAuthorized: true, + runnerExecutionTarget: { kind: "remote", transport: "sandbox", remoteCwd: ".", leaseId: "lease", runnerTask: task } }); + const registration = await owner.registration({} as never, base.binding.runId); + const failure = new NativeSessionCloseUnrecoverableError({ runnerSuspended: false, providerDrained: true, + semanticTools: { pending: [{ callId: "question-call" }] } }, false); + let closePromise: Promise | undefined; + const close = vi.fn(() => closePromise ??= (async () => { + await registration.release(); + await registration.checkpoint("unsettled").catch(() => undefined); + throw failure; + })()); + const session = { close }; + backend.bindManagedSession(session as never); + const collectStopped = vi.fn(async () => { expect(owner.isRetired()).toBe(true); }); + const retirementFailed = vi.fn(async () => {}); + state.execute.mockReset().mockImplementationOnce(async options => { + await options.onSession?.(session); return result; + }); + try { + await executePaperclipNativeSession({ db: leaseDb(base), execution: base, runnerInstanceId: "runner", + instructionWorkingCopy: { hasChanges: async () => false, collectStopped, retirementFailed }, + runnerExecutionTarget: { kind: "remote", transport: "sandbox", environmentId, remoteCwd: `/tmp/${environmentId}`, + sandboxLeaseAcquisition: { outcome: "created", providerLeaseId: "parked-sandbox" } } }); + const graceful = await closeWarmNativeSessionsForEnvironment({ environmentId, reason: "idle sweep" }); + expect(graceful).toMatchObject({ closed: 0, failed: 1, failures: [{ runId: base.binding.runId, + providerLeaseId: "parked-sandbox", environmentId }] }); + expect(graceful.failures[0]!.reason).toContain("native_session_close_unrecoverable: provider_transport_failed: runner did not durably suspend before checkpoint"); + expect(graceful.failures[0]!.reason).toContain("question-call"); + expect(collectStopped).not.toHaveBeenCalled(); + const deleted = await closeWarmNativeSessionsForEnvironment({ environmentId, reason: "environment deleted", forceStopUnsettled: true }); + expect(task.mock.calls.filter(([operation]) => operation.kind === "stop")).toHaveLength(1); + expect(task.mock.calls.some(([operation]) => operation.kind === "read_state")).toBe(false); + if (mode === "contained") { + expect(deleted).toEqual({ closed: 1, busy: 0, failed: 0, failures: [] }); + expect(collectStopped).toHaveBeenCalledOnce(); + expect(await closeWarmNativeSessionsForEnvironment({ environmentId, reason: "environment deleted", forceStopUnsettled: true })) + .toEqual({ closed: 0, busy: 0, failed: 0, failures: [] }); + } else { + expect(deleted).toMatchObject({ closed: 0, failed: 1, failures: [{ runId: base.binding.runId }] }); + expect(deleted.failures[0]!.reason).toContain("runner did not durably suspend before checkpoint"); + expect(deleted.failures[0]!.reason).toContain("forced Runner stop failed: handoff unconfirmed"); + expect(collectStopped).not.toHaveBeenCalled(); + } + } finally { + close.mockResolvedValue(undefined); + collectStopped.mockResolvedValue(undefined); + expect(await closeWarmNativeSessionsForEnvironment({ environmentId, reason: "test cleanup" })).toMatchObject({ failed: 0 }); + ownerFactory.mockRestore(); + } + }, + ); + + it.each(["destroyed", "stopped", "unproven"] as const)( + "releases an idle warm owner whose sandbox lease is gone when its environment is deleted (%s)", async receipt => { + const { base } = fixture(`idle-gone-${receipt}`); + const environmentId = `idle-gone-env-${receipt}`; + const inactive = new Error("Daytona sandbox lease gone-sandbox is no longer active."); + const close = vi.fn(async (): Promise => { throw inactive; }); + const retirementFailed = vi.fn(async () => {}); + state.execute.mockReset().mockImplementationOnce(async options => { + await options.onSession?.({ close }); return result; + }); + const lease = { id: "lease-gone", companyId: base.binding.companyId, environmentId, heartbeatRunId: "pending-cleanup-run", + provider: "daytona", providerLeaseId: "gone-sandbox", status: "released", releasedAt: new Date(), cleanupStatus: "success", + leasePolicy: "reuse_by_environment", + metadata: receipt === "unproven" ? {} : { remoteExecutionTermination: { schema: "paperclip.remote-termination.v1", + companyId: base.binding.companyId, runId: "pending-cleanup-run", leaseId: "lease-gone", provider: "daytona", + providerLeaseId: "gone-sandbox", state: receipt, confirmedAt: new Date().toISOString() } } }; + const where = vi.fn(async () => [lease]); + const db = { select: () => ({ from: () => ({ where }) }) } as never; + try { + await executePaperclipNativeSession({ db: leaseDb(base), execution: base, runnerInstanceId: "runner", + instructionWorkingCopy: { hasChanges: async () => false, collectStopped: async () => {}, retirementFailed }, + runnerExecutionTarget: { kind: "remote", transport: "sandbox", environmentId, remoteCwd: `/tmp/${environmentId}`, + sandboxLeaseAcquisition: { outcome: "created", providerLeaseId: "gone-sandbox" } } }); + const deleted = await closeWarmNativeSessionsForEnvironment({ environmentId, reason: "environment deleted", forceStopUnsettled: true, db }); + expect(where).toHaveBeenCalledOnce(); + if (receipt === "destroyed") { + expect(deleted).toEqual({ closed: 1, busy: 0, failed: 0, failures: [] }); + expect(await closeWarmNativeSessionsForEnvironment({ environmentId, reason: "environment deleted", forceStopUnsettled: true, db })) + .toEqual({ closed: 0, busy: 0, failed: 0, failures: [] }); + } else { + expect(deleted).toMatchObject({ closed: 0, failed: 1, failures: [{ runId: base.binding.runId, providerLeaseId: "gone-sandbox", + reason: "close failed: Daytona sandbox lease gone-sandbox is no longer active." }] }); + } + } finally { + close.mockResolvedValue(undefined); + await closeWarmNativeSessionsForEnvironment({ environmentId, reason: "test cleanup" }); + } + }, + ); + it("retains failed retirement for retry and never launches over an uncontained owner", async () => { const { base, next, run } = fixture("identity-close-failure"); const close = vi.fn().mockRejectedValueOnce(new Error("containment unavailable")).mockResolvedValue(undefined); @@ -12474,7 +12590,7 @@ describe("native warm session supervision", () => { await executePaperclipNativeSession({ db: leaseDb(warmExecution), execution: warmExecution, runnerInstanceId: "runner" }); expect(readNativeSessionIdleWork()).toBe("present"); if (retirement === "controller restart") { - expect(await closeIdleWarmNativeSessionsForRestart()).toEqual({ closed: 0, busy: 0, failed: 1 }); + expect(await closeIdleWarmNativeSessionsForRestart()).toMatchObject({ closed: 0, busy: 0, failed: 1 }); } else { await vi.waitFor(() => expect(close).toHaveBeenCalledOnce()); } @@ -12486,7 +12602,7 @@ describe("native warm session supervision", () => { expect(state.execute).toHaveBeenCalledOnce(); expect(readNativeSessionIdleWork()).toBe("present"); close.mockResolvedValue(undefined); - expect(await closeIdleWarmNativeSessionsForRestart()).toEqual({ closed: 1, busy: 0, failed: 0 }); + expect(await closeIdleWarmNativeSessionsForRestart()).toEqual({ closed: 1, busy: 0, failed: 0, failures: [] }); expect(close).toHaveBeenCalledTimes(3); expect(readNativeSessionIdleWork()).toBe("none"); } finally { @@ -12521,18 +12637,18 @@ describe("native warm session supervision", () => { it("does no work when no warm owner exists", async () => { await closeIdleWarmNativeSessionsForRestart(); expect(await prepareSleep({ canRetire: () => true })) - .toEqual({ closed: 0, busy: 0, failed: 0 }); + .toEqual({ closed: 0, busy: 0, failed: 0, failures: [] }); expect(readNativeSessionIdleWork()).toBe("none"); }); it("leaves an idle owner intact without current drain authority", async () => { const f = await start("sleep-stale-hold"); expect(await prepareSleep({ canRetire: () => false })) - .toEqual({ closed: 0, busy: 0, failed: 0 }); + .toEqual({ closed: 0, busy: 0, failed: 0, failures: [] }); expect(f.close).not.toHaveBeenCalled(); expect(readNativeSessionIdleWork()).toBe("present"); expect(await prepareSleep({ canRetire: () => true })) - .toEqual({ closed: 1, busy: 0, failed: 0 }); + .toEqual({ closed: 1, busy: 0, failed: 0, failures: [] }); expect(f.close).toHaveBeenCalledExactlyOnceWith({ reason: "tenant idle sleep" }); expect(readNativeSessionIdleWork()).toBe("none"); }); @@ -12547,7 +12663,7 @@ describe("native warm session supervision", () => { heartbeat.getTaskDrainStatus(); expect(f.close).not.toHaveBeenCalled(); expect(await heartbeat.prepareIdleWarmNativeSessionsForSleep({ canRetire: () => true })) - .toEqual({ closed: 1, busy: 0, failed: 0 }); + .toEqual({ closed: 1, busy: 0, failed: 0, failures: [] }); expect(readNativeSessionIdleWork()).toBe("none"); }); @@ -12563,11 +12679,11 @@ describe("native warm session supervision", () => { expect(readNativeSessionIdleWork()).toBe("present"); current = false; finish(); - expect(await preparation).toEqual({ closed: 1, busy: 0, failed: 0 }); + expect(await preparation).toEqual({ closed: 1, busy: 0, failed: 0, failures: [] }); expect(second.close).not.toHaveBeenCalled(); expect(readNativeSessionIdleWork()).toBe("present"); expect(await prepareSleep({ canRetire: () => true })) - .toEqual({ closed: 1, busy: 0, failed: 0 }); + .toEqual({ closed: 1, busy: 0, failed: 0, failures: [] }); expect(readNativeSessionIdleWork()).toBe("none"); } finally { finish(); await preparation; } }); @@ -12576,13 +12692,13 @@ describe("native warm session supervision", () => { const close = vi.fn(async () => undefined); await start("sleep-busy-turn", close, async () => { expect(await prepareSleep({ canRetire: () => true })) - .toEqual({ closed: 0, busy: 1, failed: 0 }); + .toEqual({ closed: 0, busy: 1, failed: 0, failures: [] }); expect(close).not.toHaveBeenCalled(); }); expect(close).not.toHaveBeenCalled(); expect(readNativeSessionIdleWork()).toBe("present"); expect(await prepareSleep({ canRetire: () => true })) - .toEqual({ closed: 1, busy: 0, failed: 0 }); + .toEqual({ closed: 1, busy: 0, failed: 0, failures: [] }); }); it("collects managed edits after the provider checkpoint before clearing the owner", async () => { @@ -12604,7 +12720,7 @@ describe("native warm session supervision", () => { runId: "sleep-files", root, preparationKey: "sleep-files", hasChanges: async () => false, collectStopped, }); expect(await prepareSleep({ canRetire: () => true })) - .toEqual({ closed: 1, busy: 0, failed: 0 }); + .toEqual({ closed: 1, busy: 0, failed: 0, failures: [] }); expect(await readFile(saved, "utf8")).toBe("Edits collected from stopped provider"); expect(JSON.parse(await readFile(providerCheckpoint, "utf8"))).toEqual({ session: "sleep-files", resume: "retained" }); expect(collectStopped).toHaveBeenCalledOnce(); @@ -12620,10 +12736,10 @@ describe("native warm session supervision", () => { const close = vi.fn(async () => undefined).mockRejectedValueOnce(failure); const f = await start("sleep-retry", close); expect(await prepareSleep({ canRetire: () => true })) - .toEqual({ closed: 0, busy: 0, failed: 1 }); + .toMatchObject({ closed: 0, busy: 0, failed: 1 }); expect(readNativeSessionIdleWork()).toBe("present"); expect(await prepareSleep({ canRetire: () => true })) - .toEqual({ closed: 1, busy: 0, failed: 0 }); + .toEqual({ closed: 1, busy: 0, failed: 0, failures: [] }); const next = { ...f.current, binding: { ...f.current.binding, runId: "sleep-retry-next" } }; state.execute.mockImplementationOnce(async options => { expect(options.existingSession).toBeUndefined(); @@ -12690,7 +12806,7 @@ describe("native warm session supervision", () => { environmentId: "other-environment", reason: "environment deleted", }), - ).resolves.toEqual({ closed: 0, busy: 0, failed: 0 }); + ).resolves.toEqual({ closed: 0, busy: 0, failed: 0, failures: [] }); const closeResult = shutdownKind === "controller restart" ? closeIdleWarmNativeSessionsForRestart() : closeWarmNativeSessionsForEnvironment({ @@ -12918,9 +13034,9 @@ describe("native warm session supervision", () => { if (checkpointFails) { expect(readNativeSessionIdleWork()).toBe("present"); close.mockResolvedValue(undefined); - await expect(closeIdleWarmNativeSessionsForRestart()).resolves.toEqual({ closed: 1, busy: 0, failed: 0 }); + await expect(closeIdleWarmNativeSessionsForRestart()).resolves.toEqual({ closed: 1, busy: 0, failed: 0, failures: [] }); } else { - await expect(closeIdleWarmNativeSessionsForRestart()).resolves.toEqual({ closed: 0, busy: 0, failed: 0 }); + await expect(closeIdleWarmNativeSessionsForRestart()).resolves.toEqual({ closed: 0, busy: 0, failed: 0, failures: [] }); } }); @@ -13753,7 +13869,7 @@ describe("native warm session supervision", () => { await writeFile(join(durableRoot, "control-plane", "control-plane-state.json"), JSON.stringify(durableControlPlaneState(durableIdentity))); await mkdir(join(durableRoot, "runner"), { recursive: true }); await writeFile(join(durableRoot, "runner", "runner-state.json"), JSON.stringify(durableRunnerState(durableIdentity, "ready"))); - if (path === "adoption") expect(await closeIdleWarmNativeSessionsForRestart()).toEqual({ closed: 1, busy: 0, failed: 0 }); + if (path === "adoption") expect(await closeIdleWarmNativeSessionsForRestart()).toEqual({ closed: 1, busy: 0, failed: 0, failures: [] }); const continuation = executePaperclipNativeSession({ db: { ...leaseDb(second), select: () => ({ from: () => ({ where: () => ({ limit: () => Promise.resolve([{ status: "succeeded", runnerProfileJson: { nativeExecutionInput: first } }]) }) }) }) } as unknown as Db, @@ -17837,7 +17953,7 @@ describe("runnerd provider runtime wiring", () => { } if (retirement === "environment deletion") { expect(await closeWarmNativeSessionsForEnvironment({ environmentId: target.environmentId, reason: "environment deleted" })) - .toEqual({ closed: 1, busy: 0, failed: 0 }); + .toEqual({ closed: 1, busy: 0, failed: 0, failures: [] }); expect(close).toHaveBeenCalledExactlyOnceWith({ reason: "environment deleted" }); expect(detachControllerForRestart).not.toHaveBeenCalled(); return; @@ -17845,7 +17961,7 @@ describe("runnerd provider runtime wiring", () => { if (retirement === "detach failure") { detachControllerForRestart.mockRejectedValue(new Error("fixture journal drain failed")); for (let attempt = 0; attempt < 2; attempt++) { - expect(await closeIdleWarmNativeSessionsForRestart()).toEqual({ closed: 0, busy: 0, failed: 1 }); + expect(await closeIdleWarmNativeSessionsForRestart()).toMatchObject({ closed: 0, busy: 0, failed: 1 }); expect(close).not.toHaveBeenCalled(); expect(readNativeSessionIdleWork()).toBe("present"); expect(JSON.parse(await readFile(path, "utf8")).releasedRunner).toEqual(envelope.releasedRunner); @@ -17853,8 +17969,8 @@ describe("runnerd provider runtime wiring", () => { detachControllerForRestart.mockResolvedValue(undefined); } if (retirement !== "idle timeout") { - expect(await closeIdleWarmNativeSessionsForRestart()).toEqual({ closed: 1, busy: 0, failed: 0 }); - expect(await closeIdleWarmNativeSessionsForRestart()).toEqual({ closed: 0, busy: 0, failed: 0 }); + expect(await closeIdleWarmNativeSessionsForRestart()).toEqual({ closed: 1, busy: 0, failed: 0, failures: [] }); + expect(await closeIdleWarmNativeSessionsForRestart()).toEqual({ closed: 0, busy: 0, failed: 0, failures: [] }); } else await vi.waitFor(() => expect(detachControllerForRestart).toHaveBeenCalledOnce()); expect(readNativeSessionIdleWork()).toBe("none"); expect(close).not.toHaveBeenCalled(); diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 418dca7eb98..2bc9715293d 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -123,7 +123,7 @@ import { uploadVerifiedProviderPack } from "./provider-pack-upload.js"; import { selectRemotePiCompanion } from "./remote-pi-companion.js"; import { scheduledResumeContinuationAllowed } from "./scheduled-resume-continuation.js"; import { readNativeLocalProcessStop, recordedLocalExecutionStopped, PRIOR_OWNER_FENCE_REFUSED, PROCESS_IDENTITY_RECORDED, PROCESS_START_REQUESTED, withReleasedNativeRunProcess } from "../native-local-process-stop.js"; -import { remoteLeaseCleanupScope } from "../remote-execution-termination.js"; +import { hasRemoteTerminationReceipt, remoteLeaseCleanupScope, resolveRemoteTerminationLease } from "../remote-execution-termination.js"; import { connectorToolDefinitionsForSkills } from "../connector-runtime.js"; import { NATIVE_PREPARATION_FENCING_VERSION, setNativeExecutionActive } from "./native-active-runs.js"; import { preserveNativeFailedWorkspace } from "./native-failed-workspace-recovery.js"; @@ -728,9 +728,41 @@ type WarmNativeSession = { lastActivityAt: string; }; +const WARM_CLOSE_FAILURE_DETAIL_MAX = 2_000; + +export function describeWarmCloseFailure(error: unknown): string { + const describe = (value: unknown): string => { + if (value instanceof AggregateError) return [value.message, ...value.errors.map(describe)].join("; "); + if (!(value instanceof Error)) return String(value); + const code = "code" in value && typeof value.code === "string" ? `${value.code}: ` : ""; + const settlement = value instanceof NativeSessionCloseUnrecoverableError && value.settlement + ? ` ${JSON.stringify(value.settlement)}` : ""; + const cause = value.cause === undefined ? "" : ` (cause: ${describe(value.cause)})`; + return `${code}${value.message}${settlement}${cause}`; + }; + const text = describe(error).replaceAll(/[\r\n]+/g, " "); + return text.length > WARM_CLOSE_FAILURE_DETAIL_MAX ? `${text.slice(0, WARM_CLOSE_FAILURE_DETAIL_MAX)}…` : text; +} + +export type WarmNativeSessionCloseFailure = { + runId: string; + issueId: string; + environmentId: string | null; + providerLeaseId: string | null; + reason: string; +}; + +export type WarmNativeSessionCloseResult = { + closed: number; + busy: number; + failed: number; + failures: WarmNativeSessionCloseFailure[]; +}; + async function closeWarmNativeSession(entry: WarmNativeSession, reason: string, options?: { preserveInstructionsForRunId?: string; discardUnsettledSession?: boolean; + onForcedStopFailed?: (error: unknown) => void; }) { if (entry.controllerDetachedForRestart) return detachReleasedWarmNativeSession(entry); // Revoke before awaiting process retirement/checkpoint IO. @@ -753,8 +785,16 @@ async function closeWarmNativeSession(entry: WarmNativeSession, reason: string, const discardable = options?.discardUnsettledSession === true; const owner = discardable ? pluginTaskSessionOwners.get(entry.session) : undefined; const retiredWithoutCheckpoint = discardable && (owner - ? owner.isRetired() || await owner.retireSupersededTask().catch(() => false) + ? owner.isRetired() || await owner.retireSupersededTask().catch(stopError => { + options?.onForcedStopFailed?.(stopError); + return false; + }) : error instanceof NativeSessionCloseUnrecoverableError && error.settlement?.runnerSuspended === true); + if (retiredWithoutCheckpoint) { + logger.warn({ runId: entry.runId, issueId: entry.issueId, environmentId: entry.environmentId, + providerLeaseId: entry.providerLeaseId ?? null, reason, closeFailure: describeWarmCloseFailure(error) }, + "warm native session close did not settle; its Runner was stopped and the session discarded"); + } if (!retiredWithoutCheckpoint) { try { await entry.instructionWorkingCopy?.retirementFailed?.(); } catch (receiptError) { throw new AggregateError([error, receiptError], "Warm instruction retirement and receipt both failed"); } @@ -982,15 +1022,27 @@ export async function reserveWarmNativeInstructionDirectory(input: { export async function closeWarmNativeSessionsForEnvironment(input: { environmentId: string; reason: string; -}): Promise<{ closed: number; busy: number; failed: number }> { + forceStopUnsettled?: boolean; + db?: Db; +}): Promise { return closeIdleWarmNativeSessions(input); } +async function warmSandboxDestroyed(db: Db, entry: WarmNativeSession): Promise { + if (!entry.providerLeaseId || !entry.environmentId) return false; + const leases = await db.select().from(environmentLeases).where(and( + eq(environmentLeases.companyId, entry.companyId), + eq(environmentLeases.environmentId, entry.environmentId), + eq(environmentLeases.providerLeaseId, entry.providerLeaseId), + )); + const resolved = await Promise.all(leases.map(lease => resolveRemoteTerminationLease(db, lease))); + return resolved.some(lease => hasRemoteTerminationReceipt(lease) + && (lease.metadata?.remoteExecutionTermination as { state?: unknown } | undefined)?.state === "destroyed"); +} + /** Detach released owners and checkpoint unreleased owners before a controller * exits. Active turns keep their separate authenticated restart handoff. */ -export async function closeIdleWarmNativeSessionsForRestart(): Promise<{ - closed: number; busy: number; failed: number; -}> { +export async function closeIdleWarmNativeSessionsForRestart(): Promise { return closeIdleWarmNativeSessions({ reason: "controller restart", closeBusyOnRelease: true, @@ -1000,7 +1052,7 @@ export async function closeIdleWarmNativeSessionsForRestart(): Promise<{ export async function prepareIdleWarmNativeSessionsForSleep(input: { canRetire: () => boolean; -}): Promise<{ closed: number; busy: number; failed: number }> { +}): Promise { return closeIdleWarmNativeSessions({ reason: "tenant idle sleep", canRetire: input.canRetire }); } @@ -1010,10 +1062,13 @@ async function closeIdleWarmNativeSessions(input: { closeBusyOnRelease?: boolean; detachReleased?: boolean; canRetire?: () => boolean; -}): Promise<{ closed: number; busy: number; failed: number }> { + forceStopUnsettled?: boolean; + db?: Db; +}): Promise { let closed = 0; let busy = 0; let failed = 0; + const failures: WarmNativeSessionCloseFailure[] = []; for (const [sessionId, entry] of [...warmNativeSessions]) { if (input.canRetire && !input.canRetire()) break; if (input.environmentId !== undefined && entry.environmentId !== input.environmentId) { @@ -1036,17 +1091,39 @@ async function closeIdleWarmNativeSessions(input: { // Failure must not make a live directory eligible for a replacement owner. entry.busy = true; if (!detach) entry.closeOnReleaseReason = input.reason; + let forcedStopFailure: unknown; const closing = Promise.resolve().then(() => - detach ? detachReleasedWarmNativeSession(entry) : closeWarmNativeSession(entry, input.reason), + detach ? detachReleasedWarmNativeSession(entry) : closeWarmNativeSession(entry, input.reason, input.forceStopUnsettled ? { + discardUnsettledSession: true, + onForcedStopFailed: error => { forcedStopFailure = error; }, + } : undefined), ); closingWarmNativeSessions.set(sessionId, closing); try { await closing; forgetWarmNativeSession(sessionId, entry); closed += 1; - } catch { + } catch (error) { if (detach) entry.closeOnReleaseReason = "released Runner local detach is unconfirmed"; + const destroyed = !detach && input.forceStopUnsettled && input.db + ? await warmSandboxDestroyed(input.db, entry).catch(proofError => { + logger.warn({ err: proofError, runId: entry.runId }, "could not read the warm session's sandbox lease"); + return false; + }) + : false; + if (destroyed) { + logger.warn({ runId: entry.runId, issueId: entry.issueId, environmentId: entry.environmentId, + providerLeaseId: entry.providerLeaseId ?? null, reason: input.reason, closeFailure: describeWarmCloseFailure(error) }, + "warm native session's sandbox was already destroyed; released its owner"); + forgetWarmNativeSession(sessionId, entry); + closed += 1; + continue; + } failed += 1; + const forced = forcedStopFailure === undefined ? "" : `; forced Runner stop failed: ${describeWarmCloseFailure(forcedStopFailure)}`; + failures.push({ runId: entry.runId, issueId: entry.issueId, environmentId: entry.environmentId, + providerLeaseId: entry.providerLeaseId ?? null, + reason: `${detach ? "released Runner detach failed" : "close failed"}: ${describeWarmCloseFailure(error)}${forced}` }); } finally { entry.busy = false; if (closingWarmNativeSessions.get(sessionId) === closing) { @@ -1055,7 +1132,8 @@ async function closeIdleWarmNativeSessions(input: { } if (input.canRetire && !input.canRetire()) break; } - return { closed, busy, failed }; + if (failures.length > 0) logger.warn({ reason: input.reason, failures }, "idle warm native sessions did not close"); + return { closed, busy, failed, failures }; } function readBoundedNativeFile( @@ -8522,8 +8600,11 @@ function armWarmNativeSessionIdleTimer( if (!released) current.closeOnReleaseReason = "warm native session idle timeout"; const closing = (released ? detachReleasedWarmNativeSession(current) : closeWarmNativeSession(current, current.closeOnReleaseReason!)) .then(() => { forgetWarmNativeSession(sessionId, current); }) - .catch(() => { + .catch(error => { if (released) current.closeOnReleaseReason = "released Runner local detach is unconfirmed"; + logger.warn({ runId: current.runId, issueId: current.issueId, environmentId: current.environmentId, + providerLeaseId: current.providerLeaseId ?? null, closeFailure: describeWarmCloseFailure(error) }, + released ? "idle warm native session detach failed" : "idle warm native session close failed"); }) .finally(() => { current.busy = false; diff --git a/server/src/services/native-runtime/plugin-runner-release-resume.integration.test.ts b/server/src/services/native-runtime/plugin-runner-release-resume.integration.test.ts index 5ea1f9940e9..5c470a255af 100644 --- a/server/src/services/native-runtime/plugin-runner-release-resume.integration.test.ts +++ b/server/src/services/native-runtime/plugin-runner-release-resume.integration.test.ts @@ -254,12 +254,12 @@ describeEmbeddedPostgres("plugin Runner release/resume through the Core executio const pid = handle!.child.pid; const { heartbeatService } = await import("../heartbeat.js"); expect(await heartbeatService(db).prepareIdleWarmNativeSessionsForSleep({ canRetire: () => true })) - .toEqual({ closed: 0, busy: 0, failed: 0 }); + .toEqual({ closed: 0, busy: 0, failed: 0, failures: [] }); expect(stops).toBe(0); expect(handle!.isExitConfirmed?.()).toBe(false); if (adopted) { if (restart) { - expect(await closeIdleWarmNativeSessionsForRestart()).toEqual({ closed: 1, busy: 0, failed: 0 }); - expect(await closeIdleWarmNativeSessionsForRestart()).toEqual({ closed: 0, busy: 0, failed: 0 }); + expect(await closeIdleWarmNativeSessionsForRestart()).toEqual({ closed: 1, busy: 0, failed: 0, failures: [] }); + expect(await closeIdleWarmNativeSessionsForRestart()).toEqual({ closed: 0, busy: 0, failed: 0, failures: [] }); } else await new Promise(done => setTimeout(done, 5_200)); expect(stops).toBe(0); expect(handle!.isExitConfirmed?.()).toBe(false); expect(await proof()).toMatchObject({ source: releases[0], taskId }); diff --git a/tests/restart-chaos/README.md b/tests/restart-chaos/README.md index a1684280485..d270b74ba1a 100644 --- a/tests/restart-chaos/README.md +++ b/tests/restart-chaos/README.md @@ -864,6 +864,14 @@ pending is retried for the same settle window; if it is still blocked then, the wave fails (not as a harness error) and the instance is kept, because its database owns the cleanup work. The result is in `/sandbox-sweep.json`. +**Clean warm shutdown.** Every warm wave (`--daytona-warm` or `--lifecycle warm`) +also grades "no leaked sandboxes / clean warm shutdown" (`warm-shutdown.ts`). It fails +the wave when the environment delete stayed blocked, when a wave sandbox is still +running after cleanup, or when the server log shows a warm session that did not close +on environment delete or controller shutdown. Idle closes that needed a forced Runner +stop, or whose sandbox was already destroyed, are counted but do not fail. The result +is in `/warm-shutdown.json` and in the wave's `SUMMARY.md`. + ## Provider message faults (PV1–PV3) and the saved question (Q1) ```sh diff --git a/tests/restart-chaos/main.ts b/tests/restart-chaos/main.ts index f0de3073dca..ded4a0efb10 100644 --- a/tests/restart-chaos/main.ts +++ b/tests/restart-chaos/main.ts @@ -8,7 +8,7 @@ */ import { execFileSync, spawnSync } from "node:child_process"; import { existsSync, readFileSync, writeFileSync } from "node:fs"; -import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { mkdir, readdir, readFile, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { setTimeout as delay } from "node:timers/promises"; @@ -53,6 +53,7 @@ import { isBrokenStdoutError, markTimelineEchoBroken, Timeline } from "./timelin import { guardedRunnerOutputs, worktreeFingerprint } from "./wave-guard.js"; import { checkCrossServer, readCrossServerEvidence } from "./multi-server.js"; import { sessionFaultServerEnvironment } from "./session-faults.js"; +import { controllerLogNames, gradeWarmShutdown, isWarmWave, warmShutdownMarkdown, type WarmShutdownCleanup } from "./warm-shutdown.js"; import type { FaultKind, FaultReceipt, OracleArtifacts, TriggerId, WavePlan } from "./types.js"; const repositoryRoot = path.resolve(import.meta.dirname, "../.."); @@ -303,6 +304,8 @@ async function main(): Promise { let diskFullSeam: DiskFullFaultSeam | null = null; let speedSection: string | null = null; let storageSection: string | null = null; + let warmShutdownSection: string | null = null; + let daytonaCleanup: WarmShutdownCleanup | null = null; let harnessError: unknown; process.on("uncaughtException", error => { if (isSocketTypeOfServiceRace(error)) { timeline.add("harness_socket_race_ignored", { error: String(error) }); return; } @@ -884,6 +887,7 @@ async function main(): Promise { sandboxSweep ??= await owned.sweep(0); // Cleanup only through Paperclip; the harness never changes a sandbox. const cleanup = await owned.cleanUp(); + daytonaCleanup = cleanup; timeline.add("daytona_cleanup", cleanup); console.info(`Daytona R6 sweep: ${sandboxSweep.liveSandboxes.length} live, ${sandboxSweep.archivedSandboxes.length} archived after release, ` + `${sandboxSweep.orphanSandboxes.length} orphaned, ` @@ -915,6 +919,17 @@ async function main(): Promise { timeline.add("supervisor_stop_failed", { error: String(error) }); } } + if (isWarmWave(plan.options)) { + const logDir = path.dirname(supervisor.paths.serverLogPath); + const controllerLogs = controllerLogNames(await readdir(logDir).catch(() => [] as string[])); + const serverLog = (await Promise.all(controllerLogs.map(name => readFile(path.join(logDir, name), "utf8").catch(() => "")))).join("\n"); + const warmShutdown = gradeWarmShutdown({ cleanup: daytonaCleanup, serverLog }); + timeline.add("warm_shutdown", { ...warmShutdown }); + if (warmShutdown.verdict === "fail") anyFailure = true; + warmShutdownSection = warmShutdownMarkdown(warmShutdown); + await writeFile(path.join(waveDir, "warm-shutdown.json"), `${JSON.stringify(warmShutdown, null, 2)}\n`, { mode: 0o600 }) + .catch(error => timeline.add("warm_shutdown_write_failed", { error: String(error) })); + } storageChaos?.stop(); await credentialRotation?.settled(); if (credentialRotation?.failures.length) harnessError ??= new Error(`ST5R credential rotation failed: ${credentialRotation.failures.join("; ")}`); @@ -941,7 +956,7 @@ async function main(): Promise { if (lines.length) { await appendLedger(resultsRoot, lines); const summary = await writeSummary(resultsRoot, plan.waveId); - await writeFile(path.join(waveDir, "SUMMARY.md"), [summary, speedSection, storageSection].filter(Boolean).join("\n"), { mode: 0o600 }); + await writeFile(path.join(waveDir, "SUMMARY.md"), [summary, warmShutdownSection, speedSection, storageSection].filter(Boolean).join("\n"), { mode: 0o600 }); console.info(`\n${summary}`); } } diff --git a/tests/restart-chaos/warm-shutdown.test.ts b/tests/restart-chaos/warm-shutdown.test.ts new file mode 100644 index 00000000000..f5eb4fdc368 --- /dev/null +++ b/tests/restart-chaos/warm-shutdown.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, it } from "vitest"; +import { controllerLogNames, gradeWarmShutdown, isWarmWave, warmShutdownMarkdown } from "./warm-shutdown.js"; + +const esc = String.fromCharCode(27); +const clean = { environmentDeleted: true, environmentDeleteBlocked: null, remainingSandboxes: [] }; + +describe("clean warm shutdown check", () => { + it("applies to warm local and warm Daytona waves only", () => { + expect(isWarmWave({ lifecycle: "warm" })).toBe(true); + expect(isWarmWave({ daytonaWarm: true })).toBe(true); + expect(isWarmWave({ lifecycle: "per_turn" })).toBe(false); + expect(isWarmWave({})).toBe(false); + }); + + it("grades every controller's log, including the second controller of a multi-server wave", () => { + expect(controllerLogNames(["timeline.jsonl", "server-b.log", "server.log", "speed.json", "server.log.bak"])) + .toEqual(["server-b.log", "server.log"]); + }); + + it("passes a deleted environment with no sandboxes left and no unclosed session", () => { + const check = gradeWarmShutdown({ cleanup: clean, serverLog: `[14:20:17] ${esc}[32mINFO${esc}[39m: server stopped\n` }); + expect(check).toMatchObject({ verdict: "pass", problems: [], leakedSandboxes: [] }); + }); + + it("fails the staging shape: a 409 delete, started sandboxes and a bare failed count at shutdown", () => { + const blocked = "DELETE /api/environments/env?destroyReusableSandboxLeases=true returned 409: warm native runner did not shut down cleanly"; + const check = gradeWarmShutdown({ + cleanup: { environmentDeleted: false, environmentDeleteBlocked: blocked, remainingSandboxes: ["sbx-1:started", "sbx-2:started"] }, + serverLog: `[14:20:17] ${esc}[33mWARN${esc}[39m: ${esc}[36midle native session cleanup failed before controller shutdown${esc}[39m {"idleSessions":{"failed":2}}\n`, + }); + expect(check.verdict).toBe("fail"); + expect(check.environmentDeleteBlocked).toBe(blocked); + expect(check.leakedSandboxes).toEqual(["sbx-1:started", "sbx-2:started"]); + expect(check.shutdownCloseFailures).toEqual(['[14:20:17] WARN: idle native session cleanup failed before controller shutdown {"idleSessions":{"failed":2}}']); + expect(check.problems).toHaveLength(3); + expect(warmShutdownMarkdown(check)).toContain("Verdict: FAIL"); + }); + + it("reports a forced stop without failing when cleanup still finished", () => { + const check = gradeWarmShutdown({ cleanup: clean, + serverLog: "WARN: warm native session close did not settle; its Runner was stopped and the session discarded {}\n" }); + expect(check).toMatchObject({ verdict: "pass", idleCloseFailures: [expect.stringContaining("did not settle")] }); + expect(gradeWarmShutdown({ cleanup: clean, + serverLog: "WARN: warm native session's sandbox was already destroyed; released its owner {}\n" }).idleCloseFailures).toHaveLength(1); + }); + + it("grades a warm local wave from the server log alone", () => { + expect(gradeWarmShutdown({ cleanup: null, serverLog: "" }).verdict).toBe("pass"); + expect(gradeWarmShutdown({ cleanup: null, + serverLog: 'WARN: idle warm native sessions did not close {"reason":"controller restart","failures":[]}' }).verdict).toBe("fail"); + expect(gradeWarmShutdown({ cleanup: null, + serverLog: 'WARN: idle warm native sessions did not close {"reason":"terminal heartbeat run destroyed its environment lease"}' }).verdict).toBe("pass"); + }); +}); diff --git a/tests/restart-chaos/warm-shutdown.ts b/tests/restart-chaos/warm-shutdown.ts new file mode 100644 index 00000000000..3006b68c68c --- /dev/null +++ b/tests/restart-chaos/warm-shutdown.ts @@ -0,0 +1,56 @@ +export interface WarmShutdownCleanup { + environmentDeleted: boolean; + environmentDeleteBlocked: string | null; + remainingSandboxes: string[]; +} + +export interface WarmShutdownCheck { + verdict: "pass" | "fail"; + environmentDeleted: boolean | null; + environmentDeleteBlocked: string | null; + leakedSandboxes: string[]; + shutdownCloseFailures: string[]; + idleCloseFailures: string[]; + problems: string[]; +} + +const SHUTDOWN_CLOSE_FAILURE = /idle native session cleanup failed before controller shutdown|idle warm native sessions did not close.*"reason":"(environment deleted|controller restart)"/; +const IDLE_CLOSE_FAILURE = /idle warm native session (close|detach) failed|warm native session close did not settle|warm native session's sandbox was already destroyed/; +const ANSI = new RegExp(`${String.fromCharCode(27)}\\[[0-9;]*m`, "g"); + +export function controllerLogNames(names: readonly string[]): string[] { + return names.filter(name => /^server(-[a-z0-9]+)?\.log$/.test(name)).sort(); +} + +export function isWarmWave(options: { lifecycle?: string; daytonaWarm?: boolean }): boolean { + return options.lifecycle === "warm" || options.daytonaWarm === true; +} + +export function gradeWarmShutdown(input: { cleanup: WarmShutdownCleanup | null; serverLog: string }): WarmShutdownCheck { + const lines = input.serverLog.split("\n").map(line => line.replace(ANSI, "").trim()).filter(Boolean); + const shutdownCloseFailures = lines.filter(line => SHUTDOWN_CLOSE_FAILURE.test(line)); + const idleCloseFailures = lines.filter(line => IDLE_CLOSE_FAILURE.test(line)); + const problems: string[] = []; + if (input.cleanup?.environmentDeleteBlocked) problems.push(`environment delete blocked: ${input.cleanup.environmentDeleteBlocked}`); + else if (input.cleanup && !input.cleanup.environmentDeleted) problems.push("environment was not deleted"); + if (input.cleanup?.remainingSandboxes.length) problems.push(`sandboxes still running after cleanup: ${input.cleanup.remainingSandboxes.join(", ")}`); + for (const line of shutdownCloseFailures) problems.push(`warm session did not close: ${line}`); + return { + verdict: problems.length ? "fail" : "pass", + environmentDeleted: input.cleanup?.environmentDeleted ?? null, + environmentDeleteBlocked: input.cleanup?.environmentDeleteBlocked ?? null, + leakedSandboxes: input.cleanup?.remainingSandboxes ?? [], + shutdownCloseFailures, + idleCloseFailures, + problems, + }; +} + +export function warmShutdownMarkdown(check: WarmShutdownCheck): string { + const out = ["## Clean warm shutdown (no leaked sandboxes)", "", + `Verdict: ${check.verdict.toUpperCase()}. Environment deleted: ${check.environmentDeleted ?? "n/a"}; sandboxes left running: ${check.leakedSandboxes.length}; ` + + `warm sessions that did not close: ${check.shutdownCloseFailures.length}; idle closes that needed a forced stop or failed: ${check.idleCloseFailures.length}.`, ""]; + for (const problem of check.problems) out.push(`- ${problem.slice(0, 2_000)}`); + if (check.problems.length) out.push(""); + return out.join("\n"); +}