diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 58e352d9a17..f5632ecb178 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1136,8 +1136,25 @@ An interrupted conversation does not permanently block its task. For local conve Shutdown, process loss, and provider failure use the existing durable failure retry counter and delay. Ordinary failure recovery permits at most two automatic retries in a failure chain. Accepted-interaction infrastructure recovery retains its existing bounded policy. Repeated scheduler visits reuse the same successor; restarting the server does not reset the counter. After exhaustion, automatic attempts stop. A new explicit user message can start a fresh run and failure budget. Productive max-turn continuation and confirmed workspace waits keep their separate existing semantics. +Graceful shutdown interrupts an owned legacy adapter through its cancellation +control, then waits within one shared 30-second window for provider stop and +adapter settlement across all affected runs. Before interruption becomes visible, +it commits a settlement fence on the exact sandbox lease. Cleanup on this server +or another server defers while that fence is pending. If the controller dies, +cleanup records exact stop-only retention and a repair hold after the deadline, +without claiming a successful restore. The shipped Compose services allow 90 +seconds for settlement and subsequent shutdown cleanup. +The sandbox lease remains available during workspace export and instruction +collection. Lease release and recovery classification follow settlement and use +the current execution evidence. If settlement exceeds the deadline with an +active sandbox source, shutdown records a restore failure and pins the exact +source for stop and retention before cleanup. A later run must pass the normal +workspace repair gate. Native runner ownership keeps its existing restart path. + Real gates still apply: company and task ownership, active provider ownership, budget limits, agent availability, dependencies, pending approval/review paths, and explicit pause holds. Native runner reattachment and finalization retain their existing ownership protocol. Process, HTTP, and gateway adapters retain their recovery rules because invoking those adapters can itself repeat an external action rather than start a conversation turn. +A source that already stopped and was released remains fenced until its export settles; restart reconciliation scans these released sources too. Both provider reuse and destruction check the physical allocation for a shutdown fence or file-repair pin, including historical lease rows. + An operator Stop waits for provider termination. Remote sandbox providers may return a stopped/deleted receipt after their control-plane operation completes. Paperclip binds that receipt to the company, run, and exact lease; successful file cleanup, a terminal run row, or an in-sandbox shutdown event is not sufficient. Legacy conversational runs receive their cancellation acknowledgement after all remote leases have confirmed termination. Stop alone never creates a continuation. A user message queued during remote cleanup is reconsidered when the provider confirms termination; it still passes normal admission and adopts pending comment IDs in order. Once stopped, the next explicit wake uses the same queue. A compatible saved ACP session can resume, and an unavailable or incompatible session can start fresh with the full task context. Run credentials and scratch paths remain scoped to the new run. A subtree pause requires Resume; a message does not bypass it. Cancelled runs and runs with a recorded pending stop lose run-scoped API write authority, while reads remain available for diagnostics. This applies to ordinary tasks as well as conversations. Task and interaction mutations recheck that authority in the write transaction, so a cancelled run cannot overwrite a recovery disposition with a late Done or resolve an interaction after revocation. A task handoff that intentionally stops its own run can commit only with a server receipt tied to that exact request. Direct sandbox CLI adapters such as Grok register cancellation before preparation and keep ownership until host-owned termination is verified, including a sandbox acquired before adapter registration. A failed stop does not acknowledge cancellation or abandon an outstanding remote command. Interrupted workspace restore failures remain recorded for recovery; a stop receipt proves termination, not successful file restoration. diff --git a/docker/docker-compose.quickstart.yml b/docker/docker-compose.quickstart.yml index f750bda19b6..8123f5efb71 100644 --- a/docker/docker-compose.quickstart.yml +++ b/docker/docker-compose.quickstart.yml @@ -16,6 +16,8 @@ services: # backstop if something leaks anyway — the container hits its own limit and # dies visibly rather than starving the host of pids. pids_limit: 2048 + # Includes the shared 30s adapter-settlement window and final cleanup. + stop_grace_period: 90s ports: - "${PAPERCLIP_PORT:-3100}:3100" environment: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index dc6dda42383..03e58dfc52e 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -28,6 +28,8 @@ services: # this container hits its own ceiling and dies visibly rather than starving # the whole host of pids. pids_limit: 2048 + # Includes the shared 30s adapter-settlement window and final cleanup. + stop_grace_period: 90s ports: - "3100:3100" environment: diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index de0a695142d..1d63a948523 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -1,3 +1,4 @@ +import { allowLegacyShutdownWorkspaceCleanup, legacyShutdownWorkspaceResourceProtected } from "./legacy-shutdown-workspace-settlement.js"; import { beginIdleTrackedWork } from "./task-admission.js"; import { hasStopOnlyCleanup, prepareSandboxStopAndRetain, readStopOnlyCleanup, settleStopOnlyCleanup, stopOnlyCleanupKey } from "./sandbox-stop-and-retain.js"; import { JsonRpcCallError, readEnvironmentAcquisitionDiagnostic, readEnvironmentCreationCleanupError } from "@paperclipai/plugin-sdk"; @@ -1943,6 +1944,9 @@ function createSandboxEnvironmentDriver( lease.metadata?.agentId === input.agentId, ) : []; + for (let index = reusableCandidateLeases.length - 1; index >= 0; index--) { + if (await legacyShutdownWorkspaceResourceProtected(db, reusableCandidateLeases[index]!)) reusableCandidateLeases.splice(index, 1); + } const reusableExistingLeases = reusableCandidateLeases.filter((lease) => reusableSandboxLeaseScopeMatches({ lease, @@ -2358,6 +2362,9 @@ function createSandboxEnvironmentDriver( lease.metadata?.agentId === input.agentId, ) : []; + for (let index = reusableCandidateLeases.length - 1; index >= 0; index--) { + if (await legacyShutdownWorkspaceResourceProtected(db, reusableCandidateLeases[index]!)) reusableCandidateLeases.splice(index, 1); + } const reusableExistingLeases = reusableCandidateLeases.filter((lease) => reusableSandboxLeaseScopeMatches({ lease, @@ -3327,6 +3334,7 @@ function createSandboxEnvironmentDriver( lease: EnvironmentLease; failureReason: string; }): Promise { + if (await legacyShutdownWorkspaceResourceProtected(db, input.lease)) return null; let cleanupStatus: "success" | "failed" = "success"; let termination: ReturnType; const metadata = input.lease.metadata ?? {}; @@ -3888,6 +3896,7 @@ export function environmentRuntimeService( scopeCondition?: ReturnType; failureReason: string; }): Promise { + if (!(await allowLegacyShutdownWorkspaceCleanup(db, input.leaseRow))) return null; const now = new Date(); const attemptId = randomUUID(); // Persist recovery ownership before any provider work. Re-check scope and @@ -4065,8 +4074,16 @@ export function environmentRuntimeService( // error through `onLeaseReleaseError` for its log path. Keep the order // serial. const released: EnvironmentRuntimeLeaseRecord[] = []; - for (const leaseRow of leaseRows) { + for (let leaseRow of leaseRows) { try { + if (!(await allowLegacyShutdownWorkspaceCleanup(db, leaseRow, new Date(), { + ownerStopOnly: cancelActiveWork === true && providerResourceDisposition === "stop_and_retain", + }))) continue; + if (leaseRow.metadata?.legacyShutdownWorkspaceSettlement) { + leaseRow = (await db.select().from(environmentLeases).where(and( + eq(environmentLeases.id, leaseRow.id), eq(environmentLeases.companyId, leaseRow.companyId), + )))[0] ?? leaseRow; + } const environment = leaseRow.environmentId ? await environmentsSvc.getById(leaseRow.environmentId) : null; @@ -4211,6 +4228,14 @@ export function environmentRuntimeService( environment: Environment | null; lease: EnvironmentLease; }): Promise { + if (!(await allowLegacyShutdownWorkspaceCleanup(db, input.lease))) { + throw new Error("Shutdown workspace settlement is still pending."); + } + if (input.lease.metadata?.legacyShutdownWorkspaceSettlement) { + const current = await environmentsSvc.getLeaseById(input.lease.id); + if (!current) throw new Error("Shutdown workspace source is unavailable."); + input = { ...input, lease: current }; + } const driver = requireDriverKey(getLeaseDriverKey(input.lease, input.environment)); if (!driver.retryPendingSandboxTeardown) { throw new Error( @@ -4407,6 +4432,7 @@ export function environmentRuntimeService( }, async resumeRunLease(input: EnvironmentDriverLeaseInput): Promise { + if (await legacyShutdownWorkspaceResourceProtected(db, input.lease)) return null; const driver = requireDriverKey(getLeaseDriverKey(input.lease, input.environment)); if (!driver.resumeRunLease) { throw new Error(`Environment driver "${driver.driver}" does not support lease resume.`); @@ -4415,6 +4441,13 @@ export function environmentRuntimeService( }, async destroyRunLease(input: EnvironmentDriverLeaseInput): Promise { + if (await legacyShutdownWorkspaceResourceProtected(db, input.lease)) return null; + if (input.lease.metadata?.legacyShutdownWorkspaceSettlement) { + if (!(await allowLegacyShutdownWorkspaceCleanup(db, input.lease))) return null; + const current = await environmentsSvc.getLeaseById(input.lease.id); + if (!current || hasStopOnlyCleanup(current)) return null; + input = { ...input, lease: current }; + } const driver = requireDriverKey(getLeaseDriverKey(input.lease, input.environment)); if (!driver.destroyRunLease) { throw new Error(`Environment driver "${driver.driver}" does not support lease destroy.`); diff --git a/server/src/services/heartbeat/recovery.test.ts b/server/src/services/heartbeat/recovery.test.ts index a925edfaa79..6d76f2df511 100644 --- a/server/src/services/heartbeat/recovery.test.ts +++ b/server/src/services/heartbeat/recovery.test.ts @@ -1,15 +1,17 @@ import { randomUUID } from "node:crypto"; import type { Db } from "@paperclipai/db"; -import { agents, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issues, nativeRunFinalizations } from "@paperclipai/db"; +import { agents, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, environments, environmentLeases, heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations } from "@paperclipai/db"; import { eq } from "drizzle-orm"; import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; -import { environmentRuntimeService } from "../environment-runtime.js"; +import { classifyEnvironmentCapabilities, environmentRuntimeService } from "../environment-runtime.js"; import { environmentService } from "../environments.js"; import { createHeartbeatRecovery, type HeartbeatRecoveryDependencies } from "./recovery.js"; import type { HotRestartIntent } from "../hot-restart.js"; import type { NativeRestartRecoveryClaim, NativeRestartRecoveryDisposition } from "../native-runtime/index.js"; import { NATIVE_OWNERSHIP_UNVERIFIED_ERROR_CODE } from "../native-runtime/native-runner-ownership.js"; +import { allowLegacyShutdownWorkspaceCleanup, beginLegacyShutdownWorkspaceSettlement, finishLegacyShutdownWorkspaceSettlement, legacyShutdownWorkspaceResourceProtected, sweepLegacyShutdownWorkspaceSettlements } from "../legacy-shutdown-workspace-settlement.js"; +import { adapterExecutionControls, createAdapterExecutionControl } from "../adapter-execution-control.js"; type Run = typeof heartbeatRuns.$inferSelect; @@ -71,7 +73,7 @@ function callbacks(db: Db) { terminateHeartbeatRunProcess: vi.fn(async () => undefined), mergeRunStopMetadataForAgent: vi.fn((_agent, _outcome, options) => options?.resultJson ?? null), classifyAndPersistRunLiveness: vi.fn(async run => run), - releaseEnvironmentLeasesForRun: vi.fn(async () => undefined), + releaseEnvironmentLeasesForRun: vi.fn(async () => undefined), acknowledgeRemoteStop: vi.fn(async () => undefined), resumeRemoteStopComments: vi.fn(async () => undefined), dispatchPendingNativeStatusWakeups: vi.fn(async () => undefined), @@ -184,6 +186,287 @@ describePostgres("heartbeat recovery module ownership", () => { return (await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, issueId, invocationSource: "automation", status: "running", runtimeMode: "legacy" }).returning())[0]; } + it.each(["restored", "restore_failed"])("keeps the sandbox available until adapter workspace settlement: %s", async outcome => { + const run = await seedRun(); + const deps = callbacks(db); + deps.setRunStatusIfRunning.mockImplementation(async (id, status, patch) => { + const [updated] = await db.update(heartbeatRuns).set({ ...patch, status }).where(eq(heartbeatRuns.id, id)).returning(); + return { updated: true, run: updated }; + }); + deps.getRun.mockImplementation(async id => (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, id)))[0] ?? null); + const control = createAdapterExecutionControl(); + adapterExecutionControls.set(run.id, control); + const drain = createHeartbeatRecovery(db, deps).drainRunningRunsForShutdown("SIGTERM", new Date(), [run.id]); + try { + await vi.waitFor(() => expect(control.controller.signal.aborted).toBe(true)); + expect(deps.releaseEnvironmentLeasesForRun).not.toHaveBeenCalled(); + expect(deps.scheduleBoundedRetryForRun).not.toHaveBeenCalled(); + expect((await deps.getRun(run.id))?.status).toBe("interrupted"); + const resultJson = outcome === "restore_failed" ? { workspaceRestoreFailure: "restore_failed", + workspaceRestoreRecovery: { schema: "paperclip.workspace-restore-recovery.v1", leaseIds: [randomUUID()] } } + : { workspaceRestored: true }; + await db.update(heartbeatRuns).set({ resultJson }).where(eq(heartbeatRuns.id, run.id)); + control.finish(); + await drain; + expect(deps.releaseEnvironmentLeasesForRun).toHaveBeenCalledOnce(); + expect(deps.classifyAndPersistRunLiveness).toHaveBeenCalledWith(expect.objectContaining({ + resultJson: expect.objectContaining(resultJson) }), expect.anything()); + } finally { + control.finish(); + adapterExecutionControls.delete(run.id); + await drain; + await db.delete(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)); + } + }); + + it("starts all owned shutdown cancellations within one settlement window", async () => { + const runs = await Promise.all([seedRun(), seedRun()]); + const controls = runs.map(run => { + const control = createAdapterExecutionControl(); adapterExecutionControls.set(run.id, control); return control; + }); + const deps = callbacks(db); + deps.setRunStatusIfRunning.mockImplementation(async (id, status, patch) => { + const [updated] = await db.update(heartbeatRuns).set({ ...patch, status }).where(eq(heartbeatRuns.id, id)).returning(); + return { updated: true, run: updated }; + }); + const drain = createHeartbeatRecovery(db, deps).drainRunningRunsForShutdown("SIGTERM", new Date(), runs.map(run => run.id)); + try { + await vi.waitFor(() => expect(controls.every(control => control.controller.signal.aborted)).toBe(true)); + expect(deps.releaseEnvironmentLeasesForRun).not.toHaveBeenCalled(); + } finally { + controls.forEach(control => control.finish()); + await drain; + for (const run of runs) { adapterExecutionControls.delete(run.id); await db.delete(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)); } + } + }); + + it.each(["ephemeral", "reuse_by_environment"] as const)("protects a %s export from another server and pins it after controller loss", async leasePolicy => { + const run = await seedRun(); + await db.update(heartbeatRuns).set({ contextSnapshot: { issueId: run.issueId } }).where(eq(heartbeatRuns.id, run.id)); + const [lease] = await db.insert(environmentLeases).values({ companyId: run.companyId, heartbeatRunId: run.id, + issueId: run.issueId, status: "active", provider: "daytona", providerLeaseId: randomUUID(), leasePolicy, + metadata: { driver: "sandbox", sandboxProviderPlugin: true, pluginId: "fixture-plugin", remoteCwd: "/home/daytona/workspace" } }).returning(); + try { + const deadline = new Date(Date.now() + 30_000); + await beginLegacyShutdownWorkspaceSettlement(db, run, deadline); + await db.update(heartbeatRuns).set({ status: "interrupted", errorCode: "server_shutdown_interrupted", finishedAt: new Date() }).where(eq(heartbeatRuns.id, run.id)); + const foreignDeps = callbacks(db); + const foreign = createHeartbeatRecovery(db, foreignDeps); + expect(await foreign.sweepOrphanedActiveLeases({ backoffMs: 0 })).toMatchObject({ recovered: 0 }); + expect(await environmentRuntimeService(db).releaseRunLeases(run.id)).toEqual([]); + await db.update(environmentLeases).set({ status: "pending_cleanup" }).where(eq(environmentLeases.id, lease.id)); + expect(await foreign.sweepPendingCleanupLeases({ backoffMs: 0 })).toMatchObject({ destroyed: 0 }); + await db.update(environmentLeases).set({ status: "active" }).where(eq(environmentLeases.id, lease.id)); + let source = (await db.select().from(environmentLeases).where(eq(environmentLeases.id, lease.id)))[0]; + expect(source.status).toBe("active"); + expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, run.issueId!))).toHaveLength(0); + // No local adapter map or timeout handler survives a container SIGKILL. + expect(await allowLegacyShutdownWorkspaceCleanup(db, source, new Date(deadline.getTime() + 1))).toBe(true); + source = (await db.select().from(environmentLeases).where(eq(environmentLeases.id, lease.id)))[0]; + expect(source).toMatchObject({ status: "pending_cleanup", leasePolicy: "retain_on_failure", + metadata: { legacyShutdownWorkspaceSettlement: { state: "expired" }, sandboxStopAndRetain: { leaseId: lease.id } } }); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0].resultJson) + .toMatchObject({ workspaceRestoreFailure: "restore_failed", workspaceRestoreRecovery: { leaseIds: [lease.id] } }); + expect(await foreign.sweepOrphanedActiveLeases({ backoffMs: 0 })).toMatchObject({ recovered: 0 }); + } finally { + await db.delete(environmentLeases).where(eq(environmentLeases.id, lease.id)); + await db.delete(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, run.issueId!)); + await db.delete(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)); + } + }); + + async function seedShutdownSource() { + const run = await seedRun(); + const [lease] = await db.insert(environmentLeases).values({ companyId: run.companyId, heartbeatRunId: run.id, + issueId: run.issueId, status: "active", provider: "daytona", providerLeaseId: randomUUID(), leasePolicy: "ephemeral", + metadata: { driver: "sandbox", sandboxProviderPlugin: true, pluginId: "fixture-plugin", remoteCwd: "/home/daytona/workspace" } }).returning(); + await beginLegacyShutdownWorkspaceSettlement(db, run, new Date(Date.now() + 30_000)); + return { run, lease }; + } + async function deleteShutdownSource(f: Awaited>) { + await db.delete(environmentLeases).where(eq(environmentLeases.id, f.lease.id)); + await db.delete(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.run.issueId!)); + await db.delete(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)); + adapterExecutionControls.delete(f.run.id); + } + + it("does not turn a stale pending snapshot into a repair hold after settlement", async () => { + const f = await seedShutdownSource(); + try { + const old = (await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id)))[0]; + await db.update(heartbeatRuns).set({ status: "interrupted", contextSnapshot: { issueId: f.run.issueId }, resultJson: { workspaceRestored: true } }).where(eq(heartbeatRuns.id, f.run.id)); + await finishLegacyShutdownWorkspaceSettlement(db, f.run); + const single = createDb(database.connectionString, { maxConnections: 1 }); + try { expect(await allowLegacyShutdownWorkspaceCleanup(single, old, new Date(Date.now() + 60_000))).toBe(true); } + finally { await single.$client.end({ timeout: 1 }); } + expect((await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id)))[0]) + .toMatchObject({ status: "active", leasePolicy: "ephemeral", metadata: { legacyShutdownWorkspaceSettlement: { state: "settled" } } }); + expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.run.issueId!))).toHaveLength(0); + } finally { await deleteShutdownSource(f); } + }); + + it("allows the aborted owner to stop and retain its command while excluding outside cleanup", async () => { + const f = await seedShutdownSource(); + const control = createAdapterExecutionControl(); control.controller.abort(); + adapterExecutionControls.set(f.run.id, control); + const stop = vi.fn(async () => ({ providerLeaseId: f.lease.providerLeaseId, state: "stopped" })); + const release = vi.fn(async () => null); + const runtime = environmentRuntimeService(db, { drivers: [{ driver: "sandbox", resolveCapabilities: async () => classifyEnvironmentCapabilities({}), + acquireRunLease: async () => { throw new Error("Unexpected acquire"); }, releaseRunLease: release, + retryPendingSandboxTeardown: stop }] }); + try { + await runtime.releaseRunLeases(f.run.id, "released", undefined, "stop_and_retain", false); + expect(stop).not.toHaveBeenCalled(); + await runtime.releaseRunLeases(f.run.id, "released", undefined, "stop_and_retain", true); + expect(stop).toHaveBeenCalledOnce(); expect(release).not.toHaveBeenCalled(); + expect((await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id)))[0]) + .toMatchObject({ status: "released", cleanupStatus: "success", metadata: { remoteExecutionTermination: { state: "stopped" }, + sandboxStopAndRetainReceipt: { leaseId: f.lease.id } } }); + } finally { control.finish(); await deleteShutdownSource(f); } + }); + + it.each(["ephemeral", "reuse_by_environment"] as const)("recovers a stopped %s source after its shutdown controller disappears", async leasePolicy => { + const f = await seedShutdownSource(); + await db.update(environmentLeases).set({ leasePolicy }).where(eq(environmentLeases.id, f.lease.id)); + const control = createAdapterExecutionControl(); control.controller.abort(); adapterExecutionControls.set(f.run.id, control); + const [environmentRow] = await db.insert(environments).values({ name: "Guarded source", driver: "sandbox" }).returning(); + const environment = (await environmentService(db).getById(environmentRow.id))!; + const leaseSnapshot = (await environmentService(db).getLeaseById(f.lease.id))!; + leaseSnapshot.metadata = f.lease.metadata; + const resume = vi.fn(async () => null); const destroy = vi.fn(async () => null); + const stop = vi.fn(async () => ({ providerLeaseId: f.lease.providerLeaseId, state: "stopped" })); + const runtime = environmentRuntimeService(db, { drivers: [{ driver: "sandbox", resolveCapabilities: async () => classifyEnvironmentCapabilities({}), + acquireRunLease: async () => { throw new Error("Unexpected acquire"); }, releaseRunLease: vi.fn(), retryPendingSandboxTeardown: stop, resumeRunLease: resume, destroyRunLease: destroy }] }); + try { + await runtime.releaseRunLeases(f.run.id, "released", undefined, "stop_and_retain", true); + adapterExecutionControls.delete(f.run.id); // SIGKILL before export/finally settled. + await db.update(heartbeatRuns).set({ status: "interrupted", finishedAt: new Date() }).where(eq(heartbeatRuns.id, f.run.id)); + const [source] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id)); + expect(source.status).toBe("released"); + // An old reuse receipt must not bypass the current allocation's fence. + expect(await legacyShutdownWorkspaceResourceProtected(db, { provider: source.provider, providerLeaseId: source.providerLeaseId })).toBe(true); + await sweepLegacyShutdownWorkspaceSettlements(db, new Date(Date.now() + 60_000)); + expect((await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id)))[0]) + .toMatchObject({ status: "released", leasePolicy: "retain_on_failure", metadata: { + legacyShutdownWorkspaceSettlement: { state: "expired" }, workspaceRestoreRecovery: { runId: f.run.id } } }); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].resultJson) + .toMatchObject({ workspaceRestoreFailure: "restore_failed", workspaceRestoreRecovery: { leaseIds: [f.lease.id] } }); + expect(await legacyShutdownWorkspaceResourceProtected(db, source)).toBe(true); + // Direct operations and pre-fence snapshots cannot bypass the source pin. + expect(await runtime.resumeRunLease({ environment, lease: leaseSnapshot })).toBeNull(); + expect(await runtime.destroyRunLease({ environment, lease: leaseSnapshot })).toBeNull(); + expect(resume).not.toHaveBeenCalled(); expect(destroy).not.toHaveBeenCalled(); + expect(stop).toHaveBeenCalledOnce(); + } finally { control.finish(); await deleteShutdownSource(f); await db.delete(environments).where(eq(environments.id, environment.id)); } + }); + + it.each(["ephemeral", "reuse_by_environment"] as const)("recovers failed ordinary cleanup of a fenced %s source", async leasePolicy => { + const f = await seedShutdownSource(); + const stop = vi.fn(async () => ({ providerLeaseId: f.lease.providerLeaseId, state: "stopped" })); + const destroy = vi.fn(async () => null); + const runtime = environmentRuntimeService(db, { drivers: [{ driver: "sandbox", resolveCapabilities: async () => classifyEnvironmentCapabilities({}), + acquireRunLease: async () => { throw new Error("Unexpected acquire"); }, releaseRunLease: destroy, destroyRunLease: destroy, + retryPendingSandboxTeardown: stop }] }); + try { + await db.update(heartbeatRuns).set({ status: "interrupted", finishedAt: new Date(), contextSnapshot: { issueId: f.run.issueId } }).where(eq(heartbeatRuns.id, f.run.id)); + await db.update(environmentLeases).set({ status: "pending_cleanup", leasePolicy, cleanupStatus: "failed", + releasedAt: new Date(), failureReason: "release_cleanup_failed" }).where(eq(environmentLeases.id, f.lease.id)); + const [source] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id)); + expect(await allowLegacyShutdownWorkspaceCleanup(db, source, new Date(Date.now() + 60_000))).toBe(true); + const lease = (await environmentService(db).getLeaseById(f.lease.id))!; + expect(lease).toMatchObject({ status: "pending_cleanup", leasePolicy: "retain_on_failure", metadata: { + legacyShutdownWorkspaceSettlement: { state: "expired" }, sandboxStopAndRetain: { leaseId: f.lease.id } } }); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].resultJson) + .toMatchObject({ workspaceRestoreRecovery: { leaseIds: [f.lease.id] } }); + const deps = callbacks(db); deps.environmentRuntime = runtime; + expect(await createHeartbeatRecovery(db, deps).sweepPendingCleanupLeases({ backoffMs: 0 })).toMatchObject({ destroyed: 1 }); + expect((await environmentService(db).getLeaseById(f.lease.id))!).toMatchObject({ status: "released", cleanupStatus: "success" }); + expect(stop).toHaveBeenCalledOnce(); expect(destroy).not.toHaveBeenCalled(); + } finally { await deleteShutdownSource(f); } + }); + + it("waits for a claimed ordinary cleanup attempt before converting its shutdown source", async () => { + const f = await seedShutdownSource(); const expiresAt = Date.now() + 120_000; + try { + await db.update(heartbeatRuns).set({ status: "interrupted", finishedAt: new Date(), contextSnapshot: { issueId: f.run.issueId } }).where(eq(heartbeatRuns.id, f.run.id)); + const [marked] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id)); + await db.update(environmentLeases).set({ status: "pending_cleanup", cleanupStatus: "failed", releasedAt: new Date(), + metadata: { ...marked.metadata, pendingCleanupInFlight: true, pendingCleanupLeaseExpiresAtMs: expiresAt } }).where(eq(environmentLeases.id, f.lease.id)); + const [source] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id)); + expect(await allowLegacyShutdownWorkspaceCleanup(db, source, new Date(expiresAt - 1))).toBe(false); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].resultJson?.workspaceRestoreRecovery).toBeUndefined(); + expect(await allowLegacyShutdownWorkspaceCleanup(db, source, new Date(expiresAt + 1))).toBe(true); + expect((await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id)))[0]) + .toMatchObject({ status: "pending_cleanup", metadata: { sandboxStopAndRetain: { leaseId: f.lease.id } } }); + } finally { await deleteShutdownSource(f); } + }); + + it("retries lease release when ordinary completion wins the shutdown status race", async () => { + const f = await seedShutdownSource(); const control = createAdapterExecutionControl(); + adapterExecutionControls.set(f.run.id, control); + const deps = callbacks(db); + deps.setRunStatusIfRunning.mockImplementation(async () => { + await db.update(heartbeatRuns).set({ status: "succeeded", resultJson: { workspaceRestored: true } }).where(eq(heartbeatRuns.id, f.run.id)); + // The adapter's first release was deferred by the fence; its finally ended. + control.finish(); return { updated: false, run: null }; + }); + deps.getRun.mockImplementation(async id => (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, id)))[0] ?? null); + deps.releaseEnvironmentLeasesForRun.mockImplementation(async input => { + expect(input.status).toBe("succeeded"); + expect((await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id)))[0].metadata) + .toMatchObject({ legacyShutdownWorkspaceSettlement: { state: "settled" } }); + await db.update(environmentLeases).set({ status: "released", cleanupStatus: "success", releasedAt: new Date() }).where(eq(environmentLeases.id, f.lease.id)); + }); + try { + await createHeartbeatRecovery(db, deps).drainRunningRunsForShutdown("SIGTERM", new Date(), [f.run.id]); + expect(deps.releaseEnvironmentLeasesForRun).toHaveBeenCalledOnce(); + expect((await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id)))[0].status).toBe("released"); + expect(deps.scheduleBoundedRetryForRun).not.toHaveBeenCalled(); + } finally { control.finish(); await deleteShutdownSource(f); } + }); + + it("pins an unjoined sandbox restore for repair before bounded shutdown cleanup", async () => { + const run = await seedRun(); + await db.update(heartbeatRuns).set({ contextSnapshot: { issueId: run.issueId } }).where(eq(heartbeatRuns.id, run.id)); + const [lease] = await db.insert(environmentLeases).values({ companyId: run.companyId, heartbeatRunId: run.id, + issueId: run.issueId, status: "active", provider: "daytona", providerLeaseId: randomUUID(), leasePolicy: "ephemeral", + metadata: { driver: "sandbox", sandboxProviderPlugin: true, pluginId: "fixture-plugin", remoteCwd: "/home/daytona/workspace" } }).returning(); + const deps = callbacks(db); + deps.setRunStatusIfRunning.mockImplementation(async (id, status, patch) => { + expect((await db.select().from(environmentLeases).where(eq(environmentLeases.id, lease.id)))[0].metadata) + .toMatchObject({ legacyShutdownWorkspaceSettlement: { state: "pending", runId: run.id } }); + const [updated] = await db.update(heartbeatRuns).set({ ...patch, status }).where(eq(heartbeatRuns.id, id)).returning(); + return { updated: true, run: updated }; + }); + deps.getRun.mockImplementation(async id => (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, id)))[0] ?? null); + deps.releaseEnvironmentLeasesForRun.mockImplementation(async () => { + expect((await deps.getRun(run.id))?.resultJson).toMatchObject({ workspaceRestoreFailure: "restore_failed", + workspaceRestoreRecovery: { schema: "paperclip.workspace-restore-recovery.v1", leaseIds: [lease.id] } }); + expect((await db.select().from(environmentLeases).where(eq(environmentLeases.id, lease.id)))[0]).toMatchObject({ + status: "pending_cleanup", leasePolicy: "retain_on_failure", metadata: { sandboxStopAndRetain: { leaseId: lease.id } } }); + }); + const control = createAdapterExecutionControl(); + adapterExecutionControls.set(run.id, control); + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + const drain = createHeartbeatRecovery(db, deps).drainRunningRunsForShutdown("SIGTERM", new Date(), [run.id]); + try { + await vi.waitFor(() => expect(control.controller.signal.aborted).toBe(true)); + await vi.advanceTimersByTimeAsync(30_000); + await drain; + expect(deps.releaseEnvironmentLeasesForRun).toHaveBeenCalledOnce(); + expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, run.issueId!))) + .toEqual([expect.objectContaining({ evidence: expect.objectContaining({ workspaceRestoreFailure: "restore_failed" }) })]); + } finally { + vi.useRealTimers(); + control.finish(); + adapterExecutionControls.delete(run.id); + await drain; + await db.delete(environmentLeases).where(eq(environmentLeases.id, lease.id)); + await db.delete(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, run.issueId!)); + await db.delete(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)); + } + }); + it("uses the shared live-execution set after construction across two factories", async () => { const run = await seedRun(); const deps = callbacks(db); diff --git a/server/src/services/heartbeat/recovery.ts b/server/src/services/heartbeat/recovery.ts index a727f4ad5f4..4a6db618b5a 100644 --- a/server/src/services/heartbeat/recovery.ts +++ b/server/src/services/heartbeat/recovery.ts @@ -13,7 +13,7 @@ import { } from "../legacy-controller-lease.js"; import { remoteTerminationReceipt } from "../remote-execution-termination.js"; import { runUsedConversationAdapter } from "../conversation-continuation.js"; -import { legacyExecutionNeedsReconciliationWithEvidence } from "../legacy-execution-recovery.js"; +import { legacyExecutionNeedsReconciliationWithEvidence, terminalizeLegacyExecution } from "../legacy-execution-recovery.js"; import { randomUUID } from "node:crypto"; import { and, @@ -72,6 +72,8 @@ import { type HotRestartReportRun, } from "../hot-restart.js"; import { serverVersion } from "../../version.js"; +import { adapterExecutionControls, waitForAdapterStop } from "../adapter-execution-control.js"; +import { allowLegacyShutdownWorkspaceCleanup, beginLegacyShutdownWorkspaceSettlement, finishLegacyShutdownWorkspaceSettlement, sweepLegacyShutdownWorkspaceSettlements } from "../legacy-shutdown-workspace-settlement.js"; import type { environmentService } from "../environments.js"; import type { environmentRuntimeService, ProviderResourceDisposition } from "../environment-runtime.js"; @@ -1086,12 +1088,13 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD const retryRunIds: string[] = []; const restartSuspendedRunIds: string[] = []; - for (const { run, agent } of activeRuns) { + const settlementDeadline = new Date(Date.now() + 30_000); + const settlements = await Promise.allSettled(activeRuns.map(async ({ run, agent }) => { // Shutdown owns only this boot's legacy executions. Expired foreign // owners belong to the reaper, not another container's drain. if (run.runtimeMode === "legacy" && run.controllerBootId && - run.controllerBootId !== legacyControllerBootId) continue; - if (isNativeRunnerOwnershipHeld(run)) continue; + run.controllerBootId !== legacyControllerBootId) return; + if (isNativeRunnerOwnershipHeld(run)) return; if ( run.runtimeMode === "native" && agent.adapterType === "paperclip_runner" @@ -1152,10 +1155,12 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD }, }); restartSuspendedRunIds.push(run.id); - continue; + return; } const message = `Interrupted by graceful server shutdown (${signal})`; const running = runningProcesses.get(run.id); + const control = run.runtimeMode === "legacy" ? adapterExecutionControls.get(run.id) : undefined; + if (control) await beginLegacyShutdownWorkspaceSettlement(db, run, settlementDeadline); try { if (run.runtimeMode === "native") { await cancelHeartbeatNativeRun({ @@ -1165,7 +1170,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD runtimeMode: run.runtimeMode, }); } - if (running) { + if (running && !control) { await terminateHeartbeatRunProcess({ pid: running.child.pid, processGroupId: running.processGroupId, @@ -1173,7 +1178,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD }); } } finally { - runningProcesses.delete(run.id); + if (!control && runningProcesses.get(run.id) === running) runningProcesses.delete(run.id); } const persistedCancellationResult = @@ -1199,8 +1204,54 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD }), }, ); - if (!interruptedStatus.updated || !interruptedStatus.run) continue; + if (!interruptedStatus.updated || !interruptedStatus.run) { + // A competing terminal write does not prove the owned adapter settled. + if (control) { + control.controller.abort(new Error(message)); + try { + await waitForAdapterStop(control.settled, Math.max(1, settlementDeadline.getTime() - Date.now())); + await finishLegacyShutdownWorkspaceSettlement(db, run); + const current = await getRun(run.id, { includeExecutionEvidence: true }); + if (current && HEARTBEAT_RUN_TERMINAL_STATUSES.some(status => status === current.status)) { + await releaseEnvironmentLeasesForRun({ runId: current.id, companyId: current.companyId, + agentId: current.agentId, status: current.status, failureReason: current.error ?? undefined }); + } + } catch { /* The durable fence retains this source after controller loss. */ } + } + return; + } let interrupted = interruptedStatus.run; + if (control) { + // Keep the exact lease open while its adapter stops the provider and + // copies files back. Provider release closes command admission and + // must not race the adapter's git export or instruction collection. + control.controller.abort(new Error(message)); + try { + await waitForAdapterStop(control.settled, Math.max(1, settlementDeadline.getTime() - Date.now()), { + runId: run.id, adapterType: agent.adapterType, runtimeMode: run.runtimeMode, + abortRequested: true, + }, control); + await finishLegacyShutdownWorkspaceSettlement(db, run); + } catch { + logger.warn({ runId: run.id }, "Shutdown adapter settlement timed out; retaining the sandbox through normal cleanup"); + const sources = await db.select().from(environmentLeases).where(and( + eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.heartbeatRunId, run.id), + sql`${environmentLeases.metadata}->'legacyShutdownWorkspaceSettlement'->>'state' = 'pending'`, + )); + // Stop-only settlement can already have released the source. Claim + // its current fence and pin recovery before classifying or retrying. + for (const source of sources) await allowLegacyShutdownWorkspaceCleanup(db, source, + new Date(Math.max(Date.now(), settlementDeadline.getTime()))); + if (runningProcesses.get(run.id) === running && running) { + await terminateHeartbeatRunProcess({ pid: running.child.pid, processGroupId: running.processGroupId, + graceMs: Math.max(1, running.graceSec) * 1000 }); + if (runningProcesses.get(run.id) === running) runningProcesses.delete(run.id); + } + } + // Settlement can save restore-failure evidence after the shutdown CAS. + // Classification and retry must use the current row, not its snapshot. + interrupted = (await getRun(run.id, { includeExecutionEvidence: true })) ?? interrupted; + } await setWakeupStatus(run.wakeupRequestId, "cancelled", { finishedAt: now, error: null, @@ -1243,6 +1294,10 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run), }); interruptedRunIds.push(interrupted.id); + })); + for (const [index, settlement] of settlements.entries()) { + if (settlement.status === "rejected") logger.warn({ runId: activeRuns[index]!.run.id, + errorKind: "shutdown_settlement_failed" }, "Shutdown settlement failed; durable cleanup protection remains in place"); } if (interruptedRunIds.length > 0) { @@ -1436,6 +1491,10 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD let recovered = 0; for (const { lease } of rows) { + if (!(await allowLegacyShutdownWorkspaceCleanup(db, lease))) { + await deferOrphanedActiveLease(lease.id); + continue; + } // A provider resource id names one physical sandbox. A different lease // row can still hold that same resource in a live status, so this sweep // must not tear down a sandbox that a different lease still owns. @@ -1554,6 +1613,10 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD let capped = 0; for (const row of rows) { if (pendingCleanupAttemptsInFlight.has(row.id)) continue; + if (!(await allowLegacyShutdownWorkspaceCleanup(db, row))) { + await deferPendingCleanupLease(row.id); + continue; + } const metadata = { ...(row.metadata ?? {}) } as Record; const attempts = readPendingCleanupRetryAttempts(metadata); @@ -2370,6 +2433,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD // pending_cleanup sweep, so this tick can stop the recovered sandbox. // Isolate the sweep so its failure never hides the reaper result. try { + await sweepLegacyShutdownWorkspaceSettlements(db); const orphanedActiveLeaseSweep = await sweepOrphanedActiveLeases({ backoffMs: staleThresholdMs, }); diff --git a/server/src/services/legacy-shutdown-workspace-settlement.ts b/server/src/services/legacy-shutdown-workspace-settlement.ts new file mode 100644 index 00000000000..fcf2fbd5448 --- /dev/null +++ b/server/src/services/legacy-shutdown-workspace-settlement.ts @@ -0,0 +1,137 @@ +import { and, eq, gt, inArray, ne, or, sql } from "drizzle-orm"; +import { environmentLeases, heartbeatRuns, issues, type Db } from "@paperclipai/db"; +import { adapterExecutionControls } from "./adapter-execution-control.js"; +import { legacyControllerBootId } from "./legacy-controller-lease.js"; +import { LEGACY_WORKSPACE_RECOVERY_SCHEMA } from "./workspace-restore-recovery-state.js"; + +const SCHEMA = "paperclip.legacy-shutdown-workspace-settlement.v1"; +type Lease = Pick; +type Run = Pick; + +export function pendingLegacyShutdownWorkspaceSettlement(lease: Lease) { + const value = lease.metadata?.legacyShutdownWorkspaceSettlement; + if (!value || typeof value !== "object" || Array.isArray(value)) return null; + const marker = value as Record; + return marker.schema === SCHEMA && marker.state === "pending" && marker.runId === lease.heartbeatRunId + && typeof marker.deadlineAt === "string" ? marker : null; +} + +/** Historical reuse rows may name the same allocation as the fenced source. */ +export async function legacyShutdownWorkspaceResourceProtected(db: Db, + lease: Pick) { + if (!lease.provider || !lease.providerLeaseId) return false; + const [protectedLease] = await db.select({ id: environmentLeases.id }).from(environmentLeases).where(and( + eq(environmentLeases.provider, lease.provider), eq(environmentLeases.providerLeaseId, lease.providerLeaseId), + or(sql`${environmentLeases.metadata}->'legacyShutdownWorkspaceSettlement'->>'schema' = ${SCHEMA} + and ${environmentLeases.metadata}->'legacyShutdownWorkspaceSettlement'->>'state' = 'pending'`, + sql`${environmentLeases.metadata}->'workspaceRestoreRecovery'->>'schema' = ${LEGACY_WORKSPACE_RECOVERY_SCHEMA}`), + )).limit(1); + return !!protectedLease; +} + +/** Released stop receipts still need reconciliation if export never settled. */ +export async function sweepLegacyShutdownWorkspaceSettlements(db: Db, now = new Date()) { + const rows = await db.select().from(environmentLeases).where(and( + sql`${environmentLeases.metadata}->'legacyShutdownWorkspaceSettlement'->>'schema' = ${SCHEMA}`, + sql`${environmentLeases.metadata}->'legacyShutdownWorkspaceSettlement'->>'state' = 'pending'`, + sql`${environmentLeases.metadata}->'legacyShutdownWorkspaceSettlement'->>'deadlineAt' <= ${now.toISOString()}`, + )).orderBy(sql`${environmentLeases.metadata}->'legacyShutdownWorkspaceSettlement'->>'deadlineAt'`).limit(50); + for (const lease of rows) await allowLegacyShutdownWorkspaceCleanup(db, lease, now); +} + +/** Commit the fence while the run is still live, before exposing interruption. */ +export async function beginLegacyShutdownWorkspaceSettlement(db: Db, run: Run, deadlineAt: Date) { + await db.update(environmentLeases).set({ metadata: sql`coalesce(${environmentLeases.metadata}, '{}'::jsonb) || + ${JSON.stringify({ legacyShutdownWorkspaceSettlement: { schema: SCHEMA, state: "pending", runId: run.id, + controllerBootId: legacyControllerBootId, deadlineAt: deadlineAt.toISOString() } })}::jsonb`, + }).where(and(eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.heartbeatRunId, run.id), + eq(environmentLeases.status, "active"), ne(environmentLeases.provider, "local"), + or(sql`${environmentLeases.metadata}->>'driver' = 'sandbox'`, sql`${environmentLeases.metadata}->>'sandboxProviderPlugin' = 'true'`))); +} + +/** The owner joins settlement, or records exact retention before timeout cleanup. */ +export async function finishLegacyShutdownWorkspaceSettlement(db: Db, run: Run, state: "settled" | "expired" = "settled") { + await db.update(environmentLeases).set({ metadata: sql`jsonb_set(${environmentLeases.metadata}, + '{legacyShutdownWorkspaceSettlement,state}', ${JSON.stringify(state)}::jsonb)` }).where(and( + eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.heartbeatRunId, run.id), + sql`${environmentLeases.metadata}->'legacyShutdownWorkspaceSettlement'->>'schema' = ${SCHEMA}`, + sql`${environmentLeases.metadata}->'legacyShutdownWorkspaceSettlement'->>'controllerBootId' = ${legacyControllerBootId}`, + sql`${environmentLeases.metadata}->'legacyShutdownWorkspaceSettlement'->>'state' = 'pending'`, + )); +} + +/** A restart or another cleanup worker must never infer saved files from timeout. */ +export async function allowLegacyShutdownWorkspaceCleanup(db: Db, lease: Lease, now = new Date(), + options: { ownerStopOnly?: boolean } = {}) { + if (!pendingLegacyShutdownWorkspaceSettlement(lease)) return true; + return db.transaction(async tx => { + const [hint] = await tx.select().from(environmentLeases).where(and( + eq(environmentLeases.id, lease.id), eq(environmentLeases.companyId, lease.companyId), + eq(environmentLeases.heartbeatRunId, lease.heartbeatRunId!), + )); + if (!hint) return false; + if (!pendingLegacyShutdownWorkspaceSettlement(hint)) return true; + // Match the terminal recorder's issue -> run -> lease lock order. Settlement + // and expiry compete on the current lease row, not the caller's old snapshot. + if (hint.issueId) await tx.select({ id: issues.id }).from(issues).where(and( + eq(issues.id, hint.issueId), eq(issues.companyId, lease.companyId))).for("update"); + const [run] = await tx.select().from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, lease.companyId), eq(heartbeatRuns.id, lease.heartbeatRunId!), + eq(heartbeatRuns.runtimeMode, "legacy"), + )).for("update"); + const [current] = await tx.select().from(environmentLeases).where(and( + eq(environmentLeases.id, lease.id), eq(environmentLeases.companyId, lease.companyId), + eq(environmentLeases.heartbeatRunId, lease.heartbeatRunId!), + )).for("update"); + if (!run || !current) return false; + const marker = pendingLegacyShutdownWorkspaceSettlement(current); + if (!marker) return true; + // Stopping and retaining the exact owner's command is a prerequisite for + // some adapters to settle. This exception never permits provider destruction. + if (options.ownerStopOnly && marker.controllerBootId === legacyControllerBootId + && adapterExecutionControls.get(run.id)?.controller.signal.aborted) return true; + const deadline = Date.parse(marker.deadlineAt as string); + if (!Number.isFinite(deadline) || deadline > now.getTime()) return false; + // A killed container cannot certify export. Preserve source and hold in + // this same transaction before any worker may perform provider cleanup. + if (current.status !== "active") { + const { hasConfirmedSandboxStopAndRetain, readStopOnlyCleanup } = await import("./sandbox-stop-and-retain.js"); + if (!(current.status === "released" && hasConfirmedSandboxStopAndRetain(current)) + && current.status !== "pending_cleanup") return false; + // The shutdown marker was committed on this exact active source before + // cancellation. A stop receipt does not certify the later file export. + const [competing] = await tx.select({ id: environmentLeases.id }).from(environmentLeases).where(and( + ne(environmentLeases.id, current.id), eq(environmentLeases.provider, current.provider!), + eq(environmentLeases.providerLeaseId, current.providerLeaseId!), + or(inArray(environmentLeases.status, ["active", "pending_cleanup"]), + gt(environmentLeases.acquiredAt, current.acquiredAt)), + )).limit(1); + if (competing) return false; + if (current.status === "pending_cleanup" && !readStopOnlyCleanup(current)) { + const expiresAt = current.metadata?.pendingCleanupLeaseExpiresAtMs; + if (current.metadata?.pendingCleanupInFlight === true + && (typeof expiresAt !== "number" || !Number.isFinite(expiresAt) || expiresAt > now.getTime())) return false; + // A failed ordinary release can race the shutdown marker. Re-enter the + // recorder's active-source path within this claim transaction so it + // replaces destructive retry with an exact stop-only recovery intent. + await tx.update(environmentLeases).set({ status: "active" }).where(eq(environmentLeases.id, current.id)); + } else { + await tx.update(environmentLeases).set({ leasePolicy: "retain_on_failure", + metadata: sql`${environmentLeases.metadata} || ${JSON.stringify({ workspaceRestoreRecovery: { + schema: LEGACY_WORKSPACE_RECOVERY_SCHEMA, runId: run.id, + } })}::jsonb`, + }).where(eq(environmentLeases.id, current.id)); + } + } + const { terminalizeLegacyExecution } = await import("./legacy-execution-recovery.js"); + const preserved = await terminalizeLegacyExecution({ db: tx as unknown as Db, run, status: run.status, + recordRestoreFailureOnly: true, patch: { resultJson: { workspaceRestoreFailure: "restore_failed" } } }); + if (!preserved) return false; + await tx.update(environmentLeases).set({ metadata: sql`jsonb_set(${environmentLeases.metadata}, + '{legacyShutdownWorkspaceSettlement,state}', '"expired"'::jsonb)` }).where(and( + eq(environmentLeases.id, lease.id), eq(environmentLeases.companyId, lease.companyId), + eq(environmentLeases.heartbeatRunId, run.id), + )); + return true; + }); +}