diff --git a/server/src/__tests__/environment-retained-orphan-sandbox.test.ts b/server/src/__tests__/environment-retained-orphan-sandbox.test.ts index 8c7b329bf6f..480e3deb85b 100644 --- a/server/src/__tests__/environment-retained-orphan-sandbox.test.ts +++ b/server/src/__tests__/environment-retained-orphan-sandbox.test.ts @@ -1,6 +1,6 @@ import { spawn } from "node:child_process"; import { randomUUID } from "node:crypto"; -import { eq } from "drizzle-orm"; +import { eq, ne } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { agents, @@ -8,6 +8,7 @@ import { createDb, environmentLeases, environments, + executionWorkspaces, heartbeatRuns, issueRecoveryActions, issues, @@ -59,6 +60,9 @@ suite("retained orphan sandbox on the shared ownership path", () => { await db.delete(issueRecoveryActions); await db.delete(environmentLeases); await db.delete(heartbeatRuns); + await db.update(issues).set({ executionWorkspaceId: null }).where(eq(issues.id, issueId)); + await db.delete(issues).where(ne(issues.id, issueId)); + await db.delete(executionWorkspaces); }); afterAll(async () => { @@ -292,6 +296,127 @@ suite("retained orphan sandbox on the shared ownership path", () => { expect((await row(leaseId)).status).toBe("pending_cleanup"); }); + async function workspace() { + const [created] = await db.insert(executionWorkspaces).values({ companyId, mode: "isolated_workspace", + strategyType: "git_worktree", name: "Task workspace" }).returning({ id: executionWorkspaces.id }); + return created!.id; + } + + /** A warm sandbox the previous release left behind: a reuse_by_environment + * lease the plugin stopped and kept for the next turn on its workspace. This + * is the exact row shape releases before this change wrote; it carries no + * new marker. */ + async function warmLease(runId: string, input: { status?: string; workspaceId: string | null; receipt?: boolean; + providerLeaseId?: string; createdAt?: Date }) { + const id = randomUUID(), providerLeaseId = input.providerLeaseId ?? `sandbox://fake/${id}`; + const now = input.createdAt ?? new Date(); + await db.insert(environmentLeases).values({ id, companyId, environmentId, issueId, heartbeatRunId: runId, + executionWorkspaceId: input.workspaceId, status: input.status ?? "released", cleanupStatus: "success", + leasePolicy: "reuse_by_environment", provider: "fake", providerLeaseId, acquiredAt: now, releasedAt: now, + createdAt: now, updatedAt: now, + metadata: { agentId, reuseLease: true, ...(input.receipt === false ? {} : { remoteExecutionTermination: { + schema: "paperclip.remote-termination.v1", companyId, runId, leaseId: id, provider: "fake", providerLeaseId, + state: "stopped", confirmedAt: now.toISOString() } }) } }); + return id; + } + + it("destroys a task's warm sandbox once the task moves to another workspace", async () => { + const [oldWorkspace, newWorkspace] = [await workspace(), await workspace()]; + const leaseId = await warmLease(await run("failed"), { workspaceId: oldWorkspace }); + // A Retry kept a fresh workspace: no acquire can match the old lineage again. + await db.update(issues).set({ executionWorkspaceId: newWorkspace }).where(eq(issues.id, issueId)); + const { kind } = await pass(); + expect(kind).toMatchObject({ settled: 1 }); + const released = await row(leaseId); + expect(released).toMatchObject({ status: "pending_cleanup", failureReason: "retained_orphan_sandbox_released" }); + // The successor is never blocked on this destroy. + expect(hasRecoveredCleanupStopReceipt(released)).toBe(true); + expect(await hasPendingRunCleanup(db, companyId, released.heartbeatRunId!)).toBe(false); + }); + + it("destroys a warm sandbox behind a failed lease, which no turn can resume", async () => { + const current = await workspace(); + await db.update(issues).set({ executionWorkspaceId: current }).where(eq(issues.id, issueId)); + const leaseId = await warmLease(await run("failed"), { status: "failed", workspaceId: current }); + const { kind } = await pass(); + expect(kind).toMatchObject({ settled: 1 }); + expect((await row(leaseId)).status).toBe("pending_cleanup"); + }); + + it.each([ + ["the task can still resume it on its own workspace", async () => { + const current = await workspace(); + await db.update(issues).set({ executionWorkspaceId: current }).where(eq(issues.id, issueId)); + return [await warmLease(await run("succeeded"), { workspaceId: current })]; + }], + ["it has no confirmed stop receipt", async () => { + await db.update(issues).set({ executionWorkspaceId: await workspace() }).where(eq(issues.id, issueId)); + return [await warmLease(await run("failed"), { status: "failed", workspaceId: await workspace(), receipt: false })]; + }], + ["a newer lease on the same sandbox handed it on", async () => { + const current = await workspace(); + await db.update(issues).set({ executionWorkspaceId: current }).where(eq(issues.id, issueId)); + const providerLeaseId = `sandbox://fake/${randomUUID()}`; + const older = await warmLease(await run("failed"), { status: "failed", workspaceId: current, providerLeaseId, + createdAt: new Date(Date.now() - 60_000) }); + const newer = await warmLease(await run("succeeded"), { workspaceId: current, providerLeaseId }); + return [older, newer]; + }], + ["another task still shares its workspace", async () => { + const [shared, fresh] = [await workspace(), await workspace()]; + await db.update(issues).set({ executionWorkspaceId: fresh }).where(eq(issues.id, issueId)); + await db.insert(issues).values({ companyId, title: "Sibling on the shared workspace", executionWorkspaceId: shared }); + return [await warmLease(await run("failed"), { workspaceId: shared })]; + }], + ["the task still has a run in flight", async () => { + const [oldWorkspace, newWorkspace] = [await workspace(), await workspace()]; + await db.update(issues).set({ executionWorkspaceId: newWorkspace }).where(eq(issues.id, issueId)); + const leaseId = await warmLease(await run("failed"), { workspaceId: oldWorkspace }); + await run("running"); + return [leaseId]; + }], + ] as const)("keeps a warm sandbox while %s", async (_label, arrange) => { + const ids = await arrange(); + const before = await Promise.all(ids.map(row)); + const { kind } = await pass(); + expect(kind).toMatchObject({ settled: 0 }); + expect(await Promise.all(ids.map(row))).toEqual(before); + }); + + it("stops, then destroys, a running warm sandbox once its task moves to another workspace", async () => { + const [oldWorkspace, newWorkspace] = [await workspace(), await workspace()]; + const runId = await run("succeeded"); + // A keep-running release: retained, no stop, no receipt. Its idle timer + // lived in the process that kept it. + const leaseId = await warmLease(runId, { status: "retained", workspaceId: oldWorkspace, receipt: false }); + await db.update(environmentLeases).set({ releasedAt: null }).where(eq(environmentLeases.id, leaseId)); + await db.update(issues).set({ executionWorkspaceId: newWorkspace }).where(eq(issues.id, issueId)); + + expect((await pass()).kind).toMatchObject({ settled: 1 }); + const stopping = await row(leaseId); + expect(stopping).toMatchObject({ status: "pending_cleanup", failureReason: "sandbox_stop_pending" }); + const intent = stopping.metadata?.sandboxStopAndRetain as { requestId: string; recoveredOrphanedRunLease?: boolean }; + expect(intent).toMatchObject({ recoveredOrphanedRunLease: true, supersededWarmSandbox: true }); + // The cleanup sweep's first stop is due at once. + expect(await nextEnvironmentCleanupAt(db)).toBeLessThanOrEqual(Date.now()); + + // The provider confirms the stop; the next pass hands it to destruction. + const stopped = await settleStopOnlyCleanup(db, stopping, { attemptId: intent.requestId, + receipt: { providerLeaseId: stopping.providerLeaseId, state: "stopped" } }); + expect(stopped).toMatchObject({ status: "released", cleanupStatus: "success" }); + expect((await pass()).kind).toMatchObject({ settled: 1 }); + expect(await row(leaseId)).toMatchObject({ status: "pending_cleanup", failureReason: "retained_orphan_sandbox_released" }); + }); + + it("keeps a running warm sandbox its task can still resume after a restart", async () => { + const current = await workspace(); + await db.update(issues).set({ executionWorkspaceId: current }).where(eq(issues.id, issueId)); + const leaseId = await warmLease(await run("succeeded"), { status: "retained", workspaceId: current, receipt: false }); + const before = await row(leaseId); + expect((await pass()).kind).toMatchObject({ settled: 0 }); + expect(await row(leaseId)).toEqual(before); + }); + it("leaves an ordinary retained sandbox (no orphan marker) alone", async () => { const leaseId = await retainedOrphan(await run("failed")); const { [ORPHANED_SANDBOX_RECOVERY_KEY]: _marker, ...metadata } = (await row(leaseId)).metadata!; diff --git a/server/src/services/environment-retained-orphan-settlement.ts b/server/src/services/environment-retained-orphan-settlement.ts index 285952b4c68..f876aab7a79 100644 --- a/server/src/services/environment-retained-orphan-settlement.ts +++ b/server/src/services/environment-retained-orphan-settlement.ts @@ -1,11 +1,11 @@ -import { and, eq, inArray, ne, notInArray, sql } from "drizzle-orm"; +import { and, eq, inArray, ne, notInArray, or, sql } from "drizzle-orm"; import { environmentLeases, heartbeatRuns, issueRecoveryActions, type Db } from "@paperclipai/db"; import { writeEnvironmentCleanup } from "./environment-cleanup-notifications.js"; import { nativeWorkspaceExportPendingForLease } from "./native-runtime/native-workspace-finalization-state.js"; import type { OwnerProof } from "./owner-proof.js"; import type { InProgressRecord, SettleOutcome } from "./recovery/ownership-registry.js"; import { hasRemoteTerminationReceipt } from "./remote-execution-termination.js"; -import { ORPHANED_SANDBOX_RECOVERY_KEY } from "./sandbox-stop-and-retain.js"; +import { ORPHANED_SANDBOX_RECOVERY_KEY, prepareSupersededWarmSandboxStop } from "./sandbox-stop-and-retain.js"; /** Failure reason on a retained orphan sandbox released for destruction. */ export const RETAINED_ORPHAN_RELEASED_REASON = "retained_orphan_sandbox_released"; @@ -17,8 +17,64 @@ const OPEN_RUN_STATUSES = ["queued", "scheduled_retry", "running"]; type LeaseRow = typeof environmentLeases.$inferSelect; /** A retained orphan: the ownership settle stopped and kept its sandbox, and - * the stop is confirmed. */ + * the stop is confirmed. Or an abandoned warm sandbox (see below). */ export function retainedOrphanSandboxCondition() { + return or(orphanedRunSandboxCondition(), abandonedWarmSandboxCondition(), supersededRunningWarmSandboxCondition()); +} + +/** The newest lease row for its sandbox: an older row was handed on. */ +function newestRowForSandbox() { + return sql`not exists (select 1 from environment_leases newer where newer.id <> ${environmentLeases.id} + and newer.provider = ${environmentLeases.provider} and newer.provider_lease_id = ${environmentLeases.providerLeaseId} + and newer.created_at > ${environmentLeases.createdAt})`; +} + +/** No task is bound to this lease's workspace any more (its own task moved to + * a fresh one, and no other task shares it), so reuse, which matches the exact + * workspace and agent, can never pick the sandbox again. */ +function taskMovedToAnotherWorkspace() { + return and(sql`${environmentLeases.executionWorkspaceId} is not null`, + sql`not exists (select 1 from issues bound where bound.company_id = ${environmentLeases.companyId} + and bound.execution_workspace_id = ${environmentLeases.executionWorkspaceId})`); +} + +/** + * A warm sandbox kept running for reuse (`retained` after a keep-running + * release) whose task moved to another workspace. Its idle timer lived in the + * process that kept it, so after a restart nothing else would ever stop it. + * The settle stops it through the stop-only contract; once the provider + * confirms the stop, it is an abandoned warm sandbox and is destroyed. + */ +export function supersededRunningWarmSandboxCondition() { + return and(eq(environmentLeases.status, "retained"), eq(environmentLeases.leasePolicy, "reuse_by_environment"), + eq(environmentLeases.cleanupStatus, "success"), sql`${environmentLeases.heartbeatRunId} is not null`, + sql`${environmentLeases.provider} is not null`, sql`${environmentLeases.providerLeaseId} is not null`, + newestRowForSandbox(), taskMovedToAnotherWorkspace()); +} + +/** + * A warm sandbox no turn can resume: a reuse_by_environment lease whose + * sandbox the provider confirmed stopped and kept, on the newest row for that + * sandbox, where either + * - the row is `failed`, which reuse never resumes; or + * - the row is `released` (stopped, kept for reuse) but bound to a workspace the + * task no longer runs in: a Retry on a remote task keeps a fresh workspace, + * and reuse matches only the same workspace. + * Releases before this change wrote exactly these rows, so the condition reads + * existing columns only. A row without a confirmed stop is never listed. + */ +export function abandonedWarmSandboxCondition() { + return and(eq(environmentLeases.leasePolicy, "reuse_by_environment"), eq(environmentLeases.cleanupStatus, "success"), + sql`${environmentLeases.heartbeatRunId} is not null`, sql`${environmentLeases.provider} is not null`, + sql`${environmentLeases.providerLeaseId} is not null`, + sql`${environmentLeases.metadata}#>>'{remoteExecutionTermination,state}' = 'stopped'`, + sql`${environmentLeases.metadata}#>>'{remoteExecutionTermination,leaseId}' = ${environmentLeases.id}::text`, + sql`${environmentLeases.metadata}#>>'{remoteExecutionTermination,runId}' = ${environmentLeases.heartbeatRunId}::text`, + newestRowForSandbox(), + or(eq(environmentLeases.status, "failed"), and(eq(environmentLeases.status, "released"), taskMovedToAnotherWorkspace()))); +} + +function orphanedRunSandboxCondition() { return and(eq(environmentLeases.status, "released"), eq(environmentLeases.cleanupStatus, "success"), // An ephemeral lease only: a reusable lease stays for warm reuse, and a // retain-on-failure lease belongs to workspace restore recovery. @@ -127,15 +183,23 @@ export async function settleRetainedOrphanSandbox(db: Db, record: InProgressReco const runId = record.runId; return db.transaction(async transaction => { const tx = transaction as unknown as Db; + // Re-judge the row under its lock with the same listing predicate. const [lease] = await tx.select().from(environmentLeases).where(and(eq(environmentLeases.id, record.recordId), - eq(environmentLeases.heartbeatRunId, runId), eq(environmentLeases.status, "released"), - eq(environmentLeases.cleanupStatus, "success"))).for("update"); - if (!lease || !hasRemoteTerminationReceipt(lease) || lease.leasePolicy !== "ephemeral" || !lease.metadata?.[ORPHANED_SANDBOX_RECOVERY_KEY] - || !lease.provider || !lease.providerLeaseId) { - return { settled: false, reason: "no_longer_a_retained_orphan" }; - } + eq(environmentLeases.heartbeatRunId, runId), retainedOrphanSandboxCondition())).for("update"); + if (!lease || !lease.provider || !lease.providerLeaseId) return { settled: false, reason: "no_longer_a_retained_orphan" }; const held = await heldBy(tx, lease, runId, context.now); if (held) return { settled: false, reason: held }; + if (lease.status === "retained") { + // Still running: stop it first. The confirmed stop makes it destroyable. + const stopping = await prepareSupersededWarmSandboxStop(tx, lease, + { recoveredOrphanedRunLease: true, supersededWarmSandbox: true, ownerProof: context.proof.reason }); + return stopping ? { settled: true, reason: context.proof.reason } : { settled: false, reason: "lease_changed_since_proof" }; + } + if (!hasRemoteTerminationReceipt(lease) + || (lease.leasePolicy === "ephemeral" ? !lease.metadata?.[ORPHANED_SANDBOX_RECOVERY_KEY] : lease.leasePolicy !== "reuse_by_environment")) { + return { settled: false, reason: "no_longer_a_retained_orphan" }; + } + const status = lease.status; const now = context.now; // The destroy is queued through the recognized stopped-cleanup contract // (`workspaceRecoveryCleanup` with the verified stop receipt), so admission @@ -149,7 +213,7 @@ export async function settleRetainedOrphanSandbox(db: Db, record: InProgressReco pendingCleanupInFlight: false, pendingCleanupRetryAfterMs: 0, pendingCleanupRetryAttempts: 0, pendingCleanupLeaseExpiresAtMs: 0 })}::jsonb`, }).where(and(eq(environmentLeases.id, lease.id), eq(environmentLeases.heartbeatRunId, runId), - eq(environmentLeases.status, "released"), eq(environmentLeases.cleanupStatus, "success"), + eq(environmentLeases.status, status), eq(environmentLeases.cleanupStatus, "success"), sql`${environmentLeases.metadata} = ${JSON.stringify(lease.metadata)}::jsonb`)) .returning({ id: environmentLeases.id }), { companyId: lease.companyId, runId, leaseId: lease.id }); return released.length ? { settled: true, reason: context.proof.reason } : { settled: false, reason: "lease_changed_since_proof" }; diff --git a/server/src/services/sandbox-stop-and-retain.ts b/server/src/services/sandbox-stop-and-retain.ts index 6269ed3b59d..27e4122b3cd 100644 --- a/server/src/services/sandbox-stop-and-retain.ts +++ b/server/src/services/sandbox-stop-and-retain.ts @@ -152,7 +152,16 @@ function stopOnlyAttempt(lease: Lease, attemptId: string) { sql`${environmentLeases.metadata}->>'pluginId' is not distinct from ${readStopOnlyCleanup(lease)?.pluginId ?? null}`); } -async function prepareStopAndRetain(db: Db, lease: Lease, fromStatus: "active" | "released", recovery: Record = {}) { +/** A warm sandbox kept running for reuse whose task moved to another workspace: + * stop it through the stop-only contract so the provider confirms the stop + * before anything destroys it. The ownership settle verifies the move. */ +export async function prepareSupersededWarmSandboxStop(db: Db, lease: typeof environmentLeases.$inferSelect, recovery: Record) { + if (lease.status !== "retained" || lease.leasePolicy !== "reuse_by_environment" || !lease.heartbeatRunId + || !lease.providerLeaseId || lease.metadata?.pendingCleanupInFlight === true) return null; + return prepareStopAndRetain(db, lease, "retained", recovery); +} + +async function prepareStopAndRetain(db: Db, lease: Lease, fromStatus: "active" | "released" | "retained", recovery: Record = {}) { const requestId = randomUUID(), now = new Date(); const intent = { schema: "paperclip.sandbox-stop-and-retain.v1", requestId, companyId: lease.companyId, runId: lease.heartbeatRunId, leaseId: lease.id,