diff --git a/server/src/__tests__/legacy-workspace-restore-lock-contention.test.ts b/server/src/__tests__/legacy-workspace-restore-lock-contention.test.ts new file mode 100644 index 00000000000..938de577029 --- /dev/null +++ b/server/src/__tests__/legacy-workspace-restore-lock-contention.test.ts @@ -0,0 +1,66 @@ +import { randomUUID } from "node:crypto"; +import { eq } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { agents, companies, createDb, environmentLeases, environments, executionWorkspaces, heartbeatRuns, issues, projects } from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { terminalizeLegacyExecution } from "../services/legacy-execution-recovery.js"; +import { hasRequiredWorkspaceRecovery } from "../services/workspace-restore-recovery-state.js"; + +const externalTestDatabaseUrl = process.env.PAPERCLIP_TEST_DATABASE_URL?.trim(); +const support = externalTestDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); +(support.supported ? describe : describe.skip)("workspace restore hold under environment lock contention", () => { + let db: ReturnType; + let temporary: Awaited>; + beforeAll(async () => { if (externalTestDatabaseUrl) db = createDb(externalTestDatabaseUrl); + else { temporary = await startEmbeddedPostgresTestDatabase("workspace-restore-lock-"); db = createDb(temporary.connectionString); } }, 30_000); + afterAll(async () => { await db?.$client.end(); await temporary?.cleanup(); }); + + async function seed(runtimeMode: string) { + const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), runId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Workspace recovery", issuePrefix: companyId.slice(0, 8) }); + await db.insert(agents).values({ id: agentId, companyId, name: "Engineer", role: "engineer", adapterType: "codex_local" }); + await db.insert(issues).values({ id: issueId, companyId, title: "Restore files", status: "in_progress", assigneeAgentId: agentId }); + const [project] = await db.insert(projects).values({ companyId, name: "Workspace project" }).returning(); + const [workspace] = await db.insert(executionWorkspaces).values({ companyId, projectId: project.id, + sourceIssueId: issueId, mode: "isolated", strategyType: "git_worktree", name: "Fixture workspace", cwd: "/work/fixture", + }).returning(); + const [run] = await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, issueId, status: "failed", runtimeMode, + finishedAt: new Date(), contextSnapshot: { issueId, executionWorkspaceId: workspace.id }, + }).returning(); + const [environment] = await db.insert(environments).values({ name: `Shared sandbox ${runId}`, driver: "sandbox" }).returning(); + const [lease] = await db.insert(environmentLeases).values({ companyId, issueId, executionWorkspaceId: workspace.id, heartbeatRunId: runId, + environmentId: environment.id, status: "active", leasePolicy: "ephemeral", provider: "daytona", providerLeaseId: randomUUID(), + metadata: { remoteCwd: "/work/fixture", workspaceRealization: { authoritativeRoot: "/work/fixture" }, driver: "sandbox", + pluginId: randomUUID(), sandboxProviderPlugin: true }, + }).returning(); + return { runId, run, lease }; + } + + // A second write to a row this transaction already wrote re-runs its + // foreign-key checks, which take KEY SHARE locks on the referenced rows. + // Another run's lease acquisition holds the shared environment FOR UPDATE. + it.each(["legacy", "native"])("retains an active %s source while another run holds its environment", async (runtimeMode) => { + const f = await seed(runtimeMode); + let locked!: () => void, release!: () => void; + const isLocked = new Promise(resolve => { locked = resolve; }); + const released = new Promise(resolve => { release = resolve; }); + const holder = db.transaction(async tx => { + await tx.select({ id: environments.id }).from(environments).where(eq(environments.id, f.lease.environmentId!)).for("update"); + locked(); + await released; + }); + await isLocked; + let returned: Awaited>; + try { + returned = await terminalizeLegacyExecution({ db, run: f.run, status: "failed", recordRestoreFailureOnly: runtimeMode === "native", + workspaceRestoreSource: f.lease, patch: { resultJson: { workspaceRestoreFailure: "restore_failed" } } }); + } finally { release(); await holder; } + expect(hasRequiredWorkspaceRecovery(returned!.resultJson)).toBe(true); + const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id)); + expect(lease).toMatchObject({ status: "pending_cleanup", leasePolicy: "retain_on_failure", metadata: expect.objectContaining({ + remoteCwd: "/work/fixture", + workspaceRestoreRecovery: { schema: "paperclip.workspace-restore-recovery.v1", runId: f.runId, originalLeasePolicy: "ephemeral" }, + sandboxStopAndRetain: expect.objectContaining({ leaseId: f.lease.id, runId: f.runId }), + }) }); + }); +}); diff --git a/server/src/services/legacy-workspace-restore-recovery.ts b/server/src/services/legacy-workspace-restore-recovery.ts index 99a5ceb5691..9c448052608 100644 --- a/server/src/services/legacy-workspace-restore-recovery.ts +++ b/server/src/services/legacy-workspace-restore-recovery.ts @@ -108,18 +108,13 @@ export async function preserveLegacyWorkspaceRestoreSources(db: Db, run: Run, so // The stop-only worker rechecks exact allocation ownership, including // competing leases, before dispatch. Persist the intent even when cleanup // must wait, so no terminal-run sweep can fall back to destruction. - const prepared = await prepareSandboxStopAndRetain(db, lease); - if (!prepared) throw new Error("Workspace recovery lease ownership changed before terminal commit."); // The original policy lets the settled hold return a warm sandbox to reuse // (settleRecoveredWorkspaceRestoreHolds). It grants nothing while held. - await db.update(environmentLeases).set({ leasePolicy: "retain_on_failure", - metadata: sql`${environmentLeases.metadata} || ${JSON.stringify({ workspaceRestoreRecovery: { + const prepared = await prepareSandboxStopAndRetain(db, lease, undefined, { leasePolicy: "retain_on_failure", + metadata: { workspaceRestoreRecovery: { schema: LEGACY_WORKSPACE_RECOVERY_SCHEMA, runId: run.id, originalLeasePolicy: lease.leasePolicy, - } })}::jsonb`, - }).where(and( - eq(environmentLeases.id, prepared.id), eq(environmentLeases.companyId, run.companyId), - eq(environmentLeases.heartbeatRunId, run.id), eq(environmentLeases.status, "pending_cleanup"), - )); + } } }); + if (!prepared) throw new Error("Workspace recovery lease ownership changed before terminal commit."); retained.push(lease.id); } return retained; diff --git a/server/src/services/sandbox-stop-and-retain.ts b/server/src/services/sandbox-stop-and-retain.ts index aea0762201a..20f8c2de44a 100644 --- a/server/src/services/sandbox-stop-and-retain.ts +++ b/server/src/services/sandbox-stop-and-retain.ts @@ -96,9 +96,13 @@ export function readStopOnlyCleanup(lease: Lease) { * an explicit stop into ordinary destructive release. Invalid pins stay pending. * `recovery` marks an ownership settle on the intent and on the lease * (`ORPHANED_SANDBOX_RECOVERY_KEY`); the pending-cleanup sweep dispatches such - * a stop's first attempt without its retry backoff. */ -export async function prepareSandboxStopAndRetain(db: Db, lease: Lease, recovery?: Record) { - return prepareStopAndRetain(db, lease, "active", recovery); + * a stop's first attempt without its retry backoff. `retention` sets a repair + * hold's policy and metadata in the same write: a second write to the row in + * one transaction re-runs its foreign-key checks, which wait on any run that + * holds the shared environment row. */ +export async function prepareSandboxStopAndRetain(db: Db, lease: Lease, recovery?: Record, + retention?: { leasePolicy: string; metadata: Record }) { + return prepareStopAndRetain(db, lease, "active", recovery, retention); } /** Historical release rows can lack a stop receipt. Re-inspect the exact @@ -189,7 +193,8 @@ export async function prepareUnconfirmedSourceSandboxStop(db: Db, lease: typeof return prepareStopAndRetain(db, lease, lease.status); } -async function prepareStopAndRetain(db: Db, lease: Lease, fromStatus: string, recovery: Record = {}) { +async function prepareStopAndRetain(db: Db, lease: Lease, fromStatus: string, recovery: Record = {}, + retention?: { leasePolicy: string; metadata: Record }) { const requestId = randomUUID(), now = new Date(); const intent = { schema: "paperclip.sandbox-stop-and-retain.v1", requestId, companyId: lease.companyId, runId: lease.heartbeatRunId, leaseId: lease.id, @@ -198,12 +203,14 @@ async function prepareStopAndRetain(db: Db, lease: Lease, fromStatus: string, re ? { builtinProvider: lease.provider } : { pluginId: lease.metadata?.pluginId }), ...recovery }; const [updated] = await writeEnvironmentCleanup(db, tx => tx.update(environmentLeases).set({ status: "pending_cleanup", cleanupStatus: "failed", failureReason: "sandbox_stop_pending", releasedAt: now, updatedAt: now, + ...(retention ? { leasePolicy: retention.leasePolicy } : {}), metadata: sql`(coalesce(${environmentLeases.metadata}, '{}'::jsonb) - 'remoteExecutionTermination' - 'sandboxStopAndRetainReceipt') || ${JSON.stringify({ [SANDBOX_STOP_AND_RETAIN_KEY]: intent, pendingCleanupAttemptId: requestId, ...(fromStatus === "active" && Object.keys(recovery).length ? { [ORPHANED_SANDBOX_RECOVERY_KEY]: { ...recovery, runId: lease.heartbeatRunId, recoveredAt: now.toISOString() } } : {}), pendingCleanupInFlight: false, pendingCleanupLeaseExpiresAtMs: 0, pendingCleanupRetryAfterMs: 0, pendingCleanupRetryAttempts: 0, pendingCleanupRetryCapWarned: false, + ...retention?.metadata, })}::jsonb`, }).where(and(eq(environmentLeases.id, lease.id), eq(environmentLeases.companyId, lease.companyId), eq(environmentLeases.heartbeatRunId, lease.heartbeatRunId!), eq(environmentLeases.status, fromStatus),