diff --git a/doc/native-workspace-finalization-recovery.md b/doc/native-workspace-finalization-recovery.md index 469514c278e..2be7f1b67db 100644 --- a/doc/native-workspace-finalization-recovery.md +++ b/doc/native-workspace-finalization-recovery.md @@ -219,3 +219,28 @@ plugin RPC. Its intent pins the registered built-in provider and its receipt say `builtin.stopLease`. This provider owns no real process or filesystem; the receipt models its lifecycle. Missing built-in stop support or an unconfirmed receipt also remains pending through restart, without calling release or destroy. + +### Legacy copy-back after a confirmed stop + +A legacy adapter can report failed workspace copy-back after cancellation has +already stopped and released its sandbox lease. The host records the exact lease +snapshot from before adapter dispatch. For an originally ephemeral source, the +recorder can preserve the released allocation for repair only while the current +lease is still ephemeral and its company, run, issue, workspace, provider, +allocation, acquisition time, plugin, and realized root still match. It requires +both the dedicated stop-and-retain receipt and the matching stopped-execution +receipt. A competing owner or changed source fails closed. The recorder leaves +release status and receipts intact, excludes the allocation from reuse, and +creates the existing board-owned file-repair obligation without replaying work +or changing a newer run or the task's status. + +This late-recording path does not admit reusable leases. Provider resume can +precede the database handoff, so an old stop receipt and a database row lock +cannot prove that reuse has not started. Existing active-source and previously +recorded recovery paths retain their behavior. A missing or rejected original +snapshot does not authorize adopting another released lease. + +A retained source receipt records control-plane evidence; it does not prove the +provider still holds the original bytes or that copy-back succeeded. Operators +must verify and recover the exact source separately. Later successful runs and +completed tasks do not certify recovery of earlier workspace files. diff --git a/server/src/__tests__/legacy-workspace-restore-recovery.test.ts b/server/src/__tests__/legacy-workspace-restore-recovery.test.ts index 24d12c6674c..6278de98192 100644 --- a/server/src/__tests__/legacy-workspace-restore-recovery.test.ts +++ b/server/src/__tests__/legacy-workspace-restore-recovery.test.ts @@ -1,12 +1,13 @@ import { randomUUID } from "node:crypto"; import { and, eq, sql } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; -import { agents, companies, createDb, environmentLeases, environments, heartbeatRuns, issueRecoveryActions, issues } from "@paperclipai/db"; +import { agents, companies, createDb, environmentLeases, environments, executionWorkspaces, heartbeatRuns, issueRecoveryActions, issues, projects } from "@paperclipai/db"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; import { recordLegacyWorkspaceRestoreFailure, legacyExecutionNeedsReconciliationWithEvidence, terminalizeLegacyExecution } from "../services/legacy-execution-recovery.js"; import { hasConversationContinuationPolicy, conversationRecoveryActionPredicate } from "../services/conversation-continuation.js"; import { preserveWorkspaceRestoreRecoveryMetadataSql } from "../services/legacy-workspace-restore-recovery.js"; -import { settleStopOnlyCleanup } from "../services/sandbox-stop-and-retain.js"; +import { hasConfirmedSandboxStopAndRetain, prepareSandboxStopAndRetain, settleStopOnlyCleanup } from "../services/sandbox-stop-and-retain.js"; +import { environmentService } from "../services/environments.js"; import { hasRequiredWorkspaceRecovery } from "../services/workspace-restore-recovery-state.js"; const externalTestDatabaseUrl = process.env.PAPERCLIP_TEST_DATABASE_URL?.trim(); @@ -22,14 +23,20 @@ const support = externalTestDatabaseUrl ? { supported: true } : await getEmbedde await db.insert(companies).values({ id: companyId, name: "Workspace recovery", issuePrefix: companyId.slice(0, 8) }); await db.insert(agents).values({ id: agentId, companyId, name: "Engineer", role: "engineer", adapterType: "codex_local" }); await db.insert(issues).values({ id: issueId, companyId, title: "Restore files", status: options.taskDone ? "done" : "in_progress", assigneeAgentId: agentId }); - const [run] = await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "running", runtimeMode: "legacy", contextSnapshot: { issueId } }).returning(); + const [project] = await db.insert(projects).values({ companyId, name: "Workspace project" }).returning(); + const [workspace] = await db.insert(executionWorkspaces).values({ companyId, projectId: project.id, + sourceIssueId: issueId, mode: "isolated", strategyType: "git_worktree", name: "Fixture workspace", cwd: "/work/fixture", + }).returning(); + const [run] = await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, issueId, status: "running", runtimeMode: "legacy", + contextSnapshot: { issueId, executionWorkspaceId: workspace.id }, + }).returning(); if (options.local) await db.insert(environments).values({ name: "Local", driver: "local" }).onConflictDoNothing(); const [environment] = options.local ? await db.select().from(environments).where(eq(environments.driver, "local")) : await db.insert(environments).values({ name: `Isolated test sandbox ${runId}`, driver: "sandbox" }).returning(); - const [lease] = await db.insert(environmentLeases).values({ companyId, issueId, heartbeatRunId: runId, environmentId: environment.id, + const [lease] = await db.insert(environmentLeases).values({ companyId, issueId, executionWorkspaceId: workspace.id, heartbeatRunId: runId, environmentId: environment.id, status: "active", leasePolicy: options.reusable ? "reuse_by_environment" : "ephemeral", provider: options.local ? "local" : "daytona", providerLeaseId: randomUUID(), - metadata: { driver: options.local ? "local" : "sandbox", pluginId: randomUUID(), sandboxProviderPlugin: !options.local }, + metadata: { remoteCwd: "/work/fixture", workspaceRealization: { authoritativeRoot: "/work/fixture" }, driver: options.local ? "local" : "sandbox", pluginId: randomUUID(), sandboxProviderPlugin: !options.local }, }).returning(); return { companyId, runId, issueId, run, lease }; } @@ -38,6 +45,22 @@ const support = externalTestDatabaseUrl ? { supported: true } : await getEmbedde } }; const readLease = async (id: string) => (await db.select().from(environmentLeases).where(eq(environmentLeases.id, id)))[0]; + async function stopAndRetain(f: Awaited>, retry = false) { + let stopping = (await prepareSandboxStopAndRetain(db, f.lease))!; + expect(stopping.status).toBe("pending_cleanup"); + if (retry) { + [stopping] = await db.update(environmentLeases).set({ metadata: { + ...stopping.metadata, pendingCleanupAttemptId: randomUUID(), + } }).where(eq(environmentLeases.id, f.lease.id)).returning(); + } + const stopped = (await settleStopOnlyCleanup(db, stopping, { + attemptId: String(stopping.metadata!.pendingCleanupAttemptId), + receipt: { providerLeaseId: f.lease.providerLeaseId, state: "stopped" }, + }))!; + expect(stopped.status).toBe("released"); + return stopped; + } + it.each([false, true])("persists terminal status, stop-only source and board action atomically (task done=%s)", async taskDone => { const f = await seed({ taskDone, reusable: true }); expect(await legacyExecutionNeedsReconciliationWithEvidence(db, { ...f.run, status: "failed", ...patch })).toBe(true); @@ -69,6 +92,239 @@ const support = externalTestDatabaseUrl ? { supported: true } : await getEmbedde expect((await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId)))).toHaveLength(1); }); + it("records the exact stopped source when provider cancellation finishes before copy-back", async () => { + const f = await seed({ taskDone: true }); + const stopping = await prepareSandboxStopAndRetain(db, f.lease); + expect(stopping).not.toBeNull(); + const stopped = await settleStopOnlyCleanup(db, stopping!, { + attemptId: String(stopping!.metadata?.pendingCleanupAttemptId), + receipt: { providerLeaseId: f.lease.providerLeaseId, state: "stopped" }, + }); + expect(stopped?.status).toBe("released"); + await db.update(heartbeatRuns).set({ status: "cancelled", finishedAt: new Date(), + resultJson: { executionCancellation: { state: "acknowledged" } }, + }).where(eq(heartbeatRuns.id, f.runId)); + // This is the lease captured by the host before invoking the adapter. + await recordLegacyWorkspaceRestoreFailure(db, f.run, patch.resultJson, f.lease); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + expect(run.status).toBe("cancelled"); + expect(run.resultJson?.executionCancellation).toEqual({ state: "acknowledged" }); + expect(run.resultJson?.workspaceRestoreRecovery).toMatchObject({ leaseIds: [f.lease.id] }); + const retained = await readLease(f.lease.id); + expect(retained).toMatchObject({ status: "released", leasePolicy: "retain_on_failure", cleanupStatus: "success", + releasedAt: stopped!.releasedAt, metadata: { remoteExecutionTermination: stopped!.metadata!.remoteExecutionTermination, + sandboxStopAndRetainReceipt: stopped!.metadata!.sandboxStopAndRetainReceipt } }); + const [action] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId)); + expect(action).toMatchObject({ status: "resolved", outcome: "blocked", ownerType: "board", + evidence: { automaticRecovery: { replay: "blocked" }, workspaceRestoreRecovery: { leaseIds: [f.lease.id] } } }); + expect((await db.select().from(issues).where(eq(issues.id, f.issueId)))[0].status).toBe("done"); + // A lost commit response can cause the exact adapter receipt to be recorded twice. + await recordLegacyWorkspaceRestoreFailure(db, f.run, patch.resultJson, f.lease); + const repeated = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId)); + expect(repeated).toHaveLength(1); + expect(repeated[0]).toMatchObject({ id: action.id, status: "resolved", outcome: "blocked", attemptCount: 1, + evidence: { automaticRecovery: { replay: "blocked" }, workspaceRestoreRecovery: { leaseIds: [f.lease.id] } } }); + expect(await readLease(f.lease.id)).toEqual(retained); + }); + + it.each(["retry", "builtin"])("keeps the original stop proof after %s dispatch", async scenario => { + const f = await seed(); + if (scenario === "builtin") { + [f.lease] = await db.update(environmentLeases).set({ provider: "fake", metadata: { + driver: "sandbox", remoteCwd: "/work/fixture", + } }).where(eq(environmentLeases.id, f.lease.id)).returning(); + } + const stopped = await stopAndRetain(f, scenario === "retry"); + if (scenario === "retry") expect((stopped.metadata!.sandboxStopAndRetainReceipt as Record).requestId) + .not.toBe(stopped.metadata!.pendingCleanupAttemptId); + await recordLegacyWorkspaceRestoreFailure(db, f.run, patch.resultJson, f.lease); + expect(await readLease(f.lease.id)).toMatchObject({ leasePolicy: "retain_on_failure", status: "released", + metadata: { sandboxStopAndRetainReceipt: stopped.metadata!.sandboxStopAndRetainReceipt } }); + }); + + it.each(["id", "companyId", "heartbeatRunId", "environmentId", "issueId", "executionWorkspaceId", "provider", "providerLeaseId", "acquiredAt", "pluginId", "remoteCwd", "workspaceRealization"])( + "rejects a stopped source with a different original %s", async field => { + const f = await seed(); + const stopped = await stopAndRetain(f); + const source = { ...f.lease, metadata: { ...f.lease.metadata } }; + if (["pluginId", "remoteCwd", "workspaceRealization"].includes(field)) source.metadata[field] = "different-original-value"; + else if (field === "acquiredAt") source.acquiredAt = new Date(source.acquiredAt.getTime() - 1); + else Object.assign(source, { [field]: randomUUID() }); + await recordLegacyWorkspaceRestoreFailure(db, f.run, patch.resultJson, source); + expect(await readLease(f.lease.id)).toEqual(stopped); + expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId))).toEqual([]); + }, + ); + + it.each(["missing_dedicated", "missing_termination", "destroyed", "foreign_receipt", "wrong_plugin", "wrong_method", "empty_request", "invalid_time", "before_acquisition", "after_release", "unequal_confirmation", "pending_intent", "cleanup_in_flight", "expired_allocation", "missing_release_time", "retained_without_stop", "failed_cleanup"])( + "does not infer retention from %s", async scenario => { + const f = await seed(); + const stopped = await stopAndRetain(f); + const metadata = structuredClone(stopped.metadata!); + const dedicated = metadata.sandboxStopAndRetainReceipt as Record; + const termination = metadata.remoteExecutionTermination as Record; + if (scenario === "missing_dedicated") delete metadata.sandboxStopAndRetainReceipt; + if (scenario === "missing_termination") delete metadata.remoteExecutionTermination; + if (scenario === "destroyed") termination.state = "destroyed"; + if (scenario === "foreign_receipt") dedicated.runId = randomUUID(); + if (scenario === "wrong_plugin") dedicated.pluginId = randomUUID(); + if (scenario === "wrong_method") dedicated.method = "environmentReleaseLease"; + if (scenario === "empty_request") dedicated.requestId = " "; + if (scenario === "invalid_time") dedicated.confirmedAt = termination.confirmedAt = "not-a-time"; + if (scenario === "before_acquisition") dedicated.confirmedAt = termination.confirmedAt = new Date(stopped.acquiredAt.getTime() - 1).toISOString(); + if (scenario === "after_release") dedicated.confirmedAt = termination.confirmedAt = new Date(stopped.releasedAt!.getTime() + 1).toISOString(); + if (scenario === "unequal_confirmation") dedicated.confirmedAt = new Date(stopped.releasedAt!.getTime() - 1).toISOString(); + if (scenario === "pending_intent") metadata.sandboxStopAndRetain = {}; + if (scenario === "cleanup_in_flight") metadata.pendingCleanupInFlight = true; + const [invalid] = await db.update(environmentLeases).set({ metadata, + ...(scenario === "retained_without_stop" ? { status: "retained" } : {}), + ...(scenario === "expired_allocation" ? { expiresAt: new Date(stopped.acquiredAt.getTime() - 1) } : {}), + ...(scenario === "missing_release_time" ? { releasedAt: null } : {}), + ...(scenario === "failed_cleanup" ? { cleanupStatus: "failed" } : {}), + }).where(eq(environmentLeases.id, f.lease.id)).returning(); + await recordLegacyWorkspaceRestoreFailure(db, f.run, patch.resultJson, f.lease); + expect(await readLease(f.lease.id)).toEqual(invalid); + expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId))).toEqual([]); + }, + ); + + it("rejects invalid lease timestamps instead of treating receipt metadata as proof", async () => { + const f = await seed(); + const stopped = await stopAndRetain(f); + expect(hasConfirmedSandboxStopAndRetain(stopped)).toBe(true); + for (const field of ["acquiredAt", "releasedAt", "expiresAt"] as const) { + expect(hasConfirmedSandboxStopAndRetain({ ...stopped, [field]: new Date(Number.NaN) })).toBe(false); + } + }); + + it.each(["missing_root", "empty_root", "invalid_realized_root", "run_workspace_mismatch"])( + "requires the original execution root after %s", async scenario => { + const f = await seed(); + const metadata = { ...f.lease.metadata }; + if (scenario === "missing_root") delete metadata.remoteCwd; + if (scenario === "empty_root") metadata.remoteCwd = " "; + if (scenario === "invalid_realized_root") metadata.workspaceRealization = { authoritativeRoot: " " }; + [f.lease] = await db.update(environmentLeases).set({ metadata }).where(eq(environmentLeases.id, f.lease.id)).returning(); + if (scenario === "run_workspace_mismatch") await db.update(heartbeatRuns).set({ contextSnapshot: { + ...f.run.contextSnapshot, executionWorkspaceId: randomUUID(), + } }).where(eq(heartbeatRuns.id, f.runId)); + const stopped = await stopAndRetain(f); + await recordLegacyWorkspaceRestoreFailure(db, f.run, patch.resultJson, f.lease); + expect(await readLease(f.lease.id)).toEqual(stopped); + expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId))).toEqual([]); + }, + ); + + it("does not search arbitrary stopped leases when the original host snapshot is missing", async () => { + const f = await seed(); + const stopped = await stopAndRetain(f); + await recordLegacyWorkspaceRestoreFailure(db, f.run, patch.resultJson); + expect(await readLease(f.lease.id)).toEqual(stopped); + expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId))).toEqual([]); + }); + + it.each(["active", "stopped", "older_stopped", "older_destroyed"])("rejects a competing %s allocation owner across companies", async state => { + const f = await seed(), other = await seed(); + const stopped = await stopAndRetain(f); + [other.lease] = await db.update(environmentLeases).set({ providerLeaseId: f.lease.providerLeaseId, + acquiredAt: new Date(f.lease.acquiredAt.getTime() + (state.startsWith("older_") ? -1_000 : 1)), + }).where(eq(environmentLeases.id, other.lease.id)).returning(); + if (state !== "active") { + const stoppedOther = await stopAndRetain(other); + if (state === "older_destroyed") await db.update(environmentLeases).set({ metadata: { + ...stoppedOther.metadata, remoteExecutionTermination: { + ...(stoppedOther.metadata!.remoteExecutionTermination as Record), state: "destroyed", + }, + } }).where(eq(environmentLeases.id, other.lease.id)); + } + const competing = await readLease(other.lease.id); + await recordLegacyWorkspaceRestoreFailure(db, f.run, patch.resultJson, f.lease); + expect(await readLease(f.lease.id)).toEqual(stopped); + expect(await readLease(other.lease.id)).toEqual(competing); + expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId))).toEqual([]); + }); + + it("does not claim a released reusable source from a stop receipt that may precede resume", async () => { + const f = await seed({ reusable: true }); + const stopped = await stopAndRetain(f); + await recordLegacyWorkspaceRestoreFailure(db, f.run, patch.resultJson, f.lease); + expect(await readLease(f.lease.id)).toEqual(stopped); + expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId))).toEqual([]); + }); + + it.each(["original_reusable", "current_reusable", "expired", "root_rebound", "run_rebound"])( + "leaves a stopped source unchanged after %s", async scenario => { + const f = await seed({ reusable: scenario === "original_reusable" }); + const other = await seed(); + const stopped = await stopAndRetain(f); + const [changed] = await db.update(environmentLeases).set({ + ...(scenario === "original_reusable" ? { leasePolicy: "ephemeral" } : {}), + ...(scenario === "current_reusable" ? { leasePolicy: "reuse_by_environment" } : {}), + ...(scenario === "expired" ? { status: "expired" } : {}), + ...(scenario === "root_rebound" ? { metadata: { ...stopped.metadata, remoteCwd: "/work/replacement" } } : {}), + ...(scenario === "run_rebound" ? { heartbeatRunId: other.runId } : {}), + }).where(eq(environmentLeases.id, f.lease.id)).returning(); + await recordLegacyWorkspaceRestoreFailure(db, f.run, patch.resultJson, f.lease); + expect(await readLease(f.lease.id)).toEqual(changed); + expect(await readLease(other.lease.id)).toEqual(other.lease); + expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId))).toEqual([]); + }, + ); + + it("does not adopt the expired source when a normal reusable handoff wins first", async () => { + const f = await seed({ reusable: true }); + await stopAndRetain(f); + const newerRunId = randomUUID(); + await db.insert(heartbeatRuns).values({ id: newerRunId, companyId: f.companyId, agentId: f.run.agentId, + status: "running", runtimeMode: "legacy", contextSnapshot: { issueId: f.issueId } }); + // The production handoff also checks the immutable task/agent scope. + [f.lease] = await db.update(environmentLeases).set({ metadata: { + ...(await readLease(f.lease.id)).metadata, agentId: f.run.agentId, + } }).where(eq(environmentLeases.id, f.lease.id)).returning(); + const newer = await environmentService(db).acquireLease({ companyId: f.companyId, + environmentId: f.lease.environmentId!, issueId: f.issueId, executionWorkspaceId: f.lease.executionWorkspaceId, heartbeatRunId: newerRunId, + leasePolicy: "reuse_by_environment", provider: f.lease.provider!, providerLeaseId: f.lease.providerLeaseId, + metadata: { ...f.lease.metadata, agentId: f.run.agentId }, replacesReusableLeaseId: f.lease.id, + }); + const expired = await readLease(f.lease.id); + expect(expired.status).toBe("expired"); + await recordLegacyWorkspaceRestoreFailure(db, f.run, patch.resultJson, f.lease); + expect(await readLease(f.lease.id)).toEqual(expired); + expect(await readLease(newer.id)).toEqual(newer); + expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId))).toEqual([]); + }); + + it("rolls back the stopped-source hold and action if recording cannot commit", async () => { + const f = await seed(); + const stopped = await stopAndRetain(f); + await expect(db.transaction(async tx => { + await recordLegacyWorkspaceRestoreFailure(tx as unknown as typeof db, f.run, patch.resultJson, f.lease); + throw new Error("recording transaction failed"); + })).rejects.toThrow("recording transaction failed"); + expect(await readLease(f.lease.id)).toEqual(stopped); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)))[0].resultJson).toBeNull(); + expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId))).toEqual([]); + }); + + it("preserves newer task execution while retaining only the original stopped source", async () => { + const f = await seed(), newerRunId = randomUUID(); + await stopAndRetain(f); + await db.insert(heartbeatRuns).values({ id: newerRunId, companyId: f.companyId, agentId: f.run.agentId, + status: "running", runtimeMode: "legacy", contextSnapshot: { issueId: f.issueId } }); + await db.update(issues).set({ executionRunId: newerRunId, checkoutRunId: newerRunId }).where(eq(issues.id, f.issueId)); + const [newerLease] = await db.insert(environmentLeases).values({ companyId: f.companyId, issueId: f.issueId, + heartbeatRunId: newerRunId, environmentId: f.lease.environmentId, provider: "daytona", providerLeaseId: randomUUID(), + status: "active", leasePolicy: "ephemeral", metadata: { driver: "sandbox", pluginId: randomUUID() }, + }).returning(); + await recordLegacyWorkspaceRestoreFailure(db, f.run, patch.resultJson, f.lease); + expect(await readLease(f.lease.id)).toMatchObject({ leasePolicy: "retain_on_failure" }); + expect(await readLease(newerLease.id)).toEqual(newerLease); + expect((await db.select().from(issues).where(eq(issues.id, f.issueId)))[0]) + .toMatchObject({ status: "in_progress", executionRunId: newerRunId, checkoutRunId: newerRunId }); + const [action] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId)); + expect(action.evidence.automaticRecovery).toMatchObject({ replay: "blocked" }); + }); + it("leaves local lock retry and successful remote cleanup policy unchanged", async () => { const local = await seed({ local: true }); const failure = { ...patch, resultJson: { ...patch.resultJson, workspaceRestoreFailure: "restore_lock_timeout" } }; diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 4df871c3d5d..a549ee7ce42 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -4738,6 +4738,9 @@ export function heartbeatService( await bindIssueToPersistedExecutionWorkspace(persistedExecutionWorkspace); const workspaceRealization = realizationResult.workspaceRealization; const executionTarget = realizationResult.executionTarget; + // Preserve the host-owned source before adapter context can share lease + // metadata. A later copy-back failure must not adopt a rebound source. + const workspaceRestoreSource = structuredClone(realizationResult.lease); let instructionCopy: Awaited> = null; let instructionSave: Record | null = null; const instructionPreparationKey = createHash("sha256").update(JSON.stringify({ @@ -7157,7 +7160,7 @@ export function heartbeatService( }; // Retention is the fallback even if recording this receipt fails. providerResourceDispositionForRun = "stop_and_retain"; - await recordLegacyWorkspaceRestoreFailure(db, run, requiredWorkspaceRestoreEvidence); + await recordLegacyWorkspaceRestoreFailure(db, run, requiredWorkspaceRestoreEvidence, workspaceRestoreSource); } for (const stream of ["stdout", "stderr"] as const) { const tail = identityRedactor.finish(stream); diff --git a/server/src/services/legacy-execution-recovery.ts b/server/src/services/legacy-execution-recovery.ts index 9d56e1891e0..828ad951553 100644 --- a/server/src/services/legacy-execution-recovery.ts +++ b/server/src/services/legacy-execution-recovery.ts @@ -1,5 +1,5 @@ import { hasRequiredWorkspaceRecovery, preserveWorkspaceRestoreRecoveryMetadata, LEGACY_WORKSPACE_RECOVERY_SCHEMA } from "./workspace-restore-recovery-state.js"; -import { hasUnrestoredRemoteWorkspace, preserveLegacyWorkspaceRestoreSources } from "./legacy-workspace-restore-recovery.js"; +import { hasUnrestoredRemoteWorkspace, preserveLegacyWorkspaceRestoreSources, type LegacyWorkspaceRestoreSource } from "./legacy-workspace-restore-recovery.js"; import { isPreDispatchReviewWaitVerified } from "./pre-dispatch-review-wait.js"; import { hasWorkspaceRestoreFailure } from "@paperclipai/shared"; import { normalizeMaxTurnStopReason } from "./heartbeat-stop-metadata.js"; @@ -74,6 +74,7 @@ export async function terminalizeLegacyExecution(input: { writeConditions?: SQL[]; /** Adapter settlement records a copy-back failure before host finalization. */ recordRestoreFailureOnly?: boolean; + workspaceRestoreSource?: LegacyWorkspaceRestoreSource; }) { const { db, run, status } = input; let patch = input.patch; @@ -122,7 +123,8 @@ export async function terminalizeLegacyExecution(input: { ) .returning(); if (!updated) return null; - const retainedLeaseIds = await preserveLegacyWorkspaceRestoreSources(tx as unknown as Db, input.recordRestoreFailureOnly ? { ...updated, status: "failed" } : updated); + const retainedLeaseIds = await preserveLegacyWorkspaceRestoreSources(tx as unknown as Db, + input.recordRestoreFailureOnly ? { ...updated, status: "failed" } : updated, input.workspaceRestoreSource); if (retainedLeaseIds.length) { [updated] = await tx.update(heartbeatRuns).set({ resultJson: { ...updated.resultJson, @@ -310,9 +312,9 @@ export async function settleInterruptedNativeBootstrap( /** Capture the required-file obligation before unrelated host writes. Keeps * execution ownership while the adapter's host finalization is still running. */ -export async function recordLegacyWorkspaceRestoreFailure(db: Db, run: Run, evidence: Record) { +export async function recordLegacyWorkspaceRestoreFailure(db: Db, run: Run, evidence: Record, source?: LegacyWorkspaceRestoreSource) { if (run.runtimeMode !== "legacy" || !hasWorkspaceRestoreFailure(evidence)) return; await terminalizeLegacyExecution({ db, run, status: run.status, - recordRestoreFailureOnly: true, patch: { resultJson: evidence }, + recordRestoreFailureOnly: true, patch: { resultJson: evidence }, workspaceRestoreSource: source, }); } diff --git a/server/src/services/legacy-workspace-restore-recovery.ts b/server/src/services/legacy-workspace-restore-recovery.ts index 9f7822375af..fc1419ddfb8 100644 --- a/server/src/services/legacy-workspace-restore-recovery.ts +++ b/server/src/services/legacy-workspace-restore-recovery.ts @@ -1,28 +1,57 @@ +import { isDeepStrictEqual } from "node:util"; import { and, eq, inArray, isNotNull, ne, or, sql } from "drizzle-orm"; import { environmentLeases, environments, heartbeatRuns, type Db } from "@paperclipai/db"; import { hasWorkspaceRestoreFailure, WORKSPACE_RESTORE_FAILURE_CODES } from "@paperclipai/shared"; -import { prepareSandboxStopAndRetain } from "./sandbox-stop-and-retain.js"; +import { hasConfirmedSandboxStopAndRetain, prepareSandboxStopAndRetain } from "./sandbox-stop-and-retain.js"; import { hasRequiredWorkspaceRecovery, LEGACY_WORKSPACE_RECOVERY_SCHEMA } from "./workspace-restore-recovery-state.js"; type Run = typeof heartbeatRuns.$inferSelect; +/** The host's realized lease snapshot from before adapter execution. Never + * reconstruct this identity from a terminal status or a search of old leases. */ +export type LegacyWorkspaceRestoreSource = Pick; function needsRemoteRestore(run: Run): boolean { return run.runtimeMode === "legacy" && ["failed", "timed_out", "interrupted", "cancelled"].includes(run.status) && hasWorkspaceRestoreFailure(run.resultJson); } -function leaseScope(run: Run) { +function leaseScope(run: Run, source?: LegacyWorkspaceRestoreSource) { return and(eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.heartbeatRunId, run.id), or(eq(environmentLeases.status, "active"), and( sql`${environmentLeases.metadata}->'workspaceRestoreRecovery'->>'schema' = ${LEGACY_WORKSPACE_RECOVERY_SCHEMA}`, - sql`${environmentLeases.metadata}->'workspaceRestoreRecovery'->>'runId' = ${run.id}`)), + sql`${environmentLeases.metadata}->'workspaceRestoreRecovery'->>'runId' = ${run.id}`), + source ? and(eq(environmentLeases.id, source.id), eq(environmentLeases.status, "released")) : undefined), isNotNull(environmentLeases.providerLeaseId), isNotNull(environmentLeases.provider), ne(environmentLeases.provider, "local"), or(eq(environments.driver, "sandbox"), sql`${environmentLeases.metadata}->>'driver' = 'sandbox'`, sql`${environmentLeases.metadata}->>'sandboxProviderPlugin' = 'true'`)); } +function matchesOriginalSource(lease: typeof environmentLeases.$inferSelect, run: Run, source: LegacyWorkspaceRestoreSource) { + const issueId = run.nativeIssueId ?? run.issueId ?? run.contextSnapshot?.issueId ?? null; + const workspaceId = run.contextSnapshot?.executionWorkspaceId; + const remoteCwd = source.metadata?.remoteCwd; + const realization = source.metadata?.workspaceRealization; + if (typeof remoteCwd !== "string" || !remoteCwd.trim() + || (typeof workspaceId === "string" && source.executionWorkspaceId !== workspaceId)) return false; + if (realization !== undefined && realization !== null) { + if (typeof realization !== "object" || Array.isArray(realization)) return false; + const root = (realization as Record).authoritativeRoot; + if (root !== undefined && (typeof root !== "string" || !root.trim())) return false; + } + return source.leasePolicy === "ephemeral" + && lease.leasePolicy === source.leasePolicy + && source.companyId === run.companyId && source.heartbeatRunId === run.id && source.issueId === issueId + && (["id", "companyId", "heartbeatRunId", "environmentId", "issueId", "executionWorkspaceId", "provider", "providerLeaseId"] as const) + .every(key => lease[key] === source[key]) + && lease.acquiredAt.getTime() === source.acquiredAt.getTime() + && (["pluginId", "sandboxProviderPlugin", "remoteCwd", "workspaceRealization"] as const) + .every(key => isDeepStrictEqual(lease.metadata?.[key], source.metadata?.[key])); +} + /** A local lock timeout has no remote source to retain and keeps its old policy. */ export async function hasUnrestoredRemoteWorkspace(db: Db, run: Run): Promise { if (!needsRemoteRestore(run)) return false; @@ -35,14 +64,40 @@ export async function hasUnrestoredRemoteWorkspace(db: Db, run: Run): Promise { +export async function preserveLegacyWorkspaceRestoreSources(db: Db, run: Run, source?: LegacyWorkspaceRestoreSource): Promise { if (!needsRemoteRestore(run)) return []; const rows = await db.select({ lease: environmentLeases }).from(environmentLeases) .leftJoin(environments, eq(environments.id, environmentLeases.environmentId)) - .where(leaseScope(run)).for("update", { of: environmentLeases }); + .where(leaseScope(run, source)).for("update", { of: environmentLeases }); const retained: string[] = []; for (const { lease } of rows) { - if (lease.status !== "active") { retained.push(lease.id); continue; } + if (lease.status !== "active") { + const recovery = lease.metadata?.workspaceRestoreRecovery as Record | undefined; + if (recovery?.schema === LEGACY_WORKSPACE_RECOVERY_SCHEMA && recovery.runId === run.id) { + retained.push(lease.id); continue; + } + // Cancellation can finish the dedicated stop before copy-back reports its + // failure. Only the host's original source and two matching receipts can + // turn that released row into a file-repair hold. + if (!source || !matchesOriginalSource(lease, run, source) || !hasConfirmedSandboxStopAndRetain(lease)) continue; + const [other] = await db.select({ id: environmentLeases.id }).from(environmentLeases).where(and( + ne(environmentLeases.id, lease.id), eq(environmentLeases.provider, lease.provider!), + eq(environmentLeases.providerLeaseId, lease.providerLeaseId!), + )).limit(1); + // An ephemeral source has no legitimate reuse history. Any other physical + // owner makes retention ambiguous, including an older owner that later + // destroyed the allocation. Do not expose cross-company owner identity. + if (other) continue; + await db.update(environmentLeases).set({ leasePolicy: "retain_on_failure", + metadata: sql`${environmentLeases.metadata} || ${JSON.stringify({ workspaceRestoreRecovery: { + schema: LEGACY_WORKSPACE_RECOVERY_SCHEMA, runId: run.id, + } })}::jsonb`, + }).where(eq(environmentLeases.id, lease.id)); + // Reusable sources are excluded: provider resume precedes the database + // handoff, so a row lock cannot fence an in-flight resume. Keep the release + // status and receipts; this repair hold grants no execute right. + retained.push(lease.id); continue; + } // The stop-only worker rechecks exact allocation ownership, including // competing leases, before dispatch. Persist the intent even when cleanup // must wait, so no terminal-run sweep can fall back to destruction. diff --git a/server/src/services/sandbox-stop-and-retain.ts b/server/src/services/sandbox-stop-and-retain.ts index e9823dcabd4..d90c2d3e625 100644 --- a/server/src/services/sandbox-stop-and-retain.ts +++ b/server/src/services/sandbox-stop-and-retain.ts @@ -2,13 +2,46 @@ import { randomUUID } from "node:crypto"; import { and, eq, sql, type SQL } from "drizzle-orm"; import { environmentLeases, type Db } from "@paperclipai/db"; import { isBuiltinSandboxProvider } from "./sandbox-provider-runtime.js"; -import { remoteTerminationReceipt } from "./remote-execution-termination.js"; +import { hasRemoteTerminationReceipt, remoteTerminationReceipt } from "./remote-execution-termination.js"; import { hasNativeWorkspaceExportResume, readNativeWorkspaceExportResume, settleNativeWorkspaceExportResume } from "./native-runtime/native-workspace-export-resume.js"; type Lease = Pick; export const SANDBOX_STOP_AND_RETAIN_KEY = "sandboxStopAndRetain"; +/** A successful stop is not by itself proof that the allocation was retained. + * Require the dedicated stop-only dispatch receipt and its independent stop + * confirmation, both bound to the original lease. No provider work is allowed. */ +export function hasConfirmedSandboxStopAndRetain(lease: typeof environmentLeases.$inferSelect): boolean { + const acquiredAt = lease.acquiredAt.getTime(); + const releasedAt = lease.releasedAt?.getTime(); + const expiresAt = lease.expiresAt?.getTime(); + if (!Number.isFinite(acquiredAt) || releasedAt === undefined || !Number.isFinite(releasedAt) + || (expiresAt !== undefined && (!Number.isFinite(expiresAt) || expiresAt <= Date.now()))) return false; + const value = lease.metadata?.sandboxStopAndRetainReceipt; + if (!value || typeof value !== "object" || Array.isArray(value)) return false; + const receipt = value as Record; + const termination = lease.metadata?.remoteExecutionTermination as Record | undefined; + if (lease.status !== "released" || !hasRemoteTerminationReceipt(lease) + || termination?.state !== "stopped" || hasStopOnlyCleanup(lease) + || lease.metadata?.pendingCleanupInFlight === true + || receipt.schema !== "paperclip.sandbox-stop-and-retain-receipt.v1" + || receipt.companyId !== lease.companyId || receipt.runId !== lease.heartbeatRunId + || receipt.leaseId !== lease.id || receipt.provider !== lease.provider + || receipt.providerLeaseId !== lease.providerLeaseId + || typeof receipt.requestId !== "string" || !receipt.requestId.trim() + || typeof receipt.confirmedAt !== "string" || receipt.confirmedAt !== termination.confirmedAt) return false; + const confirmedAt = Date.parse(receipt.confirmedAt); + if (!Number.isFinite(confirmedAt) || confirmedAt < acquiredAt || confirmedAt > releasedAt) return false; + if (receipt.builtinProvider !== undefined) { + return receipt.builtinProvider === lease.provider && isBuiltinSandboxProvider(lease.provider!) + && receipt.method === "builtin.stopLease" && receipt.pluginId === undefined + && lease.metadata?.pluginId == null && !lease.metadata?.sandboxProviderPlugin; + } + return receipt.method === "environmentStopLease" && typeof receipt.pluginId === "string" + && !!receipt.pluginId.trim() && receipt.pluginId === lease.metadata?.pluginId; +} + export function hasStopOnlyCleanup(lease: Pick): boolean { return hasNativeWorkspaceExportResume(lease) || Object.prototype.hasOwnProperty.call(lease.metadata ?? {}, SANDBOX_STOP_AND_RETAIN_KEY);