diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 65cf2148a2b..94688dc5a03 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -2429,6 +2429,16 @@ when Stop found no live controller. The worker retains the exact prepared source and obtains independent provider-stop evidence through normal recovery. It does not convert the no-op acknowledgement into a dispatched interrupt receipt or permission to continue the cancelled turn. +Terminal release of a warm (`keep_running`) run that is not successful stops and +retains its reusable sandbox. This release can finish before the preservation +worker sees a terminal coordinator. A released reusable lease with a confirmed +stop-and-retain receipt for the same run, lease and allocation therefore +remains a preservation source. The worker records it as file recovery work and +keeps its release status, policy and receipts. It dispatches no second stop and +grants no execute right on the reusable sandbox. A released reusable lease without +that receipt stays ineligible. Until this recovery saves the files, the +task's remote workspace keeps the ended writer's hold. Successor runs wait for that +recovery instead of overwriting unsaved files. A reusable lease can also owe native copyback: the export-repair action must select its exact immutable descriptor and the finalizer's accepted successful result under the same company, task, run and completion contract. Lease reuse diff --git a/server/src/services/native-runtime/native-cancelled-workspace-preservation.test.ts b/server/src/services/native-runtime/native-cancelled-workspace-preservation.test.ts index 0916bd8c90f..7c6ec6ff92f 100644 --- a/server/src/services/native-runtime/native-cancelled-workspace-preservation.test.ts +++ b/server/src/services/native-runtime/native-cancelled-workspace-preservation.test.ts @@ -207,6 +207,91 @@ import { preserveCancelledNativeWorkspaces, preserveNativeFailedWorkspace, } finally { await worker.stop(); } }); + /** A reassignment Stop of a warm (keep_running) run: the executor settles its + * coordinator, then terminal release stops and retains the reusable sandbox + * before the preservation worker's next check. */ + async function stopAndRetainReusableSource(f: Awaited>, options: { stopReceipt?: boolean } = {}) { + const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.leaseId)); + const confirmedAt = new Date(lease.acquiredAt.getTime() + 1_000); + const releasedAt = new Date(confirmedAt.getTime() + 1); + const termination = { schema: "paperclip.remote-termination.v1", companyId: f.companyId, runId: f.runId, + leaseId: f.leaseId, provider: "daytona", providerLeaseId: f.reference.providerLeaseId, state: "stopped", + confirmedAt: confirmedAt.toISOString() }; + const receipt = { schema: "paperclip.sandbox-stop-and-retain-receipt.v1", companyId: f.companyId, runId: f.runId, + leaseId: f.leaseId, provider: "daytona", providerLeaseId: f.reference.providerLeaseId, requestId: randomUUID(), + method: "environmentStopLease", pluginId: lease.metadata?.pluginId, confirmedAt: confirmedAt.toISOString() }; + await db.update(environmentLeases).set({ status: "released", leasePolicy: "reuse_by_environment", + cleanupStatus: "success", releasedAt, metadata: { ...lease.metadata, pendingCleanupInFlight: false, + ...(options.stopReceipt === false ? {} : { remoteExecutionTermination: termination, sandboxStopAndRetainReceipt: receipt }) }, + }).where(eq(environmentLeases.id, f.leaseId)); + await db.update(executionWorkspaces).set({ mode: "shared_workspace" }).where(eq(executionWorkspaces.id, f.workspaceId)); + await db.update(heartbeatRuns).set({ errorCode: "issue_reassigned", error: "Cancelled before issue reassignment", + runnerProfileJson: { remoteWorkspace: f.reference, nativeResourceDisposition: "keep_running" }, + }).where(eq(heartbeatRuns.id, f.runId)); + await db.update(nativeRunFinalizations).set({ failureCode: "native_retry_cancelled", recoveryState: null }) + .where(eq(nativeRunFinalizations.runId, f.runId)); + return (await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.leaseId)))[0]; + } + + it.each(["worker", "startup-scan"] as const)("preserves files of a cancelled run whose reusable sandbox was already stopped and retained (%s)", async kind => { + const f = await seed(); + const stopped = await stopAndRetainReusableSource(f); + const worker = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => { throw error; } }); + try { + if (kind === "startup-scan") await preserveCancelledNativeWorkspaces(db); + else await registerCancelledWorkspacePreservationWork(db, worker, () => true, async () => true).ready; + const [retained] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + expect(retained.resultJson).toMatchObject({ nativeCancellation: f.nativeCancellation, workspaceRestoreFailure: "restore_failed", + workspaceRestoreRecovery: { schema: "paperclip.workspace-restore-recovery.v1", leaseIds: [f.leaseId] } }); + expect(retained).toMatchObject({ status: "cancelled", errorCode: "issue_reassigned", finishedAt: f.run.finishedAt }); + // The stopped allocation keeps its receipts. Preservation dispatches no + // second stop and grants no execute right on the reusable sandbox. + expect((await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.leaseId)))[0]).toEqual(stopped); + const actions = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.companyId, f.companyId)); + expect(actions).toHaveLength(1); + expect(actions[0].evidence).toMatchObject({ runId: f.runId, workspaceRestoreRecovery: { leaseIds: [f.leaseId] } }); + expect((await inspectWorkspaceRecoverySource(db, { companyId: f.companyId, issueId: f.issueId, + runId: f.runId, actionId: actions[0].id })).sources).toHaveLength(1); + if (kind === "worker") expect(worker.nextWakeAt()).toBeNull(); + // A repeated pass records nothing again and starts no work. + await preserveCancelledNativeWorkspaces(db); + const repeated = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => { throw error; } }); + try { + await registerCancelledWorkspacePreservationWork(db, repeated, () => true, async () => true).ready; + expect(repeated.nextWakeAt()).toBeNull(); + } finally { await repeated.stop(); } + expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.companyId, f.companyId))).toEqual(actions); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)))[0].resultJson).toEqual(retained.resultJson); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId))).toHaveLength(1); + expect(await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, f.companyId))).toHaveLength(0); + expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)))[0]) + .toMatchObject({ phase: "terminal_failure", leaseOwner: null, resultId: null }); + } finally { await worker.stop(); } + }, 120_000); + + it.each(["live-owner", "pending-stop", "no-stop-receipt", "running"] as const) + ("does not preserve a stopped reusable sandbox for %s", async invalid => { + const f = await seed(); + const stopped = await stopAndRetainReusableSource(f, { stopReceipt: invalid !== "no-stop-receipt" }); + if (invalid === "live-owner") await db.update(nativeRunFinalizations).set({ phase: "observed", leaseOwner: randomUUID(), + leaseExpiresAt: new Date(Date.now() + 60_000) }).where(eq(nativeRunFinalizations.runId, f.runId)); + if (invalid === "pending-stop") await db.update(heartbeatRuns).set({ resultJson: { + nativeCancellation: { ...f.nativeCancellation, dispatchState: "pending" } } }).where(eq(heartbeatRuns.id, f.runId)); + if (invalid === "running") await db.update(heartbeatRuns).set({ status: "running" }).where(eq(heartbeatRuns.id, f.runId)); + const errors: unknown[] = []; + const worker = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => { errors.push(error); } }); + try { + await registerCancelledWorkspacePreservationWork(db, worker, () => true, async () => true).ready; + await preserveCancelledNativeWorkspaces(db); + expect(errors).toEqual([]); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + expect(run.resultJson).not.toHaveProperty("workspaceRestoreRecovery"); + expect((await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.leaseId)))[0]).toEqual(stopped); + expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.companyId, f.companyId))).toHaveLength(0); + expect(await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, f.companyId))).toHaveLength(0); + } finally { await worker.stop(); } + }); + it("rolls preservation back with its outer transaction", async () => { const f = await seed(); await expect(db.transaction(async tx => { diff --git a/server/src/services/native-runtime/native-failed-workspace-recovery.ts b/server/src/services/native-runtime/native-failed-workspace-recovery.ts index 9681695bef3..c8da89e0d59 100644 --- a/server/src/services/native-runtime/native-failed-workspace-recovery.ts +++ b/server/src/services/native-runtime/native-failed-workspace-recovery.ts @@ -16,7 +16,20 @@ import { logger } from "../../middleware/logger.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; import { preserveRemoteWorkspacePreparationFailure } from "./native-workspace-finalization-state.js"; import { pinFailedObservedWorkspace } from "./native-observed-workspace-preservation.js"; +import { hasConfirmedSandboxStopAndRetain } from "../sandbox-stop-and-retain.js"; +/** Leases that can still hold a cancelled run's prepared files. Terminal + * release can stop and retain a reusable sandbox before preservation runs. + * Its confirmed stop receipt keeps those files as a recovery source; the + * restore recorder rechecks the receipt and the exact workspace binding. */ +function cancelledWorkspaceSourceLease() { + return or(eq(environmentLeases.status, "active"), and( + inArray(environmentLeases.status, ["pending_cleanup", "released"]), eq(environmentLeases.leasePolicy, "ephemeral"), + ), and(eq(environmentLeases.status, "released"), + inArray(environmentLeases.leasePolicy, ["reuse_by_environment", "reuse_by_execution_workspace"]), + sql`${environmentLeases.metadata}->'remoteExecutionTermination'->>'state' = 'stopped'`, + sql`${environmentLeases.metadata} ? 'sandboxStopAndRetainReceipt'`)); +} function workspaceRecoveryReference(profile: Record | null | undefined) { const remote = remoteWorkspaceRunReferenceSchema.safeParse(profile?.remoteWorkspace); @@ -178,9 +191,7 @@ export async function preserveCancelledNativeWorkspaces(db: Db) { eq(heartbeatRuns.id, environmentLeases.heartbeatRunId), eq(heartbeatRuns.companyId, environmentLeases.companyId), )).innerJoin(nativeRunFinalizations, and(eq(nativeRunFinalizations.runId, heartbeatRuns.id), eq(nativeRunFinalizations.companyId, heartbeatRuns.companyId), eq(nativeRunFinalizations.issueId, heartbeatRuns.nativeIssueId))) - .where(and(or(eq(environmentLeases.status, "active"), and( - inArray(environmentLeases.status, ["pending_cleanup", "released"]), eq(environmentLeases.leasePolicy, "ephemeral"), - )), eq(heartbeatRuns.runtimeMode, "native"), + .where(and(cancelledWorkspaceSourceLease(), eq(heartbeatRuns.runtimeMode, "native"), eq(heartbeatRuns.status, "cancelled"), eq(nativeRunFinalizations.phase, "terminal_failure"), isNull(nativeRunFinalizations.resultId), isNull(nativeRunFinalizations.leaseOwner), sql`(${heartbeatRuns.runnerProfileJson}->'nativeWorkspaceSync'->>'state' = 'prepared' OR ${heartbeatRuns.runnerProfileJson}->'remoteWorkspace'->>'state' = 'prepared')`, @@ -257,11 +268,12 @@ export function registerCancelledWorkspacePreservationWork(db: Db, if (finalization?.resultId) return null; if (finalization?.phase !== "terminal_failure" || finalization.leaseOwner) return { ...entry, dueAt: Date.now() + 30_000 }; - const [source] = await db.select({ id: environmentLeases.id }).from(environmentLeases).where(and( + const sources = await db.select().from(environmentLeases).where(and( eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.heartbeatRunId, run.id), - or(eq(environmentLeases.status, "active"), and(inArray(environmentLeases.status, ["pending_cleanup", "released"]), - eq(environmentLeases.leasePolicy, "ephemeral"))), - )).limit(1); + cancelledWorkspaceSourceLease(), + )); + const source = sources.find(lease => lease.status !== "released" || lease.leasePolicy === "ephemeral" + || hasConfirmedSandboxStopAndRetain(lease)); if (source) await preserveNativeFailedWorkspace(db, run, { allowPreterminal: true, requireCancelledStop: true }); const [current] = await db.select().from(heartbeatRuns) .where(and(eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.companyId, run.companyId)));