From 90ae3c4337431a097bdb6327b211f3fbf77db59b Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Thu, 12 Nov 2026 11:54:47 -0600 Subject: [PATCH] fix(native): process a superseded export's late stop receipt and stop its sandbox when none arrives (#21143) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work. > - A native run can be retired as superseded while its workspace export fails. Its files stay unsaved, and the run stays the writer of the task's workspace. > - #21136 records those files for the automatic workspace recovery. That record needs the sandbox's stop-and-retain receipt. > - The native run projection worker checked for that receipt for one hour after the run ended. After that, it dropped the run. Nothing woke it when the receipt arrived later. > - If the receipt came late or never, the run kept the task's workspace fenced. Every queued run on the task waited and then failed setup. Nobody owned the problem. > - This pull request makes the worker process a late receipt, stop and keep the sandbox itself when no receipt arrives, and never drop the run silently. > - The benefit is that the task's queue moves again by itself, and two writers never use one workspace. ## Linked Issues or Issue Description Refs #21136 (review thread on `native-superseded-workspace-export.ts:84`) **What happened?** `preserveSupersededWorkspaceExport` returned `recheckAt: null` when the sandbox stop receipt had not arrived within `finishedAt + 1h`. The projection worker then deleted its entry. A later stop receipt (`settleStopOnlyCleanup`) woke only the workspace recovery queue, not the projection queue. `preserveNativeFailedWorkspace` requires `hasConfirmedSandboxStopAndRetain`, so no hold was recorded. The run stayed the dirty writer, and queued runs kept failing setup. **Expected behavior** A late receipt is processed at once. When no receipt arrives, Paperclip stops and keeps the sandbox itself and waits for the provider's confirmation. The workspace stays held while the sandbox may still run. When the provider confirms that the sandbox is gone, the task continues from its last save. When the stop stays unconfirmed for hours, the task shows one plain notice, and the check continues. **Steps to reproduce** 1. Retire a native run as superseded after its workspace export fails (the #21136 shape). 2. Let the run's sandbox stop not settle (for example, the server that owned the release is replaced). 3. Wait more than one hour after the run ended. 4. See that the projection worker drops the run, and that a later stop receipt records nothing. The workspace stays held. ## What Changed - `sandbox-stop-and-retain.ts`: a confirmed stop-only receipt now also notifies `DELIVERY_QUEUES.nativeRunProjection` (scoped to the company and run). New `prepareUnconfirmedSourceSandboxStop` puts a stop-and-retain intent on the source lease from its current status. It skips a destroyed sandbox, a stop already in progress, and a cleanup in flight. - `native-superseded-workspace-export.ts`: - Within one hour of the run ending, the worker checks every minute (unchanged). - After that hour, it requests the stop-and-retain itself. The existing pending-cleanup sweep dispatches the stop and retries it until the provider confirms. The worker checks again every 15 minutes, and the receipt wakes it at once. The workspace stays held. - A sandbox that another lease holds is never stopped. - When the provider confirms that the sandbox was destroyed, the files cannot be saved. In one transaction (task, run, then workspace locks), the worker sets aside only that run's workspace epoch, wakes queued-run dispatch, and records `issue.previous_run_files_lost` on the task. - When the stop is still unconfirmed six hours after the run ended (or the sandbox is held by another lease), the task gets one `issue.previous_run_files_waiting` activity entry. The worker keeps checking. It never drops the run. - A stop-only dispatch now settles when the provider confirms that the exact sandbox is already gone. Before, the runtime accepted only a `stopped` reply and threw on `destroyed`, so a gone sandbox kept the stop pending forever. - `sandbox-stop-and-retain.ts`: new `acceptedStopOnlyReceipt` accepts `stopped`, or `destroyed` for a stop-and-retain intent (a native workspace export resume still needs `stopped`). `settleStopOnlyCleanup` stores a `destroyed` receipt as `released` / `success` / `workspace_source_unavailable`, removes the stop intent, never writes a retain receipt, and wakes the projection and workspace recovery queues. - `environment-runtime.ts`: the plugin `environmentStopLease` and built-in `stopLease` checks use `acceptedStopOnlyReceipt`. - `workspace-recovery.ts`: when its own stop settles as `destroyed`, the source is unavailable (the existing alternative-strategy path) instead of a binding failure. - Daytona plugin: stop of a sandbox that Daytona reports as not found returns `destroyed` (destroy already did this) instead of throwing. boxd already returns `destroyed`. Other sandbox plugins have no stop-only method, so nothing changes for them. - New shared `SANDBOX_ALLOCATION_HOLDER_STATUSES` (`active`, `retained`, `pending_cleanup`). The stop-only cleanup ownership check and the worker's "held by another lease" guard both use it, so a retained warm sandbox is never stopped and the two checks cannot drift. - `ui/src/lib/activity-format.ts`: plain labels for the two task activity entries. They do not ask a person to fix anything. - `doc/execution-semantics.md`: document the rule. - Not changed: the shared owner prover still refuses to treat a stopped instance on another host as proof that its child processes stopped. ## Verification - `server/src/services/superseded-workspace-export-recovery.test.ts` (4 new tests, red/green): - "checks again every minute while the run's own release may still stop its sandbox": the recheck branch. Nothing is recorded or stopped, and the workspace keeps its writer. - "stops and keeps the sandbox itself when no stop receipt arrived, and records the files once the late receipt lands": after the hour, the lease gets a stop-only intent. A second pass changes nothing. The late receipt wakes the projection queue. The next check records the hold, and the system recovery frees the workspace. Fails on master (`expected { …(19) } to match object { status: 'pending_cleanup' }`). - "names the unconfirmed stop on the task once, and keeps checking, when the provider never confirms it": the expiry branch. One notice, the check continues, no hold, and the workspace stays held. Fails on master. - "continues from the last save when the provider confirms the sandbox is gone": epoch 1 is set aside, the workspace is free, the task says so, and the result is not applied. Fails on master. - Review-round tests (all 5 fail on the first head `a208bfa58f`, pass now): - `environment-runtime.test.ts` "settles a plugin stop whose provider reports the sandbox gone (boxd, Daytona not found) through the cleanup sweep": provider dispatch through `releaseRunLeases` and `sweepPendingCleanupLeases`. The lease settles `released` with a `destroyed` receipt and no retain receipt. - `environment-runtime.test.ts` "settles a built-in stop whose provider reports the sandbox gone, without claiming retained files": the built-in `stopLease` path. No destroy call. - `superseded-workspace-export-recovery.test.ts` "continues from the last save when Paperclip's own stop finds the sandbox already gone": the worker's stop request gets a `destroyed` reply, the projection queue wakes, and the workspace is freed at epoch 2. - `superseded-workspace-export-recovery.test.ts` "never stops a sandbox a retained warm lease still holds, says so once and keeps checking": the source lease is unchanged, a recheck is returned, and `sandbox_in_use` is recorded. - Daytona `plugin.test.ts` "explicit stop confirms a sandbox Daytona no longer has as destroyed". - `environment-runtime.test.ts` (207) and Daytona `plugin.test.ts` (421) pass. - Related suites pass: `native-run-projection-work`, `retained-native-session-cleanup-work`, `workspace-recovery-stop-proof`, `native-failed-workspace-recovery`, `legacy-workspace-restore-recovery`, `environment-retained-orphan-sandbox`, `workspace-recovery-retry-copyback`, `workspace-recovery-fallback` (398 tests), and `ui/src/lib/activity-format.test.ts`. - `tsc --noEmit` for `server`. ## Risks - Low to medium risk. The new behavior acts only on an ended superseded run that failed in its workspace export, still holds a dirty workspace, and has no retained-file hold. - The worker now stops an ended run's sandbox one hour after the run ended if no stop receipt arrived. The stop uses the existing stop-only contract and keeps the sandbox. A sandbox held by another lease is never stopped. - A destroyed sandbox's unsaved files are not recoverable. The task continues from its last save and says so. - If a provider never confirms a stop, the workspace stays held (by design: no second writer). The task shows one notice, and Paperclip keeps retrying. - Shared cleanup code now treats a confirmed `destroyed` stop-only reply as settled. Every caller was checked: the cleanup sweep and run release count it as settled; the workspace recovery treats the source as unavailable; the fallback treats it as stopped (nothing can run there). No caller reads it as retained files, because no retain receipt is written. A native workspace export resume still requires `stopped`. - Each extra delivery notification per settled stop-only cleanup is cheap. The projection worker's scoped load finds nothing for runs that do not match. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, with tool use (code search, shell, test execution) in Claude Code. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- doc/execution-semantics.md | 2 +- .../daytona/src/plugin.test.ts | 9 + .../sandbox-providers/daytona/src/plugin.ts | 4 +- .../src/__tests__/environment-runtime.test.ts | 48 ++++++ server/src/services/environment-runtime.ts | 8 +- .../native-superseded-workspace-export.ts | 121 +++++++++++-- .../src/services/sandbox-stop-and-retain.ts | 47 ++++- ...perseded-workspace-export-recovery.test.ts | 160 +++++++++++++++++- server/src/services/workspace-recovery.ts | 5 + ui/src/lib/activity-format.ts | 4 + 10 files changed, 381 insertions(+), 27 deletions(-) diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 202b276eb6a..c4777023819 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -3779,7 +3779,7 @@ another provider attempt, so restarts the run already survived cannot block it. A disposition request that also ends without a result consumes the marker and fails closed as before. -Provider execution and control-plane finalization have different clocks. A healthy provider can think or execute a long tool without output. Once execution settles, recovery and finalization control steps have a 60-second deadline, checked on startup and every 15 seconds. With a healthy database and scheduler, an abandoned transition must be repaired or surfaced within 90 seconds. Terminal persistence must not wait on provider cleanup or publication; a late finalizer cannot change a reassigned or closed task or release another run's locks. In particular, a done report from a run whose task now belongs to another agent, a user or nobody is kept as an accepted result but records `completion_owner_superseded` and preserves the task status for the current owner. A finalizer retired because a newer run already committed the task's decision (`native_finalization_superseded`) also ends its run, so the run never holds the task or the runs queued behind it; the result is kept for reference and never applied, and the task's decision does not change. The native run projection worker ends a run that an older release left running in that state, at startup and whenever it is signalled. When that retirement interrupted the run's workspace export, the run's files are still unsaved and the run still holds the task's workspace as its writer, so the same worker records them as a retained-file hold (the stopped sandbox is kept). The system workspace recovery then saves them from that sandbox within its bounded retry budget and frees the workspace. If every attempt fails, the task continues from its last committed save: only that run's workspace epoch is set aside (its location stays on the workspace record), the readable files are attached to the task unapplied, and the task shows a plain notice. The superseded result is never applied in either case. Historical ambiguous runs are never automatically replayed after an upgrade. +Provider execution and control-plane finalization have different clocks. A healthy provider can think or execute a long tool without output. Once execution settles, recovery and finalization control steps have a 60-second deadline, checked on startup and every 15 seconds. With a healthy database and scheduler, an abandoned transition must be repaired or surfaced within 90 seconds. Terminal persistence must not wait on provider cleanup or publication; a late finalizer cannot change a reassigned or closed task or release another run's locks. In particular, a done report from a run whose task now belongs to another agent, a user or nobody is kept as an accepted result but records `completion_owner_superseded` and preserves the task status for the current owner. A finalizer retired because a newer run already committed the task's decision (`native_finalization_superseded`) also ends its run, so the run never holds the task or the runs queued behind it; the result is kept for reference and never applied, and the task's decision does not change. The native run projection worker ends a run that an older release left running in that state, at startup and whenever it is signalled. When that retirement interrupted the run's workspace export, the run's files are still unsaved and the run still holds the task's workspace as its writer, so the same worker records them as a retained-file hold (the stopped sandbox is kept). The system workspace recovery then saves them from that sandbox within its bounded retry budget and frees the workspace. If every attempt fails, the task continues from its last committed save: only that run's workspace epoch is set aside (its location stays on the workspace record), the readable files are attached to the task unapplied, and the task shows a plain notice. The hold needs the sandbox's stop-and-retain receipt, and that receipt wakes the worker whenever it lands. When the run's own release has not delivered it within an hour of the run ending, the worker stops and keeps the sandbox itself through the stop-only cleanup, which retries until the provider confirms; the workspace stays held meanwhile, so a sandbox that may still run never gets a second writer. If that stop stays unconfirmed for six hours, or the sandbox is held by another active, retained or pending-cleanup lease, the task gets one plain notice and the worker keeps checking. A stop-only dispatch also settles when the provider confirms the exact sandbox is already gone (a `destroyed` receipt, such as boxd's reply or Daytona's not-found): the lease records the destruction, never a retain receipt, and that receipt wakes the worker. If the provider confirms the sandbox is already gone, its files cannot be saved: only that run's epoch is set aside, the queued runs continue from the last save, and the task says so. The superseded result is never applied in any case. Historical ambiguous runs are never automatically replayed after an upgrade. A control deadline bounds the settlement of one attempt, never a later attempt of the same run. While a native run's finalization owner holds a live lease (for example, a controller that reclaimed the run after a server restart) or has a same-run resume scheduled, that owner is responsible for the run. The reconciler clears a run deadline left by an earlier attempt instead of failing the resumed work. When the live owner's own settlement deadline is due, the reconciler waits until that lease lapses before it surfaces the deadline. A run whose deadline genuinely expires fails with `execution_finalization_deadline_exceeded` and shows plain wording with Retry: its saved work is kept, and actions with unconfirmed outcomes are not repeated. Runs saved by earlier releases with the previous operator-facing wording show the same plain wording when the task is read; no migration is needed. diff --git a/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts b/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts index 1bde131bbac..aa9963536ed 100644 --- a/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts +++ b/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts @@ -3042,6 +3042,15 @@ describe("Daytona sandbox provider plugin", () => { expect(sandbox.delete).not.toHaveBeenCalled(); }); + it("explicit stop confirms a sandbox Daytona no longer has as destroyed", async () => { + process.env.DAYTONA_API_KEY = "host-key"; + mockGet.mockRejectedValue(new MockDaytonaNotFoundError("gone")); + await expect(plugin.definition.onEnvironmentStopLease!({ + driverKey: "daytona", companyId: "company-1", environmentId: "env-1", + providerLeaseId: "sandbox-gone", config: { reuseLease: false }, + })).resolves.toEqual({ providerLeaseId: "sandbox-gone", state: "destroyed" }); + }); + describe("explicit stop of a sandbox in the provider error state", () => { const stopParams = (sandboxId: string) => ({ driverKey: "daytona", companyId: "company-1", environmentId: "env-1", providerLeaseId: sandboxId, config: { reuseLease: false } }); diff --git a/packages/plugins/sandbox-providers/daytona/src/plugin.ts b/packages/plugins/sandbox-providers/daytona/src/plugin.ts index 69031f0970b..ee55bea06d3 100644 --- a/packages/plugins/sandbox-providers/daytona/src/plugin.ts +++ b/packages/plugins/sandbox-providers/daytona/src/plugin.ts @@ -2805,7 +2805,9 @@ async function stopLease(params: PluginEnvironmentReleaseLeaseParams): Promise

{ }, ); + it("settles a plugin stop whose provider reports the sandbox gone (boxd, Daytona not found) through the cleanup sweep", async () => { + const seeded = await seedReusablePluginSandboxLease("paperclip_runner"); + const lease = seeded.reusableLease; + await db.update(environmentLeases).set({ leasePolicy: "ephemeral", metadata: { ...lease.metadata, reuseLease: false } }) + .where(eq(environmentLeases.id, lease.id)); + await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, seeded.runId)); + let gone = false; + const call = vi.fn(async (_id: string, method: string) => { + if (method !== "environmentStopLease") throw new Error(`unexpected ${method}`); + if (!gone) throw new Error("provider unavailable"); + return { providerLeaseId: lease.providerLeaseId, state: "destroyed" }; + }); + const runtime = () => environmentRuntimeService(db, { pluginWorkerManager: { + isRunning: () => true, call, + getWorker: () => ({ supportedMethods: ["environmentReleaseLease", "environmentDestroyLease", "environmentStopLease"] }), + } as unknown as PluginWorkerManager }); + await runtime().releaseRunLeases(seeded.runId, "failed", undefined, "stop_and_retain"); + expect(await environmentService(db).getLeaseById(lease.id)).toMatchObject({ status: "pending_cleanup" }); + gone = true; + await heartbeatService(db, { environmentRuntime: runtime() }).sweepPendingCleanupLeases({ backoffMs: 0 }); + const settled = (await environmentService(db).getLeaseById(lease.id))!; + expect(settled).toMatchObject({ status: "released", cleanupStatus: "success", failureReason: "workspace_source_unavailable", + metadata: { remoteExecutionTermination: { state: "destroyed", providerLeaseId: lease.providerLeaseId } } }); + expect(settled.metadata).not.toHaveProperty("sandboxStopAndRetainReceipt"); + expect(call.mock.calls.every(entry => entry[1] === "environmentStopLease")).toBe(true); + }); + it("renews the stop-only cleanup claim while the provider stop is still running", async () => { const seeded = await seedReusablePluginSandboxLease("paperclip_runner"); const lease = seeded.reusableLease; @@ -1899,6 +1926,27 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { expect(destroy).not.toHaveBeenCalled(); }); + it("settles a built-in stop whose provider reports the sandbox gone, without claiming retained files", async () => { + const { companyId, environment, runId } = await seedEnvironment({ + driver: "sandbox", config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false }, + }); + const acquired = await runtime.acquireRunLease({ companyId, environment, issueId: null, + heartbeatRunId: runId, persistedExecutionWorkspace: null }); + const provider = sandboxProviderRuntime.requireSandboxProvider("fake"), originalStop = provider.stopLease!; + const destroy = vi.spyOn(provider, "destroyLease"); + try { + provider.stopLease = vi.fn(async () => ({ providerLeaseId: acquired.lease.providerLeaseId!, state: "destroyed" })) as never; + await runtime.releaseRunLeases(runId, "released", undefined, "stop_and_retain", true); + const settled = (await environmentService(db).getLeaseById(acquired.lease.id))!; + expect(settled).toMatchObject({ status: "released", cleanupStatus: "success", failureReason: "workspace_source_unavailable", + metadata: { remoteExecutionTermination: { state: "destroyed", providerLeaseId: acquired.lease.providerLeaseId } } }); + expect(settled.metadata).not.toHaveProperty("sandboxStopAndRetainReceipt"); + expect(settled.metadata).not.toHaveProperty("sandboxStopAndRetain"); + expect(await remoteExecutionHasStopped(db, companyId, runId)).toBe(true); + expect(destroy).not.toHaveBeenCalled(); + } finally { provider.stopLease = originalStop; } + }); + it.each(["stop_failed", "wrong_receipt", "missing_capability", "changed_binding"])( "preserves a built-in stop through restart without destructive fallback: %s", async outcome => { const { companyId, environment, runId } = await seedEnvironment({ diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index bec487ba3e4..d3366b8bb0d 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -27,7 +27,7 @@ import { beginIdleTrackedWork } from "./task-admission.js"; import { historicalProviderCredentials } from "./historical-provider-credentials.js"; import { ownedWorkspaceRecoveryLease, type WorkspaceRecoverySourceIdentity } from "./workspace-recovery-source.js"; import type { NativeWorkspaceFinalizationOwnership } from "./native-runtime/native-workspace-finalization-ownership.js"; -import { hasStopOnlyCleanup, prepareSandboxStopAndRetain, readStopOnlyCleanup, SANDBOX_STOP_AND_RETAIN_KEY, settleStopOnlyCleanup, stopOnlyCleanupKey } from "./sandbox-stop-and-retain.js"; +import { acceptedStopOnlyReceipt, hasStopOnlyCleanup, prepareSandboxStopAndRetain, readStopOnlyCleanup, SANDBOX_ALLOCATION_HOLDER_STATUSES, SANDBOX_STOP_AND_RETAIN_KEY, settleStopOnlyCleanup, stopOnlyCleanupKey } from "./sandbox-stop-and-retain.js"; import { JsonRpcCallError, readEnvironmentAcquisitionDiagnostic, readEnvironmentCreationCleanupError } from "@paperclipai/plugin-sdk"; import { preserveEnvironmentSyncOutErrorDiagnostic } from "./environment-sync-out-error.js"; import { remoteTerminationReceipt } from "./remote-execution-termination.js"; @@ -2447,7 +2447,7 @@ function createSandboxEnvironmentDriver( const [current] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, input.lease.id)).limit(1); const [otherOwner] = await db.select({ id: environmentLeases.id }).from(environmentLeases).where(and( ne(environmentLeases.id, input.lease.id), eq(environmentLeases.provider, input.lease.provider!), - eq(environmentLeases.providerLeaseId, input.lease.providerLeaseId!), inArray(environmentLeases.status, ["active", "retained", "pending_cleanup"]), + eq(environmentLeases.providerLeaseId, input.lease.providerLeaseId!), inArray(environmentLeases.status, [...SANDBOX_ALLOCATION_HOLDER_STATUSES]), )).limit(1); if (!current || current.status !== "pending_cleanup" || current.companyId !== input.lease.companyId || current.metadata?.pendingCleanupAttemptId !== input.lease.metadata?.pendingCleanupAttemptId @@ -2521,7 +2521,7 @@ function createSandboxEnvironmentDriver( config: workerConfig, providerLeaseId: input.lease.providerLeaseId, leaseMetadata: input.lease.metadata ?? {}, cancelActiveWork: true, resourceDisposition: "stop_and_retain", }, Math.min(resolvePluginSandboxRpcTimeoutMs(workerConfig) ?? 60_000, 60_000))); - if (remoteTerminationReceipt(input.lease, receipt)?.state !== "stopped") { + if (!acceptedStopOnlyReceipt(input.lease, receipt)) { throw new Error("Workspace export recovery did not confirm the retained sandbox stopped."); } return receipt; @@ -2574,7 +2574,7 @@ function createSandboxEnvironmentDriver( } await assertExportResumeOwnership(); const receipt = await provider.stopLease({ config: cleanupConfig, providerLeaseId: input.lease.providerLeaseId }); - if (remoteTerminationReceipt(input.lease, receipt)?.state !== "stopped") { + if (!acceptedStopOnlyReceipt(input.lease, receipt)) { throw new Error("Built-in sandbox stop did not confirm the exact retained allocation."); } return receipt; diff --git a/server/src/services/native-runtime/native-superseded-workspace-export.ts b/server/src/services/native-runtime/native-superseded-workspace-export.ts index 30f3da2c689..0fc48e702db 100644 --- a/server/src/services/native-runtime/native-superseded-workspace-export.ts +++ b/server/src/services/native-runtime/native-superseded-workspace-export.ts @@ -1,15 +1,27 @@ -import { and, eq, inArray, isNotNull, isNull, sql } from "drizzle-orm"; -import { environmentLeases, heartbeatRuns, issueRecoveryActions, nativeRunFinalizations, type Db } from "@paperclipai/db"; -import { remoteWorkspaceRunReferenceSchema } from "@paperclipai/shared"; +import { and, eq, inArray, isNotNull, isNull, ne, sql } from "drizzle-orm"; +import { activityLog, environmentLeases, executionWorkspaces, heartbeatRuns, issueRecoveryActions, issues, + nativeRunFinalizations, type Db } from "@paperclipai/db"; +import { readRemoteWorkspaceAuthority, remoteWorkspaceRunReferenceSchema, type RemoteWorkspaceRunReference } from "@paperclipai/shared"; import { SUPERSEDED_WORKSPACE_EXPORT_KEY, SUPERSEDED_WORKSPACE_EXPORT_SCHEMA } from "../workspace-recovery-fallback-discovery.js"; -import { hasRemoteTerminationReceipt } from "../remote-execution-termination.js"; +import { hasRecoveredCleanupStopReceipt, hasRemoteTerminationReceipt } from "../remote-execution-termination.js"; +import { abandonUnverifiedRemoteWorkspaceEpoch } from "../remote-workspace-epoch-abandonment.js"; +import { notifyQueuedRunDispatch } from "../run-dispatch-notifications.js"; +import { hasConfirmedSandboxStopAndRetain, hasStopOnlyCleanup, prepareUnconfirmedSourceSandboxStop, + SANDBOX_ALLOCATION_HOLDER_STATUSES } from "../sandbox-stop-and-retain.js"; +import { logActivity } from "../activity-log.js"; import { hasRequiredWorkspaceRecovery } from "../workspace-restore-recovery-state.js"; /** The finalizer's verdict for a result a newer run already decided (see native-run-finalizer). */ const SUPERSEDED = "native_finalization_superseded"; -/** How long a just-ended run's source may wait for its sandbox stop receipt before the check rests. */ +/** How long the run's own release may take to deliver its sandbox stop receipt. */ const STOP_RECEIPT_WAIT_MS = 60 * 60_000; const STOP_RECEIPT_RECHECK_MS = 60_000; +/** While Paperclip's own stop request is pending. Its receipt also wakes this work. */ +const REQUESTED_STOP_RECHECK_MS = 15 * 60_000; +/** A stop still unconfirmed this long after the run ended gets one notice on the task. */ +const STOP_NOTICE_AFTER_MS = 6 * 60 * 60_000; +export const SUPERSEDED_EXPORT_STOP_NOTICE = "issue.previous_run_files_waiting"; +export const SUPERSEDED_EXPORT_FILES_GONE = "issue.previous_run_files_lost"; type Coordinator = Pick; @@ -62,8 +74,13 @@ export function supersededWorkspaceExportUnpreservedCondition() { /** * Records the retained-file obligation of an ended superseded export (the * system workspace recovery then re-exports it from the retained sandbox). - * Returns when to check again: only while a just-ended run's sandbox has not - * yet confirmed its stop, the one precondition that arrives by itself. + * The obligation needs the sandbox's stop-and-retain receipt. The run's own + * release normally delivers it, and the receipt wakes this work again. When it + * has not arrived within an hour of the run ending, Paperclip stops and keeps + * the sandbox itself and waits for that receipt. The workspace stays held + * until then: a sandbox that may still run never gets a second writer beside + * it. When the stop cannot be requested, or stays unconfirmed for hours, the + * task gets one notice. Returns when to check again. */ export async function preserveSupersededWorkspaceExport(db: Db, runId: string, preserve: (db: Db, run: typeof heartbeatRuns.$inferSelect) => Promise): Promise<{ recheckAt: number | null }> { @@ -77,9 +94,89 @@ export async function preserveSupersededWorkspaceExport(db: Db, runId: string, const reference = remoteWorkspaceRunReferenceSchema.safeParse(row.run.runnerProfileJson?.remoteWorkspace); if (!reference.success) return { recheckAt: null }; const [lease] = await db.select().from(environmentLeases).where(and(eq(environmentLeases.companyId, row.run.companyId), - eq(environmentLeases.id, reference.data.leaseId), eq(environmentLeases.heartbeatRunId, row.run.id), - inArray(environmentLeases.status, ["active", "pending_cleanup", "released"]))); - const ended = row.run.finishedAt?.getTime() ?? 0; - return { recheckAt: lease && !hasRemoteTerminationReceipt(lease) && Date.now() < ended + STOP_RECEIPT_WAIT_MS - ? Date.now() + STOP_RECEIPT_RECHECK_MS : null }; + eq(environmentLeases.id, reference.data.leaseId), eq(environmentLeases.heartbeatRunId, row.run.id))); + // A confirmed stop leaves nothing to wait for: the refusal is not about the sandbox. + if (lease && hasConfirmedSandboxStopAndRetain(lease)) return { recheckAt: null }; + const now = Date.now(), elapsed = now - (row.run.finishedAt?.getTime() ?? 0); + if (lease && !hasRemoteTerminationReceipt(lease) && elapsed < STOP_RECEIPT_WAIT_MS) + return { recheckAt: now + STOP_RECEIPT_RECHECK_MS }; + if (lease && await continueFromLastSaveWhenSandboxGone(db, row.run, reference.data, lease)) return { recheckAt: null }; + const stop = lease ? await requestSourceSandboxStop(db, lease) : "unavailable"; + if (stop === "pending" && elapsed < STOP_NOTICE_AFTER_MS) return { recheckAt: now + REQUESTED_STOP_RECHECK_MS }; + await noticeOnce(db, row.run, stop); + // Never a silent rest: the stop, another lease letting go, or a provider + // receipt can still settle it, and each later check sees that. + return { recheckAt: now + REQUESTED_STOP_RECHECK_MS }; +} + +const destroyed = (lease: typeof environmentLeases.$inferSelect) => hasRemoteTerminationReceipt(lease) + && (lease.metadata?.remoteExecutionTermination as { state?: string } | undefined)?.state === "destroyed"; + +/** + * The provider confirmed the run's sandbox is gone, so nothing can write there + * and its unsaved files cannot be saved. Set aside only this run's workspace + * epoch, so the task's queued runs continue from the last save, and say so on + * the task. Every allocation of the run must carry a provider stop receipt. + */ +async function continueFromLastSaveWhenSandboxGone(db: Db, run: typeof heartbeatRuns.$inferSelect, + reference: RemoteWorkspaceRunReference, lease: typeof environmentLeases.$inferSelect) { + const issueId = run.nativeIssueId ?? run.issueId; + if (!destroyed(lease) || !issueId) return false; + return db.transaction(async tx => { + // The existing lock order: task, run, then (in the abandonment) the workspace. + const [task] = await tx.select({ id: issues.id }).from(issues).where(and(eq(issues.id, issueId), + eq(issues.companyId, run.companyId))).for("update"); + if (!task) return false; + const [current] = await tx.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.id, run.id), + eq(heartbeatRuns.companyId, run.companyId))).for("update"); + if (!current || current.status !== "failed" || !current.finishedAt + || current.controllerLeaseExpiresAt && current.controllerLeaseExpiresAt.getTime() > Date.now()) return false; + const leases = await tx.select().from(environmentLeases).where(and(eq(environmentLeases.companyId, run.companyId), + eq(environmentLeases.heartbeatRunId, run.id))); + if (!leases.some(row => row.id === lease.id && destroyed(row)) + || leases.some(row => !hasRemoteTerminationReceipt(row) && !hasRecoveredCleanupStopReceipt(row))) return false; + const [workspace] = await tx.select({ metadata: executionWorkspaces.metadata }).from(executionWorkspaces) + .where(and(eq(executionWorkspaces.companyId, run.companyId), eq(executionWorkspaces.id, reference.workspaceId))); + const authority = workspace ? readRemoteWorkspaceAuthority(workspace.metadata) : null; + if (!authority?.location || authority.writerRunId !== run.id || authority.epoch !== reference.epoch + || authority.snapshotId !== reference.snapshotId || authority.location.leaseId !== lease.id) return false; + await abandonUnverifiedRemoteWorkspaceEpoch(tx, { companyId: run.companyId, workspaceId: reference.workspaceId, + runId: run.id, epoch: authority.epoch, snapshotId: authority.snapshotId, location: authority.location }); + await notifyQueuedRunDispatch(tx, { companyId: run.companyId }); + await logActivity(tx as unknown as Db, { companyId: run.companyId, actorType: "system", actorId: "native-workspace-recovery", + action: SUPERSEDED_EXPORT_FILES_GONE, entityType: "issue", entityId: issueId, agentId: run.agentId, runId: run.id, + details: { runId: run.id, reason: "sandbox_destroyed" } }); + return true; + }); +} + +type SourceStop = "pending" | "in_use" | "unavailable"; + +async function requestSourceSandboxStop(db: Db, lease: typeof environmentLeases.$inferSelect): Promise { + if (hasStopOnlyCleanup(lease) || lease.metadata?.pendingCleanupInFlight === true) return "pending"; + if (!lease.provider || lease.provider === "local" || !lease.providerLeaseId || destroyed(lease)) return "unavailable"; + // A sandbox another lease holds is never stopped from here. + const [other] = await db.select({ id: environmentLeases.id }).from(environmentLeases).where(and( + ne(environmentLeases.id, lease.id), eq(environmentLeases.provider, lease.provider), + eq(environmentLeases.providerLeaseId, lease.providerLeaseId), inArray(environmentLeases.status, [...SANDBOX_ALLOCATION_HOLDER_STATUSES]))) + .limit(1); + if (other) return "in_use"; + // The pending-cleanup sweep dispatches the stop and retries it until the + // provider confirms. A lost race is re-read on the next check. + await prepareUnconfirmedSourceSandboxStop(db, lease); + return "pending"; +} + +/** One activity entry on the task, in plain words. The files are kept and + * Paperclip keeps working on it; nothing asks a person to fix anything. */ +async function noticeOnce(db: Db, run: typeof heartbeatRuns.$inferSelect, stop: SourceStop) { + const issueId = run.nativeIssueId ?? run.issueId; + if (!issueId) return; + const [existing] = await db.select({ id: activityLog.id }).from(activityLog).where(and( + eq(activityLog.companyId, run.companyId), eq(activityLog.entityType, "issue"), eq(activityLog.entityId, issueId), + eq(activityLog.action, SUPERSEDED_EXPORT_STOP_NOTICE), eq(activityLog.runId, run.id))).limit(1); + if (existing) return; + await logActivity(db, { companyId: run.companyId, actorType: "system", actorId: "native-workspace-recovery", + action: SUPERSEDED_EXPORT_STOP_NOTICE, entityType: "issue", entityId: issueId, agentId: run.agentId, runId: run.id, + details: { runId: run.id, reason: stop === "pending" ? "stop_unconfirmed" : stop === "in_use" ? "sandbox_in_use" : "sandbox_unavailable" } }); } diff --git a/server/src/services/sandbox-stop-and-retain.ts b/server/src/services/sandbox-stop-and-retain.ts index fe0c2b1ee99..aea0762201a 100644 --- a/server/src/services/sandbox-stop-and-retain.ts +++ b/server/src/services/sandbox-stop-and-retain.ts @@ -50,6 +50,20 @@ export function hasConfirmedSandboxStopAndRetain(lease: typeof environmentLeases && !!receipt.pluginId.trim() && receipt.pluginId === lease.metadata?.pluginId; } +/** Lease statuses through which another lease still holds the same provider + * allocation. Stop-only cleanup never stops an allocation another lease holds. */ +export const SANDBOX_ALLOCATION_HOLDER_STATUSES = ["active", "retained", "pending_cleanup"] as const; + +/** A stop-only dispatch settles when the provider confirms the exact sandbox + * stopped, or that it is already gone (for example, boxd reports `destroyed` + * for a machine it cannot find). A gone sandbox runs nothing and keeps no + * files, so its settle records the destruction and never a retain receipt. + * A native workspace export resume still needs a stopped sandbox. */ +export function acceptedStopOnlyReceipt(lease: Lease, value: unknown): boolean { + const state = remoteTerminationReceipt(lease, value)?.state; + return state === "stopped" || state === "destroyed" && !hasNativeWorkspaceExportResume(lease); +} + export function hasStopOnlyCleanup(lease: Pick): boolean { return hasNativeWorkspaceExportResume(lease) || Object.prototype.hasOwnProperty.call(lease.metadata ?? {}, SANDBOX_STOP_AND_RETAIN_KEY); @@ -162,7 +176,20 @@ export async function prepareSupersededWarmSandboxStop(db: Db, lease: typeof env return prepareStopAndRetain(db, lease, "retained", recovery); } -async function prepareStopAndRetain(db: Db, lease: Lease, fromStatus: "active" | "released" | "retained", recovery: Record = {}) { +/** An ended run still owes unsaved files from this sandbox, and no stop-and-retain + * receipt ever arrived for it. Stop and keep it through the stop-only contract, + * whatever cleanup it was waiting on: its pending-cleanup sweep then dispatches + * the stop and retries it until the provider confirms. A destroyed sandbox, a + * stop already in progress and a sandbox another lease holds are left alone; + * the caller verifies that the run has ended. */ +export async function prepareUnconfirmedSourceSandboxStop(db: Db, lease: typeof environmentLeases.$inferSelect) { + if (!lease.heartbeatRunId || !lease.provider || lease.provider === "local" || !lease.providerLeaseId + || lease.metadata?.pendingCleanupInFlight === true || hasStopOnlyCleanup(lease) + || (lease.metadata?.remoteExecutionTermination as { state?: string } | undefined)?.state === "destroyed") return null; + return prepareStopAndRetain(db, lease, lease.status); +} + +async function prepareStopAndRetain(db: Db, lease: Lease, fromStatus: string, recovery: Record = {}) { const requestId = randomUUID(), now = new Date(); const intent = { schema: "paperclip.sandbox-stop-and-retain.v1", requestId, companyId: lease.companyId, runId: lease.heartbeatRunId, leaseId: lease.id, @@ -192,23 +219,31 @@ export async function settleStopOnlyCleanup(db: Db, lease: Lease, options: { att const intent = readStopOnlyCleanup(lease); if (!intent) return null; const receipt = remoteTerminationReceipt(lease, options.receipt), stopped = receipt?.state === "stopped"; + // The provider confirmed the sandbox is gone: the stop settles with that + // receipt, and nothing claims its files were retained. + const gone = receipt?.state === "destroyed"; const now = new Date(); - const removeIntent: SQL = stopped ? sql`- 'sandboxStopAndRetain'` : sql``; + const removeIntent: SQL = stopped ? sql`- 'sandboxStopAndRetain'` + : gone ? sql`- 'sandboxStopAndRetain' - 'sandboxStopAndRetainReceipt'` : sql``; const recordedIntent = lease.metadata?.[SANDBOX_STOP_AND_RETAIN_KEY] as Record | undefined; const legacyOrphan = stopped && recordedIntent?.recoveredOrphanedRunLease === true && !lease.metadata?.[ORPHANED_SANDBOX_RECOVERY_KEY] ? { [ORPHANED_SANDBOX_RECOVERY_KEY]: { recoveredOrphanedRunLease: true, ownerProof: recordedIntent.ownerProof ?? null, runId: lease.heartbeatRunId, recoveredAt: now.toISOString(), fromStopIntent: true } } : {}; const [updated] = await writeEnvironmentCleanup(db, async tx => { - if (stopped) { + if (stopped || gone) { await notifyRetainedNativeSessionCleanup(tx, { companyId: lease.companyId, runId: lease.heartbeatRunId! }); - // A stop receipt is new evidence for a save hold waiting on it. + // A stop receipt is new evidence for a save hold waiting on it, and for + // an ended run whose unsaved files wait on it before they get a hold. await notifyDeliveryWork(tx, DELIVERY_QUEUES.workspaceRecovery); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.nativeRunProjection, + JSON.stringify({ companyId: lease.companyId, runId: lease.heartbeatRunId })); } return tx.update(environmentLeases).set({ - status: stopped ? "released" : "pending_cleanup", cleanupStatus: stopped ? "success" : "failed", - failureReason: stopped ? null : "sandbox_stop_pending", + status: stopped || gone ? "released" : "pending_cleanup", cleanupStatus: stopped || gone ? "success" : "failed", + failureReason: stopped ? null : gone ? "workspace_source_unavailable" : "sandbox_stop_pending", releasedAt: now, lastUsedAt: now, updatedAt: now, metadata: sql`(${environmentLeases.metadata} - 'remoteExecutionTermination' ${removeIntent}) || ${JSON.stringify({ + ...(gone ? { remoteExecutionTermination: receipt } : {}), ...(stopped ? { remoteExecutionTermination: receipt, sandboxStopAndRetainReceipt: { schema: "paperclip.sandbox-stop-and-retain-receipt.v1", requestId: intent.requestId, companyId: lease.companyId, runId: lease.heartbeatRunId, leaseId: lease.id, diff --git a/server/src/services/superseded-workspace-export-recovery.test.ts b/server/src/services/superseded-workspace-export-recovery.test.ts index 03b65a6e329..10b1ae28289 100644 --- a/server/src/services/superseded-workspace-export-recovery.test.ts +++ b/server/src/services/superseded-workspace-export-recovery.test.ts @@ -3,14 +3,17 @@ import { readFile } from "node:fs/promises"; import path from "node:path"; import { eq } from "drizzle-orm"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { environmentLeases, executionWorkspaces, heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations, +import { activityLog, environmentLeases, executionWorkspaces, heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations, nativeRunResults } from "@paperclipai/db"; import type { AdapterSandboxExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; import { readRemoteWorkspaceAuthority } from "@paperclipai/shared"; import { nativeRunProjectionWork } from "./native-runtime/native-run-projection-work.js"; import { readRemoteWorkspaceWriterHold } from "./remote-workspace-writer-hold.js"; import { createSavedWorkspaceLoader } from "./remote-workspaces.js"; -import { prepareStoppedSandboxForRecovery, settleStopOnlyCleanup } from "./sandbox-stop-and-retain.js"; +import { prepareStoppedSandboxForRecovery, readStopOnlyCleanup, settleStopOnlyCleanup } from "./sandbox-stop-and-retain.js"; +import { DELIVERY_QUEUES, subscribeDeliveryWork } from "./delivery-work-notifications.js"; +import { remoteTerminationReceipt } from "./remote-execution-termination.js"; +import { SUPERSEDED_EXPORT_FILES_GONE, SUPERSEDED_EXPORT_STOP_NOTICE } from "./native-runtime/native-superseded-workspace-export.js"; import { readSystemRecovery } from "./system-recovery-state.js"; import { adoptSystemRecovery } from "./system-recovery.js"; import { RETAINED_SOURCE_FALLBACK_OUTCOME } from "./workspace-recovery-fallback.js"; @@ -45,7 +48,7 @@ describe("a superseded run whose workspace export failed", () => { * still the dirty writer of the task's workspace. No retained-file * obligation exists, so nothing ever settles that writer. */ - async function supersededExport() { + async function supersededExport(stop: "confirmed" | "never_requested" | "destroyed" = "confirmed") { const f = await acceptedRemoteExport(); const now = new Date(); await db.update(nativeRunFinalizations).set({ phase: "terminal_failure", leaseOwner: null, leaseExpiresAt: null, @@ -68,6 +71,20 @@ describe("a superseded run whose workspace export failed", () => { await db.update(issues).set({ status: "in_progress" }).where(eq(issues.id, f.issueId)); const [lease] = await db.update(environmentLeases).set({ leasePolicy: "ephemeral" }) .where(eq(environmentLeases.id, f.lease.id)).returning(); + if (stop === "never_requested") { + // The run's own release never stopped its sandbox (for example, the + // server that owned it was replaced mid-release): no receipt exists. + const { remoteExecutionTermination: _receipt, ...metadata } = lease.metadata ?? {}; + await db.update(environmentLeases).set({ status: "active", releasedAt: null, cleanupStatus: null, metadata }) + .where(eq(environmentLeases.id, f.lease.id)); + return f; + } + if (stop === "destroyed") { + await db.update(environmentLeases).set({ metadata: { ...lease.metadata, + remoteExecutionTermination: remoteTerminationReceipt(lease, { providerLeaseId: f.providerLeaseId, state: "destroyed" }) } }) + .where(eq(environmentLeases.id, f.lease.id)); + return f; + } const stopping = (await prepareStoppedSandboxForRecovery(db, lease))!; await settleStopOnlyCleanup(db, stopping, { attemptId: String(stopping.metadata!.pendingCleanupAttemptId), receipt: { providerLeaseId: f.providerLeaseId, state: "stopped" } }); @@ -169,6 +186,143 @@ describe("a superseded run whose workspace export failed", () => { .toMatchObject({ phase: "terminal_failure", failureCode: "native_finalization_superseded" }); }, 60_000); + async function checkThroughProjectionWorker(f: { companyId: string; runId: string }) { + const work = nativeRunProjectionWork(db); + const [entry] = (await work.load({ companyId: f.companyId, runId: f.runId })).filter(item => item.id === f.runId); + return entry ? await work.check(entry) : undefined; + } + const endedAgo = (f: { runId: string }, ms: number) => db.update(heartbeatRuns) + .set({ finishedAt: new Date(Date.now() - ms) }).where(eq(heartbeatRuns.id, f.runId)); + const lease = async (id: string) => (await db.select().from(environmentLeases).where(eq(environmentLeases.id, id)))[0]!; + const notices = async (f: { companyId: string; runId: string }) => (await db.select().from(activityLog) + .where(eq(activityLog.companyId, f.companyId))).filter(row => row.action === SUPERSEDED_EXPORT_STOP_NOTICE && row.runId === f.runId); + async function writerHold(f: { companyId: string; workspaceId: string }) { + const authority = readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)!; + return readRemoteWorkspaceWriterHold(db, { companyId: f.companyId, authority }); + } + + it("checks again every minute while the run's own release may still stop its sandbox", async () => { + const f = await supersededExport("never_requested"); + const before = Date.now(); + const next = await checkThroughProjectionWorker(f); + expect(next?.dueAt).toBeGreaterThanOrEqual(before + 60_000); + expect(next!.dueAt).toBeLessThan(before + 5 * 60_000); + // Nothing is recorded or stopped yet, and the workspace keeps its writer. + expect(await preservation(f)).toBeNull(); + expect(await lease(f.lease.id)).toMatchObject({ status: "active" }); + expect(await writerHold(f)).toMatchObject({ kind: "held", writer: { runId: f.runId } }); + expect(await notices(f)).toEqual([]); + }, 60_000); + + it("stops and keeps the sandbox itself when no stop receipt arrived, and records the files once the late receipt lands", async () => { + const f = await supersededExport("never_requested"); + await endedAgo(f, 2 * 60 * 60_000); + const before = Date.now(); + const next = await checkThroughProjectionWorker(f); + // Paperclip asked for the stop-and-retain; the cleanup sweep dispatches it. + const stopping = await lease(f.lease.id); + expect(stopping).toMatchObject({ status: "pending_cleanup" }); + expect(readStopOnlyCleanup(stopping)).toMatchObject({ runId: f.runId, leaseId: f.lease.id }); + // The check is not dropped, and the workspace stays held while the sandbox may still run. + expect(next?.dueAt).toBeGreaterThanOrEqual(before + 15 * 60_000); + expect(await preservation(f)).toBeNull(); + expect(await writerHold(f)).toMatchObject({ kind: "held", writer: { runId: f.runId } }); + // A second pass does not request another stop. + await checkThroughProjectionWorker(f); + expect((await lease(f.lease.id)).metadata).toEqual(stopping.metadata); + + // The provider's late receipt wakes the native run projection work... + let woken = 0; + const unsubscribe = subscribeDeliveryWork(db, DELIVERY_QUEUES.nativeRunProjection, () => { woken++; }); + try { + expect(await settleStopOnlyCleanup(db, stopping, { attemptId: String(stopping.metadata!.pendingCleanupAttemptId), + receipt: { providerLeaseId: f.providerLeaseId, state: "stopped" } })).not.toBeNull(); + await vi.waitFor(() => expect(woken).toBeGreaterThan(0)); + } finally { unsubscribe(); } + // ...which records the files for the system workspace recovery. + expect(await checkThroughProjectionWorker(f)).toBeNull(); + const hold = await preservation(f); + expect(hold?.evidence).toMatchObject({ supersededWorkspaceExport: { runId: f.runId } }); + const input = { ...f.input, actionId: hold!.id }; + await adoptSystemRecovery(db, input); + await recoverRetainedWorkspace(input); + expect(await writerHold(f)).toEqual({ kind: "free" }); + expect(await notices(f)).toEqual([]); + }, 60_000); + + it("names the unconfirmed stop on the task once, and keeps checking, when the provider never confirms it", async () => { + const f = await supersededExport("never_requested"); + await endedAgo(f, 2 * 60 * 60_000); + await checkThroughProjectionWorker(f); + await endedAgo(f, 7 * 60 * 60_000); + const before = Date.now(); + const next = await checkThroughProjectionWorker(f); + expect(next?.dueAt).toBeGreaterThanOrEqual(before + 15 * 60_000); + await checkThroughProjectionWorker(f); + const logged = await notices(f); + expect(logged).toHaveLength(1); + expect(logged[0]).toMatchObject({ entityType: "issue", entityId: f.issueId, actorType: "system", + details: { runId: f.runId, reason: "stop_unconfirmed" } }); + // No second writer: the files are not set aside while the sandbox may still run. + expect(await preservation(f)).toBeNull(); + expect(await writerHold(f)).toMatchObject({ kind: "held", writer: { runId: f.runId } }); + }, 60_000); + + it("continues from the last save when the provider confirms the sandbox is gone", async () => { + const f = await supersededExport("destroyed"); + expect(await checkThroughProjectionWorker(f)).toBeNull(); + // Only this run's epoch is set aside; the queued runs start from the last save. + const after = await workspace(f.workspaceId); + expect(readRemoteWorkspaceAuthority(after.metadata)).toMatchObject({ writerRunId: null, dirty: false, location: null, epoch: 2 }); + expect(after.metadata?.remoteWorkspaceAbandonedEpochs).toEqual([expect.objectContaining({ epoch: 1, runId: f.runId })]); + expect(await writerHold(f)).toEqual({ kind: "free" }); + const said = (await db.select().from(activityLog).where(eq(activityLog.companyId, f.companyId))) + .filter(row => row.action === SUPERSEDED_EXPORT_FILES_GONE); + expect(said).toEqual([expect.objectContaining({ entityId: f.issueId, runId: f.runId })]); + // Nothing is left to do, and the superseded result is still not applied. + expect(await settleThroughProjectionWorker(f)).toBe(0); + expect(await resumePreservedResultFinalization(db, f.companyId, f.runId)).toBeFalsy(); + expect(await notices(f)).toEqual([]); + }, 60_000); + + it("continues from the last save when Paperclip's own stop finds the sandbox already gone", async () => { + const f = await supersededExport("never_requested"); + await endedAgo(f, 2 * 60 * 60_000); + await checkThroughProjectionWorker(f); + const stopping = await lease(f.lease.id); + // The provider answers the stop-only dispatch with its "gone" receipt + // (boxd: destroyed; Daytona: not found, reported as destroyed). + let woken = 0; + const unsubscribe = subscribeDeliveryWork(db, DELIVERY_QUEUES.nativeRunProjection, () => { woken++; }); + try { + expect(await settleStopOnlyCleanup(db, stopping, { attemptId: String(stopping.metadata!.pendingCleanupAttemptId), + receipt: { providerLeaseId: f.providerLeaseId, state: "destroyed" } })).toMatchObject({ status: "released" }); + await vi.waitFor(() => expect(woken).toBeGreaterThan(0)); + } finally { unsubscribe(); } + expect((await lease(f.lease.id)).metadata).not.toHaveProperty("sandboxStopAndRetainReceipt"); + expect(await checkThroughProjectionWorker(f)).toBeNull(); + expect(readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)) + .toMatchObject({ writerRunId: null, dirty: false, epoch: 2 }); + expect(await writerHold(f)).toEqual({ kind: "free" }); + expect(await preservation(f)).toBeNull(); + }, 60_000); + + it("never stops a sandbox a retained warm lease still holds, says so once and keeps checking", async () => { + const f = await supersededExport("never_requested"); + await endedAgo(f, 2 * 60 * 60_000); + const source = await lease(f.lease.id); + const { id: _id, ...shape } = source; + await db.insert(environmentLeases).values({ ...shape, id: randomUUID(), heartbeatRunId: null, status: "retained", + leasePolicy: "reuse_by_environment", releasedAt: null, metadata: { remoteCwd: source.metadata?.remoteCwd } }); + const before = Date.now(); + const next = await checkThroughProjectionWorker(f); + expect(next?.dueAt).toBeGreaterThanOrEqual(before + 15 * 60_000); + // The source lease is untouched: no stop intent was written. + expect(await lease(f.lease.id)).toEqual(source); + expect((await notices(f)).map(row => row.details)).toEqual([expect.objectContaining({ reason: "sandbox_in_use" })]); + expect(await writerHold(f)).toMatchObject({ kind: "held", writer: { runId: f.runId } }); + }, 60_000); + it("leaves a superseded result alone when it never failed in the workspace export", async () => { const f = await supersededExport(); await db.update(nativeRunFinalizations).set({ failureDetail: { originalFailureCode: "native_status_conflict", diff --git a/server/src/services/workspace-recovery.ts b/server/src/services/workspace-recovery.ts index 31177ac4c57..f95b57d7b94 100644 --- a/server/src/services/workspace-recovery.ts +++ b/server/src/services/workspace-recovery.ts @@ -320,6 +320,11 @@ export async function recoverRetainedWorkspace(input: Source & { db: Db; environ ?? await settleStopOnlyCleanup(db, pending, { attemptId: ownership.token })); if (!settled || !hasRemoteTerminationReceipt(settled)) throw new SystemRecoveryFailure("agent_stop_unconfirmed", "recheck_evidence"); + // The provider confirmed the sandbox is gone: its files cannot be read. + if ((settled.metadata?.remoteExecutionTermination as Record | undefined)?.state === "destroyed") { + sourceUnavailable = true; + continue; + } stoppedLease = settled; } const target = await resolveEnvironmentExecutionTarget({ db, companyId: input.companyId, adapterType: "paperclip_runner", diff --git a/ui/src/lib/activity-format.ts b/ui/src/lib/activity-format.ts index 5aa4cfc5409..79328fe10d5 100644 --- a/ui/src/lib/activity-format.ts +++ b/ui/src/lib/activity-format.ts @@ -77,6 +77,8 @@ const ACTIVITY_ROW_VERBS: Record = { "issue.workspace_recovered": "recovered saved work for", "issue.retry_continued_from_saved_work": "continued from saved work on", "issue.workspace_recovery_requested": "requested workspace recovery for", + "issue.previous_run_files_waiting": "is still working to save the previous run's files on", + "issue.previous_run_files_lost": "continued from the last save on", "issue.recovery_action_resolved": "resolved the recovery action on", "issue.recovery_action_escalated": "escalated the recovery action on", "agent.created": "created", @@ -181,6 +183,8 @@ const ISSUE_ACTIVITY_LABELS: Record = { "issue.workspace_recovered": "Saved work recovered.", "issue.retry_continued_from_saved_work": "Retry continued from saved work.", "issue.workspace_recovery_requested": "Requested workspace recovery", + "issue.previous_run_files_waiting": "Still working to save the previous run's unsaved files. New runs start once they are saved. Paperclip keeps trying on its own.", + "issue.previous_run_files_lost": "Could not save the previous run's unsaved files because they were already gone. This task continues from its last save.", "issue.recovery_action_resolved": "Resolved the recovery action", "issue.recovery_action_escalated": "Escalated the recovery action", "issue.accepted_plan_decomposition_updated": "updated the accepted-plan decomposition",