diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 029a0a96de8..ae92ce4b2a0 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -2221,7 +2221,10 @@ The lease carries the verified stop forward as `workspaceRecoveryCleanup.stopReceipt`. The saved message, Retry and the next remote workspace writer wait for that stop, not for the destroy. So they continue right after recovery, instead of after the cleanup retry. A lease without a -matching stop receipt still blocks them. +matching stop receipt still blocks them. The same holds when the Stop or +interrupt landed before the provider turn opened: the saved message then goes +through ordinary admission once the acknowledged Stop has a verified stop, and +that check also accepts the carried-forward receipt. The deadline is a backstop, not the normal failure path. When native task preparation fails after a fresh controller claim and before any provider turn, diff --git a/server/src/services/acknowledged-native-stop.ts b/server/src/services/acknowledged-native-stop.ts index e140b6968ad..6ad77cc89f9 100644 --- a/server/src/services/acknowledged-native-stop.ts +++ b/server/src/services/acknowledged-native-stop.ts @@ -3,7 +3,7 @@ import { environmentLeases, heartbeatRunEvents, heartbeatRuns, type Db } from "@ import { parseObject } from "../adapters/utils.js"; import { resolveProviderApiPlacement } from "./native-runtime/provider-api-placement.js"; import { hasNativeLocalProcessStop } from "./native-local-process-stop.js"; -import { hasRemoteTerminationReceipt } from "./remote-execution-termination.js"; +import { hasRecoveredCleanupStopReceipt, hasRemoteTerminationReceipt } from "./remote-execution-termination.js"; import { validatePrpEvent } from "../vendor/paperclip-runner/index.js"; import { nativeSha256 } from "./native-runtime/canonical.js"; import { isConfirmedCursorCloudTerminalObservation } from "./native-runtime/cursor-cloud-runtime.js"; @@ -96,8 +96,10 @@ export async function acknowledgedNativeStopExecutionHasStopped(db: Db, run: Pic const leases = await db.select().from(environmentLeases).where(and( eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.heartbeatRunId, run.id), )); - // Provider PIDs are meaningful only on their own host. - if (leases.some(lease => lease.provider !== "local")) return leases.every(hasRemoteTerminationReceipt); + // Provider PIDs are meaningful only on their own host. A recovered sandbox + // queued for destroy keeps its verified stop; the destroy is only cleanup. + if (leases.some(lease => lease.provider !== "local")) + return leases.every(lease => hasRemoteTerminationReceipt(lease) || hasRecoveredCleanupStopReceipt(lease)); if (leases.some(lease => !lease.releasedAt || lease.cleanupStatus === "failed")) return false; if (!run.processPid && !run.processGroupId) return hasNativeLocalProcessStop(db, run.companyId, run.id); const absent = (pid: number) => { diff --git a/server/src/services/workspace-recovery.test.ts b/server/src/services/workspace-recovery.test.ts index ad2d0314ff1..580a10b3263 100644 --- a/server/src/services/workspace-recovery.test.ts +++ b/server/src/services/workspace-recovery.test.ts @@ -3783,6 +3783,64 @@ describe("automatic retained workspace recovery", () => { } }, 30_000); + it.each(["verified-stop", "no-stop-receipt"] as const) + ("starts the interrupt redirect once after recovery when the interrupt landed before the provider turn, without waiting for the destroy (%s)", async kind => { + // An interrupt comment while an ephemeral-lease run was still starting: the + // startup owner fenced the handle before any provider turn, its files were + // recovered, and the re-stopped sandbox was queued for destroy. The queued + // follow-up was cancelled before start and its comment saved for recovery. + const f = await seed("native", true, "destroy", false, true); + const agentId = (await run(f.runId)).agentId; + await stopBeforeSessionOpen(f, { errorCode: "operator_interrupted" }); + const stopped = await run(f.runId); + await db.update(heartbeatRuns).set({ resultJson: { ...stopped.resultJson, operatorInterrupted: true, + interruptionSource: "issue_comment_interrupt", interruptedIssueId: f.issueId, interruptedByActorType: "user", + interruptedByActorId: "operator", cancelledByActorType: "user", cancelledByUserId: "operator" } }) + .where(eq(heartbeatRuns.id, f.runId)); + const [comment] = await db.insert(issueComments).values({ companyId: f.companyId, issueId: f.issueId, + authorType: "user", authorUserId: "operator", body: "Use the other approach instead." }).returning(); + const context = { issueId: f.issueId, wakeReason: "issue_commented", wakeCommentId: comment.id, wakeCommentIds: [comment.id] }; + await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId, runtimeMode: "legacy", status: "cancelled", + invocationSource: "automation", errorCode: "execution_reconciliation_required", finishedAt: new Date(), contextSnapshot: context }); + const [saved] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId, + source: "automation", reason: "issue_commented", status: "deferred_issue_execution", + requestedByActorType: "user", requestedByActorId: "operator", + payload: { issueId: f.issueId, commentId: comment.id, _paperclipWakeContext: context, + executionWait: { reason: "execution_recovery", message: "Waiting for execution recovery. Your message is saved." } } }).returning(); + await recoverRetainedWorkspace(f.input); + expect(readSystemRecovery((await action(f.actionId)).evidence)).toMatchObject({ state: "complete" }); + await scheduleRecoveredWorkspaceCleanup(db, await run(f.runId)); + const [queued] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id)); + expect(queued).toMatchObject({ status: "pending_cleanup", metadata: { workspaceRecoveryCleanup: { runId: f.runId } } }); + if (kind === "no-stop-receipt") { + const cleanup = queued.metadata!.workspaceRecoveryCleanup as Record; + await db.update(environmentLeases).set({ metadata: { ...queued.metadata, + workspaceRecoveryCleanup: { ...cleanup, stopReceipt: undefined } } }).where(eq(environmentLeases.id, f.lease.id)); + } + // No provider turn ran, so the no-replay hold does not gate the task. + expect(await getExecutionBlocker(db, f.companyId, f.issueId)).toBeNull(); + // Recovery hands the saved comment to ordinary admission (onRecovered), twice + // to prove a repeated sweep adds no second executor. + for (let pass = 0; pass < 2; pass += 1) await heartbeatService(db).resumeRemoteStopComments(await run(f.runId)); + const turns = await pendingTurns(f); + const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, saved.id)); + if (kind === "verified-stop") { + // The destroy is still pending: the follow-up waits on the stop, not on it. + expect((await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id)))[0]) + .toMatchObject({ status: "pending_cleanup" }); + expect(turns).toHaveLength(1); + expect(turns[0]).toMatchObject({ agentId, contextSnapshot: { wakeCommentId: comment.id } }); + expect(wake.status).not.toBe("deferred_issue_execution"); + } else { + // Without a verified stop, the saved comment keeps waiting. + expect(turns).toEqual([]); + expect(wake.status).toBe("deferred_issue_execution"); + } + // The no-replay audit row is untouched. + expect(await action(f.actionId)).toMatchObject({ status: "resolved", outcome: "blocked", + evidence: { automaticRecovery: { replay: "blocked" } } }); + }, 30_000); + it("keeps a snapshot-only recovery held for a new assignee because the task workspace lacks the saved files", async () => { const f = await seed("native", false); await db.update(heartbeatRuns).set({ status: "cancelled", errorCode: "issue_reassigned" }).where(eq(heartbeatRuns.id, f.runId));